diff --git a/backend/apps/config_app.py b/backend/apps/config_app.py index fb6a0a4f0..58e2b008b 100644 --- a/backend/apps/config_app.py +++ b/backend/apps/config_app.py @@ -6,6 +6,7 @@ from apps.datamate_app import router as datamate_router from apps.vectordatabase_app import router as vectordatabase_router from apps.dify_app import router as dify_router +from apps.idata_app import router as idata_router from apps.file_management_app import file_management_config_router as file_manager_router from apps.image_app import router as proxy_router from apps.knowledge_summary_app import router as summary_router @@ -39,6 +40,7 @@ app.include_router(proxy_router) app.include_router(tool_config_router) app.include_router(dify_router) +app.include_router(idata_router) # Choose user management router based on IS_SPEED_MODE if IS_SPEED_MODE: diff --git a/backend/apps/data_process_app.py b/backend/apps/data_process_app.py index 3ac8b45cf..9138d5ef1 100644 --- a/backend/apps/data_process_app.py +++ b/backend/apps/data_process_app.py @@ -11,6 +11,7 @@ ConvertStateRequest, TaskRequest, ) +from consts.exceptions import OfficeConversionException from data_process.tasks import process_and_forward, process_sync from services.data_process_service import get_data_process_service @@ -311,3 +312,35 @@ async def convert_state(request: ConvertStateRequest): status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail=f"Error converting state: {str(e)}" ) + + +@router.post("/convert_to_pdf") +async def convert_office_to_pdf( + object_name: str = Form(...), + pdf_object_name: str = Form(...) +): + """ + Convert an Office document stored in MinIO to PDF. + + Parameters: + object_name: Source Office file path in MinIO + pdf_object_name: Destination PDF path in MinIO + """ + try: + await service.convert_office_to_pdf_impl( + object_name=object_name, + pdf_object_name=pdf_object_name, + ) + return JSONResponse(status_code=HTTPStatus.OK, content={"success": True}) + except OfficeConversionException as exc: + logger.error(f"Office conversion failed for '{object_name}': {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=str(exc) + ) + except Exception as exc: + logger.error(f"Unexpected error during conversion for '{object_name}': {exc}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=f"Office conversion failed: {exc}" + ) diff --git a/backend/apps/file_management_app.py b/backend/apps/file_management_app.py index 9ed87cfae..5b7c7bc3c 100644 --- a/backend/apps/file_management_app.py +++ b/backend/apps/file_management_app.py @@ -9,22 +9,29 @@ from fastapi import APIRouter, Body, File, Form, Header, HTTPException, Path as PathParam, Query, UploadFile from fastapi.responses import JSONResponse, RedirectResponse, StreamingResponse +from consts.exceptions import FileTooLargeException, NotFoundException, OfficeConversionException, UnsupportedFileTypeException from consts.model import ProcessParams from services.file_management_service import upload_to_minio, upload_files_impl, \ - get_file_url_impl, get_file_stream_impl, delete_file_impl, list_files_impl + get_file_url_impl, get_file_stream_impl, delete_file_impl, list_files_impl, \ + preview_file_impl from utils.file_management_utils import trigger_data_process logger = logging.getLogger("file_management_app") -def build_content_disposition_header(filename: Optional[str]) -> str: +def build_content_disposition_header(filename: Optional[str], inline: bool = False) -> str: """ Build a Content-Disposition header that keeps the original filename. + Args: + filename: Original filename to include in header + inline: If True, use 'inline' disposition (for preview); otherwise 'attachment' (for download) + - ASCII filenames are returned directly. - Non-ASCII filenames include both an ASCII fallback and RFC 5987 encoded value so modern browsers keep the original name. """ + disposition = "inline" if inline else "attachment" safe_name = (filename or "download").strip() or "download" def _sanitize_ascii(value: str) -> str: @@ -40,26 +47,26 @@ def _sanitize_ascii(value: str) -> str: try: safe_name.encode("ascii") - return f'attachment; filename="{_sanitize_ascii(safe_name)}"' + return f'{disposition}; filename="{_sanitize_ascii(safe_name)}"' except UnicodeEncodeError: try: encoded = quote(safe_name, safe="") except Exception: # quote failure, fallback to sanitized ASCII only logger.warning("Failed to encode filename '%s', using fallback", safe_name) - return f'attachment; filename="{_sanitize_ascii(safe_name)}"' + return f'{disposition}; filename="{_sanitize_ascii(safe_name)}"' fallback = _sanitize_ascii( safe_name.encode("ascii", "ignore").decode("ascii") or "download" ) - return f'attachment; filename="{fallback}"; filename*=UTF-8\'\'{encoded}' + return f'{disposition}; filename="{fallback}"; filename*=UTF-8\'\'{encoded}' except Exception as exc: # pragma: no cover logger.warning( "Failed to encode filename '%s': %s. Using fallback.", safe_name, exc, ) - return 'attachment; filename="download"' + return f'{disposition}; filename="download"' # Create API router file_management_runtime_router = APIRouter(prefix="/file") @@ -567,3 +574,69 @@ async def get_storage_file_batch_urls( "failed_count": sum(1 for r in results if not r.get("success", False)), "results": results } + +@file_management_config_router.get("/preview/{object_name:path}") +async def preview_file( + object_name: str = PathParam(..., description="File object name to preview"), + filename: Optional[str] = Query(None, description="Original filename for display (optional)") +): + """ + Preview file inline in browser + + - **object_name**: File object name in storage + - **filename**: Original filename for Content-Disposition header (optional) + + Returns file stream with Content-Disposition: inline for browser preview + """ + try: + # Get file stream from preview service + file_stream, content_type = await preview_file_impl(object_name=object_name) + + # Use provided filename or extract from object_name + display_filename = filename + if not display_filename: + display_filename = object_name.split("/")[-1] if "/" in object_name else object_name + + # Build Content-Disposition header for inline display + content_disposition = build_content_disposition_header(display_filename, inline=True) + + return StreamingResponse( + file_stream, + media_type=content_type, + headers={ + "Content-Disposition": content_disposition, + "Cache-Control": "public, max-age=3600", + "ETag": f'"{object_name}"', + } + ) + + except FileTooLargeException as e: + logger.warning(f"[preview_file] File too large: object_name={object_name}, error={str(e)}") + raise HTTPException( + status_code=HTTPStatus.REQUEST_ENTITY_TOO_LARGE, + detail=str(e) + ) + except NotFoundException as e: + logger.error(f"[preview_file] File not found: object_name={object_name}, error={str(e)}") + raise HTTPException( + status_code=HTTPStatus.NOT_FOUND, + detail=f"File not found: {object_name}" + ) + except UnsupportedFileTypeException as e: + logger.error(f"[preview_file] Unsupported file type: object_name={object_name}, error={str(e)}") + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail=f"File format not supported for preview: {str(e)}" + ) + except OfficeConversionException as e: + logger.error(f"[preview_file] Conversion failed: object_name={object_name}, error={str(e)}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=f"Failed to preview file: {str(e)}" + ) + except Exception as e: + logger.error(f"[preview_file] Unexpected error: object_name={object_name}, error={str(e)}") + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, + detail=f"Failed to preview file: {str(e)}" + ) \ No newline at end of file diff --git a/backend/apps/idata_app.py b/backend/apps/idata_app.py new file mode 100644 index 000000000..278c1b60f --- /dev/null +++ b/backend/apps/idata_app.py @@ -0,0 +1,109 @@ +""" +iData App Layer +FastAPI endpoints for iData knowledge space operations. + +This module provides API endpoints to interact with iData's API, +including fetching knowledge spaces and transforming responses to a format +compatible with the frontend. +""" +import logging +from http import HTTPStatus + +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from services.idata_service import ( + fetch_idata_knowledge_spaces_impl, + fetch_idata_datasets_impl, +) + +router = APIRouter(prefix="/idata") +logger = logging.getLogger("idata_app") + + +@router.get("/knowledge-space") +async def fetch_idata_knowledge_spaces_api( + idata_api_base: str = Query(..., description="iData API base URL"), + api_key: str = Query(..., description="iData API key"), + user_id: str = Query(..., description="iData user ID"), +): + """ + Fetch knowledge spaces from iData API. + + Returns knowledge spaces in a format with id and name for frontend compatibility. + """ + try: + # Normalize URL by removing trailing slash + idata_api_base = idata_api_base.rstrip('/') + except Exception as e: + logger.error(f"Invalid iData configuration: {e}") + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + f"Invalid URL format: {str(e)}" + ) + + try: + result = fetch_idata_knowledge_spaces_impl( + idata_api_base=idata_api_base, + api_key=api_key, + user_id=user_id, + ) + return JSONResponse( + status_code=HTTPStatus.OK, + content=result + ) + except AppException: + # Re-raise AppException to be handled by global middleware + raise + except Exception as e: + logger.error(f"Failed to fetch iData knowledge spaces: {e}") + raise AppException( + ErrorCode.IDATA_SERVICE_ERROR, + f"Failed to fetch iData knowledge spaces: {str(e)}" + ) + + +@router.get("/datasets") +async def fetch_idata_datasets_api( + idata_api_base: str = Query(..., description="iData API base URL"), + api_key: str = Query(..., description="iData API key"), + user_id: str = Query(..., description="iData user ID"), + knowledge_space_id: str = Query(..., description="Knowledge space ID"), +): + """ + Fetch datasets (knowledge bases) from iData API. + + Returns knowledge bases in a format consistent with DataMate for frontend compatibility. + """ + try: + # Normalize URL by removing trailing slash + idata_api_base = idata_api_base.rstrip('/') + except Exception as e: + logger.error(f"Invalid iData configuration: {e}") + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + f"Invalid URL format: {str(e)}" + ) + + try: + result = fetch_idata_datasets_impl( + idata_api_base=idata_api_base, + api_key=api_key, + user_id=user_id, + knowledge_space_id=knowledge_space_id, + ) + return JSONResponse( + status_code=HTTPStatus.OK, + content=result + ) + except AppException: + # Re-raise AppException to be handled by global middleware + raise + except Exception as e: + logger.error(f"Failed to fetch iData datasets: {e}") + raise AppException( + ErrorCode.IDATA_SERVICE_ERROR, + f"Failed to fetch iData datasets: {str(e)}" + ) diff --git a/backend/apps/northbound_app.py b/backend/apps/northbound_app.py index a39877ded..e6aaf4eb6 100644 --- a/backend/apps/northbound_app.py +++ b/backend/apps/northbound_app.py @@ -1,12 +1,12 @@ import logging from http import HTTPStatus -from typing import Optional, Dict +from typing import Optional, Dict, Any import uuid -from fastapi import APIRouter, Body, Header, Request, HTTPException +from fastapi import APIRouter, Body, Header, Request, HTTPException, Query from fastapi.responses import JSONResponse -from consts.exceptions import UnauthorizedError, LimitExceededError, SignatureValidationError +from consts.exceptions import LimitExceededError, UnauthorizedError from services.northbound_service import ( NorthboundContext, get_conversation_history, @@ -14,86 +14,89 @@ start_streaming_chat, stop_chat, get_agent_info_list, - update_conversation_title + update_conversation_title, ) -from utils.auth_utils import get_current_user_id, validate_aksk_authentication +from utils.auth_utils import validate_bearer_token, get_user_and_tenant_by_access_key router = APIRouter(prefix="/nb/v1", tags=["northbound"]) -def _get_header(headers: Dict[str, str], name: str) -> Optional[str]: - for k, v in headers.items(): - if k.lower() == name.lower(): - return v - return None +async def _get_northbound_context(request: Request) -> NorthboundContext: + """ + Build northbound context from request. + Authentication: Bearer Token (API Key) in Authorization header + - Authorization: Bearer -async def _parse_northbound_context(request: Request) -> NorthboundContext: - """ - Build northbound context from headers. + The user_id and tenant_id are derived from the access_key by querying + user_token_info_t and user_tenant_t tables. - - X-Access-Key: Access key for AK/SK authentication - - X-Timestamp: Timestamp for signature validation - - X-Signature: HMAC-SHA256 signature signed with secret key - - Authorization: Bearer , jwt contains sub (user_id) - - X-Request-Id: optional, generated if not provided + Optional headers: + - X-Request-Id: Request ID, generated if not provided """ - # 1. Verify AK/SK signature + # 1. Validate Bearer Token and extract access_key try: - # Get request body for signature verification - request_body = "" - if request.method in ["POST", "PUT", "PATCH"]: - try: - body_bytes = await request.body() - request_body = body_bytes.decode('utf-8') if body_bytes else "" - except Exception as e: - logging.warning( - f"Cannot read request body for signature verification: {e}") - request_body = "" - - validate_aksk_authentication(request.headers, request_body) - except (UnauthorizedError, LimitExceededError, SignatureValidationError) as e: - raise e + auth_header = request.headers.get("Authorization") + is_valid, token_info = validate_bearer_token(auth_header) + + if not is_valid or not token_info: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail="Invalid or missing bearer token" + ) + + # Extract access_key from the token + access_key = auth_header.replace("Bearer ", "") if auth_header.startswith("Bearer ") else auth_header + + # Get user_id and tenant_id from access_key + user_tenant_info = get_user_and_tenant_by_access_key(access_key) + resolved_user_id = user_tenant_info.get("user_id") + resolved_tenant_id = user_tenant_info.get("tenant_id") + token_id = user_tenant_info.get("token_id") + + except HTTPException: + raise + except LimitExceededError as e: + logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) + raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, + detail="Too Many Requests: rate limit exceeded") + except UnauthorizedError as e: + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail=str(e) + ) except Exception as e: - logging.error(f"Failed to parse northbound context: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Internal Server Error: cannot parse northbound context") - - # 2. Parse JWT token - auth_header = _get_header(request.headers, "Authorization") - if not auth_header: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: No authorization header found") + logging.error(f"Failed to validate bearer token: {str(e)}", exc_info=e) + raise HTTPException( + status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: invalid API key" + ) - # Use auth_utils to parse JWT token - try: - user_id, tenant_id = get_current_user_id(auth_header) + if not resolved_user_id: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Missing user information for this access key" + ) - if not user_id: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: missing user_id in JWT token") - if not tenant_id: - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: unregistered user_id in JWT token") + if not resolved_tenant_id: + raise HTTPException( + status_code=HTTPStatus.BAD_REQUEST, + detail="Missing tenant information for this access key" + ) - except HTTPException as e: - # Preserve explicit HTTP errors raised during JWT parsing - raise e - except Exception as e: - logging.error(f"Failed to parse JWT token: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR, - detail="Internal Server Error: cannot parse JWT token") + request_id = request.headers.get("X-Request-Id") or str(uuid.uuid4()) - request_id = _get_header( - request.headers, "X-Request-Id") or str(uuid.uuid4()) + # Get authorization header if present, otherwise use a placeholder + auth_header_value = request.headers.get("Authorization", "Bearer placeholder") return NorthboundContext( request_id=request_id, - tenant_id=tenant_id, - user_id=str(user_id), - authorization=auth_header, + tenant_id=resolved_tenant_id, + user_id=resolved_user_id, + authorization=auth_header_value, + token_id=token_id, ) @@ -105,34 +108,27 @@ async def health_check(): @router.post("/chat/run") async def run_chat( request: Request, - conversation_id: str = Body(..., embed=True), + conversation_id: Optional[int] = Body(None, embed=True), agent_name: str = Body(..., embed=True), query: str = Body(..., embed=True), + meta_data: Optional[Dict[str, Any]] = Body(None, embed=True), idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"), ): try: - ctx: NorthboundContext = await _parse_northbound_context(request) + ctx: NorthboundContext = await _get_northbound_context(request) return await start_streaming_chat( ctx=ctx, - external_conversation_id=conversation_id, + conversation_id=conversation_id, agent_name=agent_name, query=query, + meta_data=meta_data, idempotency_key=idempotency_key, ) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: - # Propagate HTTP errors from context parsing without altering status/detail raise e except Exception as e: logging.error(f"Failed to run chat: {str(e)}", exc_info=e) @@ -141,22 +137,25 @@ async def run_chat( @router.get("/chat/stop/{conversation_id}") -async def stop_chat_stream(request: Request, conversation_id: str): +async def stop_chat_stream( + request: Request, + conversation_id: int, + meta_data: Optional[str] = Query(None, description="Optional metadata as JSON string"), +): + import json + parsed_meta_data = None + if meta_data: + try: + parsed_meta_data = json.loads(meta_data) + except json.JSONDecodeError: + pass try: - ctx: NorthboundContext = await _parse_northbound_context(request) - return await stop_chat(ctx=ctx, external_conversation_id=conversation_id) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") + ctx: NorthboundContext = await _get_northbound_context(request) + return await stop_chat(ctx=ctx, conversation_id=conversation_id, meta_data=parsed_meta_data) except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: raise e except Exception as e: @@ -166,22 +165,17 @@ async def stop_chat_stream(request: Request, conversation_id: str): @router.get("/conversations/{conversation_id}") -async def get_history(request: Request, conversation_id: str): +async def get_history( + request: Request, + conversation_id: int, +): try: - ctx: NorthboundContext = await _parse_northbound_context(request) - return await get_conversation_history(ctx=ctx, external_conversation_id=conversation_id) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") + ctx: NorthboundContext = await _get_northbound_context(request) + return await get_conversation_history(ctx=ctx, conversation_id=conversation_id) except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: raise e except Exception as e: @@ -193,20 +187,12 @@ async def get_history(request: Request, conversation_id: str): @router.get("/agents") async def list_agents(request: Request): try: - ctx: NorthboundContext = await _parse_northbound_context(request) + ctx: NorthboundContext = await _get_northbound_context(request) return await get_agent_info_list(ctx=ctx) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: raise e except Exception as e: @@ -218,20 +204,12 @@ async def list_agents(request: Request): @router.get("/conversations") async def list_convs(request: Request): try: - ctx: NorthboundContext = await _parse_northbound_context(request) + ctx: NorthboundContext = await _get_northbound_context(request) return await list_conversations(ctx=ctx) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: raise e except Exception as e: @@ -243,34 +221,35 @@ async def list_convs(request: Request): @router.put("/conversations/{conversation_id}/title") async def update_convs_title( request: Request, - conversation_id: str, - title: str, + conversation_id: int, + title: str = Query(..., description="New title"), + meta_data: Optional[str] = Query(None, description="Optional metadata as JSON string"), idempotency_key: Optional[str] = Header(None, alias="Idempotency-Key"), ): + import json + parsed_meta_data = None + if meta_data: + try: + parsed_meta_data = json.loads(meta_data) + except json.JSONDecodeError: + pass try: - ctx: NorthboundContext = await _parse_northbound_context(request) + ctx: NorthboundContext = await _get_northbound_context(request) result = await update_conversation_title( ctx=ctx, - external_conversation_id=conversation_id, + conversation_id=conversation_id, title=title, + meta_data=parsed_meta_data, idempotency_key=idempotency_key, ) headers_out = { "Idempotency-Key": result.get("idempotency_key", ""), "X-Request-Id": ctx.request_id} return JSONResponse(content=result, headers=headers_out) - except UnauthorizedError as e: - logging.error(f"Unauthorized: AK/SK authentication failed: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: AK/SK authentication failed") except LimitExceededError as e: logging.error(f"Too Many Requests: rate limit exceeded: {str(e)}", exc_info=e) raise HTTPException(status_code=HTTPStatus.TOO_MANY_REQUESTS, detail="Too Many Requests: rate limit exceeded") - except SignatureValidationError as e: - logging.error(f"Unauthorized: invalid signature: {str(e)}", exc_info=e) - raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, - detail="Unauthorized: invalid signature") except HTTPException as e: raise e except Exception as e: diff --git a/backend/apps/remote_mcp_app.py b/backend/apps/remote_mcp_app.py index cfc82146b..009e5cffa 100644 --- a/backend/apps/remote_mcp_app.py +++ b/backend/apps/remote_mcp_app.py @@ -387,7 +387,13 @@ async def add_mcp_from_config( except MCPContainerError as e: logger.error( f"Failed to start MCP container {service_name}: {e}") - errors.append(f"{service_name}: {str(e)}") + error_str = str(e) + # Check if error is related to image not found + if "not found" in error_str.lower() or "404" in error_str: + errors.append( + f"{service_name}: Image not found - MCP service startup image is missing") + else: + errors.append(f"{service_name}: {error_str}") except Exception as e: logger.error( f"Unexpected error adding MCP {service_name}: {e}") diff --git a/backend/apps/user_management_app.py b/backend/apps/user_management_app.py index c38b4e73c..d50cdc1f0 100644 --- a/backend/apps/user_management_app.py +++ b/backend/apps/user_management_app.py @@ -1,9 +1,10 @@ import logging from dotenv import load_dotenv -from fastapi import APIRouter, Request, HTTPException +from fastapi import APIRouter, Header, Query, Request, HTTPException from fastapi.responses import JSONResponse from http import HTTPStatus +from typing import Optional from supabase_auth.errors import AuthApiError, AuthWeakPasswordError @@ -11,7 +12,7 @@ from consts.exceptions import NoInviteCodeException, IncorrectInviteCodeException, UserRegistrationException from services.user_management_service import get_authorized_client, validate_token, \ check_auth_service_health, signup_user_with_invitation, signin_user, refresh_user_token, \ - get_session_by_authorization, get_user_info + get_session_by_authorization, get_user_info, create_token, list_tokens_by_user, delete_token from services.user_service import delete_user_and_cleanup from consts.exceptions import UnauthorizedError from utils.auth_utils import get_current_user_id @@ -44,7 +45,8 @@ async def signup(request: UserSignUpRequest): try: user_data = await signup_user_with_invitation(email=request.email, password=request.password, - invite_code=request.invite_code) + invite_code=request.invite_code, + auto_login=request.auto_login) success_message = "🎉 User account registered successfully! Please start experiencing the AI assistant service." return JSONResponse(status_code=HTTPStatus.OK, content={"message":success_message, "data":user_data}) @@ -273,3 +275,107 @@ async def revoke_user_account(request: Request): logging.error(f"User revoke failed: {str(e)}") raise HTTPException( status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="User revoke failed") + +@router.post("/tokens") +async def create_token_endpoint( + authorization: Optional[str] = Header(None) +): + """Create a new token for the authenticated user. + + The user_id is extracted from the Authorization header (JWT token). + Returns the complete token including the secret key. + """ + try: + if not authorization: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: No authorization header found") + + user_id, _ = get_current_user_id(authorization) + if not user_id: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: missing user_id in JWT token") + + result = create_token(str(user_id)) + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "success", "data": result} + ) + except HTTPException as e: + raise e + except Exception as e: + logging.error(f"Failed to create token: {str(e)}", exc_info=e) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Internal Server Error") + + +@router.get("/tokens") +async def list_tokens_endpoint( + user_id: str = Query(..., description="User ID to query tokens for"), + authorization: Optional[str] = Header(None) +): + """List all tokens for the specified user. + + Returns token information with masked access keys (middle part replaced with *). + """ + try: + if not authorization: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: No authorization header found") + + request_user_id, _ = get_current_user_id(authorization) + if not request_user_id: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: missing user_id in JWT token") + + # Only allow users to list their own tokens + if str(request_user_id) != user_id: + raise HTTPException(status_code=HTTPStatus.FORBIDDEN, + detail="Forbidden: cannot list tokens for other users") + + tokens = list_tokens_by_user(user_id) + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "success", "data": tokens} + ) + except HTTPException as e: + raise e + except Exception as e: + logging.error(f"Failed to list tokens: {str(e)}", exc_info=e) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Internal Server Error") + + +@router.delete("/tokens/{token_id}") +async def delete_token_endpoint( + token_id: int, + authorization: Optional[str] = Header(None) +): + """Soft delete a token. + + Only the owner of the token can delete it. + """ + try: + if not authorization: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: No authorization header found") + + user_id, _ = get_current_user_id(authorization) + if not user_id: + raise HTTPException(status_code=HTTPStatus.UNAUTHORIZED, + detail="Unauthorized: missing user_id in JWT token") + + success = delete_token(token_id, str(user_id)) + if not success: + raise HTTPException(status_code=HTTPStatus.NOT_FOUND, + detail="Token not found or not owned by user") + + return JSONResponse( + status_code=HTTPStatus.OK, + content={"message": "success", "data": {"token_id": token_id}} + ) + except HTTPException as e: + raise e + except Exception as e: + logging.error(f"Failed to delete token: {str(e)}", exc_info=e) + raise HTTPException( + status_code=HTTPStatus.INTERNAL_SERVER_ERROR, detail="Internal Server Error") diff --git a/backend/consts/const.py b/backend/consts/const.py index 32404bab4..2a48da8ce 100644 --- a/backend/consts/const.py +++ b/backend/consts/const.py @@ -36,6 +36,21 @@ class VectorDatabaseType(str, Enum): ROOT_DIR = os.getenv("ROOT_DIR") +# Preview Configuration +FILE_PREVIEW_SIZE_LIMIT = 100 * 1024 * 1024 # 100MB +# Limit concurrent Office-to-PDF conversions +MAX_CONCURRENT_CONVERSIONS = 5 +# Supported Office file MIME types +OFFICE_MIME_TYPES = [ + 'application/msword', # .doc + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document', # .docx + 'application/vnd.ms-excel', # .xls + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet', # .xlsx + 'application/vnd.ms-powerpoint', # .ppt + 'application/vnd.openxmlformats-officedocument.presentationml.presentation' # .pptx +] + + # Supabase Configuration SUPABASE_URL = os.getenv('SUPABASE_URL') SUPABASE_KEY = os.getenv('SUPABASE_KEY') @@ -261,6 +276,7 @@ class VectorDatabaseType(str, Enum): APP_NAME = "APP_NAME" APP_DESCRIPTION = "APP_DESCRIPTION" ICON_TYPE = "ICON_TYPE" +ICON_KEY = "ICON_KEY" AVATAR_URI = "AVATAR_URI" CUSTOM_ICON_URL = "CUSTOM_ICON_URL" TENANT_NAME = "TENANT_NAME" @@ -307,4 +323,4 @@ class VectorDatabaseType(str, Enum): MODEL_ENGINE_ENABLED = os.getenv("MODEL_ENGINE_ENABLED") # APP Version -APP_VERSION = "v1.8.0.2" +APP_VERSION = "v1.8.1" diff --git a/backend/consts/error_code.py b/backend/consts/error_code.py index 73decbbf3..072243de4 100644 --- a/backend/consts/error_code.py +++ b/backend/consts/error_code.py @@ -121,6 +121,13 @@ class ErrorCode(Enum): MODEL_CONFIG_INVALID = "090102" # Invalid model configuration MODEL_HEALTH_CHECK_FAILED = "090103" # Health check failed MODEL_PROVIDER_ERROR = "090104" # Model provider error + MODEL_PROMPT_GENERATION_FAILED = "090105" # Model prompt generation failed + # 02 - Model API errors + MODEL_API_KEY_INVALID = "090201" # API key is invalid or expired + MODEL_API_KEY_NO_PERMISSION = "090202" # API key does not have permission + MODEL_RATE_LIMIT_EXCEEDED = "090203" # Rate limit exceeded + MODEL_SERVICE_UNAVAILABLE = "090204" # Model service is temporarily unavailable + MODEL_CONNECTION_ERROR = "090205" # Failed to connect to model service # ==================== 10 Memory / 记忆管理 ==================== # 01 - Memory @@ -157,6 +164,14 @@ class ErrorCode(Enum): # 03 - ME Service ME_CONNECTION_FAILED = "130301" # ME service connection failed + # 04 - iData Service + IDATA_SERVICE_ERROR = "130401" # iData service error + IDATA_CONFIG_INVALID = "130402" # Invalid iData configuration + IDATA_CONNECTION_ERROR = "130403" # iData connection error + IDATA_AUTH_ERROR = "130404" # iData auth error + IDATA_RATE_LIMIT = "130405" # iData rate limit + IDATA_RESPONSE_ERROR = "130406" # iData response error + # ==================== 14 Northbound / 北向接口 ==================== # 01 - Request NORTHBOUND_REQUEST_FAILED = "140101" # Northbound request failed @@ -216,4 +231,10 @@ class ErrorCode(Enum): ErrorCode.DIFY_CONNECTION_ERROR: 502, ErrorCode.DIFY_RESPONSE_ERROR: 502, ErrorCode.DIFY_RATE_LIMIT: 429, + # iData (module 13) + ErrorCode.IDATA_CONFIG_INVALID: 400, + ErrorCode.IDATA_AUTH_ERROR: 401, + ErrorCode.IDATA_CONNECTION_ERROR: 502, + ErrorCode.IDATA_RESPONSE_ERROR: 502, + ErrorCode.IDATA_RATE_LIMIT: 429, } diff --git a/backend/consts/error_message.py b/backend/consts/error_message.py index aa7bf45e3..4ff1141c7 100644 --- a/backend/consts/error_message.py +++ b/backend/consts/error_message.py @@ -84,6 +84,13 @@ class ErrorMessage: ErrorCode.MODEL_CONFIG_INVALID: "Model configuration is invalid.", ErrorCode.MODEL_HEALTH_CHECK_FAILED: "Model health check failed.", ErrorCode.MODEL_PROVIDER_ERROR: "Model provider error.", + ErrorCode.MODEL_PROMPT_GENERATION_FAILED: "Model is unavailable. Please check the model status and try again.", + # 02 - Model API errors + ErrorCode.MODEL_API_KEY_INVALID: "Model API key is invalid or expired. Please check your API key configuration.", + ErrorCode.MODEL_API_KEY_NO_PERMISSION: "Model API key does not have permission. Please check your API key permissions.", + ErrorCode.MODEL_RATE_LIMIT_EXCEEDED: "Rate limit exceeded. Please try again later.", + ErrorCode.MODEL_SERVICE_UNAVAILABLE: "Model service is temporarily unavailable. Please try again later.", + ErrorCode.MODEL_CONNECTION_ERROR: "Failed to connect to model service. Please check your network and model configuration.", # ==================== 10 Memory / 记忆管理 ==================== ErrorCode.MEMORY_NOT_FOUND: "Memory not found.", diff --git a/backend/consts/exceptions.py b/backend/consts/exceptions.py index e9d270673..369c24aab 100644 --- a/backend/consts/exceptions.py +++ b/backend/consts/exceptions.py @@ -43,7 +43,7 @@ def __init__(self, error_code: ErrorCode, message: str = None, details: dict = N def to_dict(self) -> dict: return { - "code": int(self.error_code.value), + "code": str(self.error_code.value), # Keep as string to preserve leading zeros "message": self.message, "details": self.details if self.details else None } @@ -115,6 +115,21 @@ class IncorrectInviteCodeException(Exception): pass +class OfficeConversionException(Exception): + """Raised when Office-to-PDF conversion via data-process service fails.""" + pass + + +class UnsupportedFileTypeException(Exception): + """Raised when a file type is not supported for the requested operation.""" + pass + + +class FileTooLargeException(Exception): + """Raised when a file exceeds the maximum allowed size for the requested operation.""" + pass + + class UserRegistrationException(Exception): """Raised when user registration fails.""" pass diff --git a/backend/consts/model.py b/backend/consts/model.py index ae8e576a8..6aea42fa9 100644 --- a/backend/consts/model.py +++ b/backend/consts/model.py @@ -31,6 +31,7 @@ class UserSignUpRequest(BaseModel): email: EmailStr password: str = Field(..., min_length=6) invite_code: Optional[str] = None + auto_login: Optional[bool] = True # Whether to return session after signup class UserSignInRequest(BaseModel): @@ -114,6 +115,7 @@ class AppConfig(BaseModel): appName: str appDescription: str iconType: str + iconKey: Optional[str] = "search" customIconUrl: Optional[str] = None avatarUri: Optional[str] = None modelEngineEnabled: bool = False diff --git a/backend/consts/provider.py b/backend/consts/provider.py index 7fd783015..38bbc4027 100644 --- a/backend/consts/provider.py +++ b/backend/consts/provider.py @@ -6,11 +6,21 @@ class ProviderEnum(str, Enum): SILICON = "silicon" OPENAI = "openai" MODELENGINE = "modelengine" + DASHSCOPE = "dashscope" + TOKENPONY = "tokenpony" # Silicon Flow SILICON_BASE_URL = "https://api.siliconflow.cn/v1/" SILICON_GET_URL = "https://api.siliconflow.cn/v1/models" +# Dashcope +DASHSCOPE_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1/" +DASHSCOPE_GET_URL = "https://dashscope.aliyuncs.com/api/v1/models" + +# TokenPony +TOKENPONY_BASE_URL = "https://api.tokenpony.cn/v1/" +TOKENPONY_GET_URL = "https://api.tokenpony.cn/v1/models" + # ModelEngine # Base URL and API key are loaded from environment variables at runtime diff --git a/backend/database/attachment_db.py b/backend/database/attachment_db.py index d7764b3a2..2e6249468 100644 --- a/backend/database/attachment_db.py +++ b/backend/database/attachment_db.py @@ -169,6 +169,42 @@ def get_file_size_from_minio(object_name: str, bucket: Optional[str] = None) -> return minio_client.get_file_size(object_name, bucket) +def file_exists(object_name: str, bucket: Optional[str] = None) -> bool: + """ + Check if a file exists in the bucket. + + Args: + object_name: Object name in storage + bucket: Bucket name, if not specified will use default bucket + + Returns: + bool: True if file exists, False otherwise + """ + try: + return minio_client.file_exists(object_name, bucket) + except Exception: + return False + + +def copy_file(source_object: str, dest_object: str, bucket: Optional[str] = None) -> Dict[str, Any]: + """ + Copy a file within the same bucket (atomic operation in MinIO). + + Args: + source_object: Source object name + dest_object: Destination object name + bucket: Bucket name, if not specified will use default bucket + + Returns: + Dict[str, Any]: Result containing success flag and error message (if any) + """ + success, result = minio_client.copy_file(source_object, dest_object, bucket) + if success: + return {"success": True, "object_name": result} + else: + return {"success": False, "error": result} + + def list_files(prefix: str = "", bucket: Optional[str] = None) -> List[Dict[str, Any]]: """ List files in bucket @@ -269,6 +305,7 @@ def get_content_type(file_path: str) -> str: '.pptx': 'application/vnd.openxmlformats-officedocument.presentationml.presentation', '.txt': 'text/plain', '.csv': 'text/csv', + '.md': 'text/markdown', '.html': 'text/html', '.htm': 'text/html', '.json': 'application/json', diff --git a/backend/database/client.py b/backend/database/client.py index c82f78df3..37e5dba03 100644 --- a/backend/database/client.py +++ b/backend/database/client.py @@ -213,6 +213,33 @@ def get_file_stream(self, object_name: str, bucket: Optional[str] = None) -> Tup """ return self._storage_client.get_file_stream(object_name, bucket) + def file_exists(self, object_name: str, bucket: Optional[str] = None) -> bool: + """ + Check if file exists in MinIO + + Args: + object_name: Object name + bucket: Bucket name, if not specified use default bucket + + Returns: + bool: True if file exists, False otherwise + """ + return self._storage_client.exists(object_name, bucket) + + def copy_file(self, source_object: str, dest_object: str, bucket: Optional[str] = None) -> Tuple[bool, str]: + """ + Copy a file within the same bucket (atomic operation) + + Args: + source_object: Source object name + dest_object: Destination object name + bucket: Bucket name, if not specified use default bucket + + Returns: + Tuple[bool, str]: (Success status, Destination object name or error message) + """ + return self._storage_client.copy_file(source_object, dest_object, bucket) + # Create global database and MinIO client instances db_client = PostgresClient() diff --git a/backend/database/conversation_db.py b/backend/database/conversation_db.py index 0267d77c2..18c0ee9fc 100644 --- a/backend/database/conversation_db.py +++ b/backend/database/conversation_db.py @@ -4,7 +4,7 @@ from sqlalchemy import asc, desc, func, insert, select, update -from .client import as_dict, get_db_session +from .client import as_dict, db_client, get_db_session from .db_models import ( ConversationMessage, ConversationMessageUnit, @@ -328,11 +328,12 @@ def rename_conversation(conversation_id: int, new_title: str, user_id: Optional[ # Ensure conversation_id is of integer type conversation_id = int(conversation_id) - # Prepare update data + # Prepare update data with UTF-8 encoding for title update_data = { "conversation_title": new_title, "update_time": func.current_timestamp() } + update_data = db_client.clean_string_values(update_data) if user_id: update_data = add_update_tracking(update_data, user_id) diff --git a/backend/database/db_models.py b/backend/database/db_models.py index 36f475f53..80dcc87eb 100644 --- a/backend/database/db_models.py +++ b/backend/database/db_models.py @@ -1,4 +1,5 @@ from sqlalchemy import BigInteger, Boolean, Column, Integer, JSON, Numeric, PrimaryKeyConstraint, Sequence, String, Text, TIMESTAMP +from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import DeclarativeBase from sqlalchemy.sql import func @@ -483,3 +484,31 @@ class AgentVersion(TableBase): source_version_no = Column(Integer, doc="Source version number. If this version is a rollback, record the source version") source_type = Column(String(30), doc="Source type: NORMAL (normal publish) / ROLLBACK (rollback and republish)") status = Column(String(30), default="RELEASED", doc="Version status: RELEASED / DISABLED / ARCHIVED") + + +class UserTokenInfo(TableBase): + """ + User token (AK/SK) information table + """ + __tablename__ = "user_token_info_t" + __table_args__ = {"schema": SCHEMA} + + token_id = Column(Integer, Sequence("user_token_info_t_token_id_seq", schema=SCHEMA), + primary_key=True, nullable=False, doc="Token ID, unique primary key") + access_key = Column(String(100), nullable=False, doc="Access Key (AK)") + user_id = Column(String(100), nullable=False, doc="User ID who owns this token") + + +class UserTokenUsageLog(TableBase): + """ + User token usage log table + """ + __tablename__ = "user_token_usage_log_t" + __table_args__ = {"schema": SCHEMA} + + token_usage_id = Column(Integer, Sequence("user_token_usage_log_t_token_usage_id_seq", schema=SCHEMA), + primary_key=True, nullable=False, doc="Token usage log ID, unique primary key") + token_id = Column(Integer, nullable=False, doc="Foreign key to user_token_info_t.token_id") + call_function_name = Column(String(100), doc="API function name being called") + related_id = Column(Integer, doc="Related resource ID (e.g., conversation_id)") + meta_data = Column(JSONB, doc="Additional metadata for this usage log entry, stored as JSON") diff --git a/backend/database/token_db.py b/backend/database/token_db.py new file mode 100644 index 000000000..70d53a42e --- /dev/null +++ b/backend/database/token_db.py @@ -0,0 +1,189 @@ +""" +Database operations for user API token (API Key) management. +""" +import secrets +from typing import Any, Dict, List, Optional + +from database.client import get_db_session +from database.db_models import UserTokenInfo, UserTokenUsageLog + + +def generate_access_key() -> str: + """Generate a random access key with format nexent-xxxxx...""" + random_part = secrets.token_hex(12) # 24 hex characters for more entropy + return f"nexent-{random_part}" + + +def create_token(access_key: str, user_id: str) -> Dict[str, Any]: + """Create a new token record in the database. + + Args: + access_key: The access key (API Key). + user_id: The user ID who owns this token. + + Returns: + Dictionary containing the created token information. + """ + with get_db_session() as session: + token = UserTokenInfo( + access_key=access_key, + user_id=user_id, + created_by=user_id, + updated_by=user_id, + delete_flag='N' + ) + session.add(token) + session.flush() + + return { + "token_id": token.token_id, + "access_key": token.access_key, + "user_id": token.user_id + } + + +def list_tokens_by_user(user_id: str) -> List[Dict[str, Any]]: + """List all active tokens for the specified user. + + Args: + user_id: The user ID to query tokens for. + + Returns: + List of token information with masked access keys. + """ + with get_db_session() as session: + tokens = session.query(UserTokenInfo).filter( + UserTokenInfo.user_id == user_id, + UserTokenInfo.delete_flag == 'N' + ).order_by(UserTokenInfo.create_time.desc()).all() + + return [ + { + "token_id": token.token_id, + "access_key": token.access_key, + "user_id": token.user_id, + "create_time": token.create_time.isoformat() if token.create_time else None + } + for token in tokens + ] + + +def get_token_by_id(token_id: int) -> UserTokenInfo: + """Get a token by its ID. + + Args: + token_id: The token ID to query. + + Returns: + UserTokenInfo object if found and active, None otherwise. + """ + with get_db_session() as session: + return session.query(UserTokenInfo).filter( + UserTokenInfo.token_id == token_id, + UserTokenInfo.delete_flag == 'N' + ).first() + + +def get_token_by_access_key(access_key: str) -> Optional[Dict[str, Any]]: + """Get a token by its access key. + + Args: + access_key: The access key to query. + + Returns: + Token information dict if found and active, None otherwise. + """ + with get_db_session() as session: + token = session.query(UserTokenInfo).filter( + UserTokenInfo.access_key == access_key, + UserTokenInfo.delete_flag == 'N' + ).first() + + if token: + return { + "token_id": token.token_id, + "access_key": token.access_key, + "user_id": token.user_id, + "delete_flag": token.delete_flag + } + return None + + +def delete_token(token_id: int, user_id: str) -> bool: + """Soft delete a token by setting delete_flag to 'Y'. + + Args: + token_id: The token ID to delete. + user_id: The user ID who owns this token (for authorization). + + Returns: + True if the token was deleted, False if not found or not owned by user. + """ + with get_db_session() as session: + token = session.query(UserTokenInfo).filter( + UserTokenInfo.token_id == token_id, + UserTokenInfo.user_id == user_id, + UserTokenInfo.delete_flag == 'N' + ).first() + + if not token: + return False + + token.delete_flag = 'Y' + token.updated_by = user_id + return True + + +def log_token_usage( + token_id: int, + call_function_name: str, + related_id: Optional[int], + created_by: str, + metadata: Optional[Dict[str, Any]] = None +) -> int: + """Log token usage to the database. + + Args: + token_id: The token ID used. + call_function_name: The API function name being called. + related_id: Related resource ID (e.g., conversation_id). + created_by: User ID who initiated the call. + metadata: Optional additional metadata for this usage log entry. + + Returns: + The created token_usage_id. + """ + with get_db_session() as session: + usage_log = UserTokenUsageLog( + token_id=token_id, + call_function_name=call_function_name, + related_id=related_id, + created_by=created_by, + meta_data=metadata + ) + session.add(usage_log) + session.flush() + return usage_log.token_usage_id + + +def get_latest_usage_metadata(token_id: int, related_id: int, call_function_name: str) -> Optional[Dict[str, Any]]: + """Get the latest metadata for a given token, related_id and function name. + + Args: + token_id: The token ID used. + related_id: Related resource ID (e.g., conversation_id). + call_function_name: The API function name. + + Returns: + The metadata dict if found, None otherwise. + """ + with get_db_session() as session: + usage_log = session.query(UserTokenUsageLog).filter( + UserTokenUsageLog.token_id == token_id, + UserTokenUsageLog.related_id == related_id, + UserTokenUsageLog.call_function_name == call_function_name + ).order_by(UserTokenUsageLog.create_time.desc()).first() + + if usage_log and usage_log.meta_data: + return usage_log.meta_data + return None diff --git a/backend/database/tool_db.py b/backend/database/tool_db.py index 0001315a7..2a64c47d6 100644 --- a/backend/database/tool_db.py +++ b/backend/database/tool_db.py @@ -4,6 +4,7 @@ from database.agent_db import logger from database.client import get_db_session, filter_property, as_dict from database.db_models import ToolInstance, ToolInfo +from consts.model import ToolSourceEnum def create_tool(tool_info, version_no: int = 0): @@ -36,7 +37,7 @@ def create_or_update_tool_by_tool_info(tool_info, tenant_id: str, user_id: str, Args: tool_info: Dictionary containing tool information tenant_id: Tenant ID for filtering, mandatory - user_id: Optional user ID for filtering + user_id: User ID for updating (will be set as the last updater) version_no: Version number to filter. Default 0 = draft/editing state Returns: @@ -47,9 +48,10 @@ def create_or_update_tool_by_tool_info(tool_info, tenant_id: str, user_id: str, with get_db_session() as session: # Query if there is an existing ToolInstance + # Note: Do not filter by user_id to avoid creating duplicate instances + # for the same agent_id and tool_id when different users save query = session.query(ToolInstance).filter( ToolInstance.tenant_id == tenant_id, - ToolInstance.user_id == user_id, ToolInstance.agent_id == tool_info_dict['agent_id'], ToolInstance.delete_flag != 'Y', ToolInstance.tool_id == tool_info_dict['tool_id'], @@ -62,7 +64,12 @@ def create_or_update_tool_by_tool_info(tool_info, tenant_id: str, user_id: str, if hasattr(tool_instance, key): setattr(tool_instance, key, value) else: - create_tool(tool_info_dict, version_no) + # Create a new ToolInstance + new_tool_instance = ToolInstance( + **filter_property(tool_info_dict, ToolInstance)) + session.add(new_tool_instance) + session.flush() # Flush to get the ID + tool_instance = new_tool_instance return tool_instance @@ -190,13 +197,23 @@ def check_tool_list_initialized(tenant_id: str) -> bool: def update_tool_table_from_scan_tool_list(tenant_id: str, user_id: str, tool_list: List[ToolInfo]): """ scan all tools and update the tool table in PG database, remove the duplicate tools + For MCP tools, use name&source&usage as unique key to allow same tool name from different MCP servers """ with get_db_session() as session: # get all existing tools (including complete information) existing_tools = session.query(ToolInfo).filter(ToolInfo.delete_flag != 'Y', ToolInfo.author == tenant_id).all() - existing_tool_dict = { - f"{tool.name}&{tool.source}": tool for tool in existing_tools} + # Build existing_tool_dict with different keys for MCP vs non-MCP tools + existing_tool_dict = {} + for tool in existing_tools: + if tool.source == ToolSourceEnum.MCP.value: + # For MCP tools, use name + source + usage (MCP server name) as unique key + key = f"{tool.name}&{tool.source}&{tool.usage or ''}" + else: + # For other tools, use name + source as unique key + key = f"{tool.name}&{tool.source}" + existing_tool_dict[key] = tool + # set all tools to unavailable for tool in existing_tools: tool.is_available = False @@ -208,9 +225,15 @@ def update_tool_table_from_scan_tool_list(tenant_id: str, user_id: str, tool_lis is_available = True if re.match( r'^[a-zA-Z_][a-zA-Z0-9_]*$', tool.name) is not None else False - if f"{tool.name}&{tool.source}" in existing_tool_dict: - # by tool name and source to update the existing tool - existing_tool = existing_tool_dict[f"{tool.name}&{tool.source}"] + # Use same key generation logic as above + if tool.source == ToolSourceEnum.MCP.value: + tool_key = f"{tool.name}&{tool.source}&{tool.usage or ''}" + else: + tool_key = f"{tool.name}&{tool.source}" + + if tool_key in existing_tool_dict: + # by tool name, source, and usage (for MCP) to update the existing tool + existing_tool = existing_tool_dict[tool_key] for key, value in filtered_tool_data.items(): setattr(existing_tool, key, value) existing_tool.updated_by = user_id @@ -308,6 +331,7 @@ def delete_tools_by_agent_id(agent_id, tenant_id, user_id, version_no: int = 0): ToolInstance.delete_flag: 'Y', 'updated_by': user_id }) + def search_last_tool_instance_by_tool_id(tool_id: int, tenant_id: str, user_id: str, version_no: int = 0): """ Query the latest ToolInstance by tool_id. @@ -331,4 +355,4 @@ def search_last_tool_instance_by_tool_id(tool_id: int, tenant_id: str, user_id: ToolInstance.delete_flag != 'Y' ).order_by(ToolInstance.update_time.desc()) tool_instance = query.first() - return as_dict(tool_instance) if tool_instance else None \ No newline at end of file + return as_dict(tool_instance) if tool_instance else None diff --git a/backend/middleware/exception_handler.py b/backend/middleware/exception_handler.py index 14d9ebb38..6ec521f12 100644 --- a/backend/middleware/exception_handler.py +++ b/backend/middleware/exception_handler.py @@ -74,7 +74,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: return JSONResponse( status_code=http_status, content={ - "code": int(exc.error_code.value), + "code": exc.error_code.value, # Keep as string to preserve leading zeros "message": exc.message, "trace_id": trace_id, "details": exc.details if exc.details else None @@ -88,7 +88,7 @@ async def dispatch(self, request: Request, call_next: Callable) -> Response: return JSONResponse( status_code=exc.status_code, content={ - "code": int(error_code.value), + "code": error_code.value, # Keep as string to preserve leading zeros "message": exc.detail, "trace_id": trace_id } @@ -141,7 +141,7 @@ def create_error_response( return JSONResponse( status_code=status, content={ - "code": int(error_code.value), + "code": error_code.value, # Keep as string to preserve leading zeros "message": message or ErrorMessage.get_message(error_code), "trace_id": trace_id, "details": details diff --git a/backend/services/config_sync_service.py b/backend/services/config_sync_service.py index 7c7c66e7e..9fe50813a 100644 --- a/backend/services/config_sync_service.py +++ b/backend/services/config_sync_service.py @@ -13,6 +13,7 @@ DEFAULT_APP_NAME_ZH, DEFAULT_GROUP_ID, ICON_TYPE, + ICON_KEY, LANGUAGE, MODEL_CONFIG_MAPPING, LANGUAGE, @@ -139,6 +140,7 @@ def build_app_config(language: str, tenant_id: str) -> dict: "defaultGroupId": tenant_config_manager.get_app_config(DEFAULT_GROUP_ID, tenant_id=tenant_id) or "", "icon": { "type": tenant_config_manager.get_app_config(ICON_TYPE, tenant_id=tenant_id) or "preset", + "iconKey": tenant_config_manager.get_app_config(ICON_KEY, tenant_id=tenant_id) or "search", "avatarUri": tenant_config_manager.get_app_config(AVATAR_URI, tenant_id=tenant_id) or "", "customUrl": tenant_config_manager.get_app_config(CUSTOM_ICON_URL, tenant_id=tenant_id) or "" }, diff --git a/backend/services/data_process_service.py b/backend/services/data_process_service.py index bce279a4c..8c44c15e6 100644 --- a/backend/services/data_process_service.py +++ b/backend/services/data_process_service.py @@ -4,6 +4,7 @@ import io import logging import os +import shutil import tempfile import threading import time @@ -18,12 +19,18 @@ from transformers import CLIPProcessor, CLIPModel from nexent.data_process.core import DataProcessCore -from consts.const import CLIP_MODEL_PATH, IMAGE_FILTER, REDIS_BACKEND_URL, REDIS_URL +from consts.const import CLIP_MODEL_PATH, IMAGE_FILTER, MAX_CONCURRENT_CONVERSIONS, REDIS_BACKEND_URL, REDIS_URL +from consts.exceptions import OfficeConversionException from consts.model import BatchTaskRequest +from database.attachment_db import delete_file, file_exists, get_file_size_from_minio, get_file_stream, upload_file +from utils.file_management_utils import convert_office_to_pdf from data_process.app import app as celery_app from data_process.tasks import process, forward from data_process.utils import get_task_info, get_all_task_ids_from_redis +# Limit concurrent LibreOffice processes to avoid resource exhaustion +_conversion_semaphore = asyncio.Semaphore(MAX_CONCURRENT_CONVERSIONS) + # Configure logging logger = logging.getLogger("data_process.service") @@ -551,6 +558,81 @@ async def process_uploaded_text_file(self, file_content: bytes, filename: str, c "chunking_strategy": chunking_strategy } + async def convert_office_to_pdf_impl(self, object_name: str, pdf_object_name: str) -> None: + """Full conversion pipeline: download → convert → upload → validate → cleanup. + + All five steps run inside data-process so that LibreOffice only needs to be + installed in this container. + + Args: + object_name: Source Office file path in MinIO. + pdf_object_name: Destination PDF path in MinIO (final, not temp). + """ + async with _conversion_semaphore: + temp_dir = None + try: + temp_dir = tempfile.mkdtemp(prefix='office_convert_') + + # Step 1: Download original Office file from MinIO + original_stream = get_file_stream(object_name) + if original_stream is None: + raise OfficeConversionException(f"Source file not found in storage: {object_name}") + + original_filename = os.path.basename(object_name) + input_path = os.path.join(temp_dir, original_filename) + with open(input_path, 'wb') as f: + while chunk := original_stream.read(8192): + f.write(chunk) + + # Step 2: Local conversion using LibreOffice + try: + pdf_path = await convert_office_to_pdf(input_path, temp_dir, timeout=30) + except Exception as exc: + raise OfficeConversionException(f"LibreOffice conversion failed: {exc}") from exc + + # Step 3: Upload converted PDF to MinIO + result = upload_file(file_path=pdf_path, object_name=pdf_object_name) + if not result.get('success'): + raise OfficeConversionException( + f"Failed to upload PDF to MinIO: {result.get('error', 'Unknown error')}" + ) + + # Step 4: Validate the uploaded PDF (header check + minimum size) + remote_size = get_file_size_from_minio(pdf_object_name) + if remote_size <= 0: + raise OfficeConversionException("PDF validation failed: cannot read remote file size") + if remote_size < 100: + raise OfficeConversionException( + f"PDF validation failed: file too small ({remote_size} bytes)" + ) + remote_stream = get_file_stream(pdf_object_name) + if remote_stream is None: + raise OfficeConversionException("PDF validation failed: cannot read uploaded file") + try: + header = remote_stream.read(5) + finally: + try: + remote_stream.close() + except Exception: + pass + if not header.startswith(b'%PDF-'): + raise OfficeConversionException("PDF validation failed: invalid PDF header") + + except OfficeConversionException: + # Clean up any partially-uploaded remote PDF so a future retry starts clean + if file_exists(pdf_object_name): + delete_file(pdf_object_name) + raise + except Exception as exc: + raise OfficeConversionException(f"Unexpected error during conversion: {exc}") from exc + finally: + # Step 5: Clean up local temporary directory + if temp_dir and os.path.exists(temp_dir): + try: + shutil.rmtree(temp_dir) + except Exception as cleanup_err: + logger.warning(f"Failed to cleanup temp dir '{temp_dir}': {cleanup_err}") + def convert_celery_states_to_custom(self, process_celery_state: Optional[str], forward_celery_state: Optional[str]) -> str: """Map Celery task states to a custom frontend state string. diff --git a/backend/services/file_management_service.py b/backend/services/file_management_service.py index 8215be810..39b3af858 100644 --- a/backend/services/file_management_service.py +++ b/backend/services/file_management_service.py @@ -1,20 +1,33 @@ import asyncio +import hashlib import logging import os from io import BytesIO from pathlib import Path -from typing import List, Optional +from typing import List, Optional, Tuple +import httpx from fastapi import UploadFile -from consts.const import UPLOAD_FOLDER, MAX_CONCURRENT_UPLOADS, MODEL_CONFIG_MAPPING +from consts.const import ( + DATA_PROCESS_SERVICE, + FILE_PREVIEW_SIZE_LIMIT, + MAX_CONCURRENT_UPLOADS, + MODEL_CONFIG_MAPPING, + OFFICE_MIME_TYPES, + UPLOAD_FOLDER, +) +from consts.exceptions import FileTooLargeException, NotFoundException, OfficeConversionException, UnsupportedFileTypeException from database.attachment_db import ( - upload_fileobj, - get_file_url, + copy_file, + delete_file, + file_exists, get_content_type, + get_file_size_from_minio, get_file_stream, - delete_file, - list_files + get_file_url, + list_files, + upload_fileobj, ) from services.vectordatabase_service import ElasticSearchService, get_vector_db_core from utils.config_utils import tenant_config_manager, get_model_name_from_config @@ -28,6 +41,10 @@ upload_dir.mkdir(exist_ok=True) upload_semaphore = asyncio.Semaphore(MAX_CONCURRENT_UPLOADS) +# Per-file locks prevent duplicate conversions of the same file +_conversion_locks: dict[str, asyncio.Lock] = {} +_conversion_locks_guard = asyncio.Lock() + logger = logging.getLogger("file_management_service") @@ -195,4 +212,134 @@ def get_llm_model(tenant_id: str): max_context_tokens=main_model_config.get("max_tokens"), ssl_verify=main_model_config.get("ssl_verify", True), ) - return long_text_to_text_model \ No newline at end of file + return long_text_to_text_model + + +async def preview_file_impl(object_name: str) -> Tuple[BytesIO, str]: + """ + Preview a file by returning its contents as a stream. + + Args: + object_name: File object name in storage + + Returns: + Tuple[BytesIO, str]: (file_stream, content_type) + """ + file_size = get_file_size_from_minio(object_name) + if file_size > FILE_PREVIEW_SIZE_LIMIT: + raise FileTooLargeException( + f"File size {file_size} bytes exceeds the {FILE_PREVIEW_SIZE_LIMIT // (1024 * 1024)} MB preview limit" + ) + + content_type = get_content_type(object_name) + + # PDF, images, and text files - return directly + if content_type == 'application/pdf' or content_type.startswith('image/') or content_type in ['text/plain', 'text/csv', 'text/markdown']: + file_stream = get_file_stream(object_name) + if file_stream is None: + raise NotFoundException("File not found or failed to read from storage") + return file_stream, content_type + + # Office documents - convert to PDF with caching + elif content_type in OFFICE_MIME_TYPES: + name_without_ext = object_name.rsplit('.', 1)[0] if '.' in object_name else object_name + hash_suffix = hashlib.md5(object_name.encode()).hexdigest()[:8] + pdf_object_name = f"preview/converted/{name_without_ext}_{hash_suffix}.pdf" + temp_pdf_object_name = f"preview/converting/{name_without_ext}_{hash_suffix}.pdf.tmp" + + # Fast path: return from cache without acquiring any lock + cached_stream = _get_cached_pdf_stream(pdf_object_name) + if cached_stream is not None: + return cached_stream, 'application/pdf' + + # Slow path: convert with locking + file_stream = await _convert_office_to_cached_pdf(object_name, pdf_object_name, temp_pdf_object_name) + return file_stream, 'application/pdf' + + # Unsupported file type + else: + raise UnsupportedFileTypeException(f"Unsupported file type for preview: {content_type}") + + +def _get_cached_pdf_stream(pdf_object_name: str) -> Optional[BytesIO]: + """ + Return the cached PDF stream if available, or None if missing or corrupted. + + If the file exists but cannot be read, the corrupted entry is deleted so + a subsequent call will trigger a fresh conversion. + """ + if file_exists(pdf_object_name): + file_stream = get_file_stream(pdf_object_name) + if file_stream is None: + logger.warning(f"Corrupted cache detected (cannot read), deleting: {pdf_object_name}") + delete_file(pdf_object_name) + return None + return file_stream + return None + + +async def _convert_office_to_cached_pdf( + object_name: str, + pdf_object_name: str, + temp_pdf_object_name: str, +) -> BytesIO: + """ + Convert an Office document to PDF and store the result in MinIO. + + Args: + object_name: Source Office file path in MinIO + pdf_object_name: Final cached PDF path in MinIO + temp_pdf_object_name: Temporary PDF path used during conversion + + Returns: + BytesIO stream of the converted PDF + """ + # Get or create a lock for this specific file to prevent duplicate conversions + async with _conversion_locks_guard: + if object_name not in _conversion_locks: + _conversion_locks[object_name] = asyncio.Lock() + file_lock = _conversion_locks[object_name] + + try: + async with file_lock: + # Double-check: another request may have completed the conversion while we waited + cached_stream = _get_cached_pdf_stream(pdf_object_name) + if cached_stream is not None: + return cached_stream + + # Conversion semaphore is enforced inside the data-process service + try: + # Request conversion: data-process downloads, converts, uploads to temp path, validates + async with httpx.AsyncClient(timeout=120.0) as client: + response = await client.post( + f"{DATA_PROCESS_SERVICE}/tasks/convert_to_pdf", + data={ + "object_name": object_name, + "pdf_object_name": temp_pdf_object_name, + }, + ) + if response.status_code != 200: + raise Exception( + f"data-process conversion returned {response.status_code}: {response.text}" + ) + + # Atomic move from temp to final location, then clean up temp + copy_result = copy_file(source_object=temp_pdf_object_name, dest_object=pdf_object_name) + if not copy_result.get('success'): + raise Exception(f"Failed to finalize PDF cache: {copy_result.get('error', 'Unknown error')}") + delete_file(temp_pdf_object_name) + + except Exception as e: + if file_exists(temp_pdf_object_name): + delete_file(temp_pdf_object_name) + logger.error(f"Office conversion failed: {str(e)}") + raise OfficeConversionException(f"Failed to convert Office document to PDF: {str(e)}") from e + finally: + # Clean up the file lock (prevents memory leak for many unique files) + async with _conversion_locks_guard: + _conversion_locks.pop(object_name, None) + + file_stream = get_file_stream(pdf_object_name) + if file_stream is None: + raise NotFoundException("Converted PDF not found or failed to read from storage") + return file_stream diff --git a/backend/services/idata_service.py b/backend/services/idata_service.py new file mode 100644 index 000000000..691130dc0 --- /dev/null +++ b/backend/services/idata_service.py @@ -0,0 +1,359 @@ +""" +iData Service Layer +Handles API calls to iData for knowledge space operations. + +This service layer provides functionality to interact with iData's API, +including fetching knowledge spaces and transforming responses +to a format compatible with the frontend. +""" +import json +import logging +from typing import Any, Dict, List + +import httpx + +from consts.error_code import ErrorCode +from consts.exceptions import AppException +from nexent.utils.http_client_manager import http_client_manager + +logger = logging.getLogger("idata_service") + + +def _validate_idata_base_params( + idata_api_base: str, + api_key: str, + user_id: str, +) -> None: + """ + Validate common iData API parameters. + + Args: + idata_api_base: iData API base URL + api_key: iData API key + user_id: iData user ID + + Raises: + AppException: If any parameter is invalid + """ + if not idata_api_base or not isinstance(idata_api_base, str): + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + "iData API URL is required and must be a non-empty string" + ) + + if not (idata_api_base.startswith("http://") or idata_api_base.startswith("https://")): + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + "iData API URL must start with http:// or https://" + ) + + if not api_key or not isinstance(api_key, str): + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + "iData API key is required and must be a non-empty string" + ) + + if not user_id or not isinstance(user_id, str): + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + "iData user ID is required and must be a non-empty string" + ) + + +def _normalize_api_base(idata_api_base: str) -> str: + """ + Normalize API base URL by removing trailing slash. + + Args: + idata_api_base: iData API base URL + + Returns: + Normalized API base URL + """ + return idata_api_base.rstrip("/") + + +def _make_idata_request( + api_base: str, + url: str, + headers: Dict[str, str], + request_body: Dict[str, Any], +) -> Dict[str, Any]: + """ + Make HTTP POST request to iData API and handle common errors. + + Args: + api_base: Normalized API base URL + url: Full request URL + headers: Request headers + request_body: Request body as dictionary + + Returns: + Parsed JSON response + + Raises: + AppException: If request fails or response is invalid + """ + logger.info(f"Making iData API request to: {url}") + + try: + # Use shared HttpClientManager for connection pooling + # Note: ssl_verify is set to False as per requirement (self-signed certificate) + client = http_client_manager.get_sync_client( + base_url=api_base, + timeout=10.0, + verify_ssl=False + ) + response = client.post(url, headers=headers, json=request_body) + response.raise_for_status() + + return response.json() + + except httpx.RequestError as e: + logger.error(f"iData API request failed: {str(e)}") + raise AppException( + ErrorCode.IDATA_CONNECTION_ERROR, + f"iData API request failed: {str(e)}" + ) + except httpx.HTTPStatusError as e: + logger.error( + f"iData API HTTP error: {str(e)}, status_code: {e.response.status_code}") + # Map HTTP status to specific error code + if e.response.status_code == 401: + logger.error("Raising IDATA_AUTH_ERROR for 401 error") + raise AppException( + ErrorCode.IDATA_AUTH_ERROR, + f"iData authentication failed: {str(e)}" + ) + elif e.response.status_code == 403: + logger.error("Raising IDATA_AUTH_ERROR for 403 error") + raise AppException( + ErrorCode.IDATA_AUTH_ERROR, + f"iData access forbidden: {str(e)}" + ) + elif e.response.status_code == 429: + logger.error("Raising IDATA_RATE_LIMIT for 429 error") + raise AppException( + ErrorCode.IDATA_RATE_LIMIT, + f"iData API rate limit exceeded: {str(e)}" + ) + else: + logger.error( + f"Raising IDATA_SERVICE_ERROR for status {e.response.status_code}") + raise AppException( + ErrorCode.IDATA_SERVICE_ERROR, + f"iData API HTTP error {e.response.status_code}: {str(e)}" + ) + except json.JSONDecodeError as e: + logger.error(f"Failed to parse iData API response: {str(e)}") + raise AppException( + ErrorCode.IDATA_RESPONSE_ERROR, + f"Failed to parse iData API response: {str(e)}" + ) + + +def _parse_idata_response(result: Dict[str, Any]) -> List[Dict[str, Any]]: + """ + Parse iData API response and validate format. + + Args: + result: Parsed JSON response from iData API + + Returns: + List of data items from response + + Raises: + AppException: If response format is invalid + """ + # Expected format: {"code": "1", "msg": "...", "data": [...], "msgParams": null} + code = result.get("code", "") + if code != "1": + msg = result.get("msg", "Unknown error") + logger.error( + f"iData API returned error code: {code}, message: {msg}") + raise AppException( + ErrorCode.IDATA_SERVICE_ERROR, + f"iData API error: {msg}" + ) + + data = result.get("data", []) + if not isinstance(data, list): + logger.error( + f"Unexpected iData API response format: data is not a list") + raise AppException( + ErrorCode.IDATA_RESPONSE_ERROR, + "Unexpected iData API response format: data is not a list" + ) + + return data + + +def fetch_idata_knowledge_spaces_impl( + idata_api_base: str, + api_key: str, + user_id: str, +) -> List[Dict[str, str]]: + """ + Fetch knowledge spaces from iData API. + + Args: + idata_api_base: iData API base URL + api_key: iData API key with Bearer token + user_id: iData user ID + + Returns: + List of dictionaries containing knowledge spaces with id and name: + [ + { + "id": "6cbf949946bf4b769c073259406b04f8", + "name": "test1" + }, + ... + ] + + Raises: + AppException: If API request fails or response is invalid + """ + # Validate inputs + _validate_idata_base_params(idata_api_base, api_key, user_id) + + # Normalize API base URL + api_base = _normalize_api_base(idata_api_base) + + # Build request URL + url = f"{api_base}/apiaccess/modelmate/north/machine/v1/knowledgeSpaces/query" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + # Request body + request_body = { + "userId": user_id + } + + # Make request and parse response + result = _make_idata_request(api_base, url, headers, request_body) + data = _parse_idata_response(result) + + # Extract id and name from each knowledge space + knowledge_spaces = [] + for item in data: + if not isinstance(item, dict): + continue + + space_id = item.get("id") + space_name = item.get("name") + + if space_id and space_name: + knowledge_spaces.append({ + "id": str(space_id), + "name": str(space_name) + }) + + return knowledge_spaces + + +def fetch_idata_datasets_impl( + idata_api_base: str, + api_key: str, + user_id: str, + knowledge_space_id: str, +) -> Dict[str, Any]: + """ + Fetch datasets (knowledge bases) from iData API and transform to DataMate-compatible format. + + Args: + idata_api_base: iData API base URL + api_key: iData API key with Bearer token + user_id: iData user ID + knowledge_space_id: Knowledge space ID + + Returns: + Dictionary containing knowledge bases in DataMate-compatible format: + { + "indices": ["dataset_id_1", "dataset_id_2", ...], + "count": 2, + "indices_info": [ + { + "name": "dataset_id_1", + "display_name": "知识库名称", + "stats": { + "base_info": { + "doc_count": 10, + "process_source": "iData" + } + } + }, + ... + ] + } + + Raises: + AppException: If API request fails or response is invalid + """ + # Validate inputs + _validate_idata_base_params(idata_api_base, api_key, user_id) + + if not knowledge_space_id or not isinstance(knowledge_space_id, str): + raise AppException( + ErrorCode.IDATA_CONFIG_INVALID, + "Knowledge space ID is required and must be a non-empty string" + ) + + # Normalize API base URL + api_base = _normalize_api_base(idata_api_base) + + # Build request URL + url = f"{api_base}/apiaccess/modelmate/north/machine/v1/knowledgeBases/query" + + headers = { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json" + } + + # Request body + request_body = { + "userId": user_id, + "knowledgeSpaceId": knowledge_space_id + } + + # Make request and parse response + result = _make_idata_request(api_base, url, headers, request_body) + data = _parse_idata_response(result) + + # Transform to DataMate-compatible format + indices = [] + indices_info = [] + + for knowledge_base in data: + if not isinstance(knowledge_base, dict): + continue + + kb_id = knowledge_base.get("id", "") + kb_name = knowledge_base.get("name", "") + file_count = knowledge_base.get("fileCount", 0) + + if not kb_id: + continue + + indices.append(kb_id) + + # Create indices_info entry (compatible with DataMate format) + indices_info.append({ + "name": kb_id, + "display_name": kb_name, + "stats": { + "base_info": { + "doc_count": file_count, + "process_source": "iData" + } + } + }) + + return { + "indices": indices, + "count": len(indices), + "indices_info": indices_info + } diff --git a/backend/services/model_management_service.py b/backend/services/model_management_service.py index 4b8265028..a18c16c36 100644 --- a/backend/services/model_management_service.py +++ b/backend/services/model_management_service.py @@ -3,7 +3,7 @@ from consts.const import LOCALHOST_IP, LOCALHOST_NAME, DOCKER_INTERNAL_HOST from consts.model import ModelConnectStatusEnum -from consts.provider import ProviderEnum, SILICON_BASE_URL +from consts.provider import ProviderEnum, SILICON_BASE_URL, DASHSCOPE_BASE_URL, TOKENPONY_BASE_URL from database.model_management_db import ( create_model_record, @@ -142,6 +142,10 @@ async def batch_create_models_for_tenant(user_id: str, tenant_id: str, batch_pay elif provider == ProviderEnum.MODELENGINE.value: # ModelEngine models carry their own base_url in each model dict model_url = "" + elif provider == ProviderEnum.DASHSCOPE.value: + model_url = DASHSCOPE_BASE_URL + elif provider == ProviderEnum.TOKENPONY.value: + model_url = TOKENPONY_BASE_URL else: model_url = "" diff --git a/backend/services/model_provider_service.py b/backend/services/model_provider_service.py index a302eb999..8c397dc70 100644 --- a/backend/services/model_provider_service.py +++ b/backend/services/model_provider_service.py @@ -11,6 +11,8 @@ from services.model_health_service import embedding_dimension_check from services.providers.base import AbstractModelProvider from services.providers.silicon_provider import SiliconModelProvider +from services.providers.tokenpony_provider import TokenPonyModelProvider +from services.providers.dashscope_provider import DashScopeModelProvider from services.providers.modelengine_provider import ModelEngineProvider, get_model_engine_raw_url, MODEL_ENGINE_NORTH_PREFIX from utils.model_name_utils import split_repo_name, add_repo_to_name @@ -40,6 +42,12 @@ async def get_provider_models(model_data: dict) -> List[dict]: elif model_data["provider"] == ProviderEnum.MODELENGINE.value: provider = ModelEngineProvider() model_list = await provider.get_models(model_data) + elif model_data["provider"] == ProviderEnum.DASHSCOPE.value: + provider = DashScopeModelProvider() + model_list = await provider.get_models(model_data) + elif model_data["provider"] == ProviderEnum.TOKENPONY.value: + provider = TokenPonyModelProvider() + model_list = await provider.get_models(model_data) return model_list @@ -117,7 +125,8 @@ async def prepare_model_dict(provider: str, model: dict, model_url: str, model_a # dimension by performing a real connectivity check. if model["model_type"] in ["embedding", "multi_embedding"]: if provider != ProviderEnum.MODELENGINE.value: - model_dict["base_url"] = f"{model_url}embeddings" + # Ensure proper slash between base URL and endpoint + model_dict["base_url"] = f"{model_url.rstrip('/')}/embeddings" else: # For ModelEngine embedding models, append the embeddings path model_dict["base_url"] = f"{model_url.rstrip('/')}/{MODEL_ENGINE_NORTH_PREFIX}/embeddings" diff --git a/backend/services/northbound_service.py b/backend/services/northbound_service.py index 6f9164269..a6eaed77d 100644 --- a/backend/services/northbound_service.py +++ b/backend/services/northbound_service.py @@ -13,11 +13,7 @@ ) from consts.model import AgentRequest from database.conversation_db import get_conversation_messages -from database.partner_db import ( - add_mapping_id, - get_external_id_by_internal, - get_internal_id_by_external -) +from database.token_db import log_token_usage, get_latest_usage_metadata from services.agent_service import ( run_agent_stream, stop_agent_tasks, @@ -40,6 +36,7 @@ class NorthboundContext: tenant_id: str user_id: str authorization: str + token_id: int = 0 # ----------------------------- @@ -114,26 +111,6 @@ def _build_idempotency_key(*parts: Any) -> str: return ":".join(processed) -# ----------------------------- -# ID mapping helpers -# ----------------------------- -async def to_external_conversation_id(internal_id: int) -> str: - if not internal_id: - raise Exception("invalid internal conversation id") - external_id = get_external_id_by_internal(internal_id=internal_id, mapping_type="CONVERSATION") - if not external_id: - logger.error(f"cannot find external id for conversation_id: {internal_id}") - raise Exception("cannot find external id") - return external_id - - -async def to_internal_conversation_id(external_id: str) -> int: - if not external_id: - raise Exception("invalid external conversation id") - internal_id = get_internal_id_by_external(external_id=external_id, mapping_type="CONVERSATION") - return internal_id - - # ----------------------------- # Agent resolver # ----------------------------- @@ -146,30 +123,30 @@ async def get_agent_info_by_name(agent_name: str, tenant_id: str) -> int: async def start_streaming_chat( ctx: NorthboundContext, - external_conversation_id: str, + conversation_id: Optional[int], agent_name: str, query: str, + meta_data: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None ) -> StreamingResponse: try: # Simple rate limit await check_and_consume_rate_limit(ctx.tenant_id) - internal_conversation_id = await to_internal_conversation_id(external_conversation_id) - # Add mapping to postgres database - if internal_conversation_id is None: - logging.info(f"Conversation {external_conversation_id} not found, creating a new conversation") - # Create a new conversation and get its internal ID + # If conversation_id is not provided, create a new conversation + if conversation_id is None: + logging.info("No conversation_id provided, creating a new conversation") new_conversation = create_new_conversation(title="New Conversation", user_id=ctx.user_id) - internal_conversation_id = new_conversation["conversation_id"] - # Add the new mapping to the database - add_mapping_id(internal_id=internal_conversation_id, external_id=external_conversation_id, tenant_id=ctx.tenant_id, user_id=ctx.user_id) + conversation_id = new_conversation["conversation_id"] + logging.info(f"Created new conversation with id: {conversation_id}") + + internal_conversation_id = conversation_id # Get history according to internal_conversation_id - history_resp = await get_conversation_history(ctx, external_conversation_id) + history_resp = await get_conversation_history_internal(ctx, internal_conversation_id) agent_id = await get_agent_id_by_name(agent_name=agent_name, tenant_id=ctx.tenant_id) # Idempotency: only prevent concurrent duplicate starts - composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, external_conversation_id, agent_id, query) + composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, str(conversation_id), agent_id, query) await idempotency_start(composed_key) agent_request = AgentRequest( conversation_id=internal_conversation_id, @@ -192,7 +169,7 @@ async def start_streaming_chat( except UnauthorizedError as _: raise UnauthorizedError("Cannot authenticate.") except Exception as e: - raise Exception(f"Failed to start streaming chat for external conversation id {external_conversation_id}: {str(e)}") + raise Exception(f"Failed to start streaming chat for conversation_id {conversation_id}: {str(e)}") try: response = await run_agent_stream( @@ -207,34 +184,82 @@ async def start_streaming_chat( if composed_key: asyncio.create_task(_release_idempotency_after_delay(composed_key)) - # Attach request id header + # Log token usage + if ctx.token_id > 0: + try: + log_token_usage( + token_id=ctx.token_id, + call_function_name="run_chat", + related_id=conversation_id, + created_by=ctx.user_id, + metadata=meta_data + ) + except Exception as e: + logger.warning(f"Failed to log token usage: {str(e)}") + + # Attach request id header and conversation_id (internal id) response.headers["X-Request-Id"] = ctx.request_id - response.headers["conversation_id"] = external_conversation_id + response.headers["conversation_id"] = str(conversation_id) return response -async def stop_chat(ctx: NorthboundContext, external_conversation_id: str) -> Dict[str, Any]: +async def stop_chat(ctx: NorthboundContext, conversation_id: int, meta_data: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: try: - internal_id = await to_internal_conversation_id(external_conversation_id) - - stop_result = stop_agent_tasks(internal_id, ctx.user_id) - return {"message": stop_result.get("message", "success"), "data": external_conversation_id, "requestId": ctx.request_id} + stop_result = stop_agent_tasks(conversation_id, ctx.user_id) + + # Log token usage + if ctx.token_id > 0: + try: + log_token_usage( + token_id=ctx.token_id, + call_function_name="stop_chat_stream", + related_id=conversation_id, + created_by=ctx.user_id, + metadata=meta_data + ) + except Exception as e: + logger.warning(f"Failed to log token usage: {str(e)}") + + return {"message": stop_result.get("message", "success"), "data": conversation_id, "requestId": ctx.request_id} except Exception as e: - raise Exception(f"Failed to stop chat for external conversation id {external_conversation_id}: {str(e)}") + raise Exception(f"Failed to stop chat for conversation_id {conversation_id}: {str(e)}") async def list_conversations(ctx: NorthboundContext) -> Dict[str, Any]: conversations = get_conversation_list_service(ctx.user_id) # get_conversation_list_service is sync - for item in conversations: - item["conversation_id"] = await to_external_conversation_id(int(item["conversation_id"])) - return {"message": "success", "data": conversations, "requestId": ctx.request_id} + # Add meta_data from token usage log if available + if ctx.token_id > 0: + for item in conversations: + # Ensure we do not leak empty meta_data keys + if "meta_data" in item and not item.get("meta_data"): + item.pop("meta_data", None) + + conversation_id = item.get("conversation_id") + if conversation_id: + try: + meta_data = get_latest_usage_metadata( + token_id=ctx.token_id, + related_id=int(conversation_id), + call_function_name="run_chat" + ) + # Only return meta_data when there is a usage log record and meta_data is non-empty + if meta_data: + item["meta_data"] = meta_data + else: + item.pop("meta_data", None) + except Exception as e: + logger.warning(f"Failed to get meta_data for conversation {conversation_id}: {str(e)}") + item.pop("meta_data", None) + + # Now return internal conversation_id directly + return {"message": "success", "data": conversations, "requestId": ctx.request_id} -async def get_conversation_history(ctx: NorthboundContext, external_conversation_id: str) -> Dict[str, Any]: - internal_id = await to_internal_conversation_id(external_conversation_id) - history = get_conversation_messages(internal_id) +async def get_conversation_history_internal(ctx: NorthboundContext, conversation_id: int) -> Dict[str, Any]: + """Internal helper to get conversation history without logging.""" + history = get_conversation_messages(conversation_id) # Remove unnecessary fields result = [] for message in history: @@ -244,44 +269,63 @@ async def get_conversation_history(ctx: NorthboundContext, external_conversation }) response = { - "conversation_id": external_conversation_id, + "conversation_id": conversation_id, "history": result } - # Ensure external id in response return {"message": "success", "data": response, "requestId": ctx.request_id} +async def get_conversation_history(ctx: NorthboundContext, conversation_id: int) -> Dict[str, Any]: + try: + return await get_conversation_history_internal(ctx, conversation_id) + except Exception as e: + raise Exception(f"Failed to get conversation history for conversation_id {conversation_id}: {str(e)}") + + async def get_agent_info_list(ctx: NorthboundContext) -> Dict[str, Any]: try: - agent_info_list = await list_all_agent_info_impl(tenant_id=ctx.tenant_id) + agent_info_list = await list_all_agent_info_impl(tenant_id=ctx.tenant_id, user_id=ctx.user_id) # Remove internal information that partner don't need for agent_info in agent_info_list: agent_info.pop("agent_id", None) + return {"message": "success", "data": agent_info_list, "requestId": ctx.request_id} except Exception as e: raise Exception(f"Failed to get agent info list for tenant {ctx.tenant_id}: {str(e)}") -async def update_conversation_title(ctx: NorthboundContext, external_conversation_id: str, title: str, idempotency_key: Optional[str] = None) -> Dict[str, Any]: +async def update_conversation_title(ctx: NorthboundContext, conversation_id: int, title: str, meta_data: Optional[Dict[str, Any]] = None, idempotency_key: Optional[str] = None) -> Dict[str, Any]: composed_key: Optional[str] = None try: - internal_id = await to_internal_conversation_id(external_conversation_id) - # Idempotency: avoid concurrent duplicate title update for same conversation - composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, external_conversation_id, title) + composed_key = idempotency_key or _build_idempotency_key(ctx.tenant_id, str(conversation_id), title) await idempotency_start(composed_key) - update_conversation_title_service(internal_id, title, ctx.user_id) + update_conversation_title_service(conversation_id, title, ctx.user_id) + + # Log token usage + if ctx.token_id > 0: + try: + log_token_usage( + token_id=ctx.token_id, + call_function_name="update_conversation_title", + related_id=conversation_id, + created_by=ctx.user_id, + metadata=meta_data + ) + except Exception as e: + logger.warning(f"Failed to log token usage: {str(e)}") + return { "message": "success", - "data": external_conversation_id, + "data": conversation_id, "requestId": ctx.request_id, "idempotency_key": composed_key, } except LimitExceededError as _: raise LimitExceededError("Duplicate request is still running, please wait.") except Exception as e: - raise Exception(f"Failed to update conversation title for external conversation id {external_conversation_id}: {str(e)}") + raise Exception(f"Failed to update conversation title for conversation_id {conversation_id}: {str(e)}") finally: if composed_key: asyncio.create_task(_release_idempotency_after_delay(composed_key)) diff --git a/backend/services/prompt_service.py b/backend/services/prompt_service.py index a505f28f4..3706c3cc5 100644 --- a/backend/services/prompt_service.py +++ b/backend/services/prompt_service.py @@ -7,8 +7,10 @@ from jinja2 import StrictUndefined, Template from consts.const import LANGUAGE -from consts.model import AgentInfoRequest -from database.agent_db import update_agent, search_agent_info_by_agent_id, query_all_agent_info_by_tenant_id, \ +from consts.error_code import ErrorCode +from consts.error_message import ErrorMessage +from consts.exceptions import AppException +from database.agent_db import search_agent_info_by_agent_id, query_all_agent_info_by_tenant_id, \ query_sub_agents_id_list from database.tool_db import query_tools_by_ids from services.agent_service import ( @@ -28,18 +30,30 @@ def gen_system_prompt_streamable(agent_id: int, model_id: int, task_description: str, user_id: str, tenant_id: str, language: str, tool_ids: Optional[List[int]] = None, sub_agent_ids: Optional[List[int]] = None): - for system_prompt in generate_and_save_system_prompt_impl( - agent_id=agent_id, - model_id=model_id, - task_description=task_description, - user_id=user_id, - tenant_id=tenant_id, - language=language, - tool_ids=tool_ids, - sub_agent_ids=sub_agent_ids - ): - # SSE format, each message ends with \n\n - yield f"data: {json.dumps({'success': True, 'data': system_prompt}, ensure_ascii=False)}\n\n" + try: + for system_prompt in generate_and_save_system_prompt_impl( + agent_id=agent_id, + model_id=model_id, + task_description=task_description, + user_id=user_id, + tenant_id=tenant_id, + language=language, + tool_ids=tool_ids, + sub_agent_ids=sub_agent_ids + ): + # SSE format, each message ends with \n\n + yield f"data: {json.dumps({'success': True, 'data': system_prompt}, ensure_ascii=False)}\n\n" + except Exception as e: + # Catch model unavailable or other errors and return error through SSE + logger.error(f"Error generating prompt: {e}") + # Use original error code if it's an AppException, otherwise use default + if isinstance(e, AppException): + error_code = e.error_code + error_message = e.message + else: + error_code = ErrorCode.MODEL_PROMPT_GENERATION_FAILED + error_message = ErrorMessage.get_message(error_code) + yield f"data: {json.dumps({'success': False, 'error': {'code': error_code.value, 'message': error_message}}, ensure_ascii=False)}\n\n" def generate_and_save_system_prompt_impl(agent_id: int, @@ -200,6 +214,14 @@ def generate_and_save_system_prompt_impl(agent_id: int, "Updating agent with business_description and prompt segments") logger.info("Prompt generation and agent update completed successfully") + # Check if any content was generated - if all fields are empty, model likely failed + all_fields = ["duty", "constraint", "few_shots", + "agent_var_name", "agent_display_name", "agent_description"] + has_content = any(final_results.get(field, "").strip() + for field in all_fields) + if not has_content: + raise Exception("Failed to generate prompt content.") + def generate_system_prompt(sub_agent_info_list, task_description, tool_info_list, tenant_id: str, model_id: int, language: str = LANGUAGE["ZH"]): """Main function for generating system prompts""" @@ -222,15 +244,18 @@ def generate_system_prompt(sub_agent_info_list, task_description, tool_info_list "agent_var_name": False, "agent_display_name": False, "agent_description": False} # Start all generation threads - threads = _start_generation_threads( + threads, error_holder = _start_generation_threads( content, prompt_for_generate, produce_queue, latest, stop_flags, tenant_id, model_id) # Stream results - yield from _stream_results(produce_queue, latest, stop_flags, threads) + yield from _stream_results(produce_queue, latest, stop_flags, threads, error_holder) def _start_generation_threads(content, prompt_for_generate, produce_queue, latest, stop_flags, tenant_id, model_id): """Start all prompt generation threads""" + # Shared error tracking across threads + error_holder = {"error": None} + def make_callback(tag): def callback_fn(current_text): latest[tag] = current_text @@ -243,6 +268,7 @@ def run_and_flag(tag, sys_prompt): model_id, content, sys_prompt, make_callback(tag), tenant_id) except Exception as e: logger.error(f"Error in {tag} generation: {e}") + error_holder["error"] = e finally: stop_flags[tag] = True @@ -266,10 +292,10 @@ def run_and_flag(tag, sys_prompt): thread.start() threads.append(thread) - return threads + return threads, error_holder -def _stream_results(produce_queue, latest, stop_flags, threads): +def _stream_results(produce_queue, latest, stop_flags, threads, error_holder): """Stream prompt generation results""" # Real-time streaming output for the first three sections @@ -277,6 +303,13 @@ def _stream_results(produce_queue, latest, stop_flags, threads): "agent_var_name": "", "agent_display_name": "", "agent_description": ""} while not all(stop_flags.values()): + # Check if error occurred in any thread - raise immediately + if error_holder.get("error"): + # Wait for threads to finish + for thread in threads: + thread.join(timeout=5) + raise error_holder["error"] + try: produce_queue.get(timeout=0.5) except queue.Empty: @@ -293,6 +326,10 @@ def _stream_results(produce_queue, latest, stop_flags, threads): yield result_data last_results[tag] = latest[tag] + # Check if error occurred before final output + if error_holder.get("error"): + raise error_holder["error"] + # Wait for all threads to complete for thread in threads: thread.join(timeout=5) diff --git a/backend/services/providers/dashscope_provider.py b/backend/services/providers/dashscope_provider.py new file mode 100644 index 000000000..4ecbcbb1d --- /dev/null +++ b/backend/services/providers/dashscope_provider.py @@ -0,0 +1,132 @@ +import httpx +from typing import Dict, List +import asyncio +from consts.const import DEFAULT_LLM_MAX_TOKENS +from consts.provider import DASHSCOPE_GET_URL +from services.providers.base import AbstractModelProvider, _classify_provider_error + + +class DashScopeModelProvider(AbstractModelProvider): + """Concrete implementation for DashScope (Aliyun) provider.""" + + async def get_models(self, provider_config: Dict) -> List[Dict]: + """ + Fetch models from DashScope API, categorize them, and return + the requested model type. + + Args: + provider_config: Configuration dict containing model_type and api_key + + Returns: + List of models with canonical fields. Returns error dict if API call fails. + """ + try: + target_model_type: str = provider_config["model_type"] + model_api_key: str = provider_config["api_key"] + + headers = {"Authorization": f"Bearer {model_api_key}"} + base_url = DASHSCOPE_GET_URL + + all_models: List[Dict] = [] + current_page = 1 + + # Fetch all models with pagination asynchronously + async with httpx.AsyncClient(verify=False) as client: + while True: + params = {"page_size": 100, "page_no": current_page} + response = await client.get(base_url, headers=headers, params=params) + if response.status_code == 429: + await asyncio.sleep(2) + continue + response.raise_for_status() + + data = response.json() + models = data.get("output", {}).get("models", []) + + # Break loop if no more models on the current page + if not models: + break + + all_models.extend(models) + if len(models) < 100: + break + current_page += 1 + await asyncio.sleep(0.5) + + # Initialize containers for the 6 main categories + categorized_models = { + "chat": [], # Maps to "llm" + "vlm": [], # Maps to "vlm" + "embedding": [], # Maps to "embedding" / "multi_embedding" + "reranker": [], # Maps to "reranker" + "tts": [], # Maps to "tts" + "stt": [] # Maps to "stt" + } + + # Classify models and inject canonical fields expected downstream + for model_obj in all_models: + # Extract key fields for logical determination (lowercased for robustness) + m_id = model_obj.get('model', '').lower() + desc = model_obj.get('description', '') + metadata = model_obj.get('inference_metadata', {}) + req_mod = metadata.get('request_modality', []) + res_mod = metadata.get('response_modality', []) + model_obj.setdefault("object", model_obj.get("object", "model")) + model_obj.setdefault("owned_by", model_obj.get("owned_by", "dashscope")) + cleaned_model = { + "id": m_id, + "object": model_obj.get("object"), + "created": 0, + "owned_by": model_obj.get("owned_by"), + "model_tag": "", + "model_type": "", + "max_tokens": DEFAULT_LLM_MAX_TOKENS + } + # 1. Embedding + if 'embedding' in m_id.lower() or '向量' in desc: + cleaned_model.update({"model_tag": "embedding", "model_type": "embedding"}) + categorized_models['embedding'].append(cleaned_model) + continue + + # 2. Reranker + if 'rerank' in m_id.lower() or '重排序' in desc: + cleaned_model.update({"model_tag": "reranker", "model_type": "reranker"}) + categorized_models['reranker'].append(cleaned_model) + continue + + # 3. STT + if 'Audio' in req_mod and 'Text' in res_mod: + cleaned_model.update({"model_tag": "stt", "model_type": "stt"}) + categorized_models['stt'].append(cleaned_model) + continue + + # 4. TTS + if 'Audio' in res_mod and 'Video' not in res_mod: + cleaned_model.update({"model_tag": "tts", "model_type": "tts"}) + categorized_models['tts'].append(cleaned_model) + continue + + # 5. VLM + vision_mods = {'Image', 'Video'} + if (set(req_mod) & vision_mods) or (set(res_mod) & vision_mods) or '视觉' in desc: + cleaned_model.update({"model_tag": "chat", "model_type": "vlm"}) + categorized_models['vlm'].append(cleaned_model) + continue + + # 6. Chat / LLM + if 'Text' in req_mod or 'Text' in res_mod: + cleaned_model.update({"model_tag": "chat", "model_type": "llm"}) + categorized_models['chat'].append(cleaned_model) + + # Return the specific list based on the requested target_model_type + if target_model_type == "llm": + return categorized_models["chat"] + elif target_model_type in ("embedding", "multi_embedding"): + return categorized_models["embedding"] + elif target_model_type in categorized_models: + return categorized_models[target_model_type] + else: + return [] + except (httpx.HTTPStatusError, httpx.ConnectTimeout, httpx.ConnectError, Exception) as e: + return _classify_provider_error("DashScope", exception=e) + diff --git a/backend/services/providers/tokenpony_provider.py b/backend/services/providers/tokenpony_provider.py new file mode 100644 index 000000000..42e5d178c --- /dev/null +++ b/backend/services/providers/tokenpony_provider.py @@ -0,0 +1,112 @@ +import httpx +import ssl + +from typing import Dict, List + + +from consts.const import DEFAULT_LLM_MAX_TOKENS +from consts.provider import TOKENPONY_GET_URL +from services.providers.base import AbstractModelProvider, _classify_provider_error + + +class TokenPonyModelProvider(AbstractModelProvider): + """Concrete implementation for TokenPony provider.""" + + async def get_models(self, provider_config: Dict) -> List[Dict]: + """ + Fetch models from TokenPony API, categorize them based on modality/ID, + and return the requested model type. + + Args: + provider_config: Configuration dict containing model_type and api_key + + Returns: + List of models with canonical fields. Returns error dict if API call fails. + """ + try: + target_model_type: str = provider_config["model_type"] + model_api_key: str = provider_config["api_key"] + + headers = {"Authorization": f"Bearer {model_api_key}"} + url = TOKENPONY_GET_URL + + + ssl_context = ssl.create_default_context() + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE + ssl_context.set_ciphers("DEFAULT@SECLEVEL=1") + + async with httpx.AsyncClient(http2=True) as client: + response = await client.get(url, headers=headers) + response.raise_for_status() + # OpenAI standard response puts the model list inside the "data" array + all_models: List[Dict] = response.json().get("data", []) + + # Initialize containers for the 6 main categories + categorized_models = { + "chat": [], # Maps to "llm" + "vlm": [], # Maps to "vlm" + "embedding": [], # Maps to "embedding" / "multi_embedding" + "reranker": [], # Maps to "reranker" + "tts": [], # Maps to "tts" + "stt": [] # Maps to "stt" + } + + # Classify models and inject canonical fields expected downstream + for model_obj in all_models: + m_id = model_obj['id'].lower() + model_obj.setdefault("object", model_obj.get("object", "model")) + model_obj.setdefault("owned_by", model_obj.get("owned_by", "tokenpony")) + cleaned_model = { + "id": m_id, + "object": model_obj.get("object"), + "created": 0, + "owned_by": model_obj.get("owned_by"), + "model_tag": "", + "model_type": "", + "max_tokens": DEFAULT_LLM_MAX_TOKENS + } + # 1. reranker + if 'rerank' in m_id: + cleaned_model.update({"model_tag": "reranker", "model_type": "reranker"}) + categorized_models['reranker'].append(cleaned_model) + #2. embedding + elif 'embedding' in m_id or m_id.startswith('bge-'): + cleaned_model.update({"model_tag": "embedding", "model_type": "embedding"}) + categorized_models['embedding'].append(cleaned_model) + + # 3. STT (Speech-to-Text / Audio understanding) + elif 'stt' in m_id: + cleaned_model.update({"model_tag": "stt", "model_type": "stt"}) + categorized_models['stt'].append(cleaned_model) + + + # 4. TTS (Text-to-Speech) + elif 'tts' in m_id: + cleaned_model.update({"model_tag": "tts", "model_type": "tts"}) + categorized_models['tts'].append(cleaned_model) + + # 5. VLM (Vision Language Model / Image & Video Generation) + + elif any(keyword in m_id for keyword in ['-vl', 'vl-', 'ocr', 'vision']): + cleaned_model.update({"model_tag": "chat", "model_type": "vlm"}) + categorized_models['vlm'].append(cleaned_model) + + # 6. Chat (Pure Text Conversation / Reasoning) + # Fallback check added: 'not metadata' catches standard OpenAI models that lack modality data + else : + cleaned_model.update({"model_tag": "chat", "model_type": "llm"}) + categorized_models['chat'].append(cleaned_model) + + # Return the specific list based on the requested target_model_type + if target_model_type == "llm": + return categorized_models["chat"] + elif target_model_type in ("embedding", "multi_embedding"): + return categorized_models["embedding"] + elif target_model_type in categorized_models: + return categorized_models[target_model_type] + else: + return [] + + except (httpx.HTTPStatusError, httpx.ConnectTimeout, httpx.ConnectError, Exception) as e: + return _classify_provider_error("TokenPony", exception=e) diff --git a/backend/services/user_management_service.py b/backend/services/user_management_service.py index 792887ec5..39ea8cfbe 100644 --- a/backend/services/user_management_service.py +++ b/backend/services/user_management_service.py @@ -1,6 +1,13 @@ import logging from typing import Optional, Any, Tuple, Dict, List +from database.token_db import ( + create_token as create_token_record, + generate_access_key, + list_tokens_by_user as list_tokens_by_user_record, + delete_token as delete_token_record, +) + import aiohttp from fastapi import Header from supabase import Client @@ -122,11 +129,12 @@ async def check_auth_service_health(): async def signup_user_with_invitation(email: EmailStr, password: str, - invite_code: Optional[str] = None): + invite_code: Optional[str] = None, + auto_login: Optional[bool] = True): """User registration with invitation code support""" client = get_supabase_client() logging.info( - f"Receive registration request: email={email}, invite_code={'provided' if invite_code else 'not provided'}") + f"Receive registration request: email={email}, invite_code={'provided' if invite_code else 'not provided'}, auto_login={auto_login}") # Default user role is USER user_role = "USER" @@ -221,7 +229,7 @@ async def signup_user_with_invitation(email: EmailStr, f"Failed to use invitation code {invite_code} for user {user_id}: {str(e)}") logging.info( - f"User {email} registered successfully, role: {user_role}, tenant: {tenant_id}") + f"User {email} registered successfully, role: {user_role}, tenant: {tenant_id}, auto_login={auto_login}") if user_role == "ADMIN": await generate_tts_stt_4_admin(tenant_id, user_id) @@ -229,7 +237,7 @@ async def signup_user_with_invitation(email: EmailStr, # Initialize tool list for the new tenant (only once per tenant) await init_tool_list_for_tenant(tenant_id, user_id) - return await parse_supabase_response(False, response, user_role) + return await parse_supabase_response(False, response, user_role, auto_login) else: logging.error( "Supabase registration request returned no user object") @@ -237,7 +245,7 @@ async def signup_user_with_invitation(email: EmailStr, "Registration service is temporarily unavailable, please try again later") -async def parse_supabase_response(is_admin, response, user_role): +async def parse_supabase_response(is_admin, response, user_role, auto_login: bool = True): """Parse Supabase response and build standardized user registration response""" user_data = { "id": response.user.id, @@ -246,7 +254,7 @@ async def parse_supabase_response(is_admin, response, user_role): } session_data = None - if response.session: + if response.session and auto_login: session_data = { "access_token": response.session.access_token, "refresh_token": response.session.refresh_token, @@ -472,3 +480,45 @@ def format_role_permissions(permissions: List[Dict[str, Any]]) -> Dict[str, List "permissions": formatted_permissions, "accessibleRoutes": accessible_routes } + + +# ----------------------------- +# Token Management +# ----------------------------- + +def create_token(user_id: str) -> Dict[str, Any]: + """Create a new API token for the specified user. + + Args: + user_id: The user ID who owns this token. + + Returns: + Dictionary containing the API token information including token_id. + """ + access_key = generate_access_key() + return create_token_record(access_key, user_id) + + +def list_tokens_by_user(user_id: str) -> List[Dict[str, Any]]: + """List all tokens for the specified user. + + Args: + user_id: The user ID to query token pairs for. + + Returns: + List of token information with masked access keys. + """ + return list_tokens_by_user_record(user_id) + + +def delete_token(token_id: int, user_id: str) -> bool: + """Soft delete a token. + + Args: + token_id: The token ID to delete. + user_id: The user ID who owns this token (for authorization). + + Returns: + True if the token was deleted, False if not found or not owned by user. + """ + return delete_token_record(token_id, user_id) diff --git a/backend/utils/auth_utils.py b/backend/utils/auth_utils.py index a27a48b38..7b40576e2 100644 --- a/backend/utils/auth_utils.py +++ b/backend/utils/auth_utils.py @@ -1,9 +1,9 @@ import logging -import hashlib -import hmac import time +import hmac +import hashlib from datetime import datetime, timedelta -from typing import Optional, Tuple +from typing import Dict, Optional, Tuple import jwt from fastapi import Request @@ -20,189 +20,195 @@ DEBUG_JWT_EXPIRE_SECONDS, LANGUAGE, ) -from consts.exceptions import LimitExceededError, SignatureValidationError, UnauthorizedError +from consts.exceptions import LimitExceededError, UnauthorizedError from database.user_tenant_db import get_user_tenant_by_user_id +from database.token_db import get_token_by_access_key # Module logger logger = logging.getLogger(__name__) # --------------------------------------------------------------------------- -# AK/SK authentication helpers (merged from aksk_auth_utils.py) +# Shared test constants # --------------------------------------------------------------------------- -# Mock AK/SK configuration (replace with DB/config lookup in production) -MOCK_ACCESS_KEY = "mock_access_key_12345" -MOCK_SECRET_KEY = "mock_secret_key_67890abcdef" -MOCK_JWT_SECRET_KEY = "mock_jwt_secret_key_67890abcdef" +# Fixed test secret used by generate_test_jwt and unit tests. +MOCK_JWT_SECRET_KEY = "nexent-mock-jwt-secret" -# Timestamp validity window in seconds (prevent replay attacks) -TIMESTAMP_VALIDITY_WINDOW = 300 +# --------------------------------------------------------------------------- +# AK/SK (Access Key / Secret Key) authentication helpers +# --------------------------------------------------------------------------- +# Validity window in seconds for X-Timestamp header. +TIMESTAMP_VALIDITY_WINDOW = 5 * 60 -def get_aksk_config(tenant_id: str) -> Tuple[str, str]: - """ - Get AK/SK configuration according to tenant_id - Returns: - Tuple[str, str]: (access_key, secret_key) +def calculate_hmac_signature(secret_key: str, access_key: str, timestamp: str, body: str) -> str: """ + Calculate HMAC-SHA256 signature for AK/SK authentication. - # TODO: get ak/sk according to tenant_id from DB - return MOCK_ACCESS_KEY, MOCK_SECRET_KEY + Returns a lowercase hex digest of length 64. + """ + message = f"{access_key}\n{timestamp}\n{body}".encode("utf-8") + return hmac.new(secret_key.encode("utf-8"), message, hashlib.sha256).hexdigest() def validate_timestamp(timestamp: str) -> bool: - """ - Validate timestamp is within validity window + """Validate that timestamp is within allowed window.""" + try: + ts = int(timestamp) + except (TypeError, ValueError): + return False - Args: - timestamp: timestamp string + now = int(time.time()) + return abs(now - ts) <= TIMESTAMP_VALIDITY_WINDOW - Returns: - bool: whether timestamp is valid + +def extract_aksk_headers(headers: Dict[str, str]) -> Tuple[str, str, str]: + """Extract AK/SK headers or raise UnauthorizedError when missing.""" + access_key = headers.get("X-Access-Key") if headers else None + timestamp = headers.get("X-Timestamp") if headers else None + signature = headers.get("X-Signature") if headers else None + + if not access_key or not timestamp or not signature: + raise UnauthorizedError("Missing AK/SK authentication headers") + + return access_key, timestamp, signature + + +def get_aksk_config(tenant_id: str) -> Tuple[str, str]: """ - try: - timestamp_int = int(timestamp) - current_time = int(time.time()) + Get (access_key, secret_key) configuration for a tenant. - if abs(current_time - timestamp_int) > TIMESTAMP_VALIDITY_WINDOW: - logger.warning( - f"Timestamp validation failed: current={current_time}, provided={timestamp_int}" - ) - return False + This is intentionally a thin indirection so tests can monkeypatch it. + """ + raise UnauthorizedError("AK/SK authentication is not configured") - return True - except (ValueError, TypeError) as e: - logger.error(f"Invalid timestamp format: {timestamp}, error: {e}") + +def verify_aksk_signature(access_key: str, timestamp: str, signature: str, body: str, tenant_id: str = None) -> bool: + """Verify AK/SK signature; returns False instead of raising on mismatch.""" + tenant = tenant_id or DEFAULT_TENANT_ID + try: + expected_access_key, secret_key = get_aksk_config(tenant) + except Exception: return False + if access_key != expected_access_key: + return False -def calculate_hmac_signature(secret_key: str, access_key: str, timestamp: str, request_body: str = "") -> str: - """ - Calculate HMAC-SHA256 signature + expected_sig = calculate_hmac_signature(secret_key, access_key, timestamp, body) + return hmac.compare_digest(expected_sig, signature) - Args: - secret_key: secret key - access_key: access key - timestamp: timestamp - request_body: request body (optional) - Returns: - str: HMAC-SHA256 signature (hex string) +def validate_aksk_authentication(headers: Dict[str, str], body: str, tenant_id: str = None) -> bool: """ - string_to_sign = f"{access_key}{timestamp}{request_body}" - signature = hmac.new( - secret_key.encode("utf-8"), - string_to_sign.encode("utf-8"), - hashlib.sha256, - ).hexdigest() - return signature - - -def verify_aksk_signature( - access_key: str, timestamp: str, signature: str, request_body: str = "" -) -> bool: - """ - Validate AK/SK signature - - Args: - access_key: access key - timestamp: timestamp - signature: provided signature - request_body: request body (optional) + Validate AK/SK authentication. - Returns: - bool: whether signature is valid + Returns True when valid, otherwise raises domain exceptions. """ - try: - if not validate_timestamp(timestamp): - raise SignatureValidationError("Timestamp is invalid or expired") + from consts.exceptions import SignatureValidationError # imported lazily for test-time stubbing - # TODO: get ak/sk according to tenant_id from DB - mock_access_key, mock_secret_key = get_aksk_config( - tenant_id="tenant_id") + try: + access_key, ts, sig = extract_aksk_headers(headers) - if access_key != mock_access_key: - logger.warning(f"Invalid access key: {access_key}") - return False + if not validate_timestamp(ts): + raise UnauthorizedError("Invalid or expired timestamp") - expected_signature = calculate_hmac_signature( - mock_secret_key, access_key, timestamp, request_body - ) + # Call with positional args so tests can monkeypatch with simple lambdas. + if tenant_id is None: + ok = verify_aksk_signature(access_key, ts, sig, body) + else: + ok = verify_aksk_signature(access_key, ts, sig, body, tenant_id) - if not hmac.compare_digest(signature, expected_signature): - logger.warning( - f"Signature mismatch: expected={expected_signature}, provided={signature}" - ) - return False + if not ok: + raise SignatureValidationError("Invalid signature") return True - except Exception as e: - logger.error(f"Error during signature verification: {e}") - return False + except (UnauthorizedError, SignatureValidationError): + raise + except Exception as exc: + logger.exception("Unexpected error during AK/SK authentication") + raise UnauthorizedError("Authentication failed") from exc +# --------------------------------------------------------------------------- +# Bearer Token (API Key) authentication +# --------------------------------------------------------------------------- -def extract_aksk_headers(headers: dict) -> Tuple[str, str, str]: + +def validate_bearer_token(authorization: Optional[str]) -> Tuple[bool, Optional[dict]]: """ - Extract AK/SK related information from request headers + Validate Bearer token (API Key) from Authorization header. Args: - headers: request headers dictionary + authorization: Authorization header value (e.g., "Bearer nexent-xxxxx") Returns: - Tuple[str, str, str]: (access_key, timestamp, signature) - - Raises: - UnauthorizedError: when required headers are missing + Tuple of (is_valid, token_info_dict) + - is_valid: True if token exists and is active + - token_info: Token information dict if valid, None otherwise """ + if not authorization: + logger.warning("No authorization header provided") + return False, None - def get_header(headers: dict, name: str) -> Optional[str]: - for k, v in headers.items(): - if k.lower() == name.lower(): - return v - return None - - access_key = get_header(headers, "X-Access-Key") - timestamp = get_header(headers, "X-Timestamp") - signature = get_header(headers, "X-Signature") + # Extract token from "Bearer " format + token = authorization.replace("Bearer ", "") if authorization.startswith("Bearer ") else authorization - if not access_key: - raise UnauthorizedError("Missing X-Access-Key header") - if not timestamp: - raise UnauthorizedError("Missing X-Timestamp header") - if not signature: - raise UnauthorizedError("Missing X-Signature header") + if not token: + logger.warning("Empty bearer token") + return False, None - return access_key, timestamp, signature + # Look up token in database + try: + token_info = get_token_by_access_key(token) + if token_info and token_info.get("delete_flag") != "Y": + logger.debug(f"Token validated successfully for user {token_info.get('user_id')}") + return True, token_info + else: + logger.warning(f"Invalid or inactive token: {token[:20]}...") + return False, None + except Exception as e: + logger.error(f"Error validating bearer token: {str(e)}") + return False, None -def validate_aksk_authentication(headers: dict, request_body: str = "") -> bool: +def get_user_and_tenant_by_access_key(access_key: str) -> Dict[str, str]: """ - Validate AK/SK authentication + Get user_id and tenant_id from access_key by querying user_token_info_t and user_tenant_t. Args: - headers: request headers dictionary - request_body: request body (optional) + access_key: The access key (API Key) from the Authorization header. Returns: - bool: whether authentication is successful + Dict containing user_id and tenant_id. Raises: - UnauthorizedError: when authentication fails - SignatureValidationError: when signature verification fails + UnauthorizedError: If the access key is not found or invalid. """ - try: - access_key, timestamp, signature = extract_aksk_headers(headers) - - if not verify_aksk_signature(access_key, timestamp, signature, request_body): - raise SignatureValidationError("Invalid signature") - - return True - except (UnauthorizedError, SignatureValidationError, LimitExceededError) as e: - raise e - except Exception as e: - logger.error(f"Unexpected error during AK/SK authentication: {e}") - raise UnauthorizedError("Authentication failed") + if not access_key: + raise UnauthorizedError("Invalid access key") + + # Query token from user_token_info_t + token_info = get_token_by_access_key(access_key) + if not token_info or token_info.get("delete_flag") == "Y": + raise UnauthorizedError("Invalid or inactive access key") + + user_id = token_info.get("user_id") + if not user_id: + raise UnauthorizedError("No user associated with this access key") + + # Query tenant from user_tenant_t + user_tenant_record = get_user_tenant_by_user_id(user_id) + if user_tenant_record and user_tenant_record.get("tenant_id"): + tenant_id = user_tenant_record["tenant_id"] + else: + tenant_id = DEFAULT_TENANT_ID + logger.warning(f"No tenant relationship found for user {user_id}, using default tenant") + + return { + "user_id": user_id, + "tenant_id": tenant_id, + "token_id": token_info.get("token_id") + } def get_supabase_client(): diff --git a/backend/utils/file_management_utils.py b/backend/utils/file_management_utils.py index 2a1aa3801..57025e350 100644 --- a/backend/utils/file_management_utils.py +++ b/backend/utils/file_management_utils.py @@ -1,5 +1,7 @@ +import asyncio import logging import os +import subprocess import traceback from pathlib import Path from typing import List @@ -337,3 +339,66 @@ def get_file_size(source_type: str, path_or_url: str) -> int: logging.error(f"Error getting file size for {path_or_url}: {str(e)}") return 0 + +async def convert_office_to_pdf(input_path: str, output_dir: str, timeout: int = 30) -> str: + """ + Convert Office document to PDF using LibreOffice. + + Args: + input_path: Path to input Office file + output_dir: Directory for output PDF file + timeout: Conversion timeout in seconds (default: 30s) + + Returns: + str: Path to generated PDF file + """ + if not os.path.exists(input_path): + raise FileNotFoundError(f"Input file not found: {input_path}") + + def _run_libreoffice_conversion(): + """Synchronous LibreOffice conversion to run in thread executor.""" + cmd = [ + 'libreoffice', + '--headless', + '--convert-to', 'pdf', + '--outdir', output_dir, + input_path + ] + return subprocess.run( + cmd, + capture_output=True, + text=True, + timeout=timeout + ) + + try: + # Run blocking subprocess in thread executor to avoid blocking event loop + result = await asyncio.to_thread(_run_libreoffice_conversion) + + if result.returncode != 0: + error_msg = result.stderr or result.stdout or "Unknown conversion error" + logger.error(f"LibreOffice conversion failed: {error_msg}") + raise RuntimeError(f"Office to PDF conversion failed: {error_msg}") + + # Find generated PDF file + input_filename = os.path.basename(input_path) + pdf_filename = os.path.splitext(input_filename)[0] + '.pdf' + pdf_path = os.path.join(output_dir, pdf_filename) + + if not os.path.exists(pdf_path): + raise RuntimeError(f"Converted PDF not found: {pdf_path}") + + return pdf_path + + except subprocess.TimeoutExpired: + logger.error(f"Office to PDF conversion timeout after {timeout}s: {input_path}") + raise TimeoutError(f"Office to PDF conversion timeout (>{timeout}s)") + + except FileNotFoundError as e: + # LibreOffice executable not found in PATH + logger.error(f"LibreOffice not available: {str(e)}") + raise FileNotFoundError( + "LibreOffice is not installed or not available in PATH. " + ) from e + + diff --git a/backend/utils/llm_utils.py b/backend/utils/llm_utils.py index 0ede9a263..d1aa6fcf3 100644 --- a/backend/utils/llm_utils.py +++ b/backend/utils/llm_utils.py @@ -2,8 +2,9 @@ from typing import Callable, List, Optional from consts.const import MESSAGE_ROLE, THINK_END_PATTERN, THINK_START_PATTERN +from consts.error_code import ErrorCode +from consts.exceptions import AppException from database.model_management_db import get_model_by_model_id -from nexent.core.utils.observer import MessageObserver from nexent.core.models import OpenAIModel from utils.config_utils import get_model_name_from_config @@ -122,8 +123,23 @@ def call_llm_for_system_prompt( return result except Exception as exc: logger.error("Failed to generate prompt from LLM: %s", str(exc)) - raise + # Parse error code from exception message and raise appropriate AppException + # Use specific error codes for different scenarios + error_msg = str(exc) + if "401" in error_msg or "api key" in error_msg.lower() or "unauthorized" in error_msg.lower(): + raise AppException(ErrorCode.MODEL_API_KEY_INVALID) + elif "403" in error_msg or "forbidden" in error_msg.lower(): + raise AppException(ErrorCode.MODEL_API_KEY_NO_PERMISSION) + elif "404" in error_msg or "not found" in error_msg.lower(): + raise AppException(ErrorCode.MODEL_NOT_FOUND) + elif "429" in error_msg or "rate limit" in error_msg.lower(): + raise AppException(ErrorCode.MODEL_RATE_LIMIT_EXCEEDED) + elif "500" in error_msg or "502" in error_msg or "503" in error_msg or "504" in error_msg: + raise AppException(ErrorCode.MODEL_SERVICE_UNAVAILABLE) + elif "connection" in error_msg.lower() or "timeout" in error_msg.lower() or "refused" in error_msg.lower(): + raise AppException(ErrorCode.MODEL_CONNECTION_ERROR) + else: + raise AppException(ErrorCode.MODEL_PROMPT_GENERATION_FAILED) __all__ = ["call_llm_for_system_prompt", "_process_thinking_tokens"] - diff --git a/doc/docs/en/quick-start/installation.md b/doc/docs/en/quick-start/installation.md index 662eb7c3d..f01576513 100644 --- a/doc/docs/en/quick-start/installation.md +++ b/doc/docs/en/quick-start/installation.md @@ -44,13 +44,35 @@ After executing this command, the system will provide two different versions for - **Terminal Tool**: Enables openssh-server for AI agent shell command execution - **Regional optimization**: Mainland China users can use optimized image sources ->⚠️ **Important Note**: When deploying v1.8.0 or later for the first time, please pay special attention to the `suadmin` super administrator account information output in the Docker logs. This account has the highest system privileges, and the password is only displayed upon first generation. It cannot be viewed again later, so please be sure to save it securely. +### ⚠️ Important Notes +1️⃣ **When deploying v1.8.0 or later for the first time**, please pay special attention to the `suadmin` super administrator account information output in the Docker logs. This account has the highest system privileges, and the password is only displayed upon first generation. It cannot be viewed again later, so please be sure to save it securely. + +2️⃣ Forgot to note the `suadmin` account password? Follow these steps: +```bash +# Step1: Delete su account record in supabase container +docker exec -it supabase-db-mini bash +psql -U postgres +select id, email from auth.users; +# Get the user_id of suadmin@nexent.com account +delete from auth.users where id = 'your_user_id'; +delete from auth.identities where user_id = 'your_user_id'; + +# Step2: Delete su account record in nexent database +docker exec -it nexent-postgresql bash +psql -U root -d nexent +delete from nexent.user_tenant_t where user_id = 'your_user_id'; + +# Step3: Redeploy and record the su account password +``` ### 3. Access Your Installation When deployment completes successfully: 1. Open **http://localhost:3000** in your browser -2. Refer to the [User Guide](../user-guide/home-page) to develop agents +2. Log in with the super administrator account +3. Access tenant resources → Create tenant and tenant administrator +4. Log in with the tenant administrator account +5. Refer to the [User Guide](../user-guide/home-page) to develop agents ## 🏗️ Service Architecture diff --git a/doc/docs/zh/opensource-memorial-wall.md b/doc/docs/zh/opensource-memorial-wall.md index 54bac7c28..068d5902f 100644 --- a/doc/docs/zh/opensource-memorial-wall.md +++ b/doc/docs/zh/opensource-memorial-wall.md @@ -711,3 +711,19 @@ Nexent 加油!希望能达成所愿! ::: info sisyphus0x - 2026-03-04 对多智能体编排和协同工作很感兴趣,学习一下 ::: + +::: info hmh_mike - 2026-03-05 +感觉很有意思,试用一下看看对工作有没有帮助 +::: + +::: tip GZX- 2026-03-08 +感谢 Nexent 期待与Nexent一起进步。 +::: + +::: info xingzhewujiang - 2026-03-09 +偶然发现Nexent是一个开源的零代码智能体自动生成平台,非常值的研究与尝试,祝福Nexent让零代码走向AI全球。 +::: + +::: info ichigoichie - 2026-03-10 +被 Nexent 官网吸引,希望深入了解产品并应用于工作场景,提升工作效率。 +::: diff --git a/doc/docs/zh/quick-start/installation.md b/doc/docs/zh/quick-start/installation.md index 64840b8d0..87df5abde 100644 --- a/doc/docs/zh/quick-start/installation.md +++ b/doc/docs/zh/quick-start/installation.md @@ -45,12 +45,34 @@ bash deploy.sh - **区域优化**: 中国大陆用户可使用优化的镜像源 ->⚠️ **重要提示**:首次部署 v1.8.0 及以上版本时,需特别留意 Docker 日志中输出的 `suadmin` 超级管理员账号信息。该账号为系统最高权限账户,密码仅在首次生成时显示,后续无法再次查看,请务必妥善保存。 +### ⚠️ 重要提示 +1️⃣ **首次部署 v1.8.0 及以上版本时**,需特别留意 Docker 日志中输出的 `suadmin` 超级管理员账号信息。该账号为系统最高权限账户,密码仅在首次生成时显示,后续无法再次查看,请务必妥善保存。 +> 该账号仅用于权限管理,无权开发智能体或创建知识库。请登录该账号,依次完成:访问租户资源→创建租户→创建租户管理员,然后使用租户管理员账号登录,即可使用全部功能。角色权限详情参见 [用户管理](../user-guide/user-management) +2️⃣ 忘记留意 `suadmin` 账号密码?请按照以下步骤操作: +```bash +# Step1: 在supabase容器中删除su账号记录 +docker exec -it supabase-db-mini bash +psql -U postgres +select id, email from auth.users; +#获取到suadmin@nexent.com账号的user_id +delete from auth.users where id = '你的user_id'; +delete from auth.identities where user_id = '你的user_id'; + +#Step2:在nexent的数据库中删除su账号记录 +docker exec -it nexent-postgresql bash +psql -U root -d nexent +delete from nexent.user_tenant_t where user_id = '你的user_id'; + +#Step3:重新部署并记录su账号密码 +``` ### 3. 访问您的安装 部署成功完成后: 1. 在浏览器中打开 **http://localhost:3000** +2. 登录超级管理员账号 +3. 访问租户资源 → 创建租户及租户管理员 +4. 登录租户管理员账号 2. 参考 [用户指南](../user-guide/home-page) 进行智能体的开发 diff --git a/docker/create-su.sh b/docker/create-su.sh index 8d290a726..639e64553 100644 --- a/docker/create-su.sh +++ b/docker/create-su.sh @@ -54,10 +54,32 @@ wait_for_postgresql_ready() { create_default_super_admin_user() { local email="suadmin@nexent.com" local password - password="$(generate_random_password)" + + # Get password from command line argument, or generate random one if not provided + if [ -n "$1" ]; then + password="$1" + else + # Fallback to random password if no argument provided (for backward compatibility) + password="$(generate_random_password)" + echo " ⚠️ Warning: No password provided, using random password" + fi echo "🔧 Creating super admin user..." - RESPONSE=$(docker exec nexent-config bash -c "curl -s -X POST http://kong:8000/auth/v1/signup -H \"apikey: ${SUPABASE_KEY}\" -H \"Authorization: Bearer ${SUPABASE_KEY}\" -H \"Content-Type: application/json\" -d '{\"email\":\"${email}\",\"password\":\"${password}\",\"email_confirm\":true}'" 2>/dev/null) + + # Determine which container to use for curl command + local curl_container="nexent-config" + if [ "$DEPLOYMENT_MODE" = "infrastructure" ] || ! docker ps | grep -q "nexent-config"; then + # In infrastructure mode or if nexent-config is not running, use supabase-db-mini + if docker ps | grep -q "supabase-db-mini"; then + curl_container="supabase-db-mini" + echo " ℹ️ Using supabase-db-mini container (infrastructure mode)" + else + echo " ❌ Neither nexent-config nor supabase-db-mini container is available." + return 1 + fi + fi + + RESPONSE=$(docker exec "$curl_container" bash -c "curl -s -X POST http://kong:8000/auth/v1/signup -H \"apikey: ${SUPABASE_KEY}\" -H \"Authorization: Bearer ${SUPABASE_KEY}\" -H \"Content-Type: application/json\" -d '{\"email\":\"${email}\",\"password\":\"${password}\",\"email_confirm\":true}'" 2>/dev/null) if [ -z "$RESPONSE" ]; then echo " ❌ No response received from Supabase." @@ -65,21 +87,24 @@ create_default_super_admin_user() { elif echo "$RESPONSE" | grep -q '"access_token"' && echo "$RESPONSE" | grep -q '"user"'; then echo " ✅ Default super admin user has been successfully created." echo "" - echo " Please save the following credentials carefully, which would ONLY be shown once." + echo " Please save the following credentials carefully." echo " 📧 Email: ${email}" - echo " 🔏 Password: ${password}" + if [ -n "$1" ]; then + echo " 🔏 Password: [User provided password]" + else + echo " 🔏 Password: ${password}" + fi # Extract user.id from RESPONSE JSON local user_id - # Try using Python to parse JSON (most reliable) - user_id=$(echo "$RESPONSE" | docker exec -i nexent-config python3 -c "import sys, json; data = json.load(sys.stdin); print(data.get('user', {}).get('id', ''))" 2>/dev/null) - - # Fallback to jq if Python fails - if [ -z "$user_id" ] && command -v jq >/dev/null 2>&1; then + # Try using jq first (if available in the container or on host) + if docker exec "$curl_container" command -v jq >/dev/null 2>&1; then + user_id=$(echo "$RESPONSE" | docker exec -i "$curl_container" jq -r '.user.id // empty' 2>/dev/null) + elif command -v jq >/dev/null 2>&1; then user_id=$(echo "$RESPONSE" | jq -r '.user.id // empty' 2>/dev/null) fi - # Final fallback: use grep and sed + # Fallback: use grep and sed (works without any special tools) if [ -z "$user_id" ]; then user_id=$(echo "$RESPONSE" | grep -o '"user"[^}]*"id":"[^"]*"' | sed -n 's/.*"id":"\([^"]*\)".*/\1/p' 2>/dev/null) fi @@ -150,7 +175,8 @@ create_default_super_admin_user() { } # Main execution -if create_default_super_admin_user; then +# Pass password as first argument if provided +if create_default_super_admin_user "$1"; then exit 0 else exit 1 diff --git a/docker/deploy.sh b/docker/deploy.sh index 83d3f7947..e30e6e75a 100755 --- a/docker/deploy.sh +++ b/docker/deploy.sh @@ -865,24 +865,98 @@ select_terminal_tool() { echo "" } -generate_random_password() { - # Generate a URL/JSON safe random password (alphanumeric only) - local pwd="" - if command -v openssl >/dev/null 2>&1; then - pwd=$(openssl rand -base64 32 | tr -dc 'A-Za-z0-9' | head -c 20) - else - pwd=$(tr -dc 'A-Za-z0-9' /dev/null | tr -d '[:space:]') + if [ "$user_exists" = "1" ]; then + return 0 # User exists + elif [ "$user_exists" = "0" ]; then + return 1 # User does not exist + fi fi - if [ -z "$pwd" ]; then - # Fallback (should be extremely rare) - pwd=$(date +%s%N | tr -dc '0-9' | head -c 20) + + # Fallback: Try to sign in with a dummy password to check if user exists + # This is less reliable but works when database access is not available + local test_response + test_response=$(docker exec "$curl_container" bash -c "curl -s -X POST http://kong:8000/auth/v1/token?grant_type=password -H \"apikey: ${SUPABASE_KEY}\" -H \"Content-Type: application/json\" -d '{\"email\":\"${email}\",\"password\":\"dummy_password_check\"}'" 2>/dev/null) + + if echo "$test_response" | grep -q '"error_code":"invalid_credentials"'; then + return 0 # User exists (wrong password means user exists) + elif echo "$test_response" | grep -q '"error_code":"email_not_confirmed"'; then + return 0 # User exists + else + return 1 # User likely does not exist fi - echo "$pwd" +} + +prompt_super_admin_password() { + # Prompt user to enter password for super admin user with confirmation + # Note: All prompts go to stderr, only password is returned via stdout + local password="" + local password_confirm="" + local max_attempts=3 + local attempts=0 + + echo "" >&2 + echo "🔐 Super Admin User Password Setup" >&2 + echo " Email: suadmin@nexent.com" >&2 + echo "" >&2 + + while [ $attempts -lt $max_attempts ]; do + # First password input + echo " 🔐 Please enter password for super admin user:" >&2 + read -s password + echo "" >&2 + + # Check if password is empty + if [ -z "$password" ]; then + echo " ❌ Password cannot be empty. Please try again." >&2 + attempts=$((attempts + 1)) + continue + fi + + # Confirm password input + echo " 🔐 Please confirm the password:" >&2 + read -s password_confirm + echo "" >&2 + + # Check if passwords match + if [ "$password" != "$password_confirm" ]; then + echo " ❌ Passwords do not match. Please try again." >&2 + attempts=$((attempts + 1)) + continue + fi + + # Passwords match, return the password via stdout + echo "$password" + return 0 + done + + # Max attempts reached + echo " ❌ Maximum attempts reached. Failed to set password." >&2 + return 1 } create_default_super_admin_user() { # Call the dedicated script for creating super admin user local script_path="$SCRIPT_DIR/create-su.sh" + local email="suadmin@nexent.com" if [ ! -f "$script_path" ]; then echo " ❌ ERROR create-su.sh not found at $script_path" @@ -892,15 +966,43 @@ create_default_super_admin_user() { # Make sure the script is executable chmod +x "$script_path" + # Check if super admin user already exists + echo "" + echo "🔍 Checking if super admin user exists..." + local check_result + check_super_admin_user_exists + check_result=$? + + if [ $check_result -eq 0 ]; then + echo " ✅ Super admin user (${email}) already exists." + echo " 💡 Skipping user creation. If you need to reset the password, please do so manually." + return 0 + elif [ $check_result -eq 1 ]; then + echo " ℹ️ Super admin user (${email}) does not exist. Proceeding with creation..." + else + echo " ⚠️ Warning: Could not determine if user exists. Proceeding with creation..." + fi + + # Prompt for password + local password + password="$(prompt_super_admin_password)" + local prompt_result=$? + + if [ $prompt_result -ne 0 ] || [ -z "$password" ]; then + echo " ❌ Failed to get password from user." + return 1 + fi + # Export necessary environment variables for the script export SUPABASE_KEY export POSTGRES_USER export POSTGRES_DB export DEPLOYMENT_VERSION export SUPABASE_POSTGRES_DB + export DEPLOYMENT_MODE - # Execute the script with current environment variables - if bash "$script_path"; then + # Execute the script with password as argument + if bash "$script_path" "$password"; then return 0 else return 1 @@ -984,6 +1086,12 @@ main_deploy() { # Special handling for infrastructure mode if [ "$DEPLOYMENT_MODE" = "infrastructure" ]; then generate_env_for_infrastructure || { echo "❌ Environment generation failed"; exit 1; } + + # Create default super admin user (only for full version) + if [ "$DEPLOYMENT_VERSION" = "full" ]; then + create_default_super_admin_user || { echo "❌ Default super admin user creation failed"; exit 1; } + fi + echo "🎉 Infrastructure deployment completed successfully!" echo " You can now start the core services manually using dev containers" echo " Environment file available at: $(cd .. && pwd)/.env" diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml index e9d344461..8eef651ae 100644 --- a/docker/docker-compose.prod.yml +++ b/docker/docker-compose.prod.yml @@ -272,6 +272,7 @@ services: mc admin policy attach myadmin readwrite --user=$MINIO_ACCESS_KEY mc mb myadmin/$MINIO_DEFAULT_BUCKET mc anonymous set download myadmin/$MINIO_DEFAULT_BUCKET + mc ilm rule add myadmin/$MINIO_DEFAULT_BUCKET --prefix 'preview/' --expiry-days 7 --id expire-converted-pdfs wait $$MINIO_PID " diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 221ff0c89..321f29665 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -298,6 +298,7 @@ services: mc admin policy attach myadmin readwrite --user=$MINIO_ACCESS_KEY mc mb myadmin/$MINIO_DEFAULT_BUCKET mc anonymous set download myadmin/$MINIO_DEFAULT_BUCKET + mc ilm rule add myadmin/$MINIO_DEFAULT_BUCKET --prefix 'preview/' --expiry-days 7 --id expire-converted-pdfs wait $$MINIO_PID " diff --git a/docker/sql/v1.8.1_0306_add_user_token_info.sql b/docker/sql/v1.8.1_0306_add_user_token_info.sql new file mode 100644 index 000000000..040530334 --- /dev/null +++ b/docker/sql/v1.8.1_0306_add_user_token_info.sql @@ -0,0 +1,118 @@ +-- Migration: Add user_token_info_t and user_token_usage_log_t tables +-- Date: 2026-03-06 +-- Description: Create user token (AK/SK) management tables with audit fields + +-- Set search path to nexent schema +SET search_path TO nexent; + +-- Create the user_token_info_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.user_token_info_t ( + token_id SERIAL4 PRIMARY KEY NOT NULL, + access_key VARCHAR(100) NOT NULL, + user_id VARCHAR(100) NOT NULL, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "user_token_info_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.user_token_info_t IS 'User token (AK/SK) information table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.user_token_info_t.token_id IS 'Token ID, unique primary key'; +COMMENT ON COLUMN nexent.user_token_info_t.access_key IS 'Access Key (AK)'; +COMMENT ON COLUMN nexent.user_token_info_t.user_id IS 'User ID who owns this token'; +COMMENT ON COLUMN nexent.user_token_info_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.user_token_info_t.delete_flag IS 'Soft delete flag, Y means deleted'; + +-- Create unique index on access_key to ensure uniqueness +CREATE UNIQUE INDEX IF NOT EXISTS idx_user_token_info_access_key ON nexent.user_token_info_t(access_key) WHERE delete_flag = 'N'; + +-- Create index on user_id for query performance +CREATE INDEX IF NOT EXISTS idx_user_token_info_user_id ON nexent.user_token_info_t(user_id) WHERE delete_flag = 'N'; + +-- Create a function to update the update_time column +CREATE OR REPLACE FUNCTION update_user_token_info_update_time() +RETURNS TRIGGER AS $$ +BEGIN + NEW.update_time = CURRENT_TIMESTAMP; + RETURN NEW; +END; +$$ LANGUAGE plpgsql; + +-- Add comment to the function +COMMENT ON FUNCTION update_user_token_info_update_time() IS 'Function to update the update_time column when a record in user_token_info_t is updated'; + +-- Create a trigger to call the function before each update +DROP TRIGGER IF EXISTS update_user_token_info_update_time_trigger ON nexent.user_token_info_t; +CREATE TRIGGER update_user_token_info_update_time_trigger +BEFORE UPDATE ON nexent.user_token_info_t +FOR EACH ROW +EXECUTE FUNCTION update_user_token_info_update_time(); + +-- Add comment to the trigger +COMMENT ON TRIGGER update_user_token_info_update_time_trigger ON nexent.user_token_info_t IS 'Trigger to call update_user_token_info_update_time function before each update on user_token_info_t table'; + + +-- Create the user_token_usage_log_t table in the nexent schema +CREATE TABLE IF NOT EXISTS nexent.user_token_usage_log_t ( + token_usage_id SERIAL4 PRIMARY KEY NOT NULL, + token_id INT4 NOT NULL, + call_function_name VARCHAR(100), + related_id INT4, + meta_data JSONB, + create_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + update_time TIMESTAMP WITHOUT TIME ZONE DEFAULT CURRENT_TIMESTAMP, + created_by VARCHAR(100), + updated_by VARCHAR(100), + delete_flag VARCHAR(1) DEFAULT 'N' +); + +ALTER TABLE "user_token_usage_log_t" OWNER TO "root"; + +-- Add comment to the table +COMMENT ON TABLE nexent.user_token_usage_log_t IS 'User token usage log table'; + +-- Add comments to the columns +COMMENT ON COLUMN nexent.user_token_usage_log_t.token_usage_id IS 'Token usage log ID, unique primary key'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.token_id IS 'Foreign key to user_token_info_t.token_id'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.call_function_name IS 'API function name being called'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.related_id IS 'Related resource ID (e.g., conversation_id)'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.meta_data IS 'Additional metadata for this usage log entry, stored as JSON'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.create_time IS 'Creation time, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.update_time IS 'Update time, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.created_by IS 'Creator ID, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.updated_by IS 'Last updater ID, audit field'; +COMMENT ON COLUMN nexent.user_token_usage_log_t.delete_flag IS 'Soft delete flag, Y means deleted'; + +-- Create index on token_id for query performance +CREATE INDEX IF NOT EXISTS idx_user_token_usage_log_token_id ON nexent.user_token_usage_log_t(token_id); + +-- Create index on call_function_name for query performance +CREATE INDEX IF NOT EXISTS idx_user_token_usage_log_function_name ON nexent.user_token_usage_log_t(call_function_name); + +-- Add foreign key constraint +ALTER TABLE nexent.user_token_usage_log_t +ADD CONSTRAINT fk_user_token_usage_log_token_id +FOREIGN KEY (token_id) +REFERENCES nexent.user_token_info_t(token_id) +ON DELETE CASCADE; + + +-- Migration: Remove partner_mapping_id_t table for northbound conversation ID mapping +-- Date: 2026-03-10 +-- Description: Remove the external-internal conversation ID mapping table as northbound APIs now use internal conversation IDs directly +-- Note: This table is no longer needed after refactoring northbound authentication logic + +-- Drop the partner_mapping_id_t table if it exists +DROP TABLE IF EXISTS nexent.partner_mapping_id_t CASCADE; + +-- Drop the associated sequence if it exists +DROP SEQUENCE IF EXISTS nexent.partner_mapping_id_t_id_seq; diff --git a/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx b/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx index f5815a094..c16ab969e 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/ToolManagement.tsx @@ -1,17 +1,17 @@ "use client"; -import { useState, useEffect, useCallback, useMemo } from "react"; +import { useState, useEffect, useCallback } from "react"; import { useTranslation } from "react-i18next"; import ToolConfigModal from "./tool/ToolConfigModal"; import { ToolGroup, Tool, ToolParam } from "@/types/agentConfig"; import { Tabs, Collapse, message, Tooltip } from "antd"; import { useAgentConfigStore } from "@/stores/agentConfigStore"; import { useToolList } from "@/hooks/agent/useToolList"; -import { useModelList } from "@/hooks/model/useModelList"; import { usePrefetchKnowledgeBases } from "@/hooks/useKnowledgeBaseSelector"; import { useConfig } from "@/hooks/useConfig"; import { updateToolConfig } from "@/services/agentConfigService"; import { useQueryClient } from "@tanstack/react-query"; +import { useConfirmModal } from "@/hooks/useConfirmModal"; import { Settings, AlertTriangle } from "lucide-react"; @@ -26,6 +26,7 @@ const TOOLS_REQUIRING_KB_SELECTION = [ "knowledge_base_search", "dify_search", "datamate_search", + "idata_search", ]; // Tool types that require Embedding model @@ -40,10 +41,11 @@ const TOOLS_REQUIRING_VLM = [ function getToolKbType( toolName: string -): "knowledge_base_search" | "dify_search" | "datamate_search" | null { +): "knowledge_base_search" | "dify_search" | "datamate_search" | "idata_search" | null { if (!TOOLS_REQUIRING_KB_SELECTION.includes(toolName)) return null; if (toolName === "dify_search") return "dify_search"; if (toolName === "datamate_search") return "datamate_search"; + if (toolName === "idata_search") return "idata_search"; return "knowledge_base_search"; } @@ -74,6 +76,7 @@ export default function ToolManagement({ }: ToolManagementProps) { const { t } = useTranslation("common"); const queryClient = useQueryClient(); + const { confirm } = useConfirmModal(); // Get current agent permission from store const currentAgentPermission = useAgentConfigStore( @@ -98,73 +101,7 @@ export default function ToolManagement({ // Use tool list hook for data management const { availableTools } = useToolList(); - // Get config for model checks - const { modelConfig: tenantModelConfig } = useConfig(); - - // Get VLM models to check availability - const { availableVlmModels, models } = useModelList(); - - // Check if VLM is properly configured: - // 1. Must have at least one VLM model that passed health check (available) - // 2. Must have a VLM model selected in tenant configuration - const isVlmConfigured = useMemo(() => { - // Check if there's any available VLM model - if (!availableVlmModels || availableVlmModels.length === 0) { - return false; - } - - // Check if tenant configuration has selected a VLM model - try { - const selectedVlmModelName = tenantModelConfig?.vlm?.modelName || tenantModelConfig?.vlm?.displayName; - - if (!selectedVlmModelName) { - return false; - } - - // Check if the selected VLM model exists in available models - const isSelectedModelAvailable = availableVlmModels.some( - (model) => model.name === selectedVlmModelName || model.displayName === selectedVlmModelName - ); - - return isSelectedModelAvailable; - } catch (error) { - return false; - } - }, [availableVlmModels, models, tenantModelConfig]); - - // Get Embedding models to check availability - const { availableEmbeddingModels } = useModelList(); - - // Check if Embedding is properly configured: - // 1. Must have at least one Embedding model that passed health check (available) - // 2. Must have an Embedding model selected in tenant configuration - const isEmbeddingConfigured = useMemo(() => { - // Check if there's any available Embedding model - if (!availableEmbeddingModels || availableEmbeddingModels.length === 0) { - return false; - } - - // Check if tenant configuration has selected an Embedding model - try { - const selectedEmbeddingModelName = - tenantModelConfig?.embedding?.modelName || tenantModelConfig?.embedding?.displayName; - - if (!selectedEmbeddingModelName) { - return false; - } - - // Check if the selected Embedding model exists in available models - const isSelectedModelAvailable = availableEmbeddingModels.some( - (model) => - model.name === selectedEmbeddingModelName || - model.displayName === selectedEmbeddingModelName - ); - - return isSelectedModelAvailable; - } catch (error) { - return false; - } - }, [availableEmbeddingModels, models, tenantModelConfig]); + const { isVlmAvailable, isEmbeddingAvailable } = useConfig(); // Prefetch knowledge bases for KB tools const { prefetchKnowledgeBases } = usePrefetchKnowledgeBases(); @@ -235,9 +172,7 @@ export default function ToolManagement({ (t) => parseInt(t.id) === parseInt(tool.id) ); // Merge configured tool with original tool to ensure all fields are present - const toolToUse = configuredTool - ? { ...tool, ...configuredTool, initParams: configuredTool.initParams } - : tool; + const toolToUse = configuredTool ? { ...tool, ...configuredTool, initParams: configuredTool.initParams } : tool; // Get merged parameters (for editing mode, merge with instance params) const mergedParams = await mergeToolParamsWithInstance( @@ -264,96 +199,96 @@ export default function ToolManagement({ } // Get latest tools directly from store to avoid stale closure issues - const currentSelectdTools = - useAgentConfigStore.getState().editedAgent.tools; + const currentSelectdTools = useAgentConfigStore.getState().editedAgent.tools; const isCurrentlySelected = currentSelectdTools.some( (t) => parseInt(t.id) === numericId ); if (isCurrentlySelected) { // If already selected, deselect it - const newSelectedTools = currentSelectdTools.filter( - (t) => parseInt(t.id) !== numericId - ); + const newSelectedTools = currentSelectdTools.filter((t) => parseInt(t.id) !== numericId); updateTools(newSelectedTools); } else { - // If not selected, determine tool params and check if modal is needed - const configuredTool = currentSelectdTools.find( - (t) => parseInt(t.id) === numericId - ); - // Merge configured tool with original tool to ensure all fields are present - const toolToUse = configuredTool - ? { ...tool, ...configuredTool, initParams: configuredTool.initParams } - : tool; - - // Get merged parameters (for editing mode, merge with instance params) - const mergedParams = await mergeToolParamsWithInstance( - tool, - toolToUse, - isCreatingMode ? undefined : currentAgentId! - ); + // Helper function to proceed with tool selection after duplicate check + async function proceedWithToolSelection() { + // Get latest tools again to ensure we have the most up-to-date list + const currentSelectdTools = + useAgentConfigStore.getState().editedAgent.tools; + + // Determine tool params and check if modal is needed + const configuredTool = currentSelectdTools.find( + (t) => parseInt(t.id) === numericId + ); + // Merge configured tool with original tool to ensure all fields are present + const toolToUse = configuredTool + ? { ...tool, ...configuredTool, initParams: configuredTool.initParams } + : tool; + + // Get merged parameters (for editing mode, merge with instance params) + const mergedParams = await mergeToolParamsWithInstance( + tool, + toolToUse, + isCreatingMode ? undefined : currentAgentId! + ); + + // Check if there are empty required params + const hasEmptyRequiredParams = mergedParams.some( + (param: ToolParam) => + param.required && + (param.value === undefined || + param.value === "" || + param.value === null) + ); + + if (hasEmptyRequiredParams) { + // Need to configure, open modal + setSelectedTool(toolToUse); + setToolParams(mergedParams); + setIsToolModalOpen(true); + } else { + // No required params missing, add directly + const newSelectedTools = [ + ...currentSelectdTools, + { + ...toolToUse, + initParams: mergedParams, + }, + ]; + updateTools(newSelectedTools); + } + } - // Check if there are empty required params - const hasEmptyRequiredParams = mergedParams.some( - (param: ToolParam) => - param.required && - (param.value === undefined || - param.value === "" || - param.value === null) + // If not selected, check for duplicate tool names first + const duplicateTool = currentSelectdTools.find( + (selectedTool) => selectedTool.name === tool.name ); - if (hasEmptyRequiredParams) { - // Need to configure, open modal - setSelectedTool(toolToUse); - setToolParams(mergedParams); - setIsToolModalOpen(true); - } else { - // No required params missing, add directly - const newSelectedTools = [ - ...currentSelectdTools, - { - ...toolToUse, - initParams: mergedParams, - }, - ]; - updateTools(newSelectedTools); - - // In non-creating mode, immediately save tool config to backend - if (!isCreatingMode && currentAgentId) { - try { - // Convert params to backend format - const paramsObj = mergedParams.reduce( - (acc, param) => { - acc[param.name] = param.value; - return acc; - }, - {} as Record - ); - - const isEnabled = true; // New tool is enabled by default - const result = await updateToolConfig( - numericId, - currentAgentId, - paramsObj, - isEnabled - ); - - if (result.success) { - // Invalidate queries to refresh tool info - queryClient.invalidateQueries({ - queryKey: ["toolInfo", numericId, currentAgentId], - }); - } else { - message.error( - result.message || t("toolConfig.message.saveError") - ); - } - } catch (error) { - console.error("Failed to save tool config:", error); - message.error(t("toolConfig.message.saveError")); - } - } + if (duplicateTool) { + // Show confirmation modal for duplicate tool name + return new Promise((resolve) => { + confirm({ + title: t("toolPool.duplicateToolName.title"), + content: t("toolPool.duplicateToolName.content", { + toolName: tool.name, + }), + okText: t("toolPool.duplicateToolName.confirm"), + cancelText: t("toolPool.duplicateToolName.cancel"), + danger: true, + onOk: async () => { + // User confirmed, proceed with tool selection + await proceedWithToolSelection(); + resolve(); + }, + onCancel: () => { + // User cancelled, do nothing + resolve(); + }, + }); + }); } + + // No duplicate, proceed with normal tool selection + await proceedWithToolSelection(); } }; @@ -428,8 +363,8 @@ export default function ToolManagement({ const isSelected = originalSelectedToolIdsSet.has( tool.id ); - const isDisabledDueToVlm = isToolDisabledDueToVlm(tool.name, isVlmConfigured); - const isDisabledDueToEmbedding = isToolDisabledDueToEmbedding(tool.name, isEmbeddingConfigured); + const isDisabledDueToVlm = isToolDisabledDueToVlm(tool.name, isVlmAvailable); + const isDisabledDueToEmbedding = isToolDisabledDueToEmbedding(tool.name, isEmbeddingAvailable); const isDisabled = isDisabledDueToVlm || isDisabledDueToEmbedding || isReadOnly; // Tooltip priority: permission > VLM > Embedding const tooltipTitle = isReadOnly @@ -533,8 +468,8 @@ export default function ToolManagement({ > {group.tools.map((tool) => { const isSelected = originalSelectedToolIdsSet.has(tool.id); - const isDisabledDueToVlm = isToolDisabledDueToVlm(tool.name, isVlmConfigured); - const isDisabledDueToEmbedding = isToolDisabledDueToEmbedding(tool.name, isEmbeddingConfigured); + const isDisabledDueToVlm = isToolDisabledDueToVlm(tool.name, isVlmAvailable); + const isDisabledDueToEmbedding = isToolDisabledDueToEmbedding(tool.name, isEmbeddingAvailable); const isDisabled = isDisabledDueToVlm || isDisabledDueToEmbedding || isReadOnly; // Tooltip priority: permission > VLM > Embedding const tooltipTitle = isReadOnly diff --git a/frontend/app/[locale]/agents/components/agentConfig/tool/ToolConfigModal.tsx b/frontend/app/[locale]/agents/components/agentConfig/tool/ToolConfigModal.tsx index 2a616326b..fc927d51d 100644 --- a/frontend/app/[locale]/agents/components/agentConfig/tool/ToolConfigModal.tsx +++ b/frontend/app/[locale]/agents/components/agentConfig/tool/ToolConfigModal.tsx @@ -27,6 +27,7 @@ import { useConfig } from "@/hooks/useConfig"; import { useKnowledgeBasesForToolConfig } from "@/hooks/useKnowledgeBaseSelector"; import { useKnowledgeBaseConfigChangeHandler } from "@/hooks/useKnowledgeBaseConfigChangeHandler"; import { API_ENDPOINTS } from "@/services/api"; +import knowledgeBaseService from "@/services/knowledgeBaseService"; import log from "@/lib/logger"; export interface ToolConfigModalProps { @@ -45,6 +46,7 @@ const TOOLS_REQUIRING_KB_SELECTION = [ "knowledge_base_search", "dify_search", "datamate_search", + "idata_search", ]; export default function ToolConfigModal({ @@ -91,6 +93,26 @@ export default function ToolConfigModal({ apiKey: "", }); + // iData configuration state + const [idataConfig, setIdataConfig] = useState<{ + serverUrl: string; + apiKey: string; + userId: string; + knowledgeSpaceId: string; + }>({ + serverUrl: "", + apiKey: "", + userId: "", + knowledgeSpaceId: "", + }); + + // iData knowledge spaces state + const [idataKnowledgeSpaces, setIdataKnowledgeSpaces] = useState< + Array<{ id: string; name: string }> + >([]); + const [idataKnowledgeSpacesLoading, setIdataKnowledgeSpacesLoading] = + useState(false); + // DataMate URL from knowledge base configuration const [knowledgeBaseDataMateUrl, setKnowledgeBaseDataMateUrl] = useState(""); @@ -117,11 +139,13 @@ export default function ToolConfigModal({ | "knowledge_base_search" | "dify_search" | "datamate_search" + | "idata_search" | null => { if (!toolRequiresKbSelection) return null; const name = tool?.name; if (name === "dify_search") return "dify_search"; if (name === "datamate_search") return "datamate_search"; + if (name === "idata_search") return "idata_search"; return "knowledge_base_search"; }, [tool?.name, toolRequiresKbSelection]); @@ -147,6 +171,46 @@ export default function ToolConfigModal({ } }, [toolKbType, difyServerUrlParam, difyApiKeyParam]); + // Get iData configuration from initial params + const idataServerUrlParam = useMemo(() => { + return currentParams.find((param) => param.name === "server_url"); + }, [currentParams]); + + const idataApiKeyParam = useMemo(() => { + return currentParams.find((param) => param.name === "api_key"); + }, [currentParams]); + + const idataUserIdParam = useMemo(() => { + return currentParams.find((param) => param.name === "user_id"); + }, [currentParams]); + + const idataKnowledgeSpaceIdParam = useMemo(() => { + return currentParams.find((param) => param.name === "knowledge_space_id"); + }, [currentParams]); + + // Initialize iData config from params + useEffect(() => { + if (toolKbType === "idata_search") { + const serverUrl = idataServerUrlParam?.value || ""; + const apiKey = idataApiKeyParam?.value || ""; + const userId = idataUserIdParam?.value || ""; + const knowledgeSpaceId = idataKnowledgeSpaceIdParam?.value || ""; + + setIdataConfig({ + serverUrl, + apiKey, + userId, + knowledgeSpaceId, + }); + } + }, [ + toolKbType, + idataServerUrlParam, + idataApiKeyParam, + idataUserIdParam, + idataKnowledgeSpaceIdParam, + ]); + // Fetch knowledge bases for tool config based on tool type (now uses React Query caching) // For datamate_search, use the server_url from the form as config const datamateServerUrl = useMemo(() => { @@ -157,6 +221,40 @@ export default function ToolConfigModal({ return ""; }, [toolKbType, currentParams]); + // Fetch iData knowledge spaces when config is available + useEffect(() => { + if ( + toolKbType === "idata_search" && + idataConfig.serverUrl && + idataConfig.apiKey && + idataConfig.userId + ) { + setIdataKnowledgeSpacesLoading(true); + knowledgeBaseService + .getIdataKnowledgeSpaces( + idataConfig.serverUrl, + idataConfig.apiKey, + idataConfig.userId + ) + .then((spaces) => { + setIdataKnowledgeSpaces(spaces); + setIdataKnowledgeSpacesLoading(false); + }) + .catch((error) => { + log.error("Failed to fetch iData knowledge spaces:", error); + setIdataKnowledgeSpaces([]); + setIdataKnowledgeSpacesLoading(false); + }); + } else if (toolKbType === "idata_search") { + setIdataKnowledgeSpaces([]); + } + }, [ + toolKbType, + idataConfig.serverUrl, + idataConfig.apiKey, + idataConfig.userId, + ]); + const { data: knowledgeBases = [], isLoading: kbLoading, @@ -168,7 +266,19 @@ export default function ToolConfigModal({ ? difyConfig : toolKbType === "datamate_search" ? { serverUrl: datamateServerUrl } - : undefined + : toolKbType === "idata_search" + ? idataConfig.serverUrl && + idataConfig.apiKey && + idataConfig.userId && + idataConfig.knowledgeSpaceId + ? { + serverUrl: idataConfig.serverUrl, + apiKey: idataConfig.apiKey, + userId: idataConfig.userId, + knowledgeSpaceId: idataConfig.knowledgeSpaceId, + } + : undefined + : undefined ); // Handle config change: clear knowledge base selection and refetch @@ -210,10 +320,92 @@ export default function ToolConfigModal({ ? difyConfig : toolKbType === "datamate_search" ? { serverUrl: datamateServerUrl } - : undefined, + : toolKbType === "idata_search" + ? { + serverUrl: idataConfig.serverUrl, + apiKey: idataConfig.apiKey, + userId: idataConfig.userId, + } + : undefined, onConfigChange: handleKbConfigChange, }); + // Handle iData knowledge space ID change: clear knowledge base selection and refetch + const prevKnowledgeSpaceIdRef = useRef(""); + useEffect(() => { + if ( + toolKbType === "idata_search" && + idataConfig.knowledgeSpaceId && + idataConfig.serverUrl && + idataConfig.apiKey && + idataConfig.userId + ) { + // Only trigger if knowledge space ID actually changed + // Skip if this is the initial load (prevKnowledgeSpaceIdRef is empty and we have a value from initialParams) + if (prevKnowledgeSpaceIdRef.current === idataConfig.knowledgeSpaceId) { + return; + } + + // If prevKnowledgeSpaceIdRef is empty, this is likely the initial load + // Don't clear dataset_ids on initial load, only when space ID actually changes + if (prevKnowledgeSpaceIdRef.current === "") { + // This is initial load, just update the ref without clearing + prevKnowledgeSpaceIdRef.current = idataConfig.knowledgeSpaceId; + return; + } + + // Update ref + prevKnowledgeSpaceIdRef.current = idataConfig.knowledgeSpaceId; + + // Clear previous knowledge base selection when space ID changes + setSelectedKbIds([]); + setSelectedKbDisplayNames([]); + + // Clear form value for dataset_ids field + const kbFieldIndex = currentParams.findIndex( + (p) => p.name === "dataset_ids" + ); + if (kbFieldIndex >= 0) { + form.setFieldValue(`param_${kbFieldIndex}`, []); + const updatedParams = [...currentParams]; + updatedParams[kbFieldIndex] = { + ...updatedParams[kbFieldIndex], + value: [], + }; + setCurrentParams(updatedParams); + } + + // Refetch knowledge bases with new space ID + refetchKnowledgeBases(); + } else if (toolKbType === "idata_search") { + // Reset ref when config is cleared + prevKnowledgeSpaceIdRef.current = ""; + } + }, [ + toolKbType, + idataConfig.knowledgeSpaceId, + idataConfig.serverUrl, + idataConfig.apiKey, + idataConfig.userId, + refetchKnowledgeBases, + currentParams, + form, + ]); + + // Reset prevKnowledgeSpaceIdRef when modal opens/closes + useEffect(() => { + if (!isOpen) { + // Reset ref when modal closes + prevKnowledgeSpaceIdRef.current = ""; + } else if (isOpen && toolKbType === "idata_search") { + // Initialize ref with current knowledgeSpaceId when modal opens + // This prevents clearing dataset_ids on initial load + if (idataConfig.knowledgeSpaceId) { + prevKnowledgeSpaceIdRef.current = idataConfig.knowledgeSpaceId; + } + } + }, [isOpen, toolKbType, idataConfig.knowledgeSpaceId]); + // Get current embedding model from config for model matching const currentEmbeddingModel = useMemo(() => { try { @@ -746,51 +938,10 @@ export default function ToolConfigModal({ newSelectedTools = [...currentTools, updatedTool]; } - // For editing mode (when currentAgentId exists), always call API - // For creating mode (isCreatingMode=true), update local state only - if (isCreatingMode) { - // In creating mode, just update local state - updateTools(newSelectedTools); - message.success(t("toolConfig.message.saveSuccess")); - handleClose(); // Close modal - return; - } - - if (!currentAgentId) { - // Should not happen in normal editing mode, but handle gracefully - updateTools(newSelectedTools); - message.success(t("toolConfig.message.saveSuccess")); - handleClose(); // Close modal - return; - } - - // Edit mode: call API to persist changes - try { - setIsLoading(true); - const isEnabled = true; // New tool is enabled by default - const result = await updateToolConfig( - parseInt(toolToSave.id), - currentAgentId, - paramsObj, - isEnabled - ); - setIsLoading(false); - - if (result.success) { - // Update local state and invalidate queries - updateTools(newSelectedTools); - queryClient.invalidateQueries({ - queryKey: ["toolInfo", parseInt(toolToSave.id), currentAgentId], - }); - message.success(t("toolConfig.message.saveSuccess")); - handleClose(); // Close modal - } else { - message.error(result.message || t("toolConfig.message.saveError")); - } - } catch (error) { - setIsLoading(false); - message.error(t("toolConfig.message.saveError")); - } + // Update local state only - actual save will happen when user clicks "Save Agent" + updateTools(newSelectedTools); + message.success(t("toolConfig.message.saveSuccess")); + handleClose(); // Close modal // Call original onSave if provided if (onSave) { @@ -888,7 +1039,8 @@ export default function ToolConfigModal({ const getToolType = (): | "knowledge_base_search" | "dify_search" - | "datamate_search" => { + | "datamate_search" + | "idata_search" => { return toolKbType || "knowledge_base_search"; }; @@ -1025,13 +1177,47 @@ export default function ToolConfigModal({ ); const renderParamInput = (param: ToolParam, index: number) => { + // Get field name for form + const fieldName = `param_${index}`; + // Get options from frontend configuration based on tool name and parameter name const options = getToolParamOptions(tool.name, param.name); // Determine if this parameter should be rendered as a select dropdown const isSelectType = options && options.length > 0; + // Special handling for iData knowledge_space_id parameter + const isIdataKnowledgeSpaceId = + toolKbType === "idata_search" && param.name === "knowledge_space_id"; + const inputComponent = (() => { + // Handle iData knowledge space ID selector + if (isIdataKnowledgeSpaceId) { + const currentValue = form.getFieldValue(fieldName); + return ( + setEditTitle(e.target.value)} - onKeyDown={handleKeyDown} - onBlur={handleSubmit} - className="text-xl font-bold text-center h-9 max-w-xs" - autoFocus - /> - ) : ( -

- {title} -

- )} - - - -
- {/* Right side controls - now handled by navigation bar */} -
- +
+ {isEditing ? ( + setEditTitle(e.target.value)} + onKeyDown={handleKeyDown} + onBlur={handleSubmit} + className="text-xl font-bold text-center h-9 max-w-xs" + autoFocus + /> + ) : ( +

+ {title} +

+ )}
diff --git a/frontend/app/[locale]/chat/components/chatInput.tsx b/frontend/app/[locale]/chat/components/chatInput.tsx index 7665b934c..9b175c8cd 100644 --- a/frontend/app/[locale]/chat/components/chatInput.tsx +++ b/frontend/app/[locale]/chat/components/chatInput.tsx @@ -1257,7 +1257,7 @@ export function ChatInput({ ) : ( -
+
{ +const categorizeConversations = (conversations: ConversationListItem[]) => { const now = new Date(); const today = new Date( now.getFullYear(), @@ -59,248 +60,278 @@ const categorizeDialogs = (dialogs: ConversationListItem[]) => { ).getTime(); const weekAgo = today - 7 * 24 * 60 * 60 * 1000; - const todayDialogs: ConversationListItem[] = []; - const weekDialogs: ConversationListItem[] = []; - const olderDialogs: ConversationListItem[] = []; + const todayConversations: ConversationListItem[] = []; + const weekConversations: ConversationListItem[] = []; + const olderConversations: ConversationListItem[] = []; - dialogs.forEach((dialog) => { - const dialogTime = dialog.create_time; + conversations.forEach((conversations) => { + const conversationTime = conversations.create_time; - if (dialogTime >= today) { - todayDialogs.push(dialog); - } else if (dialogTime >= weekAgo) { - weekDialogs.push(dialog); + if (conversationTime >= today) { + todayConversations.push(conversations); + } else if (conversationTime >= weekAgo) { + weekConversations.push(conversations); } else { - olderDialogs.push(dialog); + olderConversations.push(conversations); } }); return { - today: todayDialogs, - week: weekDialogs, - older: olderDialogs, + today: todayConversations, + week: weekConversations, + older: olderConversations, }; }; +// Chat sidebar props type +export interface ChatSidebarProps { + streamingConversations: Set; + completedConversations: Set; + conversationManagement: ConversationManagement; + /** Called when user clicks a conversation - loads messages and updates selection */ + onConversationSelect: (conversation: ConversationListItem) => void | Promise; +} + +const CONVERSATION_TITLE_MAX_LENGTH = 100; + export function ChatSidebar({ - conversationList, - selectedConversationId, - openDropdownId, streamingConversations, completedConversations, - onNewConversation, - onDialogClick, - onRename, - onDelete, - onSettingsClick, - onDropdownOpenChange, - onToggleSidebar, - expanded, - userEmail, - userAvatarUrl + conversationManagement, + onConversationSelect, }: ChatSidebarProps) { const { t } = useTranslation(); const { confirm } = useConfirmModal(); - const router = useRouter(); - const { today, week, older } = categorizeDialogs(conversationList); + const { today, week, older } = categorizeConversations(conversationManagement.conversationList); const [editingId, setEditingId] = useState(null); - const [editingTitle, setEditingTitle] = useState(""); - const inputRef = useRef(null); - - const [animationComplete, setAnimationComplete] = useState(false); + const [renameValue, setRenameValue] = useState(""); + const [renameError, setRenameError] = useState(null); + const [collapsed, setCollapsed] = useState(false); + const [openDropdownId, setOpenDropdownId] = useState(null); - useEffect(() => { - // Reset animation state when expanded changes - setAnimationComplete(false); + const onToggleSidebar = () => setCollapsed((prev) => !prev); - // Set animation complete after the transition duration (200ms) - const timer = setTimeout(() => { - setAnimationComplete(true); - }, 200); - - return () => clearTimeout(timer); - }, [expanded]); - - // Handle edit start - const handleStartEdit = (dialogId: number, title: string) => { - setEditingId(dialogId); - setEditingTitle(title); - // Close any open dropdown menus - onDropdownOpenChange(false, null); + const handleRenameClick = (conversationId: number, currentTitle: string) => { + setEditingId(conversationId); + setRenameValue(currentTitle); + setRenameError(null); + setOpenDropdownId(null); + }; - // Use setTimeout to ensure that the input box is focused after the DOM is updated - setTimeout(() => { - if (inputRef.current) { - inputRef.current.focus(); - inputRef.current.select(); - } - }, 10); + const validateRenameTitle = (title: string): string | null => { + const trimmedTitle = title.trim(); + if (!trimmedTitle) { + return t("chatLeftSidebar.renameErrorEmpty"); + } + if (trimmedTitle.length > CONVERSATION_TITLE_MAX_LENGTH) { + return t("chatLeftSidebar.renameErrorTooLong", { + max: CONVERSATION_TITLE_MAX_LENGTH, + }); + } + return null; }; - // Handle edit submission - const handleSubmitEdit = () => { - if (editingId !== null && editingTitle.trim()) { - onRename(editingId, editingTitle.trim()); + const handleRename = async (conversationId: number, newTitle: string) => { + const trimmedTitle = newTitle.trim(); + if (!trimmedTitle) return false; + try { + await conversationService.rename(conversationId, trimmedTitle); + await conversationManagement.fetchConversationList(); + if (conversationManagement.selectedConversationId === conversationId) { + conversationManagement.setConversationTitle(trimmedTitle); + } setEditingId(null); + setRenameError(null); + return true; + } catch (error) { + log.error(t("chatInterface.renameFailed"), error); + setRenameError(t("chatLeftSidebar.renameErrorSubmitFailed")); + message.error(t("chatLeftSidebar.renameErrorSubmitFailed")); + return false; } }; - // Handle edit cancellation - const handleCancelEdit = () => { - setEditingId(null); - }; + const handleRenameSubmit = async (conversationId: number) => { + const validationError = validateRenameTitle(renameValue); + if (validationError) { + setRenameError(validationError); + message.warning(validationError); + return; + } - // Handle key events - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter") { - handleSubmitEdit(); - } else if (e.key === "Escape") { - handleCancelEdit(); + const success = await handleRename(conversationId, renameValue); + if (success) { + setRenameValue(""); } }; - // Handle delete click - const handleDeleteClick = (dialogId: number) => { - // Close dropdown menus - onDropdownOpenChange(false, null); + const handleRenameCancel = () => { + setEditingId(null); + setRenameValue(""); + setRenameError(null); + }; + + // Handle delete + const handleDelete = (conversationId: number) => { - // Show confirmation modal confirm({ title: t("chatLeftSidebar.confirmDeletionTitle"), content: t("chatLeftSidebar.confirmDeletionDescription"), - onOk: () => { - onDelete(dialogId); + onOk: async () => { + try { + await conversationService.delete(conversationId); + await conversationManagement.fetchConversationList(); + if (conversationManagement.selectedConversationId === conversationId) { + conversationManagement.setSelectedConversationId(null); + conversationManagement.setConversationTitle( + t("chatInterface.newConversation") + ); + conversationManagement.handleNewConversation(); + } + } catch (error) { + log.error(t("chatInterface.deleteFailed"), error); + } }, }); }; // Render dialog list items - const renderDialogList = (dialogs: ConversationListItem[], title: string) => { - if (dialogs.length === 0) return null; + const renderConversationList = (conversation: ConversationListItem[], title: string) => { + if (conversation.length === 0) return null; return ( -
+

{title}

- {dialogs.map((dialog) => ( -
- {editingId === dialog.conversation_id ? ( - // Edit mode -
- setEditingTitle(e.target.value)} - onKeyDown={handleKeyDown} - onBlur={handleSubmitEdit} - className="h-8 text-base" - autoFocus - /> -
- ) : ( - // Display mode - <> - - {dialog.conversation_title}

+ {conversation.map((conversation) => { + const isEditing = editingId === conversation.conversation_id; + return ( +
+
+ + {conversation.conversation_title} + + ) : null} + placement="bottom" + > +
{ + if (!isEditing) { + onConversationSelect(conversation); } - placement="right" - styles={{ root: { maxWidth: "300px" } }} - > - - - + )} +
+
+ +
- - onDropdownOpenChange( - open, - dialog.conversation_id.toString() - ) +
+ setOpenDropdownId(open ? conversation.conversation_id : null)} + menu={{ + items: [ + { + key: "rename", + label: ( + + + {t("chatLeftSidebar.rename")} + + ), + }, + { + key: "delete", + label: ( + + + {t("chatLeftSidebar.delete")} + + ), + }, + ], + onClick: ({ key }) => { + if (key === "rename") { + handleRenameClick( + conversation.conversation_id, + conversation.conversation_title + ); + } else if (key === "delete") { + handleDelete(conversation.conversation_id); } - menu={{ - items: [ - { - key: "rename", - label: ( - - - {t("chatLeftSidebar.rename")} - - ), - }, - { - key: "delete", - label: ( - - - {t("chatLeftSidebar.delete")} - - ), - }, - ], - onClick: ({ key }) => { - if (key === "rename") { - handleStartEdit( - dialog.conversation_id, - dialog.conversation_title - ); - } else if (key === "delete") { - handleDeleteClick(dialog.conversation_id); - } - }, - }} - placement="bottomRight" - trigger={["click"]} - > - - - - )} -
- ))} + }, + }} + placement="bottomRight" + trigger={["click"]} + > + +
+
+
+ ); + })}
); }; @@ -311,40 +342,30 @@ export function ChatSidebar({ <> {/* Expand/Collapse button */}
- - + - - + + +
{/* New conversation button */} -
- - + + - - + + +
{/* Spacer */} @@ -354,20 +375,26 @@ export function ChatSidebar({ }; return ( - <> -
- {expanded || !animationComplete ? ( -
+ + {!collapsed ? ( +
- - - + + + +
+
+ +
+
+
+ {conversationManagement.conversationList.length > 0 ? + ( + <> + {renderConversationList(today, t("chatLeftSidebar.today"))} + {renderConversationList(week, t("chatLeftSidebar.last7Days"))} + {renderConversationList(older, t("chatLeftSidebar.older"))} + + ) : ( +
+

+ {t("chatLeftSidebar.recentConversations")} +

- - - +
+ )} +
- - -
- {conversationList.length > 0 ? ( - <> - {renderDialogList(today, t("chatLeftSidebar.today"))} - {renderDialogList(week, t("chatLeftSidebar.last7Days"))} - {renderDialogList(older, t("chatLeftSidebar.older"))} - - ) : ( -
-

- {t("chatLeftSidebar.recentConversations")} -

- -
- )} -
-
) : ( renderCollapsedSidebar() )} -
- + + ); } diff --git a/frontend/app/[locale]/chat/components/chatRightPanel.tsx b/frontend/app/[locale]/chat/components/chatRightPanel.tsx index 83b25c4b5..9eb9f6a7d 100644 --- a/frontend/app/[locale]/chat/components/chatRightPanel.tsx +++ b/frontend/app/[locale]/chat/components/chatRightPanel.tsx @@ -473,13 +473,13 @@ export function ChatRightPanel({