feat: browser_external_cdp

backend/app/model/chat.py

@@ -47,6 +47,8 @@ class Chat(BaseModel):
     api_url: str | None = None  # for cloud version, user don't need to set api_url
     language: str = "en"
     browser_port: int = 9222
+    use_external_cdp: bool = False
+    cdp_browsers: list[dict] = []
     max_retries: int = 3
     allow_local_system: bool = False
     installed_mcp: McpServers = {"mcpServers": {}}

backend/app/service/chat_service.py

@@ -236,6 +236,26 @@ def build_context_for_workforce(task_lock: TaskLock, options: Chat) -> str:
 @sync_step
 @traceroot.trace()
 async def step_solve(options: Chat, request: Request, task_lock: TaskLock):
+    # Log CDP browsers received from frontend
+    logger.info(f"[BACKEND CDP] ========================================")
+    logger.info(f"[BACKEND CDP] Received task request for project: {options.project_id}")
+    logger.info(f"[BACKEND CDP] browser_port: {options.browser_port}")
+    logger.info(f"[BACKEND CDP] use_external_cdp: {options.use_external_cdp}")
+
+    if hasattr(options, 'cdp_browsers') and options.cdp_browsers:
+        logger.info(f"[BACKEND CDP] cdp_browsers count: {len(options.cdp_browsers)}")
+        for idx, browser in enumerate(options.cdp_browsers):
+            port = browser.get('port', 'N/A')
+            is_external = browser.get('isExternal', 'N/A')
+            name = browser.get('name', 'Unnamed')
+            browser_id = browser.get('id', 'N/A')
+            logger.info(f"[BACKEND CDP]   Browser {idx + 1}: port={port}, isExternal={is_external}, name=\"{name}\", id={browser_id}")
+    else:
+        logger.warn(f"[BACKEND CDP] ⚠️  NO CDP browsers configured - cdp_browsers is empty or missing")
+        logger.warn(f"[BACKEND CDP] ⚠️  Agents will all use default browser port: {options.browser_port}")
+
+    logger.info(f"[BACKEND CDP] ========================================")
+
     # if True:
     #     import faulthandler
 

backend/app/utils/single_agent_worker.py

@@ -19,7 +19,7 @@ def __init__(
         description: str,
         worker: ListenChatAgent,
         use_agent_pool: bool = True,
-        pool_initial_size: int = 1,
+        pool_initial_size: int = 0,  # Changed from 1 to 0 to avoid pre-creating clones that waste CDP resources
         pool_max_size: int = 10,
         auto_scale_pool: bool = True,
         use_structured_output_handler: bool = True,
@@ -64,14 +64,19 @@ async def _process_task(self, task: Task, dependencies: list[Task]) -> TaskState
             TaskState: `TaskState.DONE` if processed successfully, otherwise
                 `TaskState.FAILED`.
         """
+        # Log task details before getting agent (for clone tracking)
+        task_content_preview = task.content[:100] + "..." if len(task.content) > 100 else task.content
+        logger.info(f"[TASK REQUEST] Requesting agent for task_id={task.id}, content_preview='{task_content_preview}'")
+
         # Get agent efficiently (from pool or by cloning)
         worker_agent = await self._get_worker_agent()
         worker_agent.process_task_id = task.id  # type: ignore  rewrite line
 
         logger.info("Starting task processing", extra={
             "task_id": task.id,
             "worker_agent_id": worker_agent.agent_id,
-            "dependencies_count": len(dependencies)
+            "dependencies_count": len(dependencies),
+            "task_content_preview": task_content_preview
         })
 
         response_content = ""

backend/app/utils/toolkit/hybrid_browser_python_toolkit.py

@@ -141,6 +141,7 @@ def __init__(
         enabled_tools: List[str] | None = None,
         browser_log_to_file: bool = False,
         session_id: str | None = None,
+        log_base_dir: str | None = None,
         default_start_url: str = "https://google.com/",
         default_timeout: int | None = None,
         short_timeout: int | None = None,
@@ -156,6 +157,7 @@ def __init__(
         self._stealth = stealth
         self._cache_dir = cache_dir
         self._browser_log_to_file = browser_log_to_file
+        self._log_base_dir = log_base_dir
         self._default_start_url = default_start_url
         self._session_id = session_id or "default"
 
@@ -179,7 +181,12 @@ def __init__(
         # Set up log file if needed
         if self.log_to_file:
             # Create log directory if it doesn't exist
-            log_dir = "browser_log"
+            # If log_base_dir is provided, use task-specific directory; otherwise use default backend/browser_log
+            if log_base_dir:
+                log_dir = os.path.join(log_base_dir, "browser_logs")
+            else:
+                log_dir = "browser_log"  # Backward compatibility: use default location
+
             os.makedirs(log_dir, exist_ok=True)
 
             timestamp = datetime.datetime.now().strftime("%Y%m%d_%H%M%S")
@@ -320,6 +327,7 @@ def clone_for_new_session(self, new_session_id: str | None = None) -> "HybridBro
             enabled_tools=self.enabled_tools.copy(),
             browser_log_to_file=self._browser_log_to_file,
             session_id=new_session_id,
+            log_base_dir=self._log_base_dir,
             default_start_url=self._default_start_url,
             default_timeout=self._default_timeout,
             short_timeout=self._short_timeout,

backend/app/utils/toolkit/hybrid_browser_toolkit.py

@@ -160,7 +160,7 @@ async def get_connection(self, session_id: str, config: Dict[str, Any]) -> WebSo
                         is_healthy = False
 
                 if is_healthy:
-                    logger.debug(f"Reusing healthy WebSocket connection for session {session_id}")
+                    logger.info(f"[CONNECTION POOL] Reusing healthy WebSocket connection for session {session_id}")
                     return wrapper
                 else:
                     # Connection is unhealthy, clean it up
@@ -172,11 +172,12 @@ async def get_connection(self, session_id: str, config: Dict[str, Any]) -> WebSo
                     del self._connections[session_id]
 
             # Create a new connection
-            logger.info(f"Creating new WebSocket connection for session {session_id}")
+            cdp_url = config.get('cdpUrl', 'NOT SET')
+            logger.info(f"[CONNECTION POOL] Creating new WebSocket connection for session {session_id}, CDP URL: {cdp_url}")
             wrapper = WebSocketBrowserWrapper(config)
             await wrapper.start()
             self._connections[session_id] = wrapper
-            logger.info(f"Successfully created WebSocket connection for session {session_id}")
+            logger.info(f"[CONNECTION POOL] Successfully created WebSocket connection for session {session_id}, CDP URL: {cdp_url}")
             return wrapper
 
     async def close_connection(self, session_id: str):
@@ -246,7 +247,10 @@ def __init__(
         logger.info(f"[HybridBrowserToolkit] Initializing with api_task_id: {api_task_id}")
         self.api_task_id = api_task_id
         logger.debug(f"[HybridBrowserToolkit] api_task_id set to: {self.api_task_id}")
-
+
+        # Store log_dir for use in clone()
+        self._log_dir = log_dir
+
         # Set default user_data_dir if not provided
         if user_data_dir is None:
             # Use browser port to determine profile directory
@@ -258,14 +262,15 @@ def __init__(
         else:
             logger.info(f"[HybridBrowserToolkit] Using provided user_data_dir: {user_data_dir}")
 
-        logger.debug(f"[HybridBrowserToolkit] Calling super().__init__ with session_id: {session_id}")
+        logger.info(f"[HybridBrowserToolkit] Calling super().__init__ with session_id: {session_id}, cdp_url: {cdp_url}")
         super().__init__(
             headless=headless,
             user_data_dir=user_data_dir,
             stealth=stealth,
             cache_dir=cache_dir,
             enabled_tools=enabled_tools,
             browser_log_to_file=browser_log_to_file,
+            log_dir=log_dir,
             session_id=session_id,
             default_start_url=default_start_url,
             default_timeout=default_timeout,
@@ -281,24 +286,28 @@ def __init__(
             cdp_keep_current_page=cdp_keep_current_page,
             full_visual_mode=full_visual_mode,
         )
-        logger.info(f"[HybridBrowserToolkit] Initialization complete for api_task_id: {self.api_task_id}")
+        logger.info(f"[HybridBrowserToolkit] Initialization complete for api_task_id: {self.api_task_id}, session_id: {self._session_id}")
 
     async def _ensure_ws_wrapper(self):
         """Ensure WebSocket wrapper is initialized using connection pool."""
-        logger.debug(f"[HybridBrowserToolkit] _ensure_ws_wrapper called for api_task_id: {getattr(self, 'api_task_id', 'NOT SET')}")
+        import traceback
         global websocket_connection_pool
 
         # Get session ID from config or use default
         session_id = self._ws_config.get("session_id", "default")
-        logger.debug(f"[HybridBrowserToolkit] Using session_id: {session_id}")
 
         # Log when connecting to browser
-        cdp_url = self._ws_config.get("cdp_url", f"http://localhost:{env('browser_port', '9222')}")
-        logger.info(f"[PROJECT BROWSER] Connecting to browser via CDP at {cdp_url}")
+        cdp_url = self._ws_config.get("cdpUrl", f"http://localhost:{env('browser_port', '9222')}")
+
+        # Log stack trace to see who's calling this
+        stack = traceback.extract_stack()
+        caller_info = f"{stack[-2].filename}:{stack[-2].lineno} in {stack[-2].name}"
+
+        logger.info(f"[TOOLKIT SESSION {session_id}] Connecting to browser via CDP at {cdp_url} (api_task_id: {getattr(self, 'api_task_id', 'UNKNOWN')}, toolkit_id: {id(self)}, caller: {caller_info})")
 
         # Get or create connection from pool
         self._ws_wrapper = await websocket_connection_pool.get_connection(session_id, self._ws_config)
-        logger.info(f"[HybridBrowserToolkit] WebSocket wrapper initialized for session: {session_id}")
+        logger.info(f"[TOOLKIT SESSION {session_id}] WebSocket wrapper initialized, CDP URL in config: {cdp_url}")
 
         # Additional health check
         if self._ws_wrapper.websocket is None:
@@ -314,21 +323,30 @@ def clone_for_new_session(self, new_session_id: str | None = None) -> "HybridBro
 
         # For cloned sessions, use the same user_data_dir to share login state
         # This allows multiple agents to use the same browser profile without conflicts
-        logger.info(f"Cloning session {new_session_id} with shared user_data_dir: {self._user_data_dir}")
+        parent_session_id = self._session_id if hasattr(self, '_session_id') else 'UNKNOWN'
+        logger.info(f"[TOOLKIT CLONE] Parent session {parent_session_id} -> New session {new_session_id}, shared user_data_dir: {self._user_data_dir}")
+
+        # Use the same CDP URL as parent
+        cdp_url = self.config_loader.get_browser_config().cdp_url
+        logger.info(f"[TOOLKIT CLONE] Parent session {parent_session_id} -> New session {new_session_id}, CDP URL: {cdp_url}")
+
+        # When cloning with cdp_keep_current_page=True, don't use default_start_url
+        cdp_keep_current = self.config_loader.get_browser_config().cdp_keep_current_page
+        start_url = None if cdp_keep_current else self._default_start_url
 
         # Use the same session_id to share the same browser instance
         # This ensures all clones use the same WebSocket connection and browser
-        return HybridBrowserToolkit(
+        cloned_toolkit = HybridBrowserToolkit(
             self.api_task_id,
             headless=self._headless,
             user_data_dir=self._user_data_dir,  # Use the same user_data_dir
             stealth=self._stealth,
             cache_dir=f"{self._cache_dir.rstrip('/')}/_clone_{new_session_id}/",
-            enabled_tools=self.enabled_tools.copy(),
+            enabled_tools=getattr(self, '_enabled_tools', None).copy() if getattr(self, '_enabled_tools', None) else None,
             browser_log_to_file=self._browser_log_to_file,
-            log_dir=self.config_loader.get_toolkit_config().log_dir,
+            log_dir=self._log_dir,  # Use the same log_dir as parent
             session_id=new_session_id,
-            default_start_url=self._default_start_url,
+            default_start_url=start_url,
             default_timeout=self._default_timeout,
             short_timeout=self._short_timeout,
             navigation_timeout=self._navigation_timeout,
@@ -338,31 +356,30 @@ def clone_for_new_session(self, new_session_id: str | None = None) -> "HybridBro
             dom_content_loaded_timeout=self._dom_content_loaded_timeout,
             viewport_limit=self._viewport_limit,
             connect_over_cdp=self.config_loader.get_browser_config().connect_over_cdp,
-            cdp_url=f"http://localhost:{env('browser_port', '9222')}",
-            cdp_keep_current_page=self.config_loader.get_browser_config().cdp_keep_current_page,
+            cdp_url=cdp_url,
+            cdp_keep_current_page=cdp_keep_current,
             full_visual_mode=self._full_visual_mode,
         )
 
+        return cloned_toolkit
+
     @classmethod
     def toolkit_name(cls) -> str:
         return "Browser Toolkit"
 
     async def close(self):
-        """Close the browser toolkit and release WebSocket connection."""
-        try:
-            # Close browser if needed
-            if self._ws_wrapper:
-                await super().browser_close()
-        except Exception as e:
-            logger.error(f"Error closing browser: {e}")
+        """Close the browser toolkit - but keep browser and connections open for reuse."""
+        logger.info(f"[HybridBrowserToolkit] close() called - browser and connections will remain open for reuse")
 
-        # Release connection from pool
-        session_id = self._ws_config.get("session_id", "default")
-        await websocket_connection_pool.close_connection(session_id)
-        logger.info(f"Released WebSocket connection for session {session_id}")
+        # DISABLED: Do not close browser - keep it open for reuse across tasks
+        # DISABLED: Do not release WebSocket connection - keep it in pool for reuse
+        # DISABLED: Do not release CDP browser port - use fixed port, no occupation management
 
     def __del__(self):
         """Cleanup when object is garbage collected."""
+        logger.info(f"[HybridBrowserToolkit] __del__ called for api_task_id: {getattr(self, 'api_task_id', 'UNKNOWN')} - browser will remain open")
+
+        # Log cleanup
         if hasattr(self, "_ws_wrapper") and self._ws_wrapper:
-            session_id = self._ws_config.get("session_id", "default")
+            session_id = self._ws_config.get("session_id", "default") if hasattr(self, "_ws_config") else "unknown"
             logger.debug(f"HybridBrowserToolkit for session {session_id} is being garbage collected")