fix tab focus and check uncheck

Author: Dingway98

src/agent/index.ts

@@ -332,6 +332,29 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
     if (!this.context) {
       throw new HyperagentError("No context found");
     }
+
+    // Poll context for new pages to catch any that opened since the last check
+    // This handles race conditions where the 'page' event might not have fired yet
+    // or where we missed it during a heavy operation.
+    const pages = this.context.pages();
+    if (pages.length > 0) {
+      const lastPage = pages[pages.length - 1];
+      // If the last page is different and not closed, switch to it
+      // We prefer the newest page as it's likely the result of the user's last action
+      if (
+        lastPage &&
+        !lastPage.isClosed() &&
+        lastPage !== this._currentPage
+      ) {
+        if (this.debug) {
+          console.log(
+            `[HyperAgent] Polling detected new page, switching focus: ${lastPage.url()}`
+          );
+        }
+        this._currentPage = lastPage;
+      }
+    }
+
     if (!this.currentPage || this.currentPage.isClosed()) {
       this._currentPage = await this.context.newPage();
 
@@ -406,6 +429,7 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
         mcpClient: this.mcpClient,
         variables: this._variables,
         cdpActions: this.cdpActionsEnabled,
+        activePage: () => this.getCurrentPage(),
       },
       taskState,
       mergedParams
@@ -458,6 +482,7 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
           mcpClient: this.mcpClient,
           variables: this._variables,
           cdpActions: this.cdpActionsEnabled,
+          activePage: () => this.getCurrentPage(),
         },
         taskState,
         mergedParams
@@ -1062,12 +1087,18 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
 
   private setupHyperPage(page: Page): HyperPage {
     const hyperPage = page as HyperPage;
-    hyperPage.ai = (task: string, params?: TaskParams) =>
-      this.executeTask(task, params, page);
-    hyperPage.aiAction = (instruction: string, params?: TaskParams) =>
-      this.executeSingleAction(instruction, page, params);
-    hyperPage.aiAsync = (task: string, params?: TaskParams) =>
-      this.executeTaskAsync(task, params, page);
+    hyperPage.ai = async (task: string, params?: TaskParams) => {
+      const activePage = await this.getCurrentPage();
+      return this.executeTask(task, params, activePage);
+    };
+    hyperPage.aiAction = async (instruction: string, params?: TaskParams) => {
+      const activePage = await this.getCurrentPage();
+      return this.executeSingleAction(instruction, activePage, params);
+    };
+    hyperPage.aiAsync = async (task: string, params?: TaskParams) => {
+      const activePage = await this.getCurrentPage();
+      return this.executeTaskAsync(task, params, activePage);
+    };
     hyperPage.extract = async (task, outputSchema, params) => {
       if (!task && !outputSchema) {
         throw new HyperagentError(
@@ -1080,11 +1111,14 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
         ...params,
         outputSchema,
       };
+      
+      const activePage = await this.getCurrentPage();
+      
       if (task) {
         const res = await this.executeTask(
           `You have to perform an extraction on the current page. You have to perform the extraction according to the task: ${task}. Make sure your final response only contains the extracted content`,
           taskParams,
-          page
+          activePage
         );
         if (outputSchema) {
           const outputText = res.output;
@@ -1106,7 +1140,7 @@ export class HyperAgent<T extends BrowserProviders = "Local"> {
         const res = await this.executeTask(
           "You have to perform a data extraction on the current page. Make sure your final response only contains the extracted content",
           taskParams,
-          page
+          activePage
         );
         if (typeof res.output !== "string" || res.output === "") {
           throw new Error(

src/agent/shared/find-element.ts

@@ -10,6 +10,7 @@ import type { ExamineDomResult } from "../examine-dom/types";
 import type { AccessibilityNode } from "@/context-providers/a11y-dom/types";
 import { captureDOMState } from "./dom-capture";
 import type { A11yDOMState } from "@/context-providers/a11y-dom/types";
+import { waitForSettledDOM } from "@/utils/waitForSettledDOM";
 
 export interface FindElementOptions {
   /**
@@ -70,25 +71,23 @@ export async function findElementWithInstruction(
 
   // Retry loop with DOM refresh (matches aiAction's findElementWithRetry pattern)
   for (let attempt = 0; attempt < maxRetries; attempt++) {
-    
     if (debug) {
       if (attempt === 0) {
         console.log(`[findElement] Starting attempt ${attempt + 1}`);
       } else {
-        console.log(
-          `[findElement] Retry ${attempt + 1}/${maxRetries}`
-        );
+        console.log(`[findElement] Retry ${attempt + 1}/${maxRetries}`);
       }
     }
 
+    await waitForSettledDOM(page);
     // Fetch FRESH a11y tree using the robust shared utility
     // captureDOMState handles DOM settling and retries for bad snapshots internally for this *single* capture attempt
     // We still need our outer loop for retrying the *finding* logic (e.g. if the LLM can't find the element)
     const domState = await captureDOMState(page, {
       debug,
       // Don't retry capture inside captureDOMState too aggressively since we have an outer loop here
       // But we do want it to handle transient CDP errors
-      maxRetries: 2 
+      maxRetries: 2,
     });
 
     if (debug) {

src/agent/tools/agent.ts

@@ -157,7 +157,10 @@ const runAction = async (
   };
 
   if (ctx.cdpActions) {
-    const { cdpClient, frameContextManager } = await initializeRuntimeContext(page, ctx.debug);
+    const { cdpClient, frameContextManager } = await initializeRuntimeContext(
+      page,
+      ctx.debug
+    );
     actionCtx.cdp = {
       resolveElement,
       dispatchCDPAction,
@@ -233,27 +236,32 @@ export const runAgentTask = async (
   ];
 
   let output = "";
-  const page = taskState.startingPage;
+  let page = taskState.startingPage;
   const useDomCache = params?.useDomCache === true;
   const enableDomStreaming = params?.enableDomStreaming === true;
-  
+
   // Track schema validation errors across steps
   if (!ctx.schemaErrors) {
     ctx.schemaErrors = [];
   }
-  
+
   const navigationDirtyHandler = (): void => {
     markDomSnapshotDirty(page);
   };
-  page.on("framenavigated", navigationDirtyHandler);
-  page.on("framedetached", navigationDirtyHandler);
-  page.on("load", navigationDirtyHandler);
-
-  const cleanupDomListeners = (): void => {
-    page.off?.("framenavigated", navigationDirtyHandler);
-    page.off?.("framedetached", navigationDirtyHandler);
-    page.off?.("load", navigationDirtyHandler);
+
+  const setupDomListeners = (p: Page) => {
+    p.on("framenavigated", navigationDirtyHandler);
+    p.on("framedetached", navigationDirtyHandler);
+    p.on("load", navigationDirtyHandler);
+  };
+
+  const cleanupDomListeners = (p: Page) => {
+    p.off?.("framenavigated", navigationDirtyHandler);
+    p.off?.("framedetached", navigationDirtyHandler);
+    p.off?.("load", navigationDirtyHandler);
   };
+
+  setupDomListeners(page);
   let currStep = 0;
   let consecutiveFailuresOrWaits = 0;
   const MAX_CONSECUTIVE_FAILURES_OR_WAITS = 5;
@@ -263,8 +271,23 @@ export const runAgentTask = async (
   try {
     // Initialize context at the start of the task
     await initializeRuntimeContext(page, ctx.debug);
-    
+
     while (true) {
+      // Check for page context switch
+      if (ctx.activePage) {
+        const newPage = await ctx.activePage();
+        if (newPage && newPage !== page) {
+          if (ctx.debug) {
+            console.log(`[Agent] Switching active page context to ${newPage.url()}`);
+          }
+          cleanupDomListeners(page);
+          page = newPage;
+          setupDomListeners(page);
+          await initializeRuntimeContext(page, ctx.debug);
+          markDomSnapshotDirty(page);
+        }
+      }
+
       // Status Checks
       const status: TaskStatus = taskState.status;
       if (status === TaskStatus.PAUSED) {
@@ -292,16 +315,19 @@ export const runAgentTask = async (
       const domChunks: string | null = null;
       try {
         const domFetchStart = performance.now();
-        
+
+        await waitForSettledDOM(page);
         domState = await captureDOMState(page, {
           useCache: useDomCache,
           debug: ctx.debug,
           enableVisualMode: params?.enableVisualMode ?? false,
           debugStepDir: ctx.debug ? debugStepDir : undefined,
           enableStreaming: enableDomStreaming,
-          onFrameChunk: enableDomStreaming ? () => {
-            // captureDOMState handles aggregation
-          } : undefined
+          onFrameChunk: enableDomStreaming
+            ? () => {
+                // captureDOMState handles aggregation
+              }
+            : undefined,
         });
 
         const domDuration = performance.now() - domFetchStart;
@@ -366,14 +392,14 @@ export const runAgentTask = async (
         trimmedScreenshot,
         Object.values(ctx.variables)
       );
-      
+
       // Append accumulated schema errors from previous steps
       if (ctx.schemaErrors && ctx.schemaErrors.length > 0) {
         const errorSummary = ctx.schemaErrors
           .slice(-3) // Only keep last 3 errors to avoid context bloat
-          .map(err => `Step ${err.stepIndex}: ${err.error}`)
-          .join('\n');
-        
+          .map((err) => `Step ${err.stepIndex}: ${err.error}`)
+          .join("\n");
+
         msgs = [
           ...msgs,
           {
@@ -395,7 +421,7 @@ export const runAgentTask = async (
       const agentOutput = await (async () => {
         const maxAttempts = 3;
         let currentMsgs = msgs;
-        
+
         for (let attempt = 0; attempt < maxAttempts; attempt++) {
           const structuredResult = await retry({
             func: () =>
@@ -431,7 +457,7 @@ export const runAgentTask = async (
 
           const providerId = ctx.llm?.getProviderId?.() ?? "unknown-provider";
           const modelId = ctx.llm?.getModelId?.() ?? "unknown-model";
-          
+
           // Try to get detailed Zod validation error
           let validationError = "Unknown validation error";
           if (structuredResult.rawText) {
@@ -446,27 +472,28 @@ export const runAgentTask = async (
               }
             }
           }
-          
+
           console.error(
             `[LLM][StructuredOutput] Failed to parse response from ${providerId} (${modelId}). Raw response: ${
               structuredResult.rawText?.trim() || "<empty>"
             } (attempt ${attempt + 1}/${maxAttempts})`
           );
-          
+
           // Store error for cross-step learning
           ctx.schemaErrors?.push({
             stepIndex: currStep,
             error: validationError,
             rawResponse: structuredResult.rawText || "",
           });
-          
+
           // Append error feedback for next retry
           if (attempt < maxAttempts - 1) {
             currentMsgs = [
               ...currentMsgs,
               {
                 role: "assistant",
-                content: structuredResult.rawText || "Failed to generate response",
+                content:
+                  structuredResult.rawText || "Failed to generate response",
               },
               {
                 role: "user",
@@ -629,7 +656,7 @@ export const runAgentTask = async (
 
     logPerf(ctx.debug, `[Perf][runAgentTask] Task ${taskId}`, taskStart);
   } finally {
-    cleanupDomListeners();
+    cleanupDomListeners(page);
   }
 
   const taskOutput: TaskOutput = {

src/agent/tools/types.ts

@@ -2,6 +2,7 @@ import { AgentActionDefinition } from "@/types/agent/actions/types";
 import { MCPClient } from "../mcp/client";
 import { HyperAgentLLM } from "@/llm/types";
 import { HyperVariable } from "@/types/agent/types";
+import { Page } from "playwright-core";
 
 export interface AgentCtx {
   mcpClient?: MCPClient;
@@ -17,4 +18,5 @@ export interface AgentCtx {
     error: string;
     rawResponse: string;
   }>;
+  activePage?: () => Promise<Page>;
 }