feat(playground): implement PlaygroundSDK for execution management

apps/android-playground/package.json

@@ -12,6 +12,8 @@
     "@ant-design/icons": "^5.3.1",
     "@midscene/android": "workspace:*",
     "@midscene/core": "workspace:*",
+    "@midscene/playground": "workspace:*",
+
     "@midscene/shared": "workspace:*",
     "@midscene/visualizer": "workspace:*",
     "@midscene/web": "workspace:*",
@@ -21,6 +23,7 @@
     "dayjs": "^1.11.11",
     "react": "18.3.1",
     "react-dom": "18.3.1",
+    "react-resizable-panels": "2.0.22",
     "socket.io-client": "4.8.1"
   },
   "devDependencies": {

apps/android-playground/src/App.tsx

@@ -1,5 +1,6 @@
 import './App.less';
 import type { DeviceAction } from '@midscene/core';
+import { PlaygroundSDK } from '@midscene/playground';
 import { SCRCPY_SERVER_PORT } from '@midscene/shared/constants';
 import { overrideAIConfig } from '@midscene/shared/env';
 import {
@@ -10,17 +11,12 @@ import {
   PromptInput,
   type ReplayScriptsInfo,
   allScriptsFromDump,
-  cancelTask,
-  getActionSpace,
-  getTaskProgress,
   globalThemeConfig,
-  overrideServerConfig,
-  requestPlaygroundServer,
   useEnvConfig,
   useServerValid,
 } from '@midscene/visualizer';
 import { Col, ConfigProvider, Form, Layout, Row, message } from 'antd';
-import { useCallback, useEffect, useRef, useState } from 'react';
+import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
 import { type Socket, io } from 'socket.io-client';
 import AdbDevice from './adb-device';
 import ScrcpyPlayer, { type ScrcpyRefMethods } from './scrcpy-player';
@@ -60,6 +56,13 @@ export default function App() {
   const serverValid = useServerValid(true);
   const [actionSpace, setActionSpace] = useState<DeviceAction<any>[]>([]);
 
+  // Initialize PlaygroundSDK only once
+  const playgroundSDK = useMemo(() => {
+    return new PlaygroundSDK({
+      type: 'remote-execution',
+    });
+  }, []);
+
   // Socket connection and device management
   const socketRef = useRef<Socket | null>(null);
   // Add a ref to ScrcpyPlayer
@@ -216,7 +219,7 @@ export default function App() {
       // set polling interval to 500ms
       pollIntervalRef.current = setInterval(async () => {
         try {
-          const data = await getTaskProgress(requestId);
+          const data = await playgroundSDK.getTaskProgress(requestId);
 
           if (data.tip) {
             setLoadingProgressText(data.tip);
@@ -226,7 +229,7 @@ export default function App() {
         }
       }, 500);
     },
-    [clearPollingInterval],
+    [clearPollingInterval, playgroundSDK],
   );
 
   // clean up the polling when the component unmounts
@@ -270,15 +273,15 @@ export default function App() {
   // Override AI configuration
   useEffect(() => {
     overrideAIConfig(config);
-    overrideServerConfig(config);
-  }, [config]);
+    playgroundSDK.overrideConfig(config);
+  }, [config, playgroundSDK]);
 
   // Initialize actionSpace
   useEffect(() => {
     const loadActionSpace = async () => {
       try {
         if (selectedDeviceId) {
-          const space = await getActionSpace(selectedDeviceId);
+          const space = await playgroundSDK.getActionSpace(selectedDeviceId);
           setActionSpace(space || []);
         } else {
           setActionSpace([]);
@@ -292,7 +295,7 @@ export default function App() {
     if (serverValid) {
       loadActionSpace();
     }
-  }, [serverValid, selectedDeviceId]);
+  }, [serverValid, selectedDeviceId, playgroundSDK]);
 
   // handle run button click
   const handleRun = useCallback(async () => {
@@ -330,7 +333,9 @@ export default function App() {
         typeof action.paramSchema === 'object' &&
         'shape' in action.paramSchema
       ) {
-        const shape = (action.paramSchema as any).shape || {};
+        const shape =
+          (action.paramSchema as { shape: Record<string, unknown> }).shape ||
+          {};
         const shapeKeys = Object.keys(shape);
         return shapeKeys.length > 0; // Only need structured params if there are actual fields
       }
@@ -349,7 +354,9 @@ export default function App() {
           typeof action.paramSchema === 'object' &&
           'shape' in action.paramSchema
         ) {
-          const shape = (action.paramSchema as any).shape || {};
+          const shape =
+            (action.paramSchema as { shape: Record<string, unknown> }).shape ||
+            {};
 
           // Check if any field is required (not optional)
           // For this we need to implement the unwrapZodType logic here or import it
@@ -389,16 +396,19 @@ export default function App() {
     startPollingProgress(thisRunningId);
 
     try {
-      const res = await requestPlaygroundServer(
-        selectedDeviceId,
+      const res = (await playgroundSDK.executeAction(
         type,
-        prompt,
         {
+          type,
+          prompt,
+          params,
+        },
+        {
+          context: selectedDeviceId,
           requestId: thisRunningId,
           deepThink,
-          params, // Pass params to server
         },
-      );
+      )) as PlaygroundResult;
 
       // stop polling
       clearPollingInterval();
@@ -449,10 +459,10 @@ export default function App() {
     setLoading(false);
     resetResult();
     if (currentRequestIdRef.current) {
-      await cancelTask(currentRequestIdRef.current);
+      await playgroundSDK.cancelTask(currentRequestIdRef.current);
     }
     messageApi.info('Operation stopped');
-  }, [messageApi, clearPollingInterval]);
+  }, [messageApi, clearPollingInterval, playgroundSDK]);
 
   return (
     <ConfigProvider theme={globalThemeConfig()}>

apps/chrome-extension/package.json

@@ -12,12 +12,13 @@
   "dependencies": {
     "@ant-design/icons": "^5.3.1",
     "@midscene/core": "workspace:*",
+    "@midscene/playground": "workspace:*",
     "@midscene/recorder": "workspace:*",
     "@midscene/report": "workspace:*",
     "@midscene/shared": "workspace:*",
     "@midscene/visualizer": "workspace:*",
     "@midscene/web": "workspace:*",
-    "@midscene/playground": "workspace:*",
+
     "@types/file-saver": "2.0.7",
     "antd": "^5.21.6",
     "canvas-confetti": "1.9.3",

apps/chrome-extension/src/components/playground.tsx

@@ -4,7 +4,7 @@ import Icon, {
   ArrowDownOutlined,
 } from '@ant-design/icons';
 import type { DeviceAction, UIContext } from '@midscene/core';
-import { noReplayAPIs } from '@midscene/playground';
+import { PlaygroundSDK, noReplayAPIs } from '@midscene/playground';
 import {
   ContextPreview,
   type PlaygroundResult,
@@ -19,12 +19,6 @@ import { useCallback, useEffect, useRef, useState } from 'react';
 import { EnvConfigReminder } from '.';
 import PlaygroundIcon from '../icons/playground.svg?react';
 import { getExtensionVersion } from '../utils/chrome';
-import {
-  createDisplayContent,
-  executeAction,
-  formatErrorMessage,
-  validateStructuredParams,
-} from '../utils/playground-helpers';
 import {
   clearStoredMessages,
   getMsgsFromStorage,
@@ -36,6 +30,9 @@ import { ChromeExtensionProxyPage } from '@midscene/web/chrome-extension';
 
 declare const __SDK_VERSION__: string;
 
+// Constants
+const DEFAULT_AGENT_ERROR = 'Agent is required for local execution mode';
+
 const { Text } = Typography;
 
 export interface PlaygroundProps {
@@ -133,6 +130,31 @@ export function BrowserExtensionPlayground({
     (state) => state.forceSameTabNavigation,
   );
 
+  // Initialize SDK for Local Execution type (stable instance to avoid re-creating per render)
+  const sdkRef = useRef<PlaygroundSDK | null>(null);
+  const currentAgent = useRef<any>(null);
+
+  // Initialize SDK with agent when needed - optimized to cache based on agent
+  const initializeSDK = useCallback(
+    (agent?: any) => {
+      const targetAgent = agent || getAgent();
+      if (!targetAgent) {
+        throw new Error(DEFAULT_AGENT_ERROR);
+      }
+
+      // Only recreate if agent has changed or SDK doesn't exist
+      if (!sdkRef.current || currentAgent.current !== targetAgent) {
+        sdkRef.current = new PlaygroundSDK({
+          type: 'local-execution',
+          agent: targetAgent,
+        });
+        currentAgent.current = targetAgent;
+      }
+      return sdkRef.current;
+    },
+    [getAgent],
+  );
+
   // References
   const runResultRef = useRef<HTMLHeadingElement>(null);
   const currentAgentRef = useRef<any>(null);
@@ -185,7 +207,9 @@ export function BrowserExtensionPlayground({
     const loadActionSpace = async () => {
       try {
         const page = new ChromeExtensionProxyPage(forceSameTabNavigation);
-        const space = await page.actionSpace();
+        // Use a temporary agent for actionSpace loading - this doesn't need to match execution agent
+        const sdk = initializeSDK();
+        const space = await sdk.getActionSpace(page);
 
         setActionSpace(space || []);
       } catch (error) {
@@ -194,7 +218,7 @@ export function BrowserExtensionPlayground({
     };
 
     loadActionSpace();
-  }, [getAgent, forceSameTabNavigation, config]);
+  }, [forceSameTabNavigation, config]);
 
   // store light messages to localStorage (big result data is stored separately)
   useEffect(() => {
@@ -282,7 +306,10 @@ export function BrowserExtensionPlayground({
     const needsStructuredParams = !!action?.paramSchema;
 
     if (needsStructuredParams) {
-      const validation = validateStructuredParams(value, action);
+      // Get the agent that will be used for execution
+      const agent = getAgent(forceSameTabNavigation);
+      const sdk = initializeSDK(agent);
+      const validation = sdk.validateStructuredParams(value, action);
       if (!validation.valid) {
         message.error(validation.errorMessage || 'Validation failed');
         return;
@@ -294,8 +321,11 @@ export function BrowserExtensionPlayground({
 
     const startTime = Date.now();
 
+    const activeAgent = getAgent(forceSameTabNavigation);
+
     // Create display content for user input - dynamically from actionSpace
-    const displayContent = createDisplayContent(
+    const sdk = initializeSDK(activeAgent);
+    const displayContent = sdk.createDisplayContent(
       value,
       needsStructuredParams,
       action,
@@ -312,7 +342,6 @@ export function BrowserExtensionPlayground({
     setLoading(true);
 
     const result: PlaygroundResult = { ...blankResult };
-    const activeAgent = getAgent(forceSameTabNavigation);
     const thisRunningId = Date.now();
     const actionType = value.type;
 
@@ -350,20 +379,16 @@ export function BrowserExtensionPlayground({
         setInfoList((prev) => [...prev, progressItem]);
       };
 
-      // Execute the action using the new helper function
-      result.result = await executeAction(
-        activeAgent,
-        actionType,
-        actionSpace,
-        value,
-        {
-          deepThink,
-          screenshotIncluded,
-          domIncluded,
-        },
-      );
+      // Execute the action using the SDK with the same agent
+      const sdk = initializeSDK(activeAgent);
+      result.result = await sdk.executeAction(actionType, value, {
+        deepThink,
+        screenshotIncluded,
+        domIncluded,
+      });
     } catch (e: any) {
-      result.error = formatErrorMessage(e);
+      const sdk = initializeSDK(activeAgent);
+      result.error = sdk.formatErrorMessage(e);
       console.error(e);
     }
 
@@ -383,14 +408,6 @@ export function BrowserExtensionPlayground({
       console.error(e);
     }
 
-    try {
-      console.log('destroy agent.page', activeAgent?.page);
-      await activeAgent?.page?.destroy();
-      console.log('destroy agent.page done', activeAgent?.page);
-    } catch (e) {
-      console.error(e);
-    }
-
     currentAgentRef.current = null;
     setLoading(false);
 
@@ -449,7 +466,13 @@ export function BrowserExtensionPlayground({
 
     // Reset hasNewMessage for future runs
 
-    console.log(`time taken: ${Date.now() - startTime}ms`);
+    console.log('Chrome Extension playground execution completed:', {
+      timeTaken: `${Date.now() - startTime}ms`,
+      actionType,
+      hasResult: !!result.result,
+      hasDump: !!result.dump,
+      hasError: !!result.error,
+    });
   }, [
     form,
     getAgent,
@@ -705,3 +728,5 @@ export function BrowserExtensionPlayground({
     </div>
   );
 }
+
+export default BrowserExtensionPlayground;