Use touchX command with extra args to dispatch touch events to sim-se…

…rver (#543) This PR builds on top of changes from software-mansion-labs/simulator-server#173 where we add support for multitouch events to be dispatched to device (Android for the time being). This change restructures the multitouch handling code both on the front-end and in extension code such that we follow the new protocol. Now, instead of having separate command for multitouch, we allow for the dispatchTouch method to take an array of touches (typically one or two). Then, we send `touchDown` or similar command with additional coordinates after the x and y of the first pointer are added. The main change on the frontend was that we don't send leading point and anchor points but instead send two concrete touch points,. This removes coupling because sim-server and frontend code as otherwise we'd need to have code that calculates touch points on both sides (including logic related to rounding etc) This PR also updates some parts of the rpc code that I needed for one stage of the changes since we werent handling `null` value correctly (which apparently has typeof = 'object' in JS) ## Test plan This has been tested on custom sim-server build that includes software-mansion-labs/simulator-server#173 by opening an app with zoomable control on Android (I used photo gallery)
software-mansion · Sep 17, 2024 · 66ee860 · 66ee860
1 parent f126d51
commit 66ee860
Show file tree

Hide file tree

Showing 9 changed files with 171 additions and 103 deletions.
diff --git a/packages/vscode-extension/src/common/Project.ts b/packages/vscode-extension/src/common/Project.ts
@@ -80,6 +80,11 @@ export type InspectDataStackItem = {
   };
 };
 
+export type TouchPoint = {
+  xRatio: number;
+  yRatio: number;
+};
+
 export type InspectData = {
   stack: InspectDataStackItem[] | undefined;
   frame: {
@@ -124,14 +129,7 @@ export interface ProjectInterface {
 
   resetAppPermissions(permissionType: AppPermissionType): Promise<void>;
 
-  dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down"): Promise<void>;
-  dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ): Promise<void>;
+  dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down"): Promise<void>;
   dispatchKeyPress(keyCode: number, direction: "Up" | "Down"): Promise<void>;
   dispatchPaste(text: string): Promise<void>;
   inspectElementAt(

diff --git a/packages/vscode-extension/src/devices/DeviceBase.ts b/packages/vscode-extension/src/devices/DeviceBase.ts
@@ -1,7 +1,7 @@
 import { Disposable } from "vscode";
 import { Preview } from "./preview";
 import { BuildResult } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings } from "../common/Project";
+import { AppPermissionType, DeviceSettings, TouchPoint } from "../common/Project";
 import { DeviceInfo, DevicePlatform } from "../common/DeviceManager";
 import { tryAcquiringLock } from "../utilities/common";
 
@@ -44,18 +44,8 @@ export abstract class DeviceBase implements Disposable {
     this.preview?.dispose();
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.preview?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.preview?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.preview?.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

diff --git a/packages/vscode-extension/src/devices/preview.ts b/packages/vscode-extension/src/devices/preview.ts
@@ -4,6 +4,7 @@ import { exec, ChildProcess, lineReader } from "../utilities/subprocess";
 import { extensionContext } from "../utilities/extensionContext";
 import { Logger } from "../Logger";
 import { Platform } from "../utilities/platform";
+import { TouchPoint } from "../common/Project";
 
 export class Preview implements Disposable {
   private subprocess?: ChildProcess;
@@ -50,20 +51,9 @@ export class Preview implements Disposable {
     });
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.subprocess?.stdin?.write(`touch${type} ${xRatio} ${yRatio}\n`);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    // this.subprocess?.stdin?.write(
-    //   `multitouch${type} ${xRatio} ${yRatio} ${xAnchorRatio} ${yAnchorRatio}\n` // TODO set proper multitouch simserver command
-    // );
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    const touchesCoords = touches.map((pt) => `${pt.xRatio} ${pt.yRatio}`).join(" ");
+    this.subprocess?.stdin?.write(`touch${type} ${touchesCoords}\n`);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

diff --git a/packages/vscode-extension/src/panels/WebviewController.ts b/packages/vscode-extension/src/panels/WebviewController.ts
@@ -99,7 +99,7 @@ export class WebviewController implements Disposable {
   private setWebviewMessageListener(webview: Webview) {
     webview.onDidReceiveMessage(
       (message: WebviewEvent) => {
-        const isTouchEvent = message.command === "call" && message.method === "dispatchTouch";
+        const isTouchEvent = message.command === "call" && message.method === "dispatchTouches";
         if (!isTouchEvent) {
           Logger.log("Message from webview", message);
         }
@@ -132,7 +132,7 @@ export class WebviewController implements Disposable {
     const callableObject = this.callableObjects.get(object);
     if (callableObject && method in callableObject) {
       const argsWithCallbacks = args.map((arg: any) => {
-        if (typeof arg === "object" && "__callbackId" in arg) {
+        if (typeof arg === "object" && arg !== null && "__callbackId" in arg) {
           const callbackId = arg.__callbackId;
           let callback = this.idToCallback.get(callbackId)?.deref();
           if (!callback) {

diff --git a/packages/vscode-extension/src/project/deviceSession.ts b/packages/vscode-extension/src/project/deviceSession.ts
@@ -4,7 +4,13 @@ import { Devtools } from "./devtools";
 import { DeviceBase } from "../devices/DeviceBase";
 import { Logger } from "../Logger";
 import { BuildManager, BuildResult, DisposableBuild } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings, ReloadAction, StartupMessage } from "../common/Project";
+import {
+  AppPermissionType,
+  DeviceSettings,
+  ReloadAction,
+  StartupMessage,
+  TouchPoint,
+} from "../common/Project";
 import { DevicePlatform } from "../common/DeviceManager";
 import { AndroidEmulatorDevice } from "../devices/AndroidEmulatorDevice";
 import { getLaunchConfiguration } from "../utilities/launchConfiguration";
@@ -180,18 +186,8 @@ export class DeviceSession implements Disposable {
     return false;
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.device.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.device.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.device.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts
@@ -16,6 +16,7 @@ import {
   ProjectState,
   ReloadAction,
   StartupMessage,
+  TouchPoint,
   ZoomLevelType,
 } from "../common/Project";
 import { EventEmitter } from "stream";
@@ -336,18 +337,8 @@ export class Project
     }
   }
 
-  public async dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.deviceSession?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public async dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.deviceSession?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public async dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.deviceSession?.sendTouches(touches, type);
   }
 
   public async dispatchKeyPress(keyCode: number, direction: "Up" | "Down") {

diff --git a/packages/vscode-extension/src/webview/components/Preview.tsx b/packages/vscode-extension/src/webview/components/Preview.tsx
@@ -182,18 +182,28 @@ type Props = {
   onZoomChanged: (zoomLevel: ZoomLevelType) => void;
 };
 
-function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
-  interface TouchPoint {
-    x: number;
-    y: number;
-  }
+interface Point {
+  x: number;
+  y: number;
+}
+
+function calculateMirroredTouchPosition(touchPoint: Point, anchorPoint: Point) {
+  const { x: pointX, y: pointY } = touchPoint;
+  const { x: mirrorX, y: mirrorY } = anchorPoint;
+  const mirroredPointX = 2 * mirrorX - pointX;
+  const mirroredPointY = 2 * mirrorY - pointY;
+  const clampedX = clamp(mirroredPointX, 0, 1);
+  const clampedY = clamp(mirroredPointY, 0, 1);
+  return { x: clampedX, y: clampedY };
+}
 
+function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
   const wrapperDivRef = useRef<HTMLDivElement>(null);
   const [isPressing, setIsPressing] = useState(false);
   const [isMultiTouching, setIsMultiTouching] = useState(false);
   const [isPanning, setIsPanning] = useState(false);
-  const [touchPoint, setTouchPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
-  const [anchorPoint, setAnchorPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
+  const [touchPoint, setTouchPoint] = useState<Point>({ x: 0.5, y: 0.5 });
+  const [anchorPoint, setAnchorPoint] = useState<Point>({ x: 0.5, y: 0.5 });
   const previewRef = useRef<HTMLImageElement>(null);
   const [showPreviewRequested, setShowPreviewRequested] = useState(false);
 
@@ -250,25 +260,22 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     setAnchorPoint({ x: anchorX, y: anchorY });
   }
 
-  function getMirroredTouchPosition(mirrorPoint: TouchPoint) {
-    const { x: pointX, y: pointY } = touchPoint;
-    const { x: mirrorX, y: mirrorY } = mirrorPoint;
-    const mirroredPointX = 2 * mirrorX - pointX;
-    const mirroredPointY = 2 * mirrorY - pointY;
-    const clampedX = clamp(mirroredPointX, 0, 1);
-    const clampedY = clamp(mirroredPointY, 0, 1);
-    return { x: clampedX, y: clampedY };
-  }
-
   type MouseMove = "Move" | "Down" | "Up";
   function sendTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
     const { x, y } = getTouchPosition(event);
-    project.dispatchTouch(x, y, type);
+    project.dispatchTouches([{ xRatio: x, yRatio: y }], type);
   }
 
   function sendMultiTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
-    const { x, y } = getTouchPosition(event);
-    project.dispatchMultiTouch(x, y, anchorPoint.x, anchorPoint.y, type);
+    const pt = getTouchPosition(event);
+    const secondPt = calculateMirroredTouchPosition(pt, anchorPoint);
+    project.dispatchTouches(
+      [
+        { xRatio: pt.x, yRatio: pt.y },
+        { xRatio: secondPt.x, yRatio: secondPt.y },
+      ],
+      type
+    );
   }
 
   function onInspectorItemSelected(item: InspectDataStackItem) {
@@ -465,7 +472,7 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     device: device!,
   });
 
-  const mirroredTouchPosition = getMirroredTouchPosition(anchorPoint);
+  const mirroredTouchPosition = calculateMirroredTouchPosition(touchPoint, anchorPoint);
   const normalTouchMarkerSize = 33;
   const smallTouchMarkerSize = 9;
 

diff --git a/test-apps/expo-router/app/index.js b/test-apps/expo-router/app/index.js
@@ -1,24 +1,24 @@
-import React, { useState, useEffect, useRef, useContext } from "react";
-import { Button, TextInput, View, Text, useColorScheme } from "react-native";
-import { Link } from "expo-router";
-import { NiceButton } from "./components/NiceButton";
-import { UglyButton } from "./components/UglyButton";
-import Constants from "expo-constants";
+import React, { useState, useEffect, useRef, useContext } from 'react';
+import { Button, TextInput, View, Text, useColorScheme } from 'react-native';
+import { Link } from 'expo-router';
+import { NiceButton } from './components/NiceButton';
+import { UglyButton } from './components/UglyButton';
+import Constants from 'expo-constants';
 
 const obj = {
-  something: "lsdkjfhjdshf",
+  something: 'lsdkjfhjdshf',
   arrayOfThings: [
     {
       number: 1,
-      string: "sdjfh",
+      string: 'sdjfh',
       andObject: {
         prop1: 77,
         prop2: 2837,
       },
     },
     {
       number: 2,
-      string: "skdfh",
+      string: 'skdfh',
       andObject: {
         prop1: 919,
         prop2: 22,
@@ -32,7 +32,7 @@ function two(uu) {
   for (let i = 0; i < 10; i++) {
     b += i;
   }
-  console.log("P", uu.a + b);
+  console.log('P', uu.a + b);
 }
 
 function one() {
@@ -51,37 +51,37 @@ function Home() {
   return (
     <View
       ref={ref}
-      style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
-    >
+      style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}>
       <Link href="/details">Go to details</Link>
+      <Link href="/rotato">Go to rotato</Link>
       <Link href="/another">another</Link>
       <Link href="/location">test location</Link>
       <Link href="/another?id=100">/another?id=100</Link>
       <NiceButton
         onPress={() => {
           let a = 2;
-          console.log("Nice button pressed", obj);
+          console.log('Nice button pressed', obj);
           one();
           a++;
-          console.warn("Yollo");
-          console.warn("Yollo3");
+          console.warn('Yollo');
+          console.warn('Yollo3');
           // console.warn('Nice button pressed again');
           // console.log('WWW', window.__REACT_DEVTOOLS_PORT__);
         }}
       />
       <TextInput
         style={{
           height: 40,
-          borderColor: "gray",
+          borderColor: 'gray',
           borderWidth: 1,
-          width: "70%",
+          width: '70%',
         }}
       />
       <Button
         title="Throw error 1"
         a
         onPress={() => {
-          throw new Error("from button");
+          throw new Error('from button');
         }}
       />
       <UglyButton />
@@ -92,9 +92,9 @@ function Home() {
 
 let EntryPoint = Home;
 
-const storybookEnabled = Constants.expoConfig.extra.storybookEnabled === "true";
+const storybookEnabled = Constants.expoConfig.extra.storybookEnabled === 'true';
 if (storybookEnabled) {
-  const StorybookUI = require("../.storybook").default;
+  const StorybookUI = require('../.storybook').default;
   EntryPoint = () => {
     return (
       <View style={{ flex: 1 }}>