Use touchX command with extra args to dispatch touch events to sim-server

packages/vscode-extension/src/common/Project.ts

@@ -34,7 +34,7 @@
 
 // important: order of values in this enum matters
 export enum StartupMessage {
   InitializingDevice = "Initializing device",
   StartingPackager = "Starting packager",
   BootingDevice = "Booting device",
   Building = "Building",
@@ -80,6 +80,11 @@
   };
 };
 
+export type TouchPoint = {
+  xRatio: number;
+  yRatio: number;
+};
+
 export type InspectData = {
   stack: InspectDataStackItem[] | undefined;
   frame: {
@@ -124,14 +129,7 @@
 
   resetAppPermissions(permissionType: AppPermissionType): Promise<void>;
 
-  dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down"): Promise<void>;
-  dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ): Promise<void>;
+  dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down"): Promise<void>;
   dispatchKeyPress(keyCode: number, direction: "Up" | "Down"): Promise<void>;
   dispatchPaste(text: string): Promise<void>;
   inspectElementAt(

packages/vscode-extension/src/devices/DeviceBase.ts

@@ -1,7 +1,7 @@
 import { Disposable } from "vscode";
 import { Preview } from "./preview";
 import { BuildResult } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings } from "../common/Project";
+import { AppPermissionType, DeviceSettings, TouchPoint } from "../common/Project";
 import { DeviceInfo, DevicePlatform } from "../common/DeviceManager";
 import { tryAcquiringLock } from "../utilities/common";
 
@@ -44,18 +44,8 @@ export abstract class DeviceBase implements Disposable {
     this.preview?.dispose();
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.preview?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.preview?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.preview?.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

packages/vscode-extension/src/devices/preview.ts

@@ -4,6 +4,7 @@ import { exec, ChildProcess, lineReader } from "../utilities/subprocess";
 import { extensionContext } from "../utilities/extensionContext";
 import { Logger } from "../Logger";
 import { Platform } from "../utilities/platform";
+import { TouchPoint } from "../common/Project";
 
 export class Preview implements Disposable {
   private subprocess?: ChildProcess;
@@ -50,20 +51,9 @@ export class Preview implements Disposable {
     });
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.subprocess?.stdin?.write(`touch${type} ${xRatio} ${yRatio}\n`);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    // this.subprocess?.stdin?.write(
-    //   `multitouch${type} ${xRatio} ${yRatio} ${xAnchorRatio} ${yAnchorRatio}\n` // TODO set proper multitouch simserver command
-    // );
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    const fff = `touch${type} ` + touches.map((pt) => `${pt.xRatio} ${pt.yRatio}`).join(" ") + "\n";
+    this.subprocess?.stdin?.write(fff);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

packages/vscode-extension/src/panels/WebviewController.ts

@@ -99,7 +99,7 @@ export class WebviewController implements Disposable {
   private setWebviewMessageListener(webview: Webview) {
     webview.onDidReceiveMessage(
       (message: WebviewEvent) => {
-        const isTouchEvent = message.command === "call" && message.method === "dispatchTouch";
+        const isTouchEvent = message.command === "call" && message.method === "dispatchTouches";
         if (!isTouchEvent) {
           Logger.log("Message from webview", message);
         }
@@ -132,7 +132,7 @@ export class WebviewController implements Disposable {
     const callableObject = this.callableObjects.get(object);
     if (callableObject && method in callableObject) {
       const argsWithCallbacks = args.map((arg: any) => {
-        if (typeof arg === "object" && "__callbackId" in arg) {
+        if (typeof arg === "object" && arg !== null && "__callbackId" in arg) {
           const callbackId = arg.__callbackId;
           let callback = this.idToCallback.get(callbackId)?.deref();
           if (!callback) {

packages/vscode-extension/src/project/deviceSession.ts

@@ -4,7 +4,13 @@ import { Devtools } from "./devtools";
 import { DeviceBase } from "../devices/DeviceBase";
 import { Logger } from "../Logger";
 import { BuildManager, BuildResult, DisposableBuild } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings, ReloadAction, StartupMessage } from "../common/Project";
+import {
+  AppPermissionType,
+  DeviceSettings,
+  ReloadAction,
+  StartupMessage,
+  TouchPoint,
+} from "../common/Project";
 import { DevicePlatform } from "../common/DeviceManager";
 import { AndroidEmulatorDevice } from "../devices/AndroidEmulatorDevice";
 import { getLaunchConfiguration } from "../utilities/launchConfiguration";
@@ -180,18 +186,8 @@ export class DeviceSession implements Disposable {
     return false;
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.device.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.device.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.device.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {

packages/vscode-extension/src/project/project.ts

@@ -16,6 +16,7 @@ import {
   ProjectState,
   ReloadAction,
   StartupMessage,
+  TouchPoint,
   ZoomLevelType,
 } from "../common/Project";
 import { EventEmitter } from "stream";
@@ -336,18 +337,8 @@ export class Project
     }
   }
 
-  public async dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.deviceSession?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public async dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.deviceSession?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public async dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.deviceSession?.sendTouches(touches, type);
   }
 
   public async dispatchKeyPress(keyCode: number, direction: "Up" | "Down") {

packages/vscode-extension/src/webview/components/Preview.tsx

@@ -182,18 +182,28 @@ type Props = {
   onZoomChanged: (zoomLevel: ZoomLevelType) => void;
 };
 
-function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
-  interface TouchPoint {
-    x: number;
-    y: number;
-  }
+interface Point {
+  x: number;
+  y: number;
+}
+
+function calculateMirroredTouchPosition(touchPoint: Point, anchorPoint: Point) {
+  const { x: pointX, y: pointY } = touchPoint;
+  const { x: mirrorX, y: mirrorY } = anchorPoint;
+  const mirroredPointX = 2 * mirrorX - pointX;
+  const mirroredPointY = 2 * mirrorY - pointY;
+  const clampedX = clamp(mirroredPointX, 0, 1);
+  const clampedY = clamp(mirroredPointY, 0, 1);
+  return { x: clampedX, y: clampedY };
+}
 
+function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
   const wrapperDivRef = useRef<HTMLDivElement>(null);
   const [isPressing, setIsPressing] = useState(false);
   const [isMultiTouching, setIsMultiTouching] = useState(false);
   const [isPanning, setIsPanning] = useState(false);
-  const [touchPoint, setTouchPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
-  const [anchorPoint, setAnchorPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
+  const [touchPoint, setTouchPoint] = useState<Point>({ x: 0.5, y: 0.5 });
+  const [anchorPoint, setAnchorPoint] = useState<Point>({ x: 0.5, y: 0.5 });
   const previewRef = useRef<HTMLImageElement>(null);
   const [showPreviewRequested, setShowPreviewRequested] = useState(false);
 
@@ -250,25 +260,22 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     setAnchorPoint({ x: anchorX, y: anchorY });
   }
 
-  function getMirroredTouchPosition(mirrorPoint: TouchPoint) {
-    const { x: pointX, y: pointY } = touchPoint;
-    const { x: mirrorX, y: mirrorY } = mirrorPoint;
-    const mirroredPointX = 2 * mirrorX - pointX;
-    const mirroredPointY = 2 * mirrorY - pointY;
-    const clampedX = clamp(mirroredPointX, 0, 1);
-    const clampedY = clamp(mirroredPointY, 0, 1);
-    return { x: clampedX, y: clampedY };
-  }
-
   type MouseMove = "Move" | "Down" | "Up";
   function sendTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
     const { x, y } = getTouchPosition(event);
-    project.dispatchTouch(x, y, type);
+    project.dispatchTouches([{ xRatio: x, yRatio: y }], type);
   }
 
   function sendMultiTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
-    const { x, y } = getTouchPosition(event);
-    project.dispatchMultiTouch(x, y, anchorPoint.x, anchorPoint.y, type);
+    const pt = getTouchPosition(event);
+    const secondPt = calculateMirroredTouchPosition(pt, anchorPoint);
+    project.dispatchTouches(
+      [
+        { xRatio: pt.x, yRatio: pt.y },
+        { xRatio: secondPt.x, yRatio: secondPt.y },
+      ],
+      type
+    );
   }
 
   function onInspectorItemSelected(item: InspectDataStackItem) {
@@ -465,7 +472,7 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     device: device!,
   });
 
-  const mirroredTouchPosition = getMirroredTouchPosition(anchorPoint);
+  const mirroredTouchPosition = calculateMirroredTouchPosition(touchPoint, anchorPoint);
   const normalTouchMarkerSize = 33;
   const smallTouchMarkerSize = 9;