From 66ee8604981aa08c5b7269fc1fe09d81e806db74 Mon Sep 17 00:00:00 2001
From: Krzysztof Magiera <krzys@swmansion.com>
Date: Tue, 17 Sep 2024 14:51:47 +0200
Subject: [PATCH] Use touchX command with extra args to dispatch touch events
 to sim-server (#543)

This PR builds on top of changes from
https://github.com/software-mansion-labs/simulator-server/pull/173 where
we add support for multitouch events to be dispatched to device (Android
for the time being).

This change restructures the multitouch handling code both on the
front-end and in extension code such that we follow the new protocol.
Now, instead of having separate command for multitouch, we allow for the
dispatchTouch method to take an array of touches (typically one or two).
Then, we send `touchDown` or similar command with additional coordinates
after the x and y of the first pointer are added.

The main change on the frontend was that we don't send leading point and
anchor points but instead send two concrete touch points,. This removes
coupling because sim-server and frontend code as otherwise we'd need to
have code that calculates touch points on both sides (including logic
related to rounding etc)

This PR also updates some parts of the rpc code that I needed for one
stage of the changes since we werent handling `null` value correctly
(which apparently has typeof = 'object' in JS)

## Test plan

This has been tested on custom sim-server build that includes
https://github.com/software-mansion-labs/simulator-server/pull/173 by
opening an app with zoomable control on Android (I used photo gallery)
---
 .../vscode-extension/src/common/Project.ts    | 14 ++-
 .../src/devices/DeviceBase.ts                 | 16 +---
 .../vscode-extension/src/devices/preview.ts   | 18 +---
 .../src/panels/WebviewController.ts           |  4 +-
 .../src/project/deviceSession.ts              | 22 ++---
 .../vscode-extension/src/project/project.ts   | 15 +--
 .../src/webview/components/Preview.tsx        | 49 ++++++----
 test-apps/expo-router/app/index.js            | 40 ++++----
 test-apps/expo-router/app/rotato.js           | 96 +++++++++++++++++++
 9 files changed, 171 insertions(+), 103 deletions(-)
 create mode 100644 test-apps/expo-router/app/rotato.js

diff --git a/packages/vscode-extension/src/common/Project.ts b/packages/vscode-extension/src/common/Project.ts
index 8f197c9c9..61b2243c1 100644
--- a/packages/vscode-extension/src/common/Project.ts
+++ b/packages/vscode-extension/src/common/Project.ts
@@ -80,6 +80,11 @@ export type InspectDataStackItem = {
   };
 };
 
+export type TouchPoint = {
+  xRatio: number;
+  yRatio: number;
+};
+
 export type InspectData = {
   stack: InspectDataStackItem[] | undefined;
   frame: {
@@ -124,14 +129,7 @@ export interface ProjectInterface {
 
   resetAppPermissions(permissionType: AppPermissionType): Promise<void>;
 
-  dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down"): Promise<void>;
-  dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ): Promise<void>;
+  dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down"): Promise<void>;
   dispatchKeyPress(keyCode: number, direction: "Up" | "Down"): Promise<void>;
   dispatchPaste(text: string): Promise<void>;
   inspectElementAt(
diff --git a/packages/vscode-extension/src/devices/DeviceBase.ts b/packages/vscode-extension/src/devices/DeviceBase.ts
index 464a78682..a58973d73 100644
--- a/packages/vscode-extension/src/devices/DeviceBase.ts
+++ b/packages/vscode-extension/src/devices/DeviceBase.ts
@@ -1,7 +1,7 @@
 import { Disposable } from "vscode";
 import { Preview } from "./preview";
 import { BuildResult } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings } from "../common/Project";
+import { AppPermissionType, DeviceSettings, TouchPoint } from "../common/Project";
 import { DeviceInfo, DevicePlatform } from "../common/DeviceManager";
 import { tryAcquiringLock } from "../utilities/common";
 
@@ -44,18 +44,8 @@ export abstract class DeviceBase implements Disposable {
     this.preview?.dispose();
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.preview?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.preview?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.preview?.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {
diff --git a/packages/vscode-extension/src/devices/preview.ts b/packages/vscode-extension/src/devices/preview.ts
index bfa4ced17..845912093 100644
--- a/packages/vscode-extension/src/devices/preview.ts
+++ b/packages/vscode-extension/src/devices/preview.ts
@@ -4,6 +4,7 @@ import { exec, ChildProcess, lineReader } from "../utilities/subprocess";
 import { extensionContext } from "../utilities/extensionContext";
 import { Logger } from "../Logger";
 import { Platform } from "../utilities/platform";
+import { TouchPoint } from "../common/Project";
 
 export class Preview implements Disposable {
   private subprocess?: ChildProcess;
@@ -50,20 +51,9 @@ export class Preview implements Disposable {
     });
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.subprocess?.stdin?.write(`touch${type} ${xRatio} ${yRatio}\n`);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    // this.subprocess?.stdin?.write(
-    //   `multitouch${type} ${xRatio} ${yRatio} ${xAnchorRatio} ${yAnchorRatio}\n` // TODO set proper multitouch simserver command
-    // );
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    const touchesCoords = touches.map((pt) => `${pt.xRatio} ${pt.yRatio}`).join(" ");
+    this.subprocess?.stdin?.write(`touch${type} ${touchesCoords}\n`);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {
diff --git a/packages/vscode-extension/src/panels/WebviewController.ts b/packages/vscode-extension/src/panels/WebviewController.ts
index 3b15e5b2a..b05bc6a35 100644
--- a/packages/vscode-extension/src/panels/WebviewController.ts
+++ b/packages/vscode-extension/src/panels/WebviewController.ts
@@ -99,7 +99,7 @@ export class WebviewController implements Disposable {
   private setWebviewMessageListener(webview: Webview) {
     webview.onDidReceiveMessage(
       (message: WebviewEvent) => {
-        const isTouchEvent = message.command === "call" && message.method === "dispatchTouch";
+        const isTouchEvent = message.command === "call" && message.method === "dispatchTouches";
         if (!isTouchEvent) {
           Logger.log("Message from webview", message);
         }
@@ -132,7 +132,7 @@ export class WebviewController implements Disposable {
     const callableObject = this.callableObjects.get(object);
     if (callableObject && method in callableObject) {
       const argsWithCallbacks = args.map((arg: any) => {
-        if (typeof arg === "object" && "__callbackId" in arg) {
+        if (typeof arg === "object" && arg !== null && "__callbackId" in arg) {
           const callbackId = arg.__callbackId;
           let callback = this.idToCallback.get(callbackId)?.deref();
           if (!callback) {
diff --git a/packages/vscode-extension/src/project/deviceSession.ts b/packages/vscode-extension/src/project/deviceSession.ts
index b004755d5..beee99fa1 100644
--- a/packages/vscode-extension/src/project/deviceSession.ts
+++ b/packages/vscode-extension/src/project/deviceSession.ts
@@ -4,7 +4,13 @@ import { Devtools } from "./devtools";
 import { DeviceBase } from "../devices/DeviceBase";
 import { Logger } from "../Logger";
 import { BuildManager, BuildResult, DisposableBuild } from "../builders/BuildManager";
-import { AppPermissionType, DeviceSettings, ReloadAction, StartupMessage } from "../common/Project";
+import {
+  AppPermissionType,
+  DeviceSettings,
+  ReloadAction,
+  StartupMessage,
+  TouchPoint,
+} from "../common/Project";
 import { DevicePlatform } from "../common/DeviceManager";
 import { AndroidEmulatorDevice } from "../devices/AndroidEmulatorDevice";
 import { getLaunchConfiguration } from "../utilities/launchConfiguration";
@@ -180,18 +186,8 @@ export class DeviceSession implements Disposable {
     return false;
   }
 
-  public sendTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.device.sendTouch(xRatio, yRatio, type);
-  }
-
-  public sendMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.device.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public sendTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.device.sendTouches(touches, type);
   }
 
   public sendKey(keyCode: number, direction: "Up" | "Down") {
diff --git a/packages/vscode-extension/src/project/project.ts b/packages/vscode-extension/src/project/project.ts
index 262d14a38..280e2799d 100644
--- a/packages/vscode-extension/src/project/project.ts
+++ b/packages/vscode-extension/src/project/project.ts
@@ -16,6 +16,7 @@ import {
   ProjectState,
   ReloadAction,
   StartupMessage,
+  TouchPoint,
   ZoomLevelType,
 } from "../common/Project";
 import { EventEmitter } from "stream";
@@ -336,18 +337,8 @@ export class Project
     }
   }
 
-  public async dispatchTouch(xRatio: number, yRatio: number, type: "Up" | "Move" | "Down") {
-    this.deviceSession?.sendTouch(xRatio, yRatio, type);
-  }
-
-  public async dispatchMultiTouch(
-    xRatio: number,
-    yRatio: number,
-    xAnchorRatio: number,
-    yAnchorRatio: number,
-    type: "Up" | "Move" | "Down"
-  ) {
-    this.deviceSession?.sendMultiTouch(xRatio, yRatio, xAnchorRatio, yAnchorRatio, type);
+  public async dispatchTouches(touches: Array<TouchPoint>, type: "Up" | "Move" | "Down") {
+    this.deviceSession?.sendTouches(touches, type);
   }
 
   public async dispatchKeyPress(keyCode: number, direction: "Up" | "Down") {
diff --git a/packages/vscode-extension/src/webview/components/Preview.tsx b/packages/vscode-extension/src/webview/components/Preview.tsx
index e70f78028..ebc5e3361 100644
--- a/packages/vscode-extension/src/webview/components/Preview.tsx
+++ b/packages/vscode-extension/src/webview/components/Preview.tsx
@@ -182,18 +182,28 @@ type Props = {
   onZoomChanged: (zoomLevel: ZoomLevelType) => void;
 };
 
-function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
-  interface TouchPoint {
-    x: number;
-    y: number;
-  }
+interface Point {
+  x: number;
+  y: number;
+}
+
+function calculateMirroredTouchPosition(touchPoint: Point, anchorPoint: Point) {
+  const { x: pointX, y: pointY } = touchPoint;
+  const { x: mirrorX, y: mirrorY } = anchorPoint;
+  const mirroredPointX = 2 * mirrorX - pointX;
+  const mirroredPointY = 2 * mirrorY - pointY;
+  const clampedX = clamp(mirroredPointX, 0, 1);
+  const clampedY = clamp(mirroredPointY, 0, 1);
+  return { x: clampedX, y: clampedY };
+}
 
+function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Props) {
   const wrapperDivRef = useRef<HTMLDivElement>(null);
   const [isPressing, setIsPressing] = useState(false);
   const [isMultiTouching, setIsMultiTouching] = useState(false);
   const [isPanning, setIsPanning] = useState(false);
-  const [touchPoint, setTouchPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
-  const [anchorPoint, setAnchorPoint] = useState<TouchPoint>({ x: 0.5, y: 0.5 });
+  const [touchPoint, setTouchPoint] = useState<Point>({ x: 0.5, y: 0.5 });
+  const [anchorPoint, setAnchorPoint] = useState<Point>({ x: 0.5, y: 0.5 });
   const previewRef = useRef<HTMLImageElement>(null);
   const [showPreviewRequested, setShowPreviewRequested] = useState(false);
 
@@ -250,25 +260,22 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     setAnchorPoint({ x: anchorX, y: anchorY });
   }
 
-  function getMirroredTouchPosition(mirrorPoint: TouchPoint) {
-    const { x: pointX, y: pointY } = touchPoint;
-    const { x: mirrorX, y: mirrorY } = mirrorPoint;
-    const mirroredPointX = 2 * mirrorX - pointX;
-    const mirroredPointY = 2 * mirrorY - pointY;
-    const clampedX = clamp(mirroredPointX, 0, 1);
-    const clampedY = clamp(mirroredPointY, 0, 1);
-    return { x: clampedX, y: clampedY };
-  }
-
   type MouseMove = "Move" | "Down" | "Up";
   function sendTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
     const { x, y } = getTouchPosition(event);
-    project.dispatchTouch(x, y, type);
+    project.dispatchTouches([{ xRatio: x, yRatio: y }], type);
   }
 
   function sendMultiTouch(event: MouseEvent<HTMLDivElement>, type: MouseMove) {
-    const { x, y } = getTouchPosition(event);
-    project.dispatchMultiTouch(x, y, anchorPoint.x, anchorPoint.y, type);
+    const pt = getTouchPosition(event);
+    const secondPt = calculateMirroredTouchPosition(pt, anchorPoint);
+    project.dispatchTouches(
+      [
+        { xRatio: pt.x, yRatio: pt.y },
+        { xRatio: secondPt.x, yRatio: secondPt.y },
+      ],
+      type
+    );
   }
 
   function onInspectorItemSelected(item: InspectDataStackItem) {
@@ -465,7 +472,7 @@ function Preview({ isInspecting, setIsInspecting, zoomLevel, onZoomChanged }: Pr
     device: device!,
   });
 
-  const mirroredTouchPosition = getMirroredTouchPosition(anchorPoint);
+  const mirroredTouchPosition = calculateMirroredTouchPosition(touchPoint, anchorPoint);
   const normalTouchMarkerSize = 33;
   const smallTouchMarkerSize = 9;
 
diff --git a/test-apps/expo-router/app/index.js b/test-apps/expo-router/app/index.js
index 69feb0f34..fb0545096 100644
--- a/test-apps/expo-router/app/index.js
+++ b/test-apps/expo-router/app/index.js
@@ -1,16 +1,16 @@
-import React, { useState, useEffect, useRef, useContext } from "react";
-import { Button, TextInput, View, Text, useColorScheme } from "react-native";
-import { Link } from "expo-router";
-import { NiceButton } from "./components/NiceButton";
-import { UglyButton } from "./components/UglyButton";
-import Constants from "expo-constants";
+import React, { useState, useEffect, useRef, useContext } from 'react';
+import { Button, TextInput, View, Text, useColorScheme } from 'react-native';
+import { Link } from 'expo-router';
+import { NiceButton } from './components/NiceButton';
+import { UglyButton } from './components/UglyButton';
+import Constants from 'expo-constants';
 
 const obj = {
-  something: "lsdkjfhjdshf",
+  something: 'lsdkjfhjdshf',
   arrayOfThings: [
     {
       number: 1,
-      string: "sdjfh",
+      string: 'sdjfh',
       andObject: {
         prop1: 77,
         prop2: 2837,
@@ -18,7 +18,7 @@ const obj = {
     },
     {
       number: 2,
-      string: "skdfh",
+      string: 'skdfh',
       andObject: {
         prop1: 919,
         prop2: 22,
@@ -32,7 +32,7 @@ function two(uu) {
   for (let i = 0; i < 10; i++) {
     b += i;
   }
-  console.log("P", uu.a + b);
+  console.log('P', uu.a + b);
 }
 
 function one() {
@@ -51,20 +51,20 @@ function Home() {
   return (
     <View
       ref={ref}
-      style={{ flex: 1, justifyContent: "center", alignItems: "center" }}
-    >
+      style={{ flex: 1, justifyContent: 'center', alignItems: 'center' }}>
       <Link href="/details">Go to details</Link>
+      <Link href="/rotato">Go to rotato</Link>
       <Link href="/another">another</Link>
       <Link href="/location">test location</Link>
       <Link href="/another?id=100">/another?id=100</Link>
       <NiceButton
         onPress={() => {
           let a = 2;
-          console.log("Nice button pressed", obj);
+          console.log('Nice button pressed', obj);
           one();
           a++;
-          console.warn("Yollo");
-          console.warn("Yollo3");
+          console.warn('Yollo');
+          console.warn('Yollo3');
           // console.warn('Nice button pressed again');
           // console.log('WWW', window.__REACT_DEVTOOLS_PORT__);
         }}
@@ -72,16 +72,16 @@ function Home() {
       <TextInput
         style={{
           height: 40,
-          borderColor: "gray",
+          borderColor: 'gray',
           borderWidth: 1,
-          width: "70%",
+          width: '70%',
         }}
       />
       <Button
         title="Throw error 1"
         a
         onPress={() => {
-          throw new Error("from button");
+          throw new Error('from button');
         }}
       />
       <UglyButton />
@@ -92,9 +92,9 @@ function Home() {
 
 let EntryPoint = Home;
 
-const storybookEnabled = Constants.expoConfig.extra.storybookEnabled === "true";
+const storybookEnabled = Constants.expoConfig.extra.storybookEnabled === 'true';
 if (storybookEnabled) {
-  const StorybookUI = require("../.storybook").default;
+  const StorybookUI = require('../.storybook').default;
   EntryPoint = () => {
     return (
       <View style={{ flex: 1 }}>
diff --git a/test-apps/expo-router/app/rotato.js b/test-apps/expo-router/app/rotato.js
new file mode 100644
index 000000000..16e34f389
--- /dev/null
+++ b/test-apps/expo-router/app/rotato.js
@@ -0,0 +1,96 @@
+import { View, StyleSheet } from 'react-native';
+import { useRouter } from 'expo-router';
+
+import Animated, {
+  useAnimatedStyle,
+  useSharedValue,
+} from 'react-native-reanimated';
+import {
+  GestureDetector,
+  Gesture,
+  GestureHandlerRootView,
+} from 'react-native-gesture-handler';
+
+function Photo() {
+  const translationX = useSharedValue(0);
+  const translationY = useSharedValue(0);
+  const scale = useSharedValue(1);
+  const rotation = useSharedValue(0);
+
+  const style = useAnimatedStyle(() => {
+    return {
+      transform: [
+        { translateX: translationX.value },
+        { translateY: translationY.value },
+        { scale: scale.value },
+        { rotateZ: `${rotation.value}rad` },
+      ],
+    };
+  });
+
+  const rotationGesture = Gesture.Rotation().onChange((e) => {
+    'worklet';
+    rotation.value += e.rotationChange;
+  });
+
+  const scaleGesture = Gesture.Pinch().onChange((e) => {
+    'worklet';
+    scale.value *= e.scaleChange;
+  });
+
+  const panGesture = Gesture.Pan()
+    .averageTouches(true)
+    .onChange((e) => {
+      'worklet';
+      translationX.value += e.changeX;
+      translationY.value += e.changeY;
+    });
+
+  const doubleTapGesture = Gesture.Tap()
+    .numberOfTaps(2)
+    .onEnd((_e, success) => {
+      'worklet';
+      if (success) {
+        scale.value *= 1.25;
+      }
+    });
+
+  const gesture = Gesture.Simultaneous(
+    rotationGesture,
+    scaleGesture,
+    panGesture,
+    doubleTapGesture
+  );
+
+  return (
+    <GestureDetector gesture={gesture}>
+      <Animated.View style={[styles.button, style]} />
+    </GestureDetector>
+  );
+}
+
+const styles = StyleSheet.create({
+  home: {
+    width: '100%',
+    height: '100%',
+    alignSelf: 'center',
+    backgroundColor: 'plum',
+  },
+  button: {
+    width: 200,
+    height: 200,
+    backgroundColor: 'green',
+    alignSelf: 'center',
+  },
+});
+
+export default function Details() {
+  const router = useRouter();
+  return (
+    <GestureHandlerRootView style={{ flex: 1 }}>
+      <View>
+        <Photo />
+      </View>
+    </GestureHandlerRootView>
+  );
+}