feat: 新增 Action 字段 MultiSwipe 用于多指触控 (#448)

fix #447 fix #419
MaaXYZ · Dec 6, 2024 · 0f3d875 · 0f3d875
1 parent 6001df0
commit 0f3d875
Show file tree

Hide file tree

Showing 40 changed files with 723 additions and 65 deletions.
diff --git a/docs/en_us/3.1-PipelineProtocol.md b/docs/en_us/3.1-PipelineProtocol.md
@@ -70,7 +70,7 @@ This loop continues until the "next" of a task is empty, which signifies that th
 
 - `action`: *string*  
     Action to execute. Optional, default is `DoNothing`.  
-    Possible values: `DoNothing` | `Click` | `Swipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`.  
+    Possible values: `DoNothing` | `Click` | `Swipe` | `MultiSwipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`.  
     See [Action Types](#action-types) for details.
 
 - `next`: *string* | *list<string, >*  
@@ -443,7 +443,7 @@ Additional properties for this action:
 
 ### `Swipe`
 
-Swipes.
+Linear swipe.
 
 Additional properties for this action:
 
@@ -454,14 +454,64 @@ Additional properties for this action:
     Additional movement from the `begin` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0].
 
 - `end`: *true* | *string* | *array<int, 4>*  
-    The end point of the swipe. Required. The values are the same as `Click`.`target`.
+    The end point of the swipe. Optional, default is true. The values are the same as `Click`.`target`.
 
 - `end_offset`: *array<int, 4>*  
     Additional movement from the `end` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0].
 
 - `duration`: *uint*  
     Duration of the swipe in milliseconds. Optional, default is 200.
 
+### `MultiSwipe`
+
+Multi-finger linear swipe.
+
+Additional properties for this action:
+
+- `swipes`: *list<object,>*  
+    Multi swipe array. Required.  
+    The order of the array elements has no effect, only based on `starting`.
+
+  - `starting`: *uint*  
+    Swipe start time, in milliseconds. Optional, default 0.  
+    `MultiSwipe` additional field, the swipe will start pressing at the `starting` millisecond in this action.
+
+  - `begin`: *true* | *string* | *array<int, 4>*  
+      The starting point of the swipe. Optional, default is true. The values are the same as `Click`.`target`.
+
+  - `begin_offset`: *array<int, 4>*  
+      Additional movement from the `begin` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0].
+
+  - `end`: *true* | *string* | *array<int, 4>*  
+      The end point of the swipe. Optional, default is true. The values are the same as `Click`.`target`.
+
+  - `end_offset`: *array<int, 4>*  
+      Additional movement from the `end` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0].
+
+  - `duration`: *uint*  
+      Duration of the swipe in milliseconds. Optional, default is 200.
+
+For example:
+
+```jsonc
+{
+    "A": {
+        "action": "MultiSwipe",
+        "swipes": [
+            {
+                "begin": xxx,
+                "end": xxx
+            },
+            {
+                "starting": 500,
+                "begin": xxx,
+                "end": xxx
+            }
+        ]
+    }
+}
+```
+
 ### `Key`
 
 Presses a key.

diff --git a/docs/zh_cn/3.1-任务流水线协议.md b/docs/zh_cn/3.1-任务流水线协议.md
@@ -75,7 +75,7 @@ Orange 的 next 中，
 
 - `action`: *string*  
     执行的动作。可选，默认 `DoNothing` 。  
-    可选的值：`DoNothing` | `Click` | `Swipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`  
+    可选的值：`DoNothing` | `Click` | `Swipe` | `MultiSwipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`  
     详见 [动作类型](#动作类型)。
 
 - `next` : *string* | *list<string, >*  
@@ -453,7 +453,7 @@ Orange 的 next 中，
 
 ### `Swipe`
 
-滑动。  
+线性滑动。  
 
 该动作属性需额外部分字段：
 
@@ -464,14 +464,64 @@ Orange 的 next 中，
     在 `begin` 的基础上额外移动再作为起点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。
 
 - `end`: *true* | *string* | *array<int, 4>*  
-    滑动终点。必选。值同上述 `Click`.`target` 。
+    滑动终点。可选，默认 true 。值同上述 `Click`.`target` 。
 
 - `end_offset`: *array<int, 4>*  
     在 `end` 的基础上额外移动再作为终点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。
 
 - `duration`: *uint*  
     滑动持续时间，单位毫秒。可选，默认 200 。
 
+### `MultiSwipe`
+
+多指线性滑动。  
+
+该动作属性需额外部分字段：
+
+- `swipes`: *list<object,>*  
+    多个滑动的数组。必选。  
+    数组元素顺序没有影响，只基于 `starting` 确定顺序。
+
+  - `starting`: *uint*  
+    滑动起始时间，单位毫秒。可选，默认 0 。  
+    `MultiSwipe` 额外字段，该滑动会在本 action 中第 `starting` 毫秒才开始。
+
+  - `begin`: *true* | *string* | *array<int, 4>*  
+    滑动起点。可选，默认 true 。值同上述 `Click`.`target` 。
+
+  - `begin_offset`: *array<int, 4>*  
+    在 `begin` 的基础上额外移动再作为起点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。
+
+  - `end`: *true* | *string* | *array<int, 4>*  
+    滑动终点。可选，默认 true 。值同上述 `Click`.`target` 。
+
+  - `end_offset`: *array<int, 4>*  
+    在 `end` 的基础上额外移动再作为终点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。
+
+  - `duration`: *uint*  
+    滑动持续时间，单位毫秒。可选，默认 200 。
+
+举例：
+
+```jsonc
+{
+    "A": {
+        "action": "MultiSwipe",
+        "swipes": [
+            {
+                "begin": xxx,
+                "end": xxx
+            },
+            {
+                "starting": 500,
+                "begin": xxx,
+                "end": xxx
+            }
+        ]
+    }
+}
+```
+
 ### `Key`
 
 按键。  

diff --git a/sample/resource/pipeline/common.json b/sample/resource/pipeline/common.json
@@ -0,0 +1,67 @@
+{
+    "Pinch": {
+        "action": "MultiSwipe",
+        "swipes": [
+            {
+                "begin": [
+                    100,
+                    100,
+                    0,
+                    0
+                ],
+                "end": [
+                    500,
+                    300,
+                    0,
+                    0
+                ]
+            },
+            {
+                "begin": [
+                    1000,
+                    600,
+                    0,
+                    0
+                ],
+                "end": [
+                    700,
+                    400,
+                    0,
+                    0
+                ]
+            }]
+    },
+    "Unpinch": {
+        "action": "MultiSwipe",
+        "swipes": [
+            {
+                "begin": [
+                    500,
+                    300,
+                    0,
+                    0
+                ],
+                "end": [
+                    100,
+                    100,
+                    0,
+                    0
+                ]
+            },
+            {
+                "begin": [
+                    700,
+                    400,
+                    0,
+                    0
+                ],
+                "end": [
+                    1000,
+                    600,
+                    0,
+                    0
+                ]
+            }
+        ]
+    }
+}
diff --git a/source/MaaAdbControlUnit/Base/UnitBase.h b/source/MaaAdbControlUnit/Base/UnitBase.h
@@ -76,13 +76,16 @@ class InputBase
     , public ControlUnitSink
 {
 public:
+    using SwipeParam = ControlUnitAPI::SwipeParam;
+
     virtual ~InputBase() override = default;
 
     virtual bool init() = 0;
 
 public:
     virtual bool click(int x, int y) = 0;
     virtual bool swipe(int x1, int y1, int x2, int y2, int duration) = 0;
+    virtual bool multi_swipe(const std::vector<SwipeParam>& swipes) = 0;
 
     virtual bool touch_down(int contact, int x, int y, int pressure) = 0;
     virtual bool touch_move(int contact, int x, int y, int pressure) = 0;

diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp
@@ -116,7 +116,7 @@ bool MuMuPlayerExtras::swipe(int x1, int y1, int x2, int y2, int duration)
 
     if (duration <= 0) {
         LogWarn << "duration out of range" << VAR(duration);
-        duration = 500;
+        duration = 200;
     }
 
     int display_id = get_display_id();
@@ -142,6 +142,12 @@ bool MuMuPlayerExtras::swipe(int x1, int y1, int x2, int y2, int duration)
     return true;
 }
 
+bool MuMuPlayerExtras::multi_swipe(const std::vector<SwipeParam>& swipes)
+{
+    LogError << "MuMuPlayerExtras not supports" << VAR(swipes.size());
+    return false;
+}
+
 bool MuMuPlayerExtras::touch_down(int contact, int x, int y, int pressure)
 {
     if (!input_event_touch_down_func_) {

diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h
@@ -30,6 +30,7 @@ class MuMuPlayerExtras
 public: // from InputBase
     virtual bool click(int x, int y) override;
     virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override;
+    virtual bool multi_swipe(const std::vector<SwipeParam>& swipes) override;
 
     virtual bool touch_down(int contact, int x, int y, int pressure) override;
     virtual bool touch_move(int contact, int x, int y, int pressure) override;

diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.cpp b/source/MaaAdbControlUnit/Input/AdbShellInput.cpp
@@ -59,6 +59,12 @@ bool AdbShellInput::swipe(int x1, int y1, int x2, int y2, int duration)
     return output_opt && output_opt->empty();
 }
 
+bool AdbShellInput::multi_swipe(const std::vector<SwipeParam>& swipes)
+{
+    LogError << "AdbShellInput not supports" << VAR(swipes.size());
+    return false;
+}
+
 bool AdbShellInput::touch_down(int contact, int x, int y, int pressure)
 {
     LogError << "AdbShellInput not supports" << VAR(contact) << VAR(x) << VAR(y) << VAR(pressure);

diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.h b/source/MaaAdbControlUnit/Input/AdbShellInput.h
@@ -17,6 +17,7 @@ class AdbShellInput : public InputBase
 
     virtual bool click(int x, int y) override;
     virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override;
+    virtual bool multi_swipe(const std::vector<SwipeParam>& swipes) override;
 
     virtual bool touch_down(int contact, int x, int y, int pressure) override;
     virtual bool touch_move(int contact, int x, int y, int pressure) override;

diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.cpp b/source/MaaAdbControlUnit/Input/MtouchHelper.cpp
@@ -118,7 +118,7 @@ bool MtouchHelper::swipe(int x1, int y1, int x2, int y2, int duration)
     }
     if (duration <= 0) {
         LogWarn << "duration out of range" << VAR(duration);
-        duration = 500;
+        duration = 200;
     }
 
     auto [touch_x1, touch_y1] = screen_to_touch(x1, y1);
@@ -142,6 +142,52 @@ bool MtouchHelper::swipe(int x1, int y1, int x2, int y2, int duration)
     return ret;
 }
 
+bool MtouchHelper::multi_swipe(const std::vector<SwipeParam>& swipes)
+{
+    if (!pipe_ios_) {
+        LogError << "pipe_ios_ is nullptr";
+        return false;
+    }
+
+    std::vector<SwipeParam> correction = swipes;
+
+    for (SwipeParam& s : correction) {
+        if (s.x1 < 0 || s.x1 >= display_width_ || s.y1 < 0 || s.y1 >= display_height_ || s.x2 < 0 || s.x2 >= display_width_ || s.y2 < 0
+            || s.y2 >= display_height_) {
+            LogWarn << "swipe point out of range" << VAR(s.x1) << VAR(s.y1) << VAR(s.x2) << VAR(s.y2);
+            s.x1 = std::clamp(s.x1, 0, display_width_ - 1);
+            s.y1 = std::clamp(s.y1, 0, display_height_ - 1);
+            s.x2 = std::clamp(s.x2, 0, display_width_ - 1);
+            s.y2 = std::clamp(s.y2, 0, display_height_ - 1);
+        }
+        if (s.duration <= 0) {
+            LogWarn << "duration out of range" << VAR(s.duration);
+            s.duration = 200;
+        }
+
+        auto [touch_x1, touch_y1] = screen_to_touch(s.x1, s.y1);
+        auto [touch_x2, touch_y2] = screen_to_touch(s.x2, s.y2);
+
+        LogInfo << VAR(s.x1) << VAR(s.y1) << VAR(touch_x1) << VAR(touch_y1) << VAR(s.x2) << VAR(s.y2) << VAR(touch_x2) << VAR(touch_y2)
+                << VAR(s.duration);
+
+        s.x1 = touch_x1;
+        s.y1 = touch_y1;
+        s.x2 = touch_x2;
+        s.y2 = touch_y2;
+    }
+
+    bool ret = true;
+
+    micro_multi_swipe(
+        correction,
+        [&](int contact, int x, int y) { ret &= pipe_ios_->write(std::format(kDownFormat, contact, x, y, press_)); },
+        [&](int contact, int x, int y) { ret &= pipe_ios_->write(std::format(kMoveFormat, contact, x, y, press_)); },
+        [&](int contact, [[maybe_unused]] int x, [[maybe_unused]] int y) { ret &= pipe_ios_->write(std::format(kUpFormat, contact)); });
+
+    return ret;
+}
+
 bool MtouchHelper::touch_down(int contact, int x, int y, int pressure)
 {
     if (!pipe_ios_) {

diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.h b/source/MaaAdbControlUnit/Input/MtouchHelper.h
@@ -18,6 +18,7 @@ class MtouchHelper : public InputBase
 public: // from InputBase
     virtual bool click(int x, int y) override;
     virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override;
+    virtual bool multi_swipe(const std::vector<SwipeParam>& swipes) override;
 
     virtual bool touch_down(int contact, int x, int y, int pressure) override;
     virtual bool touch_move(int contact, int x, int y, int pressure) override;

diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp
@@ -167,6 +167,16 @@ bool ControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration)
     return input_->swipe(x1, y1, x2, y2, duration);
 }
 
+bool ControlUnitMgr::multi_swipe(const std::vector<SwipeParam>& swipes)
+{
+    if (!input_) {
+        LogError << "input_ is null";
+        return false;
+    }
+
+    return input_->multi_swipe(swipes);
+}
+
 bool ControlUnitMgr::touch_down(int contact, int x, int y, int pressure)
 {
     if (!input_) {

diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h
@@ -41,6 +41,7 @@ class ControlUnitMgr
 
     virtual bool click(int x, int y) override;
     virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override;
+    virtual bool multi_swipe(const std::vector<SwipeParam>& swipes) override;
 
     virtual bool touch_down(int contact, int x, int y, int pressure) override;
     virtual bool touch_move(int contact, int x, int y, int pressure) override;