From 0f3d87577a9f4bd4e145b237067c8607d595d3bb Mon Sep 17 00:00:00 2001 From: MistEO Date: Fri, 6 Dec 2024 14:03:55 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=20Action=20=E5=AD=97?= =?UTF-8?q?=E6=AE=B5=20MultiSwipe=20=E7=94=A8=E4=BA=8E=E5=A4=9A=E6=8C=87?= =?UTF-8?q?=E8=A7=A6=E6=8E=A7=20(#448)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fix https://github.com/MaaXYZ/MaaFramework/issues/447 fix https://github.com/MaaXYZ/MaaFramework/issues/419 --- docs/en_us/3.1-PipelineProtocol.md | 56 +++++++++- ...64\347\272\277\345\215\217\350\256\256.md" | 56 +++++++++- sample/resource/pipeline/common.json | 67 +++++++++++ source/MaaAdbControlUnit/Base/UnitBase.h | 3 + .../EmulatorExtras/MuMuPlayerExtras.cpp | 8 +- .../EmulatorExtras/MuMuPlayerExtras.h | 1 + .../MaaAdbControlUnit/Input/AdbShellInput.cpp | 6 + .../MaaAdbControlUnit/Input/AdbShellInput.h | 1 + .../MaaAdbControlUnit/Input/MtouchHelper.cpp | 48 +++++++- source/MaaAdbControlUnit/Input/MtouchHelper.h | 1 + .../Manager/ControlUnitMgr.cpp | 10 ++ .../Manager/ControlUnitMgr.h | 1 + .../MaaAdbControlUnit/Manager/InputAgent.cpp | 10 ++ source/MaaAdbControlUnit/Manager/InputAgent.h | 1 + .../CarouselImage/CarouselImage.cpp | 7 ++ .../CarouselImage/CarouselImage.h | 1 + .../ReplayRecording/Record.h | 7 +- .../ReplayRecording/RecordParser.cpp | 33 +++++- .../ReplayRecording/RecordParser.h | 1 + .../ReplayRecording/ReplayRecording.cpp | 22 ++++ .../ReplayRecording/ReplayRecording.h | 1 + .../Controller/ControllerAgent.cpp | 104 ++++++++++++------ .../MaaFramework/Controller/ControllerAgent.h | 33 +++++- .../Controller/CustomControllerAgent.cpp | 9 ++ .../Controller/CustomControllerAgent.h | 1 + .../Controller/GeneralControllerAgent.cpp | 28 +++++ .../Controller/GeneralControllerAgent.h | 1 + .../Resource/DefaultPipelineMgr.cpp | 2 + .../MaaFramework/Resource/PipelineResMgr.cpp | 48 ++++++++ source/MaaFramework/Resource/PipelineResMgr.h | 5 + source/MaaFramework/Resource/PipelineTypes.h | 9 +- .../MaaFramework/Task/Component/Actuator.cpp | 25 +++++ source/MaaFramework/Task/Component/Actuator.h | 1 + .../MaaWin32ControlUnit/Input/SeizeInput.cpp | 2 +- .../Input/SendMessageInput.cpp | 2 +- .../Manager/ControlUnitMgr.cpp | 9 ++ .../Manager/ControlUnitMgr.h | 1 + source/include/ControlUnit/ControlUnitAPI.h | 12 ++ source/include/Utils/MicroControl.hpp | 89 +++++++++++++-- tools/pipeline.schema.json | 66 ++++++++++- 40 files changed, 723 insertions(+), 65 deletions(-) create mode 100644 sample/resource/pipeline/common.json diff --git a/docs/en_us/3.1-PipelineProtocol.md b/docs/en_us/3.1-PipelineProtocol.md index 9386177f6..b62053317 100644 --- a/docs/en_us/3.1-PipelineProtocol.md +++ b/docs/en_us/3.1-PipelineProtocol.md @@ -70,7 +70,7 @@ This loop continues until the "next" of a task is empty, which signifies that th - `action`: *string* Action to execute. Optional, default is `DoNothing`. - Possible values: `DoNothing` | `Click` | `Swipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`. + Possible values: `DoNothing` | `Click` | `Swipe` | `MultiSwipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom`. See [Action Types](#action-types) for details. - `next`: *string* | *list* @@ -443,7 +443,7 @@ Additional properties for this action: ### `Swipe` -Swipes. +Linear swipe. Additional properties for this action: @@ -454,7 +454,7 @@ Additional properties for this action: Additional movement from the `begin` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0]. - `end`: *true* | *string* | *array* - The end point of the swipe. Required. The values are the same as `Click`.`target`. + The end point of the swipe. Optional, default is true. The values are the same as `Click`.`target`. - `end_offset`: *array* Additional movement from the `end` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0]. @@ -462,6 +462,56 @@ Additional properties for this action: - `duration`: *uint* Duration of the swipe in milliseconds. Optional, default is 200. +### `MultiSwipe` + +Multi-finger linear swipe. + +Additional properties for this action: + +- `swipes`: *list* + Multi swipe array. Required. + The order of the array elements has no effect, only based on `starting`. + + - `starting`: *uint* + Swipe start time, in milliseconds. Optional, default 0. + `MultiSwipe` additional field, the swipe will start pressing at the `starting` millisecond in this action. + + - `begin`: *true* | *string* | *array* + The starting point of the swipe. Optional, default is true. The values are the same as `Click`.`target`. + + - `begin_offset`: *array* + Additional movement from the `begin` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0]. + + - `end`: *true* | *string* | *array* + The end point of the swipe. Optional, default is true. The values are the same as `Click`.`target`. + + - `end_offset`: *array* + Additional movement from the `end` before swiping, where the four values are added together. Optional, default is [0, 0, 0, 0]. + + - `duration`: *uint* + Duration of the swipe in milliseconds. Optional, default is 200. + +For example: + +```jsonc +{ + "A": { + "action": "MultiSwipe", + "swipes": [ + { + "begin": xxx, + "end": xxx + }, + { + "starting": 500, + "begin": xxx, + "end": xxx + } + ] + } +} +``` + ### `Key` Presses a key. diff --git "a/docs/zh_cn/3.1-\344\273\273\345\212\241\346\265\201\346\260\264\347\272\277\345\215\217\350\256\256.md" "b/docs/zh_cn/3.1-\344\273\273\345\212\241\346\265\201\346\260\264\347\272\277\345\215\217\350\256\256.md" index 47606328c..7615019ad 100644 --- "a/docs/zh_cn/3.1-\344\273\273\345\212\241\346\265\201\346\260\264\347\272\277\345\215\217\350\256\256.md" +++ "b/docs/zh_cn/3.1-\344\273\273\345\212\241\346\265\201\346\260\264\347\272\277\345\215\217\350\256\256.md" @@ -75,7 +75,7 @@ Orange 的 next 中， - `action`: *string* 执行的动作。可选，默认 `DoNothing` 。 - 可选的值：`DoNothing` | `Click` | `Swipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom` + 可选的值：`DoNothing` | `Click` | `Swipe` | `MultiSwipe` | `Key` | `InputText` | `StartApp` | `StopApp` | `StopTask` | `Custom` 详见 [动作类型](#动作类型)。 - `next` : *string* | *list* @@ -453,7 +453,7 @@ Orange 的 next 中， ### `Swipe` -滑动。 +线性滑动。 该动作属性需额外部分字段： @@ -464,7 +464,7 @@ Orange 的 next 中， 在 `begin` 的基础上额外移动再作为起点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。 - `end`: *true* | *string* | *array* - 滑动终点。必选。值同上述 `Click`.`target` 。 + 滑动终点。可选，默认 true 。值同上述 `Click`.`target` 。 - `end_offset`: *array* 在 `end` 的基础上额外移动再作为终点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。 @@ -472,6 +472,56 @@ Orange 的 next 中， - `duration`: *uint* 滑动持续时间，单位毫秒。可选，默认 200 。 +### `MultiSwipe` + +多指线性滑动。 + +该动作属性需额外部分字段： + +- `swipes`: *list* + 多个滑动的数组。必选。 + 数组元素顺序没有影响，只基于 `starting` 确定顺序。 + + - `starting`: *uint* + 滑动起始时间，单位毫秒。可选，默认 0 。 + `MultiSwipe` 额外字段，该滑动会在本 action 中第 `starting` 毫秒才开始。 + + - `begin`: *true* | *string* | *array* + 滑动起点。可选，默认 true 。值同上述 `Click`.`target` 。 + + - `begin_offset`: *array* + 在 `begin` 的基础上额外移动再作为起点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。 + + - `end`: *true* | *string* | *array* + 滑动终点。可选，默认 true 。值同上述 `Click`.`target` 。 + + - `end_offset`: *array* + 在 `end` 的基础上额外移动再作为终点，四个值分别相加。可选，默认 [0, 0, 0, 0] 。 + + - `duration`: *uint* + 滑动持续时间，单位毫秒。可选，默认 200 。 + +举例： + +```jsonc +{ + "A": { + "action": "MultiSwipe", + "swipes": [ + { + "begin": xxx, + "end": xxx + }, + { + "starting": 500, + "begin": xxx, + "end": xxx + } + ] + } +} +``` + ### `Key` 按键。 diff --git a/sample/resource/pipeline/common.json b/sample/resource/pipeline/common.json new file mode 100644 index 000000000..b2ddb1de4 --- /dev/null +++ b/sample/resource/pipeline/common.json @@ -0,0 +1,67 @@ +{ + "Pinch": { + "action": "MultiSwipe", + "swipes": [ + { + "begin": [ + 100, + 100, + 0, + 0 + ], + "end": [ + 500, + 300, + 0, + 0 + ] + }, + { + "begin": [ + 1000, + 600, + 0, + 0 + ], + "end": [ + 700, + 400, + 0, + 0 + ] + }] + }, + "Unpinch": { + "action": "MultiSwipe", + "swipes": [ + { + "begin": [ + 500, + 300, + 0, + 0 + ], + "end": [ + 100, + 100, + 0, + 0 + ] + }, + { + "begin": [ + 700, + 400, + 0, + 0 + ], + "end": [ + 1000, + 600, + 0, + 0 + ] + } + ] + } +} \ No newline at end of file diff --git a/source/MaaAdbControlUnit/Base/UnitBase.h b/source/MaaAdbControlUnit/Base/UnitBase.h index aef28b092..3fdee2be0 100644 --- a/source/MaaAdbControlUnit/Base/UnitBase.h +++ b/source/MaaAdbControlUnit/Base/UnitBase.h @@ -76,6 +76,8 @@ class InputBase , public ControlUnitSink { public: + using SwipeParam = ControlUnitAPI::SwipeParam; + virtual ~InputBase() override = default; virtual bool init() = 0; @@ -83,6 +85,7 @@ class InputBase public: virtual bool click(int x, int y) = 0; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) = 0; + virtual bool multi_swipe(const std::vector& swipes) = 0; virtual bool touch_down(int contact, int x, int y, int pressure) = 0; virtual bool touch_move(int contact, int x, int y, int pressure) = 0; diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp index 8ff6b6a08..616318995 100644 --- a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp +++ b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp @@ -116,7 +116,7 @@ bool MuMuPlayerExtras::swipe(int x1, int y1, int x2, int y2, int duration) if (duration <= 0) { LogWarn << "duration out of range" << VAR(duration); - duration = 500; + duration = 200; } int display_id = get_display_id(); @@ -142,6 +142,12 @@ bool MuMuPlayerExtras::swipe(int x1, int y1, int x2, int y2, int duration) return true; } +bool MuMuPlayerExtras::multi_swipe(const std::vector& swipes) +{ + LogError << "MuMuPlayerExtras not supports" << VAR(swipes.size()); + return false; +} + bool MuMuPlayerExtras::touch_down(int contact, int x, int y, int pressure) { if (!input_event_touch_down_func_) { diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h index 1c73a41ab..649be05b5 100644 --- a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h +++ b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h @@ -30,6 +30,7 @@ class MuMuPlayerExtras public: // from InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.cpp b/source/MaaAdbControlUnit/Input/AdbShellInput.cpp index db734899f..bcb558d0d 100644 --- a/source/MaaAdbControlUnit/Input/AdbShellInput.cpp +++ b/source/MaaAdbControlUnit/Input/AdbShellInput.cpp @@ -59,6 +59,12 @@ bool AdbShellInput::swipe(int x1, int y1, int x2, int y2, int duration) return output_opt && output_opt->empty(); } +bool AdbShellInput::multi_swipe(const std::vector& swipes) +{ + LogError << "AdbShellInput not supports" << VAR(swipes.size()); + return false; +} + bool AdbShellInput::touch_down(int contact, int x, int y, int pressure) { LogError << "AdbShellInput not supports" << VAR(contact) << VAR(x) << VAR(y) << VAR(pressure); diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.h b/source/MaaAdbControlUnit/Input/AdbShellInput.h index b162e0e7e..9af3e1f07 100644 --- a/source/MaaAdbControlUnit/Input/AdbShellInput.h +++ b/source/MaaAdbControlUnit/Input/AdbShellInput.h @@ -17,6 +17,7 @@ class AdbShellInput : public InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.cpp b/source/MaaAdbControlUnit/Input/MtouchHelper.cpp index d83c133f8..9f1c46ef6 100644 --- a/source/MaaAdbControlUnit/Input/MtouchHelper.cpp +++ b/source/MaaAdbControlUnit/Input/MtouchHelper.cpp @@ -118,7 +118,7 @@ bool MtouchHelper::swipe(int x1, int y1, int x2, int y2, int duration) } if (duration <= 0) { LogWarn << "duration out of range" << VAR(duration); - duration = 500; + duration = 200; } auto [touch_x1, touch_y1] = screen_to_touch(x1, y1); @@ -142,6 +142,52 @@ bool MtouchHelper::swipe(int x1, int y1, int x2, int y2, int duration) return ret; } +bool MtouchHelper::multi_swipe(const std::vector& swipes) +{ + if (!pipe_ios_) { + LogError << "pipe_ios_ is nullptr"; + return false; + } + + std::vector correction = swipes; + + for (SwipeParam& s : correction) { + if (s.x1 < 0 || s.x1 >= display_width_ || s.y1 < 0 || s.y1 >= display_height_ || s.x2 < 0 || s.x2 >= display_width_ || s.y2 < 0 + || s.y2 >= display_height_) { + LogWarn << "swipe point out of range" << VAR(s.x1) << VAR(s.y1) << VAR(s.x2) << VAR(s.y2); + s.x1 = std::clamp(s.x1, 0, display_width_ - 1); + s.y1 = std::clamp(s.y1, 0, display_height_ - 1); + s.x2 = std::clamp(s.x2, 0, display_width_ - 1); + s.y2 = std::clamp(s.y2, 0, display_height_ - 1); + } + if (s.duration <= 0) { + LogWarn << "duration out of range" << VAR(s.duration); + s.duration = 200; + } + + auto [touch_x1, touch_y1] = screen_to_touch(s.x1, s.y1); + auto [touch_x2, touch_y2] = screen_to_touch(s.x2, s.y2); + + LogInfo << VAR(s.x1) << VAR(s.y1) << VAR(touch_x1) << VAR(touch_y1) << VAR(s.x2) << VAR(s.y2) << VAR(touch_x2) << VAR(touch_y2) + << VAR(s.duration); + + s.x1 = touch_x1; + s.y1 = touch_y1; + s.x2 = touch_x2; + s.y2 = touch_y2; + } + + bool ret = true; + + micro_multi_swipe( + correction, + [&](int contact, int x, int y) { ret &= pipe_ios_->write(std::format(kDownFormat, contact, x, y, press_)); }, + [&](int contact, int x, int y) { ret &= pipe_ios_->write(std::format(kMoveFormat, contact, x, y, press_)); }, + [&](int contact, [[maybe_unused]] int x, [[maybe_unused]] int y) { ret &= pipe_ios_->write(std::format(kUpFormat, contact)); }); + + return ret; +} + bool MtouchHelper::touch_down(int contact, int x, int y, int pressure) { if (!pipe_ios_) { diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.h b/source/MaaAdbControlUnit/Input/MtouchHelper.h index 4038b823e..c04cfa448 100644 --- a/source/MaaAdbControlUnit/Input/MtouchHelper.h +++ b/source/MaaAdbControlUnit/Input/MtouchHelper.h @@ -18,6 +18,7 @@ class MtouchHelper : public InputBase public: // from InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp index 5aa55b5cd..c12fe2e23 100644 --- a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp +++ b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp @@ -167,6 +167,16 @@ bool ControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration) return input_->swipe(x1, y1, x2, y2, duration); } +bool ControlUnitMgr::multi_swipe(const std::vector& swipes) +{ + if (!input_) { + LogError << "input_ is null"; + return false; + } + + return input_->multi_swipe(swipes); +} + bool ControlUnitMgr::touch_down(int contact, int x, int y, int pressure) { if (!input_) { diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h index e788abc86..d8df4cbcd 100644 --- a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h +++ b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h @@ -41,6 +41,7 @@ class ControlUnitMgr virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Manager/InputAgent.cpp b/source/MaaAdbControlUnit/Manager/InputAgent.cpp index e8ff3a256..71a70c6e8 100644 --- a/source/MaaAdbControlUnit/Manager/InputAgent.cpp +++ b/source/MaaAdbControlUnit/Manager/InputAgent.cpp @@ -135,6 +135,16 @@ bool InputAgent::swipe(int x1, int y1, int x2, int y2, int duration) return active_unit_->swipe(x1, y1, x2, y2, duration); } +bool InputAgent::multi_swipe(const std::vector& swipes) +{ + if (!active_unit_) { + LogError << "No available input method" << VAR(active_unit_); + return false; + } + + return active_unit_->multi_swipe(swipes); +} + bool InputAgent::touch_down(int contact, int x, int y, int pressure) { if (!active_unit_) { diff --git a/source/MaaAdbControlUnit/Manager/InputAgent.h b/source/MaaAdbControlUnit/Manager/InputAgent.h index c61d2cbc6..500c7ce84 100644 --- a/source/MaaAdbControlUnit/Manager/InputAgent.h +++ b/source/MaaAdbControlUnit/Manager/InputAgent.h @@ -28,6 +28,7 @@ class InputAgent : public InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp index 8cc7279d1..ea94187ee 100644 --- a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp +++ b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp @@ -109,6 +109,13 @@ bool CarouselImage::swipe(int x1, int y1, int x2, int y2, int duration) return true; } +bool CarouselImage::multi_swipe(const std::vector& swipes) +{ + std::ignore = swipes; + + return true; +} + bool CarouselImage::touch_down(int contact, int x, int y, int pressure) { std::ignore = contact; diff --git a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h index 07c9df6b1..5fb16dd4a 100644 --- a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h +++ b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h @@ -32,6 +32,7 @@ class CarouselImage : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaDbgControlUnit/ReplayRecording/Record.h b/source/MaaDbgControlUnit/ReplayRecording/Record.h index fe3737d16..94c722d0e 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/Record.h +++ b/source/MaaDbgControlUnit/ReplayRecording/Record.h @@ -40,6 +40,7 @@ struct Record int x2 = 0; int y2 = 0; int duration = 0; + int starting = 0; }; struct TouchParam @@ -72,7 +73,7 @@ struct Record }; using Param = std:: - variant; + variant, TouchParam, PressKeyParam, InputTextParam, AppParam, ScreencapParam>; struct Action { @@ -82,6 +83,7 @@ struct Record connect, click, swipe, + multi_swipe, touch_down, touch_move, touch_up, @@ -124,6 +126,9 @@ inline std::ostream& operator<<(std::ostream& os, Record::Action::Type type) case Record::Action::Type::swipe: os << "swipe"; break; + case Record::Action::Type::multi_swipe: + os << "multi_swipe"; + break; case Record::Action::Type::touch_down: os << "touch_down"; break; diff --git a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp index b78235315..9a90916ae 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp +++ b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp @@ -73,11 +73,11 @@ std::optional RecordParser::parse_record(const json::value& record_json, std::string type_str = record_json.get("type", std::string()); static const std::unordered_map kTypeMap = { { "connect", Record::Action::Type::connect }, { "click", Record::Action::Type::click }, - { "swipe", Record::Action::Type::swipe }, { "touch_down", Record::Action::Type::touch_down }, - { "touch_move", Record::Action::Type::touch_move }, { "touch_up", Record::Action::Type::touch_up }, - { "press_key", Record::Action::Type::press_key }, { "input_text", Record::Action::Type::input_text }, - { "screencap", Record::Action::Type::screencap }, { "start_app", Record::Action::Type::start_app }, - { "stop_app", Record::Action::Type::stop_app }, + { "swipe", Record::Action::Type::swipe }, { "multi_swipe", Record::Action::Type::multi_swipe }, + { "touch_down", Record::Action::Type::touch_down }, { "touch_move", Record::Action::Type::touch_move }, + { "touch_up", Record::Action::Type::touch_up }, { "press_key", Record::Action::Type::press_key }, + { "input_text", Record::Action::Type::input_text }, { "screencap", Record::Action::Type::screencap }, + { "start_app", Record::Action::Type::start_app }, { "stop_app", Record::Action::Type::stop_app }, }; auto it = kTypeMap.find(type_str); @@ -99,6 +99,9 @@ std::optional RecordParser::parse_record(const json::value& record_json, case Record::Action::Type::swipe: action_opt = parse_swipe(record_json); break; + case Record::Action::Type::multi_swipe: + action_opt = parse_multi_swipe(record_json); + break; case Record::Action::Type::touch_down: case Record::Action::Type::touch_move: case Record::Action::Type::touch_up: @@ -224,6 +227,26 @@ std::optional RecordParser::parse_swipe(const json::value& record return result; } +std::optional RecordParser::parse_multi_swipe(const json::value& record_json) +{ + auto swipes_opt = record_json.find("swipes"); + if (!swipes_opt) { + LogError << "Failed to find swipes:" << VAR(record_json); + return std::nullopt; + } + + std::vector result; + for (const json::value& swipe : *swipes_opt) { + auto s_opt = parse_swipe(swipe); + if (!s_opt) { + return std::nullopt; + } + auto s = std::get(*std::move(s_opt)); + result.emplace_back(std::move(s)); + } + return result; +} + std::optional RecordParser::parse_touch(const json::value& record_json) { Record::TouchParam result; diff --git a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h index 6e39faaa5..a5103743c 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h +++ b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h @@ -18,6 +18,7 @@ class RecordParser static std::optional parse_connect(const json::value& record_json); static std::optional parse_click(const json::value& record_json); static std::optional parse_swipe(const json::value& record_json); + static std::optional parse_multi_swipe(const json::value& record_json); static std::optional parse_touch(const json::value& record_json); static std::optional parse_press_key(const json::value& record_json); static std::optional parse_input_text(const json::value& record_json); diff --git a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp index 96d499a08..173397acd 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp +++ b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp @@ -186,6 +186,28 @@ bool ReplayRecording::swipe(int x1, int y1, int x2, int y2, int duration) return record.success; } +bool ReplayRecording::multi_swipe(const std::vector& swipes) +{ + LogInfo << VAR(swipes.size()); + + if (record_index_ >= recording_.records.size()) { + LogError << "record index out of range" << VAR(record_index_) << VAR(recording_.records.size()); + return false; + } + + const Record& record = recording_.records.at(record_index_); + if (record.action.type != Record::Action::Type::multi_swipe) { + LogError << "record type is not swipe" << VAR(record.action.type) << VAR(record.raw_data); + return false; + } + + // TODO: 现在点击的点是随机区域，没法直接检查 + + sleep(record.cost); + ++record_index_; + return record.success; +} + bool ReplayRecording::touch_down(int contact, int x, int y, int pressure) { LogInfo << VAR(contact) << VAR(x) << VAR(y) << VAR(pressure); diff --git a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h index fb0848c12..216543fb2 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h +++ b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h @@ -33,6 +33,7 @@ class ReplayRecording : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaFramework/Controller/ControllerAgent.cpp b/source/MaaFramework/Controller/ControllerAgent.cpp index 8136df9a1..fea4bf6b2 100644 --- a/source/MaaFramework/Controller/ControllerAgent.cpp +++ b/source/MaaFramework/Controller/ControllerAgent.cpp @@ -204,6 +204,26 @@ bool ControllerAgent::swipe(const cv::Point& p1, const cv::Point& p2, int durati return wait(id) == MaaStatus_Succeeded; } +bool ControllerAgent::multi_swipe(const std::vector& swipes) +{ + std::vector dst_vec; + for (const auto& src : swipes) { + auto p1 = rand_point(src.r1); + auto p2 = rand_point(src.r2); + dst_vec.emplace_back(SwipeParam { + .x1 = p1.x, + .y1 = p1.y, + .x2 = p2.x, + .y2 = p2.y, + .duration = static_cast(src.duration), + .starting = static_cast(src.starting), + }); + } + + auto id = post_multi_swipe_impl(dst_vec); + return wait(id) == MaaStatus_Succeeded; +} + bool ControllerAgent::press_key(int keycode) { auto id = post_press_key_impl(keycode); @@ -280,6 +300,17 @@ MaaCtrlId ControllerAgent::post_swipe_impl(int x1, int y1, int x2, int y2, int d return post({ .type = Action::Type::swipe, .param = std::move(param) }); } +MaaCtrlId ControllerAgent::post_multi_swipe_impl(const std::vector& swipes) +{ + std::vector dst = swipes; + for (SwipeParam& d : dst) { + std::tie(d.x1, d.y1) = preproc_touch_point(d.x1, d.y1); + std::tie(d.x2, d.y2) = preproc_touch_point(d.x2, d.y2); + } + + return post({ .type = Action::Type::multi_swipe, .param = std::move(dst) }); +} + MaaCtrlId ControllerAgent::post_press_key_impl(int keycode) { PressKeyParam param { .keycode = keycode }; @@ -364,11 +395,8 @@ bool ControllerAgent::handle_click(const ClickParam& param) bool ret = _click(param); if (recording()) { - json::value info = { - { "type", "click" }, - { "x", param.x }, - { "y", param.y }, - }; + json::value info = param; + info |= { { "type", "click" } }; append_recording(std::move(info), start_time, ret); } @@ -385,10 +413,25 @@ bool ControllerAgent::handle_swipe(const SwipeParam& param) bool ret = _swipe(param); if (recording()) { - json::value info = { - { "type", "swipe" }, { "x1", param.x1 }, { "y1", param.y1 }, - { "x2", param.x2 }, { "y2", param.y2 }, { "duration", param.duration }, - }; + json::value info = param; + info |= { { "type", "swipe" } }; + append_recording(std::move(info), start_time, ret); + } + + return ret; +} + +bool ControllerAgent::handle_multi_swipe(const std::vector& param) +{ + std::chrono::steady_clock::time_point start_time; + if (recording()) { + start_time = std::chrono::steady_clock::now(); + } + + bool ret = _multi_swipe(param); + + if (recording()) { + json::value info = { { "type", "multi_swipe" }, { "swipes", json::array(param) } }; append_recording(std::move(info), start_time, ret); } @@ -405,9 +448,8 @@ bool ControllerAgent::handle_touch_down(const TouchParam& param) bool ret = _touch_down(param); if (recording()) { - json::value info = { - { "type", "touch_down" }, { "contact", param.contact }, { "x", param.x }, { "y", param.y }, { "pressure", param.pressure }, - }; + json::value info = param; + info |= { { "type", "touch_down" } }; append_recording(std::move(info), start_time, ret); } @@ -424,9 +466,8 @@ bool ControllerAgent::handle_touch_move(const TouchParam& param) bool ret = _touch_move(param); if (recording()) { - json::value info = { - { "type", "touch_move" }, { "contact", param.contact }, { "x", param.x }, { "y", param.y }, { "pressure", param.pressure }, - }; + json::value info = param; + info |= { { "type", "touch_move" } }; append_recording(std::move(info), start_time, ret); } @@ -443,10 +484,8 @@ bool ControllerAgent::handle_touch_up(const TouchParam& param) bool ret = _touch_up(param); if (recording()) { - json::value info = { - { "type", "touch_up" }, - { "contact", param.contact }, - }; + json::value info = param; + info |= { { "type", "touch_up" } }; append_recording(std::move(info), start_time, ret); } @@ -463,10 +502,8 @@ bool ControllerAgent::handle_press_key(const PressKeyParam& param) bool ret = _press_key(param); if (recording()) { - json::value info = { - { "type", "press_key" }, - { "keycode", param.keycode }, - }; + json::value info = param; + info |= { { "type", "press_key" } }; append_recording(std::move(info), start_time, ret); } @@ -483,10 +520,8 @@ bool ControllerAgent::handle_input_text(const InputTextParam& param) bool ret = _input_text(param); if (recording()) { - json::value info = { - { "type", "input_text" }, - { "input_text", param.text }, - }; + json::value info = param; + info |= { { "type", "input_text" } }; append_recording(std::move(info), start_time, ret); } @@ -534,10 +569,8 @@ bool ControllerAgent::handle_start_app(const AppParam& param) bool ret = _start_app(param); if (recording()) { - json::value info = { - { "type", "start_app" }, - { "package", param.package }, - }; + json::value info = param; + info |= { { "type", "start_app" } }; append_recording(std::move(info), start_time, ret); } return ret; @@ -553,10 +586,8 @@ bool ControllerAgent::handle_stop_app(const AppParam& param) bool ret = _stop_app(param); if (recording()) { - json::value info = { - { "type", "stop_app" }, - { "package", param.package }, - }; + json::value info = param; + info |= { { "type", "stop_app" } }; append_recording(std::move(info), start_time, ret); } return ret; @@ -661,6 +692,9 @@ bool ControllerAgent::run_action(typename AsyncRunner::Id id, Action act case Action::Type::swipe: ret = handle_swipe(std::get(action.param)); break; + case Action::Type::multi_swipe: + ret = handle_multi_swipe(std::get>(action.param)); + break; case Action::Type::touch_down: ret = handle_touch_down(std::get(action.param)); diff --git a/source/MaaFramework/Controller/ControllerAgent.h b/source/MaaFramework/Controller/ControllerAgent.h index b05778cea..018349a86 100644 --- a/source/MaaFramework/Controller/ControllerAgent.h +++ b/source/MaaFramework/Controller/ControllerAgent.h @@ -6,6 +6,8 @@ #include #include +#include + #include "API/MaaTypes.h" #include "Base/AsyncRunner.hpp" #include "Utils/MessageNotifier.hpp" @@ -21,6 +23,8 @@ struct ClickParam { int x = 0; int y = 0; + + MEO_JSONIZATION(x, y); }; struct SwipeParam @@ -30,6 +34,9 @@ struct SwipeParam int x2 = 0; int y2 = 0; int duration = 0; + int starting = 0; + + MEO_JSONIZATION(x1, y1, x2, y2, duration, starting); }; struct TouchParam @@ -38,24 +45,33 @@ struct TouchParam int x = 0; int y = 0; int pressure = 0; + + MEO_JSONIZATION(contact, x, y, pressure); }; struct PressKeyParam { int keycode = 0; + + MEO_JSONIZATION(keycode); }; struct InputTextParam { std::string text; + + MEO_JSONIZATION(text); }; struct AppParam { std::string package; + + MEO_JSONIZATION(package); }; -using Param = std::variant; +using Param = + std::variant, TouchParam, PressKeyParam, InputTextParam, AppParam>; struct Action { @@ -65,6 +81,7 @@ struct Action connect, click, swipe, + multi_swipe, touch_down, touch_move, touch_up, @@ -117,6 +134,17 @@ class ControllerAgent : public MaaController bool click(const cv::Point& p); bool swipe(const cv::Rect& r1, const cv::Rect& r2, int duration); bool swipe(const cv::Point& p1, const cv::Point& p2, int duration); + + struct SwipeParamWithRect + { + cv::Rect r1 {}; + cv::Rect r2 {}; + uint duration = 0; + uint starting = 0; + }; + + bool multi_swipe(const std::vector& swipes); + bool press_key(int keycode); bool input_text(const std::string& text); cv::Mat screencap(); @@ -132,6 +160,7 @@ class ControllerAgent : public MaaController virtual std::optional _screencap() = 0; virtual bool _click(ClickParam param) = 0; virtual bool _swipe(SwipeParam param) = 0; + virtual bool _multi_swipe(std::vector param) = 0; virtual bool _touch_down(TouchParam param) = 0; virtual bool _touch_move(TouchParam param) = 0; virtual bool _touch_up(TouchParam param) = 0; @@ -145,6 +174,7 @@ class ControllerAgent : public MaaController MaaCtrlId post_connection_impl(); MaaCtrlId post_click_impl(int x, int y); MaaCtrlId post_swipe_impl(int x1, int y1, int x2, int y2, int duration); + MaaCtrlId post_multi_swipe_impl(const std::vector& swipes); MaaCtrlId post_press_key_impl(int keycode); MaaCtrlId post_input_text_impl(const std::string& text); MaaCtrlId post_start_app_impl(const std::string& text); @@ -158,6 +188,7 @@ class ControllerAgent : public MaaController bool handle_connect(); bool handle_click(const ClickParam& param); bool handle_swipe(const SwipeParam& param); + bool handle_multi_swipe(const std::vector& param); bool handle_touch_down(const TouchParam& param); bool handle_touch_move(const TouchParam& param); bool handle_touch_up(const TouchParam& param); diff --git a/source/MaaFramework/Controller/CustomControllerAgent.cpp b/source/MaaFramework/Controller/CustomControllerAgent.cpp index 33340f298..e4c6ca8ef 100644 --- a/source/MaaFramework/Controller/CustomControllerAgent.cpp +++ b/source/MaaFramework/Controller/CustomControllerAgent.cpp @@ -121,6 +121,15 @@ bool CustomControllerAgent::_swipe(SwipeParam param) return controller_->swipe(param.x1, param.y1, param.x2, param.y2, param.duration, controller_arg_); } +bool CustomControllerAgent::_multi_swipe(std::vector param) +{ + std::ignore = param; + + LogError << "CustomController not support MulitSwipe now, We welcome your PR to implement it!" << VAR(param); + + return false; +} + bool CustomControllerAgent::_touch_down(TouchParam param) { LogFunc << VAR_VOIDP(controller_) << VAR_VOIDP(controller_->touch_down) << VAR(param.contact) << VAR(param.x) << VAR(param.y) diff --git a/source/MaaFramework/Controller/CustomControllerAgent.h b/source/MaaFramework/Controller/CustomControllerAgent.h index 5d49b5a7a..21ea2d7bb 100644 --- a/source/MaaFramework/Controller/CustomControllerAgent.h +++ b/source/MaaFramework/Controller/CustomControllerAgent.h @@ -23,6 +23,7 @@ class CustomControllerAgent : public ControllerAgent virtual std::optional _screencap() override; virtual bool _click(ClickParam param) override; virtual bool _swipe(SwipeParam param) override; + virtual bool _multi_swipe(std::vector param) override; virtual bool _touch_down(TouchParam param) override; virtual bool _touch_move(TouchParam param) override; virtual bool _touch_up(TouchParam param) override; diff --git a/source/MaaFramework/Controller/GeneralControllerAgent.cpp b/source/MaaFramework/Controller/GeneralControllerAgent.cpp index a20a3c10d..83cfa2fb7 100644 --- a/source/MaaFramework/Controller/GeneralControllerAgent.cpp +++ b/source/MaaFramework/Controller/GeneralControllerAgent.cpp @@ -131,6 +131,34 @@ bool GeneralControllerAgent::_swipe(SwipeParam param) return true; } +bool GeneralControllerAgent::_multi_swipe(std::vector param) +{ + if (!control_unit_) { + LogError << "controller is nullptr" << VAR(control_unit_); + return false; + } + + using ApiParam = MAA_CTRL_UNIT_NS::ControlUnitAPI::SwipeParam; + std::vector api_param; + std::ranges::transform(param, std::back_inserter(api_param), [](const SwipeParam& swipe) { + return ApiParam { + .x1 = swipe.x1, + .y1 = swipe.y1, + .x2 = swipe.x2, + .y2 = swipe.y2, + .duration = swipe.duration, + .starting = swipe.starting, + }; + }); + + if (!control_unit_->multi_swipe(api_param)) { + LogError << "controller swipe failed" << VAR(json::array(param)); + return false; + } + + return true; +} + bool GeneralControllerAgent::_touch_down(TouchParam param) { if (!control_unit_) { diff --git a/source/MaaFramework/Controller/GeneralControllerAgent.h b/source/MaaFramework/Controller/GeneralControllerAgent.h index a9f25e2ab..1802ac85b 100644 --- a/source/MaaFramework/Controller/GeneralControllerAgent.h +++ b/source/MaaFramework/Controller/GeneralControllerAgent.h @@ -24,6 +24,7 @@ class GeneralControllerAgent : public ControllerAgent virtual std::optional _screencap() override; virtual bool _click(ClickParam param) override; virtual bool _swipe(SwipeParam param) override; + virtual bool _multi_swipe(std::vector param) override; virtual bool _touch_down(TouchParam param) override; virtual bool _touch_move(TouchParam param) override; virtual bool _touch_up(TouchParam param) override; diff --git a/source/MaaFramework/Resource/DefaultPipelineMgr.cpp b/source/MaaFramework/Resource/DefaultPipelineMgr.cpp index 1d75b6101..11d600b28 100644 --- a/source/MaaFramework/Resource/DefaultPipelineMgr.cpp +++ b/source/MaaFramework/Resource/DefaultPipelineMgr.cpp @@ -111,6 +111,8 @@ bool DefaultPipelineMgr::parse_action(const json::value& input) { "click", Type::Click }, { "Swipe", Type::Swipe }, { "swipe", Type::Swipe }, + { "MultiSwipe", Type::MultiSwipe }, + { "multiswipe", Type::MultiSwipe }, { "PressKey", Type::Key }, { "presskey", Type::Key }, { "Key", Type::Key }, diff --git a/source/MaaFramework/Resource/PipelineResMgr.cpp b/source/MaaFramework/Resource/PipelineResMgr.cpp index de588e04e..416f1e5ff 100644 --- a/source/MaaFramework/Resource/PipelineResMgr.cpp +++ b/source/MaaFramework/Resource/PipelineResMgr.cpp @@ -1124,6 +1124,8 @@ bool PipelineResMgr::parse_action( { "click", Type::Click }, { "Swipe", Type::Swipe }, { "swipe", Type::Swipe }, + { "MultiSwipe", Type::MultiSwipe }, + { "multiswipe", Type::MultiSwipe }, { "PressKey", Type::Key }, { "presskey", Type::Key }, { "Key", Type::Key }, @@ -1167,6 +1169,16 @@ bool PipelineResMgr::parse_action( return parse_swipe(input, std::get(out_param), same_type ? std::get(parent_param) : default_param); } break; + case Type::MultiSwipe: { + auto default_multi = default_mgr.get_action_param(Type::MultiSwipe); + auto default_single = default_mgr.get_action_param(Type::Swipe); + out_param = default_multi; + return parse_multi_swipe( + input, + std::get(out_param), + same_type ? std::get(parent_param) : default_multi, default_single); + } break; + case Type::Key: { auto default_param = default_mgr.get_action_param(Type::Key); out_param = default_param; @@ -1244,6 +1256,42 @@ bool PipelineResMgr::parse_swipe(const json::value& input, Action::SwipeParam& o return false; } + // only for multi swipe + if (!get_and_check_value(input, "starting", output.starting, default_value.starting)) { + LogError << "failed to get_and_check_value starting" << VAR(input); + return false; + } + + return true; +} + +bool PipelineResMgr::parse_multi_swipe( + const json::value& input, + Action::MultiSwipeParam& output, + const Action::MultiSwipeParam& default_multi, + const Action::SwipeParam& default_single) +{ + auto swipes_opt = input.find("swipes"); + if (!swipes_opt) { + output = default_multi; + return true; + } + + if (!swipes_opt->is_array()) { + LogError << "failed to parse swipes, is not array" << VAR(input); + return false; + } + + output.swipes.clear(); + for (const json::value& swipe : swipes_opt->as_array()) { + Action::SwipeParam res; + if (!parse_swipe(swipe, res, default_single)) { + return false; + } + + output.swipes.emplace_back(std::move(res)); + } + return true; } diff --git a/source/MaaFramework/Resource/PipelineResMgr.h b/source/MaaFramework/Resource/PipelineResMgr.h index 0222099b2..09164017a 100644 --- a/source/MaaFramework/Resource/PipelineResMgr.h +++ b/source/MaaFramework/Resource/PipelineResMgr.h @@ -91,6 +91,11 @@ class PipelineResMgr : public NonCopyable const DefaultPipelineMgr& default_mgr); static bool parse_click(const json::value& input, Action::ClickParam& output, const Action::ClickParam& default_value); static bool parse_swipe(const json::value& input, Action::SwipeParam& output, const Action::SwipeParam& default_value); + static bool parse_multi_swipe( + const json::value& input, + Action::MultiSwipeParam& output, + const Action::MultiSwipeParam& default_mluti, + const Action::SwipeParam& default_single); static bool parse_press_key(const json::value& input, Action::KeyParam& output, const Action::KeyParam& default_value); static bool parse_input_text(const json::value& input, Action::TextParam& output, const Action::TextParam& default_value); static bool parse_app_info(const json::value& input, Action::AppParam& output, const Action::AppParam& default_value); diff --git a/source/MaaFramework/Resource/PipelineTypes.h b/source/MaaFramework/Resource/PipelineTypes.h index 6ba9930ce..50ddcb394 100644 --- a/source/MaaFramework/Resource/PipelineTypes.h +++ b/source/MaaFramework/Resource/PipelineTypes.h @@ -50,6 +50,7 @@ enum class Type DoNothing, Click, Swipe, + MultiSwipe, Key, Text, StartApp, @@ -71,6 +72,12 @@ struct SwipeParam Target end; uint duration = 200; + uint starting = 0; // only for MultiSwipe +}; + +struct MultiSwipeParam +{ + std::vector swipes; }; struct KeyParam @@ -95,7 +102,7 @@ struct CustomParam Target target; }; -using Param = std::variant; +using Param = std::variant; } // namespace Action struct WaitFreezesParam diff --git a/source/MaaFramework/Task/Component/Actuator.cpp b/source/MaaFramework/Task/Component/Actuator.cpp index bb39517e2..dd8dc6406 100644 --- a/source/MaaFramework/Task/Component/Actuator.cpp +++ b/source/MaaFramework/Task/Component/Actuator.cpp @@ -37,6 +37,9 @@ bool Actuator::run(const cv::Rect& reco_hit, MaaRecoId reco_id, const PipelineDa case Type::Swipe: ret = swipe(std::get(pipeline_data.action_param), reco_hit); break; + case Type::MultiSwipe: + ret = multi_swipe(std::get(pipeline_data.action_param), reco_hit); + break; case Type::Key: ret = press_key(std::get(pipeline_data.action_param)); break; @@ -92,6 +95,28 @@ bool Actuator::swipe(const MAA_RES_NS::Action::SwipeParam& param, const cv::Rect return controller()->swipe(begin, end, param.duration); } +bool Actuator::multi_swipe(const MAA_RES_NS::Action::MultiSwipeParam& param, const cv::Rect& box) +{ + if (!controller()) { + LogError << "Controller is null"; + return false; + } + + using CtrlParam = MAA_CTRL_NS::ControllerAgent::SwipeParamWithRect; + + std::vector dst; + + for (const auto& swipe : param.swipes) { + CtrlParam ctrl_param { .r1 = get_target_rect(swipe.begin, box), + .r2 = get_target_rect(swipe.end, box), + .duration = swipe.duration, + .starting = swipe.starting }; + dst.emplace_back(std::move(ctrl_param)); + } + + return controller()->multi_swipe(dst); +} + bool Actuator::press_key(const MAA_RES_NS::Action::KeyParam& param) { if (!controller()) { diff --git a/source/MaaFramework/Task/Component/Actuator.h b/source/MaaFramework/Task/Component/Actuator.h index 60221110d..398e4bd65 100644 --- a/source/MaaFramework/Task/Component/Actuator.h +++ b/source/MaaFramework/Task/Component/Actuator.h @@ -28,6 +28,7 @@ class Actuator private: bool click(const MAA_RES_NS::Action::ClickParam& param, const cv::Rect& box); bool swipe(const MAA_RES_NS::Action::SwipeParam& param, const cv::Rect& box); + bool multi_swipe(const MAA_RES_NS::Action::MultiSwipeParam& param, const cv::Rect& box); bool press_key(const MAA_RES_NS::Action::KeyParam& param); bool input_text(const MAA_RES_NS::Action::TextParam& param); diff --git a/source/MaaWin32ControlUnit/Input/SeizeInput.cpp b/source/MaaWin32ControlUnit/Input/SeizeInput.cpp index 70143c88d..2cc1e9b85 100644 --- a/source/MaaWin32ControlUnit/Input/SeizeInput.cpp +++ b/source/MaaWin32ControlUnit/Input/SeizeInput.cpp @@ -55,7 +55,7 @@ bool SeizeInput::swipe(int x1, int y1, int x2, int y2, int duration) } if (duration <= 0) { LogWarn << "duration out of range" << VAR(duration); - duration = 500; + duration = 200; } LogInfo << VAR(x1) << VAR(y1) << VAR(x2) << VAR(y2) << VAR(duration) << VAR(point1.x) << VAR(point1.y) << VAR(point2.x) << VAR(point2.y) diff --git a/source/MaaWin32ControlUnit/Input/SendMessageInput.cpp b/source/MaaWin32ControlUnit/Input/SendMessageInput.cpp index a35bcad10..a2f98e568 100644 --- a/source/MaaWin32ControlUnit/Input/SendMessageInput.cpp +++ b/source/MaaWin32ControlUnit/Input/SendMessageInput.cpp @@ -34,7 +34,7 @@ bool SendMessageInput::swipe(int x1, int y1, int x2, int y2, int duration) if (duration <= 0) { LogWarn << "duration out of range" << VAR(duration); - duration = 500; + duration = 200; } micro_swipe( diff --git a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp index bcf1ec7ad..c31b1cb8b 100644 --- a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp +++ b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp @@ -124,6 +124,15 @@ bool ControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration) return input_->swipe(x1, y1, x2, y2, duration); } +bool ControlUnitMgr::multi_swipe(const std::vector& swipes) +{ + std::ignore = swipes; + + LogError << "Win32 not support MultiSwipe. If you need it, please raise an issue with us!" << VAR(swipes.size()); + + return false; +} + bool ControlUnitMgr::touch_down(int contact, int x, int y, int pressure) { if (!input_) { diff --git a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h index 301587098..2a3767403 100644 --- a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h +++ b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h @@ -29,6 +29,7 @@ class ControlUnitMgr : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/include/ControlUnit/ControlUnitAPI.h b/source/include/ControlUnit/ControlUnitAPI.h index ae26cbe94..92a51f414 100644 --- a/source/include/ControlUnit/ControlUnitAPI.h +++ b/source/include/ControlUnit/ControlUnitAPI.h @@ -27,6 +27,18 @@ class ControlUnitAPI virtual bool click(int x, int y) = 0; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) = 0; + struct SwipeParam + { + int x1 = 0; + int y1 = 0; + int x2 = 0; + int y2 = 0; + int duration = 0; + int starting = 0; + }; + + virtual bool multi_swipe(const std::vector& swipes) = 0; + virtual bool touch_down(int contact, int x, int y, int pressure) = 0; virtual bool touch_move(int contact, int x, int y, int pressure) = 0; virtual bool touch_up(int contact) = 0; diff --git a/source/include/Utils/MicroControl.hpp b/source/include/Utils/MicroControl.hpp index b4a2e8609..820b416ba 100644 --- a/source/include/Utils/MicroControl.hpp +++ b/source/include/Utils/MicroControl.hpp @@ -2,9 +2,11 @@ #include #include +#include #include #include "Conf/Conf.h" +#include "ControlUnit/ControlUnitAPI.h" MAA_CTRL_UNIT_NS_BEGIN @@ -18,19 +20,20 @@ inline void micro_swipe( std::function touch_move, std::function touch_up) { + constexpr double kInterval = 10; // ms + const std::chrono::milliseconds delay(static_cast(kInterval)); + + const double total_step = duration / kInterval; + const double x_step_len = (x2 - x1) / total_step; + const double y_step_len = (y2 - y1) / total_step; + auto now = std::chrono::steady_clock::now(); touch_down(x1, y1); - constexpr double kInterval = 10; // ms - const double steps = duration / kInterval; - const double x_step_len = (x2 - x1) / steps; - const double y_step_len = (y2 - y1) / steps; - const std::chrono::milliseconds delay(static_cast(kInterval)); - - for (int i = 0; i < steps; ++i) { - int mx = static_cast(x1 + i * x_step_len); - int my = static_cast(y1 + i * y_step_len); + for (int step = 1; step < total_step; ++step) { + int mx = static_cast(x1 + step * x_step_len); + int my = static_cast(y1 + step * y_step_len); std::this_thread::sleep_until(now + delay); @@ -49,4 +52,72 @@ inline void micro_swipe( touch_up(x2, y2); } +inline void micro_multi_swipe( + const std::vector& swipes, + std::function touch_down, + std::function touch_move, + std::function touch_up) +{ + constexpr double kInterval = 10; // ms + const std::chrono::milliseconds delay(static_cast(kInterval)); + + struct Operating + { + double total_step = 0; + double x_step_len = 0; + double y_step_len = 0; + int step = 0; + }; + + std::vector operating(swipes.size()); + + for (size_t i = 0; i < swipes.size(); ++i) { + const ControlUnitAPI::SwipeParam& s = swipes.at(i); + Operating& o = operating.at(i); + o.total_step = s.duration / kInterval; + o.x_step_len = (s.x2 - s.x1) / o.total_step; + o.y_step_len = (s.y2 - s.y1) / o.total_step; + } + + const auto starting = std::chrono::steady_clock::now(); + auto now = starting; + + size_t over_count = 0; + while (over_count < swipes.size()) { + int now_point = static_cast(std::chrono::duration_cast(now - starting).count()); + + for (size_t i = 0; i < swipes.size(); ++i) { + const ControlUnitAPI::SwipeParam& s = swipes.at(i); + if (now_point < s.starting) { + continue; + } + + Operating& o = operating.at(i); + int contact = static_cast(i); + + if (o.step == 0) { + touch_down(contact, s.x1, s.y1); + ++o.step; + } + else if (o.step < o.total_step) { + int mx = static_cast(s.x1 + o.step * o.x_step_len); + int my = static_cast(s.y1 + o.step * o.y_step_len); + touch_move(contact, mx, my); + ++o.step; + } + else if (o.step == o.total_step) { + touch_up(contact, s.x2, s.y2); + ++o.step; + ++over_count; + } + else { // step > total + continue; + } + } + + std::this_thread::sleep_until(now + delay); + now = std::chrono::steady_clock::now(); + } +} + MAA_CTRL_UNIT_NS_END diff --git a/tools/pipeline.schema.json b/tools/pipeline.schema.json index 1a21f3cfb..4285d9908 100644 --- a/tools/pipeline.schema.json +++ b/tools/pipeline.schema.json @@ -263,10 +263,17 @@ ] }, "duration": { - "description": "滑动持续时间，单位毫秒。可选，默认 200", + "description": "滑动持续时间，单位毫秒。可选，默认 200。", "type": "integer", "default": 200 }, + "swipes": { + "description": "多个滑动的数组。必选。数组元素顺序没有影响，只基于 `starting` 确定顺序。", + "type": "array", + "items": { + "$ref": "#/definitions/swipe_type" + } + }, "key": { "description": "要按的键，仅支持 ascii。", "$ref": "#/definitions/keylist", @@ -621,16 +628,24 @@ "action": { "const": "Swipe" } + } + }, + { + "$comment": "MultiSwipe", + "properties": { + "action": { + "const": "MultiSwipe" + } }, "anyOf": [ { "required": [ - "end" + "swipes" ] }, { "required": [ - "end_code" + "swipes_code" ] } ] @@ -923,6 +938,51 @@ } } ] + }, + "swipe_type": { + "type": "object", + "properties": { + "begin": { + "description": "滑动起点。可选，默认 true。", + "$ref": "#/definitions/target_type", + "default": true + }, + "begin_offset": { + "description": "在 begin 的基础上额外移动再作为起点，四个值分别相加。可选，默认 [0, 0, 0, 0]。", + "$ref": "#/definitions/rect", + "default": [ + 0, + 0, + 0, + 0 + ] + }, + "end": { + "description": "滑动终点。必选，默认 true。", + "$ref": "#/definitions/target_type", + "default": true + }, + "end_offset": { + "description": "在 end 的基础上额外移动再作为终点，四个值分别相加。可选，默认 [0, 0, 0, 0]。", + "$ref": "#/definitions/rect", + "default": [ + 0, + 0, + 0, + 0 + ] + }, + "duration": { + "description": "滑动持续时间，单位毫秒。可选，默认 200", + "type": "integer", + "default": 200 + }, + "starting": { + "description": "滑动起始时间，单位毫秒。可选，默认 0 。该滑动会在本 action 中第 `starting` 毫秒才开始。", + "type": "integer", + "default": 0 + } + } } } } \ No newline at end of file