diff --git a/source/MaaAdbControlUnit/Base/UnitBase.h b/source/MaaAdbControlUnit/Base/UnitBase.h index aef28b092..3fdee2be0 100644 --- a/source/MaaAdbControlUnit/Base/UnitBase.h +++ b/source/MaaAdbControlUnit/Base/UnitBase.h @@ -76,6 +76,8 @@ class InputBase , public ControlUnitSink { public: + using SwipeParam = ControlUnitAPI::SwipeParam; + virtual ~InputBase() override = default; virtual bool init() = 0; @@ -83,6 +85,7 @@ class InputBase public: virtual bool click(int x, int y) = 0; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) = 0; + virtual bool multi_swipe(const std::vector& swipes) = 0; virtual bool touch_down(int contact, int x, int y, int pressure) = 0; virtual bool touch_move(int contact, int x, int y, int pressure) = 0; diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp index 8ff6b6a08..7696cefe7 100644 --- a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp +++ b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.cpp @@ -142,6 +142,15 @@ bool MuMuPlayerExtras::swipe(int x1, int y1, int x2, int y2, int duration) return true; } +bool MuMuPlayerExtras::multi_swipe(const std::vector& swipes) +{ + LogInfo << VAR(swipes.size()); + + // TODO + + return false; +} + bool MuMuPlayerExtras::touch_down(int contact, int x, int y, int pressure) { if (!input_event_touch_down_func_) { diff --git a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h index 1c73a41ab..649be05b5 100644 --- a/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h +++ b/source/MaaAdbControlUnit/EmulatorExtras/MuMuPlayerExtras.h @@ -30,6 +30,7 @@ class MuMuPlayerExtras public: // from InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.cpp b/source/MaaAdbControlUnit/Input/AdbShellInput.cpp index db734899f..bcb558d0d 100644 --- a/source/MaaAdbControlUnit/Input/AdbShellInput.cpp +++ b/source/MaaAdbControlUnit/Input/AdbShellInput.cpp @@ -59,6 +59,12 @@ bool AdbShellInput::swipe(int x1, int y1, int x2, int y2, int duration) return output_opt && output_opt->empty(); } +bool AdbShellInput::multi_swipe(const std::vector& swipes) +{ + LogError << "AdbShellInput not supports" << VAR(swipes.size()); + return false; +} + bool AdbShellInput::touch_down(int contact, int x, int y, int pressure) { LogError << "AdbShellInput not supports" << VAR(contact) << VAR(x) << VAR(y) << VAR(pressure); diff --git a/source/MaaAdbControlUnit/Input/AdbShellInput.h b/source/MaaAdbControlUnit/Input/AdbShellInput.h index b162e0e7e..9af3e1f07 100644 --- a/source/MaaAdbControlUnit/Input/AdbShellInput.h +++ b/source/MaaAdbControlUnit/Input/AdbShellInput.h @@ -17,6 +17,7 @@ class AdbShellInput : public InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.cpp b/source/MaaAdbControlUnit/Input/MtouchHelper.cpp index d83c133f8..225a9d0b2 100644 --- a/source/MaaAdbControlUnit/Input/MtouchHelper.cpp +++ b/source/MaaAdbControlUnit/Input/MtouchHelper.cpp @@ -142,6 +142,15 @@ bool MtouchHelper::swipe(int x1, int y1, int x2, int y2, int duration) return ret; } +bool MtouchHelper::multi_swipe(const std::vector& swipes) +{ + LogInfo << VAR(swipes.size()); + + // TODO + + return false; +} + bool MtouchHelper::touch_down(int contact, int x, int y, int pressure) { if (!pipe_ios_) { diff --git a/source/MaaAdbControlUnit/Input/MtouchHelper.h b/source/MaaAdbControlUnit/Input/MtouchHelper.h index 4038b823e..c04cfa448 100644 --- a/source/MaaAdbControlUnit/Input/MtouchHelper.h +++ b/source/MaaAdbControlUnit/Input/MtouchHelper.h @@ -18,6 +18,7 @@ class MtouchHelper : public InputBase public: // from InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp index 5aa55b5cd..c12fe2e23 100644 --- a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp +++ b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.cpp @@ -167,6 +167,16 @@ bool ControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration) return input_->swipe(x1, y1, x2, y2, duration); } +bool ControlUnitMgr::multi_swipe(const std::vector& swipes) +{ + if (!input_) { + LogError << "input_ is null"; + return false; + } + + return input_->multi_swipe(swipes); +} + bool ControlUnitMgr::touch_down(int contact, int x, int y, int pressure) { if (!input_) { diff --git a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h index e788abc86..d8df4cbcd 100644 --- a/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h +++ b/source/MaaAdbControlUnit/Manager/ControlUnitMgr.h @@ -41,6 +41,7 @@ class ControlUnitMgr virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaAdbControlUnit/Manager/InputAgent.cpp b/source/MaaAdbControlUnit/Manager/InputAgent.cpp index e8ff3a256..71a70c6e8 100644 --- a/source/MaaAdbControlUnit/Manager/InputAgent.cpp +++ b/source/MaaAdbControlUnit/Manager/InputAgent.cpp @@ -135,6 +135,16 @@ bool InputAgent::swipe(int x1, int y1, int x2, int y2, int duration) return active_unit_->swipe(x1, y1, x2, y2, duration); } +bool InputAgent::multi_swipe(const std::vector& swipes) +{ + if (!active_unit_) { + LogError << "No available input method" << VAR(active_unit_); + return false; + } + + return active_unit_->multi_swipe(swipes); +} + bool InputAgent::touch_down(int contact, int x, int y, int pressure) { if (!active_unit_) { diff --git a/source/MaaAdbControlUnit/Manager/InputAgent.h b/source/MaaAdbControlUnit/Manager/InputAgent.h index c61d2cbc6..500c7ce84 100644 --- a/source/MaaAdbControlUnit/Manager/InputAgent.h +++ b/source/MaaAdbControlUnit/Manager/InputAgent.h @@ -28,6 +28,7 @@ class InputAgent : public InputBase virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp index 8cc7279d1..ea94187ee 100644 --- a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp +++ b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.cpp @@ -109,6 +109,13 @@ bool CarouselImage::swipe(int x1, int y1, int x2, int y2, int duration) return true; } +bool CarouselImage::multi_swipe(const std::vector& swipes) +{ + std::ignore = swipes; + + return true; +} + bool CarouselImage::touch_down(int contact, int x, int y, int pressure) { std::ignore = contact; diff --git a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h index 07c9df6b1..5fb16dd4a 100644 --- a/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h +++ b/source/MaaDbgControlUnit/CarouselImage/CarouselImage.h @@ -32,6 +32,7 @@ class CarouselImage : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaDbgControlUnit/ReplayRecording/Record.h b/source/MaaDbgControlUnit/ReplayRecording/Record.h index fe3737d16..94c722d0e 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/Record.h +++ b/source/MaaDbgControlUnit/ReplayRecording/Record.h @@ -40,6 +40,7 @@ struct Record int x2 = 0; int y2 = 0; int duration = 0; + int starting = 0; }; struct TouchParam @@ -72,7 +73,7 @@ struct Record }; using Param = std:: - variant; + variant, TouchParam, PressKeyParam, InputTextParam, AppParam, ScreencapParam>; struct Action { @@ -82,6 +83,7 @@ struct Record connect, click, swipe, + multi_swipe, touch_down, touch_move, touch_up, @@ -124,6 +126,9 @@ inline std::ostream& operator<<(std::ostream& os, Record::Action::Type type) case Record::Action::Type::swipe: os << "swipe"; break; + case Record::Action::Type::multi_swipe: + os << "multi_swipe"; + break; case Record::Action::Type::touch_down: os << "touch_down"; break; diff --git a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp index b78235315..9a90916ae 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp +++ b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.cpp @@ -73,11 +73,11 @@ std::optional RecordParser::parse_record(const json::value& record_json, std::string type_str = record_json.get("type", std::string()); static const std::unordered_map kTypeMap = { { "connect", Record::Action::Type::connect }, { "click", Record::Action::Type::click }, - { "swipe", Record::Action::Type::swipe }, { "touch_down", Record::Action::Type::touch_down }, - { "touch_move", Record::Action::Type::touch_move }, { "touch_up", Record::Action::Type::touch_up }, - { "press_key", Record::Action::Type::press_key }, { "input_text", Record::Action::Type::input_text }, - { "screencap", Record::Action::Type::screencap }, { "start_app", Record::Action::Type::start_app }, - { "stop_app", Record::Action::Type::stop_app }, + { "swipe", Record::Action::Type::swipe }, { "multi_swipe", Record::Action::Type::multi_swipe }, + { "touch_down", Record::Action::Type::touch_down }, { "touch_move", Record::Action::Type::touch_move }, + { "touch_up", Record::Action::Type::touch_up }, { "press_key", Record::Action::Type::press_key }, + { "input_text", Record::Action::Type::input_text }, { "screencap", Record::Action::Type::screencap }, + { "start_app", Record::Action::Type::start_app }, { "stop_app", Record::Action::Type::stop_app }, }; auto it = kTypeMap.find(type_str); @@ -99,6 +99,9 @@ std::optional RecordParser::parse_record(const json::value& record_json, case Record::Action::Type::swipe: action_opt = parse_swipe(record_json); break; + case Record::Action::Type::multi_swipe: + action_opt = parse_multi_swipe(record_json); + break; case Record::Action::Type::touch_down: case Record::Action::Type::touch_move: case Record::Action::Type::touch_up: @@ -224,6 +227,26 @@ std::optional RecordParser::parse_swipe(const json::value& record return result; } +std::optional RecordParser::parse_multi_swipe(const json::value& record_json) +{ + auto swipes_opt = record_json.find("swipes"); + if (!swipes_opt) { + LogError << "Failed to find swipes:" << VAR(record_json); + return std::nullopt; + } + + std::vector result; + for (const json::value& swipe : *swipes_opt) { + auto s_opt = parse_swipe(swipe); + if (!s_opt) { + return std::nullopt; + } + auto s = std::get(*std::move(s_opt)); + result.emplace_back(std::move(s)); + } + return result; +} + std::optional RecordParser::parse_touch(const json::value& record_json) { Record::TouchParam result; diff --git a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h index 6e39faaa5..a5103743c 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h +++ b/source/MaaDbgControlUnit/ReplayRecording/RecordParser.h @@ -18,6 +18,7 @@ class RecordParser static std::optional parse_connect(const json::value& record_json); static std::optional parse_click(const json::value& record_json); static std::optional parse_swipe(const json::value& record_json); + static std::optional parse_multi_swipe(const json::value& record_json); static std::optional parse_touch(const json::value& record_json); static std::optional parse_press_key(const json::value& record_json); static std::optional parse_input_text(const json::value& record_json); diff --git a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp index 96d499a08..173397acd 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp +++ b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.cpp @@ -186,6 +186,28 @@ bool ReplayRecording::swipe(int x1, int y1, int x2, int y2, int duration) return record.success; } +bool ReplayRecording::multi_swipe(const std::vector& swipes) +{ + LogInfo << VAR(swipes.size()); + + if (record_index_ >= recording_.records.size()) { + LogError << "record index out of range" << VAR(record_index_) << VAR(recording_.records.size()); + return false; + } + + const Record& record = recording_.records.at(record_index_); + if (record.action.type != Record::Action::Type::multi_swipe) { + LogError << "record type is not swipe" << VAR(record.action.type) << VAR(record.raw_data); + return false; + } + + // TODO: 现在点击的点是随机区域，没法直接检查 + + sleep(record.cost); + ++record_index_; + return record.success; +} + bool ReplayRecording::touch_down(int contact, int x, int y, int pressure) { LogInfo << VAR(contact) << VAR(x) << VAR(y) << VAR(pressure); diff --git a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h index fb0848c12..216543fb2 100644 --- a/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h +++ b/source/MaaDbgControlUnit/ReplayRecording/ReplayRecording.h @@ -33,6 +33,7 @@ class ReplayRecording : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/MaaFramework/Controller/ControllerAgent.cpp b/source/MaaFramework/Controller/ControllerAgent.cpp index 8136df9a1..fea4bf6b2 100644 --- a/source/MaaFramework/Controller/ControllerAgent.cpp +++ b/source/MaaFramework/Controller/ControllerAgent.cpp @@ -204,6 +204,26 @@ bool ControllerAgent::swipe(const cv::Point& p1, const cv::Point& p2, int durati return wait(id) == MaaStatus_Succeeded; } +bool ControllerAgent::multi_swipe(const std::vector& swipes) +{ + std::vector dst_vec; + for (const auto& src : swipes) { + auto p1 = rand_point(src.r1); + auto p2 = rand_point(src.r2); + dst_vec.emplace_back(SwipeParam { + .x1 = p1.x, + .y1 = p1.y, + .x2 = p2.x, + .y2 = p2.y, + .duration = static_cast(src.duration), + .starting = static_cast(src.starting), + }); + } + + auto id = post_multi_swipe_impl(dst_vec); + return wait(id) == MaaStatus_Succeeded; +} + bool ControllerAgent::press_key(int keycode) { auto id = post_press_key_impl(keycode); @@ -280,6 +300,17 @@ MaaCtrlId ControllerAgent::post_swipe_impl(int x1, int y1, int x2, int y2, int d return post({ .type = Action::Type::swipe, .param = std::move(param) }); } +MaaCtrlId ControllerAgent::post_multi_swipe_impl(const std::vector& swipes) +{ + std::vector dst = swipes; + for (SwipeParam& d : dst) { + std::tie(d.x1, d.y1) = preproc_touch_point(d.x1, d.y1); + std::tie(d.x2, d.y2) = preproc_touch_point(d.x2, d.y2); + } + + return post({ .type = Action::Type::multi_swipe, .param = std::move(dst) }); +} + MaaCtrlId ControllerAgent::post_press_key_impl(int keycode) { PressKeyParam param { .keycode = keycode }; @@ -364,11 +395,8 @@ bool ControllerAgent::handle_click(const ClickParam& param) bool ret = _click(param); if (recording()) { - json::value info = { - { "type", "click" }, - { "x", param.x }, - { "y", param.y }, - }; + json::value info = param; + info |= { { "type", "click" } }; append_recording(std::move(info), start_time, ret); } @@ -385,10 +413,25 @@ bool ControllerAgent::handle_swipe(const SwipeParam& param) bool ret = _swipe(param); if (recording()) { - json::value info = { - { "type", "swipe" }, { "x1", param.x1 }, { "y1", param.y1 }, - { "x2", param.x2 }, { "y2", param.y2 }, { "duration", param.duration }, - }; + json::value info = param; + info |= { { "type", "swipe" } }; + append_recording(std::move(info), start_time, ret); + } + + return ret; +} + +bool ControllerAgent::handle_multi_swipe(const std::vector& param) +{ + std::chrono::steady_clock::time_point start_time; + if (recording()) { + start_time = std::chrono::steady_clock::now(); + } + + bool ret = _multi_swipe(param); + + if (recording()) { + json::value info = { { "type", "multi_swipe" }, { "swipes", json::array(param) } }; append_recording(std::move(info), start_time, ret); } @@ -405,9 +448,8 @@ bool ControllerAgent::handle_touch_down(const TouchParam& param) bool ret = _touch_down(param); if (recording()) { - json::value info = { - { "type", "touch_down" }, { "contact", param.contact }, { "x", param.x }, { "y", param.y }, { "pressure", param.pressure }, - }; + json::value info = param; + info |= { { "type", "touch_down" } }; append_recording(std::move(info), start_time, ret); } @@ -424,9 +466,8 @@ bool ControllerAgent::handle_touch_move(const TouchParam& param) bool ret = _touch_move(param); if (recording()) { - json::value info = { - { "type", "touch_move" }, { "contact", param.contact }, { "x", param.x }, { "y", param.y }, { "pressure", param.pressure }, - }; + json::value info = param; + info |= { { "type", "touch_move" } }; append_recording(std::move(info), start_time, ret); } @@ -443,10 +484,8 @@ bool ControllerAgent::handle_touch_up(const TouchParam& param) bool ret = _touch_up(param); if (recording()) { - json::value info = { - { "type", "touch_up" }, - { "contact", param.contact }, - }; + json::value info = param; + info |= { { "type", "touch_up" } }; append_recording(std::move(info), start_time, ret); } @@ -463,10 +502,8 @@ bool ControllerAgent::handle_press_key(const PressKeyParam& param) bool ret = _press_key(param); if (recording()) { - json::value info = { - { "type", "press_key" }, - { "keycode", param.keycode }, - }; + json::value info = param; + info |= { { "type", "press_key" } }; append_recording(std::move(info), start_time, ret); } @@ -483,10 +520,8 @@ bool ControllerAgent::handle_input_text(const InputTextParam& param) bool ret = _input_text(param); if (recording()) { - json::value info = { - { "type", "input_text" }, - { "input_text", param.text }, - }; + json::value info = param; + info |= { { "type", "input_text" } }; append_recording(std::move(info), start_time, ret); } @@ -534,10 +569,8 @@ bool ControllerAgent::handle_start_app(const AppParam& param) bool ret = _start_app(param); if (recording()) { - json::value info = { - { "type", "start_app" }, - { "package", param.package }, - }; + json::value info = param; + info |= { { "type", "start_app" } }; append_recording(std::move(info), start_time, ret); } return ret; @@ -553,10 +586,8 @@ bool ControllerAgent::handle_stop_app(const AppParam& param) bool ret = _stop_app(param); if (recording()) { - json::value info = { - { "type", "stop_app" }, - { "package", param.package }, - }; + json::value info = param; + info |= { { "type", "stop_app" } }; append_recording(std::move(info), start_time, ret); } return ret; @@ -661,6 +692,9 @@ bool ControllerAgent::run_action(typename AsyncRunner::Id id, Action act case Action::Type::swipe: ret = handle_swipe(std::get(action.param)); break; + case Action::Type::multi_swipe: + ret = handle_multi_swipe(std::get>(action.param)); + break; case Action::Type::touch_down: ret = handle_touch_down(std::get(action.param)); diff --git a/source/MaaFramework/Controller/ControllerAgent.h b/source/MaaFramework/Controller/ControllerAgent.h index b05778cea..018349a86 100644 --- a/source/MaaFramework/Controller/ControllerAgent.h +++ b/source/MaaFramework/Controller/ControllerAgent.h @@ -6,6 +6,8 @@ #include #include +#include + #include "API/MaaTypes.h" #include "Base/AsyncRunner.hpp" #include "Utils/MessageNotifier.hpp" @@ -21,6 +23,8 @@ struct ClickParam { int x = 0; int y = 0; + + MEO_JSONIZATION(x, y); }; struct SwipeParam @@ -30,6 +34,9 @@ struct SwipeParam int x2 = 0; int y2 = 0; int duration = 0; + int starting = 0; + + MEO_JSONIZATION(x1, y1, x2, y2, duration, starting); }; struct TouchParam @@ -38,24 +45,33 @@ struct TouchParam int x = 0; int y = 0; int pressure = 0; + + MEO_JSONIZATION(contact, x, y, pressure); }; struct PressKeyParam { int keycode = 0; + + MEO_JSONIZATION(keycode); }; struct InputTextParam { std::string text; + + MEO_JSONIZATION(text); }; struct AppParam { std::string package; + + MEO_JSONIZATION(package); }; -using Param = std::variant; +using Param = + std::variant, TouchParam, PressKeyParam, InputTextParam, AppParam>; struct Action { @@ -65,6 +81,7 @@ struct Action connect, click, swipe, + multi_swipe, touch_down, touch_move, touch_up, @@ -117,6 +134,17 @@ class ControllerAgent : public MaaController bool click(const cv::Point& p); bool swipe(const cv::Rect& r1, const cv::Rect& r2, int duration); bool swipe(const cv::Point& p1, const cv::Point& p2, int duration); + + struct SwipeParamWithRect + { + cv::Rect r1 {}; + cv::Rect r2 {}; + uint duration = 0; + uint starting = 0; + }; + + bool multi_swipe(const std::vector& swipes); + bool press_key(int keycode); bool input_text(const std::string& text); cv::Mat screencap(); @@ -132,6 +160,7 @@ class ControllerAgent : public MaaController virtual std::optional _screencap() = 0; virtual bool _click(ClickParam param) = 0; virtual bool _swipe(SwipeParam param) = 0; + virtual bool _multi_swipe(std::vector param) = 0; virtual bool _touch_down(TouchParam param) = 0; virtual bool _touch_move(TouchParam param) = 0; virtual bool _touch_up(TouchParam param) = 0; @@ -145,6 +174,7 @@ class ControllerAgent : public MaaController MaaCtrlId post_connection_impl(); MaaCtrlId post_click_impl(int x, int y); MaaCtrlId post_swipe_impl(int x1, int y1, int x2, int y2, int duration); + MaaCtrlId post_multi_swipe_impl(const std::vector& swipes); MaaCtrlId post_press_key_impl(int keycode); MaaCtrlId post_input_text_impl(const std::string& text); MaaCtrlId post_start_app_impl(const std::string& text); @@ -158,6 +188,7 @@ class ControllerAgent : public MaaController bool handle_connect(); bool handle_click(const ClickParam& param); bool handle_swipe(const SwipeParam& param); + bool handle_multi_swipe(const std::vector& param); bool handle_touch_down(const TouchParam& param); bool handle_touch_move(const TouchParam& param); bool handle_touch_up(const TouchParam& param); diff --git a/source/MaaFramework/Controller/CustomControllerAgent.cpp b/source/MaaFramework/Controller/CustomControllerAgent.cpp index 33340f298..e4c6ca8ef 100644 --- a/source/MaaFramework/Controller/CustomControllerAgent.cpp +++ b/source/MaaFramework/Controller/CustomControllerAgent.cpp @@ -121,6 +121,15 @@ bool CustomControllerAgent::_swipe(SwipeParam param) return controller_->swipe(param.x1, param.y1, param.x2, param.y2, param.duration, controller_arg_); } +bool CustomControllerAgent::_multi_swipe(std::vector param) +{ + std::ignore = param; + + LogError << "CustomController not support MulitSwipe now, We welcome your PR to implement it!" << VAR(param); + + return false; +} + bool CustomControllerAgent::_touch_down(TouchParam param) { LogFunc << VAR_VOIDP(controller_) << VAR_VOIDP(controller_->touch_down) << VAR(param.contact) << VAR(param.x) << VAR(param.y) diff --git a/source/MaaFramework/Controller/CustomControllerAgent.h b/source/MaaFramework/Controller/CustomControllerAgent.h index 5d49b5a7a..21ea2d7bb 100644 --- a/source/MaaFramework/Controller/CustomControllerAgent.h +++ b/source/MaaFramework/Controller/CustomControllerAgent.h @@ -23,6 +23,7 @@ class CustomControllerAgent : public ControllerAgent virtual std::optional _screencap() override; virtual bool _click(ClickParam param) override; virtual bool _swipe(SwipeParam param) override; + virtual bool _multi_swipe(std::vector param) override; virtual bool _touch_down(TouchParam param) override; virtual bool _touch_move(TouchParam param) override; virtual bool _touch_up(TouchParam param) override; diff --git a/source/MaaFramework/Controller/GeneralControllerAgent.cpp b/source/MaaFramework/Controller/GeneralControllerAgent.cpp index a20a3c10d..83cfa2fb7 100644 --- a/source/MaaFramework/Controller/GeneralControllerAgent.cpp +++ b/source/MaaFramework/Controller/GeneralControllerAgent.cpp @@ -131,6 +131,34 @@ bool GeneralControllerAgent::_swipe(SwipeParam param) return true; } +bool GeneralControllerAgent::_multi_swipe(std::vector param) +{ + if (!control_unit_) { + LogError << "controller is nullptr" << VAR(control_unit_); + return false; + } + + using ApiParam = MAA_CTRL_UNIT_NS::ControlUnitAPI::SwipeParam; + std::vector api_param; + std::ranges::transform(param, std::back_inserter(api_param), [](const SwipeParam& swipe) { + return ApiParam { + .x1 = swipe.x1, + .y1 = swipe.y1, + .x2 = swipe.x2, + .y2 = swipe.y2, + .duration = swipe.duration, + .starting = swipe.starting, + }; + }); + + if (!control_unit_->multi_swipe(api_param)) { + LogError << "controller swipe failed" << VAR(json::array(param)); + return false; + } + + return true; +} + bool GeneralControllerAgent::_touch_down(TouchParam param) { if (!control_unit_) { diff --git a/source/MaaFramework/Controller/GeneralControllerAgent.h b/source/MaaFramework/Controller/GeneralControllerAgent.h index a9f25e2ab..1802ac85b 100644 --- a/source/MaaFramework/Controller/GeneralControllerAgent.h +++ b/source/MaaFramework/Controller/GeneralControllerAgent.h @@ -24,6 +24,7 @@ class GeneralControllerAgent : public ControllerAgent virtual std::optional _screencap() override; virtual bool _click(ClickParam param) override; virtual bool _swipe(SwipeParam param) override; + virtual bool _multi_swipe(std::vector param) override; virtual bool _touch_down(TouchParam param) override; virtual bool _touch_move(TouchParam param) override; virtual bool _touch_up(TouchParam param) override; diff --git a/source/MaaFramework/Resource/DefaultPipelineMgr.cpp b/source/MaaFramework/Resource/DefaultPipelineMgr.cpp index 1d75b6101..11d600b28 100644 --- a/source/MaaFramework/Resource/DefaultPipelineMgr.cpp +++ b/source/MaaFramework/Resource/DefaultPipelineMgr.cpp @@ -111,6 +111,8 @@ bool DefaultPipelineMgr::parse_action(const json::value& input) { "click", Type::Click }, { "Swipe", Type::Swipe }, { "swipe", Type::Swipe }, + { "MultiSwipe", Type::MultiSwipe }, + { "multiswipe", Type::MultiSwipe }, { "PressKey", Type::Key }, { "presskey", Type::Key }, { "Key", Type::Key }, diff --git a/source/MaaFramework/Resource/PipelineResMgr.cpp b/source/MaaFramework/Resource/PipelineResMgr.cpp index de588e04e..416f1e5ff 100644 --- a/source/MaaFramework/Resource/PipelineResMgr.cpp +++ b/source/MaaFramework/Resource/PipelineResMgr.cpp @@ -1124,6 +1124,8 @@ bool PipelineResMgr::parse_action( { "click", Type::Click }, { "Swipe", Type::Swipe }, { "swipe", Type::Swipe }, + { "MultiSwipe", Type::MultiSwipe }, + { "multiswipe", Type::MultiSwipe }, { "PressKey", Type::Key }, { "presskey", Type::Key }, { "Key", Type::Key }, @@ -1167,6 +1169,16 @@ bool PipelineResMgr::parse_action( return parse_swipe(input, std::get(out_param), same_type ? std::get(parent_param) : default_param); } break; + case Type::MultiSwipe: { + auto default_multi = default_mgr.get_action_param(Type::MultiSwipe); + auto default_single = default_mgr.get_action_param(Type::Swipe); + out_param = default_multi; + return parse_multi_swipe( + input, + std::get(out_param), + same_type ? std::get(parent_param) : default_multi, default_single); + } break; + case Type::Key: { auto default_param = default_mgr.get_action_param(Type::Key); out_param = default_param; @@ -1244,6 +1256,42 @@ bool PipelineResMgr::parse_swipe(const json::value& input, Action::SwipeParam& o return false; } + // only for multi swipe + if (!get_and_check_value(input, "starting", output.starting, default_value.starting)) { + LogError << "failed to get_and_check_value starting" << VAR(input); + return false; + } + + return true; +} + +bool PipelineResMgr::parse_multi_swipe( + const json::value& input, + Action::MultiSwipeParam& output, + const Action::MultiSwipeParam& default_multi, + const Action::SwipeParam& default_single) +{ + auto swipes_opt = input.find("swipes"); + if (!swipes_opt) { + output = default_multi; + return true; + } + + if (!swipes_opt->is_array()) { + LogError << "failed to parse swipes, is not array" << VAR(input); + return false; + } + + output.swipes.clear(); + for (const json::value& swipe : swipes_opt->as_array()) { + Action::SwipeParam res; + if (!parse_swipe(swipe, res, default_single)) { + return false; + } + + output.swipes.emplace_back(std::move(res)); + } + return true; } diff --git a/source/MaaFramework/Resource/PipelineResMgr.h b/source/MaaFramework/Resource/PipelineResMgr.h index 0222099b2..09164017a 100644 --- a/source/MaaFramework/Resource/PipelineResMgr.h +++ b/source/MaaFramework/Resource/PipelineResMgr.h @@ -91,6 +91,11 @@ class PipelineResMgr : public NonCopyable const DefaultPipelineMgr& default_mgr); static bool parse_click(const json::value& input, Action::ClickParam& output, const Action::ClickParam& default_value); static bool parse_swipe(const json::value& input, Action::SwipeParam& output, const Action::SwipeParam& default_value); + static bool parse_multi_swipe( + const json::value& input, + Action::MultiSwipeParam& output, + const Action::MultiSwipeParam& default_mluti, + const Action::SwipeParam& default_single); static bool parse_press_key(const json::value& input, Action::KeyParam& output, const Action::KeyParam& default_value); static bool parse_input_text(const json::value& input, Action::TextParam& output, const Action::TextParam& default_value); static bool parse_app_info(const json::value& input, Action::AppParam& output, const Action::AppParam& default_value); diff --git a/source/MaaFramework/Resource/PipelineTypes.h b/source/MaaFramework/Resource/PipelineTypes.h index 6ba9930ce..50ddcb394 100644 --- a/source/MaaFramework/Resource/PipelineTypes.h +++ b/source/MaaFramework/Resource/PipelineTypes.h @@ -50,6 +50,7 @@ enum class Type DoNothing, Click, Swipe, + MultiSwipe, Key, Text, StartApp, @@ -71,6 +72,12 @@ struct SwipeParam Target end; uint duration = 200; + uint starting = 0; // only for MultiSwipe +}; + +struct MultiSwipeParam +{ + std::vector swipes; }; struct KeyParam @@ -95,7 +102,7 @@ struct CustomParam Target target; }; -using Param = std::variant; +using Param = std::variant; } // namespace Action struct WaitFreezesParam diff --git a/source/MaaFramework/Task/Component/Actuator.cpp b/source/MaaFramework/Task/Component/Actuator.cpp index bb39517e2..dd8dc6406 100644 --- a/source/MaaFramework/Task/Component/Actuator.cpp +++ b/source/MaaFramework/Task/Component/Actuator.cpp @@ -37,6 +37,9 @@ bool Actuator::run(const cv::Rect& reco_hit, MaaRecoId reco_id, const PipelineDa case Type::Swipe: ret = swipe(std::get(pipeline_data.action_param), reco_hit); break; + case Type::MultiSwipe: + ret = multi_swipe(std::get(pipeline_data.action_param), reco_hit); + break; case Type::Key: ret = press_key(std::get(pipeline_data.action_param)); break; @@ -92,6 +95,28 @@ bool Actuator::swipe(const MAA_RES_NS::Action::SwipeParam& param, const cv::Rect return controller()->swipe(begin, end, param.duration); } +bool Actuator::multi_swipe(const MAA_RES_NS::Action::MultiSwipeParam& param, const cv::Rect& box) +{ + if (!controller()) { + LogError << "Controller is null"; + return false; + } + + using CtrlParam = MAA_CTRL_NS::ControllerAgent::SwipeParamWithRect; + + std::vector dst; + + for (const auto& swipe : param.swipes) { + CtrlParam ctrl_param { .r1 = get_target_rect(swipe.begin, box), + .r2 = get_target_rect(swipe.end, box), + .duration = swipe.duration, + .starting = swipe.starting }; + dst.emplace_back(std::move(ctrl_param)); + } + + return controller()->multi_swipe(dst); +} + bool Actuator::press_key(const MAA_RES_NS::Action::KeyParam& param) { if (!controller()) { diff --git a/source/MaaFramework/Task/Component/Actuator.h b/source/MaaFramework/Task/Component/Actuator.h index 60221110d..398e4bd65 100644 --- a/source/MaaFramework/Task/Component/Actuator.h +++ b/source/MaaFramework/Task/Component/Actuator.h @@ -28,6 +28,7 @@ class Actuator private: bool click(const MAA_RES_NS::Action::ClickParam& param, const cv::Rect& box); bool swipe(const MAA_RES_NS::Action::SwipeParam& param, const cv::Rect& box); + bool multi_swipe(const MAA_RES_NS::Action::MultiSwipeParam& param, const cv::Rect& box); bool press_key(const MAA_RES_NS::Action::KeyParam& param); bool input_text(const MAA_RES_NS::Action::TextParam& param); diff --git a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp index bcf1ec7ad..c31b1cb8b 100644 --- a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp +++ b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.cpp @@ -124,6 +124,15 @@ bool ControlUnitMgr::swipe(int x1, int y1, int x2, int y2, int duration) return input_->swipe(x1, y1, x2, y2, duration); } +bool ControlUnitMgr::multi_swipe(const std::vector& swipes) +{ + std::ignore = swipes; + + LogError << "Win32 not support MultiSwipe. If you need it, please raise an issue with us!" << VAR(swipes.size()); + + return false; +} + bool ControlUnitMgr::touch_down(int contact, int x, int y, int pressure) { if (!input_) { diff --git a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h index 301587098..2a3767403 100644 --- a/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h +++ b/source/MaaWin32ControlUnit/Manager/ControlUnitMgr.h @@ -29,6 +29,7 @@ class ControlUnitMgr : public ControlUnitAPI virtual bool click(int x, int y) override; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) override; + virtual bool multi_swipe(const std::vector& swipes) override; virtual bool touch_down(int contact, int x, int y, int pressure) override; virtual bool touch_move(int contact, int x, int y, int pressure) override; diff --git a/source/include/ControlUnit/ControlUnitAPI.h b/source/include/ControlUnit/ControlUnitAPI.h index ae26cbe94..92a51f414 100644 --- a/source/include/ControlUnit/ControlUnitAPI.h +++ b/source/include/ControlUnit/ControlUnitAPI.h @@ -27,6 +27,18 @@ class ControlUnitAPI virtual bool click(int x, int y) = 0; virtual bool swipe(int x1, int y1, int x2, int y2, int duration) = 0; + struct SwipeParam + { + int x1 = 0; + int y1 = 0; + int x2 = 0; + int y2 = 0; + int duration = 0; + int starting = 0; + }; + + virtual bool multi_swipe(const std::vector& swipes) = 0; + virtual bool touch_down(int contact, int x, int y, int pressure) = 0; virtual bool touch_move(int contact, int x, int y, int pressure) = 0; virtual bool touch_up(int contact) = 0;