diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e3de84729..90f1719b0 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -111,11 +111,16 @@ jobs: pytest tests -n auto --dist loadfile pytest --ignore=tentacles/Trading/Exchange tentacles -n auto --dist loadfile else - cd ${{ matrix.package }} - if [ "${{ matrix.package }}" = "packages/tentacles_manager" ] || [ "${{ matrix.package }}" = "packages/node" ]; then - pytest tests + if [ "${{ matrix.package }}" = "packages/node" ]; then + echo "Running node tests from root dir to allow tentacles import" + pytest packages/node/tests else - pytest tests -n auto --dist loadfile + cd ${{ matrix.package }} + if [ "${{ matrix.package }}" = "packages/tentacles_manager" ]; then + pytest tests + else + pytest tests -n auto --dist loadfile + fi fi fi env: diff --git a/packages/commons/octobot_commons/asyncio_tools.py b/packages/commons/octobot_commons/asyncio_tools.py index 84abc8d71..09783c4e4 100644 --- a/packages/commons/octobot_commons/asyncio_tools.py +++ b/packages/commons/octobot_commons/asyncio_tools.py @@ -14,8 +14,11 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import asyncio +import contextlib +import time import traceback import concurrent.futures +import typing import octobot_commons.constants as constants import octobot_commons.logging as logging_util @@ -117,6 +120,27 @@ async def gather_waiting_for_all_before_raising(*coros): return maybe_exceptions +@contextlib.contextmanager +def logged_waiter(self, name: str, sleep_time: float = 30) -> typing.Generator[None, None, None]: + """ + Periodically log the time elapsed since the start of the waiter + """ + async def _waiter() -> None: + t0 = time.time() + try: + await asyncio.sleep(sleep_time) + self.logger.info(f"{name} is still processing [{time.time() - t0:.2f} seconds] ...") + except asyncio.CancelledError: + pass + task = None + try: + task = asyncio.create_task(_waiter()) + yield + finally: + if task is not None and not task.done(): + task.cancel() + + class RLock(asyncio.Lock): """ Async Lock implementing reentrancy diff --git a/packages/commons/octobot_commons/constants.py b/packages/commons/octobot_commons/constants.py index 0c51cd3c8..aa4d016d3 100644 --- a/packages/commons/octobot_commons/constants.py +++ b/packages/commons/octobot_commons/constants.py @@ -249,6 +249,7 @@ def parse_boolean_environment_var(env_key: str, default_value: str) -> bool: # DSL interpreter BASE_OPERATORS_LIBRARY = "base" CONTEXTUAL_OPERATORS_LIBRARY = "contextual" +UNRESOLVED_PARAMETER_PLACEHOLDER = "UNRESOLVED_PARAMETER" # Logging EXCEPTION_DESC = "exception_desc" diff --git a/packages/commons/octobot_commons/dataclasses/flexible_dataclass.py b/packages/commons/octobot_commons/dataclasses/flexible_dataclass.py index 93496ca4e..e4f9a5c06 100644 --- a/packages/commons/octobot_commons/dataclasses/flexible_dataclass.py +++ b/packages/commons/octobot_commons/dataclasses/flexible_dataclass.py @@ -25,7 +25,7 @@ @dataclasses.dataclass class FlexibleDataclass: - _class_field_cache: typing.ClassVar[dict] = {} + _class_field_cache: typing.ClassVar[dict] = dataclasses.field(default={}, repr=False) """ Implements from_dict which can be called to instantiate a new instance of this class from a dict. Using from_dict ignores any additional key from the given dict that is not defined as a dataclass field. diff --git a/packages/commons/octobot_commons/dsl_interpreter/__init__.py b/packages/commons/octobot_commons/dsl_interpreter/__init__.py index a685c1238..0f61588f2 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/__init__.py +++ b/packages/commons/octobot_commons/dsl_interpreter/__init__.py @@ -45,6 +45,8 @@ from octobot_commons.dsl_interpreter.parameters_util import ( format_parameter_value, resove_operator_params, + apply_resolved_parameter_value, + has_unresolved_parameters, ) from octobot_commons.dsl_interpreter.dsl_call_result import DSLCallResult @@ -67,5 +69,7 @@ "InterpreterDependency", "format_parameter_value", "resove_operator_params", + "apply_resolved_parameter_value", "DSLCallResult", + "has_unresolved_parameters", ] diff --git a/packages/commons/octobot_commons/dsl_interpreter/dsl_call_result.py b/packages/commons/octobot_commons/dsl_interpreter/dsl_call_result.py index 568058a86..aadaca7d2 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/dsl_call_result.py +++ b/packages/commons/octobot_commons/dsl_interpreter/dsl_call_result.py @@ -16,11 +16,22 @@ import dataclasses import typing + import octobot_commons.dataclasses -import octobot_commons.errors + @dataclasses.dataclass class DSLCallResult(octobot_commons.dataclasses.FlexibleDataclass): + """ + Stores a DSL call result alongside its statement (and error if any) + """ statement: str result: typing.Optional[typing.Any] = None error: typing.Optional[str] = None + + def succeeded(self) -> bool: + """ + Check if the DSL call succeeded + :return: True if the DSL call succeeded, False otherwise + """ + return self.error is None diff --git a/packages/commons/octobot_commons/dsl_interpreter/parameters_util.py b/packages/commons/octobot_commons/dsl_interpreter/parameters_util.py index 00b2fc6ef..e5e6cb9d3 100644 --- a/packages/commons/octobot_commons/dsl_interpreter/parameters_util.py +++ b/packages/commons/octobot_commons/dsl_interpreter/parameters_util.py @@ -18,6 +18,7 @@ import octobot_commons.dsl_interpreter.operator as dsl_interpreter_operator import octobot_commons.errors +import octobot_commons.constants def format_parameter_value(value: typing.Any) -> str: # pylint: disable=too-many-return-statements @@ -111,3 +112,23 @@ def resolve_operator_args_and_kwargs( ) return merged_args, remaining_kwargs + + +def apply_resolved_parameter_value(script: str, parameter: str, value: typing.Any): + """ + Apply a resolved parameter value to a DSL script. + """ + to_replace = f"{parameter}={octobot_commons.constants.UNRESOLVED_PARAMETER_PLACEHOLDER}" + if to_replace not in script: + raise octobot_commons.errors.ResolvedParameterNotFoundError( + f"Parameter {parameter} not found in script: {script}" + ) + new_value = f"{parameter}={format_parameter_value(value)}" + return script.replace(to_replace, new_value) + + +def has_unresolved_parameters(script: str) -> bool: + """ + Check if a DSL script has unresolved parameters. + """ + return octobot_commons.constants.UNRESOLVED_PARAMETER_PLACEHOLDER in script diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index f49bf08ed..c8be287fc 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -165,6 +165,12 @@ class InvalidParameterFormatError(InvalidParametersError): """ +class ResolvedParameterNotFoundError(DSLInterpreterError): + """ + Raised when a resolved parameter is not found in the script + """ + + class ErrorStatementEncountered(DSLInterpreterError): """ Raised when a error statement is encountered when executing a script diff --git a/packages/commons/octobot_commons/time_frame_manager.py b/packages/commons/octobot_commons/time_frame_manager.py index 158a46d2a..9cb4fe973 100644 --- a/packages/commons/octobot_commons/time_frame_manager.py +++ b/packages/commons/octobot_commons/time_frame_manager.py @@ -13,6 +13,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import typing + import octobot_commons.constants as constants import octobot_commons.logging as logging_util import octobot_commons.enums as enums @@ -99,7 +101,10 @@ def get_previous_time_frame(config_time_frames, time_frame, origin_time_frame): return origin_time_frame -def find_min_time_frame(time_frames, min_time_frame=None): +def find_min_time_frame( + time_frames: list[typing.Union[str, enums.TimeFrames]], + min_time_frame: typing.Optional[str] = None +) -> enums.TimeFrames: """ Find the minimum time frame :param time_frames: the time frame list diff --git a/packages/commons/tests/dsl_interpreter/test_parameters_util.py b/packages/commons/tests/dsl_interpreter/test_parameters_util.py index e8863fe0f..45bfcb7b5 100644 --- a/packages/commons/tests/dsl_interpreter/test_parameters_util.py +++ b/packages/commons/tests/dsl_interpreter/test_parameters_util.py @@ -16,6 +16,7 @@ import mock import pytest +import octobot_commons.constants as constants import octobot_commons.dsl_interpreter.parameters_util as parameters_util import octobot_commons.dsl_interpreter.operator_parameter as operator_parameter import octobot_commons.errors as commons_errors @@ -282,3 +283,80 @@ def test_partial_params_allowed(self): ) assert args == [1] assert kwargs == {} + + +class TestApplyResolvedParameterValue: + def test_replaces_single_parameter_with_int(self): + script = f"op(x=1, y={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "y", 42) + assert result == "op(x=1, y=42)" + + def test_replaces_single_parameter_with_string(self): + script = f"op(name={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "name", "hello") + assert result == "op(name='hello')" + + def test_replaces_single_parameter_with_bool(self): + script = f"op(flag={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "flag", True) + assert result == "op(flag=True)" + + def test_replaces_single_parameter_with_list(self): + script = f"op(items={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "items", [1, 2]) + assert result == "op(items=[1, 2])" + + def test_replaces_single_parameter_with_dict(self): + script = f"op(config={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value( + script, "config", {"a": 1} + ) + assert result == "op(config={'a': 1})" + + def test_replaces_single_parameter_with_none(self): + script = f"op(val={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "val", None) + assert result == "op(val=None)" + + def test_raises_when_parameter_not_found(self): + script = "op(x=1, y=2)" + with pytest.raises(commons_errors.ResolvedParameterNotFoundError, match="Parameter z not found in script"): + parameters_util.apply_resolved_parameter_value(script, "z", 42) + + def test_raises_when_placeholder_not_in_script_for_parameter(self): + script = f"op(x={constants.UNRESOLVED_PARAMETER_PLACEHOLDER}, y=2)" + with pytest.raises(commons_errors.ResolvedParameterNotFoundError, match="Parameter z not found in script"): + parameters_util.apply_resolved_parameter_value(script, "z", 42) + + def test_replaces_only_exact_parameter_pattern(self): + script = f"op(a=1, b={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + result = parameters_util.apply_resolved_parameter_value(script, "b", 100) + assert result == "op(a=1, b=100)" + # Ensure 'a' was not touched + assert "a=1" in result + + +class TestHasUnresolvedParameters: + def test_returns_true_when_placeholder_present(self): + script = f"op(x={constants.UNRESOLVED_PARAMETER_PLACEHOLDER})" + assert parameters_util.has_unresolved_parameters(script) is True + + def test_returns_true_when_multiple_placeholders(self): + placeholder = constants.UNRESOLVED_PARAMETER_PLACEHOLDER + script = f"op(a={placeholder}, b={placeholder})" + assert parameters_util.has_unresolved_parameters(script) is True + + def test_returns_false_when_no_placeholder(self): + script = "op(x=1, y=2)" + assert parameters_util.has_unresolved_parameters(script) is False + + def test_returns_false_for_empty_script(self): + assert parameters_util.has_unresolved_parameters("") is False + + def test_returns_true_when_placeholder_part_of_larger_string(self): + script = f"op(x='prefix_{constants.UNRESOLVED_PARAMETER_PLACEHOLDER}_suffix')" + assert parameters_util.has_unresolved_parameters(script) is True + + def test_returns_true_when_placeholder_alone(self): + script = constants.UNRESOLVED_PARAMETER_PLACEHOLDER + assert parameters_util.has_unresolved_parameters(script) is True diff --git a/packages/commons/tests/test_asyncio_tools.py b/packages/commons/tests/test_asyncio_tools.py index f604b2355..91494eac0 100644 --- a/packages/commons/tests/test_asyncio_tools.py +++ b/packages/commons/tests/test_asyncio_tools.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import asyncio +import mock import pytest import octobot_commons.asyncio_tools as asyncio_tools @@ -255,5 +256,51 @@ async def test_RLock_error_setup_2(): pass +async def test_logged_waiter_cancels_task_on_quick_exit(): + mock_self = mock.Mock() + mock_self.logger = mock.Mock() + + with asyncio_tools.logged_waiter(mock_self, "quick op", sleep_time=30): + await asyncio.sleep(0.001) + + mock_self.logger.info.assert_not_called() + + +async def test_logged_waiter_logs_when_body_runs_long(): + mock_self = mock.Mock() + mock_self.logger = mock.Mock() + + with asyncio_tools.logged_waiter(mock_self, "long op", sleep_time=0.05): + await asyncio.sleep(0.15) + + assert mock_self.logger.info.call_count >= 1 + call_args = mock_self.logger.info.call_args[0][0] + assert "long op" in call_args + assert "is still processing" in call_args + + +async def test_logged_waiter_cancels_on_exception(): + mock_self = mock.Mock() + mock_self.logger = mock.Mock() + + with pytest.raises(ValueError, match="body failed"): + with asyncio_tools.logged_waiter(mock_self, "failing op", sleep_time=30): + raise ValueError("body failed") + + mock_self.logger.info.assert_not_called() + + +async def test_logged_waiter_uses_custom_sleep_time(): + mock_self = mock.Mock() + mock_self.logger = mock.Mock() + + with mock.patch.object(asyncio, "sleep", wraps=asyncio.sleep) as mock_sleep: + with asyncio_tools.logged_waiter(mock_self, "custom sleep", sleep_time=0.1): + await asyncio.sleep(0.2) + + sleep_calls = [c[0][0] for c in mock_sleep.call_args_list] + assert 0.1 in sleep_calls + + def _exception_raiser(): raise RuntimeError("error") diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index e5ca58db1..8af634b73 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -25,4 +25,4 @@ class TaskResultKeys(enum.Enum): class SchedulerQueues(enum.Enum): - BOT_WORKFLOW_QUEUE = "bot_workflow_queue" + AUTOMATION_WORKFLOW_QUEUE = "automation_workflow_queue" diff --git a/packages/node/octobot_node/errors.py b/packages/node/octobot_node/errors.py index e2f3b3304..75f259149 100644 --- a/packages/node/octobot_node/errors.py +++ b/packages/node/octobot_node/errors.py @@ -20,3 +20,7 @@ class WorkflowError(Exception): class WorkflowInputError(WorkflowError): """Raised when a workflow input is invalid""" + + +class WorkflowActionExecutionError(WorkflowError): + """Raised when a workflow action execution fails""" diff --git a/packages/node/octobot_node/models.py b/packages/node/octobot_node/models.py index 25d136f43..89dc045f4 100644 --- a/packages/node/octobot_node/models.py +++ b/packages/node/octobot_node/models.py @@ -44,12 +44,10 @@ class TaskStatus(str, Enum): class TaskType(str, Enum): - START_OCTOBOT = "start_octobot" EXECUTE_ACTIONS = "execute_actions" - STOP_OCTOBOT = "stop_octobot" class Task(BaseModel): - id: uuid.UUID = uuid.uuid4() + id: str = str(uuid.uuid4()) name: typing.Optional[str] = None description: typing.Optional[str] = None content: typing.Optional[str] = None diff --git a/packages/node/octobot_node/scheduler/octobot_lib.py b/packages/node/octobot_node/scheduler/octobot_lib.py index e096b6dbe..975599019 100644 --- a/packages/node/octobot_node/scheduler/octobot_lib.py +++ b/packages/node/octobot_node/scheduler/octobot_lib.py @@ -28,6 +28,7 @@ import mini_octobot.environment import mini_octobot.enums import mini_octobot.parsers + import mini_octobot.entities # Requires mini_octobot import and importable tentacles folder # ensure environment is initialized @@ -40,25 +41,6 @@ except ImportError: logging.getLogger("octobot_node.scheduler.octobot_lib").warning("OctoBot is not installed, OctoBot actions will not be available") - # mocks to allow import - class mini_octobot_mock: - class BotActionDetails: - def from_dict(self, *args, **kwargs): - raise NotImplementedError("BotActionDetails.from_dict is not implemented") - class SingleBotActionsJob: - def __init__(self, *args, **kwargs): - raise NotImplementedError("SingleBotActionsJob.__init__ is not implemented") - async def __aenter__(self): - raise NotImplementedError("SingleBotActionsJob.__aenter__ is not implemented") - async def __aexit__(self, *args, **kwargs): - raise NotImplementedError("SingleBotActionsJob.__aexit__ is not implemented") - class parsers: - class BotActionBundleParser: - def __init__(self, *args, **kwargs): - raise NotImplementedError("BotActionBundleParser.__init__ is not implemented") - def parse(self, *args, **kwargs): - raise NotImplementedError("BotActionBundleParser.parse is not implemented") - mini_octobot = mini_octobot_mock() @dataclasses.dataclass @@ -66,36 +48,34 @@ class OctoBotActionsJobDescription(octobot_commons.dataclasses.MinimizableDatacl state: dict = dataclasses.field(default_factory=dict) auth_details: dict = dataclasses.field(default_factory=dict) params: dict = dataclasses.field(default_factory=dict) - immediate_actions: list[mini_octobot.AbstractBotActionDetails] = dataclasses.field(default_factory=list) - pending_actions: list[list[mini_octobot.AbstractBotActionDetails]] = dataclasses.field(default_factory=list) def __post_init__(self): - if self.immediate_actions and isinstance(self.immediate_actions[0], dict): - self.immediate_actions = [ - mini_octobot.parse_bot_action_details(action) for action in self.immediate_actions - ] - if self.pending_actions and self.pending_actions[0] and isinstance(self.pending_actions[0][0], dict): - self.pending_actions = [ - [mini_octobot.parse_bot_action_details(action) for action in bundle] - for bundle in self.pending_actions - ] if self.params: - if self.immediate_actions or self.pending_actions: - raise ValueError("adding extra actions to a task is not yet supported") self._parse_actions_plan(self.params) def _parse_actions_plan(self, params: dict) -> None: - action_bundles: list[list[mini_octobot.AbstractBotActionDetails]] = mini_octobot.parsers.BotActionBundleParser(params).parse() - if not action_bundles: - raise ValueError("No action bundles found in params") - self.immediate_actions = action_bundles[0] - self.pending_actions = action_bundles[1:] + to_add_actions_dag = mini_octobot.parsers.ActionsDAGParser(params).parse() + if not to_add_actions_dag: + raise ValueError("No action found in params") + automation_id = None + if not automation_id and isinstance(to_add_actions_dag.actions[0], mini_octobot.entities.ConfiguredActionDetails) and to_add_actions_dag.actions[0].config: + config = to_add_actions_dag.actions[0].config + if "automation" in config: + automation_id = config["automation"]["metadata"]["automation_id"] + if not automation_id: + raise ValueError("No automation id found in params") + self._include_actions_in_automation_state(automation_id, to_add_actions_dag) + + def _include_actions_in_automation_state(self, automation_id: str, actions: "mini_octobot.ActionsDAG"): + automation_state = mini_octobot.AutomationState.from_dict(self.state) + if not automation_state.automation.metadata.automation_id: + automation_state.set_automation(automation_id, actions) + else: + automation_state.update_automation_actions(actions.actions) + self.state = automation_state.to_dict(include_default_values=False) def get_next_execution_time(self) -> float: - return min( - bot["execution"]["current_execution"]["scheduled_to"] - for bot in self.state["bots"] - ) + return self.state["automation"]["execution"]["current_execution"]["scheduled_to"] def required_actions(func): @@ -108,17 +88,17 @@ def get_required_actions_wrapper(self, *args, **kwargs): @dataclasses.dataclass class OctoBotActionsJobResult: - processed_actions: list[mini_octobot.AbstractBotActionDetails] + processed_actions: list["mini_octobot.AbstractActionDetails"] next_actions_description: typing.Optional[OctoBotActionsJobDescription] = None + actions_dag: "mini_octobot.ActionsDAG" = dataclasses.field(default_factory=mini_octobot.ActionsDAG) @required_actions - def get_failed_actions(self) -> list[dict]: - failed_actions = [ + def get_failed_actions(self) -> list[typing.Optional[dict]]: + return [ action.result for action in self.processed_actions - if action.error_status is not mini_octobot.enums.BotActionErrorStatus.NO_ERROR.value + if action.error_status is not mini_octobot.enums.ActionErrorStatus.NO_ERROR.value ] - return failed_actions @required_actions def get_created_orders(self) -> list[dict]: @@ -128,6 +108,15 @@ def get_created_orders(self) -> list[dict]: if action.result ] return list_util.flatten_list(order_lists) if order_lists else [] + + @required_actions + def get_cancelled_orders(self) -> list[str]: + cancelled_orders = [ + action.result.get(exchange_operators.CANCELLED_ORDERS_KEY, []) + for action in self.processed_actions + if action.result + ] + return list_util.flatten_list(cancelled_orders) if cancelled_orders else [] @required_actions def get_deposit_and_withdrawal_details(self) -> list[dict]: @@ -171,30 +160,39 @@ def _parse_description(self, description: typing.Union[str, dict]) -> dict: return parsed_description async def run(self) -> OctoBotActionsJobResult: - selected_actions = self.description.immediate_actions - async with mini_octobot.SingleBotActionsJob( - self.description.state, self.description.auth_details, selected_actions - ) as single_bot_actions_job: - logging.getLogger(self.__class__.__name__).info(f"Running single bot actions job actions: {selected_actions}") - await single_bot_actions_job.run() - self.after_execution_state = single_bot_actions_job.exchange_account_details - post_execution_state_dump = single_bot_actions_job.dump() + async with mini_octobot.AutomationJob( + self.description.state, self.description.auth_details + ) as automation_job: + selected_actions = automation_job.automation_state.automation.actions_dag.get_executable_actions() + logging.getLogger(self.__class__.__name__).info(f"Running automation actions: {selected_actions}") + await automation_job.run() + automation_job.automation_state.automation.actions_dag.update_actions_results(selected_actions) + self.after_execution_state = automation_job.automation_state + post_execution_state_dump = automation_job.dump() return OctoBotActionsJobResult( - processed_actions=single_bot_actions_job.bot_actions, - next_actions_description=self.get_next_actions_description(post_execution_state_dump) + processed_actions=selected_actions, + next_actions_description=self.get_next_actions_description(post_execution_state_dump), + actions_dag=automation_job.automation_state.automation.actions_dag, ) def get_next_actions_description( self, post_execution_state: dict ) -> typing.Optional[OctoBotActionsJobDescription]: - if not self.description.pending_actions: - # completed all actions - return None - return OctoBotActionsJobDescription( - state=post_execution_state, - auth_details=self.description.auth_details, - # next immediate actions are the first remaining pending actions - immediate_actions=self.description.pending_actions[0], - # next pending actions are the remaining pending actions - pending_actions=self.description.pending_actions[1:] - ) + automation = self.after_execution_state.automation + if automation.actions_dag.get_executable_actions(): + return OctoBotActionsJobDescription( + state=post_execution_state, + auth_details=self.description.auth_details, + ) + if pending_actions := automation.actions_dag.get_pending_actions(): + raise ValueError( + f"Automation {automation.metadata.automation_id}: actions DAG dependencies issue: " + f"no executable actions while there are still " + f"{len(pending_actions)} pending actions: {pending_actions}" + ) + return None + + def __repr__(self) -> str: + parsed_state = mini_octobot.AutomationState.from_dict(self.description.state) + automation_repr = str(parsed_state.automation) if parsed_state.automation else "No automation" + return f"OctoBotActionsJob with automation:\n- {automation_repr}" \ No newline at end of file diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index 2a23ce4a2..434d03ea3 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -24,7 +24,7 @@ import octobot_node.config import octobot_node.enums import octobot_node.models -import octobot_node.scheduler.workflows.base as workflow_base +import octobot_node.scheduler.workflows_util as workflows_util DEFAULT_NAME = "octobot_node" @@ -43,7 +43,7 @@ def _sanitize(result: typing.Any) -> typing.Any: class Scheduler: INSTANCE: dbos.DBOS = None # type: ignore - BOT_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore + AUTOMATION_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore def __init__(self): self.logger = logging.getLogger(self.__class__.__name__) @@ -94,7 +94,7 @@ def stop(self) -> None: self.logger.warning("Scheduler not initialized") def create_queues(self): - self.BOT_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.BOT_WORKFLOW_QUEUE.value) + self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) async def get_periodic_tasks(self) -> list[dict]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" @@ -105,13 +105,17 @@ async def get_pending_tasks(self) -> list[dict]: return [] tasks: list[dict] = [] try: - workflows = await self.INSTANCE.list_workflows_async(status=["ENQUEUED", "PENDING"]) - for w in workflows or []: + pending_workflow_statuses = await self.INSTANCE.list_workflows_async(status=["ENQUEUED", "PENDING"]) + for pending_workflow_status in pending_workflow_statuses or []: try: - task_dict = self._parse_workflow_status(w, octobot_node.models.TaskStatus.PENDING, f"Pending task: {w.name}") + if progress_status := await workflows_util.get_progress_status(pending_workflow_status.workflow_id): + description = f"{progress_status.latest_step}" + else: + description = f"Pending task: {pending_workflow_status.workflow_id}" + task_dict = self._parse_workflow_status(pending_workflow_status, octobot_node.models.TaskStatus.PENDING, description) tasks.append(task_dict) except Exception as e: - self.logger.warning(f"Failed to process pending workflow {w.name}: {e}") + self.logger.warning(f"Failed to process pending workflow {pending_workflow_status.workflow_id}: {e}") except Exception as e: self.logger.warning(f"Failed to list pending workflows: {e}") return tasks @@ -125,67 +129,57 @@ async def get_results(self) -> list[dict]: return [] tasks: list[dict] = [] try: - workflows = await self.INSTANCE.list_workflows_async(status=["SUCCESS", "ERROR"], load_output=True) - for w in workflows or []: + completed_workflow_statuses = await self.INSTANCE.list_workflows_async(status=["SUCCESS", "ERROR"], load_output=True) + for completed_workflow_status in completed_workflow_statuses or []: try: - wf_status = w.status + wf_status = completed_workflow_status.status + task_name = completed_workflow_status.workflow_id + metadata = "" + result = "" if wf_status == "SUCCESS": - if step := await workflow_base.get_current_step(w.workflow_id): - description = f"{step.previous_step_details}" - else: - description = "Task completed" - status = octobot_node.models.TaskStatus.COMPLETED - result_obj = w.output - if isinstance(result_obj, dict): - result = result_obj.get(octobot_node.enums.TaskResultKeys.RESULT.value) - metadata = result_obj.get(octobot_node.enums.TaskResultKeys.METADATA.value) - else: - result = result_obj - metadata = "" + result = completed_workflow_status.output + execution_error = result.get("error") if isinstance(result, dict) else None + description = "Error" if execution_error else "Completed" + status = octobot_node.models.TaskStatus.FAILED if execution_error else octobot_node.models.TaskStatus.COMPLETED + if task := workflows_util.get_input_task(completed_workflow_status): + metadata = task.content_metadata + task_name = task.name else: description = "Task failed" status = octobot_node.models.TaskStatus.FAILED - result = "" - metadata = "" - result_obj = None tasks.append({ - "id": w.workflow_id, - "name": self.get_task_name(result_obj, w.workflow_id), + "id": completed_workflow_status.workflow_id, + "name": task_name, "description": description, "status": status, - "result": json.dumps(_sanitize(result)) if result is not None else "", + "result": json.dumps(_sanitize(result.get("history", result))) if isinstance(result, dict) else "", #todo change "result_metadata": metadata, - "scheduled_at": w.created_at, + "scheduled_at": completed_workflow_status.created_at, "started_at": None, - "completed_at": w.updated_at, + "completed_at": completed_workflow_status.updated_at, }) except Exception as e: - self.logger.warning(f"Failed to process result workflow {w.workflow_id}: {e}") + self.logger.exception(e, True, f"Failed to process result workflow {completed_workflow_status.workflow_id}: {e}") except Exception as e: self.logger.warning(f"Failed to list result workflows: {e}") return tasks def _parse_workflow_status( self, - w: typing.Any, + workflow_status: dbos.WorkflowStatus, status: octobot_node.models.TaskStatus, description: typing.Optional[str] = None, ) -> dict: """Map DBOS WorkflowStatus to octobot_node.models.Task dict.""" - task_id = str(w.workflow_id) - task_name = w.name if hasattr(w, "name") else str(w.workflow_id) + task_id = str(workflow_status.workflow_id) + task_name = workflow_status.name task_type = None task_actions = None - if hasattr(w, "input") and w.input: - inp = w.input - if isinstance(inp, (list, tuple)) and inp: - first = inp[0] - if hasattr(first, "type"): - task_type = first.type - elif isinstance(first, dict): - task_type = first.get("type") - task_actions = first.get("actions") + if workflow_status.input: + if task := workflows_util.get_input_task(workflow_status): + task_type = task.type + task_actions = task.content #todo confi return { "id": task_id, diff --git a/packages/node/octobot_node/scheduler/tasks.py b/packages/node/octobot_node/scheduler/tasks.py index 99d369cc4..60469d31c 100644 --- a/packages/node/octobot_node/scheduler/tasks.py +++ b/packages/node/octobot_node/scheduler/tasks.py @@ -13,43 +13,23 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . -import uuid import octobot_node.models -import octobot_node.scheduler.workflows.base as workflow_base -import octobot_commons.dataclasses.minimizable_dataclass as minimizable_dataclass +import octobot_node.scheduler.workflows_util as workflows_util +import octobot_node.scheduler.workflows.params as params from octobot_node.scheduler import SCHEDULER # avoid circular import -def _generate_instance_name() -> str: - # names can't be re-used: ensure each are unique not to mix - # workflow attributes on recovery - return str(uuid.uuid4()) - - async def trigger_task(task: octobot_node.models.Task) -> bool: - import octobot_node.scheduler.workflows.bot_workflow as bot_workflow - import octobot_node.scheduler.workflows.full_bot_workflow as full_bot_workflow + import octobot_node.scheduler.workflows.automation_workflow as automation_workflow delay = 1 handle = None # enqueue workflow instead of starting it to dispatch them to multiple workers if possible - if task.type == octobot_node.models.TaskType.START_OCTOBOT.value: - handle = await SCHEDULER.BOT_WORKFLOW_QUEUE.enqueue_async( - full_bot_workflow.FullBotWorkflow.start, - t=workflow_base.Tracker(name=f"{task.name}_{_generate_instance_name()}"), - inputs=full_bot_workflow.FullBotWorkflowStartInputs(task=task, delay=delay).to_dict(include_default_values=False) - ) - elif task.type == octobot_node.models.TaskType.STOP_OCTOBOT.value: - handle = await SCHEDULER.BOT_WORKFLOW_QUEUE.enqueue_async( - full_bot_workflow.FullBotWorkflow.stop, - t=workflow_base.Tracker(name=f"{task.name}_{_generate_instance_name()}"), - inputs=full_bot_workflow.FullBotWorkflowStopInputs(task=task, delay=delay).to_dict(include_default_values=False) - ) - elif task.type == octobot_node.models.TaskType.EXECUTE_ACTIONS.value: - handle = await SCHEDULER.BOT_WORKFLOW_QUEUE.enqueue_async( - bot_workflow.BotWorkflow.execute_octobot, - t=workflow_base.Tracker(name=f"{task.name}_{_generate_instance_name()}"), - inputs=bot_workflow.BotWorkflowInputs(task=task, delay=delay).to_dict(include_default_values=False) + if task.type == octobot_node.models.TaskType.EXECUTE_ACTIONS.value: + handle = await SCHEDULER.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + automation_workflow.AutomationWorkflow.execute_automation, + t=params.Tracker(name=workflows_util.generate_workflow_name(task.name)), + inputs=params.AutomationWorkflowInputs(task=task, delay=delay).to_dict(include_default_values=False) ) else: - raise ValueError(f"Invalid task type: {task.type}") + raise ValueError(f"Unsupported task type: {task.type}") return handle is not None diff --git a/packages/node/octobot_node/scheduler/workflows/__init__.py b/packages/node/octobot_node/scheduler/workflows/__init__.py index 66bb6fb24..032554eb5 100644 --- a/packages/node/octobot_node/scheduler/workflows/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/__init__.py @@ -15,5 +15,4 @@ # License along with this library. def register_workflows(): - import octobot_node.scheduler.workflows.bot_workflow - import octobot_node.scheduler.workflows.full_bot_workflow + import octobot_node.scheduler.workflows.automation_workflow diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py new file mode 100644 index 000000000..32f45ca07 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -0,0 +1,152 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import typing +import json +import time + +import octobot_node.models +import octobot_node.scheduler.octobot_lib as octobot_lib +import octobot_node.scheduler.task_context +import octobot_node.scheduler.workflows.base as workflow_base +import octobot_node.scheduler.workflows.params as params +import octobot_node.errors as errors + + +from octobot_node.scheduler import SCHEDULER # avoid circular import + +if typing.TYPE_CHECKING: + import mini_octobot.entities + + +INIT_STEP = "init" + + +@SCHEDULER.INSTANCE.dbos_class() +class AutomationWorkflow(workflow_base.DBOSWorkflowHelperMixin): + # use dict as input to parse minimizable dataclasses and facilitate data format updates + @staticmethod + @SCHEDULER.INSTANCE.workflow(name="execute_automation") + async def execute_automation(t: params.Tracker, inputs: dict) -> typing.Optional[dict]: + parsed_inputs = params.AutomationWorkflowInputs.from_dict(inputs) + t.logger.info( + f"AutomationWorkflow started, delay: {parsed_inputs.delay}, " + f"next actions: {parsed_inputs.progress_status.next_step if parsed_inputs.progress_status else INIT_STEP}" + ) + await AutomationWorkflow.sleep_if_needed(t, parsed_inputs.delay) + global_progress = parsed_inputs.progress_status or params.ProgressStatus() + raw_iteration_result = await AutomationWorkflow.execute_iteration(t, parsed_inputs.task) + iteration_result = params.AutomationWorkflowIterationResult.from_dict(raw_iteration_result) + # always save progress + global_progress.update(iteration_result.progress_status) + if global_progress.error: + # failed iteration, return global progress where it stopped and exit workflow + t.logger.error(f"Failed iteration: stopping workflow, error: {global_progress.error}") + return global_progress.model_dump(exclude_defaults=True) + if iteration_result.next_iteration_description: + # successful iteration and a new iteration is required, schedule next iteration, don't return anything + await AutomationWorkflow._schedule_next_iteration(t, parsed_inputs, global_progress, iteration_result) + else: + # successful iteration, no new iteration is required, return global progress and exit workflow + t.logger.info( + f"Completed all iterations, global result: " + f"{global_progress.model_dump_json(exclude_defaults=True)}" + ) + return global_progress.model_dump(exclude_defaults=True) + + @staticmethod + async def _schedule_next_iteration( + t: params.Tracker, + parsed_inputs: params.AutomationWorkflowInputs, + global_progress: params.ProgressStatus, + iteration_result: params.AutomationWorkflowIterationResult + ): + new_delay = 0 + parsed_next_iteration_description = octobot_lib.OctoBotActionsJobDescription.from_dict( + iteration_result.next_iteration_description + ) + if next_execution_time := parsed_next_iteration_description.get_next_execution_time(): + new_delay = next_execution_time - time.time() + next_task = parsed_inputs.task + next_task.content = json.dumps(iteration_result.next_iteration_description) + t.logger.info(f"Enqueuing next iteration, delay: {new_delay}, next actions: {global_progress.next_step}") + await SCHEDULER.AUTOMATION_WORKFLOW_QUEUE.enqueue_async( + AutomationWorkflow.execute_automation, t=t, + inputs=params.AutomationWorkflowInputs( + task=parsed_inputs.task, progress_status=global_progress, delay=new_delay + ).to_dict(include_default_values=False) + ) + + @staticmethod + @SCHEDULER.INSTANCE.step(name="execute_iteration") + async def execute_iteration(t: params.Tracker, task: octobot_node.models.Task) -> dict: + latest_step_result = {} + execution_error: typing.Optional[str] = None + with octobot_node.scheduler.task_context.encrypted_task(task): + latest_step = INIT_STEP + if task.type == octobot_node.models.TaskType.EXECUTE_ACTIONS.value: + t.logger.info(f"Executing task '{task.name}' ...") + result: octobot_lib.OctoBotActionsJobResult = await octobot_lib.OctoBotActionsJob( + task.content + ).run() + executed_step = "" + if result.processed_actions: + if latest_step := ", ".join([str(action.get_summary()) for action in result.processed_actions]): + executed_step = latest_step + for action in result.processed_actions: + if action.error_status is not None: + t.logger.error( + f"Error: {action.error_status} when executing action {action.id}: {action.get_summary()} " + ) + execution_error = action.error_status + latest_step_result = AutomationWorkflow._format_octobot_actions_job_result(result) + else: + raise errors.WorkflowInputError(f"Invalid task type: {task.type}") + next_actions = ( + result.actions_dag.get_executable_actions() + if result.next_actions_description and result.actions_dag else [] + ) + next_step = ", ".join([action.get_summary() for action in next_actions]) if next_actions else "" + t.logger.info( + f"Iteration completed, executed step: '{executed_step}', next immediate actions: {next_actions}" + ) + remaining_steps = ( + len(result.actions_dag.get_pending_actions()) if result.actions_dag else 0 + ) + new_progress = params.ProgressStatus( + latest_step=executed_step, + latest_step_result=latest_step_result, + next_step=next_step, + next_step_at=result.next_actions_description.get_next_execution_time() if result.next_actions_description else None, + remaining_steps=remaining_steps + 1 if result.next_actions_description else ( + 1 if result.next_actions_description else 0 + ), + error=execution_error, + ) + return params.AutomationWorkflowIterationResult( + progress_status=new_progress, + next_iteration_description=result.next_actions_description.to_dict(include_default_values=False) if result.next_actions_description else None, + ).to_dict(include_default_values=False) + + @staticmethod + def _format_octobot_actions_job_result(result: octobot_lib.OctoBotActionsJobResult) -> dict: + result_dict = { + "orders": result.get_created_orders(), + "transfers": result.get_deposit_and_withdrawal_details(), + } + if failed_actions := result.get_failed_actions(): + result_dict["errors"] = failed_actions + return result_dict diff --git a/packages/node/octobot_node/scheduler/workflows/base/__init__.py b/packages/node/octobot_node/scheduler/workflows/base/__init__.py index eda19f066..6ce8b0634 100644 --- a/packages/node/octobot_node/scheduler/workflows/base/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/base/__init__.py @@ -14,19 +14,10 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. -from octobot_node.scheduler.workflows.base.workflow_tracking import ( - Tracker, - ProgressStatus, - get_current_step -) - from octobot_node.scheduler.workflows.base.workflow_helper_mixin import ( DBOSWorkflowHelperMixin ) __all__ = [ - "Tracker", - "ProgressStatus", "DBOSWorkflowHelperMixin", - "get_current_step" ] diff --git a/packages/node/octobot_node/scheduler/workflows/base/workflow_helper_mixin.py b/packages/node/octobot_node/scheduler/workflows/base/workflow_helper_mixin.py index 9c6a78eb0..ad690a942 100644 --- a/packages/node/octobot_node/scheduler/workflows/base/workflow_helper_mixin.py +++ b/packages/node/octobot_node/scheduler/workflows/base/workflow_helper_mixin.py @@ -13,11 +13,9 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import typing import dbos as dbos_lib -import time -import octobot_node.scheduler.workflows.base.workflow_tracking as workflow_tracking +import octobot_node.scheduler.workflows.params.base_params as base_params class DBOSWorkflowHelperMixin: @@ -25,21 +23,12 @@ class DBOSWorkflowHelperMixin: def get_name(workflow_status: dbos_lib.WorkflowStatus) -> str: if workflow_status.input: for input in list(workflow_status.input.get("args", [])) + list(workflow_status.input.get("kwargs", {}).values()): - if isinstance(input, workflow_tracking.Tracker): + if isinstance(input, base_params.Tracker): return input.name raise ValueError(f"No Tracker found in workflow status: {workflow_status}") - - @staticmethod - async def register_delayed_start_step(t: workflow_tracking.Tracker, delay: float, next_step: str) -> None: - await t.set_current_step(workflow_tracking.ProgressStatus( - previous_step="delayed_start", - previous_step_details={"delay": delay}, - next_step=next_step, - next_step_at=time.time() + delay, - )) @staticmethod - async def sleep_if_needed(t: workflow_tracking.Tracker, delay: float) -> None: + async def sleep_if_needed(t: base_params.Tracker, delay: float) -> None: if delay > 0: t.logger.info(f"Sleeping for {delay} seconds ...") await dbos_lib.DBOS.sleep_async(delay) diff --git a/packages/node/octobot_node/scheduler/workflows/base/workflow_tracking.py b/packages/node/octobot_node/scheduler/workflows/base/workflow_tracking.py deleted file mode 100644 index bf8a304ab..000000000 --- a/packages/node/octobot_node/scheduler/workflows/base/workflow_tracking.py +++ /dev/null @@ -1,35 +0,0 @@ -import typing -import pydantic -import dataclasses -import logging -import dbos as dbos_lib - -import octobot_commons.dataclasses - - -CURRENT_STEP_KEY = "current_step" - - -class ProgressStatus(pydantic.BaseModel): - previous_step: str - previous_step_details: typing.Optional[dict] = None - next_step: typing.Optional[str] = None - next_step_at: typing.Optional[float] = None - remaining_steps: typing.Optional[int] = None - - -async def get_current_step(workflow_id: str) -> typing.Optional[ProgressStatus]: - return await dbos_lib.DBOS.get_event_async(workflow_id, CURRENT_STEP_KEY) - - -@dataclasses.dataclass -class Tracker(octobot_commons.dataclasses.MinimizableDataclass): - name: str - - @property - def logger(self) -> logging.Logger: - return logging.getLogger(self.name) - - async def set_current_step(self, progress_status: ProgressStatus): - await dbos_lib.DBOS.set_event_async(CURRENT_STEP_KEY, progress_status) - self.logger.info(f"Current step updated: {progress_status.model_dump(exclude_defaults=True)}") \ No newline at end of file diff --git a/packages/node/octobot_node/scheduler/workflows/bot_workflow.py b/packages/node/octobot_node/scheduler/workflows/bot_workflow.py deleted file mode 100644 index 829a634e3..000000000 --- a/packages/node/octobot_node/scheduler/workflows/bot_workflow.py +++ /dev/null @@ -1,142 +0,0 @@ -# Drakkar-Software OctoBot-Node -# Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. - -import dataclasses -import typing -import json -import copy -import time - -import octobot_node.models -import octobot_node.enums -import octobot_node.scheduler.octobot_lib as octobot_lib -import octobot_node.scheduler.task_context -import octobot_node.scheduler.workflows.base as workflow_base -import octobot_commons.dataclasses.minimizable_dataclass -import octobot_node.errors as errors - - -from octobot_node.scheduler import SCHEDULER # avoid circular import - - -@dataclasses.dataclass -class BotWorkflowInputs(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): - task: octobot_node.models.Task - delay: float = 0 - - -@dataclasses.dataclass -class BotIterationResult(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): - task_result: dict - next_iteration_time: typing.Optional[float] - next_task: typing.Optional[octobot_node.models.Task] - - -INIT_STEP = "init" - - -@SCHEDULER.INSTANCE.dbos_class() -class BotWorkflow(workflow_base.DBOSWorkflowHelperMixin): - # use dict as input to parse minimizable dataclasses and facilitate data format updates - - @staticmethod - @SCHEDULER.INSTANCE.workflow(name="execute_octobot") - async def execute_octobot(t: workflow_base.Tracker, inputs: dict) -> dict: - parsed_inputs = BotWorkflowInputs.from_dict(inputs) - should_continue = True - delay = parsed_inputs.delay - if delay > 0: - await workflow_base.DBOSWorkflowHelperMixin.register_delayed_start_step(t, delay, INIT_STEP) - next_task: octobot_node.models.Task = parsed_inputs.task - while should_continue: - await BotWorkflow.sleep_if_needed(t, delay) - raw_iteration_result = await BotWorkflow.execute_iteration(t, next_task) - iteration_result = BotIterationResult.from_dict(raw_iteration_result) - if iteration_result.next_iteration_time: - should_continue = True - delay = iteration_result.next_iteration_time - time.time() - if iteration_result.next_task is None: - raise errors.WorkflowInputError(f"iteration_result.next_task is None, this should not happen. {iteration_result=}") - next_task = iteration_result.next_task - else: - should_continue = False - t.logger.info(f"BotWorkflow completed, last iteration result: {iteration_result.task_result}") - return iteration_result.task_result - - @staticmethod - @SCHEDULER.INSTANCE.step(name="execute_iteration") - async def execute_iteration(t: workflow_base.Tracker, task: octobot_node.models.Task) -> dict: - next_iteration_time = None - task_output = {} - next_task = copy.copy(task) - error = None - with octobot_node.scheduler.task_context.encrypted_task(task): - current_step = INIT_STEP - if task.type == octobot_node.models.TaskType.EXECUTE_ACTIONS.value: - t.logger.info(f"Executing task '{task.name}' ...") - result: octobot_lib.OctoBotActionsJobResult = await octobot_lib.OctoBotActionsJob( - task.content - ).run() - if result.processed_actions: - current_step = ", ".join([str(action.dsl_script) for action in result.processed_actions]) - for action in result.processed_actions: - if action.error_status is not None: - error = action.error_status - else: - current_step = None - task_output = BotWorkflow._format_octobot_actions_job_result(result) - if result.next_actions_description: - next_iteration_time = result.next_actions_description.get_next_execution_time() - next_task.content = json.dumps(result.next_actions_description.to_dict( - include_default_values=False - )) - else: - raise errors.WorkflowInputError(f"Invalid task type: {task.type}") - t.logger.info( - f"Task '{task.name}' completed. Next immediate actions: " - f"{result.next_actions_description.immediate_actions if result.next_actions_description else None}" - ) - await t.set_current_step(workflow_base.ProgressStatus( - previous_step=current_step, - previous_step_details=task_output, - next_step=", ".join([str(action.dsl_script) for action in result.next_actions_description.immediate_actions]) if result.next_actions_description else None, - next_step_at=result.next_actions_description.get_next_execution_time() if result.next_actions_description else None, - remaining_steps=len(result.next_actions_description.pending_actions) + 1 if result.next_actions_description else ( - 1 if result.next_actions_description else 0 - ), - )) - task_result = { - octobot_node.enums.TaskResultKeys.STATUS.value: octobot_node.models.TaskStatus.COMPLETED.value, - octobot_node.enums.TaskResultKeys.RESULT.value: task_output, - octobot_node.enums.TaskResultKeys.METADATA.value: task.result_metadata, - octobot_node.enums.TaskResultKeys.TASK.value: {"name": task.name}, - octobot_node.enums.TaskResultKeys.ERROR.value: error, - } - return BotIterationResult( - task_result=task_result, - next_iteration_time=next_iteration_time, - next_task=next_task, - ).to_dict(include_default_values=False) - - @staticmethod - def _format_octobot_actions_job_result(result: octobot_lib.OctoBotActionsJobResult) -> dict: - result_dict = { - "orders": result.get_created_orders(), - "transfers": result.get_deposit_and_withdrawal_details(), - } - if failed_actions := result.get_failed_actions(): - result_dict["errors"] = failed_actions - return result_dict diff --git a/packages/node/octobot_node/scheduler/workflows/full_bot_workflow.py b/packages/node/octobot_node/scheduler/workflows/full_bot_workflow.py deleted file mode 100644 index 6da970b28..000000000 --- a/packages/node/octobot_node/scheduler/workflows/full_bot_workflow.py +++ /dev/null @@ -1,81 +0,0 @@ -# Drakkar-Software OctoBot-Node -# Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. - -import dataclasses - -import octobot_node.models -import octobot_node.enums -import octobot_node.scheduler.workflows.base as workflow_base -import octobot_node.scheduler.task_context - -from octobot_node.scheduler import SCHEDULER # avoid circular import -import octobot_commons.dataclasses.minimizable_dataclass - - -@dataclasses.dataclass -class FullBotWorkflowStartInputs(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): - task: octobot_node.models.Task - delay: float - -@dataclasses.dataclass -class FullBotWorkflowStopInputs(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): - task: octobot_node.models.Task - delay: float - - -@SCHEDULER.INSTANCE.dbos_class() -class FullBotWorkflow(workflow_base.DBOSWorkflowHelperMixin): - # use dict as inputs to parse minimizable dataclasses and facilitate data format updates - - @staticmethod - @SCHEDULER.INSTANCE.workflow(name="start_full_octobot") - async def start(t: workflow_base.Tracker, inputs: dict) -> dict: - parsed_inputs = FullBotWorkflowStartInputs.from_dict(inputs) - if parsed_inputs.delay > 0: - await workflow_base.DBOSWorkflowHelperMixin.register_delayed_start_step(t, parsed_inputs.delay, "start_bot") - await FullBotWorkflow.sleep_if_needed(t, parsed_inputs.delay) - # todo implement start logic: start bot with process name from self.get_bot_process_name() - with octobot_node.scheduler.task_context.encrypted_task(parsed_inputs.task): - parsed_inputs.task.result = "ok" - return { - octobot_node.enums.TaskResultKeys.STATUS.value: octobot_node.models.TaskStatus.COMPLETED.value, - octobot_node.enums.TaskResultKeys.RESULT.value: parsed_inputs.task.result, - octobot_node.enums.TaskResultKeys.METADATA.value: parsed_inputs.task.result_metadata, - octobot_node.enums.TaskResultKeys.TASK.value: {"name": parsed_inputs.task.name}, - octobot_node.enums.TaskResultKeys.ERROR.value: None, - } - - @staticmethod - @SCHEDULER.INSTANCE.workflow(name="stop_full_octobot") - async def stop(t: workflow_base.Tracker, inputs: dict) -> dict: - parsed_inputs = FullBotWorkflowStopInputs.from_dict(inputs) - if parsed_inputs.delay > 0: - await workflow_base.DBOSWorkflowHelperMixin.register_delayed_start_step(t, parsed_inputs.delay, "stop_bot") - await FullBotWorkflow.sleep_if_needed(t, parsed_inputs.delay) - # todo implement stop logic: stop bot with process name from self.get_bot_process_name() - with octobot_node.scheduler.task_context.encrypted_task(parsed_inputs.task): - parsed_inputs.task.result = "ok" - return { - octobot_node.enums.TaskResultKeys.STATUS.value: octobot_node.models.TaskStatus.COMPLETED.value, - octobot_node.enums.TaskResultKeys.RESULT.value: parsed_inputs.task.result, - octobot_node.enums.TaskResultKeys.METADATA.value: parsed_inputs.task.result_metadata, - octobot_node.enums.TaskResultKeys.TASK.value: {"name": parsed_inputs.task.name}, - octobot_node.enums.TaskResultKeys.ERROR.value: None, - } - - @staticmethod - def get_bot_process_name(t: workflow_base.Tracker) -> str: - return f"octobot_{t.name}" diff --git a/packages/node/octobot_node/scheduler/workflows/params/__init__.py b/packages/node/octobot_node/scheduler/workflows/params/__init__.py new file mode 100644 index 000000000..dda6d746c --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/params/__init__.py @@ -0,0 +1,30 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +from .base_params import ( + Tracker, + ProgressStatus, +) +from .automation_workflow_params import ( + AutomationWorkflowInputs, + AutomationWorkflowIterationResult, +) + +__all__ = [ + "Tracker", + "AutomationWorkflowInputs", + "AutomationWorkflowIterationResult", + "ProgressStatus", +] diff --git a/packages/node/octobot_node/scheduler/workflows/params/automation_workflow_params.py b/packages/node/octobot_node/scheduler/workflows/params/automation_workflow_params.py new file mode 100644 index 000000000..080498847 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/params/automation_workflow_params.py @@ -0,0 +1,34 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import dataclasses +import typing + +import octobot_commons.dataclasses.minimizable_dataclass +import octobot_node.models +import octobot_node.scheduler.workflows.params.base_params as base_params + + +@dataclasses.dataclass +class AutomationWorkflowInputs(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): + task: octobot_node.models.Task + progress_status: typing.Optional[base_params.ProgressStatus] = None + delay: float = 0 + + +@dataclasses.dataclass +class AutomationWorkflowIterationResult(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): + progress_status: base_params.ProgressStatus + next_iteration_description: typing.Optional[dict] diff --git a/packages/node/octobot_node/scheduler/workflows/params/base_params.py b/packages/node/octobot_node/scheduler/workflows/params/base_params.py new file mode 100644 index 000000000..7e308e94e --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/params/base_params.py @@ -0,0 +1,47 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import dataclasses +import pydantic +import typing + +import octobot_commons.logging +import octobot_commons.dataclasses + + +class ProgressStatus(pydantic.BaseModel): + latest_step: typing.Optional[str] = None + latest_step_result: typing.Optional[dict] = None + next_step: typing.Optional[str] = None + next_step_at: typing.Optional[float] = None + remaining_steps: typing.Optional[int] = None + error: typing.Optional[str] = None + + def update(self, progress_status: "ProgressStatus"): + self.latest_step = progress_status.latest_step + self.latest_step_result = progress_status.latest_step_result + self.next_step = progress_status.next_step + self.next_step_at = progress_status.next_step_at + self.remaining_steps = progress_status.remaining_steps + self.error = progress_status.error + + +@dataclasses.dataclass +class Tracker(octobot_commons.dataclasses.MinimizableDataclass): + name: str + + @property + def logger(self) -> octobot_commons.logging.BotLogger: + return octobot_commons.logging.get_logger(self.name) diff --git a/packages/node/octobot_node/scheduler/workflows_util.py b/packages/node/octobot_node/scheduler/workflows_util.py index 903af01fd..670c7606f 100644 --- a/packages/node/octobot_node/scheduler/workflows_util.py +++ b/packages/node/octobot_node/scheduler/workflows_util.py @@ -13,11 +13,43 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. - +import uuid +import typing import dbos as dbos_lib -from octobot_node.scheduler import SCHEDULER +import octobot_node.models as models +import octobot_node.scheduler.workflows.params as params +import octobot_node.scheduler async def get_workflow_handle(workflow_id: str) -> dbos_lib.WorkflowHandleAsync: - return await SCHEDULER.INSTANCE.retrieve_workflow_async(workflow_id) \ No newline at end of file + return await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async(workflow_id) + + +def generate_workflow_name(prefix: str) -> str: + return f"{prefix}_{uuid.uuid4()}" + + +async def get_progress_status(workflow_id: str) -> typing.Optional[params.ProgressStatus]: + workflow_status = await dbos_lib.DBOS.get_workflow_status_async(workflow_id) + if inputs := get_automation_workflow_inputs(workflow_status): + return inputs.progress_status + return None + + +def get_input_task(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[models.Task]: + if inputs := get_automation_workflow_inputs(workflow_status): + return inputs.task + return None + + +def get_automation_workflow_inputs(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[params.AutomationWorkflowInputs]: + for input in list(workflow_status.input.get("args", [])) + list(workflow_status.input.get("kwargs", {}).values()): + if isinstance(input, dict): + try: + parsed_inputs = params.AutomationWorkflowInputs.from_dict(input) + return parsed_inputs + except TypeError: + print(f"Failed to parse inputs: {input}") + pass + return None diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index d9e911ea1..9cb4b18e3 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -31,7 +31,7 @@ def temp_dbos_scheduler(): "name": "scheduler_test", "system_database_url": f"sqlite:///{temp_file_name}", } - if octobot_node.scheduler.SCHEDULER.BOT_WORKFLOW_QUEUE is None: + if octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE is None: octobot_node.scheduler.SCHEDULER.create_queues() dbos.DBOS(config=config) dbos.DBOS.reset_system_database() diff --git a/packages/node/tests/scheduler/test_octobot_lib.py b/packages/node/tests/scheduler/test_octobot_lib.py index 836719b8a..e533cf5ae 100644 --- a/packages/node/tests/scheduler/test_octobot_lib.py +++ b/packages/node/tests/scheduler/test_octobot_lib.py @@ -14,18 +14,23 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import pytest +import decimal +import dataclasses -import octobot_node.scheduler.octobot_lib as octobot_lib import octobot_commons.constants as common_constants +import octobot_trading.constants +import octobot_trading.errors +import octobot_trading.enums as trading_enums +import octobot_node.scheduler.octobot_lib as octobot_lib + RUN_TESTS = True + + try: - raise ImportError("test") - import octobot_trading.constants - import octobot_trading.errors + import mini_octobot.entities + import mini_octobot.enums - import octobot_wrapper.keywords.internal.overrides.custom_action_trading_mode as custom_action_trading_mode - import octobot_wrapper.keywords.internal.constants as kw_constants - import octobot_wrapper.keywords.internal.enums as kw_enums + import tentacles.Meta.DSL_operators as DSL_operators BLOCKCHAIN = octobot_trading.constants.SIMULATED_BLOCKCHAIN_NETWORK except ImportError: @@ -136,6 +141,21 @@ def deposit_action(): } +@pytest.fixture +def transfer_blockchain_action(): + return { + "params": { + "ACTIONS": "transfer", + "BLOCKCHAIN_FROM_ASSET": "BTC", + "BLOCKCHAIN_FROM_AMOUNT": 1, + "BLOCKCHAIN_FROM": BLOCKCHAIN, + "BLOCKCHAIN_TO": BLOCKCHAIN, + "BLOCKCHAIN_TO_ASSET": "BTC", + "BLOCKCHAIN_TO_ADDRESS": "0x123_simulated_transfer_to_address_BTC", + } + } + + @pytest.fixture def withdraw_action(): return { @@ -226,12 +246,18 @@ def teardown_method(self): octobot_trading.constants.ALLOW_FUNDS_TRANSFER = False async def test_run_market_order_action(self, market_order_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(market_order_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, @@ -240,14 +266,20 @@ async def test_run_market_order_action(self, market_order_action): # step 2: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script == "market('buy', 'ETH/BTC', 1)" job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script == "market('buy', 'ETH/BTC', 1)" assert len(result.get_created_orders()) == 1 order = result.get_created_orders()[0] assert order["symbol"] == "ETH/BTC" @@ -257,7 +289,7 @@ async def test_run_market_order_action(self, market_order_action): assert result.next_actions_description is None # no more actions to execute # ensure deposit is successful - post_deposit_portfolio = job2.after_execution_state.bots[0].exchange_account_elements.portfolio.content + post_deposit_portfolio = job2.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_deposit_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] < pre_trade_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] assert post_deposit_portfolio["BTC"][common_constants.PORTFOLIO_TOTAL] < pre_trade_portfolio["BTC"][common_constants.PORTFOLIO_TOTAL] @@ -266,12 +298,18 @@ async def test_run_market_order_action(self, market_order_action): assert post_deposit_portfolio["ETH"][common_constants.PORTFOLIO_TOTAL] == 0.999 async def test_run_limit_order_action(self, limit_order_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(limit_order_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, @@ -280,30 +318,42 @@ async def test_run_limit_order_action(self, limit_order_action): # step 2: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script == "limit('buy', 'ETH/BTC', 1, '-10%')" job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script == "limit('buy', 'ETH/BTC', 1, '-10%')" assert len(result.get_created_orders()) == 1 order = result.get_created_orders()[0] assert order["symbol"] == "ETH/BTC" - assert order["amount"] == 1 - assert 0.001 < order["limit_price"] < 0.2 + assert order["amount"] == decimal.Decimal("1") + assert decimal.Decimal("0.001") < order["price"] < decimal.Decimal("0.2") assert order["type"] == "limit" assert order["side"] == "buy" assert result.next_actions_description is None # no more actions to execute async def test_run_stop_loss_order_action(self, stop_loss_order_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(stop_loss_order_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["ETH"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, @@ -312,30 +362,42 @@ async def test_run_stop_loss_order_action(self, stop_loss_order_action): # step 2: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.SELL_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("stop_loss('sell', 'ETH/BTC', '10%', '-10%')") job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.SELL_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("stop_loss('sell', 'ETH/BTC', '10%', '-10%')") assert len(result.get_created_orders()) == 1 order = result.get_created_orders()[0] assert order["symbol"] == "ETH/BTC" - assert order["amount"] == 0.1 # 10% of 1 ETH - assert 0.001 < order["limit_price"] < 0.2 + assert order["amount"] == decimal.Decimal("0.1") # 10% of 1 ETH + assert decimal.Decimal("0.001") < order["price"] < decimal.Decimal("0.2") assert order["type"] == "stop_loss" assert order["side"] == "sell" assert result.next_actions_description is None # no more actions to execute async def test_run_cancel_limit_order_action(self, create_limit_and_cancel_order_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(create_limit_and_cancel_order_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, @@ -344,19 +406,25 @@ async def test_run_cancel_limit_order_action(self, create_limit_and_cancel_order # step 2: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script == "limit('buy', 'ETH/BTC', 1, '-10%')" job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("limit(") assert len(result.get_created_orders()) == 1 order = result.get_created_orders()[0] assert order["symbol"] == "ETH/BTC" - assert order["amount"] == 1 - assert 0.001 < order["limit_price"] < 0.2 + assert order["amount"] == decimal.Decimal("1") + assert decimal.Decimal("0.001") < order["price"] < decimal.Decimal("0.2") assert order["type"] == "limit" assert order["side"] == "buy" assert result.next_actions_description is not None @@ -364,24 +432,38 @@ async def test_run_cancel_limit_order_action(self, create_limit_and_cancel_order # step 3: run the cancel action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.CANCEL_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script == "cancel_order('ETH/BTC', side='buy')" job3 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job3.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.CANCEL_SIGNAL - assert result.processed_actions[0].result["cancelled_orders_count"] == 1 + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("cancel_order(") + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CANCELLED_ORDERS_KEY]) == len(result.get_cancelled_orders()) == 1 assert result.next_actions_description is None # no more actions to execute + @pytest.mark.skip(reason="restore once polymarket is fully supported") async def test_polymarket_trade_action(self, polymarket_order_action): # TODO: update once polymarket is fullly supported - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(polymarket_order_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["USDC"] == { common_constants.PORTFOLIO_AVAILABLE: 100, common_constants.PORTFOLIO_TOTAL: 100, @@ -390,29 +472,85 @@ async def test_polymarket_trade_action(self, polymarket_order_action): # TODO: u # step 2: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("market(") job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) with pytest.raises(octobot_trading.errors.FailedRequest): # TODO: update once supported result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("market(") assert len(result.get_created_orders()) == 1 order = result.get_created_orders()[0] assert order["symbol"] == "what-price-will-bitcoin-hit-in-january-2026/USDC:USDC-260131-0-YES" - assert order["amount"] == 1 + assert order["amount"] == decimal.Decimal("1") assert order["type"] == "market" assert order["side"] == "buy" + async def test_run_transfer_blockchain_only_action(self, transfer_blockchain_action): + # step 1: configure the job + job = octobot_lib.OctoBotActionsJob(transfer_blockchain_action) + result = await job.run() + assert len(result.processed_actions) == 1 + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert job.after_execution_state.automation.reference_exchange_account_elements is None + assert job.after_execution_state.automation.client_exchange_account_elements.portfolio.content is None + + # step 2: run the transfer action + next_actions_description = result.next_actions_description + assert next_actions_description is not None + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in next_actions[0].dsl_script + job2 = octobot_lib.OctoBotActionsJob( + next_actions_description.to_dict(include_default_values=False) + ) + result = await job2.run() + assert len(result.processed_actions) == 1 + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in processed_actions[0].dsl_script + assert result.next_actions_description is None # no more actions to execute + + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CREATED_TRANSACTIONS_KEY]) == len(result.get_deposit_and_withdrawal_details()) == 1 + assert len(result.get_deposit_and_withdrawal_details()) == 1 + transaction = result.get_deposit_and_withdrawal_details()[0] + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value] == "BTC" + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("1") + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.NETWORK.value] == BLOCKCHAIN + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value] == "0x123_simulated_transfer_to_address_BTC" + + + async def test_run_deposit_action(self, deposit_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(deposit_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_deposit_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_deposit_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_deposit_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 0.01, common_constants.PORTFOLIO_TOTAL: 0.01, @@ -421,30 +559,42 @@ async def test_run_deposit_action(self, deposit_action): # step 2: run the deposit action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in next_actions[0].dsl_script job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in processed_actions[0].dsl_script assert result.next_actions_description is None # no more actions to execute # ensure deposit is successful - post_deposit_portfolio = job2.after_execution_state.bots[0].exchange_account_elements.portfolio.content + post_deposit_portfolio = job2.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_deposit_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: pre_deposit_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] + deposit_action["params"]["BLOCKCHAIN_FROM_AMOUNT"], common_constants.PORTFOLIO_TOTAL: pre_deposit_portfolio["BTC"][common_constants.PORTFOLIO_TOTAL] + deposit_action["params"]["BLOCKCHAIN_FROM_AMOUNT"], } async def test_run_withdraw_action(self, withdraw_action): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(withdraw_action) result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_withdraw_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_withdraw_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_withdraw_portfolio["ETH"] == { common_constants.PORTFOLIO_AVAILABLE: 2, common_constants.PORTFOLIO_TOTAL: 2, @@ -453,56 +603,103 @@ async def test_run_withdraw_action(self, withdraw_action): # step 2: run the withdraw action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.WITHDRAW_FUNDS_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("withdraw(") job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.WITHDRAW_FUNDS_SIGNAL + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("withdraw(") assert result.next_actions_description is None # no more actions to execute # ensure withdraw is successful - post_withdraw_portfolio = job2.after_execution_state.bots[0].exchange_account_elements.portfolio.content + post_withdraw_portfolio = job2.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_withdraw_portfolio == {} # portfolio should now be empty async def test_run_multiple_actions_bundle_no_wait(self, multiple_actions_bundle_no_wait): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(multiple_actions_bundle_no_wait) + # ensure wait keywords have been considered + automation = job.description.state["automation"] + dag = automation["actions_dag"] + for action in dag["actions"]: + assert action["next_schedule"] is None # no wait keywords have been considered result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, } - # step 2: run the deposit and trade actions + # step 2: run the deposit action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 2 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL - assert next_actions_description.immediate_actions[1].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 # only the deposit action should be executable as the trade action depends on it + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in next_actions[0].dsl_script job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() - assert len(result.processed_actions) == 2 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL - assert result.processed_actions[0].result["amount"] == 1 - assert result.processed_actions[1].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + assert len(result.processed_actions) == 1 + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in processed_actions[0].dsl_script + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CREATED_TRANSACTIONS_KEY]) == len(result.get_deposit_and_withdrawal_details()) == 1 + assert len(result.get_deposit_and_withdrawal_details()) == 1 + transaction = result.get_deposit_and_withdrawal_details()[0] + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value] == "BTC" + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("1") + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.NETWORK.value] == BLOCKCHAIN + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value] == "0x123_simulated_deposit_address_BTC" + + + # step 3: run the trade action + next_actions_description = result.next_actions_description + assert next_actions_description is not None + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 # only the trade action should be executable now: all others have been executed already + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("limit(") + job3 = octobot_lib.OctoBotActionsJob( + next_actions_description.to_dict(include_default_values=False) + ) + result = await job3.run() + assert len(result.processed_actions) == 1 + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("limit(") assert len(result.get_created_orders()) == 1 limit_order = result.get_created_orders()[0] assert limit_order["symbol"] == "ETH/BTC" - assert limit_order["amount"] == 1 + assert limit_order["amount"] == decimal.Decimal("1") assert limit_order["type"] == "limit" assert limit_order["side"] == "buy" assert result.next_actions_description is None # no more actions to execute # ensure trades are taken into account in portfolio - post_deposit_portfolio = job2.after_execution_state.bots[0].exchange_account_elements.portfolio.content + post_deposit_portfolio = job3.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert "ETH" not in post_deposit_portfolio # ETH order has not been executed (still open) @@ -512,12 +709,34 @@ async def test_run_multiple_actions_bundle_no_wait(self, multiple_actions_bundle async def test_run_multiple_actions_bundle_with_wait(self, multiple_action_bundle_with_wait): - # step 1: configure the task + # step 1: configure the job job = octobot_lib.OctoBotActionsJob(multiple_action_bundle_with_wait) + # ensure wait keywords have been considered + automation = job.description.state["automation"] + dag = automation["actions_dag"] + for index, action in enumerate(dag["actions"]): + if index == len(dag["actions"]) - 1: + assert action["next_schedule"] is None + else: + assert action["next_schedule"] is not None + assert action["next_schedule"] == dataclasses.asdict(mini_octobot.entities.NextScheduleParams( + type=mini_octobot.entities.ScheduleType.RANDOM.value, + schedule=mini_octobot.entities.RandomScheduleDetails( + min_delay=0.1, + max_delay=0.15, + ), + )) + # run the job result = await job.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == kw_enums.CustomActionExclusiveFormattedContentConfigKeys.APPLY_CONFIGURATION.value - pre_trade_portfolio = job.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.ConfiguredActionDetails) + assert processed_actions[0].action == mini_octobot.enums.ActionType.APPLY_CONFIGURATION.value + assert processed_actions[0].config is not None + assert "automation" in processed_actions[0].config + assert isinstance(processed_actions[0].config["exchange_account_details"], dict) + pre_trade_portfolio = job.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert pre_trade_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 1, common_constants.PORTFOLIO_TOTAL: 1, @@ -526,19 +745,35 @@ async def test_run_multiple_actions_bundle_with_wait(self, multiple_action_bundl # step 2: run the deposit action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in next_actions[0].dsl_script job2 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job2.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.TRANSFER_FUNDS_SIGNAL - assert result.processed_actions[0].result["amount"] == 1 - assert result.next_actions_description is not None - assert len(result.next_actions_description.immediate_actions) == 1 - assert result.next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL - post_deposit_portfolio = job2.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script is not None and "blockchain_wallet_transfer" in processed_actions[0].dsl_script + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CREATED_TRANSACTIONS_KEY]) == len(result.get_deposit_and_withdrawal_details()) == 1 + transaction = processed_actions[0].result[DSL_operators.CREATED_TRANSACTIONS_KEY][0] + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value] == "BTC" + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("1") + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.NETWORK.value] == BLOCKCHAIN + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value] == "0x123_simulated_deposit_address_BTC" + next_desc = result.next_actions_description + assert next_desc is not None + parsed_state = mini_octobot.AutomationState.from_dict(next_desc.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("market(") + post_deposit_portfolio = job2.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_deposit_portfolio["BTC"] == { common_constants.PORTFOLIO_AVAILABLE: 2, common_constants.PORTFOLIO_TOTAL: 2, @@ -547,16 +782,23 @@ async def test_run_multiple_actions_bundle_with_wait(self, multiple_action_bundl # step 3: run the trade action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("market(") job3 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job3.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.BUY_SIGNAL - assert len(result.get_created_orders()) == 1 - post_trade_portfolio = job3.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("market(") + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CREATED_ORDERS_KEY]) == len(result.get_created_orders()) == 1 + post_trade_portfolio = job3.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_trade_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] < post_deposit_portfolio["BTC"][common_constants.PORTFOLIO_AVAILABLE] assert post_trade_portfolio["ETH"] == { common_constants.PORTFOLIO_AVAILABLE: 0.999, @@ -566,16 +808,28 @@ async def test_run_multiple_actions_bundle_with_wait(self, multiple_action_bundl # step 4: run the withdraw action next_actions_description = result.next_actions_description assert next_actions_description is not None - assert len(next_actions_description.immediate_actions) == 1 - assert next_actions_description.immediate_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.WITHDRAW_FUNDS_SIGNAL + parsed_state = mini_octobot.AutomationState.from_dict(next_actions_description.state) + next_actions = parsed_state.automation.actions_dag.get_executable_actions() + assert len(next_actions) == 1 + assert isinstance(next_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert next_actions[0].dsl_script.startswith("withdraw(") job4 = octobot_lib.OctoBotActionsJob( next_actions_description.to_dict(include_default_values=False) ) result = await job4.run() assert len(result.processed_actions) == 1 - assert result.processed_actions[0].config[kw_constants.CUSTOM_ACTION_OPEN_SOURCE_FORMAT_KEY][custom_action_trading_mode.CustomActionTradingMode.SIGNAL_KEY] == custom_action_trading_mode.CustomActionTradingMode.WITHDRAW_FUNDS_SIGNAL - assert result.processed_actions[0].result["amount"] == 0.999 - post_withdraw_portfolio = job4.after_execution_state.bots[0].exchange_account_elements.portfolio.content + processed_actions = result.processed_actions + assert len(processed_actions) == 1 + assert isinstance(processed_actions[0], mini_octobot.entities.DSLScriptActionDetails) + assert processed_actions[0].dsl_script.startswith("withdraw(") + assert processed_actions[0].result is not None + assert len(processed_actions[0].result[DSL_operators.CREATED_WITHDRAWALS_KEY]) == len(result.get_deposit_and_withdrawal_details()) == 1 + transaction = processed_actions[0].result[DSL_operators.CREATED_WITHDRAWALS_KEY][0] + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value] == "ETH" + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("0.999") + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.NETWORK.value] == "ethereum" + assert transaction[trading_enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value] == "0x1234567890123456789012345678901234567890" + post_withdraw_portfolio = job4.after_execution_state.automation.client_exchange_account_elements.portfolio.content assert post_withdraw_portfolio["BTC"] == post_trade_portfolio["BTC"] assert "ETH" not in post_withdraw_portfolio assert result.next_actions_description is None # no more actions to execute diff --git a/packages/node/tests/scheduler/test_tasks.py b/packages/node/tests/scheduler/test_tasks.py index 68c46e0d5..4f41f2d94 100644 --- a/packages/node/tests/scheduler/test_tasks.py +++ b/packages/node/tests/scheduler/test_tasks.py @@ -59,7 +59,7 @@ async def test_trigger_all_task_types(self, schedule_task, temp_dbos_scheduler): for task_type in octobot_node.models.TaskType: schedule_task.type = task_type.value with mock.patch.object( - temp_dbos_scheduler.BOT_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() + temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() ) as mock_enqueue_async: result = await octobot_node.scheduler.tasks.trigger_task(schedule_task) assert result is True @@ -71,9 +71,9 @@ async def test_trigger_all_task_types(self, schedule_task, temp_dbos_scheduler): inputs = call_kwargs["inputs"] assert inputs["task"] == schedule_task.model_dump(exclude_defaults=True) assert inputs["delay"] == 1 - with pytest.raises(ValueError, match="Invalid task type"): + with pytest.raises(ValueError, match="Unsupported task type"): with mock.patch.object( - temp_dbos_scheduler.BOT_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() + temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE, "enqueue_async", mock.AsyncMock() ) as mock_enqueue_async: schedule_task.type = "invalid_type" await octobot_node.scheduler.tasks.trigger_task(schedule_task) diff --git a/packages/node/tests/scheduler/test_tasks_recovery.py b/packages/node/tests/scheduler/test_tasks_recovery.py new file mode 100644 index 000000000..8dbbfd1ba --- /dev/null +++ b/packages/node/tests/scheduler/test_tasks_recovery.py @@ -0,0 +1,138 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import pytest +import tempfile +import dbos +import logging +import time + +import octobot_node.scheduler +import octobot_node.scheduler.workflows.params as params + +QUEUE = dbos.Queue(name="test_queue") + +WF_TO_CREATE = 10 +WF_SLEEP_TIME = 1.5 # note: reducing this value wont speed up the test + +async def _init_dbos_scheduler(db_file_name: str, reset_database: bool = False): + config: dbos.DBOSConfig = { + "name": "scheduler_test", + "system_database_url": f"sqlite:///{db_file_name}", + "max_executor_threads": 2, # 2 is the minimum number of threads to let dbos recover properly with pending workflows + } + dbos.DBOS(config=config) + if reset_database: + dbos.DBOS.reset_system_database() + octobot_node.scheduler.SCHEDULER.INSTANCE = dbos.DBOS + + +class TestSchedulerRecovery: + + @pytest.mark.asyncio + async def test_recover_after_shutdown(self): + completed_workflows = [] + with tempfile.NamedTemporaryFile() as temp_file: + await _init_dbos_scheduler(temp_file.name, reset_database=True) + + @octobot_node.scheduler.SCHEDULER.INSTANCE.dbos_class() + class Sleeper(): + @staticmethod + @octobot_node.scheduler.SCHEDULER.INSTANCE.workflow() + async def sleeper_workflow(t: params.Tracker, identifier: float) -> float: + t.logger.info(f"{t.name}: sleeper_workflow {identifier} started") + await dbos.DBOS.sleep_async(WF_SLEEP_TIME) + t.logger.info(f"{t.name}: sleeper_workflow {identifier} done") + completed_workflows.append(identifier) + return identifier + + logging.info(f"Launching DBOS instance 1 ...") + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + logging.info(f"DBOS instance 1 launched") + + # 1. simple execution + t0 = time.time() + for i in range(WF_TO_CREATE): + await QUEUE.enqueue_async(Sleeper.sleeper_workflow, params.Tracker(name=f"sleeper_workflow_{i}"), i) + wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( + status=["ENQUEUED", "PENDING"] + ) + assert len(wfs) == WF_TO_CREATE + for wf_status in wfs: + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async(wf_status.workflow_id) + assert 0 <= await handle.get_result() < WF_TO_CREATE + duration = time.time() - t0 + logging.info(f"Workflow batch completed in {duration} seconds") + max_duration = WF_TO_CREATE * WF_SLEEP_TIME * 0.9 # 90% of the 1 by 1 time to ensure asynchronous execution. usually 3 to 4 seconds on a normal machine + assert duration <= max_duration, f"Workflow batch part 1 completed in {duration} seconds, expected <= {max_duration}" + assert sorted(completed_workflows) == list(range(WF_TO_CREATE)) + completed_workflows.clear() + + # 2. enqueue 10 more and restart + for i in range(WF_TO_CREATE): + await QUEUE.enqueue_async(Sleeper.sleeper_workflow, params.Tracker(name=f"sleeper_workflow_{i}"), i) + logging.info(f"Destroying DBOS instance 1 ...") + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + logging.info(f"DBOS instance 1 destroyed") + + # 3. restart and check completed workflows + logging.info(f"Launching DBOS instance 2 ...") + await _init_dbos_scheduler(temp_file.name) + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + logging.info(f"DBOS instance 2 launched") + all_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async() + assert len(all_wfs) == WF_TO_CREATE * 2 + pending_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( + status=["ENQUEUED", "PENDING"] + ) + assert len(pending_wfs) == WF_TO_CREATE + # enqueue a second batch of workflows + for i in range(WF_TO_CREATE, WF_TO_CREATE*2): + await QUEUE.enqueue_async(Sleeper.sleeper_workflow, params.Tracker(name=f"sleeper_workflow_2_{i}"), i) + t0 = time.time() + for wf_status in await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async(): + handle = await octobot_node.scheduler.SCHEDULER.INSTANCE.retrieve_workflow_async(wf_status.workflow_id) + assert 0 <= await handle.get_result() < WF_TO_CREATE*2 + duration = time.time() - t0 + logging.info(f"2 parallel workflow batches completed in {duration} seconds") + max_duration = WF_TO_CREATE * WF_SLEEP_TIME * 2 * 0.9 # 90% of the 1 by 1 time to ensure asynchronous execution. usually 3 to 4 seconds on a normal machine + assert duration < max_duration, f"Workflow batch part 2 completed in {duration} seconds, expected <= {max_duration}" + assert sorted(completed_workflows) == list(range(WF_TO_CREATE*2)) + logging.info(f"Destroying DBOS instance 2 ...") + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + logging.info(f"DBOS instance 2 destroyed") + + # 4. restart and check completed workflows + logging.info(f"Launching DBOS instance 3 ...") + await _init_dbos_scheduler(temp_file.name) + octobot_node.scheduler.SCHEDULER.INSTANCE.launch() + logging.info(f"DBOS instance 3 launched") + # all 30 worflows are now historized + pending_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async( + status=["ENQUEUED", "PENDING"] + ) + assert pending_wfs == [] + all_wfs = await octobot_node.scheduler.SCHEDULER.INSTANCE.list_workflows_async() + assert len(all_wfs) == WF_TO_CREATE * 3 + logging.info(f"Destroying DBOS instance 3 ...") + octobot_node.scheduler.SCHEDULER.INSTANCE.destroy() + logging.info(f"DBOS instance 3 destroyed") + + \ No newline at end of file diff --git a/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py b/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py index b52a763da..16a74f833 100644 --- a/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py +++ b/packages/tentacles/Meta/DSL_operators/blockchain_wallet_operators/blockchain_wallet_ops.py @@ -83,11 +83,6 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: ] async def pre_compute(self) -> None: - await super().pre_compute() - if exchange_manager is None: - raise octobot_commons.errors.DSLInterpreterError( - "exchange_manager is required for blockchain_wallet_balance operator" - ) param_by_name = self.get_computed_value_by_parameter() blockchain_wallet_balance_params = BlockchainWalletBalanceParams.from_dict(param_by_name) async with octobot_trading.api.blockchain_wallet_context( @@ -95,7 +90,7 @@ async def pre_compute(self) -> None: blockchain_descriptor=blockchain_wallet_balance_params.blockchain_descriptor, wallet_descriptor=blockchain_wallet_balance_params.wallet_descriptor, ), - exchange_manager.trader + exchange_manager.trader if exchange_manager else None ) as wallet: wallet_balance = await wallet.get_balance() self.value = float( @@ -124,10 +119,6 @@ def get_parameters(cls) -> list[dsl_interpreter.OperatorParameter]: async def pre_compute(self) -> None: await super().pre_compute() - if exchange_manager is None: - raise octobot_commons.errors.DSLInterpreterError( - "exchange_manager is required for blockchain_wallet_transfer operator" - ) param_by_name = self.get_computed_value_by_parameter() transfer_funds_params = TransferFundsParams.from_dict(param_by_name) async with octobot_trading.api.blockchain_wallet_context( @@ -135,7 +126,7 @@ async def pre_compute(self) -> None: blockchain_descriptor=transfer_funds_params.blockchain_descriptor, wallet_descriptor=transfer_funds_params.wallet_descriptor, ), - exchange_manager.trader + exchange_manager.trader if exchange_manager else None ) as wallet: if transfer_funds_params.address: address = transfer_funds_params.address diff --git a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py index dfe4721f4..8f14d2a20 100644 --- a/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py +++ b/packages/tentacles/Meta/DSL_operators/python_std_operators/tests/test_base_operators.py @@ -17,6 +17,8 @@ import pytest import mock import time + +import octobot_commons.constants import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors diff --git a/packages/tentacles/Meta/Keywords/scripting_library/configuration/profile_data_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/configuration/profile_data_configuration.py index 538e7bfe9..25d0da033 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/configuration/profile_data_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/configuration/profile_data_configuration.py @@ -25,7 +25,6 @@ import octobot_commons.configuration as commons_configuration import octobot_commons.profiles as commons_profiles import octobot_commons.profiles.profile_data as commons_profile_data -import octobot_commons.tentacles_management as tentacles_management import octobot_commons.time_frame_manager as time_frame_manager import octobot_commons.symbols import octobot_commons.logging @@ -36,6 +35,7 @@ import octobot_trading.exchanges as exchanges import octobot_trading.util.test_tools.exchange_data as exchange_data_import import octobot_trading.api +import octobot_trading.enums import octobot_tentacles_manager.api import octobot_tentacles_manager.configuration @@ -358,6 +358,16 @@ def _get_is_auth_required_exchange( ) +def get_required_candles_count(profile_data: commons_profiles.ProfileData, min_candles_count: int) -> int: + for tentacle_config in profile_data.tentacles: + if common_constants.CONFIG_TENTACLES_REQUIRED_CANDLES_COUNT in tentacle_config.config: + return max( + tentacle_config.config[common_constants.CONFIG_TENTACLES_REQUIRED_CANDLES_COUNT], + min_candles_count + ) + return min_candles_count + + def _set_portfolio( profile_data: commons_profiles.ProfileData, portfolio: dict @@ -365,6 +375,18 @@ def _set_portfolio( profile_data.trader_simulator.starting_portfolio = get_formatted_portfolio(portfolio) +def update_position_levarage( + position: exchange_data_import.PositionDetails, updated_contracts_by_symbol: dict +): + leverage = float( + updated_contracts_by_symbol[ + position.contract[octobot_trading.enums.ExchangeConstantsMarginContractColumns.PAIR.value] + ].current_leverage + ) + position.contract[octobot_trading.enums.ExchangeConstantsMarginContractColumns.CURRENT_LEVERAGE.value] = leverage + position.position[octobot_trading.enums.ExchangeConstantsPositionColumns.LEVERAGE.value] = leverage + + def get_formatted_portfolio(portfolio: dict): for asset in portfolio.values(): if common_constants.PORTFOLIO_AVAILABLE not in asset: @@ -496,7 +518,7 @@ def get_traded_coins( def get_time_frames( profile_data: commons_profiles.ProfileData, for_historical_data=False -): +) -> list[str]: for config in get_config_by_tentacle(profile_data).values(): if evaluators_constants.STRATEGIES_REQUIRED_TIME_FRAME in config: return config[evaluators_constants.STRATEGIES_REQUIRED_TIME_FRAME] diff --git a/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_collect_data_and_run_backtesting.py b/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_collect_data_and_run_backtesting.py index 6946de228..b613e941d 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_collect_data_and_run_backtesting.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/tests/backtesting/test_collect_data_and_run_backtesting.py @@ -76,12 +76,12 @@ async def test_collect_candles_without_backend_and_run_backtesting(trading_mode_ ) # 2. collect candles - ccxt_clients_cache._MARKETS_BY_EXCHANGE.clear() + ccxt_clients_cache._SHARED_MARKETS_EXCHANGE_BY_EXCHANGE.clear() await scripting_library.init_exchange_market_status_and_populate_backtesting_exchange_data( exchange_data, profile_data ) # cached markets have been updated and now contain this exchange markets - assert len(ccxt_clients_cache._MARKETS_BY_EXCHANGE) == 1 + assert len(ccxt_clients_cache._SHARED_MARKETS_EXCHANGE_BY_EXCHANGE) == 1 # ensure collected datas are correct assert len(exchange_data.markets) == 2 assert sorted([market.symbol for market in exchange_data.markets]) == ["BTC/USDT", "ETH/USDT"] diff --git a/packages/tentacles/Trading/Exchange/hollaex/hollaex_exchange.py b/packages/tentacles/Trading/Exchange/hollaex/hollaex_exchange.py index 35e140c59..4c9b7fda6 100644 --- a/packages/tentacles/Trading/Exchange/hollaex/hollaex_exchange.py +++ b/packages/tentacles/Trading/Exchange/hollaex/hollaex_exchange.py @@ -58,9 +58,12 @@ async def load_symbol_markets( if self.exchange_manager.exchange_name not in _REFRESHED_EXCHANGE_FEE_TIERS_BY_EXCHANGE_NAME: authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only() # always update fees cache using all markets to avoid market filter side effects from the current client - all_markets = ccxt_clients_cache.get_exchange_parsed_markets( - ccxt_clients_cache.get_client_key(self.client, authenticated_cache) - ) + if trading_constants.USE_CCXT_SHARED_MARKETS_CACHE: + all_markets = list(self.client.markets.values()) + else: + all_markets = ccxt_clients_cache.get_exchange_parsed_markets( + ccxt_clients_cache.get_client_key(self.client, authenticated_cache) + ) await self._refresh_exchange_fee_tiers(all_markets) async def disable_quick_trade_only_pairs(self): diff --git a/packages/trading/octobot_trading/api/blockchain_wallets.py b/packages/trading/octobot_trading/api/blockchain_wallets.py index d2c8dd3d2..14e0bdfc3 100644 --- a/packages/trading/octobot_trading/api/blockchain_wallets.py +++ b/packages/trading/octobot_trading/api/blockchain_wallets.py @@ -25,7 +25,7 @@ @contextlib.asynccontextmanager async def blockchain_wallet_context( parameters: blockchain_wallets.BlockchainWalletParameters, - trader: "octobot_trading.exchanges.Trader" + trader: typing.Optional["octobot_trading.exchanges.Trader"], ) -> typing.AsyncGenerator[blockchain_wallets.BlockchainWallet, None]: wallet = blockchain_wallets.create_blockchain_wallet(parameters, trader) async with wallet.open() as wallet: diff --git a/packages/trading/octobot_trading/api/symbol_data.py b/packages/trading/octobot_trading/api/symbol_data.py index b40ebf6d2..412f86445 100644 --- a/packages/trading/octobot_trading/api/symbol_data.py +++ b/packages/trading/octobot_trading/api/symbol_data.py @@ -21,7 +21,10 @@ import octobot_trading.enums import octobot_trading.exchange_data as exchange_data import octobot_trading.util as util +import octobot_trading.exchanges.util as exchange_util +if typing.TYPE_CHECKING: + import octobot_trading.exchanges.exchange_manager def get_symbol_data(exchange_manager, symbol, allow_creation=True) -> exchange_data.ExchangeSymbolData: return exchange_manager.exchange_symbols_data.get_exchange_symbol_data(symbol, allow_creation=allow_creation) @@ -150,9 +153,12 @@ def create_new_candles_manager(candles=None, max_candles_count=None) -> exchange return manager -def force_set_mark_price(exchange_manager, symbol, price): - exchange_manager.exchange_symbols_data.get_exchange_symbol_data(symbol).prices_manager.\ - set_mark_price(decimal.Decimal(str(price)), octobot_trading.enums.MarkPriceSources.EXCHANGE_MARK_PRICE.value) +def force_set_mark_price( + exchange_manager: "octobot_trading.exchanges.exchange_manager.ExchangeManager", + symbol: str, + price: typing.Union[float, decimal.Decimal], +) -> None: + return exchange_util.force_set_mark_price(exchange_manager, symbol, price) def is_mark_price_initialized(exchange_manager, symbol: str) -> bool: diff --git a/packages/trading/octobot_trading/blockchain_wallets/blockchain_wallet_factory.py b/packages/trading/octobot_trading/blockchain_wallets/blockchain_wallet_factory.py index cd3165573..310b0435a 100644 --- a/packages/trading/octobot_trading/blockchain_wallets/blockchain_wallet_factory.py +++ b/packages/trading/octobot_trading/blockchain_wallets/blockchain_wallet_factory.py @@ -19,7 +19,6 @@ import octobot_commons.tentacles_management as tentacles_management import octobot_trading.blockchain_wallets.blockchain_wallet as blockchain_wallet import octobot_trading.blockchain_wallets.blockchain_wallet_parameters as blockchain_wallet_parameters -import octobot_trading.blockchain_wallets.simulator.blockchain_wallet_simulator as blockchain_wallet_simulator if typing.TYPE_CHECKING: import octobot_trading.exchanges @@ -36,19 +35,24 @@ def get_blockchain_wallet_class_by_blockchain() -> dict[str, type[blockchain_wal def create_blockchain_wallet( parameters: blockchain_wallet_parameters.BlockchainWalletParameters, - trader: "octobot_trading.exchanges.Trader", + trader: typing.Optional["octobot_trading.exchanges.Trader"], ) -> blockchain_wallet.BlockchainWallet: """ Create a wallet of the given type :param parameters: the parameters of the wallet to create :return: the created wallet """ + blockchain_wallet_class = None try: - return get_blockchain_wallet_class_by_blockchain()[ + blockchain_wallet_class = get_blockchain_wallet_class_by_blockchain()[ parameters.blockchain_descriptor.blockchain - ](parameters) - except (KeyError, TypeError) as err: - if trader.simulate: - # use simulator wallet with trader callbacks to interact with simulated exchange wallet - return blockchain_wallet_simulator.BlockchainWalletSimulator(parameters, trader=trader) - raise ValueError(f"Blockchain {parameters.blockchain_descriptor.blockchain} not supported") from err + ] + try: + return blockchain_wallet_class(parameters) + except TypeError: + # trader arg is required for this wallet + return blockchain_wallet_class(parameters, trader=trader) + except KeyError as err: + raise ValueError( + f"Blockchain {parameters.blockchain_descriptor.blockchain} not supported" + ) from err diff --git a/packages/trading/octobot_trading/blockchain_wallets/simulator/blockchain_wallet_simulator.py b/packages/trading/octobot_trading/blockchain_wallets/simulator/blockchain_wallet_simulator.py index 818173f23..cce2568a3 100644 --- a/packages/trading/octobot_trading/blockchain_wallets/simulator/blockchain_wallet_simulator.py +++ b/packages/trading/octobot_trading/blockchain_wallets/simulator/blockchain_wallet_simulator.py @@ -15,6 +15,7 @@ # License along with this library. import typing import enum +import time import decimal import uuid @@ -53,7 +54,7 @@ class BlockchainWalletSimulator(blockchain_wallet.BlockchainWallet): def __init__( self, parameters: blockchain_wallet_parameters.BlockchainWalletParameters, - trader: "octobot_trading.exchanges.Trader" + trader: typing.Optional["octobot_trading.exchanges.Trader"] ): if parameters.blockchain_descriptor.network != octobot_trading.constants.SIMULATED_BLOCKCHAIN_NETWORK: # this is a simulator wallet, the network must be the simulated network @@ -68,7 +69,7 @@ def __init__( free=decimal.Decimal(0) ) } if parameters.blockchain_descriptor.native_coin_symbol else {} - self._trader: "octobot_trading.exchanges.Trader" = trader + self._trader: typing.Optional["octobot_trading.exchanges.Trader"] = trader super().__init__(parameters) if parameters.wallet_descriptor.specific_config: self._apply_wallet_descriptor_specific_config(parameters.wallet_descriptor.specific_config) @@ -123,13 +124,15 @@ def _ensure_native_coin_symbol(self): @staticmethod def create_wallet_descriptor_specific_config(**kwargs) -> dict: return { - BlockchainWalletSimulatorConfigurationKeys.ASSETS.value: { - BlockchainWalletSimulatorConfigurationKeys.ASSET.value: asset, - BlockchainWalletSimulatorConfigurationKeys.AMOUNT.value: amount, - } - for asset, amount in kwargs.get( - BlockchainWalletSimulatorConfigurationKeys.ASSETS.value, {} - ).items() + BlockchainWalletSimulatorConfigurationKeys.ASSETS.value: [ + { + BlockchainWalletSimulatorConfigurationKeys.ASSET.value: asset, + BlockchainWalletSimulatorConfigurationKeys.AMOUNT.value: amount, + } + for asset, amount in kwargs.get( + BlockchainWalletSimulatorConfigurationKeys.ASSETS.value, {} + ).items() + ] } def _apply_wallet_descriptor_specific_config(self, specific_config: dict): @@ -158,6 +161,10 @@ def _get_token_balance(self, asset: str) -> blockchain_wallet_adapter.Balance: ) async def _get_trader_deposit_address(self, asset: str) -> str: + if self._trader is None: + raise octobot_trading.errors.BlockchainWalletConfigurationError( + f"No trader is provided to {self.__class__.__name__} to get a deposit address" + ) return (await self._trader.get_deposit_address(asset))[ octobot_trading.enums.ExchangeConstantsDepositAddressColumns.ADDRESS.value ] @@ -172,13 +179,17 @@ async def _transfer_coin( f"Available: {holdings.free}, required: {amount}" ) transaction_id = str(uuid.uuid4()) - if to_address == await self._get_trader_deposit_address(asset): + if self._trader and to_address == await self._get_trader_deposit_address(asset): # this is an exchange deposit: credit the exchange portfolio await self._deposit_coin_on_trader_portfolio(asset, amount, to_address, transaction_id) + tx_timestamp = ( + self._trader.exchange_manager.exchange.get_exchange_current_time() + if self._trader else int(time.time()) + ) return blockchain_wallet_adapter.Transaction( txid=transaction_id, - timestamp=self._trader.exchange_manager.exchange.get_exchange_current_time(), + timestamp=tx_timestamp, address_from=self.wallet_descriptor.address, network=self.blockchain_descriptor.network, address_to=to_address, @@ -214,7 +225,7 @@ def _get_total_withdrawals_to_address(self, asset: str, to_address: str) -> deci currency=asset, transaction_type=octobot_trading.enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, ) - ) + ) if self._trader else octobot_trading.constants.ZERO def _get_total_deposits_from_address(self, asset: str, from_address: str) -> decimal.Decimal: return sum( # type: ignore @@ -225,4 +236,4 @@ def _get_total_deposits_from_address(self, asset: str, from_address: str) -> dec currency=asset, transaction_type=octobot_trading.enums.TransactionType.BLOCKCHAIN_DEPOSIT, ) - ) + ) if self._trader else octobot_trading.constants.ZERO diff --git a/packages/trading/octobot_trading/constants.py b/packages/trading/octobot_trading/constants.py index e4f838721..2b61b010a 100644 --- a/packages/trading/octobot_trading/constants.py +++ b/packages/trading/octobot_trading/constants.py @@ -205,6 +205,7 @@ ':read ECONNRESET:read ETIMEDOUT' ) ).split(":")) +USE_CCXT_SHARED_MARKETS_CACHE = os_util.parse_boolean_environment_var("USE_CCXT_SHARED_MARKETS_CACHE", "True") # exchange proxy RETRIABLE_EXCHANGE_PROXY_ERRORS_DESC: set[str] = set(os.getenv( @@ -213,6 +214,11 @@ # used to force margin type update before positions init (if necessary) FORCED_MARGIN_TYPE = enums.MarginType(os.getenv("FORCED_MARGIN_TYPE", enums.MarginType.ISOLATED.value)) +MINIMAL_POSITION_IDENTIFICATION_DETAILS_KEYS = [ + enums.ExchangeConstantsPositionColumns.LOCAL_ID.value, # to fetch position + enums.ExchangeConstantsPositionColumns.SYMBOL.value, # to fetch position + enums.ExchangeConstantsPositionColumns.LEVERAGE.value, # to keep user configured leverage +] # API API_LOGGER_TAG = "TradingApi" @@ -243,6 +249,8 @@ # History DEFAULT_SAVED_HISTORICAL_TIMEFRAMES = [commons_enums.TimeFrames.ONE_DAY] HISTORICAL_CANDLES_FETCH_DEFAULT_TIMEOUT = 30 +MIN_CANDLES_HISTORY_SIZE = 2 # ensure that at least 2 candles are fetch to avoid issues were candles are not yet +# available on exchange ending up in empty candles fetch # 946742400 is 01/01/2000, if trade time is lower, there is an issue. MINIMUM_VAL_TRADE_TIME = 946688400 diff --git a/packages/trading/octobot_trading/exchange_data/__init__.py b/packages/trading/octobot_trading/exchange_data/__init__.py index 906ddc3b0..e43255f77 100644 --- a/packages/trading/octobot_trading/exchange_data/__init__.py +++ b/packages/trading/octobot_trading/exchange_data/__init__.py @@ -93,6 +93,7 @@ MiniTickerProducer, MiniTickerChannel, TickerUpdaterSimulator, + TickerCache, ) from octobot_trading.exchange_data import contracts from octobot_trading.exchange_data.contracts import ( @@ -103,6 +104,7 @@ get_contract_type_from_symbol, update_contracts_from_positions, update_future_contract_from_dict, + initialize_contracts_from_exchange_data, create_default_future_contract, create_default_option_contract, create_contract, @@ -211,6 +213,7 @@ "MiniTickerProducer", "MiniTickerChannel", "TickerUpdaterSimulator", + "TickerCache", "Contract", "MarginContract", "FutureContract", @@ -218,6 +221,7 @@ "get_contract_type_from_symbol", "update_contracts_from_positions", "update_future_contract_from_dict", + "initialize_contracts_from_exchange_data", "create_default_future_contract", "create_default_option_contract", "create_contract", diff --git a/packages/trading/octobot_trading/exchange_data/contracts/__init__.py b/packages/trading/octobot_trading/exchange_data/contracts/__init__.py index 7c1dfa7db..fe8bfe2bf 100644 --- a/packages/trading/octobot_trading/exchange_data/contracts/__init__.py +++ b/packages/trading/octobot_trading/exchange_data/contracts/__init__.py @@ -37,6 +37,7 @@ from octobot_trading.exchange_data.contracts import contract_factory from octobot_trading.exchange_data.contracts.contract_factory import ( get_contract_type_from_symbol, + initialize_contracts_from_exchange_data, update_contracts_from_positions, update_future_contract_from_dict, create_default_future_contract, @@ -50,6 +51,7 @@ "FutureContract", "OptionContract", "get_contract_type_from_symbol", + "initialize_contracts_from_exchange_data", "update_contracts_from_positions", "update_future_contract_from_dict", "create_default_future_contract", diff --git a/packages/trading/octobot_trading/exchange_data/contracts/contract_factory.py b/packages/trading/octobot_trading/exchange_data/contracts/contract_factory.py index 7cf3de490..bd71fd700 100644 --- a/packages/trading/octobot_trading/exchange_data/contracts/contract_factory.py +++ b/packages/trading/octobot_trading/exchange_data/contracts/contract_factory.py @@ -13,6 +13,7 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import typing import decimal import octobot_commons.logging as logging @@ -25,6 +26,8 @@ import octobot_trading.exchange_data.contracts.future_contract as future_contract import octobot_trading.exchange_data.contracts.option_contract as option_contract +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import def update_contracts_from_positions(exchange_manager, positions) -> bool: updated = False @@ -58,6 +61,12 @@ def update_contracts_from_positions(exchange_manager, positions) -> bool: return updated +def initialize_contracts_from_exchange_data(exchange_manager, exchange_data: "exchange_data_import.ExchangeData") -> None: + for position_data in exchange_data.positions: + if position_data.contract: + update_future_contract_from_dict(exchange_manager, position_data.contract) + + def update_future_contract_from_dict(exchange_manager, contract: dict) -> bool: return exchange_manager.exchange.create_pair_contract( pair=contract[enums.ExchangeConstantsMarginContractColumns.PAIR.value], diff --git a/packages/trading/octobot_trading/exchange_data/ticker/__init__.py b/packages/trading/octobot_trading/exchange_data/ticker/__init__.py index b6e40cad9..0a2c48d35 100644 --- a/packages/trading/octobot_trading/exchange_data/ticker/__init__.py +++ b/packages/trading/octobot_trading/exchange_data/ticker/__init__.py @@ -28,6 +28,9 @@ MiniTickerProducer, MiniTickerChannel, ) +from octobot_trading.exchange_data.ticker.ticker_cache import ( + TickerCache, +) __all__ = [ "TickerManager", @@ -37,4 +40,5 @@ "MiniTickerProducer", "MiniTickerChannel", "TickerUpdaterSimulator", + "TickerCache", ] diff --git a/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py b/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py new file mode 100644 index 000000000..3482eaf24 --- /dev/null +++ b/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py @@ -0,0 +1,90 @@ +import typing +import cachetools + +import octobot_commons.constants +import octobot_commons.symbols +import octobot_commons.logging + + +class TickerCache: + + def __init__(self, ttl: float, maxsize: int): + # direct cache + self._ALL_TICKERS_BY_EXCHANGE_KEY: cachetools.TTLCache[str, dict[str, dict[str, float]]] = cachetools.TTLCache( + maxsize=maxsize, ttl=ttl + ) + + # indirect caches: + # - synchronized with _ALL_TICKERS_BY_EXCHANGE_KEY + # BTCUSDT => BTC/USDT + self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY: dict[str, dict[str, octobot_commons.symbols.Symbol]] = {} + # BTCUSDT => BTC/USDT + BTCUSDT => BTC/USDT:USDT + self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY: dict[str, dict[str, octobot_commons.symbols.Symbol]] = {} + + def is_valid_symbol(self, exchange_name: str, exchange_type: str, sandboxed: bool, symbol: str) -> bool: + try: + # will raise if symbol is missing (therefore invalid) + self._ALL_TICKERS_BY_EXCHANGE_KEY[ # pylint: disable=expression-not-assigned + self.get_exchange_key(exchange_name, exchange_type, sandboxed) + ][symbol] + return True + except KeyError: + return False + + def get_all_tickers( + self, exchange_name: str, exchange_type: str, sandboxed: bool, + default: typing.Optional[dict[str, dict[str, float]]] = None + ) -> typing.Optional[dict[str, dict[str, float]]]: + return self._ALL_TICKERS_BY_EXCHANGE_KEY.get(self.get_exchange_key(exchange_name, exchange_type, sandboxed), default) + + def has_ticker_data(self, exchange_name: str, exchange_type: str, sandboxed: bool) -> bool: + return self.get_exchange_key(exchange_name, exchange_type, sandboxed) in self._ALL_TICKERS_BY_EXCHANGE_KEY + + def get_all_parsed_symbols_by_merged_symbols( + self, exchange_name: str, exchange_type: str, sandboxed: bool, default=None + ) -> typing.Optional[dict[str, octobot_commons.symbols.Symbol]]: + # populated by set_all_tickers + # WARNING: does not expire when tickers expire: use has_ticker_data to check if cache is up-to-date + if exchange_type == octobot_commons.constants.CONFIG_EXCHANGE_FUTURE: + return self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY.get( + self.get_exchange_key(exchange_name, exchange_type, sandboxed), default + ) + return self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY.get( + self.get_exchange_key(exchange_name, exchange_type, sandboxed), default + ) + + def set_all_tickers( + self, exchange_name: str, exchange_type: str, sandboxed: bool, tickers: dict, replace_all: bool = True + ): + sandbox = " sandbox" if sandboxed else "" + key = self.get_exchange_key(exchange_name, exchange_type, sandboxed) + merged_tickers = tickers if replace_all else { + **self._ALL_TICKERS_BY_EXCHANGE_KEY.get(key, {}), **tickers + } + octobot_commons.logging.get_logger(self.__class__.__name__).info( + f"Refreshed {len(tickers)} ({len(tickers)})/{len(merged_tickers)}) tickers cache for {exchange_name} {exchange_type}{sandbox}" + ) + self._ALL_TICKERS_BY_EXCHANGE_KEY[key] = merged_tickers + self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = { + octobot_commons.symbols.parse_symbol(symbol).merged_str_symbol(market_separator=""): + octobot_commons.symbols.parse_symbol(symbol) + for symbol in merged_tickers + } + if exchange_type == octobot_commons.constants.CONFIG_EXCHANGE_FUTURE: + self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = { + **self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key], + **{ + octobot_commons.symbols.parse_symbol(symbol).merged_str_base_and_quote_only_symbol(market_separator=""): + octobot_commons.symbols.parse_symbol(symbol) + for symbol in merged_tickers + } + } + + def reset_all_tickers_cache(self): + self._ALL_TICKERS_BY_EXCHANGE_KEY.clear() + self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY.clear() + self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY.clear() + + @staticmethod + def get_exchange_key(exchange_name: str, exchange_type: str, sandboxed: bool) -> str: + return f"{exchange_name}_{exchange_type or octobot_commons.constants.CONFIG_EXCHANGE_SPOT}_{sandboxed}" diff --git a/packages/trading/octobot_trading/exchanges/__init__.py b/packages/trading/octobot_trading/exchanges/__init__.py index 26fe16a48..3e6a6d2ad 100644 --- a/packages/trading/octobot_trading/exchanges/__init__.py +++ b/packages/trading/octobot_trading/exchanges/__init__.py @@ -83,6 +83,12 @@ is_proxy_config_compatible_with_websocket_connector, search_websocket_class, supports_websocket, + force_set_mark_price, + get_traded_assets, +) +from octobot_trading.exchanges import market_filters +from octobot_trading.exchanges.market_filters.market_filter_factory import ( + create_market_filter, ) from octobot_trading.exchanges import exchange_websocket_factory from octobot_trading.exchanges.exchange_websocket_factory import ( @@ -170,6 +176,8 @@ "get_auto_filled_exchange_names", "get_exchange_details", "is_error_on_this_type", + "force_set_mark_price", + "get_traded_assets", "AbstractExchange", "is_channel_managed_by_websocket", "is_channel_fully_managed_by_websocket", @@ -202,4 +210,5 @@ "ExchangeSimulatorAdapter", "retried_failed_network_request", "ExchangeDetails", + "create_market_filter", ] diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py index 3b66f33d7..a6e2b7618 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py @@ -24,7 +24,6 @@ class ProxyConnectionError(Exception): import os import ssl import aiohttp -import copy import logging import typing import ccxt @@ -114,24 +113,6 @@ def create_client( async def close_client(client): await client.close() - client.markets = {} - client.markets_by_id = {} - client.ids = [] - client.last_json_response = {} - client.last_http_response = "" - client.last_response_headers = {} - client.markets_loading = None - client.currencies = {} - client.baseCurrencies = {} - client.quoteCurrencies = {} - client.currencies_by_id = {} - client.codes = [] - client.symbols = {} - client.accounts = [] - client.accounts_by_id = {} - client.ohlcvs = {} - client.trades = {} - client.orderbooks = {} def get_unauthenticated_exchange( @@ -196,23 +177,18 @@ async def _filted_fetched_markets(*args, **kwargs): client.fetch_markets = origin_fetch_markets -def load_markets_from_cache(client, authenticated_cache: bool, market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None): +def load_markets_from_cache(client: async_ccxt.Exchange, authenticated_cache: bool, market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None): client_key = ccxt_clients_cache.get_client_key(client, authenticated_cache) - client.set_markets( - market - for market in ccxt_clients_cache.get_exchange_parsed_markets(client_key) - if market_filter is None or market_filter(market) - ) + ccxt_clients_cache.apply_exchange_markets_cache(client_key, client, market_filter) if time_difference := ccxt_clients_cache.get_exchange_time_difference(client_key): - client.options[ccxt_constants.CCXT_TIME_DIFFERENCE] = time_difference + if client.options: + client.options[ccxt_constants.CCXT_TIME_DIFFERENCE] = time_difference -def set_markets_cache(client, authenticated_cache: bool): +def set_ccxt_client_cache(client: async_ccxt.Exchange, authenticated_cache: bool): if client.markets: client_key = ccxt_clients_cache.get_client_key(client, authenticated_cache) - ccxt_clients_cache.set_exchange_parsed_markets( - client_key, copy.deepcopy(list(client.markets.values())) - ) + ccxt_clients_cache.set_exchange_markets_cache(client_key, client) if time_difference := client.options.get(ccxt_constants.CCXT_TIME_DIFFERENCE): ccxt_clients_cache.set_exchange_time_difference(client_key, time_difference) diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_clients_cache.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_clients_cache.py index 2e4270ef6..2829fdef2 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_clients_cache.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_clients_cache.py @@ -15,24 +15,28 @@ # License along with this library. import cachetools import json +import copy import typing import contextlib +import ccxt.async_support as async_ccxt import octobot_commons.constants as commons_constants - +import octobot_trading.constants as trading_constants # To avoid side effects related to a cache refresh at a fix time of the day every day, # cache should not be refreshed at the same time every day. # Use 30h and 18min as a period. It could be anything else as long as it doesn't make it so # that cache ends up refreshed approximately at the same time of the day _CACHE_TIME = commons_constants.HOURS_TO_SECONDS * 30 + commons_constants.MINUTE_TO_SECONDS * 18 -_MARKETS_BY_EXCHANGE = cachetools.TTLCache(maxsize=50, ttl=_CACHE_TIME) +_MARKETS_BY_EXCHANGE: cachetools.TTLCache[str, list[dict]] = cachetools.TTLCache(maxsize=50, ttl=_CACHE_TIME) +_SHARED_MARKETS_EXCHANGE_BY_EXCHANGE: cachetools.TTLCache[str, async_ccxt.Exchange] = cachetools.TTLCache(maxsize=50, ttl=_CACHE_TIME) # Time difference between system clock and exchange server clock, fetched when needed when loading market statuses _TIME_DIFFERENCE_BY_EXCHANGE: dict[str, float] = {} # use short cache time for authenticated markets to avoid caching them for too long _AUTH_CACHE_TIME = 15 * commons_constants.MINUTE_TO_SECONDS -_AUTH_MARKETS_BY_EXCHANGE = cachetools.TTLCache(maxsize=50, ttl=_AUTH_CACHE_TIME) +_AUTH_MARKETS_BY_EXCHANGE: cachetools.TTLCache[str, list[dict]] = cachetools.TTLCache(maxsize=50, ttl=_AUTH_CACHE_TIME) +_AUTH_SHARED_MARKETS_EXCHANGE_BY_EXCHANGE: cachetools.TTLCache[str, async_ccxt.Exchange] = cachetools.TTLCache(maxsize=50, ttl=_AUTH_CACHE_TIME) _UNAUTHENTICATED_SUFFIX = "unauthenticated" @@ -41,6 +45,29 @@ def get_client_key(client, authenticated_cache: bool) -> str: return f"{client.__class__.__name__}:{json.dumps(client.urls.get('api'))}:{suffix}" +def set_exchange_markets_cache(client_key: str, client: async_ccxt.Exchange): + if trading_constants.USE_CCXT_SHARED_MARKETS_CACHE: + set_cached_shared_markets_exchange(client_key, client) + else: + set_exchange_parsed_markets( + client_key, copy.deepcopy(list(client.markets.values())) + ) + + +def apply_exchange_markets_cache( + client_key: str, client: async_ccxt.Exchange, + market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None +): + if trading_constants.USE_CCXT_SHARED_MARKETS_CACHE: + client.set_markets_from_exchange(get_cached_shared_markets_exchange(client_key)) + else: + client.set_markets( + market + for market in get_exchange_parsed_markets(client_key) + if market_filter is None or market_filter(market) + ) + + def get_exchange_parsed_markets(client_key: str): return _get_cached_markets(client_key)[client_key] @@ -49,12 +76,28 @@ def set_exchange_parsed_markets(client_key: str, markets): _get_cached_markets(client_key)[client_key] = markets -def _get_cached_markets(client_key: str) -> cachetools.TTLCache: +def _get_cached_markets(client_key: str) -> cachetools.TTLCache[str, list[dict]]: + # used when USE_CCXT_SHARED_MARKETS_CACHE is False if _is_authenticated_cache(client_key): return _AUTH_MARKETS_BY_EXCHANGE return _MARKETS_BY_EXCHANGE +def get_cached_shared_markets_exchange(client_key: str) -> async_ccxt.Exchange: + return _get_shared_markets_exchange_cache(client_key)[client_key] + + +def set_cached_shared_markets_exchange(client_key: str, exchange: async_ccxt.Exchange): + _get_shared_markets_exchange_cache(client_key)[client_key] = exchange + + +def _get_shared_markets_exchange_cache(client_key: str) -> cachetools.TTLCache[str, async_ccxt.Exchange]: + # used when USE_CCXT_SHARED_MARKETS_CACHE is True + if _is_authenticated_cache(client_key): + return _AUTH_SHARED_MARKETS_EXCHANGE_BY_EXCHANGE + return _SHARED_MARKETS_EXCHANGE_BY_EXCHANGE + + def get_exchange_time_difference(client_key: str) -> typing.Optional[float]: return _TIME_DIFFERENCE_BY_EXCHANGE.get(client_key, None) diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py index 57ef62f5b..833f500ab 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py @@ -244,7 +244,7 @@ async def load_symbol_markets( ) try: await self._load_markets(self.client, reload, market_filter=market_filter) - ccxt_client_util.set_markets_cache(self.client, authenticated_cache) + ccxt_client_util.set_ccxt_client_cache(self.client, authenticated_cache) except ( ccxt.AuthenticationError, ccxt.ArgumentsRequired, ccxt.static_dependencies.ecdsa.der.UnexpectedDER, binascii.Error, AssertionError, IndexError @@ -283,7 +283,7 @@ async def load_symbol_markets( try: unauth_client = self._client_factory(True)[0] await self._load_markets(unauth_client, reload, market_filter=market_filter) - ccxt_client_util.set_markets_cache(unauth_client, False) + ccxt_client_util.set_ccxt_client_cache(unauth_client, False) # apply markets to target client ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter) self.logger.debug( diff --git a/packages/trading/octobot_trading/exchanges/exchange_builder.py b/packages/trading/octobot_trading/exchanges/exchange_builder.py index 5d4a14f0e..300edde47 100644 --- a/packages/trading/octobot_trading/exchanges/exchange_builder.py +++ b/packages/trading/octobot_trading/exchanges/exchange_builder.py @@ -258,14 +258,6 @@ def use_cached_markets(self, use_cached_markets: bool): def use_market_filter(self, market_filter: typing.Union[None, typing.Callable[[dict], bool]]): self.exchange_manager.market_filter = market_filter return self - - def set_rest_exchange(self, rest_exchange: typing.Optional["exchanges.RestExchange"]): - self.exchange_manager.preconfigured_exchange = rest_exchange - return self - - def leave_rest_exchange_open(self, leave_rest_exchange_open: bool): - self.exchange_manager.leave_rest_exchange_open = leave_rest_exchange_open - return self def is_ignoring_config(self, ignore_config=True): self.exchange_manager.ignore_config = ignore_config diff --git a/packages/trading/octobot_trading/exchanges/exchange_factory.py b/packages/trading/octobot_trading/exchanges/exchange_factory.py index 486e95d94..bd7b27ed7 100644 --- a/packages/trading/octobot_trading/exchanges/exchange_factory.py +++ b/packages/trading/octobot_trading/exchanges/exchange_factory.py @@ -54,14 +54,9 @@ async def create_real_exchange(exchange_manager, exchange_config_by_exchange: ty :param exchange_manager: the related exchange manager :param exchange_config_by_exchange: optional exchange configurations """ - if exchange_manager.preconfigured_exchange: - exchange_manager.exchange = exchange_manager.preconfigured_exchange - exchange_manager.exchange.exchange_manager = exchange_manager - else: - await _create_rest_exchange(exchange_manager, exchange_config_by_exchange) + await _create_rest_exchange(exchange_manager, exchange_config_by_exchange) try: - if exchange_manager.preconfigured_exchange is None: - await exchange_manager.exchange.initialize() + await exchange_manager.exchange.initialize() _create_exchange_backend(exchange_manager) if exchange_manager.exchange_only: return diff --git a/packages/trading/octobot_trading/exchanges/exchange_manager.py b/packages/trading/octobot_trading/exchanges/exchange_manager.py index ab0aca394..ba82feb7a 100644 --- a/packages/trading/octobot_trading/exchanges/exchange_manager.py +++ b/packages/trading/octobot_trading/exchanges/exchange_manager.py @@ -24,7 +24,7 @@ import octobot_trading.exchange_channel as exchange_channel import octobot_trading.exchanges as exchanges import octobot_trading.personal_data as personal_data -import octobot_trading.exchange_data as exchange_data +import octobot_trading.exchange_data import octobot_trading.constants as constants import octobot_trading.enums as enums import octobot_trading.util as util @@ -34,6 +34,9 @@ import trading_backend.exchanges +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + class ExchangeManager(util.Initializable): def __init__(self, config, exchange_class_string): super().__init__() @@ -74,8 +77,6 @@ def __init__(self, config, exchange_class_string): self.trader: exchanges.Trader = None # type: ignore self.exchange: exchanges.RestExchange = None # type: ignore - self.preconfigured_exchange: typing.Optional[exchanges.RestExchange] = None - self.leave_rest_exchange_open: bool = False self.exchange_backend: trading_backend.exchanges.Exchange = None # type: ignore self.is_broker_enabled: bool = False self.trading_modes: list = [] @@ -88,7 +89,9 @@ def __init__(self, config, exchange_class_string): self.storage_manager: storage.StorageManager = storage.StorageManager(self) self.exchange_config: exchanges.ExchangeConfig = exchanges.ExchangeConfig(self) self.exchange_personal_data: personal_data.ExchangePersonalData = personal_data.ExchangePersonalData(self) - self.exchange_symbols_data: exchange_data.ExchangeSymbolsData = exchange_data.ExchangeSymbolsData(self) + self.exchange_symbols_data: octobot_trading.exchange_data.ExchangeSymbolsData = ( + octobot_trading.exchange_data.ExchangeSymbolsData(self) + ) self.debug_info: dict[str, typing.Any] = {} @@ -134,26 +137,24 @@ async def stop(self, warning_on_missing_elements=True, enable_logs=True): # stop exchange channels if enable_logs: self.logger.debug(f"Stopping exchange channels for exchange_id: {self.id} ...") - if self.exchange is not None and not self.leave_rest_exchange_open: - try: - exchange_channel.get_exchange_channels(self.id) - await exchange_channel.stop_exchange_channels(self, should_warn=warning_on_missing_elements) - except KeyError: - # no exchange channel to stop - pass - except Exception as err: - self.logger.exception(err, True, f"Error when stopping exchange channels: {err}") + try: + exchange_channel.get_exchange_channels(self.id) + await exchange_channel.stop_exchange_channels(self, should_warn=warning_on_missing_elements) + except KeyError: + # no exchange channel to stop + pass + except Exception as err: + self.logger.exception(err, True, f"Error when stopping exchange channels: {err}") + if self.exchange is not None: + # ensure self.exchange still exists as await self.exchange.stop() + # internally uses asyncio.sleep within ccxt + exchanges.Exchanges.instance().del_exchange( + self.exchange.name, self.id, should_warn=warning_on_missing_elements + ) try: await self.exchange.stop() except Exception as err: self.logger.exception(err, True, f"Error when stopping exchange: {err}") - if self.exchange is not None: - # ensure self.exchange still exists as await self.exchange.stop() - # internally uses asyncio.sleep within ccxt - exchanges.Exchanges.instance().del_exchange( - self.exchange.name, self.id, should_warn=warning_on_missing_elements - ) - self.exchange.exchange_manager = None # type: ignore self.exchange = None # type: ignore if self.exchange_personal_data is not None: try: @@ -206,6 +207,37 @@ async def register_trader(self, trader): await self.exchange_personal_data.initialize() await self.exchange_config.initialize() + async def initialize_from_exchange_data( + self, + exchange_data: "exchange_data_import.ExchangeData", + price_by_symbol: dict[str, float], + ignore_orders_and_trades: bool, + lock_chained_orders_funds: bool, + as_simulator: bool, + ) -> None: + """ + Initialize trader positions and orders from exchange data by delegating to all relevant managers. + """ + await self.trader.initialize() + self.exchange_personal_data.portfolio_manager.portfolio_value_holder.initialize_from_exchange_data( + exchange_data, price_by_symbol + ) + if not ignore_orders_and_trades: + if exchange_data.trades: + self.exchange_personal_data.trades_manager.initialize_from_exchange_data(exchange_data) + if ( + exchange_data.orders_details.open_orders + and exchange_data.orders_details.open_orders[0] + .get(constants.STORAGE_ORIGIN_VALUE, {}) + .get(enums.ExchangeConstantsOrderColumns.TYPE.value) + ): + await self.exchange_personal_data.orders_manager.initialize_from_exchange_data(exchange_data) + if lock_chained_orders_funds: + await self.exchange_personal_data.portfolio_manager.initialize_from_exchange_data(exchange_data) + self.exchange_personal_data.positions_manager.initialize_from_exchange_data( + exchange_data, exclusively_use_exchange_position_details=not as_simulator + ) + def load_constants(self): if not self.is_backtesting: self._load_config_symbols_and_time_frames() @@ -220,7 +252,7 @@ def need_user_stream(self): return self.config[common_constants.CONFIG_TRADER][common_constants.CONFIG_ENABLED_OPTION] def reset_exchange_symbols_data(self): - self.exchange_symbols_data = exchange_data.ExchangeSymbolsData(self) + self.exchange_symbols_data = octobot_trading.exchange_data.ExchangeSymbolsData(self) def reset_exchange_personal_data(self): self.exchange_personal_data = personal_data.ExchangePersonalData(self) diff --git a/packages/trading/octobot_trading/exchanges/market_filters/__init__.py b/packages/trading/octobot_trading/exchanges/market_filters/__init__.py new file mode 100644 index 000000000..d01670038 --- /dev/null +++ b/packages/trading/octobot_trading/exchanges/market_filters/__init__.py @@ -0,0 +1,5 @@ +from octobot_trading.exchanges.market_filters.market_filter_factory import create_market_filter + +__all__ = [ + "create_market_filter", +] \ No newline at end of file diff --git a/packages/trading/octobot_trading/exchanges/market_filters/market_filter_factory.py b/packages/trading/octobot_trading/exchanges/market_filters/market_filter_factory.py new file mode 100644 index 000000000..4f68bab61 --- /dev/null +++ b/packages/trading/octobot_trading/exchanges/market_filters/market_filter_factory.py @@ -0,0 +1,52 @@ +import typing + +import octobot_commons.constants as common_constants + +import octobot_trading.enums as trading_enums +import octobot_trading.util.test_tools.exchange_data_util as exchange_data_util + +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + + +def create_market_filter( + exchange_data: typing.Optional["exchange_data_import.ExchangeData"], + to_keep_quote: typing.Optional[str], + to_keep_symbols: typing.Optional[typing.Iterable[str]] = None, + to_keep_quotes: typing.Optional[typing.Iterable[str]] = None, + force_usd_like_markets: bool = True, +) -> typing.Callable[[dict], bool]: + relevant_symbols_to_keep = set(to_keep_symbols or []) # forced symbols + if exchange_data: + relevant_symbols_to_keep.update(exchange_data_util.get_orders_and_positions_symbols(exchange_data)) # orders/positions symbols + relevant_symbols_to_keep.update(market.symbol for market in exchange_data.markets) # always in symbols in markets + merged_to_keep_quotes = set(to_keep_quotes or []) + if to_keep_quote: + merged_to_keep_quotes.add(to_keep_quote) + + def market_filter(market: dict) -> bool: + if market[trading_enums.ExchangeConstantsMarketStatusColumns.SYMBOL.value] in relevant_symbols_to_keep: + return True + base = market[trading_enums.ExchangeConstantsMarketStatusColumns.CURRENCY.value] + quote = market[trading_enums.ExchangeConstantsMarketStatusColumns.MARKET.value] + return ( + ( + # 1. all "X/to_keep_quote" markets + # => always required to run the strategy + quote in merged_to_keep_quotes or + # 2. all "to_keep_quote/X" markets + # => used in portfolio optimization. Ex: to buy BTC from USDT when BTC is the "to_keep_quote", + # BTC/USD-like market is required + base in merged_to_keep_quotes or + # 3. all USD-like/X markets + # => used in portfolio optimization. Ex: to be able to convert USD like currencies into the + # same USD-like currency + (force_usd_like_markets and base in common_constants.USD_LIKE_COINS) + ) + and ( + market[trading_enums.ExchangeConstantsMarketStatusColumns.TYPE.value] == + trading_enums.ExchangeTypes.SPOT.value + ) + ) + + return market_filter diff --git a/packages/trading/octobot_trading/exchanges/traders/trader.py b/packages/trading/octobot_trading/exchanges/traders/trader.py index d4b20d976..e5a57812f 100644 --- a/packages/trading/octobot_trading/exchanges/traders/trader.py +++ b/packages/trading/octobot_trading/exchanges/traders/trader.py @@ -15,6 +15,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import decimal +import uuid import typing import asyncio @@ -1172,3 +1173,6 @@ def _has_open_position(self, symbol): """ return len(self.exchange_manager.exchange_personal_data.positions_manager.get_symbol_positions( symbol=symbol)) != 0 + + def generate_random_order_id(self) -> str: + return str(uuid.uuid4()) diff --git a/packages/trading/octobot_trading/exchanges/traders/trader_simulator.py b/packages/trading/octobot_trading/exchanges/traders/trader_simulator.py index af6879e54..d0a08ddae 100644 --- a/packages/trading/octobot_trading/exchanges/traders/trader_simulator.py +++ b/packages/trading/octobot_trading/exchanges/traders/trader_simulator.py @@ -15,7 +15,6 @@ # License along with this library. import decimal import time -import uuid import octobot_trading.constants import octobot_trading.enums as enums @@ -59,7 +58,7 @@ async def _withdraw_on_exchange( self, asset: str, amount: decimal.Decimal, network: str, address: str, tag: str = "", params: dict = None ) -> dict: deposit_address = await self.get_deposit_address(asset) - transaction_id = str(uuid.uuid4()) + transaction_id = self.generate_random_order_id() return { enums.ExchangeConstantsTransactionColumns.TXID.value: transaction_id, enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: time.time(), diff --git a/packages/trading/octobot_trading/exchanges/util/__init__.py b/packages/trading/octobot_trading/exchanges/util/__init__.py index 84caff0b4..8f3f1fc34 100644 --- a/packages/trading/octobot_trading/exchanges/util/__init__.py +++ b/packages/trading/octobot_trading/exchanges/util/__init__.py @@ -46,6 +46,8 @@ get_auto_filled_exchange_names, get_exchange_details, is_error_on_this_type, + force_set_mark_price, + get_traded_assets, ) from octobot_trading.exchanges.util import websockets_util from octobot_trading.exchanges.util.websockets_util import ( @@ -86,4 +88,6 @@ "is_proxy_config_compatible_with_websocket_connector", "search_websocket_class", "supports_websocket", + "force_set_mark_price", + "get_traded_assets", ] diff --git a/packages/trading/octobot_trading/exchanges/util/exchange_util.py b/packages/trading/octobot_trading/exchanges/util/exchange_util.py index 9279a90e6..be5a3b6ca 100644 --- a/packages/trading/octobot_trading/exchanges/util/exchange_util.py +++ b/packages/trading/octobot_trading/exchanges/util/exchange_util.py @@ -16,6 +16,7 @@ import contextlib import typing import ccxt +import decimal import trading_backend import octobot_commons.logging as logging @@ -36,6 +37,8 @@ import octobot_trading.exchanges.exchange_details as exchange_details import octobot_trading.exchanges.exchange_builder as exchange_builder +if typing.TYPE_CHECKING: + import octobot_trading.exchanges.exchange_manager def get_rest_exchange_class( exchange_name: str, tentacles_setup_config, exchange_config_by_exchange: typing.Optional[dict[str, dict]] @@ -222,8 +225,6 @@ async def get_local_exchange_manager( is_broker_enabled: bool = False, exchange_config_by_exchange: typing.Optional[dict[str, dict]] = None, disable_unauth_retry: bool = False, market_filter: typing.Union[None, typing.Callable[[dict], bool]] = None, - rest_exchange: typing.Optional[exchanges_types.RestExchange] = None, - leave_rest_exchange_open: bool = False, ): exchange_type = exchange_config.get(common_constants.CONFIG_EXCHANGE_TYPE, get_default_exchange_type(exchange_name)) builder = builder or exchange_builder.ExchangeBuilder( @@ -241,8 +242,6 @@ async def get_local_exchange_manager( .is_broker_enabled(is_broker_enabled) \ .use_cached_markets(use_cached_markets) \ .use_market_filter(market_filter) \ - .set_rest_exchange(rest_exchange) \ - .leave_rest_exchange_open(leave_rest_exchange_open) \ .is_ignoring_config(ignore_config) \ .disable_trading_mode() \ .build() @@ -486,3 +485,23 @@ def is_error_on_this_type(error: BaseException, descriptions: typing.List[typing if all(identifier in lower_error for identifier in identifiers): return True return False + + +def get_traded_assets(exchange_manager: "octobot_trading.exchanges.exchange_manager.ExchangeManager") -> list: + # use list to maintain order + assets = [] + for symbol in exchange_manager.exchange_config.traded_symbols: + if symbol.base not in assets: + assets.append(symbol.base) + if symbol.quote not in assets: + assets.append(symbol.quote) + return assets + + +def force_set_mark_price( + exchange_manager: "octobot_trading.exchanges.exchange_manager.ExchangeManager", + symbol: str, + price: typing.Union[float, decimal.Decimal] +) -> None: + exchange_manager.exchange_symbols_data.get_exchange_symbol_data(symbol).prices_manager.\ + set_mark_price(decimal.Decimal(str(price)), enums.MarkPriceSources.EXCHANGE_MARK_PRICE.value) diff --git a/packages/trading/octobot_trading/personal_data/__init__.py b/packages/trading/octobot_trading/personal_data/__init__.py index 0a0b8321c..14f93115a 100644 --- a/packages/trading/octobot_trading/personal_data/__init__.py +++ b/packages/trading/octobot_trading/personal_data/__init__.py @@ -54,6 +54,8 @@ generate_order_id, wait_for_order_fill, get_short_order_summary, + get_enriched_orders_by_exchange_id, + get_symbol_count, apply_order_storage_details_if_any, create_orders_storage_related_elements, create_missing_virtual_orders_from_storage_order_groups, @@ -112,6 +114,7 @@ create_order_instance, create_order_from_dict, create_order_from_order_storage_details, + create_order_from_order_raw_in_storage_details_without_related_elements, OrderFactory, create_and_register_chained_order_on_base_order, OrdersProducer, @@ -331,6 +334,8 @@ "generate_order_id", "wait_for_order_fill", "get_short_order_summary", + "get_enriched_orders_by_exchange_id", + "get_symbol_count", "apply_order_storage_details_if_any", "create_orders_storage_related_elements", "create_missing_virtual_orders_from_storage_order_groups", @@ -389,6 +394,7 @@ "create_order_instance", "create_order_from_dict", "create_order_from_order_storage_details", + "create_order_from_order_raw_in_storage_details_without_related_elements", "OrderFactory", "create_and_register_chained_order_on_base_order", "OrdersProducer", diff --git a/packages/trading/octobot_trading/personal_data/orders/__init__.py b/packages/trading/octobot_trading/personal_data/orders/__init__.py index f8b6242a8..3c23ca853 100644 --- a/packages/trading/octobot_trading/personal_data/orders/__init__.py +++ b/packages/trading/octobot_trading/personal_data/orders/__init__.py @@ -144,6 +144,8 @@ wait_for_order_fill, get_short_order_summary, create_and_register_chained_order_on_base_order, + get_enriched_orders_by_exchange_id, + get_symbol_count, ) from octobot_trading.personal_data.orders import orders_storage_operations from octobot_trading.personal_data.orders.orders_storage_operations import ( @@ -183,6 +185,7 @@ create_order_instance, create_order_from_dict, create_order_from_order_storage_details, + create_order_from_order_raw_in_storage_details_without_related_elements, OrderFactory, ) @@ -280,6 +283,7 @@ "create_order_instance", "create_order_from_dict", "create_order_from_order_storage_details", + "create_order_from_order_raw_in_storage_details_without_related_elements", "OrderFactory", "OrdersProducer", "OrdersChannel", @@ -306,4 +310,6 @@ "TakeProfitLimitOrder", "TrailingStopOrder", "TrailingStopLimitOrder", + "get_enriched_orders_by_exchange_id", + "get_symbol_count", ] diff --git a/packages/trading/octobot_trading/personal_data/orders/order_factory.py b/packages/trading/octobot_trading/personal_data/orders/order_factory.py index fb3eafcbf..942aa787f 100644 --- a/packages/trading/octobot_trading/personal_data/orders/order_factory.py +++ b/packages/trading/octobot_trading/personal_data/orders/order_factory.py @@ -172,6 +172,20 @@ def create_order_from_dict( ) +def create_order_from_order_raw_in_storage_details_without_related_elements( + exchange_manager: "octobot_trading.exchanges.ExchangeManager", + order_details: dict +) -> "personal_data.Order": + """ + unlike create_order_from_order_storage_details, will not create related elements and will + parse order from raw dict + """ + order_dict = order_details[constants.STORAGE_ORIGIN_VALUE] + order = personal_data.create_order_instance_from_raw(exchange_manager.trader, order_dict) + order.update_from_storage_order_details(order_details) + return order + + async def create_order_from_order_storage_details( order_storage_details: dict, exchange_manager: "octobot_trading.exchanges.ExchangeManager", diff --git a/packages/trading/octobot_trading/personal_data/orders/order_util.py b/packages/trading/octobot_trading/personal_data/orders/order_util.py index c94414119..5ad9670e2 100644 --- a/packages/trading/octobot_trading/personal_data/orders/order_util.py +++ b/packages/trading/octobot_trading/personal_data/orders/order_util.py @@ -19,6 +19,7 @@ import contextlib import uuid import typing +import collections import octobot_commons.symbols as symbol_util import octobot_commons.constants as commons_constants @@ -881,3 +882,21 @@ def get_short_order_summary(order: typing.Union[dict, order_import.Order]) -> st f"{order[enums.ExchangeConstantsOrderColumns.TYPE.value]} {order[enums.ExchangeConstantsOrderColumns.AMOUNT.value]}{filled} " f"{order[enums.ExchangeConstantsOrderColumns.SYMBOL.value]} at {order[enums.ExchangeConstantsOrderColumns.PRICE.value]} cost: {round(cost, 8)}" ) + + +def get_enriched_orders_by_exchange_id(enriched_orders: list[dict]) -> dict[str, dict]: + return { + order_details[constants.STORAGE_ORIGIN_VALUE][ + enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + ]: order_details + for order_details in enriched_orders + } + + +def get_symbol_count(raw_trades_or_raw_orders: list[dict]) -> dict[str, int]: + return dict( + collections.Counter( + element[enums.ExchangeConstantsOrderColumns.SYMBOL.value] + for element in raw_trades_or_raw_orders + ) + ) diff --git a/packages/trading/octobot_trading/personal_data/orders/orders_manager.py b/packages/trading/octobot_trading/personal_data/orders/orders_manager.py index 8d64f0d97..698f66f34 100644 --- a/packages/trading/octobot_trading/personal_data/orders/orders_manager.py +++ b/packages/trading/octobot_trading/personal_data/orders/orders_manager.py @@ -27,12 +27,17 @@ import octobot_trading.personal_data.orders.order as order_class import octobot_trading.personal_data.orders.order_factory as order_factory import octobot_trading.personal_data.orders.order_util as order_util +import octobot_trading.personal_data.orders.orders_storage_operations as orders_storage_operations import octobot_trading.exchanges import octobot_trading.personal_data.orders.active_order_swap_strategies.active_order_swap_strategy as \ active_order_swap_strategy_import import octobot_trading.personal_data.orders.order_group as order_group_import +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + + class OrdersManager(util.Initializable): MAX_ORDERS_COUNT = 0 @@ -256,6 +261,25 @@ def disabled_order_auto_synchronization(self): finally: self.enable_order_auto_synchronization = True + async def initialize_from_exchange_data(self, exchange_data: "exchange_data_import.ExchangeData") -> None: + """ + Initialize orders from exchange data by parsing open orders and adding them to this manager. + """ + exchange_manager = self.trader.exchange_manager + pending_groups = {} + for order_details in exchange_data.orders_details.open_orders: + if constants.STORAGE_ORIGIN_VALUE in order_details: + order = order_factory.create_order_from_order_raw_in_storage_details_without_related_elements( + exchange_manager, order_details + ) + await orders_storage_operations.create_orders_storage_related_elements( + order, order_details, exchange_manager, pending_groups + ) + else: + # simple order dict (order just fetched from exchange) + order = order_factory.create_order_instance_from_raw(self.trader, order_details) + await self.upsert_order_instance(order) + # private methods def _reset_orders(self): self.orders_initialized = False diff --git a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py index bc55d5ff1..4e5b6e400 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py @@ -25,14 +25,17 @@ import octobot_trading.exchange_channel as exchange_channel import octobot_trading.constants as constants +import octobot_trading.enums as enums import octobot_trading.errors as errors import octobot_trading.personal_data as personal_data +import octobot_trading.storage as storage import octobot_trading.util as util import octobot_trading.enums as enums import octobot_trading.personal_data.portfolios.update_events as update_events if typing.TYPE_CHECKING: import octobot_trading.exchanges + import octobot_trading.util.test_tools.exchange_data as exchange_data_import class PortfolioManager(util.Initializable): @@ -340,6 +343,32 @@ def refresh_portfolio_available_from_order(self, order, is_new_order): if self.enable_portfolio_available_update_from_order: self.portfolio.update_portfolio_available(order, is_new_order=is_new_order) + async def initialize_from_exchange_data(self, exchange_data: "exchange_data_import.ExchangeData") -> None: + """ + Lock funds for chained orders from missing orders in portfolio. + """ + groups = {} + for base_order in exchange_data.orders_details.missing_orders: + for chained_order_dict in base_order.get(enums.StoredOrdersAttr.CHAINED_ORDERS.value, []): + chained_order = await personal_data.create_order_from_order_storage_details( + storage.orders_storage.from_order_document(chained_order_dict), + self.exchange_manager, + groups, + ) + if chained_order.update_with_triggering_order_fees and ( + base_order_exchange_id := base_order.get(constants.STORAGE_ORIGIN_VALUE, {}).get( + enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + ) + ): + trade = personal_data.aggregate_trades_by_exchange_order_id( + self.exchange_manager.exchange_personal_data.trades_manager.get_trades( + exchange_order_id=base_order_exchange_id + ) + ).get(base_order_exchange_id) + if trade: + chained_order.update_quantity_with_order_fees(trade) + self.portfolio.update_portfolio_available(chained_order, is_new_order=True) + def _load_portfolio(self, reset_from_config): """ Load simulated portfolio from config if required diff --git a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_value_holder.py b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_value_holder.py index 8ee5d68a8..cd90a0b86 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_value_holder.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_value_holder.py @@ -21,12 +21,16 @@ import octobot_commons.symbols as symbol_util import octobot_trading.constants as constants +import octobot_trading.exchanges as exchanges import octobot_trading.errors as errors import octobot_trading.enums as enums import octobot_trading.personal_data.portfolios.value_converter as value_converter import octobot_trading.personal_data.portfolios +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + class PortfolioValueHolder: """ PortfolioValueHolder calculates the current and the origin portfolio value in reference market for each updates @@ -57,6 +61,73 @@ def reset_portfolio_values(self): self.origin_crypto_currencies_values = {} self.current_crypto_currencies_values = {} + def initialize_from_exchange_data( + self, exchange_data: "exchange_data_import.ExchangeData", price_by_symbol: dict[str, float] + ) -> None: + """ + Initialize prices and portfolio values from exchange data. + """ + self._set_current_prices_from_exchange_data(exchange_data, price_by_symbol) + self._sync_portfolio_current_value_if_necessary() + + def _set_current_prices_from_exchange_data( + self, exchange_data: "exchange_data_import.ExchangeData", price_by_symbol: dict[str, float] + ) -> None: # todo refactor + added_symbols = set() + for market in exchange_data.markets: + price = price_by_symbol.get(market.symbol) + if price is not None: + price = decimal.Decimal(str(price)) + exchanges.force_set_mark_price(self.portfolio_manager.exchange_manager, market.symbol, price) + self.value_converter.update_last_price(market.symbol, price) + added_symbols.add(market.symbol) + ref_market = self.portfolio_manager.reference_market + for asset, value in exchange_data.portfolio_details.asset_values.items(): + if asset == ref_market: + continue + # include fetched portfolio assets values to be able to value them in ref market in case they + # are not already added from traded pairs + value_symbol = symbol_util.merge_currencies(asset, ref_market) + decimal_value = decimal.Decimal(str(value)) + if value_symbol not in added_symbols: + exchanges.force_set_mark_price(self.portfolio_manager.exchange_manager, value_symbol, decimal_value) + self.value_converter.update_last_price(value_symbol, decimal_value) + added_symbols.add(value_symbol) + + def _sync_portfolio_current_value_if_necessary(self) -> None: + if not self.portfolio_manager.portfolio.portfolio: + # portfolio is not initialized, skip portfolio values initialization + return + try: + self._sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) + portfolio_value = self.portfolio_current_value + if not portfolio_value or portfolio_value <= constants.ZERO: + if self._should_have_initialized_portfolio_values(): + # should not happen (if it does, holding ratios using portfolio_value can't + # be computed) + # This is not critial but should be fixed if seen + self.logger.error( + f"[{self.portfolio_manager.exchange_manager.exchange_name}] Portfolio current value " + f"can't be initialized: {portfolio_value=}" + ) + else: + self.logger.info( + f"[{self.portfolio_manager.exchange_manager.exchange_name}] Portfolio current value " + f"not initialized: no traded asset holdings in portfolio" + ) + except Exception as err: + self.logger.exception(err, True, f"Error when initializing trading portfolio values: {err}") + + def _should_have_initialized_portfolio_values(self) -> bool: + portfolio_assets = [ + asset + for asset, values in self.portfolio_manager.portfolio.portfolio.items() + if values.total > constants.ZERO + ] + if any(coin in portfolio_assets for coin in exchanges.get_traded_assets(self.portfolio_manager.exchange_manager)): + return True + return False + def update_origin_crypto_currencies_values(self, symbol, mark_price): """ Update origin cryptocurrencies value @@ -106,7 +177,7 @@ def get_current_crypto_currencies_values(self): :return: the current crypto-currencies values """ if not self.current_crypto_currencies_values: - self.sync_portfolio_current_value_using_available_currencies_values() + self._sync_portfolio_current_value_using_available_currencies_values() return self.current_crypto_currencies_values def get_current_holdings_values(self): @@ -171,7 +242,7 @@ def handle_profitability_recalculation(self, force_recompute_origin_portfolio): Initialize values required by portfolio profitability to perform its profitability calculation :param force_recompute_origin_portfolio: when True, force origin portfolio computation """ - self.sync_portfolio_current_value_using_available_currencies_values() + self._sync_portfolio_current_value_using_available_currencies_values() self._init_portfolio_values_if_necessary(force_recompute_origin_portfolio) def get_origin_portfolio_current_value(self, refresh_values=False): @@ -248,7 +319,7 @@ def _fill_currencies_values(self, currencies_values): if currency not in currencies_values }) - def sync_portfolio_current_value_using_available_currencies_values(self, init_price_fetchers=True): + def _sync_portfolio_current_value_using_available_currencies_values(self, init_price_fetchers=True): """ :param init_price_fetchers: When True, can init price using fetchers Update the portfolio current value with the current portfolio instance diff --git a/packages/trading/octobot_trading/personal_data/positions/positions_manager.py b/packages/trading/octobot_trading/personal_data/positions/positions_manager.py index a37b09768..f822f452b 100644 --- a/packages/trading/octobot_trading/personal_data/positions/positions_manager.py +++ b/packages/trading/octobot_trading/personal_data/positions/positions_manager.py @@ -16,11 +16,13 @@ import collections import contextlib import typing +import copy import octobot_commons.logging as logging import octobot_commons.enums as commons_enums import octobot_commons.tree as commons_tree +import octobot_trading.exchange_data.contracts.contract_factory as contract_factory import octobot_trading.personal_data.positions.position_factory as position_factory import octobot_trading.personal_data.positions.position as position_import import octobot_trading.util as util @@ -29,6 +31,9 @@ import octobot_trading.errors as errors import octobot_trading.exchange_channel as exchange_channel +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + class PositionsManager(util.Initializable): POSITION_ID_SEPARATOR = "_" @@ -232,6 +237,31 @@ def disabled_positions_update_from_order(self): finally: self._enable_position_update_from_order = True + def initialize_from_exchange_data( + self, exchange_data: "exchange_data_import.ExchangeData", + exclusively_use_exchange_position_details: bool = False + ) -> None: + """ + Initialize positions from exchange data by parsing position details and adding them to this manager. + """ + exchange_manager = self.trader.exchange_manager + contract_factory.initialize_contracts_from_exchange_data(exchange_manager, exchange_data) + for position_details in exchange_data.positions: + if not self._is_cleared_position(position_details.position): + position = position_factory.create_position_instance_from_dict( + self.trader, copy.copy(position_details.position) + ) + position.position_id = self.create_position_id(position) + self.add_position(position) + self.is_exclusively_using_exchange_position_details = exclusively_use_exchange_position_details + + @staticmethod + def _is_cleared_position(position_dict: dict) -> bool: + for key in position_dict: + if key not in constants.MINIMAL_POSITION_IDENTIFICATION_DETAILS_KEYS: + return False + return True + # private def _position_id_factory( diff --git a/packages/trading/octobot_trading/personal_data/trades/trades_manager.py b/packages/trading/octobot_trading/personal_data/trades/trades_manager.py index 27ab11b52..db7d12e2b 100644 --- a/packages/trading/octobot_trading/personal_data/trades/trades_manager.py +++ b/packages/trading/octobot_trading/personal_data/trades/trades_manager.py @@ -26,6 +26,9 @@ import octobot_trading.personal_data.trades.trade_pnl as trade_pnl import octobot_trading.util as util +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + class TradesManager(util.Initializable): # memory usage for 100000 trades: approx 180 Mo @@ -155,6 +158,15 @@ def get_trades(self, origin_order_id=None, exchange_order_id=None): ) ] + def initialize_from_exchange_data(self, exchange_data: "exchange_data_import.ExchangeData") -> None: + """ + Initialize trades from exchange data by parsing trade dicts and adding them to this manager. + """ + for trade_dict in exchange_data.trades: + trade = personal_data.create_trade_from_dict(self.trader, trade_dict) + trade.trade_id = trade.trade_id or self.trader.generate_random_order_id() + self.upsert_trade_instance(trade) + # private def _check_trades_size(self): if len(self.trades) > self.MAX_TRADES_COUNT: diff --git a/packages/trading/octobot_trading/util/test_tools/exchange_data.py b/packages/trading/octobot_trading/util/test_tools/exchange_data.py index 707ee417c..688b21192 100644 --- a/packages/trading/octobot_trading/util/test_tools/exchange_data.py +++ b/packages/trading/octobot_trading/util/test_tools/exchange_data.py @@ -13,14 +13,14 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +# todo move import dataclasses import typing import decimal import octobot_commons.dataclasses import octobot_commons.enums as common_enums -import octobot_trading.exchanges - +import octobot_trading.exchanges.util.symbol_details as symbol_details_import @dataclasses.dataclass class IncompatibleAssetDetails( @@ -62,8 +62,8 @@ class ExchangeDetails(octobot_commons.dataclasses.FlexibleDataclass, octobot_com class MarketDetails(octobot_commons.dataclasses.FlexibleDataclass, octobot_commons.dataclasses.UpdatableDataclass): id: str = "" symbol: str = "" - details: octobot_trading.exchanges.SymbolDetails = \ - dataclasses.field(default_factory=octobot_trading.exchanges.SymbolDetails) + details: symbol_details_import.SymbolDetails = \ + dataclasses.field(default_factory=symbol_details_import.SymbolDetails) time_frame: str = "" close: list[float] = dataclasses.field(default_factory=list) open: list[float] = dataclasses.field(default_factory=list) diff --git a/packages/trading/octobot_trading/util/test_tools/exchange_data_util.py b/packages/trading/octobot_trading/util/test_tools/exchange_data_util.py new file mode 100644 index 000000000..ca2c1627e --- /dev/null +++ b/packages/trading/octobot_trading/util/test_tools/exchange_data_util.py @@ -0,0 +1,55 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +# todo move +import typing + +import octobot_trading.constants as constants +import octobot_trading.enums as enums + +if typing.TYPE_CHECKING: + import octobot_trading.util.test_tools.exchange_data as exchange_data_import + + +def _get_positions_symbols(exchange_data: "exchange_data_import.ExchangeData") -> set[str]: + return set(get_positions_by_symbol(exchange_data)) + + +def _get_orders_symbols(exchange_data: "exchange_data_import.ExchangeData") -> set[str]: + return set( + order[constants.STORAGE_ORIGIN_VALUE][enums.ExchangeConstantsOrderColumns.SYMBOL.value] + for order in exchange_data.orders_details.open_orders + exchange_data.orders_details.missing_orders + if order.get(constants.STORAGE_ORIGIN_VALUE, {}).get( + enums.ExchangeConstantsOrderColumns.SYMBOL.value + ) + ) + + +def get_orders_and_positions_symbols(exchange_data: "exchange_data_import.ExchangeData") -> set[str]: + return _get_orders_symbols(exchange_data).union(_get_positions_symbols(exchange_data)) + + +def get_positions_by_symbol(exchange_data: "exchange_data_import.ExchangeData") -> dict[str, list[dict]]: + return { + position_details.position[enums.ExchangeConstantsPositionColumns.SYMBOL.value]: + [ + symbol_position_details.position + for symbol_position_details in exchange_data.positions + if symbol_position_details.position.get(enums.ExchangeConstantsPositionColumns.SYMBOL.value) == + position_details.position[enums.ExchangeConstantsPositionColumns.SYMBOL.value] + ] + for position_details in exchange_data.positions + if enums.ExchangeConstantsPositionColumns.SYMBOL.value in position_details.position + } diff --git a/packages/trading/tests/blockchain_wallets/test_wallet_factory.py b/packages/trading/tests/blockchain_wallets/test_wallet_factory.py index 37f93e79a..301d2cb79 100644 --- a/packages/trading/tests/blockchain_wallets/test_wallet_factory.py +++ b/packages/trading/tests/blockchain_wallets/test_wallet_factory.py @@ -52,18 +52,6 @@ def test_create_blockchain_wallet_simulated(mock_trader_simulate, blockchain_des assert wallet._trader == mock_trader_simulate -def test_create_blockchain_wallet_simulated_wrong_network(mock_trader_simulate, blockchain_descriptor_real, wallet_descriptor): - parameters = octobot_trading.blockchain_wallets.BlockchainWalletParameters( - blockchain_descriptor=blockchain_descriptor_real, - wallet_descriptor=wallet_descriptor - ) - - # Should raise BlockchainWalletConfigurationError when network is not SIMULATED - with pytest.raises(errors.BlockchainWalletConfigurationError) as exc_info: - octobot_trading.blockchain_wallets.create_blockchain_wallet(parameters, mock_trader_simulate) - assert constants.SIMULATED_BLOCKCHAIN_NETWORK in str(exc_info.value) - - def test_create_blockchain_wallet_real_trader_unsupported_blockchain(mock_trader_real, wallet_descriptor): blockchain_descriptor = octobot_trading.blockchain_wallets.BlockchainDescriptor( blockchain="unsupported_blockchain", diff --git a/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py b/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py new file mode 100644 index 000000000..333fa1baf --- /dev/null +++ b/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py @@ -0,0 +1,110 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock +import pytest + +import octobot_commons.constants +import octobot_commons.symbols +import octobot_trading.exchange_data as exchange_data + + +@pytest.fixture +def ticker_cache(): + cache = exchange_data.TickerCache(ttl=3600, maxsize=50) + yield cache + cache.reset_all_tickers_cache() + + +SPOT_TICKERS = { + "BTC/USDT": mock.Mock(), + "ETH/USDT": mock.Mock(), + "SOL/USDT": mock.Mock(), +} + +FUTURES_TICKERS = { + "BTC/USDT:USDT": mock.Mock(), + "ETH/USDT:USDT": mock.Mock(), + "SOL/USD:SOL": mock.Mock(), +} + + +def test_is_valid_symbol(ticker_cache): + ticker_cache.set_all_tickers("binance", "spot", False, SPOT_TICKERS) + assert ticker_cache.is_valid_symbol("binance", "spot", False, "BTC/USDT") is True + assert ticker_cache.is_valid_symbol("binance", "spot", False, "BTC2/USDT") is False + assert ticker_cache.is_valid_symbol("binance", "futures", False, "BTC/USDT:USDT") is False + ticker_cache.set_all_tickers("binance", "futures", False, FUTURES_TICKERS) + assert ticker_cache.is_valid_symbol("binance", "futures", False, "BTC/USDT:USDT") is True + ticker_cache.reset_all_tickers_cache() + assert ticker_cache.is_valid_symbol("binance", "futures", False, "BTC/USDT:USDT") is False + + +def test_get_all_tickers(ticker_cache): + assert ticker_cache.get_all_tickers("binance", "spot", False) is None + assert ticker_cache.get_all_tickers("binance", "spot", False, "default") == "default" + ticker_cache.set_all_tickers("binance", "spot", False, SPOT_TICKERS) + assert ticker_cache.get_all_tickers("binance", "spot", False) == SPOT_TICKERS + assert ticker_cache.get_all_tickers("binance", "spot", True) is None + assert ticker_cache.get_all_tickers("binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False) is None + + +def test_has_ticker_data(ticker_cache): + assert ticker_cache.has_ticker_data("binance", "spot", False) is False + ticker_cache.set_all_tickers("binance", "spot", False, SPOT_TICKERS) + assert ticker_cache.has_ticker_data("binance", "spot", False) is True + assert ticker_cache.has_ticker_data("binance", "spot", True) is False + + ticker_cache.reset_all_tickers_cache() + assert ticker_cache.has_ticker_data("binance", "spot", False) is False + + +def test_get_all_parsed_symbols_by_merged_symbols(ticker_cache): + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols("binance", "spot", False) is None + ticker_cache.set_all_tickers("binance", "spot", False, SPOT_TICKERS) + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols("binance", "spot", False) == { + "BTCUSDT": octobot_commons.symbols.parse_symbol("BTC/USDT"), + "ETHUSDT": octobot_commons.symbols.parse_symbol("ETH/USDT"), + "SOLUSDT": octobot_commons.symbols.parse_symbol("SOL/USDT"), + } + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False + ) is None + + ticker_cache.set_all_tickers( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False, FUTURES_TICKERS + ) + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False + ) == { + "BTCUSDT": octobot_commons.symbols.parse_symbol("BTC/USDT:USDT"), + "BTCUSDT:USDT": octobot_commons.symbols.parse_symbol("BTC/USDT:USDT"), + "ETHUSDT": octobot_commons.symbols.parse_symbol("ETH/USDT:USDT"), + "ETHUSDT:USDT": octobot_commons.symbols.parse_symbol("ETH/USDT:USDT"), + "SOLUSD": octobot_commons.symbols.parse_symbol("SOL/USD:SOL"), + "SOLUSD:SOL": octobot_commons.symbols.parse_symbol("SOL/USD:SOL"), + } + + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, True + ) is None + + +def test_get_exchange_key(): + assert exchange_data.TickerCache.get_exchange_key("binance", "spot", True) == "binance_spot_True" + assert exchange_data.TickerCache.get_exchange_key("binance", "spot", False) == "binance_spot_False" + assert exchange_data.TickerCache.get_exchange_key("binance", "future", False) == "binance_future_False" + assert exchange_data.TickerCache.get_exchange_key("okx", "future", False) == "okx_future_False" diff --git a/packages/trading/tests/exchanges/__init__.py b/packages/trading/tests/exchanges/__init__.py index 556a8b150..d44d3e9e8 100644 --- a/packages/trading/tests/exchanges/__init__.py +++ b/packages/trading/tests/exchanges/__init__.py @@ -26,6 +26,7 @@ import octobot_backtesting.time as backtesting_time from octobot_commons.asyncio_tools import wait_asyncio_next_cycle from octobot_commons.enums import TimeFrames +import octobot_trading.constants import octobot_trading.exchanges.connectors.ccxt.ccxt_clients_cache as ccxt_clients_cache from octobot_commons.tests.test_config import load_test_config @@ -414,9 +415,15 @@ async def cached_markets_exchange_manager(config, exchange_name, exchange_only=F def register_market_status_mocks(exchange_name): + cached_client = ccxt_client_util.ccxt_exchange_class_factory(exchange_name)() + client_key = ccxt_clients_cache.get_client_key(cached_client, False) + # save markets in cache ccxt_clients_cache.set_exchange_parsed_markets( - ccxt_clients_cache.get_client_key( - ccxt_client_util.ccxt_exchange_class_factory(exchange_name)(), False - ), - mock_exchanges_data.MOCKED_EXCHANGE_SYMBOL_DETAILS[exchange_name] + client_key, mock_exchanges_data.MOCKED_EXCHANGE_SYMBOL_DETAILS[exchange_name] ) + with mock.patch.object(octobot_trading.constants, "USE_CCXT_SHARED_MARKETS_CACHE", False): + # apply markets from cache + ccxt_clients_cache.apply_exchange_markets_cache(client_key, cached_client) + # save cached_client cache in cached exchange + ccxt_clients_cache.set_cached_shared_markets_exchange(client_key, cached_client) + diff --git a/packages/trading/tests/exchanges/market_filters/test_market_filter_factory.py b/packages/trading/tests/exchanges/market_filters/test_market_filter_factory.py new file mode 100644 index 000000000..5eb368138 --- /dev/null +++ b/packages/trading/tests/exchanges/market_filters/test_market_filter_factory.py @@ -0,0 +1,140 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import octobot_commons.symbols as commons_symbols +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as exchanges +import octobot_trading.util.test_tools.exchange_data as exchange_data_import + + +def _market(base, quote, m_type): + return { + trading_enums.ExchangeConstantsMarketStatusColumns.SYMBOL.value: commons_symbols.merge_currencies( + base, quote + ), + trading_enums.ExchangeConstantsMarketStatusColumns.CURRENCY.value: base, + trading_enums.ExchangeConstantsMarketStatusColumns.MARKET.value: quote, + trading_enums.ExchangeConstantsMarketStatusColumns.TYPE.value: m_type, + } + + +def _get_market_symbols(markets): + return [ + commons_symbols.merge_currencies( + m[trading_enums.ExchangeConstantsMarketStatusColumns.CURRENCY.value], + m[trading_enums.ExchangeConstantsMarketStatusColumns.MARKET.value], + ) + for m in markets + ] + + +MARKETS = [ + _market(base, quote, m_type) + for base, quote, m_type in [ + ("BTC", "USDT", trading_enums.ExchangeTypes.SPOT.value), + ("BTC", "USDC", trading_enums.ExchangeTypes.SPOT.value), + ("ETH", "USDT", trading_enums.ExchangeTypes.SPOT.value), + ("USDC", "USDT", trading_enums.ExchangeTypes.SPOT.value), + ("ETH", "BTC", trading_enums.ExchangeTypes.SPOT.value), + ("DAI", "USDT", trading_enums.ExchangeTypes.SPOT.value), + ("DAI", "BUSD", trading_enums.ExchangeTypes.SPOT.value), + ("ZEC", "ETH", trading_enums.ExchangeTypes.SPOT.value), + ("ZEC", "BTC", trading_enums.ExchangeTypes.SPOT.value), + ("USDT", "BNB", trading_enums.ExchangeTypes.SPOT.value), + ("XBY", "DAI", trading_enums.ExchangeTypes.SPOT.value), + ("NANO", "JPUSD", trading_enums.ExchangeTypes.SPOT.value), + ("NANO", "USDT", trading_enums.ExchangeTypes.SPOT.value), + ] +] + + +def test_create_market_filter(): + empty_exchange_data = exchange_data_import.ExchangeData() + + assert _get_market_symbols( + [m for m in MARKETS if exchanges.create_market_filter(empty_exchange_data, "BTC")(m)] + ) == ['BTC/USDT', 'BTC/USDC', 'USDC/USDT', 'ETH/BTC', 'DAI/USDT', 'DAI/BUSD', 'ZEC/BTC', 'USDT/BNB'] + + assert _get_market_symbols( + [m for m in MARKETS if exchanges.create_market_filter(empty_exchange_data, "USDT")(m)] + ) == ['BTC/USDT', 'ETH/USDT', 'USDC/USDT', 'DAI/USDT', 'DAI/BUSD', 'USDT/BNB', 'NANO/USDT'] + + exchange_data_with_orders = exchange_data_import.ExchangeData() + exchange_data_with_orders.orders_details.open_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "XBY/DAI", + } + } + ] + exchange_data_with_orders.orders_details.missing_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "NANO/JPUSD", + } + } + ] + + assert _get_market_symbols( + [m for m in MARKETS if exchanges.create_market_filter(exchange_data_with_orders, "USDT")(m)] + ) == [ + 'BTC/USDT', 'ETH/USDT', 'USDC/USDT', 'DAI/USDT', 'DAI/BUSD', + 'USDT/BNB', "XBY/DAI", "NANO/JPUSD", "NANO/USDT", + ] + + assert _get_market_symbols( + [ + m + for m in MARKETS + if exchanges.create_market_filter( + exchange_data_with_orders, + "USDT", + to_keep_symbols={"ZEC/BTC"}, + )(m) + ] + ) == [ + 'BTC/USDT', 'ETH/USDT', 'USDC/USDT', 'DAI/USDT', 'DAI/BUSD', + 'ZEC/BTC', 'USDT/BNB', "XBY/DAI", "NANO/JPUSD", "NANO/USDT", + ] + + assert _get_market_symbols( + [ + m + for m in MARKETS + if exchanges.create_market_filter( + exchange_data_with_orders, + "USDT", + to_keep_symbols={"ZEC/BTC"}, + to_keep_quotes={"USDC"}, + )(m) + ] + ) == [ + 'BTC/USDT', 'BTC/USDC', 'ETH/USDT', 'USDC/USDT', 'DAI/USDT', 'DAI/BUSD', + 'ZEC/BTC', 'USDT/BNB', "XBY/DAI", "NANO/JPUSD", "NANO/USDT", + ] + + exchange_data_with_markets = exchange_data_import.ExchangeData() + exchange_data_with_markets.markets = [ + exchange_data_import.MarketDetails(symbol="ZEC/BTC"), + ] + + assert _get_market_symbols( + [m for m in MARKETS if exchanges.create_market_filter(exchange_data_with_markets, "USDT")(m)] + ) == [ + 'BTC/USDT', 'ETH/USDT', 'USDC/USDT', 'DAI/USDT', 'DAI/BUSD', + 'ZEC/BTC', # from markets + 'USDT/BNB', "NANO/USDT", + ] diff --git a/packages/trading/tests/personal_data/portfolios/test_portfolio_value_holder.py b/packages/trading/tests/personal_data/portfolios/test_portfolio_value_holder.py index 0db51c657..a9dc7487a 100644 --- a/packages/trading/tests/personal_data/portfolios/test_portfolio_value_holder.py +++ b/packages/trading/tests/personal_data/portfolios/test_portfolio_value_holder.py @@ -166,7 +166,7 @@ def mock_create_symbol_position(symbol, position_id): portfolio_value_holder.value_converter.last_prices_by_trading_pair["ETH/BTC"] = decimal.Decimal("50") # Update current_crypto_currencies_values to include ETH with the calculated price portfolio_value_holder.current_crypto_currencies_values["ETH"] = decimal.Decimal("50") - portfolio_value_holder.sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) assert portfolio_value_holder.get_current_holdings_values() == { 'BTC': decimal.Decimal("10"), 'ETH': decimal.Decimal("5000"), @@ -255,19 +255,19 @@ async def test_update_origin_crypto_currencies_values(backtesting_trader): is False @pytest.mark.parametrize("backtesting_exchange_manager", ["spot", "margin", "futures", "options"], indirect=True) -async def test_sync_portfolio_current_value_using_available_currencies_values(backtesting_trader): +async def test__sync_portfolio_current_value_using_available_currencies_values(backtesting_trader): config, exchange_manager, trader = backtesting_trader portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager portfolio_value_holder = portfolio_manager.portfolio_value_holder assert portfolio_value_holder.portfolio_current_value == constants.ZERO - portfolio_value_holder.sync_portfolio_current_value_using_available_currencies_values() + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values() assert portfolio_value_holder.portfolio_current_value == decimal.Decimal(str(10)) portfolio_value_holder.value_converter.missing_currency_data_in_exchange.clear() exchange_manager.client_symbols.append("BTC/USDT") portfolio_manager.handle_mark_price_update("BTC/USDT", decimal.Decimal(str(100))) - portfolio_value_holder.sync_portfolio_current_value_using_available_currencies_values() + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values() assert portfolio_value_holder.portfolio_current_value == decimal.Decimal(str(20)) # now includes USDT @pytest.mark.parametrize("backtesting_exchange_manager", ["spot", "futures"], indirect=True) @@ -564,7 +564,7 @@ async def test_get_holdings_ratio_from_portfolio(backtesting_trader, currency, t exchange_manager.client_symbols.append("BTC/USDT") portfolio_value_holder.value_converter.last_prices_by_trading_pair["BTC/USDT"] = decimal.Decimal("1000") portfolio_value_holder.value_converter.missing_currency_data_in_exchange.discard("USDT") - portfolio_value_holder.sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) result = portfolio_value_holder._get_holdings_ratio_from_portfolio( currency, traded_symbols_only, include_assets_in_open_orders, coins_whitelist @@ -598,7 +598,7 @@ async def test_get_total_holdings_value(backtesting_trader, coins_whitelist, tra exchange_manager.client_symbols.append("BTC/USDT") portfolio_value_holder.value_converter.last_prices_by_trading_pair["BTC/USDT"] = decimal.Decimal("1000") portfolio_value_holder.value_converter.missing_currency_data_in_exchange.discard("USDT") - portfolio_value_holder.sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values(init_price_fetchers=False) result = portfolio_value_holder._get_total_holdings_value(coins_whitelist, traded_symbols_only)