From 28e1d372d7e7da7a311e9a8ccb79e5cd971f779c Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 26 Jun 2024 11:28:26 +0200 Subject: [PATCH 01/15] [Community] init tentacles packages --- octobot/cli.py | 64 +++++---- octobot/commands.py | 68 ++++++--- octobot/community/authentication.py | 130 ++++++++++++++++-- octobot/community/community_analysis.py | 2 +- octobot/community/feeds/abstract_feed.py | 3 + .../models/community_user_account.py | 3 + .../community_supabase_client.py | 17 +++ .../supabase_backend/configuration_storage.py | 7 + octobot/community/tentacles_packages.py | 87 ++++++++++++ octobot/constants.py | 13 +- octobot/octobot.py | 2 +- octobot/updater/updater.py | 8 +- requirements.txt | 2 +- .../community/test_authentication.py | 2 +- 14 files changed, 344 insertions(+), 64 deletions(-) create mode 100644 octobot/community/tentacles_packages.py diff --git a/octobot/cli.py b/octobot/cli.py index ae14f489a3..a4cad862ab 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -132,17 +132,6 @@ def _create_startup_config(logger): async def _apply_community_startup_info_to_config(logger, config, community_auth): try: - if not community_auth.is_initialized() and constants.USER_ACCOUNT_EMAIL and constants.USER_PASSWORD_TOKEN: - try: - await community_auth.login( - constants.USER_ACCOUNT_EMAIL, None, password_token=constants.USER_PASSWORD_TOKEN - ) - except authentication.AuthenticationError as err: - logger.debug(f"Password token auth failure ({err}). Trying with saved session.") - if not community_auth.is_initialized(): - await community_auth.async_init_account() - if not community_auth.is_logged_in(): - return startup_info = await community_auth.get_startup_info() logger.debug(f"Fetched startup info: {startup_info}") commands.download_and_select_profile( @@ -166,18 +155,42 @@ def _apply_env_variables_to_config(logger, config): ) -def _handle_forced_startup_config(logger, config, is_first_startup): +async def _get_authenticated_community_if_possible(config, logger): # switch environments if necessary octobot_community.IdentifiersProvider.use_environment_from_config(config) - - # 1. at first startup, get startup info from community when possible community_auth = octobot_community.CommunityAuthentication.create(config) - if is_first_startup: - asyncio.run(_apply_community_startup_info_to_config(logger, config, community_auth)) + try: + if not community_auth.is_initialized(): + if constants.IS_CLOUD_ENV and constants.USER_ACCOUNT_EMAIL and constants.USER_PASSWORD_TOKEN: + try: + await community_auth.login( + constants.USER_ACCOUNT_EMAIL, None, password_token=constants.USER_PASSWORD_TOKEN + ) + except authentication.AuthenticationError as err: + logger.debug(f"Password token auth failure ({err}). Trying with saved session.") + if not community_auth.is_initialized(): + # try with saved credentials if any + has_tentacles = tentacles_manager_api.is_tentacles_architecture_valid() + # When no tentacles, fetch private data. Otherwise fetch it later on in bot init + await community_auth.async_init_account(fetch_private_data=not has_tentacles) + except authentication.FailedAuthentication as err: + logger.error(f"Failed authentication when initializing community authenticator: {err}") + except Exception as err: + logger.error(f"Error when initializing community authenticator: {err}") + return community_auth + + +async def _async_load_community_data(community_auth, config, logger, is_first_startup): + if constants.IS_CLOUD_ENV and is_first_startup: + # auto config + await _apply_community_startup_info_to_config(logger, config, community_auth) + + +def _apply_forced_configs(community_auth, logger, config, is_first_startup): + asyncio.run(_async_load_community_data(community_auth, config, logger, is_first_startup)) # 2. handle profiles from env variables _apply_env_variables_to_config(logger, config) - return community_auth def _read_config(config, logger): @@ -208,21 +221,21 @@ def _repair_with_default_profile(config, logger): config.load_profiles_if_possible_and_necessary() -def _load_or_create_tentacles(config, logger): +def _load_or_create_tentacles(community_auth, config, logger): # add tentacles folder to Python path sys.path.append(os.path.realpath(os.getcwd())) - # when tentacles folder already exists if os.path.isfile(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_FILE_PATH): + # when tentacles folder already exists config.load_profiles_if_possible_and_necessary() tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config( config.get_tentacles_config_path() ) - commands.run_update_or_repair_tentacles_if_necessary(config, tentacles_setup_config) + commands.run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config) else: # when no tentacles folder has been found logger.info("OctoBot tentacles can't be found. Installing default tentacles ...") - commands.run_tentacles_install_or_update(config) + commands.run_tentacles_install_or_update(community_auth, config) config.load_profiles_if_possible_and_necessary() @@ -260,11 +273,16 @@ def start_octobot(args): # show terms _log_terms_if_unaccepted(config, logger) + community_auth = None if args.backtesting else asyncio.run( + _get_authenticated_community_if_possible(config, logger) + ) + # tries to load, install or repair tentacles - _load_or_create_tentacles(config, logger) + _load_or_create_tentacles(community_auth, config, logger) # patch setup with forced values - community_auth = None if args.backtesting else _handle_forced_startup_config(logger, config, is_first_startup) + if not args.backtesting: + _apply_forced_configs(community_auth, logger, config, is_first_startup) # Can now perform config health check (some checks require a loaded profile) configuration_manager.config_health_check(config, args.backtesting) diff --git a/octobot/commands.py b/octobot/commands.py index 3d0978cf72..6a9e21ad13 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -15,7 +15,8 @@ # License along with OctoBot. If not, see . import os -import aiohttp +import typing + import sys import asyncio import signal @@ -34,6 +35,7 @@ import octobot.api.strategy_optimizer as strategy_optimizer_api import octobot.logger as octobot_logger import octobot.constants as constants +import octobot.community.tentacles_packages as community_tentacles_packages import octobot.configuration_manager as configuration_manager COMMANDS_LOGGER_NAME = "Commands" @@ -77,13 +79,18 @@ def start_strategy_optimizer(config, commands): strategy_optimizer_api.print_optimizer_report(optimizer) -def run_tentacles_install_or_update(config): +def run_tentacles_install_or_update(community_auth, config): _check_tentacles_install_exit() - asyncio.run(install_or_update_tentacles(config)) + asyncio.run(_install_or_update_tentacles(community_auth, config)) + + +async def _install_or_update_tentacles(community_auth, config): + additional_tentacles_package_urls = community_auth.get_saved_package_urls() + await install_or_update_tentacles(config, additional_tentacles_package_urls, False) -def run_update_or_repair_tentacles_if_necessary(config, tentacles_setup_config): - asyncio.run(update_or_repair_tentacles_if_necessary(tentacles_setup_config, config)) +def run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config): + asyncio.run(update_or_repair_tentacles_if_necessary(community_auth, tentacles_setup_config, config)) def _check_tentacles_install_exit(): @@ -102,7 +109,7 @@ def _get_first_non_imported_profile_tentacles_setup_config(config): return None -async def update_or_repair_tentacles_if_necessary(selected_profile_tentacles_setup_config, config): +async def update_or_repair_tentacles_if_necessary(community_auth, selected_profile_tentacles_setup_config, config): local_profile_tentacles_setup_config = selected_profile_tentacles_setup_config logger = logging.get_logger(COMMANDS_LOGGER_NAME) if config.profile.imported: @@ -117,38 +124,57 @@ async def update_or_repair_tentacles_if_necessary(selected_profile_tentacles_set f"Please make sure that this profile works on your OctoBot.") # only update tentacles based on local (non imported) profiles tentacles installation version local_profile_tentacles_setup_config = _get_first_non_imported_profile_tentacles_setup_config(config) + + to_install_urls, to_remove_tentacles = community_tentacles_packages.get_to_install_and_remove_tentacles( + community_auth, selected_profile_tentacles_setup_config + ) + if to_remove_tentacles: + await community_tentacles_packages.uninstall_tentacles(to_remove_tentacles) + + # await install_or_update_tentacles(config, additional_tentacles_package_urls) #TMPPP if local_profile_tentacles_setup_config is None or \ not tentacles_manager_api.are_tentacles_up_to_date(local_profile_tentacles_setup_config, constants.VERSION): logger.info("OctoBot tentacles are not up to date. Updating tentacles...") _check_tentacles_install_exit() - if await install_or_update_tentacles(config): + if await install_or_update_tentacles(config, to_install_urls, False): logger.info("OctoBot tentacles are now up to date.") - elif tentacles_manager_api.load_tentacles(verbose=True): - logger.debug("OctoBot tentacles are up to date.") else: - logger.info("OctoBot tentacles are damaged. Installing default tentacles ...") - _check_tentacles_install_exit() - await install_or_update_tentacles(config) - - -async def install_or_update_tentacles(config): - await install_all_tentacles() + if to_install_urls: + logger.debug("Installing new tentacles.") + await install_or_update_tentacles(config, to_install_urls, True) + if tentacles_manager_api.load_tentacles(verbose=True): + logger.debug("OctoBot tentacles are up to date.") + else: + logger.info("OctoBot tentacles are damaged. Installing default tentacles only ...") + _check_tentacles_install_exit() + await install_or_update_tentacles(config, [], False) + + +async def install_or_update_tentacles( + config, additional_tentacles_package_urls: typing.Optional[list], only_additional: bool +): + await install_all_tentacles( + additional_tentacles_package_urls=additional_tentacles_package_urls, only_additional=only_additional + ) # reload profiles config.load_profiles() # reload tentacles return tentacles_manager_api.load_tentacles(verbose=True) -async def install_all_tentacles(tentacles_url=None): +async def install_all_tentacles( + tentacles_url=None, additional_tentacles_package_urls: typing.Optional[list] = None, only_additional: bool = False +): if tentacles_url is None: tentacles_url = configuration_manager.get_default_tentacles_url() async with aiohttp_util.ssl_fallback_aiohttp_client_session( commons_constants.KNOWN_POTENTIALLY_SSL_FAILED_REQUIRED_URL ) as aiohttp_session: - for url in [tentacles_url] + ( - constants.ADDITIONAL_TENTACLES_PACKAGE_URL.split(constants.URL_SEPARATOR) - if constants.ADDITIONAL_TENTACLES_PACKAGE_URL else [] - ): + base_urls = [tentacles_url] + ( + constants.ADDITIONAL_TENTACLES_PACKAGE_URL.split(constants.URL_SEPARATOR) + if constants.ADDITIONAL_TENTACLES_PACKAGE_URL else [] + ) + for url in (base_urls if not only_additional else []) + (additional_tentacles_package_urls or []): if url is None: continue if constants.VERSION_PLACEHOLDER in url: diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index bb62e8ba22..f57c387af4 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -31,6 +31,7 @@ import octobot.community.supabase_backend as supabase_backend import octobot.community.supabase_backend.enums as backend_enums import octobot.community.feeds as community_feeds +import octobot.community.tentacles_packages as community_tentacles_packages import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.authentication as authentication @@ -72,12 +73,14 @@ class CommunityAuthentication(authentication.Authenticator): def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singleton=True): super().__init__(use_as_singleton=use_as_singleton) + self.config = config self.backend_url = backend_url or identifiers_provider.IdentifiersProvider.BACKEND_URL self.backend_key = backend_key or identifiers_provider.IdentifiersProvider.BACKEND_KEY - self.configuration_storage = supabase_backend.SyncConfigurationStorage(config) + self.configuration_storage = supabase_backend.SyncConfigurationStorage(self.config) self.supabase_client = self._create_client() self.user_account = community_user_account.CommunityUserAccount() self.public_data = community_public_data.CommunityPublicData() + self.successfully_fetched_tentacles_package_urls = False self._community_feed = None self.initialized_event = None @@ -226,11 +229,11 @@ def is_using_the_current_loop(self): def is_initialized(self): return self.initialized_event is not None and self.initialized_event.is_set() - def init_account(self): - self._fetch_account_task = asyncio.create_task(self._initialize_account()) + def init_account(self, fetch_private_data): + self._fetch_account_task = asyncio.create_task(self._initialize_account(fetch_private_data=fetch_private_data)) - async def async_init_account(self): - self.init_account() + async def async_init_account(self, fetch_private_data): + self.init_account(fetch_private_data) await self._fetch_account_task async def _create_community_feed_if_necessary(self) -> bool: @@ -375,6 +378,12 @@ async def get_current_bot_products_subscription(self) -> dict: self.user_account.get_selected_bot_deployment_id() ) + def get_owned_packages(self) -> list[str]: + return self.user_account.owned_packages + + def has_owned_packages_to_install(self) -> bool: + return self.user_account.has_pending_packages_to_install + def is_logged_in_and_has_selected_bot(self): return (self.supabase_client.is_admin or self.is_logged_in()) and self.user_account.bot_id is not None @@ -448,7 +457,7 @@ def _login_process(self): if not self._login_completed.is_set(): self._login_completed.set() - async def _initialize_account(self, minimal=False): + async def _initialize_account(self, minimal=False, fetch_private_data=True): try: await self._ensure_async_loop() self.initialized_event = asyncio.Event() @@ -456,32 +465,125 @@ async def _initialize_account(self, minimal=False): return self._login_completed.set() if not minimal: - await self._ensure_init_community_feed() - await self.update_supports() - await self.update_selected_bot() - self.logger.debug(f"Fetched account data") - await self.init_public_data() - if not self.user_account.is_hosting_enabled(): - await self.update_is_hosting_enabled(True) + await self._init_community_data(fetch_private_data) + if self._community_feed and self._community_feed.has_registered_feed(): + await self._ensure_init_community_feed() except authentication.UnavailableError as e: - self.logger.exception(e, True, f"Error when fetching community supports, " + self.logger.exception(e, True, f"Error when fetching community data, " f"please check your internet connection.") except Exception as e: self.logger.exception(e, True, f"Error when fetching community supports: {e}({e.__class__.__name__})") finally: self.initialized_event.set() + async def _init_community_data(self, fetch_private_data): + coros = [ + self.update_supports(), + self.init_public_data(), + ] + if fetch_private_data: + coros.append(self.update_selected_bot()) + coros.append(self.fetch_private_data()) + if not self.user_account.is_hosting_enabled(): + coros.append(self.update_is_hosting_enabled(True)) + await asyncio.gather(*coros) + async def init_public_data(self, reset=False): if reset or not self.public_data.products.fetched: self.public_data.set_products(await self.supabase_client.fetch_products()) + async def fetch_private_data(self, reset=False): + try: + mqtt_uuid = None + try: + mqtt_uuid = self.get_saved_mqtt_device_uuid() + except errors.NoBotDeviceError: + pass + if reset or (not self.user_account.community_package_urls or not mqtt_uuid): + self.successfully_fetched_tentacles_package_urls = False + packages, package_urls, fetched_mqtt_uuid = await self._fetch_package_urls(mqtt_uuid) + self.successfully_fetched_tentacles_package_urls = True + self.user_account.owned_packages = packages + self.save_installed_package_urls(package_urls) + has_tentacles_to_install = \ + await community_tentacles_packages.has_tentacles_to_install_and_uninstall_tentacles_if_necessary( + self + ) + if has_tentacles_to_install: + # tentacles are not installed, save the fact that some are pending + self.user_account.has_pending_packages_to_install = True + if fetched_mqtt_uuid and fetched_mqtt_uuid != mqtt_uuid: + self.save_mqtt_device_uuid(fetched_mqtt_uuid) + except Exception as err: + self.logger.exception(err, True, f"Error when fetching package urls: {err}") + + async def _fetch_package_urls(self, mqtt_uuid: typing.Optional[str]) -> (list[str], str): + resp = await self.supabase_client.http_get( + constants.COMMUNITY_EXTENSIONS_CHECK_ENDPOINT, + headers={ + "Content-Type": "application/json", + "X-Auth-Token": constants.COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY + }, + params={"mqtt_id": mqtt_uuid} if mqtt_uuid else {}, + timeout=constants.COMMUNITY_FETCH_TIMEOUT + ) + resp.raise_for_status() + json_resp = json.loads(resp.json().get("message", {})) + if not json_resp: + return None, None, None + packages = [ + package + for package in json_resp["paid_package_slugs"] + if package + ] + urls = [ + url + for url in json_resp["package_urls"] + if url + ] + mqtt_id = json_resp["mqtt_id"] + return packages, urls, mqtt_id + + async def fetch_checkout_url(self, payment_method, redirect_url): + try: + resp = await self.supabase_client.http_post( + constants.COMMUNITY_EXTENSIONS_CHECK_ENDPOINT, + json={ + "payment_method": payment_method, + "success_url": redirect_url, + }, + headers={ + "Content-Type": "application/json", + "X-Auth-Token": constants.COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY + }, + timeout=constants.COMMUNITY_FETCH_TIMEOUT + ) + resp.raise_for_status() + json_resp = json.loads(resp.json().get("message", {})) + if not json_resp: + # valid error code but no content: user already has this product + return None + return json_resp["checkout_url"] + except Exception as err: + self.logger.exception(err, True, f"Error when fetching checkout url: {err}") + raise + + def was_connected_with_remote_packages(self): + return self.configuration_storage.has_remote_packages() + def _reset_login_token(self): if self.supabase_client is not None: self._save_value_in_config(self.supabase_client.auth._storage_key, "") + def save_installed_package_urls(self, package_urls: list[str]): + self._save_value_in_config(constants.CONFIG_COMMUNITY_PACKAGE_URLS, package_urls) + def save_mqtt_device_uuid(self, mqtt_uuid): self._save_value_in_config(constants.CONFIG_COMMUNITY_MQTT_UUID, mqtt_uuid) + def get_saved_package_urls(self) -> list[str]: + return self._get_value_in_config(constants.CONFIG_COMMUNITY_PACKAGE_URLS) or [] + def get_saved_mqtt_device_uuid(self): if mqtt_uuid := self._get_value_in_config(constants.CONFIG_COMMUNITY_MQTT_UUID): return mqtt_uuid diff --git a/octobot/community/community_analysis.py b/octobot/community/community_analysis.py index ba16d0cbcd..1ca2cd0cfa 100644 --- a/octobot/community/community_analysis.py +++ b/octobot/community/community_analysis.py @@ -43,7 +43,7 @@ async def get_stats(url, stats_key): try: async with aiohttp.ClientSession() as session: async with session.get(url) as resp: - if resp.status != 200: + if resp.status > 299: logger.error(f"Error when getting community status : error code={resp.status}") else: json_resp = await resp.json() diff --git a/octobot/community/feeds/abstract_feed.py b/octobot/community/feeds/abstract_feed.py index 7436cb791d..338d270bee 100644 --- a/octobot/community/feeds/abstract_feed.py +++ b/octobot/community/feeds/abstract_feed.py @@ -32,6 +32,9 @@ def __init__(self, feed_url, authenticator): self.is_signal_receiver = False self.is_signal_emitter = False + def has_registered_feed(self) -> bool: + return bool(self.feed_callbacks) + async def start(self): raise NotImplementedError("start is not implemented") diff --git a/octobot/community/models/community_user_account.py b/octobot/community/models/community_user_account.py index c216a7ae94..a70bfda494 100644 --- a/octobot/community/models/community_user_account.py +++ b/octobot/community/models/community_user_account.py @@ -32,6 +32,9 @@ class CommunityUserAccount: def __init__(self): self.bot_id = None self.supports = community_supports.CommunitySupports() + self.community_package_urls: list[str] = [] + self.owned_packages: list[str] = [] + self.has_pending_packages_to_install = False self._profile_raw_data = None self._selected_bot_raw_data = None diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index cb58ac0914..a01f7068f9 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -19,6 +19,7 @@ import time import typing import logging +import httpx import aiohttp import gotrue.errors @@ -645,6 +646,22 @@ def get_subscribed_channel_tables(self) -> set: def is_realtime_connected(self) -> bool: return self.realtime.socket and self.realtime.socket.connected and not self.realtime.socket.closed + async def http_get(self, url: str, *args, params=None, headers=None, **kwargs) -> httpx.Response: + """ + Perform http get using the current supabase auth token + """ + return await self.postgrest.session.get(url, *args, params=params, headers=headers, **kwargs) + + async def http_post( + self, url: str, *args, data=None, files=None, json=None, params=None, headers=None, **kwargs + ) -> httpx.Response: + """ + Perform http get using the current supabase auth token + """ + return await self.postgrest.session.post( + url, *args, data=data, files=files, json=json, params=params, headers=headers, **kwargs + ) + @staticmethod def get_formatted_time(timestamp: float) -> str: return datetime.datetime.utcfromtimestamp(timestamp).isoformat('T') diff --git a/octobot/community/supabase_backend/configuration_storage.py b/octobot/community/supabase_backend/configuration_storage.py index 6e32663a45..df2c130179 100644 --- a/octobot/community/supabase_backend/configuration_storage.py +++ b/octobot/community/supabase_backend/configuration_storage.py @@ -36,6 +36,13 @@ def set_item(self, key: str, value: str) -> None: def remove_item(self, key: str) -> None: self._save_value_in_config(key, "") + def has_remote_packages(self) -> bool: + return bool( + self.configuration.config.get(octobot.constants.CONFIG_COMMUNITY, {}).get( + octobot.constants.CONFIG_COMMUNITY_PACKAGE_URLS + ) + ) + def _save_value_in_config(self, key, value): if self.configuration is not None: if octobot.constants.CONFIG_COMMUNITY not in self.configuration.config: diff --git a/octobot/community/tentacles_packages.py b/octobot/community/tentacles_packages.py new file mode 100644 index 0000000000..5f19b8644b --- /dev/null +++ b/octobot/community/tentacles_packages.py @@ -0,0 +1,87 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2023 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import octobot.constants as constants +import octobot_tentacles_manager.api as tentacles_manager_api + + +async def has_tentacles_to_install_and_uninstall_tentacles_if_necessary(community_auth): + tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config( + community_auth.config.get_tentacles_config_path() + ) + to_install, to_remove_tentacles = get_to_install_and_remove_tentacles( + community_auth, tentacles_setup_config + ) + if to_remove_tentacles: + await uninstall_tentacles(to_remove_tentacles) + return bool(to_install) + + +def get_to_install_and_remove_tentacles( + community_auth, selected_profile_tentacles_setup_config +): + installed_community_package_urls = [ + package_url + for package_url in tentacles_manager_api.get_all_installed_package_urls( + selected_profile_tentacles_setup_config + ) + if is_community_tentacle_url(package_url) + ] + additional_tentacles_package_urls = community_auth.get_saved_package_urls() if community_auth.is_logged_in() else [] + was_connected_with_remote_packages = community_auth.was_connected_with_remote_packages() + + # do not remove tentacles: + # - if account has already been authenticated with valid extensions but is currently unauthenticated + # - if tentacles packages urls have not been fetched + # remove if: is authenticated and doesn't have access to these tentacles or has never been authenticated + can_remove_tentacles = True + if was_connected_with_remote_packages and not community_auth.is_logged_in(): + # currently unauthenticated + can_remove_tentacles = False + if not community_auth.successfully_fetched_tentacles_package_urls: + # did not fetch tentacles packages + can_remove_tentacles = False + to_remove_urls = [ + package_url + for package_url in installed_community_package_urls + if package_url not in additional_tentacles_package_urls + ] if can_remove_tentacles else [] + if to_remove_urls: + tentacles_manager_api.reload_tentacle_info() + to_remove_tentacles = [] + for to_remove_url in to_remove_urls: + installed_packages = tentacles_manager_api.get_installed_packages_from_url( + selected_profile_tentacles_setup_config, + to_remove_url + ) + for installed_package in installed_packages: + to_remove_tentacles += tentacles_manager_api.get_tentacles_from_package_name(installed_package) + + # install missing + to_install_urls = [ + package_url + for package_url in additional_tentacles_package_urls + if package_url not in installed_community_package_urls + ] + return list(set(to_install_urls)), list(set(to_remove_tentacles)) + + +def is_community_tentacle_url(url: str) -> bool: + return constants.COMMUNITY_EXTENSIONS_PACKAGES_IDENTIFIER in url + + +async def uninstall_tentacles(tentacles: list[str]): + await tentacles_manager_api.uninstall_tentacles(tentacles) + diff --git a/octobot/constants.py b/octobot/constants.py index 76974b008c..a59f798e40 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -56,7 +56,8 @@ OCTOBOT_BETA_PROGRAM_FORM_URL = "https://octobot.click/docs-join-beta" AUTOMATION_FEEDBACK_FORM_ID = "n9NKMV" WELCOME_FEEDBACK_FORM_ID = os.getenv("WELCOME_FEEDBACK_FORM_ID", None) -OCTOBOT_EXTENSION_PACKAGE_1_NAME = "Advanced OctoBot extension" +OCTOBOT_EXTENSION_PACKAGE_1_NAME = "Premium OctoBot extension" +OCTOBOT_EXTENSION_PACKAGE_1_PRICE = "99.00" COMMUNITY_FEED_CURRENT_MINIMUM_VERSION = "1.0.0" COMMUNITY_FEED_CURRENT_EXCLUDED_MAXIMUM_VERSION = "2.0.0" @@ -65,6 +66,15 @@ COMMUNITY_TRADINGVIEW_WEBHOOK_BASE_URL = os.getenv( "COMMUNITY_TRADINGVIEW_WEBHOOK_BASE_URL", "https://webhook.octobot.cloud/tradingview" ) +COMMUNITY_EXTENSIONS_IDENTIFIER = "scaleway" +COMMUNITY_EXTENSIONS_CHECK_ENDPOINT = os.getenv( + "COMMUNITY_EXTENSIONS_CHECK_ENDPOINT", "https://premium.octobot.cloud" +) +COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY = os.getenv( + "COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY", "TODO" +) +COMMUNITY_EXTENSIONS_PACKAGES_IDENTIFIER = ".cloud" +COMMUNITY_FETCH_TIMEOUT = 30 # production env SHOULD ONLY BE USED THROUGH CommunityIdentifiersProvider OCTOBOT_COMMUNITY_LANDING_URL = os.getenv("COMMUNITY_SERVER_URL", "https://octobot.cloud") @@ -90,6 +100,7 @@ CONFIG_COMMUNITY = "community" CONFIG_COMMUNITY_BOT_ID = "bot_id" CONFIG_COMMUNITY_MQTT_UUID = "mqtt_uuid" +CONFIG_COMMUNITY_PACKAGE_URLS = "package_urls" CONFIG_COMMUNITY_ENVIRONMENT = "environment" USE_BETA_EARLY_ACCESS = os_util.parse_boolean_environment_var("USE_BETA_EARLY_ACCESS", "false") USER_ACCOUNT_EMAIL = os.getenv("USER_ACCOUNT_EMAIL", "") diff --git a/octobot/octobot.py b/octobot/octobot.py index 60ecf5b64e..04584104a0 100644 --- a/octobot/octobot.py +++ b/octobot/octobot.py @@ -115,7 +115,7 @@ async def initialize(self): self.stopped = asyncio.Event() await self._ensure_clock() if not (self.community_auth.is_initialized() and self.community_auth.is_using_the_current_loop()): - self.community_auth.init_account() + self.community_auth.init_account(True) self._log_config() await self.initializer.create(True) await self._start_tools_tasks() diff --git a/octobot/updater/updater.py b/octobot/updater/updater.py index 056c593585..3243874e31 100644 --- a/octobot/updater/updater.py +++ b/octobot/updater/updater.py @@ -19,6 +19,7 @@ import octobot.configuration_manager as configuration_manager import octobot.commands as commands import octobot_commons.logging as logging +import octobot_commons.authentication as authentication class Updater: @@ -44,8 +45,13 @@ async def update_impl(self) -> bool: raise NotImplementedError("update_impl is not implemented") async def update_tentacles(self): + # todo test + authenticator = authentication.Authenticator.instance() + additional_tentacles_package_urls = authenticator.get_saved_package_urls() await commands.install_all_tentacles( - tentacles_url=configuration_manager.get_default_tentacles_url(version=await self.get_latest_version())) + tentacles_url=configuration_manager.get_default_tentacles_url(version=await self.get_latest_version()), + additional_tentacles_package_urls=additional_tentacles_package_urls + ) async def post_update(self): await self.update_tentacles() diff --git a/requirements.txt b/requirements.txt index c27e94bb96..c0bb789577 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ OctoBot-Commons==1.9.47 OctoBot-Trading==2.4.88 OctoBot-Evaluators==1.9.5 OctoBot-Tentacles-Manager==2.9.11 -OctoBot-Services==1.6.14 +OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 trading-backend==1.2.24 diff --git a/tests/unit_tests/community/test_authentication.py b/tests/unit_tests/community/test_authentication.py index 8877a11add..5795a089b3 100644 --- a/tests/unit_tests/community/test_authentication.py +++ b/tests/unit_tests/community/test_authentication.py @@ -253,7 +253,7 @@ def test_init_account(auth): with mock.patch.object(asyncio, "create_task", mock.Mock(return_value="task")) as create_task_mock, \ mock.patch.object(auth, "_initialize_account", mock.Mock(return_value="coro")) \ as _auth_and_fetch_account_mock: - auth.init_account() + auth.init_account(True) create_task_mock.assert_called_once_with("coro") _auth_and_fetch_account_mock.assert_called_once() assert auth._fetch_account_task == "task" From 03a83fbd43d15ce93002b31eac9189ccc0182e12 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Tue, 2 Jul 2024 18:29:17 +0200 Subject: [PATCH 02/15] [Community] handle version in tentacles url --- octobot/commands.py | 3 +-- octobot/community/authentication.py | 1 + .../supabase_backend/community_supabase_client.py | 9 +++++++-- octobot/community/tentacles_packages.py | 13 +++++++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/octobot/commands.py b/octobot/commands.py index 6a9e21ad13..bd78a9ca8d 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -177,8 +177,7 @@ async def install_all_tentacles( for url in (base_urls if not only_additional else []) + (additional_tentacles_package_urls or []): if url is None: continue - if constants.VERSION_PLACEHOLDER in url: - url = url.replace(constants.VERSION_PLACEHOLDER, constants.LONG_VERSION) + url = community_tentacles_packages.adapt_url_to_bot_version(url) await tentacles_manager_api.install_all_tentacles(url, aiohttp_session=aiohttp_session, bot_install_dir=os.getcwd()) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index f57c387af4..498cb81ad4 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -527,6 +527,7 @@ async def _fetch_package_urls(self, mqtt_uuid: typing.Optional[str]) -> (list[st params={"mqtt_id": mqtt_uuid} if mqtt_uuid else {}, timeout=constants.COMMUNITY_FETCH_TIMEOUT ) + # status 502 = waking up resp.raise_for_status() json_resp = json.loads(resp.json().get("message", {})) if not json_resp: diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index a01f7068f9..2b035508d0 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import asyncio +import base64 import datetime import json import time @@ -650,16 +651,20 @@ async def http_get(self, url: str, *args, params=None, headers=None, **kwargs) - """ Perform http get using the current supabase auth token """ + params = params or {} + params["access_token"] = params.get("access_token", base64.b64encode(self._get_auth_key().encode()).decode()) return await self.postgrest.session.get(url, *args, params=params, headers=headers, **kwargs) async def http_post( - self, url: str, *args, data=None, files=None, json=None, params=None, headers=None, **kwargs + self, url: str, *args, json=None, params=None, headers=None, **kwargs ) -> httpx.Response: """ Perform http get using the current supabase auth token """ + json_body = json or {} + json_body["access_token"] = json_body.get("access_token", self._get_auth_key()) return await self.postgrest.session.post( - url, *args, data=data, files=files, json=json, params=params, headers=headers, **kwargs + url, *args, json=json_body, params=params, headers=headers, **kwargs ) @staticmethod diff --git a/octobot/community/tentacles_packages.py b/octobot/community/tentacles_packages.py index 5f19b8644b..9078babc13 100644 --- a/octobot/community/tentacles_packages.py +++ b/octobot/community/tentacles_packages.py @@ -29,17 +29,26 @@ async def has_tentacles_to_install_and_uninstall_tentacles_if_necessary(communit return bool(to_install) +def adapt_url_to_bot_version(package_url: str) -> str: + if constants.VERSION_PLACEHOLDER in package_url: + package_url = package_url.replace(constants.VERSION_PLACEHOLDER, constants.LONG_VERSION) + return package_url + + def get_to_install_and_remove_tentacles( community_auth, selected_profile_tentacles_setup_config ): installed_community_package_urls = [ - package_url + adapt_url_to_bot_version(package_url) for package_url in tentacles_manager_api.get_all_installed_package_urls( selected_profile_tentacles_setup_config ) if is_community_tentacle_url(package_url) ] - additional_tentacles_package_urls = community_auth.get_saved_package_urls() if community_auth.is_logged_in() else [] + additional_tentacles_package_urls = [ + adapt_url_to_bot_version(package_url) + for package_url in community_auth.get_saved_package_urls() + ] if community_auth.is_logged_in() else [] was_connected_with_remote_packages = community_auth.was_connected_with_remote_packages() # do not remove tentacles: From 68cf640da03eb8bedb5bee87a68a0b0275dacc26 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 3 Jul 2024 15:45:58 +0200 Subject: [PATCH 03/15] [Requirements] bump --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c0bb789577..1b7c2e5f1b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ OctoBot-Commons==1.9.47 OctoBot-Trading==2.4.88 OctoBot-Evaluators==1.9.5 -OctoBot-Tentacles-Manager==2.9.11 +OctoBot-Tentacles-Manager==2.9.12 OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 From ae0b9d946f1f8861a32126a680ef5ba88f8cffc1 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 3 Jul 2024 22:48:47 +0200 Subject: [PATCH 04/15] [Community] handle index fetch and profile update --- octobot/community/authentication.py | 36 ++++++++++++++-- .../community/models/community_public_data.py | 4 +- octobot/community/models/strategy_data.py | 41 ++++++++++++++++++- .../community_supabase_client.py | 20 +++++---- octobot/constants.py | 1 + octobot/octobot.py | 25 +++++++++++ 6 files changed, 112 insertions(+), 15 deletions(-) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index 498cb81ad4..d143d613e2 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -85,6 +85,7 @@ def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singl self.initialized_event = None self._login_completed = None + self._fetched_private_data = None self._startup_info = None self._fetch_account_task = None @@ -119,8 +120,10 @@ async def get_strategy(self, strategy_id, reload=False) -> strategy_data.Strateg await self.init_public_data(reset=reload) return self.public_data.get_strategy(strategy_id) - async def get_strategy_profile_data(self, strategy_id: str) -> commons_profiles.ProfileData: - return await self.supabase_client.fetch_product_config(strategy_id) + async def get_strategy_profile_data( + self, strategy_id: str, product_slug: str = None + ) -> commons_profiles.ProfileData: + return await self.supabase_client.fetch_product_config(strategy_id, product_slug=product_slug) def is_feed_connected(self): return self._community_feed is not None and self._community_feed.is_connected_to_remote_feed() @@ -217,6 +220,11 @@ async def _ensure_async_loop(self): self._login_completed = asyncio.Event() if should_set: self._login_completed.set() + if self._fetched_private_data is not None: + should_set = self._fetched_private_data.is_set() + self._fetched_private_data = asyncio.Event() + if should_set: + self._fetched_private_data.set() # changed event loop: restart client await self.supabase_client.close() self.user_account.flush() @@ -230,6 +238,8 @@ def is_initialized(self): return self.initialized_event is not None and self.initialized_event.is_set() def init_account(self, fetch_private_data): + if fetch_private_data and self._fetched_private_data is None: + self._fetched_private_data = asyncio.Event() self._fetch_account_task = asyncio.create_task(self._initialize_account(fetch_private_data=fetch_private_data)) async def async_init_account(self, fetch_private_data): @@ -271,6 +281,12 @@ async def wait_for_login_if_processing(self): # ensure login details have been fetched await asyncio.wait_for(self._login_completed.wait(), self.LOGIN_TIMEOUT) + async def wait_for_private_data_fetch_if_processing(self): + await self.wait_for_login_if_processing() + if self.is_logged_in() and self._fetched_private_data is not None and not self._fetched_private_data.is_set(): + # ensure login details have been fetched + await asyncio.wait_for(self._fetched_private_data.wait(), constants.COMMUNITY_FETCH_TIMEOUT) + def can_authenticate(self): return bool( identifiers_provider.IdentifiersProvider.BACKEND_URL @@ -381,6 +397,9 @@ async def get_current_bot_products_subscription(self) -> dict: def get_owned_packages(self) -> list[str]: return self.user_account.owned_packages + def has_open_source_package(self) -> bool: + return bool(self.get_owned_packages()) + def has_owned_packages_to_install(self) -> bool: return self.user_account.has_pending_packages_to_install @@ -490,7 +509,13 @@ async def _init_community_data(self, fetch_private_data): async def init_public_data(self, reset=False): if reset or not self.public_data.products.fetched: - self.public_data.set_products(await self.supabase_client.fetch_products()) + await self._refresh_products() + + async def _refresh_products(self): + category_types = ["profile"] + if self.has_open_source_package(): + category_types.append("index") + self.public_data.set_products(await self.supabase_client.fetch_products(category_types)) async def fetch_private_data(self, reset=False): try: @@ -516,6 +541,11 @@ async def fetch_private_data(self, reset=False): self.save_mqtt_device_uuid(fetched_mqtt_uuid) except Exception as err: self.logger.exception(err, True, f"Error when fetching package urls: {err}") + finally: + self._fetched_private_data.set() + if self.has_open_source_package(): + # fetch indexes as well + await self._refresh_products() async def _fetch_package_urls(self, mqtt_uuid: typing.Optional[str]) -> (list[str], str): resp = await self.supabase_client.http_get( diff --git a/octobot/community/models/community_public_data.py b/octobot/community/models/community_public_data.py index c101fe076b..c98dd4518c 100644 --- a/octobot/community/models/community_public_data.py +++ b/octobot/community/models/community_public_data.py @@ -18,7 +18,7 @@ import octobot.community.models.strategy_data as strategy_data -STRATEGY_CATEGORY_TYPE = "profile" +STRATEGY_CATEGORY_TYPES = ["profile", "index"] class CommunityPublicData: @@ -36,7 +36,7 @@ def get_strategies(self) -> list[strategy_data.StrategyData]: return [ strategy_data.StrategyData.from_dict(strategy_dict) for strategy_dict in self.products.value.values() - if self._get_category_type(strategy_dict) == STRATEGY_CATEGORY_TYPE + if self._get_category_type(strategy_dict) in STRATEGY_CATEGORY_TYPES ] def _get_category_type(self, product: dict): diff --git a/octobot/community/models/strategy_data.py b/octobot/community/models/strategy_data.py index 0b3383aac2..a5c6392199 100644 --- a/octobot/community/models/strategy_data.py +++ b/octobot/community/models/strategy_data.py @@ -20,6 +20,20 @@ import octobot_commons.enums as commons_enums +CATEGORY_NAME_TRANSLATIONS_BY_SLUG = { + "coingecko-index": {"en": "Crypto Basket"} +} +FORCED_URL_PATH_BY_SLUG = { + "coingecko-index": "features/crypto-basket", +} +DEFAULT_LOGO_NAME_BY_SLUG = { + "coingecko-index": "crypto-basket.png", +} +AUTO_UPDATED_CATEGORIES = ["coingecko-index"] +DEFAULT_LOGO_NAME = "default_strategy.png" +EXTENSION_CATEGORIES = ["coingecko-index"] + + @dataclasses.dataclass class CategoryData(commons_dataclasses.FlexibleDataclass): slug: str = "" @@ -33,8 +47,18 @@ def get_url(self) -> str: if external_links: if blog_slug := external_links.get("blog"): return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/en/blog/{blog_slug}" + if features_slug := external_links.get("features"): + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/features/{features_slug}" return "" + def get_default_logo_url(self) -> str: + return DEFAULT_LOGO_NAME_BY_SLUG.get(self.slug, DEFAULT_LOGO_NAME) + + def get_name(self, locale, default_locale=constants.DEFAULT_LOCALE): + return CATEGORY_NAME_TRANSLATIONS_BY_SLUG.get(self.slug, self.name_translations).get(locale, default_locale) + + def is_auto_updated(self) -> bool: + return self.slug in AUTO_UPDATED_CATEGORIES @dataclasses.dataclass class ResultsData(commons_dataclasses.FlexibleDataclass): @@ -76,7 +100,11 @@ def get_name(self, locale, default_locale=constants.DEFAULT_LOCALE): return self.content["name_translations"].get(locale, default_locale) def get_url(self) -> str: - return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_URL}/strategies/{self.slug}" + path = FORCED_URL_PATH_BY_SLUG.get(self.category.slug, f"strategies/{self.slug}") + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/{path}" + + def get_product_url(self) -> str: + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/strategies/{self.slug}" def get_risk(self) -> commons_enums.ProfileRisk: risk = self.attributes['risk'].upper() @@ -86,3 +114,14 @@ def get_risk(self) -> commons_enums.ProfileRisk: return commons_enums.ProfileRisk[risk] except KeyError: return commons_enums.ProfileRisk.MODERATE + + def get_logo_url(self, prefix: str) -> str: + if self.logo_url: + return self.logo_url + return f"{prefix}{self.category.get_default_logo_url()}" + + def is_auto_updated(self) -> bool: + return self.category.is_auto_updated() + + def is_extension_only(self) -> bool: + return self.category.slug in EXTENSION_CATEGORIES diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index 2b035508d0..8d873ab19a 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -234,7 +234,7 @@ async def fetch_startup_info(self, bot_id) -> dict: (await self.postgres_functions().invoke("get_startup_info", {"body": {"bot_id": bot_id}}))["data"] )[0] - async def fetch_products(self) -> list: + async def fetch_products(self, category_types: list[str]) -> list: return ( await self.table("products").select( "*," @@ -243,10 +243,9 @@ async def fetch_products(self) -> list: " profitability," " reference_market_profitability" ")" - ).match({ - enums.ProductKeys.VISIBILITY.value: "public", - "category.type": "profile", - }) + ).eq( + enums.ProductKeys.VISIBILITY.value, "public" + ).in_("category.type", category_types) .execute() ).data @@ -365,14 +364,17 @@ async def fetch_exchanges(self, exchange_ids: list) -> list: f"{enums.ExchangeKeys.INTERNAL_NAME.value}" ).in_(enums.ExchangeKeys.ID.value, exchange_ids).execute()).data - async def fetch_product_config(self, product_id: str) -> commons_profiles.ProfileData: - if not product_id: + async def fetch_product_config(self, product_id: str, product_slug: str = None) -> commons_profiles.ProfileData: + if not product_id and not product_slug: raise errors.MissingProductConfigError(f"product_id is '{product_id}'") try: - product = (await self.table("products").select( + query = self.table("products").select( "slug, " "product_config:product_configs!current_config_id(config, version)" - ).eq(enums.ProductKeys.ID.value, product_id).execute()).data[0] + ) + query = query.eq(enums.ProductKeys.SLUG.value, product_slug) if product_slug \ + else query.eq(enums.ProductKeys.ID.value, product_id) + product = (await query.execute()).data[0] except IndexError: raise errors.MissingProductConfigError(f"Missing product with id '{product_id}'") profile_data = commons_profiles.ProfileData.from_dict( diff --git a/octobot/constants.py b/octobot/constants.py index a59f798e40..1a20c1bf0a 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -115,6 +115,7 @@ # Profiles to force select at startup, identified by profile id, download url or name FORCED_PROFILE = os.getenv("FORCED_PROFILE", None) RUN_IN_MAIN_THREAD = os.getenv("RUN_IN_MAIN_THREAD", False) +PROFILE_UPDATE_RESTART_MIN = float(os.getenv("PROFILE_UPDATE_RESTART_MIN", 5)) OCTOBOT_BINARY_PROJECT_NAME = "OctoBot-Binary" diff --git a/octobot/octobot.py b/octobot/octobot.py index 04584104a0..44270f9af6 100644 --- a/octobot/octobot.py +++ b/octobot/octobot.py @@ -27,6 +27,7 @@ import octobot_commons.os_clock_sync as os_clock_sync import octobot_commons.system_resources_watcher as system_resources_watcher import octobot_commons.aiohttp_util as aiohttp_util +import octobot_commons.profiles as profiles import octobot_services.api as service_api import octobot_trading.api as trading_api @@ -157,6 +158,7 @@ async def _post_initialize(self): self.automation = automation.Automation(self.bot_id, self.tentacles_setup_config) self._init_metadata_run_task = asyncio.create_task(self._store_run_metadata_when_available()) + await self._init_profile_synchronizer() async def _wait_for_run_data_init(self, exchange_managers, timeout): for exchange_manager in exchange_managers: @@ -209,6 +211,7 @@ async def stop(self): await self.exchange_producer.stop() await self.community_auth.stop() await self.service_feed_producer.stop() + await profiles.stop_profile_synchronizer() await os_clock_sync.stop_clock_synchronizer() await system_resources_watcher.stop_system_resources_watcher() await service_api.stop_services() @@ -233,6 +236,28 @@ async def _ensure_clock(self): if trading_api.is_trader_enabled_in_config(self.config) and constants.ENABLE_CLOCK_SYNCH: await os_clock_sync.start_clock_synchronizer() + async def _init_profile_synchronizer(self): + await profiles.start_profile_synchronizer( + self.get_edited_config(constants.CONFIG_KEY, dict_only=False), + self._on_profile_update + ) + + async def delayed_restart(self, delay): + await asyncio.sleep(delay) + self.octobot_api.restart_bot() + + async def _on_profile_update(self, profile_name: str): + await service_api.send_notification( + service_api.create_notification( + f"{constants.PROJECT_NAME} will restart in {constants.PROFILE_UPDATE_RESTART_MIN} minutes " + f"to apply the {profile_name} profile update.", + markdown_format=commons_enums.MarkdownFormat.ITALIC + ) + ) + asyncio.create_task(self.delayed_restart( + constants.PROFILE_UPDATE_RESTART_MIN * commons_constants.MINUTE_TO_SECONDS + )) + async def _ensure_watchers(self): if constants.ENABLE_SYSTEM_WATCHER: await system_resources_watcher.start_system_resources_watcher( From 6aa0a336854dc6ffb273a555b3fa82d028a30d43 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Wed, 3 Jul 2024 22:15:46 +0200 Subject: [PATCH 05/15] [Requirements] bump --- requirements.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 1b7c2e5f1b..153a53a863 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Drakkar-Software requirements -OctoBot-Commons==1.9.47 -OctoBot-Trading==2.4.88 +OctoBot-Commons==1.9.48 +OctoBot-Trading==2.4.89 OctoBot-Evaluators==1.9.5 -OctoBot-Tentacles-Manager==2.9.12 +OctoBot-Tentacles-Manager==2.9.13 OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 From 7e1246de29f14bf9ded0725afdf553552ffa3c20 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 11:29:23 +0200 Subject: [PATCH 06/15] [Community] fix tentacles packages api calls --- octobot/commands.py | 4 ++- octobot/community/authentication.py | 3 +- .../community_supabase_client.py | 29 +++++++++++++++++++ octobot/community/tentacles_packages.py | 5 ++++ octobot/updater/updater.py | 1 - requirements.txt | 4 +-- 6 files changed, 41 insertions(+), 5 deletions(-) diff --git a/octobot/commands.py b/octobot/commands.py index bd78a9ca8d..4c0bdf5fe7 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -141,7 +141,9 @@ async def update_or_repair_tentacles_if_necessary(community_auth, selected_profi else: if to_install_urls: logger.debug("Installing new tentacles.") - await install_or_update_tentacles(config, to_install_urls, True) + # install additional tentacles only when tentacles arch is valid. Install all tentacles otherwise + only_additional = tentacles_manager_api.is_tentacles_architecture_valid() + await install_or_update_tentacles(config, to_install_urls, only_additional) if tentacles_manager_api.load_tentacles(verbose=True): logger.debug("OctoBot tentacles are up to date.") else: diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index d143d613e2..d7f7b92458 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -536,11 +536,12 @@ async def fetch_private_data(self, reset=False): ) if has_tentacles_to_install: # tentacles are not installed, save the fact that some are pending + self.logger.info(f"New tentacles are available for installation") self.user_account.has_pending_packages_to_install = True if fetched_mqtt_uuid and fetched_mqtt_uuid != mqtt_uuid: self.save_mqtt_device_uuid(fetched_mqtt_uuid) except Exception as err: - self.logger.exception(err, True, f"Error when fetching package urls: {err}") + self.logger.exception(err, True, f"Unexpected error when fetching package urls: {err}") finally: self._fetched_private_data.set() if self.has_open_source_package(): diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index 8d873ab19a..e9c94e6ab9 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -50,8 +50,35 @@ ] # disable httpx info logs as it logs every request commons_logging.set_logging_level(_INTERNAL_LOGGERS, logging.WARNING) +HTTP_RETRY_COUNT = 5 +def _httpx_retrier(f): + async def httpx_retrier_wrapper(*args, **kwargs): + resp = None + for i in range(0, HTTP_RETRY_COUNT): + error = None + try: + resp: httpx.Response = await f(*args, **kwargs) + if resp.status_code == 502: + # waking up, retry + error = "bad gateway" + else: + return resp + except httpx.ReadTimeout as err: + error = err + # retry + commons_logging.get_logger(__name__).debug( + f"Error on {f.__name__}(args={args[1:]}) " + f"request, retrying now. Attempt {i+1} / {HTTP_RETRY_COUNT} ({error})." + ) + # no more attempts + if resp: + resp.raise_for_status() + else: + raise errors.RequestError(f"Failed to execute {f.__name__}(args={args[1:]} kwargs={kwargs})") + return httpx_retrier_wrapper + class CommunitySupabaseClient(supabase_client.AuthenticatedAsyncSupabaseClient): """ Octobot Community layer added to supabase_client.AuthenticatedSupabaseClient @@ -649,6 +676,7 @@ def get_subscribed_channel_tables(self) -> set: def is_realtime_connected(self) -> bool: return self.realtime.socket and self.realtime.socket.connected and not self.realtime.socket.closed + @_httpx_retrier async def http_get(self, url: str, *args, params=None, headers=None, **kwargs) -> httpx.Response: """ Perform http get using the current supabase auth token @@ -657,6 +685,7 @@ async def http_get(self, url: str, *args, params=None, headers=None, **kwargs) - params["access_token"] = params.get("access_token", base64.b64encode(self._get_auth_key().encode()).decode()) return await self.postgrest.session.get(url, *args, params=params, headers=headers, **kwargs) + @_httpx_retrier async def http_post( self, url: str, *args, json=None, params=None, headers=None, **kwargs ) -> httpx.Response: diff --git a/octobot/community/tentacles_packages.py b/octobot/community/tentacles_packages.py index 9078babc13..e6a4f9cf2c 100644 --- a/octobot/community/tentacles_packages.py +++ b/octobot/community/tentacles_packages.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import octobot.constants as constants +import octobot_commons.logging as logging import octobot_tentacles_manager.api as tentacles_manager_api @@ -25,6 +26,10 @@ async def has_tentacles_to_install_and_uninstall_tentacles_if_necessary(communit community_auth, tentacles_setup_config ) if to_remove_tentacles: + logging.get_logger(__name__).debug( + f"Uninstalling {len(to_remove_tentacles)} tentacles: those are not available to the current OctoBot. " + f"Tentacles: {to_remove_tentacles}" + ) await uninstall_tentacles(to_remove_tentacles) return bool(to_install) diff --git a/octobot/updater/updater.py b/octobot/updater/updater.py index 3243874e31..a8ad0cf3f3 100644 --- a/octobot/updater/updater.py +++ b/octobot/updater/updater.py @@ -45,7 +45,6 @@ async def update_impl(self) -> bool: raise NotImplementedError("update_impl is not implemented") async def update_tentacles(self): - # todo test authenticator = authentication.Authenticator.instance() additional_tentacles_package_urls = authenticator.get_saved_package_urls() await commands.install_all_tentacles( diff --git a/requirements.txt b/requirements.txt index 153a53a863..0f64855964 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ # Drakkar-Software requirements -OctoBot-Commons==1.9.48 +OctoBot-Commons==1.9.49 OctoBot-Trading==2.4.89 OctoBot-Evaluators==1.9.5 -OctoBot-Tentacles-Manager==2.9.13 +OctoBot-Tentacles-Manager==2.9.14 OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 From 6c0994bf0fc33ebfcb46ad683df0e3f8367391b5 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 14:09:25 +0200 Subject: [PATCH 07/15] [Community] fix unistalled tentacles edge case --- octobot/commands.py | 9 ++++++--- octobot/community/authentication.py | 1 - .../supabase_backend/community_supabase_client.py | 2 +- octobot/community/tentacles_packages.py | 14 ++++++++++++-- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/octobot/commands.py b/octobot/commands.py index 4c0bdf5fe7..c607e49163 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -125,11 +125,14 @@ async def update_or_repair_tentacles_if_necessary(community_auth, selected_profi # only update tentacles based on local (non imported) profiles tentacles installation version local_profile_tentacles_setup_config = _get_first_non_imported_profile_tentacles_setup_config(config) - to_install_urls, to_remove_tentacles = community_tentacles_packages.get_to_install_and_remove_tentacles( - community_auth, selected_profile_tentacles_setup_config - ) + to_install_urls, to_remove_tentacles, force_refresh_tentacles_setup_config = \ + community_tentacles_packages.get_to_install_and_remove_tentacles( + community_auth, selected_profile_tentacles_setup_config + ) if to_remove_tentacles: await community_tentacles_packages.uninstall_tentacles(to_remove_tentacles) + elif force_refresh_tentacles_setup_config: + community_tentacles_packages.refresh_tentacles_setup_config() # await install_or_update_tentacles(config, additional_tentacles_package_urls) #TMPPP if local_profile_tentacles_setup_config is None or \ diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index d7f7b92458..47cef3bb2c 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -558,7 +558,6 @@ async def _fetch_package_urls(self, mqtt_uuid: typing.Optional[str]) -> (list[st params={"mqtt_id": mqtt_uuid} if mqtt_uuid else {}, timeout=constants.COMMUNITY_FETCH_TIMEOUT ) - # status 502 = waking up resp.raise_for_status() json_resp = json.loads(resp.json().get("message", {})) if not json_resp: diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index e9c94e6ab9..d66522ea93 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -66,7 +66,7 @@ async def httpx_retrier_wrapper(*args, **kwargs): else: return resp except httpx.ReadTimeout as err: - error = err + error = f"{err} ({err.__class__.__name__})" # retry commons_logging.get_logger(__name__).debug( f"Error on {f.__name__}(args={args[1:]}) " diff --git a/octobot/community/tentacles_packages.py b/octobot/community/tentacles_packages.py index e6a4f9cf2c..a9da2d45db 100644 --- a/octobot/community/tentacles_packages.py +++ b/octobot/community/tentacles_packages.py @@ -22,7 +22,7 @@ async def has_tentacles_to_install_and_uninstall_tentacles_if_necessary(communit tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config( community_auth.config.get_tentacles_config_path() ) - to_install, to_remove_tentacles = get_to_install_and_remove_tentacles( + to_install, to_remove_tentacles, force_refresh_tentacles_setup_config = get_to_install_and_remove_tentacles( community_auth, tentacles_setup_config ) if to_remove_tentacles: @@ -31,6 +31,8 @@ async def has_tentacles_to_install_and_uninstall_tentacles_if_necessary(communit f"Tentacles: {to_remove_tentacles}" ) await uninstall_tentacles(to_remove_tentacles) + elif force_refresh_tentacles_setup_config: + refresh_tentacles_setup_config() return bool(to_install) @@ -82,6 +84,10 @@ def get_to_install_and_remove_tentacles( ) for installed_package in installed_packages: to_remove_tentacles += tentacles_manager_api.get_tentacles_from_package_name(installed_package) + force_refresh_tentacles_setup_config = False + if to_remove_urls and not to_remove_tentacles: + # True when no tentacle to uninstall but still registered tentacles should be refreshed + force_refresh_tentacles_setup_config = True # install missing to_install_urls = [ @@ -89,13 +95,17 @@ def get_to_install_and_remove_tentacles( for package_url in additional_tentacles_package_urls if package_url not in installed_community_package_urls ] - return list(set(to_install_urls)), list(set(to_remove_tentacles)) + return list(set(to_install_urls)), list(set(to_remove_tentacles)), force_refresh_tentacles_setup_config def is_community_tentacle_url(url: str) -> bool: return constants.COMMUNITY_EXTENSIONS_PACKAGES_IDENTIFIER in url +def refresh_tentacles_setup_config(): + tentacles_manager_api.refresh_all_tentacles_setup_configs() + + async def uninstall_tentacles(tentacles: list[str]): await tentacles_manager_api.uninstall_tentacles(tentacles) From 2a60a25af3df52a534d449a022cc864f7ead7a2d Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 15:07:04 +0200 Subject: [PATCH 08/15] [Community] fix wrapper --- octobot/community/supabase_backend/community_supabase_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index d66522ea93..00ea4f138d 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -75,6 +75,7 @@ async def httpx_retrier_wrapper(*args, **kwargs): # no more attempts if resp: resp.raise_for_status() + return resp else: raise errors.RequestError(f"Failed to execute {f.__name__}(args={args[1:]} kwargs={kwargs})") return httpx_retrier_wrapper From 837dae813449945aec3931306adb459d335d1f7a Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 16:41:59 +0200 Subject: [PATCH 09/15] [Tentacles] hide url when necessary --- octobot/commands.py | 10 +++++++--- requirements.txt | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/octobot/commands.py b/octobot/commands.py index c607e49163..4040f18b86 100644 --- a/octobot/commands.py +++ b/octobot/commands.py @@ -182,10 +182,14 @@ async def install_all_tentacles( for url in (base_urls if not only_additional else []) + (additional_tentacles_package_urls or []): if url is None: continue + hide_url = url in additional_tentacles_package_urls url = community_tentacles_packages.adapt_url_to_bot_version(url) - await tentacles_manager_api.install_all_tentacles(url, - aiohttp_session=aiohttp_session, - bot_install_dir=os.getcwd()) + await tentacles_manager_api.install_all_tentacles( + url, + aiohttp_session=aiohttp_session, + bot_install_dir=os.getcwd(), + hide_url=hide_url, + ) def ensure_profile(config): diff --git a/requirements.txt b/requirements.txt index 0f64855964..9f32342fb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ OctoBot-Commons==1.9.49 OctoBot-Trading==2.4.89 OctoBot-Evaluators==1.9.5 -OctoBot-Tentacles-Manager==2.9.14 +OctoBot-Tentacles-Manager==2.9.15 OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 From b8e7d2d201101f9b9e65f67f0c873307ff05d72a Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 16:11:51 +0200 Subject: [PATCH 10/15] [Community] handle logged out package user --- octobot/community/authentication.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index 47cef3bb2c..88746412cc 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -398,7 +398,10 @@ def get_owned_packages(self) -> list[str]: return self.user_account.owned_packages def has_open_source_package(self) -> bool: - return bool(self.get_owned_packages()) + return ( + bool(self.get_owned_packages()) + or (not self.is_logged_in() and self.was_connected_with_remote_packages()) + ) def has_owned_packages_to_install(self) -> bool: return self.user_account.has_pending_packages_to_install From a2d4092afe5bf757dacbca7347d4d087081019cb Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 16:33:42 +0200 Subject: [PATCH 11/15] [Community] filter indexes when already loaded --- octobot/community/authentication.py | 9 +++++++-- octobot/community/models/community_public_data.py | 7 ++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index 88746412cc..e4d4c7af8b 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -114,7 +114,7 @@ def get_packages(self): async def get_strategies(self, reload=False) -> list[strategy_data.StrategyData]: await self.init_public_data(reset=reload) - return self.public_data.get_strategies() + return self.public_data.get_strategies(self._get_compatible_strategy_categories()) async def get_strategy(self, strategy_id, reload=False) -> strategy_data.StrategyData: await self.init_public_data(reset=reload) @@ -515,10 +515,15 @@ async def init_public_data(self, reset=False): await self._refresh_products() async def _refresh_products(self): + self.public_data.set_products( + await self.supabase_client.fetch_products(self._get_compatible_strategy_categories()) + ) + + def _get_compatible_strategy_categories(self) -> list[str]: category_types = ["profile"] if self.has_open_source_package(): category_types.append("index") - self.public_data.set_products(await self.supabase_client.fetch_products(category_types)) + return category_types async def fetch_private_data(self, reset=False): try: diff --git a/octobot/community/models/community_public_data.py b/octobot/community/models/community_public_data.py index c98dd4518c..bb009dd993 100644 --- a/octobot/community/models/community_public_data.py +++ b/octobot/community/models/community_public_data.py @@ -18,9 +18,6 @@ import octobot.community.models.strategy_data as strategy_data -STRATEGY_CATEGORY_TYPES = ["profile", "index"] - - class CommunityPublicData: def __init__(self): self.products = _DataElement({}, False) @@ -32,11 +29,11 @@ def set_products(self, products): def get_product_slug(self, product_id): return self.products.value[product_id][enums.ProductKeys.SLUG.value] - def get_strategies(self) -> list[strategy_data.StrategyData]: + def get_strategies(self, strategy_categories) -> list[strategy_data.StrategyData]: return [ strategy_data.StrategyData.from_dict(strategy_dict) for strategy_dict in self.products.value.values() - if self._get_category_type(strategy_dict) in STRATEGY_CATEGORY_TYPES + if self._get_category_type(strategy_dict) in strategy_categories ] def _get_category_type(self, product: dict): From ec9493c613293097a53e0a515d1be9c6cab0f309 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 16:54:12 +0200 Subject: [PATCH 12/15] [Community] fix strategies URL --- octobot/community/models/strategy_data.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/octobot/community/models/strategy_data.py b/octobot/community/models/strategy_data.py index a5c6392199..00c5466b98 100644 --- a/octobot/community/models/strategy_data.py +++ b/octobot/community/models/strategy_data.py @@ -100,11 +100,12 @@ def get_name(self, locale, default_locale=constants.DEFAULT_LOCALE): return self.content["name_translations"].get(locale, default_locale) def get_url(self) -> str: - path = FORCED_URL_PATH_BY_SLUG.get(self.category.slug, f"strategies/{self.slug}") - return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/{path}" + if path := FORCED_URL_PATH_BY_SLUG.get(self.category.slug): + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/{path}" + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_URL}/strategies/{self.slug}" def get_product_url(self) -> str: - return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_LANDING_URL}/strategies/{self.slug}" + return f"{identifiers_provider.IdentifiersProvider.COMMUNITY_URL}/strategies/{self.slug}" def get_risk(self) -> commons_enums.ProfileRisk: risk = self.attributes['risk'].upper() From d37ab968cb4f76802acf1c603f9115dbbd526971 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Thu, 4 Jul 2024 23:34:45 +0200 Subject: [PATCH 13/15] [Community] add package check key --- octobot/community/authentication.py | 5 ++++- octobot/community/supabase_backend/__init__.py | 2 ++ .../community/supabase_backend/community_supabase_client.py | 4 ++++ octobot/constants.py | 2 +- 4 files changed, 11 insertions(+), 2 deletions(-) diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index e4d4c7af8b..6209f201ad 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -285,7 +285,10 @@ async def wait_for_private_data_fetch_if_processing(self): await self.wait_for_login_if_processing() if self.is_logged_in() and self._fetched_private_data is not None and not self._fetched_private_data.is_set(): # ensure login details have been fetched - await asyncio.wait_for(self._fetched_private_data.wait(), constants.COMMUNITY_FETCH_TIMEOUT) + await asyncio.wait_for( + self._fetched_private_data.wait(), + supabase_backend.HTTP_RETRY_COUNT * constants.COMMUNITY_FETCH_TIMEOUT + ) def can_authenticate(self): return bool( diff --git a/octobot/community/supabase_backend/__init__.py b/octobot/community/supabase_backend/__init__.py index 68d8803323..da9ccb8328 100644 --- a/octobot/community/supabase_backend/__init__.py +++ b/octobot/community/supabase_backend/__init__.py @@ -30,6 +30,7 @@ from octobot.community.supabase_backend import community_supabase_client from octobot.community.supabase_backend.community_supabase_client import ( CommunitySupabaseClient, + HTTP_RETRY_COUNT, ) __all__ = [ @@ -38,4 +39,5 @@ "ASyncConfigurationStorage", "AuthenticatedAsyncSupabaseClient", "CommunitySupabaseClient", + "HTTP_RETRY_COUNT", ] diff --git a/octobot/community/supabase_backend/community_supabase_client.py b/octobot/community/supabase_backend/community_supabase_client.py index 00ea4f138d..8ab80913aa 100644 --- a/octobot/community/supabase_backend/community_supabase_client.py +++ b/octobot/community/supabase_backend/community_supabase_client.py @@ -64,6 +64,10 @@ async def httpx_retrier_wrapper(*args, **kwargs): # waking up, retry error = "bad gateway" else: + if i > 0: + commons_logging.get_logger(__name__).debug( + f"{f.__name__}(args={args[1:]}) succeeded after {i+1} attempts" + ) return resp except httpx.ReadTimeout as err: error = f"{err} ({err.__class__.__name__})" diff --git a/octobot/constants.py b/octobot/constants.py index 1a20c1bf0a..7ed5e92d0d 100644 --- a/octobot/constants.py +++ b/octobot/constants.py @@ -71,7 +71,7 @@ "COMMUNITY_EXTENSIONS_CHECK_ENDPOINT", "https://premium.octobot.cloud" ) COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY = os.getenv( - "COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY", "TODO" + "COMMUNITY_EXTENSIONS_CHECK_ENDPOINT_KEY", "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJhcHBsaWNhdGlvbl9jbGFpbSI6W3sibmFtZXNwYWNlX2lkIjoiIiwiYXBwbGljYXRpb25faWQiOiI5YTEzMTg2Mi1iMmY2LTRlOWUtYjU1OC0yOTU4MWFjYjM0ZjUifV0sInZlcnNpb24iOjIsImF1ZCI6ImZ1bmN0aW9ucyIsImp0aSI6IjJkMTM5OWQxLTRkNjMtNGVmNi1hNTI3LWNhMDQxMDdiNmUwYSIsImlhdCI6MTcyMDEyNjc1OSwiaXNzIjoiU0NBTEVXQVkiLCJuYmYiOjE3MjAxMjY3NTksInN1YiI6InRva2VuIn0.S2StO0Jey_BGotVdIYOa1hUNyF1m-BTLr-5oy24tiIXoh6nysMn_wBx0EzTDjQ_rG9yyUWbEYENjVlUzRJukiUf-5jjmIY0sgp6gYwtn6tu5Va1HyLOHpTNLYmSFcj7S-DcJXfd0uIGJcNRSAvYftnt-SVqjray0g5SfQEoB6UDSQolfECs4Avj7O0_Wtny1LHoIX_BEqlGWetODNklNNrBJuFUtSxoGfGarVGejOyvCdk10tFXpGJQr9dKPhnNSChs6N3qk4ApH5ET6JjOUENVF6x-KZ8Ed82KFU0gdGXICMVIiCUJz-b-QU88-HG6QG2-fD8dtvRUSCt_PsZPPZ_7IDWWuA-LEdNlKCyatVz0Yx3mCDusHN7Tt3ae-dJg9wpC4VCxqy8-MHOg9uf9GREkkc8Al-Nfn04tLWrl-OY_lrJ_jJ5_6N_XTwzNGmEdN3EVAeedwfyfpiuiXJMy84WQpfmJWn1zKEUrsBmx8xrTPz1pmZBB6uRKcdjUNWV2MpiAgxFxQI8Mo_zUJvagydfylcijjen8wP1ML0y8ywF8KSUmNprBv2SUwY8AXywtP5qIusnUEv-WxtoFdOU7Rgu3bCsdlktVEo2n2S-j6R9bki43gIgAmyxCveE-lwcNYoc_MahHMrjRW2uoO5deDo_yq90OJmnvnl35cLgVbkoA" ) COMMUNITY_EXTENSIONS_PACKAGES_IDENTIFIER = ".cloud" COMMUNITY_FETCH_TIMEOUT = 30 From 68636948b4224d7db8ee5bf74a114a7ef36109b0 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Fri, 5 Jul 2024 13:31:04 +0200 Subject: [PATCH 14/15] [CCXT] update for ccxt 4.3.56 --- .../exchanges_tests/abstract_authenticated_exchange_tester.py | 3 +++ .../abstract_authenticated_future_exchange_tester.py | 2 +- additional_tests/exchanges_tests/test_bybit.py | 1 + additional_tests/exchanges_tests/test_bybit_futures.py | 1 + additional_tests/exchanges_tests/test_hollaex.py | 2 +- additional_tests/exchanges_tests/test_mexc.py | 4 ++-- requirements.txt | 4 ++-- 7 files changed, 11 insertions(+), 6 deletions(-) diff --git a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py index de170eaa71..5821b99ac9 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py @@ -59,6 +59,7 @@ class AbstractAuthenticatedExchangeTester: EXPECT_POSSIBLE_ORDER_NOT_FOUND_DURING_ORDER_CREATION = False OPEN_ORDERS_IN_CLOSED_ORDERS = False CANCELLED_ORDERS_IN_CLOSED_ORDERS = False + EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = True RECENT_TRADES_UPDATE_TIMEOUT = 15 MARKET_FILL_TIMEOUT = 15 OPEN_TIMEOUT = 15 @@ -682,6 +683,8 @@ async def local_exchange_manager(self, market_filter=None, identifiers_suffix=No self.exchange_manager = None async def get_order(self, exchange_order_id, symbol=None): + assert self.exchange_manager.exchange.connector.client.has["fetchOrder"] is \ + self.EXPECT_FETCH_ORDER_TO_BE_AVAILABLE order = await self.exchange_manager.exchange.get_order(exchange_order_id, symbol or self.SYMBOL) self._check_fetched_order_dicts([order]) return personal_data.create_order_instance_from_raw(self.exchange_manager.trader, order) diff --git a/additional_tests/exchanges_tests/abstract_authenticated_future_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_future_exchange_tester.py index 1e949bcefb..9b36da85d9 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_future_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_future_exchange_tester.py @@ -92,7 +92,7 @@ async def inner_test_get_and_set_margin_type(self, allow_empty_position=False, s if origin_margin_type is trading_enums.MarginType.ISOLATED else trading_enums.MarginType.ISOLATED if not self.exchange_manager.exchange.SUPPORTS_SET_MARGIN_TYPE: assert origin_margin_type in (trading_enums.MarginType.ISOLATED, trading_enums.MarginType.CROSS) - with pytest.raises(ccxt.NotSupported): + with pytest.raises(trading_errors.NotSupported): await self.exchange_manager.exchange.connector.set_symbol_margin_type(symbol, True) with pytest.raises(trading_errors.NotSupported): await self.set_margin_type(new_margin_type, symbol=symbol) diff --git a/additional_tests/exchanges_tests/test_bybit.py b/additional_tests/exchanges_tests/test_bybit.py index 0a84be94c6..a0159ef1ee 100644 --- a/additional_tests/exchanges_tests/test_bybit.py +++ b/additional_tests/exchanges_tests/test_bybit.py @@ -35,6 +35,7 @@ class TestBybitAuthenticatedExchange( CONVERTS_ORDER_SIZE_BEFORE_PUSHING_TO_EXCHANGES = True EXPECT_MISSING_ORDER_FEES_DUE_TO_ORDERS_TOO_OLD_FOR_RECENT_TRADES = True # when recent trades are limited and # closed orders fees are taken from recent trades + EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = False async def test_get_portfolio(self): await super().test_get_portfolio() diff --git a/additional_tests/exchanges_tests/test_bybit_futures.py b/additional_tests/exchanges_tests/test_bybit_futures.py index c7667a7410..d2720e9ed2 100644 --- a/additional_tests/exchanges_tests/test_bybit_futures.py +++ b/additional_tests/exchanges_tests/test_bybit_futures.py @@ -36,6 +36,7 @@ class TestBybitFuturesAuthenticatedExchange( EDIT_TIMEOUT = 25 # larger for bybit testnet OPEN_ORDERS_IN_CLOSED_ORDERS = True SUPPORTS_GET_LEVERAGE = False + EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = False async def test_get_portfolio(self): await super().test_get_portfolio() diff --git a/additional_tests/exchanges_tests/test_hollaex.py b/additional_tests/exchanges_tests/test_hollaex.py index 120f4ded9e..26ea2431e0 100644 --- a/additional_tests/exchanges_tests/test_hollaex.py +++ b/additional_tests/exchanges_tests/test_hollaex.py @@ -27,7 +27,7 @@ class TestHollaexAuthenticatedExchange( # enter exchange name as a class variable here EXCHANGE_NAME = "hollaex" EXCHANGE_TENTACLE_NAME = "hollaex" # specify EXCHANGE_TENTACLE_NAME as the tentacle class has no capital H - ORDER_CURRENCY = "XHT" + ORDER_CURRENCY = "ETH" SETTLEMENT_CURRENCY = "USDT" SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}" ORDER_SIZE = 50 # % of portfolio to include in test orders diff --git a/additional_tests/exchanges_tests/test_mexc.py b/additional_tests/exchanges_tests/test_mexc.py index 01fb170787..f27155e9d5 100644 --- a/additional_tests/exchanges_tests/test_mexc.py +++ b/additional_tests/exchanges_tests/test_mexc.py @@ -27,8 +27,8 @@ class TestMEXCAuthenticatedExchange( # enter exchange name as a class variable here EXCHANGE_NAME = "mexc" EXCHANGE_TENTACLE_NAME = "MEXC" - ORDER_CURRENCY = "MX" # {"code":10007,"msg":"symbol not support api"} when trading BTC/USDT or ETH/USDT - SETTLEMENT_CURRENCY = "USDT" + ORDER_CURRENCY = "BTC" + SETTLEMENT_CURRENCY = "USDC" SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}" ORDER_SIZE = 30 # % of portfolio to include in test orders CONVERTS_ORDER_SIZE_BEFORE_PUSHING_TO_EXCHANGES = True diff --git a/requirements.txt b/requirements.txt index 9f32342fb7..99e7259dfd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,12 +1,12 @@ # Drakkar-Software requirements OctoBot-Commons==1.9.49 -OctoBot-Trading==2.4.89 +OctoBot-Trading==2.4.90 OctoBot-Evaluators==1.9.5 OctoBot-Tentacles-Manager==2.9.15 OctoBot-Services==1.6.15 OctoBot-Backtesting==1.9.7 Async-Channel==2.2.1 -trading-backend==1.2.24 +trading-backend==1.2.25 ## Others colorlog==6.8.0 From 5287d6aab598047f8da11e5fa945680c0a38adc6 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Fri, 5 Jul 2024 14:38:24 +0200 Subject: [PATCH 15/15] [Version] v2.0.0 --- CHANGELOG.md | 13 +++++++++++++ README.md | 2 +- octobot/__init__.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8f3af43d97..f84bb2dfb4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)* +## [2.0.0] - 2024-07-05 +### Added +- [Extension]: New OctoBot extension to profit from the Strategy designer, secure TradingView webhooks and automatically updated crypto baskets +- [WebInterface] Light & dark mode +- [Webhook] Support OctoBot cloud powered webhooks as ngrok alternative +### Updated +- [WebInterface] Complete refresh of the UI +- [CCXT] update to ccxt 4.3.56 +### Fixed +- [Coinbase] rate limit issue +- [Exchanges] permissions check issues +- [Orders] fix requests retry issues + ## [1.0.10] - 2024-04-19 ### Fixed - [Tentacles download] Fix a rare issue related to SSL certificates when downloading tentacles diff --git a/README.md b/README.md index 9bb9bdff06..a3f60ae0e5 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# OctoBot [1.0.10](https://octobot.click/gh-changelog) +# OctoBot [2.0.0](https://octobot.click/gh-changelog) [![PyPI](https://img.shields.io/pypi/v/OctoBot.svg?logo=pypi)](https://octobot.click/gh-pypi) [![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot) [![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg?logo=docker)](https://octobot.click/gh-dockerhub) diff --git a/octobot/__init__.py b/octobot/__init__.py index ac5208e558..ada86161d9 100644 --- a/octobot/__init__.py +++ b/octobot/__init__.py @@ -16,5 +16,5 @@ PROJECT_NAME = "OctoBot" AUTHOR = "Drakkar-Software" -VERSION = "1.0.10" # major.minor.revision +VERSION = "2.0.0" # major.minor.revision LONG_VERSION = f"{VERSION}"