Dev merge

CHANGELOG.md

@@ -6,6 +6,19 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)*
 
+## [2.0.0] - 2024-07-05
+### Added
+- [Extension]: New OctoBot extension to profit from the Strategy designer, secure TradingView webhooks and automatically updated crypto baskets
+- [WebInterface] Light & dark mode
+- [Webhook] Support OctoBot cloud powered webhooks as ngrok alternative
+### Updated
+- [WebInterface] Complete refresh of the UI
+- [CCXT] update to ccxt 4.3.56
+### Fixed
+- [Coinbase] rate limit issue
+- [Exchanges] permissions check issues
+- [Orders] fix requests retry issues
+
 ## [1.0.10] - 2024-04-19
 ### Fixed
 - [Tentacles download] Fix a rare issue related to SSL certificates when downloading tentacles

README.md

@@ -1,4 +1,4 @@
-# OctoBot [1.0.10](https://octobot.click/gh-changelog)
+# OctoBot [2.0.0](https://octobot.click/gh-changelog)
 [![PyPI](https://img.shields.io/pypi/v/OctoBot.svg?logo=pypi)](https://octobot.click/gh-pypi)
 [![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot)
 [![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg?logo=docker)](https://octobot.click/gh-dockerhub)

additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py

@@ -59,6 +59,7 @@ class AbstractAuthenticatedExchangeTester:
     EXPECT_POSSIBLE_ORDER_NOT_FOUND_DURING_ORDER_CREATION = False
     OPEN_ORDERS_IN_CLOSED_ORDERS = False
     CANCELLED_ORDERS_IN_CLOSED_ORDERS = False
+    EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = True
     RECENT_TRADES_UPDATE_TIMEOUT = 15
     MARKET_FILL_TIMEOUT = 15
     OPEN_TIMEOUT = 15
@@ -682,6 +683,8 @@ async def local_exchange_manager(self, market_filter=None, identifiers_suffix=No
             self.exchange_manager = None
 
     async def get_order(self, exchange_order_id, symbol=None):
+        assert self.exchange_manager.exchange.connector.client.has["fetchOrder"] is \
+               self.EXPECT_FETCH_ORDER_TO_BE_AVAILABLE
         order = await self.exchange_manager.exchange.get_order(exchange_order_id, symbol or self.SYMBOL)
         self._check_fetched_order_dicts([order])
         return personal_data.create_order_instance_from_raw(self.exchange_manager.trader, order)

additional_tests/exchanges_tests/abstract_authenticated_future_exchange_tester.py

@@ -92,7 +92,7 @@ async def inner_test_get_and_set_margin_type(self, allow_empty_position=False, s
             if origin_margin_type is trading_enums.MarginType.ISOLATED else trading_enums.MarginType.ISOLATED
         if not self.exchange_manager.exchange.SUPPORTS_SET_MARGIN_TYPE:
             assert origin_margin_type in (trading_enums.MarginType.ISOLATED, trading_enums.MarginType.CROSS)
-            with pytest.raises(ccxt.NotSupported):
+            with pytest.raises(trading_errors.NotSupported):
                 await self.exchange_manager.exchange.connector.set_symbol_margin_type(symbol, True)
             with pytest.raises(trading_errors.NotSupported):
                 await self.set_margin_type(new_margin_type, symbol=symbol)

additional_tests/exchanges_tests/test_bybit.py

@@ -35,6 +35,7 @@ class TestBybitAuthenticatedExchange(
     CONVERTS_ORDER_SIZE_BEFORE_PUSHING_TO_EXCHANGES = True
     EXPECT_MISSING_ORDER_FEES_DUE_TO_ORDERS_TOO_OLD_FOR_RECENT_TRADES = True   # when recent trades are limited and
     # closed orders fees are taken from recent trades
+    EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = False
 
     async def test_get_portfolio(self):
         await super().test_get_portfolio()

additional_tests/exchanges_tests/test_bybit_futures.py

@@ -36,6 +36,7 @@ class TestBybitFuturesAuthenticatedExchange(
     EDIT_TIMEOUT = 25    # larger for bybit testnet
     OPEN_ORDERS_IN_CLOSED_ORDERS = True
     SUPPORTS_GET_LEVERAGE = False
+    EXPECT_FETCH_ORDER_TO_BE_AVAILABLE = False
 
     async def test_get_portfolio(self):
         await super().test_get_portfolio()

additional_tests/exchanges_tests/test_hollaex.py

@@ -27,7 +27,7 @@ class TestHollaexAuthenticatedExchange(
     # enter exchange name as a class variable here
     EXCHANGE_NAME = "hollaex"
     EXCHANGE_TENTACLE_NAME = "hollaex"  # specify EXCHANGE_TENTACLE_NAME as the tentacle class has no capital H
-    ORDER_CURRENCY = "XHT"
+    ORDER_CURRENCY = "ETH"
     SETTLEMENT_CURRENCY = "USDT"
     SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}"
     ORDER_SIZE = 50  # % of portfolio to include in test orders

additional_tests/exchanges_tests/test_mexc.py

@@ -27,8 +27,8 @@ class TestMEXCAuthenticatedExchange(
     # enter exchange name as a class variable here
     EXCHANGE_NAME = "mexc"
     EXCHANGE_TENTACLE_NAME = "MEXC"
-    ORDER_CURRENCY = "MX"  # {"code":10007,"msg":"symbol not support api"} when trading BTC/USDT or ETH/USDT
-    SETTLEMENT_CURRENCY = "USDT"
+    ORDER_CURRENCY = "BTC"
+    SETTLEMENT_CURRENCY = "USDC"
     SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}"
     ORDER_SIZE = 30  # % of portfolio to include in test orders
     CONVERTS_ORDER_SIZE_BEFORE_PUSHING_TO_EXCHANGES = True

octobot/__init__.py

@@ -16,5 +16,5 @@
 
 PROJECT_NAME = "OctoBot"
 AUTHOR = "Drakkar-Software"
-VERSION = "1.0.10"  # major.minor.revision
+VERSION = "2.0.0"  # major.minor.revision
 LONG_VERSION = f"{VERSION}"

octobot/cli.py

@@ -132,17 +132,6 @@ def _create_startup_config(logger):
 
 async def _apply_community_startup_info_to_config(logger, config, community_auth):
     try:
-        if not community_auth.is_initialized() and constants.USER_ACCOUNT_EMAIL and constants.USER_PASSWORD_TOKEN:
-            try:
-                await community_auth.login(
-                    constants.USER_ACCOUNT_EMAIL, None, password_token=constants.USER_PASSWORD_TOKEN
-                )
-            except authentication.AuthenticationError as err:
-                logger.debug(f"Password token auth failure ({err}). Trying with saved session.")
-            if not community_auth.is_initialized():
-                await community_auth.async_init_account()
-        if not community_auth.is_logged_in():
-            return
         startup_info = await community_auth.get_startup_info()
         logger.debug(f"Fetched startup info: {startup_info}")
         commands.download_and_select_profile(
@@ -166,18 +155,42 @@ def _apply_env_variables_to_config(logger, config):
     )
 
 
-def _handle_forced_startup_config(logger, config, is_first_startup):
+async def _get_authenticated_community_if_possible(config, logger):
     # switch environments if necessary
     octobot_community.IdentifiersProvider.use_environment_from_config(config)
-
-    # 1. at first startup, get startup info from community when possible
     community_auth = octobot_community.CommunityAuthentication.create(config)
-    if is_first_startup:
-        asyncio.run(_apply_community_startup_info_to_config(logger, config, community_auth))
+    try:
+        if not community_auth.is_initialized():
+            if constants.IS_CLOUD_ENV and constants.USER_ACCOUNT_EMAIL and constants.USER_PASSWORD_TOKEN:
+                try:
+                    await community_auth.login(
+                        constants.USER_ACCOUNT_EMAIL, None, password_token=constants.USER_PASSWORD_TOKEN
+                    )
+                except authentication.AuthenticationError as err:
+                    logger.debug(f"Password token auth failure ({err}). Trying with saved session.")
+            if not community_auth.is_initialized():
+                # try with saved credentials if any
+                has_tentacles = tentacles_manager_api.is_tentacles_architecture_valid()
+                # When no tentacles, fetch private data. Otherwise fetch it later on in bot init
+                await community_auth.async_init_account(fetch_private_data=not has_tentacles)
+    except authentication.FailedAuthentication as err:
+        logger.error(f"Failed authentication when initializing community authenticator: {err}")
+    except Exception as err:
+        logger.error(f"Error when initializing community authenticator: {err}")
+    return community_auth
+
+
+async def _async_load_community_data(community_auth, config, logger, is_first_startup):
+    if constants.IS_CLOUD_ENV and is_first_startup:
+        # auto config
+        await _apply_community_startup_info_to_config(logger, config, community_auth)
+
+
+def _apply_forced_configs(community_auth, logger, config, is_first_startup):
+    asyncio.run(_async_load_community_data(community_auth, config, logger, is_first_startup))
 
     # 2. handle profiles from env variables
     _apply_env_variables_to_config(logger, config)
-    return community_auth
 
 
 def _read_config(config, logger):
@@ -208,21 +221,21 @@ def _repair_with_default_profile(config, logger):
     config.load_profiles_if_possible_and_necessary()
 
 
-def _load_or_create_tentacles(config, logger):
+def _load_or_create_tentacles(community_auth, config, logger):
     # add tentacles folder to Python path
     sys.path.append(os.path.realpath(os.getcwd()))
 
-    # when tentacles folder already exists
     if os.path.isfile(tentacles_manager_constants.USER_REFERENCE_TENTACLE_CONFIG_FILE_PATH):
+        # when tentacles folder already exists
         config.load_profiles_if_possible_and_necessary()
         tentacles_setup_config = tentacles_manager_api.get_tentacles_setup_config(
             config.get_tentacles_config_path()
         )
-        commands.run_update_or_repair_tentacles_if_necessary(config, tentacles_setup_config)
+        commands.run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config)
     else:
         # when no tentacles folder has been found
         logger.info("OctoBot tentacles can't be found. Installing default tentacles ...")
-        commands.run_tentacles_install_or_update(config)
+        commands.run_tentacles_install_or_update(community_auth, config)
         config.load_profiles_if_possible_and_necessary()
 
 
@@ -260,11 +273,16 @@ def start_octobot(args):
         # show terms
         _log_terms_if_unaccepted(config, logger)
 
+        community_auth = None if args.backtesting else asyncio.run(
+            _get_authenticated_community_if_possible(config, logger)
+        )
+
         # tries to load, install or repair tentacles
-        _load_or_create_tentacles(config, logger)
+        _load_or_create_tentacles(community_auth, config, logger)
 
         # patch setup with forced values
-        community_auth = None if args.backtesting else _handle_forced_startup_config(logger, config, is_first_startup)
+        if not args.backtesting:
+            _apply_forced_configs(community_auth, logger, config, is_first_startup)
 
         # Can now perform config health check (some checks require a loaded profile)
         configuration_manager.config_health_check(config, args.backtesting)

octobot/commands.py

@@ -15,7 +15,8 @@
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
 
 import os
-import aiohttp
+import typing
+
 import sys
 import asyncio
 import signal
@@ -34,6 +35,7 @@
 import octobot.api.strategy_optimizer as strategy_optimizer_api
 import octobot.logger as octobot_logger
 import octobot.constants as constants
+import octobot.community.tentacles_packages as community_tentacles_packages
 import octobot.configuration_manager as configuration_manager
 
 COMMANDS_LOGGER_NAME = "Commands"
@@ -77,13 +79,18 @@ def start_strategy_optimizer(config, commands):
         strategy_optimizer_api.print_optimizer_report(optimizer)
 
 
-def run_tentacles_install_or_update(config):
+def run_tentacles_install_or_update(community_auth, config):
     _check_tentacles_install_exit()
-    asyncio.run(install_or_update_tentacles(config))
+    asyncio.run(_install_or_update_tentacles(community_auth, config))
+
+
+async def _install_or_update_tentacles(community_auth, config):
+    additional_tentacles_package_urls = community_auth.get_saved_package_urls()
+    await install_or_update_tentacles(config, additional_tentacles_package_urls, False)
 
 
-def run_update_or_repair_tentacles_if_necessary(config, tentacles_setup_config):
-    asyncio.run(update_or_repair_tentacles_if_necessary(tentacles_setup_config, config))
+def run_update_or_repair_tentacles_if_necessary(community_auth, config, tentacles_setup_config):
+    asyncio.run(update_or_repair_tentacles_if_necessary(community_auth, tentacles_setup_config, config))
 
 
 def _check_tentacles_install_exit():
@@ -102,7 +109,7 @@ def _get_first_non_imported_profile_tentacles_setup_config(config):
         return None
 
 
-async def update_or_repair_tentacles_if_necessary(selected_profile_tentacles_setup_config, config):
+async def update_or_repair_tentacles_if_necessary(community_auth, selected_profile_tentacles_setup_config, config):
     local_profile_tentacles_setup_config = selected_profile_tentacles_setup_config
     logger = logging.get_logger(COMMANDS_LOGGER_NAME)
     if config.profile.imported:
@@ -117,45 +124,72 @@ async def update_or_repair_tentacles_if_necessary(selected_profile_tentacles_set
                         f"Please make sure that this profile works on your OctoBot.")
             # only update tentacles based on local (non imported) profiles tentacles installation version
             local_profile_tentacles_setup_config = _get_first_non_imported_profile_tentacles_setup_config(config)
+
+    to_install_urls, to_remove_tentacles, force_refresh_tentacles_setup_config = \
+        community_tentacles_packages.get_to_install_and_remove_tentacles(
+            community_auth, selected_profile_tentacles_setup_config
+        )
+    if to_remove_tentacles:
+        await community_tentacles_packages.uninstall_tentacles(to_remove_tentacles)
+    elif force_refresh_tentacles_setup_config:
+        community_tentacles_packages.refresh_tentacles_setup_config()
+
+    # await install_or_update_tentacles(config, additional_tentacles_package_urls)    #TMPPP
     if local_profile_tentacles_setup_config is None or \
             not tentacles_manager_api.are_tentacles_up_to_date(local_profile_tentacles_setup_config, constants.VERSION):
         logger.info("OctoBot tentacles are not up to date. Updating tentacles...")
         _check_tentacles_install_exit()
-        if await install_or_update_tentacles(config):
+        if await install_or_update_tentacles(config, to_install_urls, False):
             logger.info("OctoBot tentacles are now up to date.")
-    elif tentacles_manager_api.load_tentacles(verbose=True):
-        logger.debug("OctoBot tentacles are up to date.")
     else:
-        logger.info("OctoBot tentacles are damaged. Installing default tentacles ...")
-        _check_tentacles_install_exit()
-        await install_or_update_tentacles(config)
-
-
-async def install_or_update_tentacles(config):
-    await install_all_tentacles()
+        if to_install_urls:
+            logger.debug("Installing new tentacles.")
+            # install additional tentacles only when tentacles arch is valid. Install all tentacles otherwise
+            only_additional = tentacles_manager_api.is_tentacles_architecture_valid()
+            await install_or_update_tentacles(config, to_install_urls, only_additional)
+        if tentacles_manager_api.load_tentacles(verbose=True):
+            logger.debug("OctoBot tentacles are up to date.")
+        else:
+            logger.info("OctoBot tentacles are damaged. Installing default tentacles only ...")
+            _check_tentacles_install_exit()
+            await install_or_update_tentacles(config, [], False)
+
+
+async def install_or_update_tentacles(
+    config, additional_tentacles_package_urls: typing.Optional[list], only_additional: bool
+):
+    await install_all_tentacles(
+        additional_tentacles_package_urls=additional_tentacles_package_urls, only_additional=only_additional
+    )
     # reload profiles
     config.load_profiles()
     # reload tentacles
     return tentacles_manager_api.load_tentacles(verbose=True)
 
 
-async def install_all_tentacles(tentacles_url=None):
+async def install_all_tentacles(
+    tentacles_url=None, additional_tentacles_package_urls: typing.Optional[list] = None, only_additional: bool = False
+):
     if tentacles_url is None:
         tentacles_url = configuration_manager.get_default_tentacles_url()
     async with aiohttp_util.ssl_fallback_aiohttp_client_session(
         commons_constants.KNOWN_POTENTIALLY_SSL_FAILED_REQUIRED_URL
     ) as aiohttp_session:
-        for url in [tentacles_url] + (
-                constants.ADDITIONAL_TENTACLES_PACKAGE_URL.split(constants.URL_SEPARATOR)
-                if constants.ADDITIONAL_TENTACLES_PACKAGE_URL else []
-        ):
+        base_urls = [tentacles_url] + (
+            constants.ADDITIONAL_TENTACLES_PACKAGE_URL.split(constants.URL_SEPARATOR)
+            if constants.ADDITIONAL_TENTACLES_PACKAGE_URL else []
+        )
+        for url in (base_urls if not only_additional else []) + (additional_tentacles_package_urls or []):
             if url is None:
                 continue
-            if constants.VERSION_PLACEHOLDER in url:
-                url = url.replace(constants.VERSION_PLACEHOLDER, constants.LONG_VERSION)
-            await tentacles_manager_api.install_all_tentacles(url,
-                                                              aiohttp_session=aiohttp_session,
-                                                              bot_install_dir=os.getcwd())
+            hide_url = url in additional_tentacles_package_urls
+            url = community_tentacles_packages.adapt_url_to_bot_version(url)
+            await tentacles_manager_api.install_all_tentacles(
+                url,
+                aiohttp_session=aiohttp_session,
+                bot_install_dir=os.getcwd(),
+                hide_url=hide_url,
+            )
 
 
 def ensure_profile(config):