Master update

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
         architecture: ${{ matrix.arch }}
 
     - name: Install dependencies
-      run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt
+      run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt
 
 #    - name: Black lint
 #      run: black ${{ secrets.PACKAGE_FOLDER }} --diff --check
@@ -70,7 +70,7 @@ jobs:
       run: exit 1
 
     - name: Install dependencies
-      run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt
+      run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt
 
     - name: Install tentacles on Unix
       env:
@@ -137,7 +137,7 @@ jobs:
           architecture: ${{ matrix.arch }}
 
       - name: Install dependencies
-        run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt
+        run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt
 
       - name: Build sdist
         run: python setup.py sdist

Dockerfile

@@ -1,13 +1,9 @@
-FROM python:3.10-slim-buster AS base
+FROM python:3.10-slim-bookworm AS base
 
 WORKDIR /
 
 # requires git to install requirements with git+https
-# Update to debian archive from https://gist.github.com/ishad0w/6ce1eb569c734880200c47923577426a
-RUN echo "deb http://archive.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list \
-    && echo "deb http://archive.debian.org/debian-security buster/updates main contrib non-free" >> /etc/apt/sources.list \
-    && echo "deb http://archive.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list \
-    && apt-get update \
+RUN apt-get update \
     && apt-get install -y --no-install-recommends build-essential git gcc binutils libffi-dev libssl-dev libxml2-dev libxslt1-dev libxslt-dev libjpeg62-turbo-dev zlib1g-dev \
     && python -m venv /opt/venv
 
@@ -19,10 +15,10 @@ ENV PATH="/opt/venv/bin:$PATH"
 
 COPY . .
 RUN pip install -U setuptools wheel pip>=20.0.0 \
-    && pip install --no-cache-dir --prefer-binary -r requirements.txt \
+    && pip install --no-cache-dir --prefer-binary -r requirements.txt -r full_requirements.txt \
     && python setup.py install
 
-FROM python:3.10-slim-buster
+FROM python:3.10-slim-bookworm
 
 ARG TENTACLES_URL_TAG=""
 ENV TENTACLES_URL_TAG=$TENTACLES_URL_TAG
@@ -41,11 +37,7 @@ COPY docker/* /octobot/
 # 2. Install required packages
 # 3. Finish env setup
 SHELL ["/bin/bash", "-o", "pipefail", "-c"]
-# Update to debian archive from https://gist.github.com/ishad0w/6ce1eb569c734880200c47923577426a
-RUN echo "deb http://archive.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list \
-    && echo "deb http://archive.debian.org/debian-security buster/updates main contrib non-free" >> /etc/apt/sources.list \
-    && echo "deb http://archive.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list \
-    && apt-get update \
+RUN apt-get update \
     && apt-get install -y --no-install-recommends curl libxslt-dev libxcb-xinput0 libjpeg62-turbo-dev zlib1g-dev libblas-dev liblapack-dev libatlas-base-dev libopenjp2-7 libtiff-dev \
     && rm -rf /var/lib/apt/lists/* \
     && ln -s /opt/venv/bin/OctoBot OctoBot # Make sure we use the virtualenv \

MANIFEST.in

@@ -4,5 +4,6 @@ include README.md
 include LICENSE
 include CHANGELOG.md
 include requirements.txt
+include full_requirements.txt
 
 global-exclude *.c

additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py

@@ -554,16 +554,17 @@ async def test_create_and_cancel_limit_orders(self):
 
     async def inner_test_create_and_cancel_limit_orders(self, symbol=None, settlement_currency=None, **kwargs):
         symbol = symbol or self.SYMBOL
+        side = trading_enums.TradeOrderSide.BUY
         # # DEBUG tools p1, uncomment to create specific orders
         # symbol = "ADA/USDT"
         # # end debug tools
         market_status = self.exchange_manager.exchange.get_market_status(symbol)
         exchange_data = self.get_exchange_data(symbol=symbol)
         settlement_currency = settlement_currency or self.SETTLEMENT_CURRENCY
-        price = self.get_order_price(await self.get_price(symbol=symbol), False, symbol=symbol)
+        price = self.get_order_price(await self.get_price(symbol=symbol), side == trading_enums.TradeOrderSide.SELL, symbol=symbol)
         # 1. try with "normal" order size
         default_size = self.get_order_size(
-            await self.get_portfolio(), price, symbol=symbol, settlement_currency=settlement_currency
+            await self.get_portfolio(), price, symbol=symbol, settlement_currency=(settlement_currency if side == trading_enums.TradeOrderSide.BUY else self.ORDER_CURRENCY)
         )
         # self.check_order_size_and_price(default_size, price, symbol=symbol, allow_empty_size=self.CHECK_EMPTY_ACCOUNT)
         enable_min_size_check = False
@@ -604,26 +605,27 @@ async def inner_test_create_and_cancel_limit_orders(self, symbol=None, settlemen
             assert cancelled_orders == []
             return
         try:
-            buy_limit = await self.create_limit_order(price, size, trading_enums.TradeOrderSide.BUY, symbol=symbol)
+            limit_order = await self.create_limit_order(price, size, side, symbol=symbol)
         except trading_errors.AuthenticationError as err:
             raise trading_errors.AuthenticationError(
                 f"inner_test_create_and_cancel_limit_orders#create_limit_order {err}"
             ) from err
         try:
-            self.check_created_limit_order(buy_limit, price, size, trading_enums.TradeOrderSide.BUY)
-            assert await self.order_in_open_orders(open_orders, buy_limit, symbol=symbol)
-            await self.check_can_get_order(buy_limit)
+            self.check_created_limit_order(limit_order, price, size, side)
+            assert await self.order_in_open_orders(open_orders, limit_order, symbol=symbol)
+            await self.check_can_get_order(limit_order)
             # assert free portfolio amount is smaller than total amount
             balance = await self.get_portfolio()
-            assert balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_FREE] < \
-                   balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL], (
-                    f"FALSE: {balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_FREE]} < {balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL]}"
+            locked_currency = settlement_currency if side == trading_enums.TradeOrderSide.BUY else self.ORDER_CURRENCY
+            assert balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_FREE] < \
+                   balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL], (
+                    f"FALSE: {balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_FREE]} < {balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL]}"
                    )
         finally:
             # don't leave buy_limit as open order
-            await self.cancel_order(buy_limit)
-        assert await self.order_not_in_open_orders(open_orders, buy_limit, symbol=symbol)
-        assert await self.order_in_cancelled_orders(cancelled_orders, buy_limit, symbol=symbol)
+            await self.cancel_order(limit_order)
+        assert await self.order_not_in_open_orders(open_orders, limit_order, symbol=symbol)
+        assert await self.order_in_cancelled_orders(cancelled_orders, limit_order, symbol=symbol)
 
     async def inner_test_cancel_uncancellable_order(self):
         if self.UNCANCELLABLE_ORDER_ID_SYMBOL_TYPE:
@@ -636,39 +638,44 @@ async def test_create_and_fill_market_orders(self):
             await self.inner_test_create_and_fill_market_orders()
 
     async def inner_test_create_and_fill_market_orders(self):
+        side = trading_enums.TradeOrderSide.BUY
         portfolio = await self.get_portfolio()
         current_price = await self.get_price()
-        price = self.get_order_price(current_price, False, price_diff=0)
-        size = self.get_order_size(portfolio, price)
+        order_currency = self.ORDER_CURRENCY if side == trading_enums.TradeOrderSide.SELL else self.SETTLEMENT_CURRENCY
+        price = self.get_order_price(current_price, side == trading_enums.TradeOrderSide.SELL, price_diff=0)
+        size = self.get_order_size(portfolio, price, settlement_currency=order_currency)
         if self.CHECK_EMPTY_ACCOUNT:
             assert size == trading_constants.ZERO
             return
-        buy_market = await self.create_market_order(current_price, size, trading_enums.TradeOrderSide.BUY)
+        first_market_order = await self.create_market_order(current_price, size, side)
         post_buy_portfolio = {}
         try:
-            self.check_created_market_order(buy_market, size, trading_enums.TradeOrderSide.BUY)
-            filled_order = await self.wait_for_fill(buy_market)
+            self.check_created_market_order(first_market_order, size, side)
+            filled_order = await self.wait_for_fill(first_market_order)
             parsed_filled_order = personal_data.create_order_instance_from_raw(
                 self.exchange_manager.trader,
                 filled_order
             )
-            self._check_order(parsed_filled_order, size, trading_enums.TradeOrderSide.BUY)
+            self._check_order(parsed_filled_order, size, side)
             await self.wait_for_order_exchange_id_in_trades(parsed_filled_order.exchange_order_id)
             await self.check_require_order_fees_from_trades(
                 filled_order[trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value]
             )
             self.check_raw_closed_orders([filled_order])
             post_buy_portfolio = await self.get_portfolio()
-            self.check_portfolio_changed(portfolio, post_buy_portfolio, False)
+            portfolio_increased = side == trading_enums.TradeOrderSide.SELL
+            self.check_portfolio_changed(portfolio, post_buy_portfolio, portfolio_increased)
         finally:
-            sell_size = self.get_sell_size_from_buy_order(buy_market)
+            mirror_size = self.get_sell_size_from_buy_order(first_market_order)
             # sell: reset portfolio
-            sell_market = await self.create_market_order(current_price, sell_size, trading_enums.TradeOrderSide.SELL)
-            self.check_created_market_order(sell_market, sell_size, trading_enums.TradeOrderSide.SELL)
-            await self.wait_for_fill(sell_market)
+            other_side = trading_enums.TradeOrderSide.SELL if side == trading_enums.TradeOrderSide.BUY else trading_enums.TradeOrderSide.BUY
+            second_market_order = await self.create_market_order(current_price, mirror_size, other_side)
+            self.check_created_market_order(second_market_order, mirror_size, other_side)
+            await self.wait_for_fill(second_market_order)
             post_sell_portfolio = await self.get_portfolio()
             if post_buy_portfolio:
-                self.check_portfolio_changed(post_buy_portfolio, post_sell_portfolio, True)
+                portfolio_increased = other_side == trading_enums.TradeOrderSide.SELL
+                self.check_portfolio_changed(post_buy_portfolio, post_sell_portfolio, portfolio_increased)
 
     async def check_require_order_fees_from_trades(self, filled_exchange_order_id, symbol=None):
         symbol = symbol or self.SYMBOL
@@ -771,8 +778,8 @@ async def inner_test_edit_limit_order(self):
             sell_limit = await self.create_limit_order(price, size, trading_enums.TradeOrderSide.SELL)
             self.check_created_limit_order(sell_limit, price, size, trading_enums.TradeOrderSide.SELL)
             assert await self.order_in_open_orders(open_orders, sell_limit)
-            edited_price = self.get_order_price(current_price, True, price_diff=2*self.ORDER_PRICE_DIFF)
-            edited_size = self.get_order_size(portfolio, price, order_size=2*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency())
+            edited_price = self.get_order_price(current_price, True, price_diff=1.3*self.ORDER_PRICE_DIFF)
+            edited_size = self.get_order_size(portfolio, price, order_size=1.3*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency())
             sell_limit = await self.edit_order(sell_limit, edited_price=edited_price, edited_quantity=edited_size)
             await self.wait_for_edit(sell_limit, edited_size)
             sell_limit = await self.get_order(sell_limit.exchange_order_id, sell_limit.symbol)
@@ -800,8 +807,8 @@ async def inner_test_edit_stop_order(self):
             )
             self.check_created_stop_order(stop_loss, price, size, trading_enums.TradeOrderSide.SELL)
             assert await self.order_in_open_orders(open_orders, stop_loss)
-            edited_price = self.get_order_price(current_price, False, price_diff=2*self.ORDER_PRICE_DIFF)
-            edited_size = self.get_order_size(portfolio, price, order_size=2*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency())
+            edited_price = self.get_order_price(current_price, False, price_diff=1.3*self.ORDER_PRICE_DIFF)
+            edited_size = self.get_order_size(portfolio, price, order_size=1.3*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency())
             stop_loss = await self.edit_order(stop_loss, edited_price=edited_price, edited_quantity=edited_size)
             await self.wait_for_edit(stop_loss, edited_size)
             stop_loss = await self.get_order(stop_loss.exchange_order_id, stop_loss.symbol)

additional_tests/exchanges_tests/test_hyperliquid.py

@@ -26,7 +26,7 @@ class TestHyperliquidAuthenticatedExchange(
 ):
     # enter exchange name as a class variable here
     EXCHANGE_NAME = "hyperliquid"
-    ORDER_CURRENCY = "HYPE"
+    ORDER_CURRENCY = "ETH"
     SETTLEMENT_CURRENCY = "USDC"
     SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}"
     ORDER_SIZE = 25  # % of portfolio to include in test orders

full_requirements.txt

@@ -0,0 +1,22 @@
+# Drakkar-Software full requirements
+OctoBot-Commons[full]==1.9.91
+OctoBot-Trading[full]==2.4.235
+OctoBot-Evaluators[full]==1.9.9
+OctoBot-Tentacles-Manager[full]==2.9.19
+OctoBot-Services[full]==1.6.30
+OctoBot-Backtesting[full]==1.9.8
+
+## Others
+colorlog==6.8.0
+
+# Community
+gmqtt==0.7.0
+pgpy==0.6.0
+clickhouse-connect==0.8.18
+pyiceberg==0.10.0
+pydantic<2.12 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590
+pyarrow==21.0.0
+
+# used by ccxt for protobuf "websockets" such as mexc
+# lock protobuf to avoid using .rc versions
+protobuf==5.29.5

octobot/cli.py

@@ -21,35 +21,42 @@
 
 import packaging.version as packaging_version
 
-import octobot_commons.os_util as os_util
-import octobot_commons.logging as logging
-import octobot_commons.configuration as configuration
-import octobot_commons.profiles as profiles
-import octobot_commons.authentication as authentication
-import octobot_commons.constants as common_constants
-import octobot_commons.errors as errors
-
-import octobot_services.api as service_api
-
-import octobot_tentacles_manager.api as tentacles_manager_api
-import octobot_tentacles_manager.cli as tentacles_manager_cli
-import octobot_tentacles_manager.constants as tentacles_manager_constants
-
-# make tentacles importable
-sys.path.append(os.path.dirname(sys.executable))
-
-import octobot.octobot as octobot_class
-import octobot.commands as commands
-import octobot.configuration_manager as configuration_manager
-import octobot.octobot_backtesting_factory as octobot_backtesting
-import octobot.constants as constants
-import octobot.enums as enums
-import octobot.disclaimer as disclaimer
-import octobot.logger as octobot_logger
-import octobot.community as octobot_community
-import octobot.community.errors
-import octobot.limits as limits
-
+try:
+    import octobot_commons.os_util as os_util
+    import octobot_commons.logging as logging
+    import octobot_commons.configuration as configuration
+    import octobot_commons.profiles as profiles
+    import octobot_commons.authentication as authentication
+    import octobot_commons.constants as common_constants
+    import octobot_commons.errors as errors
+
+    import octobot_services.api as service_api
+
+    import octobot_tentacles_manager.api as tentacles_manager_api
+    import octobot_tentacles_manager.cli as tentacles_manager_cli
+    import octobot_tentacles_manager.constants as tentacles_manager_constants
+
+    # make tentacles importable
+    sys.path.append(os.path.dirname(sys.executable))
+
+    import octobot.octobot as octobot_class
+    import octobot.commands as commands
+    import octobot.configuration_manager as configuration_manager
+    import octobot.octobot_backtesting_factory as octobot_backtesting
+    import octobot.constants as constants
+    import octobot.enums as enums
+    import octobot.disclaimer as disclaimer
+    import octobot.logger as octobot_logger
+    import octobot.community as octobot_community
+    import octobot.community.errors
+    import octobot.limits as limits
+except ImportError as err:
+    print(
+        "Error importing OctoBot dependencies, please install OctoBot with the [full] option. "
+        "Example: \"pip install -U octobot[full]\" "
+        "(Error: {0}: {1})".format(err.__class__.__name__, str(err)), file=sys.stderr
+    )
+    sys.exit(-1)
 
 def update_config_with_args(starting_args, config: configuration.Configuration, logger):
     try:

octobot/community/__init__.py

@@ -24,6 +24,7 @@
 )
 from octobot.community import models
 from octobot.community.models import (
+    BotLogData,
     CommunityUserAccount,
     CommunityFields,
     CommunityTentaclesPackage,

octobot/community/feeds/community_mqtt_feed.py

@@ -15,14 +15,37 @@
 #  License along with OctoBot. If not, see <https://www.gnu.org/licenses/>.
 import logging
 import typing
-import gmqtt
+import octobot_commons.constants as commons_constants
+try:
+    import gmqtt
+except ImportError:
+    if commons_constants.USE_MINIMAL_LIBS:
+        # mock gmqtt imports
+        class GmqttImportMock:
+            class Client:
+                def __init__(self, *args):
+                    raise ImportError("gmqtt not installed")
+            class Subscription:
+                def __init__(self, topic, qos):
+                    raise ImportError("gmqtt not installed")
+            class constants:
+                class MQTTv311:
+                    pass
+                class SubAckReasonCode:
+                    class UNSPECIFIED_ERROR:
+                        value = 1
+                class DEFAULT_CONFIG:
+                    pass
+                class Subscription:
+                    def __init__(self, topic, qos):
+                        pass
+        gmqtt = GmqttImportMock()
 import json
 import asyncio
 import packaging.version as packaging_version
 
 import octobot_commons.enums as commons_enums
 import octobot_commons.errors as commons_errors
-import octobot_commons.constants as commons_constants
 import octobot.community.errors as errors
 import octobot.community.feeds.abstract_feed as abstract_feed
 import octobot.constants as constants

octobot/community/history_backend/clickhouse_historical_backend_client.py

@@ -16,7 +16,25 @@
 import typing
 from datetime import datetime, timezone
 
-import clickhouse_connect.driver
+import octobot_commons.constants as commons_constants
+import octobot_commons.os_util as os_util
+
+try:
+    if os_util.is_raspberry_pi_machine():
+        raise ImportError("clickhouse_connect is not available on Raspberry Pi")
+    else:
+        import clickhouse_connect.driver
+except ImportError:
+    if commons_constants.USE_MINIMAL_LIBS:
+        # mock clickhouse_connect.driver imports
+        class ClickhouseConnectImportMock:
+            async def get_async_client(self, *args):
+                raise ImportError("clickhouse_connect not installed")
+            class driver:
+                class AsyncClient:
+                    async def close(self):
+                        pass
+        clickhouse_connect = ClickhouseConnectImportMock()
 
 import octobot_commons.logging as commons_logging
 import octobot_commons.enums as commons_enums