diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 7886fa62c..7beb8351f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -26,7 +26,7 @@ jobs: architecture: ${{ matrix.arch }} - name: Install dependencies - run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt # - name: Black lint # run: black ${{ secrets.PACKAGE_FOLDER }} --diff --check @@ -70,7 +70,7 @@ jobs: run: exit 1 - name: Install dependencies - run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install wheel && pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt - name: Install tentacles on Unix env: @@ -137,7 +137,7 @@ jobs: architecture: ${{ matrix.arch }} - name: Install dependencies - run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt + run: pip install --prefer-binary -r dev_requirements.txt -r requirements.txt -r full_requirements.txt - name: Build sdist run: python setup.py sdist diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a87ff0bc..ac67ae6d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 *It is strongly advised to perform an update of your tentacles after updating OctoBot. (start.py tentacles --install --all)* +## [2.0.15] - 2025-12-08 +### Breaking pip installation change +To install the full OctoBot (equivalent to previous versions), OctoBot needs to be installed with the [full] parameter: `pip install octobot[full]` +### Added +- [TradingModes] add cancel policies +- [DSL] add DSL and base keywords tentacles +### Updated +- Light installation: OctoBot can now be used with minimal dependencies when started with the USE_MINIMAL_LIBS=true environment variable +- Full installation: to use the full OctoBot (with user interface, etc), install octobot[full] +- [Exchanges] update to ccxt 4.5.22 +- [Hyperliquid] fix markets fetch and use uniform tickers +- Typing: add typing to most OctoBot-Trading objects +- [TradingView] deprecate email alerts +### Fixed +- [RaspberryPi]: Fix "Illegal instruction" crash +- [Exchanges] fix proxy error during markets loading +- [StaggeredOrders] fix rare orders error + ## [2.0.14] - 2025-10-29 ### Fixed - Made pyarrow dependency optionnal to prevent a rare .dll import error diff --git a/Dockerfile b/Dockerfile index 868b2e050..936522b88 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,9 @@ -FROM python:3.10-slim-buster AS base +FROM python:3.10-slim-bookworm AS base WORKDIR / # requires git to install requirements with git+https -# Update to debian archive from https://gist.github.com/ishad0w/6ce1eb569c734880200c47923577426a -RUN echo "deb http://archive.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list \ - && echo "deb http://archive.debian.org/debian-security buster/updates main contrib non-free" >> /etc/apt/sources.list \ - && echo "deb http://archive.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list \ - && apt-get update \ +RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential git gcc binutils libffi-dev libssl-dev libxml2-dev libxslt1-dev libxslt-dev libjpeg62-turbo-dev zlib1g-dev \ && python -m venv /opt/venv @@ -19,10 +15,10 @@ ENV PATH="/opt/venv/bin:$PATH" COPY . . RUN pip install -U setuptools wheel pip>=20.0.0 \ - && pip install --no-cache-dir --prefer-binary -r requirements.txt \ + && pip install --no-cache-dir --prefer-binary -r requirements.txt -r full_requirements.txt \ && python setup.py install -FROM python:3.10-slim-buster +FROM python:3.10-slim-bookworm ARG TENTACLES_URL_TAG="" ENV TENTACLES_URL_TAG=$TENTACLES_URL_TAG @@ -41,11 +37,7 @@ COPY docker/* /octobot/ # 2. Install required packages # 3. Finish env setup SHELL ["/bin/bash", "-o", "pipefail", "-c"] -# Update to debian archive from https://gist.github.com/ishad0w/6ce1eb569c734880200c47923577426a -RUN echo "deb http://archive.debian.org/debian buster main contrib non-free" > /etc/apt/sources.list \ - && echo "deb http://archive.debian.org/debian-security buster/updates main contrib non-free" >> /etc/apt/sources.list \ - && echo "deb http://archive.debian.org/debian buster-backports main contrib non-free" >> /etc/apt/sources.list \ - && apt-get update \ +RUN apt-get update \ && apt-get install -y --no-install-recommends curl libxslt-dev libxcb-xinput0 libjpeg62-turbo-dev zlib1g-dev libblas-dev liblapack-dev libatlas-base-dev libopenjp2-7 libtiff-dev \ && rm -rf /var/lib/apt/lists/* \ && ln -s /opt/venv/bin/OctoBot OctoBot # Make sure we use the virtualenv \ diff --git a/MANIFEST.in b/MANIFEST.in index a7c881c07..8986fd3cc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,5 +4,6 @@ include README.md include LICENSE include CHANGELOG.md include requirements.txt +include full_requirements.txt global-exclude *.c diff --git a/README.md b/README.md index a493d5655..d9241c26f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# OctoBot [2.0.14](https://github.com/Drakkar-Software/OctoBot/blob/master/CHANGELOG.md) +# OctoBot [2.0.15](https://github.com/Drakkar-Software/OctoBot/blob/master/CHANGELOG.md) [![PyPI](https://img.shields.io/pypi/v/OctoBot.svg?logo=pypi)](https://pypi.org/project/OctoBot) [![Downloads](https://pepy.tech/badge/octobot/month)](https://pepy.tech/project/octobot) [![Dockerhub](https://img.shields.io/docker/pulls/drakkarsoftware/octobot.svg?logo=docker)](https://hub.docker.com/r/drakkarsoftware/octobot) diff --git a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py index 5fb6ecb80..63100e0bf 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py @@ -554,16 +554,17 @@ async def test_create_and_cancel_limit_orders(self): async def inner_test_create_and_cancel_limit_orders(self, symbol=None, settlement_currency=None, **kwargs): symbol = symbol or self.SYMBOL + side = trading_enums.TradeOrderSide.BUY # # DEBUG tools p1, uncomment to create specific orders # symbol = "ADA/USDT" # # end debug tools market_status = self.exchange_manager.exchange.get_market_status(symbol) exchange_data = self.get_exchange_data(symbol=symbol) settlement_currency = settlement_currency or self.SETTLEMENT_CURRENCY - price = self.get_order_price(await self.get_price(symbol=symbol), False, symbol=symbol) + price = self.get_order_price(await self.get_price(symbol=symbol), side == trading_enums.TradeOrderSide.SELL, symbol=symbol) # 1. try with "normal" order size default_size = self.get_order_size( - await self.get_portfolio(), price, symbol=symbol, settlement_currency=settlement_currency + await self.get_portfolio(), price, symbol=symbol, settlement_currency=(settlement_currency if side == trading_enums.TradeOrderSide.BUY else self.ORDER_CURRENCY) ) # self.check_order_size_and_price(default_size, price, symbol=symbol, allow_empty_size=self.CHECK_EMPTY_ACCOUNT) enable_min_size_check = False @@ -604,26 +605,27 @@ async def inner_test_create_and_cancel_limit_orders(self, symbol=None, settlemen assert cancelled_orders == [] return try: - buy_limit = await self.create_limit_order(price, size, trading_enums.TradeOrderSide.BUY, symbol=symbol) + limit_order = await self.create_limit_order(price, size, side, symbol=symbol) except trading_errors.AuthenticationError as err: raise trading_errors.AuthenticationError( f"inner_test_create_and_cancel_limit_orders#create_limit_order {err}" ) from err try: - self.check_created_limit_order(buy_limit, price, size, trading_enums.TradeOrderSide.BUY) - assert await self.order_in_open_orders(open_orders, buy_limit, symbol=symbol) - await self.check_can_get_order(buy_limit) + self.check_created_limit_order(limit_order, price, size, side) + assert await self.order_in_open_orders(open_orders, limit_order, symbol=symbol) + await self.check_can_get_order(limit_order) # assert free portfolio amount is smaller than total amount balance = await self.get_portfolio() - assert balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_FREE] < \ - balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL], ( - f"FALSE: {balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_FREE]} < {balance[settlement_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL]}" + locked_currency = settlement_currency if side == trading_enums.TradeOrderSide.BUY else self.ORDER_CURRENCY + assert balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_FREE] < \ + balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL], ( + f"FALSE: {balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_FREE]} < {balance[locked_currency][trading_constants.CONFIG_PORTFOLIO_TOTAL]}" ) finally: # don't leave buy_limit as open order - await self.cancel_order(buy_limit) - assert await self.order_not_in_open_orders(open_orders, buy_limit, symbol=symbol) - assert await self.order_in_cancelled_orders(cancelled_orders, buy_limit, symbol=symbol) + await self.cancel_order(limit_order) + assert await self.order_not_in_open_orders(open_orders, limit_order, symbol=symbol) + assert await self.order_in_cancelled_orders(cancelled_orders, limit_order, symbol=symbol) async def inner_test_cancel_uncancellable_order(self): if self.UNCANCELLABLE_ORDER_ID_SYMBOL_TYPE: @@ -636,39 +638,44 @@ async def test_create_and_fill_market_orders(self): await self.inner_test_create_and_fill_market_orders() async def inner_test_create_and_fill_market_orders(self): + side = trading_enums.TradeOrderSide.BUY portfolio = await self.get_portfolio() current_price = await self.get_price() - price = self.get_order_price(current_price, False, price_diff=0) - size = self.get_order_size(portfolio, price) + order_currency = self.ORDER_CURRENCY if side == trading_enums.TradeOrderSide.SELL else self.SETTLEMENT_CURRENCY + price = self.get_order_price(current_price, side == trading_enums.TradeOrderSide.SELL, price_diff=0) + size = self.get_order_size(portfolio, price, settlement_currency=order_currency) if self.CHECK_EMPTY_ACCOUNT: assert size == trading_constants.ZERO return - buy_market = await self.create_market_order(current_price, size, trading_enums.TradeOrderSide.BUY) + first_market_order = await self.create_market_order(current_price, size, side) post_buy_portfolio = {} try: - self.check_created_market_order(buy_market, size, trading_enums.TradeOrderSide.BUY) - filled_order = await self.wait_for_fill(buy_market) + self.check_created_market_order(first_market_order, size, side) + filled_order = await self.wait_for_fill(first_market_order) parsed_filled_order = personal_data.create_order_instance_from_raw( self.exchange_manager.trader, filled_order ) - self._check_order(parsed_filled_order, size, trading_enums.TradeOrderSide.BUY) + self._check_order(parsed_filled_order, size, side) await self.wait_for_order_exchange_id_in_trades(parsed_filled_order.exchange_order_id) await self.check_require_order_fees_from_trades( filled_order[trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value] ) self.check_raw_closed_orders([filled_order]) post_buy_portfolio = await self.get_portfolio() - self.check_portfolio_changed(portfolio, post_buy_portfolio, False) + portfolio_increased = side == trading_enums.TradeOrderSide.SELL + self.check_portfolio_changed(portfolio, post_buy_portfolio, portfolio_increased) finally: - sell_size = self.get_sell_size_from_buy_order(buy_market) + mirror_size = self.get_sell_size_from_buy_order(first_market_order) # sell: reset portfolio - sell_market = await self.create_market_order(current_price, sell_size, trading_enums.TradeOrderSide.SELL) - self.check_created_market_order(sell_market, sell_size, trading_enums.TradeOrderSide.SELL) - await self.wait_for_fill(sell_market) + other_side = trading_enums.TradeOrderSide.SELL if side == trading_enums.TradeOrderSide.BUY else trading_enums.TradeOrderSide.BUY + second_market_order = await self.create_market_order(current_price, mirror_size, other_side) + self.check_created_market_order(second_market_order, mirror_size, other_side) + await self.wait_for_fill(second_market_order) post_sell_portfolio = await self.get_portfolio() if post_buy_portfolio: - self.check_portfolio_changed(post_buy_portfolio, post_sell_portfolio, True) + portfolio_increased = other_side == trading_enums.TradeOrderSide.SELL + self.check_portfolio_changed(post_buy_portfolio, post_sell_portfolio, portfolio_increased) async def check_require_order_fees_from_trades(self, filled_exchange_order_id, symbol=None): symbol = symbol or self.SYMBOL @@ -771,8 +778,8 @@ async def inner_test_edit_limit_order(self): sell_limit = await self.create_limit_order(price, size, trading_enums.TradeOrderSide.SELL) self.check_created_limit_order(sell_limit, price, size, trading_enums.TradeOrderSide.SELL) assert await self.order_in_open_orders(open_orders, sell_limit) - edited_price = self.get_order_price(current_price, True, price_diff=2*self.ORDER_PRICE_DIFF) - edited_size = self.get_order_size(portfolio, price, order_size=2*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency()) + edited_price = self.get_order_price(current_price, True, price_diff=1.3*self.ORDER_PRICE_DIFF) + edited_size = self.get_order_size(portfolio, price, order_size=1.3*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency()) sell_limit = await self.edit_order(sell_limit, edited_price=edited_price, edited_quantity=edited_size) await self.wait_for_edit(sell_limit, edited_size) sell_limit = await self.get_order(sell_limit.exchange_order_id, sell_limit.symbol) @@ -800,8 +807,8 @@ async def inner_test_edit_stop_order(self): ) self.check_created_stop_order(stop_loss, price, size, trading_enums.TradeOrderSide.SELL) assert await self.order_in_open_orders(open_orders, stop_loss) - edited_price = self.get_order_price(current_price, False, price_diff=2*self.ORDER_PRICE_DIFF) - edited_size = self.get_order_size(portfolio, price, order_size=2*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency()) + edited_price = self.get_order_price(current_price, False, price_diff=1.3*self.ORDER_PRICE_DIFF) + edited_size = self.get_order_size(portfolio, price, order_size=1.3*self.ORDER_SIZE, settlement_currency=self._get_edit_order_settlement_currency()) stop_loss = await self.edit_order(stop_loss, edited_price=edited_price, edited_quantity=edited_size) await self.wait_for_edit(stop_loss, edited_size) stop_loss = await self.get_order(stop_loss.exchange_order_id, stop_loss.symbol) diff --git a/additional_tests/exchanges_tests/test_hyperliquid.py b/additional_tests/exchanges_tests/test_hyperliquid.py index a7a7c1ee0..303aa4984 100644 --- a/additional_tests/exchanges_tests/test_hyperliquid.py +++ b/additional_tests/exchanges_tests/test_hyperliquid.py @@ -26,7 +26,7 @@ class TestHyperliquidAuthenticatedExchange( ): # enter exchange name as a class variable here EXCHANGE_NAME = "hyperliquid" - ORDER_CURRENCY = "HYPE" + ORDER_CURRENCY = "ETH" SETTLEMENT_CURRENCY = "USDC" SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}" ORDER_SIZE = 25 # % of portfolio to include in test orders diff --git a/full_requirements.txt b/full_requirements.txt new file mode 100644 index 000000000..dfe807335 --- /dev/null +++ b/full_requirements.txt @@ -0,0 +1,22 @@ +# Drakkar-Software full requirements +OctoBot-Commons[full]==1.9.91 +OctoBot-Trading[full]==2.4.236 +OctoBot-Evaluators[full]==1.9.9 +OctoBot-Tentacles-Manager[full]==2.9.19 +OctoBot-Services[full]==1.6.30 +OctoBot-Backtesting[full]==1.9.8 + +## Others +colorlog==6.8.0 + +# Community +gmqtt==0.7.0 +pgpy==0.6.0 +clickhouse-connect==0.8.18 +pyiceberg==0.10.0 +pydantic<2.12 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590 +pyarrow==21.0.0 + +# used by ccxt for protobuf "websockets" such as mexc +# lock protobuf to avoid using .rc versions +protobuf==5.29.5 diff --git a/octobot/__init__.py b/octobot/__init__.py index c5fe7f6d2..35476a029 100644 --- a/octobot/__init__.py +++ b/octobot/__init__.py @@ -16,5 +16,5 @@ PROJECT_NAME = "OctoBot" AUTHOR = "Drakkar-Software" -VERSION = "2.0.14" # major.minor.revision +VERSION = "2.0.15" # major.minor.revision LONG_VERSION = f"{VERSION}" diff --git a/octobot/cli.py b/octobot/cli.py index cd04ed76d..00829d9b3 100644 --- a/octobot/cli.py +++ b/octobot/cli.py @@ -21,35 +21,42 @@ import packaging.version as packaging_version -import octobot_commons.os_util as os_util -import octobot_commons.logging as logging -import octobot_commons.configuration as configuration -import octobot_commons.profiles as profiles -import octobot_commons.authentication as authentication -import octobot_commons.constants as common_constants -import octobot_commons.errors as errors - -import octobot_services.api as service_api - -import octobot_tentacles_manager.api as tentacles_manager_api -import octobot_tentacles_manager.cli as tentacles_manager_cli -import octobot_tentacles_manager.constants as tentacles_manager_constants - -# make tentacles importable -sys.path.append(os.path.dirname(sys.executable)) - -import octobot.octobot as octobot_class -import octobot.commands as commands -import octobot.configuration_manager as configuration_manager -import octobot.octobot_backtesting_factory as octobot_backtesting -import octobot.constants as constants -import octobot.enums as enums -import octobot.disclaimer as disclaimer -import octobot.logger as octobot_logger -import octobot.community as octobot_community -import octobot.community.errors -import octobot.limits as limits - +try: + import octobot_commons.os_util as os_util + import octobot_commons.logging as logging + import octobot_commons.configuration as configuration + import octobot_commons.profiles as profiles + import octobot_commons.authentication as authentication + import octobot_commons.constants as common_constants + import octobot_commons.errors as errors + + import octobot_services.api as service_api + + import octobot_tentacles_manager.api as tentacles_manager_api + import octobot_tentacles_manager.cli as tentacles_manager_cli + import octobot_tentacles_manager.constants as tentacles_manager_constants + + # make tentacles importable + sys.path.append(os.path.dirname(sys.executable)) + + import octobot.octobot as octobot_class + import octobot.commands as commands + import octobot.configuration_manager as configuration_manager + import octobot.octobot_backtesting_factory as octobot_backtesting + import octobot.constants as constants + import octobot.enums as enums + import octobot.disclaimer as disclaimer + import octobot.logger as octobot_logger + import octobot.community as octobot_community + import octobot.community.errors + import octobot.limits as limits +except ImportError as err: + print( + "Error importing OctoBot dependencies, please install OctoBot with the [full] option. " + "Example: \"pip install -U octobot[full]\" " + "(Error: {0}: {1})".format(err.__class__.__name__, str(err)), file=sys.stderr + ) + sys.exit(-1) def update_config_with_args(starting_args, config: configuration.Configuration, logger): try: diff --git a/octobot/community/__init__.py b/octobot/community/__init__.py index 783796b8d..29eaaf809 100644 --- a/octobot/community/__init__.py +++ b/octobot/community/__init__.py @@ -24,6 +24,7 @@ ) from octobot.community import models from octobot.community.models import ( + BotLogData, CommunityUserAccount, CommunityFields, CommunityTentaclesPackage, diff --git a/octobot/community/feeds/community_mqtt_feed.py b/octobot/community/feeds/community_mqtt_feed.py index 78662ee70..bbc9670f3 100644 --- a/octobot/community/feeds/community_mqtt_feed.py +++ b/octobot/community/feeds/community_mqtt_feed.py @@ -15,14 +15,37 @@ # License along with OctoBot. If not, see . import logging import typing -import gmqtt +import octobot_commons.constants as commons_constants +try: + import gmqtt +except ImportError: + if commons_constants.USE_MINIMAL_LIBS: + # mock gmqtt imports + class GmqttImportMock: + class Client: + def __init__(self, *args): + raise ImportError("gmqtt not installed") + class Subscription: + def __init__(self, topic, qos): + raise ImportError("gmqtt not installed") + class constants: + class MQTTv311: + pass + class SubAckReasonCode: + class UNSPECIFIED_ERROR: + value = 1 + class DEFAULT_CONFIG: + pass + class Subscription: + def __init__(self, topic, qos): + pass + gmqtt = GmqttImportMock() import json import asyncio import packaging.version as packaging_version import octobot_commons.enums as commons_enums import octobot_commons.errors as commons_errors -import octobot_commons.constants as commons_constants import octobot.community.errors as errors import octobot.community.feeds.abstract_feed as abstract_feed import octobot.constants as constants diff --git a/octobot/community/history_backend/clickhouse_historical_backend_client.py b/octobot/community/history_backend/clickhouse_historical_backend_client.py index d3e65778f..d1f16f49b 100644 --- a/octobot/community/history_backend/clickhouse_historical_backend_client.py +++ b/octobot/community/history_backend/clickhouse_historical_backend_client.py @@ -16,7 +16,25 @@ import typing from datetime import datetime, timezone -import clickhouse_connect.driver +import octobot_commons.constants as commons_constants +import octobot_commons.os_util as os_util + +try: + if os_util.is_raspberry_pi_machine(): + raise ImportError("clickhouse_connect is not available on Raspberry Pi") + else: + import clickhouse_connect.driver +except ImportError: + if commons_constants.USE_MINIMAL_LIBS: + # mock clickhouse_connect.driver imports + class ClickhouseConnectImportMock: + async def get_async_client(self, *args): + raise ImportError("clickhouse_connect not installed") + class driver: + class AsyncClient: + async def close(self): + pass + clickhouse_connect = ClickhouseConnectImportMock() import octobot_commons.logging as commons_logging import octobot_commons.enums as commons_enums diff --git a/octobot/community/history_backend/iceberg_historical_backend_client.py b/octobot/community/history_backend/iceberg_historical_backend_client.py index 24741f0ae..6ba1f8406 100644 --- a/octobot/community/history_backend/iceberg_historical_backend_client.py +++ b/octobot/community/history_backend/iceberg_historical_backend_client.py @@ -25,9 +25,13 @@ import dataclasses import octobot_commons.logging as commons_logging +import octobot_commons.os_util as os_util try: - import pyarrow + if os_util.is_raspberry_pi_machine(): + raise ImportError("pyarrow is not available on Raspberry Pi") + else: + import pyarrow except ImportError as err: commons_logging.get_logger().info(f"Skipped pyarrow import: {err}") class PyArrowMock(): @@ -36,16 +40,60 @@ class PyArrowMock(): Schema = None pyarrow = PyArrowMock() -import pyiceberg.catalog -import pyiceberg.schema -import pyiceberg.types -import pyiceberg.exceptions -import pyiceberg.expressions -import pyiceberg.table -import pyiceberg.table.sorting -import pyiceberg.table.statistics -import pyiceberg.table.update -import pyiceberg.table.refs +try: + import pyiceberg.catalog + import pyiceberg.schema + import pyiceberg.types + import pyiceberg.exceptions + import pyiceberg.expressions + import pyiceberg.table + import pyiceberg.table.sorting + import pyiceberg.table.statistics + import pyiceberg.table.update + import pyiceberg.table.refs +except ImportError as err: + commons_logging.get_logger().info(f"Skipped pyiceberg import: {err}") + class PyIcebergImportMock(): + # type hints mocks only + class catalog: + class Catalog: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + def load_catalog(self, *args, **kwargs): + raise ImportError("pyiceberg not installed") + class table: + class statistics: + class StatisticsFile: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class BlobMetadata: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class Table: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class DataScan: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class exceptions: + class NoSuchTableError(Exception): + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class schema: + class Schema: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class expressions: + class BooleanExpression: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class Or: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + class EqualTo: + def __init__(self, *args): + raise ImportError("pyiceberg not installed") + pyiceberg = PyIcebergImportMock() import octobot_commons.enums as commons_enums import octobot.constants as constants diff --git a/octobot/community/models/__init__.py b/octobot/community/models/__init__.py index fa56e3546..c70965216 100644 --- a/octobot/community/models/__init__.py +++ b/octobot/community/models/__init__.py @@ -18,6 +18,10 @@ from octobot.community.models.community_user_account import ( CommunityUserAccount, ) +from octobot.community.models import bot_log +from octobot.community.models.bot_log import ( + BotLogData, +) from octobot.community.models import community_fields from octobot.community.models.community_fields import ( CommunityFields, @@ -74,6 +78,7 @@ __all__ = [ "CommunityUserAccount", "CommunityFields", + "BotLogData", "CommunityTentaclesPackage", "CommunitySupports", "CommunityDonation", diff --git a/octobot/community/models/bot_log.py b/octobot/community/models/bot_log.py new file mode 100644 index 000000000..b3e890312 --- /dev/null +++ b/octobot/community/models/bot_log.py @@ -0,0 +1,26 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . +import dataclasses +import typing + +import octobot_commons.dataclasses +import octobot.community.supabase_backend.enums as supabase_enums + + +@dataclasses.dataclass +class BotLogData(octobot_commons.dataclasses.FlexibleDataclass): + log_type: supabase_enums.BotLogType + content: typing.Optional[dict] diff --git a/octobot/community/supabase_backend/enums.py b/octobot/community/supabase_backend/enums.py index 5fce4c0c6..fd15dc7fb 100644 --- a/octobot/community/supabase_backend/enums.py +++ b/octobot/community/supabase_backend/enums.py @@ -128,8 +128,9 @@ class BotLogType(enum.Enum): CLOSED_POSITION = "closed_position" UNSUPPORTED_HEDGE_POSITION = "unsupported_hedge_position" NOTHING_TO_DO = "nothing_to_do" - BOT_RESTARTED = "bot_restarted" + BOT_STARTED = "bot_started" BOT_STOPPED = "bot_stopped" + BOT_RESTARTED = "bot_restarted" MISSING_MINIMAL_FUNDS = "missing_minimal_funds" CHANGED_PRODUCT = "changed_product" IMPOSSIBLE_TO_CREATE_ALL_REQUIRED_ORDERS = "impossible_to_create_all_required_orders" diff --git a/octobot/logger.py b/octobot/logger.py index d3a3c1246..a30f578f7 100644 --- a/octobot/logger.py +++ b/octobot/logger.py @@ -34,6 +34,7 @@ import octobot_trading.exchange_channel as exchanges_channel import octobot_trading.enums as trading_enums import octobot_trading.api as trading_api +import octobot_trading.personal_data as personal_data import octobot.constants as constants import octobot.configuration_manager as configuration_manager @@ -98,8 +99,9 @@ def _load_logger_config(): common_logging.set_global_logger_level(constants.FORCED_LOG_LEVEL) except Exception as ex: config.fileConfig(constants.LOGGING_CONFIG_FILE) - logging.getLogger("Logging Configuration").warning(f"Impossible to initialize local logging configuration file," - f" using default one. {ex}") + logging.getLogger("Logging Configuration").warning( + f"Impossible to initialize local logging configuration file, using default one. {ex}" + ) async def init_exchange_chan_logger(exchange_id): @@ -293,6 +295,14 @@ def _filter_balance(balance: dict): } removed_count = len(balance) - len(filtered_balance) return trading_api.parse_decimal_portfolio(filtered_balance, False), removed_count + elif isinstance(first_value, personal_data.Asset): + filtered_balance = { + key: values + for key, values in balance.items() + if values.total + } + removed_count = len(balance) - len(filtered_balance) + return filtered_balance, removed_count return balance, 0 diff --git a/requirements.txt b/requirements.txt index 5d5501efe..493ca96b2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,15 +1,14 @@ # Drakkar-Software requirements -OctoBot-Commons==1.9.86 -OctoBot-Trading==2.4.231 -OctoBot-Evaluators==1.9.7 -OctoBot-Tentacles-Manager==2.9.18 -OctoBot-Services==1.6.26 -OctoBot-Backtesting==1.9.7 +OctoBot-Commons==1.9.91 +OctoBot-Trading==2.4.236 +OctoBot-Evaluators==1.9.9 +OctoBot-Tentacles-Manager==2.9.19 +OctoBot-Services==1.6.30 +OctoBot-Backtesting==1.9.8 Async-Channel==2.2.1 trading-backend==1.2.42 ## Others -colorlog==6.8.0 requests==2.32.5 urllib3 # required by requests, used in imports: make sure it's always available packaging==25.0 @@ -18,13 +17,7 @@ setuptools==79.0.1 # warning: setuptools>=80 breaks easy_install, need to find # see https://community.palantir.com/t/important-update-on-setuptools-pinning-the-version-below-80-0-0/3872 # Community -websockets==15.0.1 # used by supabase, a recent version is required, see https://github.com/supabase/realtime-py/blob/main/pyproject.toml -gmqtt==0.7.0 pgpy==0.6.0 -clickhouse-connect==0.8.18 -pyiceberg==0.10.0 -pydantic<2.12 # required for pyiceberg 0.10.0 https://github.com/apache/iceberg-python/issues/2590 -pyarrow==21.0.0 # Error tracking sentry-sdk==2.35.0 # always make sure sentry_aiohttp_transport.py keep working @@ -33,13 +26,10 @@ sentry-sdk==2.35.0 # always make sure sentry_aiohttp_transport.py keep working supabase==2.18.1 # Supabase client supabase_auth # Supabase authenticated API (required by supabase and enforced to allow direct import) postgrest # Supabase posgres calls (required by supabase and enforced to allow direct import) +websockets==15.0.1 # used by supabase, a recent version is required, see https://github.com/supabase/realtime-py/blob/main/pyproject.toml # async http requests aiohttp==3.12.15 # updating to aiodns==3.2.0 is incompatible (and failing CI) # raises RuntimeError: aiodns needs a SelectorEventLoop on Windows. See more: https://github.com/saghul/aiodns/issues/86 aiodns==3.1.1 # used by aiohttp - -# used by ccxt for protobuf "websockets" such as mexc -# lock protobuf to avoid using .rc versions -protobuf==5.29.5 diff --git a/setup.py b/setup.py index 5c81de1fd..38fbfefa2 100644 --- a/setup.py +++ b/setup.py @@ -13,11 +13,28 @@ # # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . +import sys +import os from setuptools import find_packages from setuptools import setup from octobot import PROJECT_NAME, AUTHOR, VERSION -PACKAGES = find_packages(exclude=["tentacles*", "tests", ]) +is_building_wheel = "bdist_wheel" in sys.argv + +EXCLUDED_PACKAGES = ["tentacles*", "tests"] +DATA_FILES = ["config/*", "strategy_optimizer/optimizer_data_files/*"] +if is_building_wheel and bool(os.getenv("USE_MINIMAL_LIBS", "false").lower() == "true"): + # exclude data files when building a wheel with minimal libs + DATA_FILES = [] +PACKAGES = [ + pkg for pkg in find_packages(exclude=EXCLUDED_PACKAGES) + if not any( + # manually excluded packages as they can't be excluded from a wheel otherwise + # because those are valid python packages + str(pkg).startswith(excluded_package) + for excluded_package in EXCLUDED_PACKAGES + ) +] # long description from README file with open('README.md', encoding='utf-8') as f: @@ -29,6 +46,7 @@ def ignore_git_requirements(requirements): REQUIRED = ignore_git_requirements(open('requirements.txt').readlines()) +FULL_REQUIRED = ignore_git_requirements(open('full_requirements.txt').readlines()) REQUIRES_PYTHON = '>=3.10' setup( @@ -42,7 +60,7 @@ def ignore_git_requirements(requirements): py_modules=['start'], packages=PACKAGES, package_data={ - "": ["config/*", "strategy_optimizer/optimizer_data_files/*"], + "": DATA_FILES, }, long_description=DESCRIPTION, long_description_content_type='text/markdown', @@ -50,6 +68,9 @@ def ignore_git_requirements(requirements): test_suite="tests", zip_safe=False, install_requires=REQUIRED, + extras_require={ + 'full': FULL_REQUIRED, + }, python_requires=REQUIRES_PYTHON, entry_points={ 'console_scripts': [ diff --git a/standard.rc b/standard.rc index 9799eadf4..96a7ee5aa 100644 --- a/standard.rc +++ b/standard.rc @@ -36,10 +36,6 @@ load-plugins= # Pickle collected data for later comparisons. persistent=yes -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - # Allow loading of arbitrary C extensions. Extensions are imported into the # active Python interpreter and may run arbitrary code. unsafe-load-any-extension=no