From feda7b10b73b3188b42ccf957ad030062eb1cc6b Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Mon, 10 Aug 2026 16:40:42 +0200 Subject: [PATCH 01/10] [Node] add global view account refresher --- packages/flow/octobot_flow/constants.py | 2 + .../flow/octobot_flow/entities/__init__.py | 10 + .../entities/global_view/__init__.py | 11 + .../exchange_account_refresh_result.py | 17 + .../global_view_account_context.py | 16 + .../global_view_account_refresh_result.py | 16 + .../global_view_refreshed_elements.py | 60 +++ packages/flow/octobot_flow/errors.py | 4 + packages/flow/octobot_flow/jobs/__init__.py | 2 + .../jobs/global_view_account_job.py | 88 ++++ .../octobot_flow/logic/accounts/__init__.py | 15 + .../accounts/account_state_persistence.py | 115 ++++++ .../logic/accounts/portfolio_history.py | 34 ++ .../logic/configuration/__init__.py | 4 + .../configuration/profile_data_factory.py | 36 ++ .../logic/exchange/orders/__init__.py | 13 + .../exchange/orders/order_change_detection.py | 54 +++ .../logic/exchange/portfolio/__init__.py | 5 + .../exchange/portfolio/valuation_unit.py | 14 + .../logic/exchange/simulator/__init__.py | 2 + .../simulator/simulated_portfolio_seeder.py | 19 + .../logic/global_view/__init__.py | 2 + .../global_view/account_refresh_builder.py | 46 +++ .../global_view/exchange_account_refresh.py | 111 +++++ .../global_view/global_view_persistence.py | 28 ++ .../test_global_view_account_refresh.py | 230 +++++++++++ .../test_account_state_persistence.py | 116 ++++++ .../logic/accounts/test_portfolio_history.py | 70 ++++ .../test_profile_data_factory.py | 74 ++++ .../orders/test_order_change_detection.py | 33 ++ .../exchange/portfolio/test_valuation_unit.py | 25 ++ .../test_simulated_portfolio_seeder.py | 80 ++++ .../test_account_refresh_builder.py | 93 +++++ .../test_global_view_account_job.py | 263 ++++++++++++ .../test_global_view_refreshed_elements.py | 47 +++ packages/node/octobot_node/constants.py | 7 + packages/node/octobot_node/enums.py | 1 + .../octobot_node/protocol/accounts_history.py | 71 ++++ .../scheduler/global_view/__init__.py | 2 + .../global_view/automation_trigger.py | 101 +++++ .../global_view/global_view_executor.py | 56 +++ .../node/octobot_node/scheduler/scheduler.py | 6 + .../node/octobot_node/scheduler/schedules.py | 2 + .../scheduler/workflows/__init__.py | 1 + .../workflows/global_view_workflow.py | 145 +++++++ .../test_global_view_workflow.py | 141 +++++++ .../util/global_view_workflow.py | 384 ++++++++++++++++++ packages/node/tests/scheduler/__init__.py | 2 + .../global_view/test_automation_trigger.py | 66 +++ .../global_view/test_global_view_workflow.py | 47 +++ packages/sync/octobot_sync/constants.py | 1 + .../base_local_collection_storage.py | 10 + .../sync/collection_providers/__init__.py | 3 + .../user_account_history_provider.py | 38 ++ .../user_account_provider.py | 3 + .../test_account_history_provider.py | 147 +++++++ 56 files changed, 2989 insertions(+) create mode 100644 packages/flow/octobot_flow/entities/global_view/__init__.py create mode 100644 packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py create mode 100644 packages/flow/octobot_flow/entities/global_view/global_view_account_context.py create mode 100644 packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py create mode 100644 packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py create mode 100644 packages/flow/octobot_flow/jobs/global_view_account_job.py create mode 100644 packages/flow/octobot_flow/logic/accounts/__init__.py create mode 100644 packages/flow/octobot_flow/logic/accounts/account_state_persistence.py create mode 100644 packages/flow/octobot_flow/logic/accounts/portfolio_history.py create mode 100644 packages/flow/octobot_flow/logic/exchange/orders/__init__.py create mode 100644 packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py create mode 100644 packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py create mode 100644 packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py create mode 100644 packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py create mode 100644 packages/flow/octobot_flow/logic/global_view/__init__.py create mode 100644 packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py create mode 100644 packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py create mode 100644 packages/flow/octobot_flow/logic/global_view/global_view_persistence.py create mode 100644 packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py create mode 100644 packages/flow/tests/logic/accounts/test_account_state_persistence.py create mode 100644 packages/flow/tests/logic/accounts/test_portfolio_history.py create mode 100644 packages/flow/tests/logic/exchange/orders/test_order_change_detection.py create mode 100644 packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py create mode 100644 packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py create mode 100644 packages/flow/tests/logic/global_view/test_account_refresh_builder.py create mode 100644 packages/flow/tests/logic/global_view/test_global_view_account_job.py create mode 100644 packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py create mode 100644 packages/node/octobot_node/protocol/accounts_history.py create mode 100644 packages/node/octobot_node/scheduler/global_view/__init__.py create mode 100644 packages/node/octobot_node/scheduler/global_view/automation_trigger.py create mode 100644 packages/node/octobot_node/scheduler/global_view/global_view_executor.py create mode 100644 packages/node/octobot_node/scheduler/workflows/global_view_workflow.py create mode 100644 packages/node/tests/functional_tests/test_global_view_workflow.py create mode 100644 packages/node/tests/functional_tests/util/global_view_workflow.py create mode 100644 packages/node/tests/scheduler/global_view/test_automation_trigger.py create mode 100644 packages/node/tests/scheduler/global_view/test_global_view_workflow.py create mode 100644 packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py create mode 100644 packages/sync/tests/sync/collection_providers/test_account_history_provider.py diff --git a/packages/flow/octobot_flow/constants.py b/packages/flow/octobot_flow/constants.py index a88c766704..8245bebb7e 100644 --- a/packages/flow/octobot_flow/constants.py +++ b/packages/flow/octobot_flow/constants.py @@ -14,3 +14,5 @@ # Copy-trading mirrored open-order grace (aligned with octobot_copy fill timeout by default) DEFAULT_COPY_TRADING_ORPHAN_CANCEL_GRACE_SECONDS = float(copy_constants.FILL_ORDER_TIMEOUT) DEFAULT_COPY_TRADING_ORPHAN_GRACE_ABORT_THRESHOLD = 2 + +PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS = 12 * 60 * 60 diff --git a/packages/flow/octobot_flow/entities/__init__.py b/packages/flow/octobot_flow/entities/__init__.py index 372ebf2d53..8efb6ffd74 100644 --- a/packages/flow/octobot_flow/entities/__init__.py +++ b/packages/flow/octobot_flow/entities/__init__.py @@ -35,6 +35,12 @@ UserAuthentication, TradingSignal, ) +from octobot_flow.entities.global_view import ( + GlobalViewAccountContext, + ExchangeAccountRefreshResult, + GlobalViewAccountRefreshResult, + GlobalViewRefreshedElements, +) __all__ = [ "AccountElements", "ExchangeAccountElements", @@ -65,4 +71,8 @@ "AdditionalActions", "UserAuthentication", "TradingSignal", + "GlobalViewAccountContext", + "ExchangeAccountRefreshResult", + "GlobalViewAccountRefreshResult", + "GlobalViewRefreshedElements", ] diff --git a/packages/flow/octobot_flow/entities/global_view/__init__.py b/packages/flow/octobot_flow/entities/global_view/__init__.py new file mode 100644 index 0000000000..44ef507b04 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/__init__.py @@ -0,0 +1,11 @@ +from octobot_flow.entities.global_view.global_view_account_context import GlobalViewAccountContext +from octobot_flow.entities.global_view.exchange_account_refresh_result import ExchangeAccountRefreshResult +from octobot_flow.entities.global_view.global_view_account_refresh_result import GlobalViewAccountRefreshResult +from octobot_flow.entities.global_view.global_view_refreshed_elements import GlobalViewRefreshedElements + +__all__ = [ + "GlobalViewAccountContext", + "ExchangeAccountRefreshResult", + "GlobalViewAccountRefreshResult", + "GlobalViewRefreshedElements", +] diff --git a/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py new file mode 100644 index 0000000000..7a835e548d --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py @@ -0,0 +1,17 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models + + +@dataclasses.dataclass +class ExchangeAccountRefreshResult: + assets: list[protocol_models.DetailedAssetsForTradingType] + portfolio_snapshot: protocol_models.PortfolioHistoricalValue + valuation_unit: str + open_orders: list[dict] + trades: list[dict] + positions: list[dict] + changed_order_ids: set[str] diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py new file mode 100644 index 0000000000..c02fcaef91 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py @@ -0,0 +1,16 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + + +@dataclasses.dataclass +class GlobalViewAccountContext: + account: protocol_models.Account + exchange_account: protocol_models.ExchangeAccount + exchange_config: protocol_models.ExchangeConfig + trading_type: protocol_models.TradingType + auth_details: exchange_data_module.ExchangeAuthDetails diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py new file mode 100644 index 0000000000..279762ecf3 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py @@ -0,0 +1,16 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models + + +@dataclasses.dataclass +class GlobalViewAccountRefreshResult: + updated_account: protocol_models.Account + changed_order_ids: set[str] + open_orders: list[dict] | None = None + trades: list[dict] | None = None + positions: list[dict] | None = None + portfolio_history_state: protocol_models.PortfolioHistoricalValuesState | None = None diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py b/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py new file mode 100644 index 0000000000..9ace068dd7 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py @@ -0,0 +1,60 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import copy + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_import + + +class GlobalViewRefreshedElements: + def __init__(self, exchange_data: exchange_data_import.ExchangeData, save_copy: bool = True): + self.open_orders: list[dict] = ( + copy.deepcopy(exchange_data.orders_details.open_orders) if save_copy else list( + exchange_data.orders_details.open_orders or [] + ) + ) if exchange_data.orders_details.open_orders else [] + self.portfolio: dict[str, dict] = ( + copy.deepcopy(exchange_data.portfolio_details.content) if save_copy else dict( + exchange_data.portfolio_details.content or {} + ) + ) if exchange_data.portfolio_details else {} + + def confirmed_change( + self, exchange_data: exchange_data_import.ExchangeData + ) -> bool: + return self._confirmed_change(GlobalViewRefreshedElements(exchange_data, save_copy=False)) + + def _confirmed_change( + self, other_global_view_refreshed_elements: "GlobalViewRefreshedElements" + ) -> bool: + if self._get_orders_signature(self.open_orders) != self._get_orders_signature( + other_global_view_refreshed_elements.open_orders + ): + return True + if self._get_portfolio_signature(self.portfolio) != self._get_portfolio_signature( + other_global_view_refreshed_elements.portfolio + ): + return True + return False + + def _get_orders_signature(self, orders: list[dict]) -> str: + return ",".join(sorted([ + self._get_order_signature(order) for order in orders + ])) + + def _get_order_signature(self, order: dict) -> str: + try: + origin_value = order[trading_constants.STORAGE_ORIGIN_VALUE] + except KeyError: + return "" + return ( + f"{origin_value.get(trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value, '')}" + f"_{origin_value.get(trading_enums.ExchangeConstantsOrderColumns.FILLED.value, 0)}" + ) + + def _get_portfolio_signature(self, portfolio: dict[str, dict]) -> str: + return ",".join(sorted([ + f"{asset}:{value}" for asset, value in portfolio.items() + ])) diff --git a/packages/flow/octobot_flow/errors.py b/packages/flow/octobot_flow/errors.py index 708d93f470..a0c0bb17c4 100644 --- a/packages/flow/octobot_flow/errors.py +++ b/packages/flow/octobot_flow/errors.py @@ -98,3 +98,7 @@ class PriorityActionError(AutomationActionError): class PendingPriorityActionsSkippedError(PriorityActionError): """raise when supplied priority actions were not executed and DAG actions would run instead""" + + +class GlobalViewUnsupportedAccountError(ConfigurationError): + """raise when global view refresh cannot run for an account type""" diff --git a/packages/flow/octobot_flow/jobs/__init__.py b/packages/flow/octobot_flow/jobs/__init__.py index c30109c891..14002b53e0 100644 --- a/packages/flow/octobot_flow/jobs/__init__.py +++ b/packages/flow/octobot_flow/jobs/__init__.py @@ -1,9 +1,11 @@ from octobot_flow.jobs.automation_job import AutomationJob from octobot_flow.jobs.automation_runner_job import AutomationRunnerJob from octobot_flow.jobs.exchange_account_job import ExchangeAccountJob +from octobot_flow.jobs.global_view_account_job import GlobalViewAccountJob __all__ = [ "AutomationJob", "AutomationRunnerJob", "ExchangeAccountJob", + "GlobalViewAccountJob", ] diff --git a/packages/flow/octobot_flow/jobs/global_view_account_job.py b/packages/flow/octobot_flow/jobs/global_view_account_job.py new file mode 100644 index 0000000000..8c44255193 --- /dev/null +++ b/packages/flow/octobot_flow/jobs/global_view_account_job.py @@ -0,0 +1,88 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities +import octobot_flow.errors +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory_module +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module +import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module + + +class GlobalViewAccountJob: + def __init__(self, user_id: str, context: octobot_flow.entities.GlobalViewAccountContext): + self.user_id = user_id + self.context = context + + async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account = self.context.account + account_specifics = account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + "Account.specifics.actual_instance is required for global view refresh." + ) + account_specifics_instance = account_specifics.actual_instance + if isinstance(account_specifics_instance, protocol_models.GenericAccount): + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + ) + if isinstance(account_specifics_instance, protocol_models.BlockchainAccount): + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + "Blockchain accounts are not supported yet." + ) + if not isinstance(account_specifics_instance, protocol_models.ExchangeAccount): + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + f"Unsupported account specifics type: {type(account_specifics_instance).__name__}." + ) + + previous_open_order_exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + self.user_id, + account.id, + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + self.context.exchange_account, + self.context.exchange_config, + self.context.trading_type, + is_simulated=account.is_simulated, + ) + exchange_data = exchange_data_module.exchange_data_factory( + exchange_internal_name=self.context.exchange_config.exchange, + exchange_type=profile_data_factory_module.exchange_type_from_trading_type(self.context.trading_type), + sandboxed=self.context.exchange_config.sandboxed, + auth_details=self.context.auth_details, + ) + tentacles_setup_config = tentacles_manager_api.get_full_tentacles_setup_config() + async with trading_exchanges.exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ) as exchange_manager: + if account.is_simulated: + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + self.context.trading_type, + previous_open_order_exchange_ids, + ) + + refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( + self.user_id, + self.context, + exchange_refresh_result, + ) + global_view_persistence_module.persist_global_view_refresh_result( + self.user_id, + account.id, + refresh_result, + ) + return refresh_result diff --git a/packages/flow/octobot_flow/logic/accounts/__init__.py b/packages/flow/octobot_flow/logic/accounts/__init__.py new file mode 100644 index 0000000000..f1d9c11add --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/__init__.py @@ -0,0 +1,15 @@ +from octobot_flow.logic.accounts.portfolio_history import merge_snapshot +from octobot_flow.logic.accounts.account_state_persistence import ( + build_portfolio_history_state, + load_portfolio_history_state, + load_previous_open_order_exchange_ids, + persist_account_trading, +) + +__all__ = [ + "merge_snapshot", + "build_portfolio_history_state", + "load_portfolio_history_state", + "load_previous_open_order_exchange_ids", + "persist_account_trading", +] diff --git a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py new file mode 100644 index 0000000000..0d9f5efb12 --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py @@ -0,0 +1,115 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import datetime + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.personal_data.orders.protocol as orders_protocol +import octobot_trading.personal_data.positions.protocol as positions_protocol +import octobot_trading.personal_data.trades.protocol as trades_protocol +import octobot_trading.personal_data.trades.trades_util as trades_util + +import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module + + +def load_previous_open_order_exchange_ids(user_id: str, account_id: str) -> set[str]: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + return set() + except Exception: + return set() + account_trading = trading_state.account_trading + if account_trading is None or account_trading.orders is None: + return set() + return order_change_detection_module.open_order_exchange_ids_from_protocol_orders( + account_trading.orders + ) + + +def load_portfolio_history_state( + user_id: str, + account_id: str, +) -> protocol_models.PortfolioHistoricalValuesState: + try: + return collection_providers.AccountHistoryProvider.instance().load_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def build_portfolio_history_state( + user_id: str, + account_id: str, + snapshot: protocol_models.PortfolioHistoricalValue, + valuation_unit: str, + evaluation_time: datetime.datetime, +) -> protocol_models.PortfolioHistoricalValuesState: + history_state = load_portfolio_history_state(user_id, account_id) + existing_values = ( + history_state.history.values + if history_state.history is not None and history_state.history.values + else [] + ) + merged_values = portfolio_history_module.merge_snapshot( + existing_values, + snapshot, + evaluation_time, + ) + updated_history = protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=merged_values, + ) + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=updated_history, + ) + + +def persist_account_trading( + user_id: str, + account_id: str, + orders: list[dict], + trades: list[dict], + positions: list[dict], +) -> None: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ) + account_trading = trading_state.account_trading + account_trading.orders = [orders_protocol.to_protocol_order(order) for order in orders] or None + account_trading.positions = [ + positions_protocol.to_protocol_position(position) for position in positions + ] or None + existing_trade_dicts = [ + trades_protocol.exchange_columns_dict_from_protocol_trade(protocol_trade) + for protocol_trade in (account_trading.trades or []) + ] + merged_trade_dicts = trades_util.merge_trades_deduped(existing_trade_dicts, trades) + account_trading.trades = [ + trades_protocol.to_protocol_trade(trade_dict) for trade_dict in merged_trade_dicts + ] or None + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + user_id, + account_id, + trading_state, + ) diff --git a/packages/flow/octobot_flow/logic/accounts/portfolio_history.py b/packages/flow/octobot_flow/logic/accounts/portfolio_history.py new file mode 100644 index 0000000000..549fbe6e34 --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/portfolio_history.py @@ -0,0 +1,34 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import datetime + +import octobot_flow.constants as flow_constants +import octobot_protocol.models as protocol_models + + +def merge_snapshot( + existing_values: list[protocol_models.PortfolioHistoricalValue], + new_snapshot: protocol_models.PortfolioHistoricalValue, + evaluation_time: datetime.datetime, +) -> list[protocol_models.PortfolioHistoricalValue]: + """ + Keep 12h-spaced snapshots plus a always-updated latest entry. + Persisted shape: twelve_hour_snapshots + [latest]. + """ + if not existing_values: + return [new_snapshot] + + twelve_hour_snapshots = ( + [existing_values[0]] + if len(existing_values) == 1 + else list(existing_values[:-1]) + ) + interval_seconds = flow_constants.PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS + last_twelve_hour_snapshot = twelve_hour_snapshots[-1] + elapsed_since_last_twelve_hour_snapshot = ( + evaluation_time - last_twelve_hour_snapshot.timestamp + ).total_seconds() + if elapsed_since_last_twelve_hour_snapshot >= interval_seconds: + twelve_hour_snapshots.append(new_snapshot) + return twelve_hour_snapshots + [new_snapshot] diff --git a/packages/flow/octobot_flow/logic/configuration/__init__.py b/packages/flow/octobot_flow/logic/configuration/__init__.py index 3d5f5bd80f..a2dec066fc 100644 --- a/packages/flow/octobot_flow/logic/configuration/__init__.py +++ b/packages/flow/octobot_flow/logic/configuration/__init__.py @@ -3,6 +3,8 @@ from octobot_flow.logic.configuration.profile_data_factory import ( create_profile_data, infer_reference_market, + profile_data_for_account, + exchange_type_from_trading_type, ) __all__ = [ @@ -10,4 +12,6 @@ "AutomationConfigurationUpdater", "create_profile_data", "infer_reference_market", + "profile_data_for_account", + "exchange_type_from_trading_type", ] diff --git a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py index 89bbdadb46..ddff947ae0 100644 --- a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py +++ b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py @@ -2,6 +2,7 @@ import octobot_commons.profiles.profile_data as profile_data_import import octobot_commons.constants +import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums import octobot_flow.entities @@ -9,6 +10,41 @@ import tentacles.Meta.Keywords.scripting_library as scripting_library +_TRADING_TYPE_TO_EXCHANGE_TYPE: dict[protocol_models.TradingType, trading_enums.ExchangeTypes] = { + protocol_models.TradingType.SPOT: trading_enums.ExchangeTypes.SPOT, + protocol_models.TradingType.FUTURES: trading_enums.ExchangeTypes.FUTURE, + protocol_models.TradingType.OPTIONS: trading_enums.ExchangeTypes.OPTION, + protocol_models.TradingType.MARGIN: trading_enums.ExchangeTypes.MARGIN, +} + + +def exchange_type_from_trading_type(trading_type: protocol_models.TradingType) -> str: + return _TRADING_TYPE_TO_EXCHANGE_TYPE[trading_type].value + + +def profile_data_for_account( + account: protocol_models.Account, + exchange_account: protocol_models.ExchangeAccount, + exchange_config: protocol_models.ExchangeConfig, + trading_type: protocol_models.TradingType, + *, + is_simulated: bool, +) -> profile_data_import.ProfileData: + profile_data = profile_data_import.ProfileData( + exchanges=[ + profile_data_import.ExchangeData( + internal_name=exchange_config.exchange, + exchange_type=exchange_type_from_trading_type(trading_type), + exchange_account_id=exchange_account.remote_account_id or account.id, + sandboxed=exchange_config.sandboxed, + ) + ] + ) + profile_data.trader.enabled = not is_simulated + profile_data.trader_simulator.enabled = is_simulated + return profile_data + + def _tentacles_for_exchange_account_details( exchange_account_details: typing.Optional[octobot_flow.entities.ExchangeAccountDetails], ) -> list[profile_data_import.TentaclesData]: diff --git a/packages/flow/octobot_flow/logic/exchange/orders/__init__.py b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py new file mode 100644 index 0000000000..4831d16378 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py @@ -0,0 +1,13 @@ +from octobot_flow.logic.exchange.orders.order_change_detection import ( + detect_changed_order_ids, + open_order_exchange_ids_from_open_orders, + open_order_exchange_ids_from_protocol_orders, + open_order_to_storage_dict, +) + +__all__ = [ + "detect_changed_order_ids", + "open_order_exchange_ids_from_open_orders", + "open_order_exchange_ids_from_protocol_orders", + "open_order_to_storage_dict", +] diff --git a/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py new file mode 100644 index 0000000000..863c3b8d17 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py @@ -0,0 +1,54 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + + +def detect_changed_order_ids( + previous_open_order_exchange_ids: set[str], + current_open_orders: list, +) -> set[str]: + if not previous_open_order_exchange_ids: + return set() + current_open_order_exchange_ids = open_order_exchange_ids_from_open_orders(current_open_orders) + return previous_open_order_exchange_ids - current_open_order_exchange_ids + + +def open_order_exchange_ids_from_protocol_orders( + protocol_orders: list[protocol_models.Order] | None, +) -> set[str]: + if not protocol_orders: + return set() + return { + str(protocol_order.exchange_id) + for protocol_order in protocol_orders + if protocol_order.exchange_id + } + + +def open_order_exchange_ids_from_open_orders(open_orders: list) -> set[str]: + exchange_ids: set[str] = set() + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_order in open_orders: + order_dict = open_order_to_storage_dict(open_order) + inner_order = order_dict.get(trading_constants.STORAGE_ORIGIN_VALUE, order_dict) + if not isinstance(inner_order, dict): + inner_order = order_dict + exchange_id = inner_order.get(order_columns.EXCHANGE_ID.value) or inner_order.get( + order_columns.ID.value + ) + if exchange_id is not None: + exchange_ids.add(str(exchange_id)) + return exchange_ids + + +def open_order_to_storage_dict(open_order) -> dict: + if isinstance(open_order, dict): + return open_order + if hasattr(open_order, "to_dict"): + return open_order.to_dict() + if hasattr(open_order, "order"): + return open_order.order + raise TypeError(f"Unsupported open order type: {type(open_order).__name__}") diff --git a/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py b/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py new file mode 100644 index 0000000000..b3e746f350 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py @@ -0,0 +1,5 @@ +from octobot_flow.logic.exchange.portfolio.valuation_unit import resolve_portfolio_valuation_unit + +__all__ = [ + "resolve_portfolio_valuation_unit", +] diff --git a/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py b/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py new file mode 100644 index 0000000000..5ece9976e9 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py @@ -0,0 +1,14 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.constants as commons_constants +import octobot_trading.enums as trading_enums + + +def resolve_portfolio_valuation_unit(exchange_manager) -> str: + quote_currency = exchange_manager.exchange.get_option_value( + trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY + ) + if quote_currency: + return str(quote_currency) + return commons_constants.DEFAULT_REFERENCE_MARKET diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py b/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py index 70957ed759..d51b37ff4b 100644 --- a/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py +++ b/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py @@ -1,7 +1,9 @@ from octobot_flow.logic.exchange.simulator.simulated_exchange_account_resolver import SimulatedExchangeAccountResolver from octobot_flow.logic.exchange.simulator.simulated_price_events_factory import SimulatedPriceEventsFactory +from octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder import seed_simulated_portfolio __all__ = [ "SimulatedExchangeAccountResolver", "SimulatedPriceEventsFactory", + "seed_simulated_portfolio", ] diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py new file mode 100644 index 0000000000..7c24db10d6 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py @@ -0,0 +1,19 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api + + +def seed_simulated_portfolio(exchange_manager, account: protocol_models.Account) -> None: + portfolio_content: dict[str, dict[str, float]] = {} + if account.assets: + for assets_for_trading_type in account.assets: + for detailed_asset in assets_for_trading_type.assets: + portfolio_content[detailed_asset.symbol] = { + commons_constants.PORTFOLIO_AVAILABLE: float(detailed_asset.available), + commons_constants.PORTFOLIO_TOTAL: float(detailed_asset.total), + } + if portfolio_content: + trading_api.set_simulated_portfolio_initial_config(exchange_manager, portfolio_content) diff --git a/packages/flow/octobot_flow/logic/global_view/__init__.py b/packages/flow/octobot_flow/logic/global_view/__init__.py new file mode 100644 index 0000000000..c78ffe6b00 --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/__init__.py @@ -0,0 +1,2 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. diff --git a/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py new file mode 100644 index 0000000000..3034281732 --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py @@ -0,0 +1,46 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import typing + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module + + +def build_updated_account( + account: protocol_models.Account, + assets: list[protocol_models.DetailedAssetsForTradingType], +) -> protocol_models.Account: + account_updates: dict[str, typing.Any] = { + "updated_at": timestamp_util.utc_now_datetime(), + } + if assets: + account_updates["assets"] = assets + return account.model_copy(update=account_updates) + + +def build_global_view_account_refresh_result( + user_id: str, + context: octobot_flow.entities.GlobalViewAccountContext, + exchange_refresh_result: octobot_flow.entities.ExchangeAccountRefreshResult, +) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + updated_account = build_updated_account(context.account, exchange_refresh_result.assets) + evaluation_time = exchange_refresh_result.portfolio_snapshot.timestamp + portfolio_history_state = account_state_persistence_module.build_portfolio_history_state( + user_id, + context.account.id, + exchange_refresh_result.portfolio_snapshot, + exchange_refresh_result.valuation_unit, + evaluation_time, + ) + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=updated_account, + changed_order_ids=exchange_refresh_result.changed_order_ids, + open_orders=exchange_refresh_result.open_orders, + trades=exchange_refresh_result.trades, + positions=exchange_refresh_result.positions, + portfolio_history_state=portfolio_history_state, + ) diff --git a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py new file mode 100644 index 0000000000..4b445c722d --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py @@ -0,0 +1,111 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.constants as commons_constants +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.errors as trading_errors +import octobot_trading.personal_data.portfolios.protocol as portfolios_protocol + +import octobot_flow.entities +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module +import octobot_flow.logic.exchange.portfolio.valuation_unit as valuation_unit_module + + +async def refresh_exchange_account( + exchange_manager, + trading_type: protocol_models.TradingType, + previous_open_order_exchange_ids: set[str], +) -> octobot_flow.entities.ExchangeAccountRefreshResult: + # Step: fetch balance and open orders from the exchange manager. + await exchange_manager.exchange.get_balance() + open_orders = await exchange_manager.exchange.get_open_orders() + trades: list[dict] = [] + positions: list[dict] = [] + + # Step: resolve valuation unit and portfolio total in that currency. + valuation_unit = valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is not None: + portfolio_manager.reference_market = valuation_unit + handle_mark_price_update = getattr(portfolio_manager, "handle_mark_price_update", None) + if handle_mark_price_update is not None: + await handle_mark_price_update() + portfolio_total = trading_api.get_current_portfolio_value(exchange_manager) + + # Step: build protocol assets and historical snapshot payload. + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + detailed_assets = portfolios_protocol.to_protocol_assets(portfolio_content) + historical_assets = _historical_assets_from_portfolio( + exchange_manager, + portfolio_content, + trading_type, + valuation_unit, + ) + evaluation_time = timestamp_util.utc_now_datetime() + portfolio_snapshot = protocol_models.PortfolioHistoricalValue( + timestamp=evaluation_time, + total=portfolio_total, + assets=historical_assets, + ) + assets_for_trading_type = [ + protocol_models.DetailedAssetsForTradingType( + trading_type=trading_type, + assets=detailed_assets, + ) + ] if detailed_assets else [] + + # Step: detect orders that disappeared since the previous refresh. + changed_order_ids = order_change_detection_module.detect_changed_order_ids( + previous_open_order_exchange_ids, + open_orders, + ) + open_order_dicts = [ + order_change_detection_module.open_order_to_storage_dict(open_order) + for open_order in open_orders + ] + return octobot_flow.entities.ExchangeAccountRefreshResult( + assets=assets_for_trading_type, + portfolio_snapshot=portfolio_snapshot, + valuation_unit=valuation_unit, + open_orders=open_order_dicts, + trades=trades, + positions=positions, + changed_order_ids=changed_order_ids, + ) + + +def _historical_assets_from_portfolio( + exchange_manager, + portfolio_content: dict, + trading_type: protocol_models.TradingType, + valuation_unit: str, +) -> list[protocol_models.HistoricalAssetsForTradingType]: + historical_asset_values: list[protocol_models.HistoricalAssetValue] = [] + for symbol, symbol_balance in portfolio_content.items(): + total_holdings = float(symbol_balance.get(commons_constants.PORTFOLIO_TOTAL) or 0) + if total_holdings == 0: + continue + try: + unit_price = float( + trading_api.get_current_crypto_currency_value(exchange_manager, symbol) + ) + except (KeyError, trading_errors.MissingPriceDataError): + unit_price = 0.0 + asset_value = unit_price * total_holdings + historical_asset_values.append( + protocol_models.HistoricalAssetValue( + symbol=str(symbol), + holdings=total_holdings, + value=asset_value, + ) + ) + if not historical_asset_values: + return [] + return [ + protocol_models.HistoricalAssetsForTradingType( + trading_type=trading_type, + assets=historical_asset_values, + ) + ] diff --git a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py new file mode 100644 index 0000000000..466c7b194f --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py @@ -0,0 +1,28 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module + + +def persist_global_view_refresh_result( + user_id: str, + account_id: str, + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, +) -> None: + collection_providers.AccountProvider.instance().update_item(user_id, refresh_result.updated_account) + account_state_persistence_module.persist_account_trading( + user_id, + account_id, + refresh_result.open_orders or [], + refresh_result.trades or [], + refresh_result.positions or [], + ) + if refresh_result.portfolio_history_state is not None: + collection_providers.AccountHistoryProvider.instance().save_state( + user_id, + account_id, + refresh_result.portfolio_history_state, + ) diff --git a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py new file mode 100644 index 0000000000..980651d56b --- /dev/null +++ b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py @@ -0,0 +1,230 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_sync.constants as sync_constants + + +def _open_order_dict(exchange_id: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _exchange_account_context() -> octobot_flow.entities.GlobalViewAccountContext: + account_id = "functional-account-1" + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id=account_id, + name="Functional account", + is_simulated=True, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + ) + + +@pytest.mark.asyncio +class TestGlobalViewAccountJobFunctional: + async def test_refresh_simulated_account(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data.portfolio_manager = None + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value=set(), + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + mock.patch.object( + account_state_persistence_module, + "build_portfolio_history_state", + side_effect=lambda user_id, account_id, snapshot, valuation_unit, evaluation_time: ( + protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=[snapshot], + ), + ) + ), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=_portfolio_content(), + ), + mock.patch.object( + trading_api, + "get_current_portfolio_value", + return_value=1500.0, + ), + mock.patch.object( + trading_api, + "get_current_crypto_currency_value", + side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + assert refresh_result.portfolio_history_state is not None + assert refresh_result.portfolio_history_state.history is not None + assert refresh_result.portfolio_history_state.history.unit == "USDC" + assert refresh_result.portfolio_history_state.history.values[-1].total == 1500.0 + assert refresh_result.updated_account.assets + assert refresh_result.changed_order_ids == set() + + async def test_refresh_real_account_detects_disappeared_orders(self): + context = _exchange_account_context() + context.account.is_simulated = False + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + exchange_manager.exchange_personal_data.portfolio_manager = None + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + mock.patch.object( + account_state_persistence_module, + "build_portfolio_history_state", + side_effect=lambda user_id, account_id, snapshot, valuation_unit, evaluation_time: ( + protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=[snapshot], + ), + ) + ), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=_portfolio_content(), + ), + mock.patch.object( + trading_api, + "get_current_portfolio_value", + return_value=1500.0, + ), + mock.patch.object( + trading_api, + "get_current_crypto_currency_value", + side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + assert refresh_result.portfolio_history_state is not None + assert refresh_result.portfolio_history_state.history.unit == commons_constants.DEFAULT_REFERENCE_MARKET + assert refresh_result.updated_account.assets + assert refresh_result.changed_order_ids == {"gone-order-1"} diff --git a/packages/flow/tests/logic/accounts/test_account_state_persistence.py b/packages/flow/tests/logic/accounts/test_account_state_persistence.py new file mode 100644 index 0000000000..c4769b0062 --- /dev/null +++ b/packages/flow/tests/logic/accounts/test_account_state_persistence.py @@ -0,0 +1,116 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _portfolio_snapshot(total: float) -> protocol_models.PortfolioHistoricalValue: + return protocol_models.PortfolioHistoricalValue( + timestamp=_TEST_TIMESTAMP, + total=total, + ) + + +def _protocol_order(exchange_id: str) -> protocol_models.Order: + return protocol_models.Order( + id=exchange_id, + symbol="BTC/USDT", + price=10000.0, + quantity=0.01, + filled=0.0, + exchange_id=exchange_id, + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_TEST_TIMESTAMP, + ) + + +class TestLoadPreviousOpenOrderExchangeIds: + def test_returns_exchange_ids_from_persisted_orders(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[ + _protocol_order("order-1"), + _protocol_order("order-2"), + ], + ), + ) + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + assert exchange_ids == {"order-1", "order-2"} + + def test_returns_empty_set_when_no_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + assert exchange_ids == set() + + +class TestBuildPortfolioHistoryState: + def test_merges_snapshot_into_history_state(self): + history_provider = mock.Mock() + history_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountHistoryProvider, + "instance", + return_value=history_provider, + ): + history_state = account_state_persistence_module.build_portfolio_history_state( + "wallet-1", + "account-1", + _portfolio_snapshot(1500.0), + "USDC", + _TEST_TIMESTAMP, + ) + assert history_state.history is not None + assert history_state.history.unit == "USDC" + assert history_state.history.values[-1].total == 1500.0 + + +class TestPersistAccountTrading: + def test_saves_updated_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[], + trades=[], + positions=[], + ) + trading_provider.save_state.assert_called_once() diff --git a/packages/flow/tests/logic/accounts/test_portfolio_history.py b/packages/flow/tests/logic/accounts/test_portfolio_history.py new file mode 100644 index 0000000000..aab9ad056d --- /dev/null +++ b/packages/flow/tests/logic/accounts/test_portfolio_history.py @@ -0,0 +1,70 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import octobot_flow.constants as flow_constants +import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module +import octobot_protocol.models as protocol_models + + +def _snapshot(timestamp: datetime.datetime, total: float) -> protocol_models.PortfolioHistoricalValue: + return protocol_models.PortfolioHistoricalValue( + timestamp=timestamp, + total=total, + ) + + +class TestMergeSnapshotLatest: + def test_every_call_replaces_latest_entry(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 1, 12, 5, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 110.0), + second_time, + ) + assert len(merged_values) == 2 + assert merged_values[-1].total == 110.0 + assert merged_values[0].total == 100.0 + + +class TestMergeSnapshot12hCadence: + def test_refresh_within_12h_does_not_add_second_twelve_hour_entry(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 1, 15, 0, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 105.0), + second_time, + ) + assert len(merged_values) == 2 + assert merged_values[0].timestamp == first_time + assert merged_values[-1].timestamp == second_time + + +class TestMergeSnapshot12hElapsed: + def test_appends_new_twelve_hour_snapshot_after_interval(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = first_time + datetime.timedelta( + seconds=flow_constants.PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS + ) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 120.0), + second_time, + ) + assert len(merged_values) == 3 + assert merged_values[0].timestamp == first_time + assert merged_values[1].timestamp == second_time + assert merged_values[-1].timestamp == second_time + + +class TestMergeSnapshotUnit: + def test_unit_preserved_by_caller_not_merge_snapshot(self): + evaluation_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [], + _snapshot(evaluation_time, 50.0), + evaluation_time, + ) + assert merged_values[0].total == 50.0 diff --git a/packages/flow/tests/logic/configuration/test_profile_data_factory.py b/packages/flow/tests/logic/configuration/test_profile_data_factory.py index 396e84ec88..32f88d0d8c 100644 --- a/packages/flow/tests/logic/configuration/test_profile_data_factory.py +++ b/packages/flow/tests/logic/configuration/test_profile_data_factory.py @@ -1,7 +1,11 @@ # Drakkar-Software OctoBot-Flow # Copyright (c) Drakkar-Software, All rights reserved. +import datetime + import octobot_commons.profiles.profile_data as profile_data_module +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums import octobot_trading.exchanges.util.exchange_data as exchange_data_module import octobot_flow.entities.accounts.exchange_account_details as exchange_account_details_module @@ -43,3 +47,73 @@ def test_omits_hollaex_tentacle_when_url_missing(self): as_simulator=True, ) assert profile_data.get_config_by_tentacle() == {} + + +class TestExchangeTypeFromTradingType: + def test_maps_spot_trading_type(self): + assert profile_data_factory_module.exchange_type_from_trading_type( + protocol_models.TradingType.SPOT, + ) == trading_enums.ExchangeTypes.SPOT.value + + +class TestProfileDataForAccount: + def test_enables_trader_for_real_account(self): + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="account-1", + name="Real account", + is_simulated=False, + created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + exchange_account, + exchange_config, + protocol_models.TradingType.SPOT, + is_simulated=False, + ) + assert profile_data.trader.enabled is True + assert profile_data.trader_simulator.enabled is False + assert profile_data.exchanges[0].internal_name == "binanceus" + + def test_enables_simulator_for_simulated_account(self): + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="sim-account-1", + name="Sim account", + is_simulated=True, + created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + exchange_account, + exchange_config, + protocol_models.TradingType.SPOT, + is_simulated=True, + ) + assert profile_data.trader.enabled is False + assert profile_data.trader_simulator.enabled is True diff --git a/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py b/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py new file mode 100644 index 0000000000..2ead3c6c33 --- /dev/null +++ b/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Flow + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module + + +def _order_dict(exchange_id: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +class TestDetectChangedOrderIds: + def test_returns_disappeared_order_ids(self): + previous_ids = {"order-1", "order-2"} + current_orders = [_order_dict("order-2")] + changed_ids = order_change_detection_module.detect_changed_order_ids( + previous_ids, + current_orders, + ) + assert changed_ids == {"order-1"} + + def test_returns_empty_when_no_previous_orders(self): + changed_ids = order_change_detection_module.detect_changed_order_ids( + set(), + [_order_dict("order-1")], + ) + assert changed_ids == set() diff --git a/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py b/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py new file mode 100644 index 0000000000..a806c9be3c --- /dev/null +++ b/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py @@ -0,0 +1,25 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import octobot_commons.constants as commons_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.exchange.portfolio.valuation_unit as valuation_unit_module + + +class TestResolvePortfolioValuationUnit: + def test_returns_exchange_default_quote_currency_when_set(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value.return_value = "USDC" + assert valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) == "USDC" + exchange_manager.exchange.get_option_value.assert_called_once_with( + trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY + ) + + def test_falls_back_to_default_reference_market_when_option_missing(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value.return_value = None + assert ( + valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) + == commons_constants.DEFAULT_REFERENCE_MARKET + ) diff --git a/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py new file mode 100644 index 0000000000..2f14dcf687 --- /dev/null +++ b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py @@ -0,0 +1,80 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api + +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +class TestSeedSimulatedPortfolio: + def test_seeds_portfolio_from_account_assets(self): + account = protocol_models.Account( + id="sim-account-1", + name="Simulated", + is_simulated=True, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=1000.0, + available=900.0, + ), + ], + ), + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ), + ), + ) + exchange_manager = mock.Mock() + with mock.patch.object( + trading_api, + "set_simulated_portfolio_initial_config", + ) as set_portfolio_mock: + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + set_portfolio_mock.assert_called_once_with( + exchange_manager, + { + "USDT": { + commons_constants.PORTFOLIO_AVAILABLE: 900.0, + commons_constants.PORTFOLIO_TOTAL: 1000.0, + }, + }, + ) + + def test_skips_when_account_has_no_assets(self): + account = protocol_models.Account( + id="sim-account-2", + name="Simulated empty", + is_simulated=True, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.GenericAccount( + account_type=protocol_models.AccountType.GENERIC, + ), + ), + ) + exchange_manager = mock.Mock() + with mock.patch.object( + trading_api, + "set_simulated_portfolio_initial_config", + ) as set_portfolio_mock: + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + set_portfolio_mock.assert_not_called() diff --git a/packages/flow/tests/logic/global_view/test_account_refresh_builder.py b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py new file mode 100644 index 0000000000..5dc10e7046 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py @@ -0,0 +1,93 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.enums as trading_enums + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _context() -> octobot_flow.entities.GlobalViewAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="account-1", + name="Test account", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ), + trading_type=protocol_models.TradingType.SPOT, + auth_details=exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id="account-1", + ), + ) + + +class TestBuildGlobalViewAccountRefreshResult: + def test_builds_refresh_result_with_history_state(self): + context = _context() + exchange_refresh_result = octobot_flow.entities.ExchangeAccountRefreshResult( + assets=[], + portfolio_snapshot=protocol_models.PortfolioHistoricalValue( + timestamp=_TEST_TIMESTAMP, + total=1000.0, + ), + valuation_unit="USDC", + open_orders=[], + trades=[], + positions=[], + changed_order_ids={"gone-order"}, + ) + expected_history_state = protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDC", + values=[exchange_refresh_result.portfolio_snapshot], + ), + ) + with mock.patch.object( + account_state_persistence_module, + "build_portfolio_history_state", + return_value=expected_history_state, + ) as build_history_mock: + refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( + "wallet-1", + context, + exchange_refresh_result, + ) + build_history_mock.assert_called_once_with( + "wallet-1", + "account-1", + exchange_refresh_result.portfolio_snapshot, + "USDC", + _TEST_TIMESTAMP, + ) + assert refresh_result.updated_account.id == "account-1" + assert refresh_result.changed_order_ids == {"gone-order"} + assert refresh_result.portfolio_history_state == expected_history_state diff --git a/packages/flow/tests/logic/global_view/test_global_view_account_job.py b/packages/flow/tests/logic/global_view/test_global_view_account_job.py new file mode 100644 index 0000000000..30a1b7d6eb --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_global_view_account_job.py @@ -0,0 +1,263 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_sync.constants as sync_constants + + +def _open_order_dict(exchange_id: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _exchange_account_context( + *, + account_id: str = "account-1", + is_simulated: bool = False, +) -> octobot_flow.entities.GlobalViewAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id=account_id, + name="Test account", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + ) + + +@pytest.mark.asyncio +class TestGlobalViewAccountJobRun: + async def test_run_returns_refresh_result_shape(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data.portfolio_manager = None + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value=set(), + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + mock.patch.object( + account_state_persistence_module, + "build_portfolio_history_state", + return_value=protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDC", + values=[], + ), + ), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=_portfolio_content(), + ), + mock.patch.object( + trading_api, + "get_current_portfolio_value", + return_value=1500.0, + ), + mock.patch.object( + trading_api, + "get_current_crypto_currency_value", + side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + assert isinstance(refresh_result, octobot_flow.entities.GlobalViewAccountRefreshResult) + assert refresh_result.updated_account.id == "account-1" + assert refresh_result.changed_order_ids == set() + assert refresh_result.portfolio_history_state is not None + assert refresh_result.open_orders == [] + + async def test_run_detects_disappeared_orders(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + exchange_manager.exchange_personal_data.portfolio_manager = None + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + mock.patch.object( + account_state_persistence_module, + "build_portfolio_history_state", + return_value=protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDC", + values=[], + ), + ), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=_portfolio_content(), + ), + mock.patch.object( + trading_api, + "get_current_portfolio_value", + return_value=1500.0, + ), + mock.patch.object( + trading_api, + "get_current_crypto_currency_value", + side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + assert refresh_result.changed_order_ids == {"gone-order-1"} + + async def test_generic_account_returns_no_op_result(self): + generic_account = protocol_models.Account( + id="generic-1", + name="Generic", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.GenericAccount( + account_type=protocol_models.AccountType.GENERIC, + ), + ), + ) + context = octobot_flow.entities.GlobalViewAccountContext( + account=generic_account, + exchange_account=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="unused", + exchange_config_ids=["exchange-config-1"], + ), + exchange_config=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ), + trading_type=protocol_models.TradingType.SPOT, + auth_details=exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + ), + ) + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + assert refresh_result.updated_account.id == "generic-1" + assert refresh_result.changed_order_ids == set() diff --git a/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py b/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py new file mode 100644 index 0000000000..2326c08875 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py @@ -0,0 +1,47 @@ +# Drakkar-Software OctoBot-Flow + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities + + +def _order_dict(exchange_id: str, *, filled: float = 0) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: filled, + } + } + + +def _exchange_data(open_orders: list[dict], portfolio: dict[str, dict]) -> exchange_data_module.ExchangeData: + return exchange_data_module.ExchangeData( + orders_details=exchange_data_module.OrdersDetails(open_orders=open_orders), + portfolio_details=exchange_data_module.PortfolioDetails(content=portfolio), + ) + + +class TestGlobalViewRefreshedElementsConfirmedChange: + def test_detects_order_disappearance(self): + before = octobot_flow.entities.GlobalViewRefreshedElements( + _exchange_data( + [_order_dict("order-1"), _order_dict("order-2")], + {"USDT": {"total": 100.0}}, + ), + ) + after_exchange_data = _exchange_data( + [_order_dict("order-2")], + {"USDT": {"total": 100.0}}, + ) + assert before.confirmed_change(after_exchange_data) is True + + def test_no_change_when_orders_and_portfolio_match(self): + orders = [_order_dict("order-1")] + portfolio = {"USDT": {"total": 100.0}} + before = octobot_flow.entities.GlobalViewRefreshedElements( + _exchange_data(orders, portfolio), + ) + after_exchange_data = _exchange_data(orders, portfolio) + assert before.confirmed_change(after_exchange_data) is False diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 081b3a972b..b81b68f3f1 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -74,6 +74,13 @@ DEFAULT_PORTFOLIO_VALUATION_UNIT = "USDT" +GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS = float( + os.getenv("GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS", "300.0") +) +GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS = float( + os.getenv("GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS", "0.1") +) + NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID = "non-trading-generic-process-octobot-strategy" NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_VERSION = "1.0.0" diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index f9cc05cf4b..01d59aedce 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -38,3 +38,4 @@ class SchedulerQueues(enum.Enum): AUTOMATION_WORKFLOW_QUEUE = "automation_workflow_queue" USER_ACTION_QUEUE = "user_action_queue" DBOS_CLEANUP_QUEUE = "dbos_cleanup_queue" + GLOBAL_VIEW_QUEUE = "global_view_queue" diff --git a/packages/node/octobot_node/protocol/accounts_history.py b/packages/node/octobot_node/protocol/accounts_history.py new file mode 100644 index 0000000000..9262ff0e54 --- /dev/null +++ b/packages/node/octobot_node/protocol/accounts_history.py @@ -0,0 +1,71 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import datetime + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers.user_account_history_provider as history_provider + +import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module + + +def get_portfolio_history_state( + user_id: str, + account_id: str, +) -> protocol_models.PortfolioHistoricalValuesState: + try: + return history_provider.AccountHistoryProvider.instance().load_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def save_portfolio_evaluation( + user_id: str, + account_id: str, + snapshot: protocol_models.PortfolioHistoricalValue, + valuation_unit: str, + evaluation_time: datetime.datetime, +) -> protocol_models.PortfolioHistoricalValuesState: + history_state = get_portfolio_history_state(user_id, account_id) + existing_values = ( + history_state.history.values + if history_state.history is not None and history_state.history.values + else [] + ) + merged_values = portfolio_history_module.merge_snapshot( + existing_values, + snapshot, + evaluation_time, + ) + updated_history = protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=merged_values, + ) + updated_state = protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=updated_history, + ) + history_provider.AccountHistoryProvider.instance().save_state( + user_id, + account_id, + updated_state, + ) + return updated_state diff --git a/packages/node/octobot_node/scheduler/global_view/__init__.py b/packages/node/octobot_node/scheduler/global_view/__init__.py new file mode 100644 index 0000000000..be14b9abe2 --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/__init__.py @@ -0,0 +1,2 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. diff --git a/packages/node/octobot_node/scheduler/global_view/automation_trigger.py b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py new file mode 100644 index 0000000000..034bf8df85 --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py @@ -0,0 +1,101 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. + +import asyncio +import logging +import time +import typing + +import dbos +import octobot_protocol.models as protocol_models + +import octobot_node.constants as node_constants +import octobot_node.scheduler.api as scheduler_api +import octobot_node.scheduler.tasks as scheduler_tasks + + +logger = logging.getLogger("GlobalViewAutomationTrigger") + + +async def trigger_account_automations( + user_id: str, + account_id: str, + changed_order_ids: set[str], +) -> None: + if not changed_order_ids: + return + matching_automations = await _matching_automations(user_id, account_id, changed_order_ids) + for automation_state in matching_automations: + await _trigger_automation_and_wait(user_id, automation_state.id) + + +async def _matching_automations( + user_id: str, + account_id: str, + changed_order_ids: set[str], +) -> list[protocol_models.AutomationState]: + automation_states = await scheduler_api.get_automation_states(user_id) + matching_automations: list[protocol_models.AutomationState] = [] + for automation_state in automation_states: + if automation_state.status != protocol_models.WorkflowStatus.RUNNING: + continue + if not automation_state.exchange_account_ids or account_id not in automation_state.exchange_account_ids: + continue + automation_order_ids = { + str(order_summary.id) + for order_summary in (automation_state.orders or []) + if order_summary.id + } + if automation_order_ids.intersection(changed_order_ids): + matching_automations.append(automation_state) + return matching_automations + + +async def _trigger_automation_and_wait(user_id: str, automation_id: str) -> None: + import octobot_node.scheduler as scheduler_module + + scheduler = scheduler_module.SCHEDULER + active_workflow_ids_before = await scheduler.resolve_active_automation_workflow_ids_for_parent_id( + user_id, + automation_id, + ) + workflow_id_before = active_workflow_ids_before[0] if active_workflow_ids_before else None + + await scheduler_tasks.send_forced_trigger_to_active_automation(automation_id, user_id) + + if workflow_id_before is None: + logger.warning( + "No active workflow to wait for after forced trigger (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) + return + await _wait_for_workflow_iteration_success(workflow_id_before) + + +async def _wait_for_workflow_iteration_success(workflow_id: str) -> None: + deadline = time.monotonic() + node_constants.GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS + poll_interval = node_constants.GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS + while time.monotonic() < deadline: + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + if workflow_status is None: + await asyncio.sleep(poll_interval) + continue + if workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: + return + if workflow_status.status in ( + dbos.WorkflowStatusString.ERROR.value, + dbos.WorkflowStatusString.CANCELLED.value, + dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED.value, + ): + logger.error( + "Automation workflow %s ended with status %s while waiting for forced trigger iteration", + workflow_id, + workflow_status.status, + ) + return + await asyncio.sleep(poll_interval) + logger.error( + "Timed out waiting for automation workflow %s to complete forced trigger iteration", + workflow_id, + ) diff --git a/packages/node/octobot_node/scheduler/global_view/global_view_executor.py b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py new file mode 100644 index 0000000000..a295daefa8 --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py @@ -0,0 +1,56 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver + + +async def refresh_account_global_view( + user_id: str, + account: protocol_models.Account, +) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account_specifics = account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "Account.specifics.actual_instance is required for global view refresh." + ) + account_specifics_instance = account_specifics.actual_instance + if isinstance(account_specifics_instance, protocol_models.GenericAccount): + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + ) + if isinstance(account_specifics_instance, protocol_models.BlockchainAccount): + raise node_errors.InvalidUserActionPayloadError("Blockchain accounts are not supported yet.") + if not isinstance(account_specifics_instance, protocol_models.ExchangeAccount): + raise node_errors.InvalidUserActionPayloadError( + f"Unsupported account specifics type: {type(account_specifics_instance).__name__}." + ) + + exchange_account = account_specifics_instance + trading_type = account_state_updater._trading_type_for_account_state_check(account) + exchange_config = exchange_account_resolver.get_exchange_config(user_id, exchange_account) + authentication = None if account.is_simulated else account_authentication_resolver.get_exchange_authentication( + user_id, + account, + ) + auth_details = account_state_updater._encrypted_exchange_auth_details( + exchange_account, + authentication, + trading_type, + exchange_config.sandboxed, + ) + context = octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=trading_type, + auth_details=auth_details, + ) + return await global_view_account_job_module.GlobalViewAccountJob(user_id, context).run() diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index e253ba9da7..a29e83b8f8 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -67,6 +67,7 @@ class Scheduler: AUTOMATION_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore USER_ACTION_QUEUE: dbos.Queue = None # type: ignore DBOS_CLEANUP_QUEUE: dbos.Queue = None # type: ignore + GLOBAL_VIEW_QUEUE: dbos.Queue = None # type: ignore @staticmethod def _wallet_filter_queue(queue_names: typing.Optional[list[str]]) -> octobot_node.enums.SchedulerQueues: @@ -151,6 +152,7 @@ def stop(self) -> None: Scheduler.AUTOMATION_WORKFLOW_QUEUE = None Scheduler.USER_ACTION_QUEUE = None Scheduler.DBOS_CLEANUP_QUEUE = None + Scheduler.GLOBAL_VIEW_QUEUE = None def create_queues(self): self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) @@ -160,6 +162,10 @@ def create_queues(self): # only one cleanup workflow can run at a time concurrency=1, ) + self.GLOBAL_VIEW_QUEUE = dbos.Queue( + name=octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, + concurrency=1, + ) async def get_periodic_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" diff --git a/packages/node/octobot_node/scheduler/schedules.py b/packages/node/octobot_node/scheduler/schedules.py index 6246ffafe1..e0d3b9049d 100644 --- a/packages/node/octobot_node/scheduler/schedules.py +++ b/packages/node/octobot_node/scheduler/schedules.py @@ -25,6 +25,7 @@ import octobot_node.constants as constants import octobot_node.scheduler.scheduler as scheduler_module import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow +import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow import octobot_node.scheduler.workflows_retention as workflows_retention @@ -271,6 +272,7 @@ async def _ensure_schedule( async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: schedule_inputs: list[dbos.ScheduleInput] = [ dbos_cleanup_workflow.get_schedule_input(), + global_view_workflow.get_schedule_input(), ] for schedule_input in schedule_inputs: await _ensure_schedule(scheduler, schedule_input) diff --git a/packages/node/octobot_node/scheduler/workflows/__init__.py b/packages/node/octobot_node/scheduler/workflows/__init__.py index 624a4184a5..8f489dd044 100644 --- a/packages/node/octobot_node/scheduler/workflows/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/__init__.py @@ -19,3 +19,4 @@ def register_workflows() -> None: import octobot_node.scheduler.workflows.automation_workflow import octobot_node.scheduler.workflows.user_action_workflow import octobot_node.scheduler.workflows.dbos_cleanup_workflow + import octobot_node.scheduler.workflows.global_view_workflow diff --git a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py new file mode 100644 index 0000000000..070687da0c --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py @@ -0,0 +1,145 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import datetime +import logging +import typing + +import dbos +import octobot_commons.logging as octobot_commons_logging +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.enums +import octobot_node.scheduler.global_view.global_view_executor as global_view_executor_module +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module +import octobot_node.scheduler.workflows_retention as workflows_retention + +from octobot_node.scheduler import SCHEDULER # avoid circular import + +WORKFLOW_NAME = "global_view_refresh" +SCHEDULE_NAME = "global_view_refresh_every_5m" +SCHEDULE_CRON = "*/5 * * * *" + + +@SCHEDULER.INSTANCE.dbos_class() +class GlobalViewRefreshWorkflow: + @staticmethod + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + async def global_view_refresh( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + return await GlobalViewRefreshWorkflow._run_global_view_refresh( + scheduled_time, + context, + ) + + @staticmethod + @SCHEDULER.INSTANCE.step(name="run_global_view_refresh") + async def _run_global_view_refresh( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + logger = octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__) + if workflows_retention.should_skip_retention_cleanup_on_this_node(): + logger.info("global_view_refresh skipped: consumer-only node") + return {"refreshed_accounts": 0, "skipped": True} + wallet_ids = collection_providers.AccountProvider.instance().list_registered_wallet_ids() + refreshed_accounts_count = 0 + for wallet_id in wallet_ids: + refreshed_accounts_count += await GlobalViewRefreshWorkflow._refresh_wallet_accounts( + wallet_id, + logger, + ) + return { + "refreshed_accounts": refreshed_accounts_count, + "scheduled_time": scheduled_time.isoformat(), + } + + @staticmethod + async def _refresh_wallet_accounts( + user_id: str, + logger: logging.Logger, + ) -> int: + account_provider = collection_providers.AccountProvider.instance() + try: + accounts = account_provider.list_items(user_id) + except Exception as error: + logger.warning( + "Skipping global view refresh for wallet %s: cannot list accounts (%s)", + user_id, + error, + ) + return 0 + if not accounts: + return 0 + refresh_results = await asyncio.gather( + *[ + GlobalViewRefreshWorkflow._refresh_single_account(user_id, account, logger) + for account in accounts + ], + return_exceptions=True, + ) + refreshed_count = 0 + for refresh_result in refresh_results: + if isinstance(refresh_result, Exception): + logger.exception( + refresh_result, + True, + f"Account global view refresh failed: {refresh_result}", + ) + continue + if refresh_result: + refreshed_count += 1 + return refreshed_count + + @staticmethod + async def _refresh_single_account( + user_id: str, + account: protocol_models.Account, + logger: logging.Logger, + ) -> bool: + try: + refresh_result = await global_view_executor_module.refresh_account_global_view( + user_id, + account, + ) + except Exception as error: + logger.exception( + error, + True, + f"Failed to refresh account {account.id} for wallet {user_id}: {error}", + ) + raise + if refresh_result.changed_order_ids: + await automation_trigger_module.trigger_account_automations( + user_id, + account.id, + refresh_result.changed_order_ids, + ) + return True + + +def get_schedule_input() -> dbos.ScheduleInput: + return { + "schedule_name": SCHEDULE_NAME, + "workflow_fn": GlobalViewRefreshWorkflow.global_view_refresh, + "schedule": SCHEDULE_CRON, + "context": None, + "automatic_backfill": True, + "queue_name": octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, + } diff --git a/packages/node/tests/functional_tests/test_global_view_workflow.py b/packages/node/tests/functional_tests/test_global_view_workflow.py new file mode 100644 index 0000000000..bf77a24a68 --- /dev/null +++ b/packages/node/tests/functional_tests/test_global_view_workflow.py @@ -0,0 +1,141 @@ +# Drakkar-Software OctoBot-Node + +import asyncio +import pathlib + +import mock +import pytest + +import octobot_node.protocol.accounts_history as accounts_history_protocol +import octobot_node.protocol.accounts_trading as accounts_trading_protocol +import octobot_node.scheduler.api as scheduler_api_module +import octobot_node.scheduler.tasks as scheduler_tasks_module + +from tests.functional_tests.util import global_view_workflow as global_view_workflow_util +from tests.scheduler import temp_dbos_scheduler + + +@pytest.fixture +def global_view_temp_dbos_scheduler(temp_dbos_scheduler): + return temp_dbos_scheduler + + +@pytest.mark.asyncio +class TestGlobalViewWorkflowFunctional: + async def test_global_view_workflow_updates_all_accounts( + self, + tmp_path: pathlib.Path, + global_view_temp_dbos_scheduler, + ): + async with global_view_workflow_util.global_view_functional_environment(tmp_path) as environment: + workflow_result = await asyncio.wait_for( + global_view_workflow_util.enqueue_and_await_global_view_refresh(), + timeout=global_view_workflow_util.WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + assert workflow_result["refreshed_accounts"] == 3 + + user_id = environment["user_id"] + account_provider = environment["account_provider"] + real_account = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_REAL_ID) + sim_account_1 = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_SIM_1_ID) + sim_account_2 = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_SIM_2_ID) + + global_view_workflow_util.assert_real_account_assets(real_account) + assert sim_account_1.assets is not None + assert sim_account_2.assets is not None + assert real_account.updated_at is not None + assert sim_account_1.updated_at is not None + assert sim_account_2.updated_at is not None + + for account_id in ( + global_view_workflow_util.ACCOUNT_REAL_ID, + global_view_workflow_util.ACCOUNT_SIM_1_ID, + global_view_workflow_util.ACCOUNT_SIM_2_ID, + ): + history_state = accounts_history_protocol.get_portfolio_history_state(user_id, account_id) + assert history_state.history is not None + assert history_state.history.values + assert history_state.history.values[-1].total is not None + assert history_state.history.unit + + async def test_global_view_workflow_triggers_automation_on_filled_order( + self, + tmp_path: pathlib.Path, + global_view_temp_dbos_scheduler, + ): + matching_automation = global_view_workflow_util.build_running_automation_state( + global_view_workflow_util.AUTOMATION_FILL_ID, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + order_ids=[global_view_workflow_util.ORDER_FILL_ID], + ) + non_matching_automation = global_view_workflow_util.build_running_automation_state( + global_view_workflow_util.AUTOMATION_NO_TRIGGER_ID, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + order_ids=[global_view_workflow_util.ORDER_STAYS_OPEN_ID], + ) + trigger_calls: list[str] = [] + + async def record_forced_trigger(automation_id: str, user_id: str) -> None: + trigger_calls.append(automation_id) + + async with global_view_workflow_util.global_view_functional_environment( + tmp_path, + sim_open_order_ids={ + global_view_workflow_util.ACCOUNT_SIM_1_ID: [ + global_view_workflow_util.ORDER_STAYS_OPEN_ID, + ], + }, + ) as environment: + user_id = environment["user_id"] + global_view_workflow_util.seed_account_trading_state( + environment["trading_provider"], + user_id, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + order_exchange_ids=[ + global_view_workflow_util.ORDER_FILL_ID, + global_view_workflow_util.ORDER_STAYS_OPEN_ID, + ], + ) + with ( + mock.patch.object( + scheduler_api_module, + "get_automation_states", + mock.AsyncMock(return_value=[matching_automation, non_matching_automation]), + ), + mock.patch.object( + scheduler_tasks_module, + "send_forced_trigger_to_active_automation", + side_effect=record_forced_trigger, + ), + ): + workflow_result = await asyncio.wait_for( + global_view_workflow_util.enqueue_and_await_global_view_refresh(), + timeout=global_view_workflow_util.WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + + assert workflow_result["refreshed_accounts"] == 3 + assert trigger_calls == [global_view_workflow_util.AUTOMATION_FILL_ID] + + trading_state = accounts_trading_protocol.get_account_trading_state( + user_id, + global_view_workflow_util.ACCOUNT_SIM_1_ID, + ) + remaining_order_ids = { + str(protocol_order.exchange_id) + for protocol_order in (trading_state.account_trading.orders or []) + if protocol_order.exchange_id + } + assert global_view_workflow_util.ORDER_FILL_ID not in remaining_order_ids + assert global_view_workflow_util.ORDER_STAYS_OPEN_ID in remaining_order_ids + + account_provider = environment["account_provider"] + for account_id in ( + global_view_workflow_util.ACCOUNT_REAL_ID, + global_view_workflow_util.ACCOUNT_SIM_1_ID, + global_view_workflow_util.ACCOUNT_SIM_2_ID, + ): + updated_account = account_provider.get_item(user_id, account_id) + assert updated_account.assets is not None + history_state = accounts_history_protocol.get_portfolio_history_state(user_id, account_id) + assert history_state.history is not None + assert history_state.history.values diff --git a/packages/node/tests/functional_tests/util/global_view_workflow.py b/packages/node/tests/functional_tests/util/global_view_workflow.py new file mode 100644 index 0000000000..5defef1c3b --- /dev/null +++ b/packages/node/tests/functional_tests/util/global_view_workflow.py @@ -0,0 +1,384 @@ +# Drakkar-Software OctoBot-Node +"""Shared helpers for global view workflow functional tests.""" + +from __future__ import annotations + +import contextlib +import datetime +import typing + +import mock +import pytest + +import octobot.community.authentication as community_authentication_module +import octobot_commons.constants as commons_constants +import octobot_commons.user_root_folder_provider as user_root_folder_provider_module +import octobot_protocol.models as protocol_models +import octobot_sync.server as sync_server_module +import octobot_sync.sync.collection_providers as collection_providers_module +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.connectors.ccxt.ccxt_connector as ccxt_connector_module +import octobot_trading.exchanges.types.rest_exchange as rest_exchange_module + +import octobot_node.scheduler.workflows_retention as workflows_retention_module + +from tests.functional_tests.util import authenticator_mocks as authenticator_mocks_module +from tests.functional_tests.test_accounts_CRUD_operations import ( + _FUNCTIONAL_BTC_HOLDINGS, + _FUNCTIONAL_ETH_HOLDINGS, + _FUNCTIONAL_SOL_HOLDINGS, + _FUNCTIONAL_USDT_HOLDINGS, + _stub_get_balance_no_network, + _stub_load_symbol_markets_no_network, +) + +_TEST_PRIVATE_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +_TEST_WALLET_PASSPHRASE = "globalViewPW1!" +_FUNCTIONAL_TIMESTAMP = datetime.datetime(2026, 4, 1, 12, 0, 0, tzinfo=datetime.UTC) +_EXCHANGE_CONFIG_ID = "global-view-functional-exchange-config" + +ACCOUNT_REAL_ID = "acc-real" +ACCOUNT_SIM_1_ID = "acc-sim-1" +ACCOUNT_SIM_2_ID = "acc-sim-2" + +AUTOMATION_FILL_ID = "automation-gv-fill" +AUTOMATION_NO_TRIGGER_ID = "automation-gv-no-trigger" +ORDER_FILL_ID = "gv-order-fill-1" +ORDER_STAYS_OPEN_ID = "gv-order-stays-open" + +WORKFLOW_RESULT_TIMEOUT_SECONDS = 120.0 + + +async def _stub_get_open_orders_no_network(self, symbol=None, since=None, limit=None, **kwargs): + return [] + + +def derive_user_id() -> str: + return sync_server_module.derive_user_id(_TEST_PRIVATE_KEY) + + +def build_exchange_config() -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id=_EXCHANGE_CONFIG_ID, + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + + +def build_exchange_account(*, remote_account_id: str) -> protocol_models.ExchangeAccount: + return protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=remote_account_id, + exchange_config_ids=[_EXCHANGE_CONFIG_ID], + ) + + +def build_simulated_account( + *, + account_id: str, + account_name: str, + usdt_total: float = 1000.0, +) -> protocol_models.Account: + return protocol_models.Account( + id=account_id, + name=account_name, + is_simulated=True, + created_at=_FUNCTIONAL_TIMESTAMP, + updated_at=_FUNCTIONAL_TIMESTAMP, + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=usdt_total, + available=usdt_total, + ) + ], + ) + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=build_exchange_account(remote_account_id=account_id), + ), + ) + + +def build_real_account() -> protocol_models.Account: + return protocol_models.Account( + id=ACCOUNT_REAL_ID, + name="Global view real account", + is_simulated=False, + created_at=_FUNCTIONAL_TIMESTAMP, + updated_at=_FUNCTIONAL_TIMESTAMP, + authentication_id="global-view-functional-auth", + specifics=protocol_models.AccountSpecifics( + actual_instance=build_exchange_account(remote_account_id=ACCOUNT_REAL_ID), + ), + ) + + +def build_functional_authentication() -> protocol_models.AccountAuthentication: + return protocol_models.AccountAuthentication( + id="global-view-functional-auth", + api_key="functional-test-api-key", + api_secret="functional-test-api-secret", + ) + + +def _protocol_order(exchange_id: str) -> protocol_models.Order: + return protocol_models.Order( + id=exchange_id, + symbol="BTC/USDT", + price=10000.0, + quantity=0.01, + filled=0.0, + exchange_id=exchange_id, + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_FUNCTIONAL_TIMESTAMP, + ) + + +def _open_order_storage_dict(exchange_id: str) -> dict: + order_columns = trading_enums.ExchangeConstantsOrderColumns + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + order_columns.EXCHANGE_ID.value: exchange_id, + order_columns.ID.value: exchange_id, + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.PRICE.value: 10000.0, + order_columns.AMOUNT.value: 0.01, + order_columns.FILLED.value: 0, + order_columns.SIDE.value: trading_enums.TradeOrderSide.BUY.value, + order_columns.TYPE.value: trading_enums.TradeOrderType.LIMIT.value, + order_columns.TRIGGER_ABOVE.value: False, + order_columns.REDUCE_ONLY.value: False, + order_columns.IS_ACTIVE.value: True, + order_columns.STATUS.value: trading_enums.OrderStatus.OPEN.value, + order_columns.TIMESTAMP.value: _FUNCTIONAL_TIMESTAMP.timestamp(), + } + } + + +def seed_account_trading_state( + trading_provider: collection_providers_module.AccountTradingProvider, + user_id: str, + *, + account_id: str, + order_exchange_ids: list[str], +) -> None: + trading_provider.save_state( + user_id, + account_id, + protocol_models.AccountTradingState( + version=collection_providers_module.AccountTradingProvider.STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_FUNCTIONAL_TIMESTAMP, + orders=[_protocol_order(order_id) for order_id in order_exchange_ids], + trades=[], + positions=[], + ), + ), + ) + + +def build_running_automation_state( + automation_id: str, + *, + account_id: str, + order_ids: list[str], +) -> protocol_models.AutomationState: + return protocol_models.AutomationState( + id=automation_id, + status=protocol_models.WorkflowStatus.RUNNING, + metadata=protocol_models.AutomationMetadata( + name=automation_id, + description=automation_id, + ), + exchange_account_ids=[account_id], + orders=[ + protocol_models.OrderSummary(id=order_id, symbol="BTC/USDT") + for order_id in order_ids + ], + ) + + +async def enqueue_and_await_global_view_refresh() -> dict[str, typing.Any]: + import octobot_node.scheduler + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module + + scheduled_time = datetime.datetime.now(datetime.UTC) + workflow_handle = await octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE.enqueue_async( + global_view_workflow_module.GlobalViewRefreshWorkflow.global_view_refresh, + scheduled_time, + None, + ) + return await workflow_handle.get_result() + + +@contextlib.asynccontextmanager +async def global_view_functional_environment( + tmp_path, + *, + sim_open_order_ids: dict[str, list[str]] | None = None, +): + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "global_view_functional_user_root" + user_root_provider.set_root(str(test_user_root)) + + authentication_instance = authenticator_mocks_module.build_community_authentication( + _TEST_PRIVATE_KEY, + _TEST_WALLET_PASSPHRASE, + ) + user_id = derive_user_id() + + account_provider = collection_providers_module.AccountProvider(base_folder=str(test_user_root)) + authentication_provider = collection_providers_module.AccountAuthenticationProvider( + base_folder=str(test_user_root), + ) + trading_provider = collection_providers_module.AccountTradingProvider(base_folder=str(test_user_root)) + history_provider = collection_providers_module.AccountHistoryProvider(base_folder=str(test_user_root)) + + configured_sim_open_order_ids = sim_open_order_ids or {} + + import octobot_flow.jobs.global_view_account_job as global_view_account_job_module + import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater_module + import octobot_trading.exchanges as trading_exchanges_module + + real_exchange_manager_from_exchange_data = trading_exchanges_module.exchange_manager_from_exchange_data + + @contextlib.asynccontextmanager + async def exchange_manager_with_simulated_open_orders( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ): + async with real_exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=price_fallback, + ) as exchange_manager: + exchange_account_id = profile_data.exchanges[0].exchange_account_id + open_order_ids_after_refresh = configured_sim_open_order_ids.get(exchange_account_id, []) + + async def patched_get_open_orders(**open_orders_kwargs): + return [ + _open_order_storage_dict(order_id) + for order_id in open_order_ids_after_refresh + ] + + exchange_manager.exchange.get_open_orders = patched_get_open_orders + + real_get_balance = exchange_manager.exchange.get_balance + + async def patched_get_balance(**balance_kwargs): + balance = await real_get_balance(**balance_kwargs) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is not None and balance is not None: + portfolio_manager.handle_balance_update(balance) + return balance + + exchange_manager.exchange.get_balance = patched_get_balance + + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is not None: + portfolio_manager.handle_mark_price_update = mock.AsyncMock(return_value=None) + yield exchange_manager + + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch.object( + collection_providers_module.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers_module.AccountAuthenticationProvider, + "instance", + return_value=authentication_provider, + ), + mock.patch.object( + collection_providers_module.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + mock.patch.object( + collection_providers_module.AccountHistoryProvider, + "instance", + return_value=history_provider, + ), + mock.patch.object( + workflows_retention_module, + "should_skip_retention_cleanup_on_this_node", + return_value=False, + ), + mock.patch.object( + account_state_updater_module, + "_fetch_api_key_rights", + mock.AsyncMock(return_value=[]), + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "load_symbol_markets", + _stub_load_symbol_markets_no_network, + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "get_balance", + _stub_get_balance_no_network, + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "get_open_orders", + _stub_get_open_orders_no_network, + ), + mock.patch.object( + trading_exchanges_module, + "exchange_manager_from_exchange_data", + exchange_manager_with_simulated_open_orders, + ), + ): + account_provider.create_exchange_config(user_id, build_exchange_config()) + authentication_provider.create_item(user_id, build_functional_authentication()) + account_provider.create_item(user_id, build_real_account()) + account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_1_ID, account_name="Sim 1")) + account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_2_ID, account_name="Sim 2")) + for account_id, order_ids in configured_sim_open_order_ids.items(): + seed_account_trading_state( + trading_provider, + user_id, + account_id=account_id, + order_exchange_ids=order_ids, + ) + try: + yield { + "user_id": user_id, + "account_provider": account_provider, + "trading_provider": trading_provider, + "history_provider": history_provider, + } + finally: + user_root_provider.set_root(previous_user_root) + + +def assert_real_account_assets(account: protocol_models.Account) -> None: + assert account.assets is not None + flattened_assets: list[protocol_models.DetailedAsset] = [] + for assets_for_trading_type in account.assets: + flattened_assets.extend(assets_for_trading_type.assets or []) + assets_by_symbol = {asset.symbol: asset for asset in flattened_assets} + assert set(assets_by_symbol) == {"USDT", "BTC", "ETH", "SOL"} + assert assets_by_symbol["USDT"].total == pytest.approx(_FUNCTIONAL_USDT_HOLDINGS) + assert assets_by_symbol["BTC"].total == pytest.approx(_FUNCTIONAL_BTC_HOLDINGS) + assert assets_by_symbol["ETH"].total == pytest.approx(_FUNCTIONAL_ETH_HOLDINGS) + assert assets_by_symbol["SOL"].total == pytest.approx(_FUNCTIONAL_SOL_HOLDINGS) diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index 9d08455d04..2825fdf5a2 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -91,6 +91,7 @@ def _ensure_scheduler_queues() -> None: octobot_node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value: "AUTOMATION_WORKFLOW_QUEUE", octobot_node_enums_module.SchedulerQueues.USER_ACTION_QUEUE.value: "USER_ACTION_QUEUE", octobot_node_enums_module.SchedulerQueues.DBOS_CLEANUP_QUEUE.value: "DBOS_CLEANUP_QUEUE", + octobot_node_enums_module.SchedulerQueues.GLOBAL_VIEW_QUEUE.value: "GLOBAL_VIEW_QUEUE", } if all(queue_name in registry.queue_info_map for queue_name in queue_bindings): for queue_name, scheduler_attribute in queue_bindings.items(): @@ -115,6 +116,7 @@ def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = None octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE = None octobot_node.scheduler.SCHEDULER.DBOS_CLEANUP_QUEUE = None + octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE = None def init_scheduler(db_file_name: str, application_version: str | None = None): diff --git a/packages/node/tests/scheduler/global_view/test_automation_trigger.py b/packages/node/tests/scheduler/global_view/test_automation_trigger.py new file mode 100644 index 0000000000..aafb77ec2d --- /dev/null +++ b/packages/node/tests/scheduler/global_view/test_automation_trigger.py @@ -0,0 +1,66 @@ +# Drakkar-Software OctoBot-Node + +import mock +import pytest + +import octobot_protocol.models as protocol_models + +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module + + +def _automation_state( + automation_id: str, + *, + account_id: str, + order_ids: list[str], +) -> protocol_models.AutomationState: + return protocol_models.AutomationState( + id=automation_id, + status=protocol_models.WorkflowStatus.RUNNING, + metadata=protocol_models.AutomationMetadata( + name=automation_id, + description=automation_id, + ), + exchange_account_ids=[account_id], + orders=[ + protocol_models.OrderSummary(id=order_id, symbol="BTC/USDT") + for order_id in order_ids + ], + ) + + +@pytest.mark.asyncio +class TestTriggerAccountAutomations: + async def test_triggers_only_matching_automation_sequentially(self): + matching_automation = _automation_state( + "automation-gv-fill", + account_id="acc-sim-1", + order_ids=["gv-order-fill-1"], + ) + non_matching_automation = _automation_state( + "automation-gv-no-trigger", + account_id="acc-sim-1", + order_ids=["gv-order-stays-open"], + ) + trigger_calls: list[str] = [] + + async def _record_trigger(user_id: str, automation_id: str) -> None: + trigger_calls.append(automation_id) + + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock, mock.patch.object( + automation_trigger_module, + "_trigger_automation_and_wait", + side_effect=_record_trigger, + ): + scheduler_api_mock.get_automation_states = mock.AsyncMock( + return_value=[matching_automation, non_matching_automation], + ) + await automation_trigger_module.trigger_account_automations( + "wallet-1", + "acc-sim-1", + {"gv-order-fill-1"}, + ) + assert trigger_calls == ["automation-gv-fill"] diff --git a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py new file mode 100644 index 0000000000..ad01d52bfc --- /dev/null +++ b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py @@ -0,0 +1,47 @@ +# Drakkar-Software OctoBot-Node + +import datetime + +import mock +import pytest + +from tests.scheduler import temp_dbos_scheduler + + +@pytest.mark.asyncio +class TestGlobalViewRefreshWorkflowRunGlobalViewRefresh: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_refreshes_all_wallets_and_accounts_in_parallel_per_wallet( + self, + global_view_workflow_module, + ): + wallet_ids = ["wallet-a"] + accounts = [mock.Mock(id="acc-1"), mock.Mock(id="acc-2")] + account_provider = mock.Mock() + account_provider.list_registered_wallet_ids.return_value = wallet_ids + account_provider.list_items.return_value = accounts + refresh_mock = mock.AsyncMock(return_value=True) + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock, mock.patch.object( + global_view_workflow_module.workflows_retention, + "should_skip_retention_cleanup_on_this_node", + return_value=False, + ), mock.patch.object( + global_view_workflow_module.GlobalViewRefreshWorkflow, + "_refresh_single_account", + refresh_mock, + ): + account_provider_class_mock.instance.return_value = account_provider + result = await global_view_workflow_module.GlobalViewRefreshWorkflow._run_global_view_refresh( + datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + None, + ) + assert result["refreshed_accounts"] == 2 + assert refresh_mock.await_count == 2 diff --git a/packages/sync/octobot_sync/constants.py b/packages/sync/octobot_sync/constants.py index ea4509d05a..b3bc4b9c81 100644 --- a/packages/sync/octobot_sync/constants.py +++ b/packages/sync/octobot_sync/constants.py @@ -42,6 +42,7 @@ EXCHANGE_ACCOUNTS_STATE_VERSION = "1.0.0" USER_ACCOUNTS_AUTH_STATE_VERSION = "1.0.0" USER_ACCOUNTS_TRADING_STATE_VERSION = "1.0.0" +USER_ACCOUNTS_HISTORY_STATE_VERSION = "1.0.0" USER_STRATEGIES_STATE_VERSION = "1.0.0" USER_DATA_STATE_VERSION = "1.0.0" USER_ACTIONS_STATE_VERSION = "1.0.0" diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index 05370b1a2f..1ddd09d511 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -61,6 +61,16 @@ def _file_path(self, storage_key: str) -> pathlib.Path: filename = f"{self._sanitize_storage_key(storage_key)}.json" return self._root / filename + def list_wallet_storage_keys(self) -> list[str]: + """Return wallet user ids that have a persisted state file at the collection root.""" + if not self._root.exists(): + return [] + return sorted( + storage_path.stem + for storage_path in self._root.glob("*.json") + if storage_path.is_file() + ) + def _missing_data_error(self, storage_key: str) -> collection_errors.CollectionNoDataError: return collection_errors.CollectionNoDataError( f"{self.collection} file does not exist for user_id {storage_key}" diff --git a/packages/sync/octobot_sync/sync/collection_providers/__init__.py b/packages/sync/octobot_sync/sync/collection_providers/__init__.py index 3a70e50ea0..66f51a8f49 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/__init__.py +++ b/packages/sync/octobot_sync/sync/collection_providers/__init__.py @@ -17,6 +17,7 @@ import octobot_sync.sync.collection_providers.user_account_authentication_provider as user_account_authentication_provider_module import octobot_sync.sync.collection_providers.user_account_provider as user_account_provider_module +import octobot_sync.sync.collection_providers.user_account_history_provider as user_account_history_provider_module import octobot_sync.sync.collection_providers.user_account_trading_provider as user_account_trading_provider_module import octobot_sync.sync.collection_providers.user_strategy_provider as user_strategy_provider_module @@ -25,11 +26,13 @@ user_account_authentication_provider_module.AccountAuthenticationProvider ) AccountTradingProvider = user_account_trading_provider_module.AccountTradingProvider +AccountHistoryProvider = user_account_history_provider_module.AccountHistoryProvider StrategyProvider = user_strategy_provider_module.StrategyProvider __all__ = [ "AccountProvider", "AccountAuthenticationProvider", "AccountTradingProvider", + "AccountHistoryProvider", "StrategyProvider", ] diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py new file mode 100644 index 0000000000..a7ecc5ca56 --- /dev/null +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py @@ -0,0 +1,38 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + + +import octobot_commons.singleton.singleton_class as singleton_class +import octobot_sync.constants as sync_constants +import octobot_protocol.models as protocol_models +import octobot_sync.enums as sync_enums + +import octobot_sync.sync.collection_backend.single_item_local_collection_provider as single_item_provider + + +class AccountHistoryProvider( + single_item_provider.SingleItemLocalCollectionProvider[protocol_models.PortfolioHistoricalValuesState], + singleton_class.Singleton, +): + """ + Singleton provider for per-account portfolio historical values. + + Each account is stored in its own encrypted file under + ``//.json``. + """ + COLLECTION = sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + STATE_VERSION = sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + STATE_CLASS = protocol_models.PortfolioHistoricalValuesState diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index c7d731ae10..34bf81ea56 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -127,3 +127,6 @@ def update_exchange_config( def delete_exchange_config(self, address: str, config_id: str) -> None: self._delete_item_for_key(address, self.EXCHANGE_CONFIGS_KEY, config_id) + + def list_registered_wallet_ids(self) -> list[str]: + return self._storage.list_wallet_storage_keys() diff --git a/packages/sync/tests/sync/collection_providers/test_account_history_provider.py b/packages/sync/tests/sync/collection_providers/test_account_history_provider.py new file mode 100644 index 0000000000..9dbf1178a1 --- /dev/null +++ b/packages/sync/tests/sync/collection_providers/test_account_history_provider.py @@ -0,0 +1,147 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime +import json + +import mock +import octobot.community.authentication as community_authentication +import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage_module +import octobot_sync.sync.collection_providers.user_account_history_provider as history_provider_module +import octobot_sync.constants as sync_constants +import octobot_protocol.models as protocol_models +import octobot_sync.enums as sync_enums +import octobot_sync.sync.collection_backend.errors as collection_errors + +_TEST_ADDRESS = "0xaaabbbcccddd" +_TEST_ACCOUNT_ID = "acc-42" +_TEST_PRIVATE_KEY = "private-key" + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def _sample_history_state(updated_at: datetime.datetime) -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDT", + values=[ + protocol_models.PortfolioHistoricalValue( + timestamp=updated_at, + total=1000.0, + ) + ], + ), + ) + + +class TestAccountHistoryProviderCollection: + def test_collection_is_USER_ACCOUNTS_HISTORY(self): + assert ( + history_provider_module.AccountHistoryProvider.COLLECTION + == sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + ) + + def test_storage_collection_matches(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + assert provider._storage.collection == sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + + def test_storage_is_single_item_local_collection_storage(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + assert isinstance(provider._storage, single_item_storage_module.SingleItemLocalCollectionStorage) + + +class TestAccountHistoryProviderStateFormat: + def test_state_version_matches_constant(self): + assert ( + history_provider_module.AccountHistoryProvider.STATE_VERSION + == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + ) + + def test_state_class_is_portfolio_historical_values_state(self): + assert ( + history_provider_module.AccountHistoryProvider.STATE_CLASS + is protocol_models.PortfolioHistoricalValuesState + ) + + +class TestAccountHistoryProviderLoadSaveState: + def test_save_and_load_state_per_account_id(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + updated_time = datetime.datetime(2026, 1, 16, tzinfo=datetime.UTC) + updated_state = _sample_history_state(updated_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + loaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, updated_state) + reloaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + assert loaded_state.history.values[0].timestamp == fixture_time + assert reloaded_state.history.values[0].timestamp == updated_time + + def test_load_state_encrypted_reads_persisted_blob(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + encrypted_blob = provider.load_state_encrypted(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + assert "iv" in encrypted_blob + assert "data" in encrypted_blob + + +class TestAccountHistoryProviderSeparateFiles: + def test_accounts_use_separate_files(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + first_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 16, tzinfo=datetime.UTC) + first_state = _sample_history_state(first_time) + second_state = _sample_history_state(second_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, "acc-1", first_state) + provider.save_state(_TEST_ADDRESS, "acc-2", second_state) + loaded_first = provider.load_state(_TEST_ADDRESS, "acc-1") + loaded_second = provider.load_state(_TEST_ADDRESS, "acc-2") + assert loaded_first.history.values[0].timestamp == first_time + assert loaded_second.history.values[0].timestamp == second_time + + +class TestAccountHistoryProviderMissingState: + def test_load_state_raises_when_file_missing(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + with _patch_wallet(): + try: + provider.load_state(_TEST_ADDRESS, "missing-account") + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + + +class TestAccountHistoryProviderWalletKey: + def test_save_requires_wallet_private_key(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + auth = mock.Mock() + auth.get_wallet_by_user_id.side_effect = KeyError("wallet not found") + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ): + try: + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + raise AssertionError("Expected KeyError from missing wallet") + except KeyError: + pass From adcbc6c478a2f4d019f31f7d6d8ac1477a87eb84 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Tue, 11 Aug 2026 11:57:42 +0200 Subject: [PATCH 02/10] [Node] fix tests & reduce logs --- octobot/community/authentication.py | 7 +- .../configuration/configuration.py | 5 +- .../profiles/profile_storage.py | 10 +- .../tests/profiles/test_profile_storage.py | 24 + .../flow/octobot_flow/entities/__init__.py | 2 - .../entities/global_view/__init__.py | 2 - .../global_view_account_context.py | 1 + .../global_view_refreshed_elements.py | 60 -- .../jobs/global_view_account_job.py | 36 +- .../octobot_flow/logic/accounts/__init__.py | 2 + .../accounts/account_state_persistence.py | 115 +++- .../logic/configuration/__init__.py | 2 - .../configuration/profile_data_factory.py | 15 +- .../logic/exchange/orders/__init__.py | 2 - .../exchange/orders/order_change_detection.py | 19 +- .../logic/exchange/portfolio/__init__.py | 5 - .../exchange/portfolio/valuation_unit.py | 14 - .../simulated_order_fill_detector.py | 61 ++ .../simulator/simulated_portfolio_seeder.py | 7 +- .../global_view/exchange_account_refresh.py | 232 +++++++- .../global_view/global_view_persistence.py | 15 +- .../exchange/tickers_repository.py | 33 ++ .../test_global_view_account_refresh.py | 165 ++++-- .../test_account_state_persistence.py | 272 ++++++++- .../test_profile_data_factory.py | 7 - .../exchange/portfolio/test_valuation_unit.py | 25 - .../test_simulated_order_fill_detector.py | 128 +++++ .../test_simulated_portfolio_seeder.py | 23 +- .../logic/global_view/portfolio_test_util.py | 69 +++ .../test_exchange_account_refresh.py | 519 ++++++++++++++++++ .../test_global_view_account_job.py | 307 +++++++++-- .../test_global_view_persistence.py | 189 +++++++ .../test_global_view_refreshed_elements.py | 47 -- .../exchange/test_tickers_repository.py | 33 ++ packages/node/octobot_node/constants.py | 2 +- .../global_view/automation_trigger.py | 78 ++- .../global_view/global_view_executor.py | 6 + .../account/create_account.py | 13 + .../util/account_state_updater.py | 45 +- .../workflows/automation_workflow.py | 53 +- .../workflows/dbos_cleanup_workflow.py | 2 - .../workflows/global_view_workflow.py | 92 +++- ...t_emit_and_copy_grid_automation_signals.py | 16 +- .../test_global_view_workflow.py | 39 +- ...heck_and_stop_dca_2_evaluators_workflow.py | 14 + ...heck_and_stop_dca_no_evaluator_workflow.py | 2 + ...default_config_octobot_process_workflow.py | 33 ++ ...test_start_check_and_stop_grid_workflow.py | 8 +- ...nd_stop_grid_workflow_hollaex_earncurve.py | 2 + .../util/global_view_workflow.py | 164 +++--- .../functional_tests/util/workflow_common.py | 31 ++ .../global_view/test_automation_trigger.py | 90 +++ .../global_view/test_global_view_workflow.py | 159 +++++- .../node/tests/scheduler/test_schedules.py | 234 ++++++-- .../account/test_create_account.py | 14 + .../util/test_account_state_updater.py | 17 - .../workflows/test_automation_workflow.py | 127 ++++- .../workflows/test_dbos_cleanup_workflow.py | 3 - .../trading/octobot_trading/api/__init__.py | 2 + .../trading/octobot_trading/api/portfolio.py | 11 + .../octobot_trading/exchange_channel.py | 13 + .../ticker/channel/ticker_updater.py | 55 +- .../exchange_data/ticker/ticker_cache.py | 35 +- .../connectors/ccxt/ccxt_client_util.py | 24 +- .../connectors/ccxt/ccxt_connector.py | 158 +++--- .../exchanges/exchange_channels.py | 23 +- .../exchanges/exchange_manager.py | 3 + .../exchanges/traders/trader.py | 7 +- .../personal_data/orders/protocol.py | 6 +- .../personal_data/portfolios/portfolio.py | 9 +- .../portfolios/portfolio_manager.py | 13 +- .../personal_data/portfolios/sub_portfolio.py | 6 +- .../portfolios/value_converter.py | 3 +- .../trading/octobot_trading/util/__init__.py | 10 + .../octobot_trading/util/config_util.py | 2 +- .../util/protocol_trading_mapping.py | 29 + packages/trading/tests/api/test_portfolio.py | 24 +- .../exchange_data/ticker/test_ticker_cache.py | 37 ++ .../ticker/test_ticker_updater.py | 82 +++ .../connectors/ccxt/test_ccxt_client_util.py | 45 ++ .../connectors/ccxt/test_ccxt_connector.py | 42 ++ .../test_trader_initialize_impl_logging.py | 39 ++ .../personal_data/orders/test_protocol.py | 35 ++ .../portfolios/test_portfolio.py | 29 + .../portfolios/test_portfolio_manager.py | 32 ++ .../portfolios/test_value_converter.py | 32 ++ .../tests/test_exchange_channel_producer.py | 72 +++ .../trading/tests/util/test_config_util.py | 43 ++ .../util/test_protocol_trading_mapping.py | 45 ++ .../community/test_authentication.py | 33 ++ 90 files changed, 3906 insertions(+), 790 deletions(-) delete mode 100644 packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py delete mode 100644 packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py delete mode 100644 packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py create mode 100644 packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py delete mode 100644 packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py create mode 100644 packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py create mode 100644 packages/flow/tests/logic/global_view/portfolio_test_util.py create mode 100644 packages/flow/tests/logic/global_view/test_exchange_account_refresh.py create mode 100644 packages/flow/tests/logic/global_view/test_global_view_persistence.py delete mode 100644 packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py create mode 100644 packages/flow/tests/repositories/exchange/test_tickers_repository.py create mode 100644 packages/trading/octobot_trading/util/protocol_trading_mapping.py create mode 100644 packages/trading/tests/exchange_data/ticker/test_ticker_updater.py create mode 100644 packages/trading/tests/exchanges/traders/test_trader_initialize_impl_logging.py create mode 100644 packages/trading/tests/personal_data/orders/test_protocol.py create mode 100644 packages/trading/tests/test_exchange_channel_producer.py create mode 100644 packages/trading/tests/util/test_protocol_trading_mapping.py diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index add3efcce5..4964ab2ef3 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -114,6 +114,7 @@ class CommunityAuthentication(authentication.Authenticator): def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singleton=True): super().__init__(use_as_singleton=use_as_singleton) + self._use_as_singleton = use_as_singleton self.config: typing.Optional[commons_configuration.Configuration] = config self.backend_url: str = backend_url or identifiers_provider.IdentifiersProvider.BACKEND_URL self.backend_key: str = backend_key or identifiers_provider.IdentifiersProvider.BACKEND_KEY @@ -600,7 +601,8 @@ def clear_local_data_if_necessary(self): self._clear_bot_scoped_config() async def stop(self): - self.logger.debug("Stopping ...") + if self._use_as_singleton: + self.logger.debug("Stopping ...") if self._fetch_account_task is not None and not self._fetch_account_task.done(): self._fetch_account_task.cancel() await self.supabase_client.aclose() @@ -615,7 +617,8 @@ async def stop(self): await session.content_client.close() await session.account_client.close() self._dk_sessions.clear() - self.logger.debug("Stopped") + if self._use_as_singleton: + self.logger.debug("Stopped") def _update_supports(self, resp_status, json_data): if resp_status == 200: diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index 405ee81065..bf42dce383 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -109,15 +109,16 @@ def load_profiles_if_possible_and_necessary(self) -> None: self.load_profiles() self.select_profile(self._get_selected_profile()) - def select_profile(self, profile_id) -> None: + def select_profile(self, profile_id, *, init_tentacles_setup: bool = True) -> None: """ Sets self.profile using its profile_id :param profile_id: id of the profile to select + :param init_tentacles_setup: when False, skip tentacles setup initialization on activation :return: None """ self.config[commons_constants.CONFIG_PROFILE] = profile_id self.profile = self.profile_by_id[profile_id] - self.profile_storage.activate_profile(self.profile) + self.profile_storage.activate_profile(self.profile, init_tentacles_setup=init_tentacles_setup) self.logger.info(f"Using {self.profile.name} profile.") self._generate_config_from_user_config_and_profile() diff --git a/packages/commons/octobot_commons/profiles/profile_storage.py b/packages/commons/octobot_commons/profiles/profile_storage.py index 364b1f3841..99384174f0 100644 --- a/packages/commons/octobot_commons/profiles/profile_storage.py +++ b/packages/commons/octobot_commons/profiles/profile_storage.py @@ -208,10 +208,16 @@ def duplicate_profile( clone.bind_profile_storage(self) return clone - def activate_profile(self, profile: profile_module.Profile) -> None: + def activate_profile( + self, + profile: profile_module.Profile, + *, + init_tentacles_setup: bool = True, + ) -> None: profile.bind_profile_storage(self) self._apply_child_overlay_config(profile) - profile.init_tentacles_setup_config() + if init_tentacles_setup: + profile.init_tentacles_setup_config() def save_active_profile( self, diff --git a/packages/commons/tests/profiles/test_profile_storage.py b/packages/commons/tests/profiles/test_profile_storage.py index 9bb5142c03..e54663cdf5 100644 --- a/packages/commons/tests/profiles/test_profile_storage.py +++ b/packages/commons/tests/profiles/test_profile_storage.py @@ -17,6 +17,7 @@ import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_module import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module import octobot_commons.profiles.profile_data_import as profile_data_import_module import octobot_sync.sync.collection_backend.errors as collection_errors @@ -802,3 +803,26 @@ def _assert_imported_profile_metadata( profile_data_import_module.IMPORTED_PROFILES_DEFAULT_EXTRA_BACKTESTING_TIMEFRAME ] assert profile.profile_id + + +class TestProfileStorageActivateProfileInitTentaclesSetup: + def test_init_tentacles_setup_runs_when_flag_true_even_if_setup_pre_bound(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + pre_bound_setup = mock.Mock() + profile.bind_tentacles_setup_config(pre_bound_setup) + profile_storage = profile_storage_module.ProfileStorage(None, None) + with mock.patch.object(profile, "init_tentacles_setup_config") as init_mock: + profile_storage.activate_profile(profile, init_tentacles_setup=True) + init_mock.assert_called_once() + + def test_init_tentacles_setup_skipped_when_flag_false(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + pre_bound_setup = mock.Mock() + profile.bind_tentacles_setup_config(pre_bound_setup) + profile_storage = profile_storage_module.ProfileStorage(None, None) + with mock.patch.object(profile, "init_tentacles_setup_config") as init_mock: + profile_storage.activate_profile(profile, init_tentacles_setup=False) + init_mock.assert_not_called() + assert profile.tentacles_setup_config is pre_bound_setup diff --git a/packages/flow/octobot_flow/entities/__init__.py b/packages/flow/octobot_flow/entities/__init__.py index 8efb6ffd74..fd61b0f372 100644 --- a/packages/flow/octobot_flow/entities/__init__.py +++ b/packages/flow/octobot_flow/entities/__init__.py @@ -39,7 +39,6 @@ GlobalViewAccountContext, ExchangeAccountRefreshResult, GlobalViewAccountRefreshResult, - GlobalViewRefreshedElements, ) __all__ = [ "AccountElements", @@ -74,5 +73,4 @@ "GlobalViewAccountContext", "ExchangeAccountRefreshResult", "GlobalViewAccountRefreshResult", - "GlobalViewRefreshedElements", ] diff --git a/packages/flow/octobot_flow/entities/global_view/__init__.py b/packages/flow/octobot_flow/entities/global_view/__init__.py index 44ef507b04..f4f14072a9 100644 --- a/packages/flow/octobot_flow/entities/global_view/__init__.py +++ b/packages/flow/octobot_flow/entities/global_view/__init__.py @@ -1,11 +1,9 @@ from octobot_flow.entities.global_view.global_view_account_context import GlobalViewAccountContext from octobot_flow.entities.global_view.exchange_account_refresh_result import ExchangeAccountRefreshResult from octobot_flow.entities.global_view.global_view_account_refresh_result import GlobalViewAccountRefreshResult -from octobot_flow.entities.global_view.global_view_refreshed_elements import GlobalViewRefreshedElements __all__ = [ "GlobalViewAccountContext", "ExchangeAccountRefreshResult", "GlobalViewAccountRefreshResult", - "GlobalViewRefreshedElements", ] diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py index c02fcaef91..09d797f278 100644 --- a/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py @@ -14,3 +14,4 @@ class GlobalViewAccountContext: exchange_config: protocol_models.ExchangeConfig trading_type: protocol_models.TradingType auth_details: exchange_data_module.ExchangeAuthDetails + has_bound_automation: bool = False diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py b/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py deleted file mode 100644 index 9ace068dd7..0000000000 --- a/packages/flow/octobot_flow/entities/global_view/global_view_refreshed_elements.py +++ /dev/null @@ -1,60 +0,0 @@ -# Drakkar-Software OctoBot-Flow -# Copyright (c) Drakkar-Software, All rights reserved. - -import copy - -import octobot_trading.constants as trading_constants -import octobot_trading.enums as trading_enums -import octobot_trading.exchanges.util.exchange_data as exchange_data_import - - -class GlobalViewRefreshedElements: - def __init__(self, exchange_data: exchange_data_import.ExchangeData, save_copy: bool = True): - self.open_orders: list[dict] = ( - copy.deepcopy(exchange_data.orders_details.open_orders) if save_copy else list( - exchange_data.orders_details.open_orders or [] - ) - ) if exchange_data.orders_details.open_orders else [] - self.portfolio: dict[str, dict] = ( - copy.deepcopy(exchange_data.portfolio_details.content) if save_copy else dict( - exchange_data.portfolio_details.content or {} - ) - ) if exchange_data.portfolio_details else {} - - def confirmed_change( - self, exchange_data: exchange_data_import.ExchangeData - ) -> bool: - return self._confirmed_change(GlobalViewRefreshedElements(exchange_data, save_copy=False)) - - def _confirmed_change( - self, other_global_view_refreshed_elements: "GlobalViewRefreshedElements" - ) -> bool: - if self._get_orders_signature(self.open_orders) != self._get_orders_signature( - other_global_view_refreshed_elements.open_orders - ): - return True - if self._get_portfolio_signature(self.portfolio) != self._get_portfolio_signature( - other_global_view_refreshed_elements.portfolio - ): - return True - return False - - def _get_orders_signature(self, orders: list[dict]) -> str: - return ",".join(sorted([ - self._get_order_signature(order) for order in orders - ])) - - def _get_order_signature(self, order: dict) -> str: - try: - origin_value = order[trading_constants.STORAGE_ORIGIN_VALUE] - except KeyError: - return "" - return ( - f"{origin_value.get(trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value, '')}" - f"_{origin_value.get(trading_enums.ExchangeConstantsOrderColumns.FILLED.value, 0)}" - ) - - def _get_portfolio_signature(self, portfolio: dict[str, dict]) -> str: - return ",".join(sorted([ - f"{asset}:{value}" for asset, value in portfolio.items() - ])) diff --git a/packages/flow/octobot_flow/jobs/global_view_account_job.py b/packages/flow/octobot_flow/jobs/global_view_account_job.py index 8c44255193..16f0eeb917 100644 --- a/packages/flow/octobot_flow/jobs/global_view_account_job.py +++ b/packages/flow/octobot_flow/jobs/global_view_account_job.py @@ -5,11 +5,13 @@ import octobot_tentacles_manager.api as tentacles_manager_api import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_flow.entities import octobot_flow.errors import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory_module +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module @@ -47,6 +49,14 @@ async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: self.user_id, account.id, ) + previous_open_orders = account_state_persistence_module.load_previous_open_orders( + self.user_id, + account.id, + ) + persist_open_orders = ( + not self.context.has_bound_automation + and bool(previous_open_orders) + ) profile_data = profile_data_factory_module.profile_data_for_account( account, self.context.exchange_account, @@ -56,7 +66,7 @@ async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: ) exchange_data = exchange_data_module.exchange_data_factory( exchange_internal_name=self.context.exchange_config.exchange, - exchange_type=profile_data_factory_module.exchange_type_from_trading_type(self.context.trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(self.context.trading_type).value, sandboxed=self.context.exchange_config.sandboxed, auth_details=self.context.auth_details, ) @@ -69,11 +79,24 @@ async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: ) as exchange_manager: if account.is_simulated: simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) - exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( - exchange_manager, - self.context.trading_type, - previous_open_order_exchange_ids, - ) + exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + self.context.trading_type, + previous_open_order_exchange_ids, + is_simulated=True, + previous_open_orders=previous_open_orders, + ) + else: + open_order_symbols = simulated_order_fill_detector_module.symbols_from_open_orders( + previous_open_orders, + ) + exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + self.context.trading_type, + previous_open_order_exchange_ids, + fetch_open_orders=bool(open_order_symbols), + open_order_symbols=open_order_symbols, + ) refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( self.user_id, @@ -84,5 +107,6 @@ async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: self.user_id, account.id, refresh_result, + persist_open_orders=persist_open_orders, ) return refresh_result diff --git a/packages/flow/octobot_flow/logic/accounts/__init__.py b/packages/flow/octobot_flow/logic/accounts/__init__.py index f1d9c11add..05166f7e2e 100644 --- a/packages/flow/octobot_flow/logic/accounts/__init__.py +++ b/packages/flow/octobot_flow/logic/accounts/__init__.py @@ -4,6 +4,7 @@ load_portfolio_history_state, load_previous_open_order_exchange_ids, persist_account_trading, + persist_account_trading_orders, ) __all__ = [ @@ -12,4 +13,5 @@ "load_portfolio_history_state", "load_previous_open_order_exchange_ids", "persist_account_trading", + "persist_account_trading_orders", ] diff --git a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py index 0d9f5efb12..e3c0986c33 100644 --- a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py +++ b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py @@ -2,16 +2,22 @@ # Copyright (c) Drakkar-Software, All rights reserved. import datetime +import typing +import octobot_commons.logging as octobot_commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_protocol.models as protocol_models import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums import octobot_trading.personal_data.orders.protocol as orders_protocol import octobot_trading.personal_data.positions.protocol as positions_protocol import octobot_trading.personal_data.trades.protocol as trades_protocol import octobot_trading.personal_data.trades.trades_util as trades_util +import octobot_flow.entities import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module @@ -24,8 +30,6 @@ def load_previous_open_order_exchange_ids(user_id: str, account_id: str) -> set[ ) except collection_errors.CollectionNoDataError: return set() - except Exception: - return set() account_trading = trading_state.account_trading if account_trading is None or account_trading.orders is None: return set() @@ -34,6 +38,27 @@ def load_previous_open_order_exchange_ids(user_id: str, account_id: str) -> set[ ) +def load_previous_open_orders(user_id: str, account_id: str) -> list[dict]: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + return [] + account_trading = trading_state.account_trading + if account_trading is None or account_trading.orders is None: + return [] + return [ + { + trading_constants.STORAGE_ORIGIN_VALUE: orders_protocol.exchange_columns_dict_from_protocol_order( + protocol_order + ), + } + for protocol_order in account_trading.orders + ] + + def load_portfolio_history_state( user_id: str, account_id: str, @@ -75,6 +100,20 @@ def build_portfolio_history_state( ) +def _normalize_order_for_protocol(order: dict) -> dict: + order_details = order.get(trading_constants.STORAGE_ORIGIN_VALUE, order) + order_id = order_details.get(trading_enums.ExchangeConstantsOrderColumns.ID.value) + exchange_id = order_details.get(trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value) + if not order_id: + if exchange_id: + order_details[trading_enums.ExchangeConstantsOrderColumns.ID.value] = exchange_id + else: + raise ValueError( + f"Order is missing both id and exchange_id: {sorted(order_details.keys())}" + ) + return order + + def persist_account_trading( user_id: str, account_id: str, @@ -88,14 +127,13 @@ def persist_account_trading( account_id, ) except collection_errors.CollectionNoDataError: - trading_state = protocol_models.AccountTradingState( - version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, - account_trading=protocol_models.AccountTrading( - updated_at=datetime.datetime.now(datetime.UTC), - ), - ) + # Abnormal: AccountTradingState must be created with the account, not invent here. + raise account_trading = trading_state.account_trading - account_trading.orders = [orders_protocol.to_protocol_order(order) for order in orders] or None + account_trading.orders = [ + orders_protocol.to_protocol_order(_normalize_order_for_protocol(order)) + for order in orders + ] or None account_trading.positions = [ positions_protocol.to_protocol_position(position) for position in positions ] or None @@ -113,3 +151,62 @@ def persist_account_trading( account_id, trading_state, ) + + +def persist_account_trading_orders( + user_id: str, + account_id: str, + orders: list[dict], +) -> None: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + # Abnormal: AccountTradingState must be created with the account, not invent here. + raise + account_trading = trading_state.account_trading + account_trading.orders = [ + orders_protocol.to_protocol_order(_normalize_order_for_protocol(order)) + for order in orders + ] or None + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + user_id, + account_id, + trading_state, + ) + + +def persist_account_trading_from_iteration_state( + user_id: typing.Optional[str], + iteration_state: typing.Optional[dict], +) -> None: + if user_id is None or iteration_state is None: + return + automation_state = octobot_flow.entities.AutomationState.from_dict(iteration_state) + exchange_account_elements = automation_state.automation.exchange_account_elements + exchange_account_details = automation_state.exchange_account_details + if exchange_account_elements is None or exchange_account_details is None: + return + exchange_account_id = exchange_account_details.exchange_details.exchange_account_id + if not exchange_account_id: + return + try: + persist_account_trading( + user_id, + exchange_account_id, + list(exchange_account_elements.orders.open_orders), + list(exchange_account_elements.trades), + [ + position_details.position + for position_details in exchange_account_elements.positions + ], + ) + except wallet_backend_errors.WalletNotFoundError: + # Trading collections are wallet-scoped; skip until the wallet is registered locally. + octobot_commons_logging.get_logger(__name__).warning( + "Skipping account trading persistence for wallet %s: wallet not registered", + user_id, + ) diff --git a/packages/flow/octobot_flow/logic/configuration/__init__.py b/packages/flow/octobot_flow/logic/configuration/__init__.py index a2dec066fc..64a2108727 100644 --- a/packages/flow/octobot_flow/logic/configuration/__init__.py +++ b/packages/flow/octobot_flow/logic/configuration/__init__.py @@ -4,7 +4,6 @@ create_profile_data, infer_reference_market, profile_data_for_account, - exchange_type_from_trading_type, ) __all__ = [ @@ -13,5 +12,4 @@ "create_profile_data", "infer_reference_market", "profile_data_for_account", - "exchange_type_from_trading_type", ] diff --git a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py index ddff947ae0..f66e174f9d 100644 --- a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py +++ b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py @@ -4,24 +4,13 @@ import octobot_commons.constants import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_flow.entities import tentacles.Meta.Keywords.scripting_library as scripting_library -_TRADING_TYPE_TO_EXCHANGE_TYPE: dict[protocol_models.TradingType, trading_enums.ExchangeTypes] = { - protocol_models.TradingType.SPOT: trading_enums.ExchangeTypes.SPOT, - protocol_models.TradingType.FUTURES: trading_enums.ExchangeTypes.FUTURE, - protocol_models.TradingType.OPTIONS: trading_enums.ExchangeTypes.OPTION, - protocol_models.TradingType.MARGIN: trading_enums.ExchangeTypes.MARGIN, -} - - -def exchange_type_from_trading_type(trading_type: protocol_models.TradingType) -> str: - return _TRADING_TYPE_TO_EXCHANGE_TYPE[trading_type].value - - def profile_data_for_account( account: protocol_models.Account, exchange_account: protocol_models.ExchangeAccount, @@ -34,7 +23,7 @@ def profile_data_for_account( exchanges=[ profile_data_import.ExchangeData( internal_name=exchange_config.exchange, - exchange_type=exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, exchange_account_id=exchange_account.remote_account_id or account.id, sandboxed=exchange_config.sandboxed, ) diff --git a/packages/flow/octobot_flow/logic/exchange/orders/__init__.py b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py index 4831d16378..2a048c3f01 100644 --- a/packages/flow/octobot_flow/logic/exchange/orders/__init__.py +++ b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py @@ -2,12 +2,10 @@ detect_changed_order_ids, open_order_exchange_ids_from_open_orders, open_order_exchange_ids_from_protocol_orders, - open_order_to_storage_dict, ) __all__ = [ "detect_changed_order_ids", "open_order_exchange_ids_from_open_orders", "open_order_exchange_ids_from_protocol_orders", - "open_order_to_storage_dict", ] diff --git a/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py index 863c3b8d17..60bfa4985f 100644 --- a/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py +++ b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py @@ -8,7 +8,7 @@ def detect_changed_order_ids( previous_open_order_exchange_ids: set[str], - current_open_orders: list, + current_open_orders: list[dict], ) -> set[str]: if not previous_open_order_exchange_ids: return set() @@ -28,27 +28,14 @@ def open_order_exchange_ids_from_protocol_orders( } -def open_order_exchange_ids_from_open_orders(open_orders: list) -> set[str]: +def open_order_exchange_ids_from_open_orders(open_orders: list[dict]) -> set[str]: exchange_ids: set[str] = set() order_columns = trading_enums.ExchangeConstantsOrderColumns for open_order in open_orders: - order_dict = open_order_to_storage_dict(open_order) - inner_order = order_dict.get(trading_constants.STORAGE_ORIGIN_VALUE, order_dict) - if not isinstance(inner_order, dict): - inner_order = order_dict + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) exchange_id = inner_order.get(order_columns.EXCHANGE_ID.value) or inner_order.get( order_columns.ID.value ) if exchange_id is not None: exchange_ids.add(str(exchange_id)) return exchange_ids - - -def open_order_to_storage_dict(open_order) -> dict: - if isinstance(open_order, dict): - return open_order - if hasattr(open_order, "to_dict"): - return open_order.to_dict() - if hasattr(open_order, "order"): - return open_order.order - raise TypeError(f"Unsupported open order type: {type(open_order).__name__}") diff --git a/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py b/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py deleted file mode 100644 index b3e746f350..0000000000 --- a/packages/flow/octobot_flow/logic/exchange/portfolio/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from octobot_flow.logic.exchange.portfolio.valuation_unit import resolve_portfolio_valuation_unit - -__all__ = [ - "resolve_portfolio_valuation_unit", -] diff --git a/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py b/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py deleted file mode 100644 index 5ece9976e9..0000000000 --- a/packages/flow/octobot_flow/logic/exchange/portfolio/valuation_unit.py +++ /dev/null @@ -1,14 +0,0 @@ -# Drakkar-Software OctoBot-Flow -# Copyright (c) Drakkar-Software, All rights reserved. - -import octobot_commons.constants as commons_constants -import octobot_trading.enums as trading_enums - - -def resolve_portfolio_valuation_unit(exchange_manager) -> str: - quote_currency = exchange_manager.exchange.get_option_value( - trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY - ) - if quote_currency: - return str(quote_currency) - return commons_constants.DEFAULT_REFERENCE_MARKET diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py new file mode 100644 index 0000000000..b54da9b52e --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py @@ -0,0 +1,61 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.logging as commons_logging +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + + +def is_simulated_order_filled(order_price: float, market_price: float, trigger_above: bool) -> bool: + if trigger_above: + return market_price >= order_price + return market_price <= order_price + + +def resolve_trigger_above(inner_order: dict) -> bool: + order_columns = trading_enums.ExchangeConstantsOrderColumns + if order_columns.TRIGGER_ABOVE.value in inner_order: + return bool(inner_order[order_columns.TRIGGER_ABOVE.value]) + order_side = inner_order.get(order_columns.SIDE.value) + if order_side == trading_enums.TradeOrderSide.SELL.value: + return True + return False + + +def resolve_simulated_open_orders( + open_orders: list[dict], + ticker_close_by_symbol: dict[str, float], +) -> list[dict]: + still_open_orders: list[dict] = [] + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + symbol = inner_order.get(order_columns.SYMBOL.value) + order_price = inner_order.get(order_columns.PRICE.value) + if symbol is None or order_price is None: + still_open_orders.append(open_order) + continue + if symbol not in ticker_close_by_symbol: + commons_logging.get_logger(__name__).warning( + "No ticker close for %s, keeping simulated order open", + symbol, + ) + still_open_orders.append(open_order) + continue + trigger_above = resolve_trigger_above(inner_order) + market_price = ticker_close_by_symbol[symbol] + if is_simulated_order_filled(float(order_price), market_price, trigger_above): + continue + still_open_orders.append(open_order) + return still_open_orders + + +def symbols_from_open_orders(open_orders: list[dict]) -> list[str]: + order_columns = trading_enums.ExchangeConstantsOrderColumns + symbols: set[str] = set() + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + symbol = inner_order.get(order_columns.SYMBOL.value) + if symbol is not None: + symbols.add(str(symbol)) + return sorted(symbols) diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py index 7c24db10d6..f6f5618b8f 100644 --- a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py +++ b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py @@ -3,7 +3,6 @@ import octobot_commons.constants as commons_constants import octobot_protocol.models as protocol_models -import octobot_trading.api as trading_api def seed_simulated_portfolio(exchange_manager, account: protocol_models.Account) -> None: @@ -16,4 +15,8 @@ def seed_simulated_portfolio(exchange_manager, account: protocol_models.Account) commons_constants.PORTFOLIO_TOTAL: float(detailed_asset.total), } if portfolio_content: - trading_api.set_simulated_portfolio_initial_config(exchange_manager, portfolio_content) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio( + portfolio_content, + update_available_funds_from_open_orders=True, + ) diff --git a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py index 4b445c722d..01a36815dc 100644 --- a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py +++ b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py @@ -1,41 +1,108 @@ # Drakkar-Software OctoBot-Flow # Copyright (c) Drakkar-Software, All rights reserved. +import asyncio +import decimal + import octobot_commons.constants as commons_constants +import octobot_commons.logging as octobot_commons_logging import octobot_commons.timestamp_util as timestamp_util import octobot_protocol.models as protocol_models import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums import octobot_trading.errors as trading_errors +import octobot_trading.exchanges.util.exchange_util as exchange_util_module +import octobot_trading.personal_data.portfolios.portfolio_util as portfolio_util_module import octobot_trading.personal_data.portfolios.protocol as portfolios_protocol import octobot_flow.entities import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module -import octobot_flow.logic.exchange.portfolio.valuation_unit as valuation_unit_module +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module + +def _get_logger() -> octobot_commons_logging.BotLogger: + return octobot_commons_logging.get_logger("ExchangeAccountRefresh") async def refresh_exchange_account( exchange_manager, trading_type: protocol_models.TradingType, previous_open_order_exchange_ids: set[str], + *, + is_simulated: bool = False, + previous_open_orders: list[dict] | None = None, + fetch_open_orders: bool = True, + open_order_symbols: list[str] | None = None, ) -> octobot_flow.entities.ExchangeAccountRefreshResult: - # Step: fetch balance and open orders from the exchange manager. - await exchange_manager.exchange.get_balance() - open_orders = await exchange_manager.exchange.get_open_orders() + await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) + + # Step: fetch balance and open orders from the exchange (real accounts only). + tickers: dict[str, dict] | None = None + if is_simulated: + if previous_open_orders is None: + raise ValueError("previous_open_orders is required for simulated account refresh.") + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_unit = trading_api.resolve_portfolio_valuation_unit(exchange_manager) + order_symbols = simulated_order_fill_detector_module.symbols_from_open_orders(previous_open_orders) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + symbols_to_fetch = sorted(set(order_symbols) | set(valuation_symbols)) + tickers = await _fetch_tickers(exchange_manager, symbols_to_fetch) + ticker_close_by_symbol = _ticker_close_by_symbol_from_tickers(tickers) + open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + previous_open_orders, + ticker_close_by_symbol, + ) + elif not fetch_open_orders: + await _fetch_and_apply_exchange_balance(exchange_manager) + open_orders = [] + else: + await _fetch_and_apply_exchange_balance(exchange_manager) + open_orders = await _fetch_open_orders_for_symbols( + exchange_manager, + open_order_symbols or [], + ) trades: list[dict] = [] positions: list[dict] = [] # Step: resolve valuation unit and portfolio total in that currency. - valuation_unit = valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) + if not is_simulated: + valuation_unit = trading_api.resolve_portfolio_valuation_unit(exchange_manager) portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager if portfolio_manager is not None: portfolio_manager.reference_market = valuation_unit - handle_mark_price_update = getattr(portfolio_manager, "handle_mark_price_update", None) - if handle_mark_price_update is not None: - await handle_mark_price_update() + if is_simulated: + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + simulated_valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + await _refresh_portfolio_valuation( + exchange_manager, + valuation_unit, + tickers=tickers, + valuation_symbols=simulated_valuation_symbols, + ) + else: + await _refresh_portfolio_valuation(exchange_manager, valuation_unit) portfolio_total = trading_api.get_current_portfolio_value(exchange_manager) # Step: build protocol assets and historical snapshot payload. portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + balance_summary = octobot_commons_logging.get_private_placeholder_if_necessary( + portfolio_util_module.get_balance_summary(portfolio_content, use_exchange_format=False) + ) + _get_logger().info( + "Fetched [%s] full [%s] portfolio: %s", + exchange_manager.exchange_name, + "simulated" if is_simulated else "real", + balance_summary, + ) detailed_assets = portfolios_protocol.to_protocol_assets(portfolio_content) historical_assets = _historical_assets_from_portfolio( exchange_manager, @@ -61,21 +128,160 @@ async def refresh_exchange_account( previous_open_order_exchange_ids, open_orders, ) - open_order_dicts = [ - order_change_detection_module.open_order_to_storage_dict(open_order) - for open_order in open_orders - ] return octobot_flow.entities.ExchangeAccountRefreshResult( assets=assets_for_trading_type, portfolio_snapshot=portfolio_snapshot, valuation_unit=valuation_unit, - open_orders=open_order_dicts, + open_orders=open_orders, trades=trades, positions=positions, changed_order_ids=changed_order_ids, ) +async def _fetch_and_apply_exchange_balance(exchange_manager) -> None: + balance = await exchange_manager.exchange.get_balance() + non_empty_balance = portfolio_util_module.filter_empty_values(balance) + decimal_balance = portfolio_util_module.parse_decimal_portfolio(non_empty_balance) + await exchange_manager.exchange_personal_data.handle_portfolio_update( + decimal_balance, + should_notify=False, + ) + + +async def _fetch_tickers(exchange_manager, symbols: list[str]) -> dict[str, dict]: + if not symbols: + return {} + tickers_repository = tickers_repository_module.TickersRepository( + exchange_manager, + known_automations=[], + fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), + ) + return await tickers_repository.fetch_tickers(symbols) + + +def _ticker_close_by_symbol_from_tickers(tickers: dict[str, dict]) -> dict[str, float]: + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol: dict[str, float] = {} + for symbol, ticker in tickers.items(): + close_price = ticker.get(close_column) + if close_price is not None: + ticker_close_by_symbol[symbol] = float(close_price) + return ticker_close_by_symbol + + +async def _refresh_portfolio_valuation( + exchange_manager, + valuation_unit: str, + *, + tickers: dict[str, dict] | None = None, + valuation_symbols: list[str] | None = None, +) -> None: + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is None: + return + if valuation_symbols is None: + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + if tickers is None and valuation_symbols: + tickers = await _fetch_tickers(exchange_manager, valuation_symbols) + if tickers: + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + symbols_to_apply = valuation_symbols if valuation_symbols is not None else list(tickers) + for symbol in symbols_to_apply: + ticker = tickers.get(symbol) + if not ticker: + continue + close_price = ticker.get(close_column) + if close_price is None: + continue + mark_price = decimal.Decimal(str(close_price)) + portfolio_manager.portfolio_value_holder.value_converter.update_last_price(symbol, mark_price) + if exchange_manager.symbol_exists(symbol): + exchange_manager.get_symbol_data(symbol).handle_ticker_update(ticker) + portfolio_manager.portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values( + init_price_fetchers=False, + ) + + +def _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content: dict, + valuation_unit: str, +) -> list[str]: + valuation_symbols: set[str] = set() + bridge_quote_currencies = [] + if valuation_unit in commons_constants.USD_LIKE_COINS: + bridge_quote_currencies = [ + quote_currency + for quote_currency in commons_constants.USD_LIKE_COINS + if quote_currency != valuation_unit + ] + for currency, symbol_balance in portfolio_content.items(): + total_holdings = float(symbol_balance.get(commons_constants.PORTFOLIO_TOTAL) or 0) + if total_holdings == 0 or currency == valuation_unit: + continue + direct_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + currency, + valuation_unit, + ) + if direct_symbol is not None: + valuation_symbols.add(direct_symbol) + continue + for bridge_quote in bridge_quote_currencies: + asset_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + currency, + bridge_quote, + ) + if asset_symbol is None: + continue + valuation_symbols.add(asset_symbol) + if bridge_quote != valuation_unit: + bridge_symbol, _is_reversed_bridge_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + bridge_quote, + valuation_unit, + ) + if bridge_symbol is not None: + valuation_symbols.add(bridge_symbol) + break + return list(valuation_symbols) + + +async def _get_open_orders_for_symbol(exchange_manager, symbol: str) -> list[dict]: + try: + return await exchange_manager.exchange.get_open_orders(symbol=symbol) + except trading_errors.UnSupportedSymbolError: + return [] + + +async def _fetch_open_orders_for_symbols(exchange_manager, symbols: list[str]) -> list[dict]: + if not symbols: + return [] + open_orders_by_symbol = await asyncio.gather(*[ + _get_open_orders_for_symbol(exchange_manager, symbol) + for symbol in symbols + ]) + orders_by_exchange_id: dict[str, dict] = {} + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_orders in open_orders_by_symbol: + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + exchange_id = inner_order.get(order_columns.EXCHANGE_ID.value) or inner_order.get( + order_columns.ID.value + ) + if exchange_id is None: + continue + orders_by_exchange_id[str(exchange_id)] = open_order + return list(orders_by_exchange_id.values()) + + def _historical_assets_from_portfolio( exchange_manager, portfolio_content: dict, diff --git a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py index 466c7b194f..7dd4a05e6d 100644 --- a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py +++ b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py @@ -11,15 +11,16 @@ def persist_global_view_refresh_result( user_id: str, account_id: str, refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, + *, + persist_open_orders: bool = False, ) -> None: collection_providers.AccountProvider.instance().update_item(user_id, refresh_result.updated_account) - account_state_persistence_module.persist_account_trading( - user_id, - account_id, - refresh_result.open_orders or [], - refresh_result.trades or [], - refresh_result.positions or [], - ) + if persist_open_orders: + account_state_persistence_module.persist_account_trading_orders( + user_id, + account_id, + refresh_result.open_orders or [], + ) if refresh_result.portfolio_history_state is not None: collection_providers.AccountHistoryProvider.instance().save_state( user_id, diff --git a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py index f8f73e90d1..9a58b3855b 100644 --- a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py @@ -2,14 +2,25 @@ import octobot_trading.exchange_data import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_import +import octobot_flow.entities import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import import octobot_trading.constants as trading_constants class TickersRepository(base_exchange_repository_import.BaseExchangeRepository): + @classmethod + async def ensure_temporary_ticker_channel(cls, exchange_manager) -> None: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [octobot_trading.exchange_data.TickerUpdater], + start_producers=False, + ) + async def fetch_tickers(self, symbols: typing.Optional[list[str]]) -> dict[str, dict]: updater = typing.cast( octobot_trading.exchange_data.TickerUpdater, @@ -17,6 +28,28 @@ async def fetch_tickers(self, symbols: typing.Optional[list[str]]) -> dict[str, ) return await updater.fetch_all_tickers(symbols) + @classmethod + async def fetch_ticker_close_by_symbol( + cls, + exchange_manager, + symbols: list[str], + ) -> dict[str, float]: + if not symbols: + return {} + tickers_repository = cls( + exchange_manager, + known_automations=[], + fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), + ) + tickers = await tickers_repository.fetch_tickers(symbols) + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol: dict[str, float] = {} + for symbol, ticker in tickers.items(): + close_price = ticker.get(close_column) + if close_price is not None: + ticker_close_by_symbol[symbol] = float(close_price) + return ticker_close_by_symbol + @staticmethod def get_cached_market_price(exchange_internal_name, exchange_type, sandboxed: bool, symbol: str) -> float: try: diff --git a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py index 980651d56b..1c8baffce7 100644 --- a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py +++ b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py @@ -8,7 +8,6 @@ import octobot_commons.constants as commons_constants import octobot_protocol.models as protocol_models -import octobot_trading.api as trading_api import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums import octobot_trading.exchanges.util.exchange_data as exchange_data_module @@ -16,14 +15,18 @@ import octobot_flow.entities import octobot_flow.jobs.global_view_account_job as global_view_account_job_module import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module import octobot_sync.constants as sync_constants +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline -def _open_order_dict(exchange_id: str) -> dict: +def _open_order_dict(exchange_id: str, symbol: str = "BTC/USDT") -> dict: return { trading_constants.STORAGE_ORIGIN_VALUE: { trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, } } @@ -45,19 +48,45 @@ def _portfolio_content() -> dict: _TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) -def _exchange_account_context() -> octobot_flow.entities.GlobalViewAccountContext: +def _empty_portfolio_history_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def _exchange_account_context( + *, + has_bound_automation: bool = False, + is_simulated: bool = True, +) -> octobot_flow.entities.GlobalViewAccountContext: account_id = "functional-account-1" exchange_account = protocol_models.ExchangeAccount( account_type=protocol_models.AccountType.EXCHANGE, remote_account_id=account_id, exchange_config_ids=["exchange-config-1"], ) + account_assets = None + if is_simulated: + account_assets = [ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=1000.0, + available=1000.0, + ), + ], + ), + ] account = protocol_models.Account( id=account_id, name="Functional account", - is_simulated=True, + is_simulated=is_simulated, created_at=_TEST_TIMESTAMP, updated_at=_TEST_TIMESTAMP, + assets=account_assets, specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), ) exchange_config = protocol_models.ExchangeConfig( @@ -77,6 +106,7 @@ def _exchange_account_context() -> octobot_flow.entities.GlobalViewAccountContex exchange_config=exchange_config, trading_type=protocol_models.TradingType.SPOT, auth_details=auth_details, + has_bound_automation=has_bound_automation, ) @@ -85,21 +115,44 @@ class TestGlobalViewAccountJobFunctional: async def test_refresh_simulated_account(self): context = _exchange_account_context() exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") - exchange_manager.exchange_personal_data.portfolio_manager = None + wire_portfolio_pipeline(exchange_manager, {}) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio = mock.Mock( + side_effect=lambda portfolio_content, update_available_funds_from_open_orders=False: ( + portfolio_manager.handle_balance_update(portfolio_content) + ), + ) @contextlib.asynccontextmanager async def fake_exchange_manager(*_args, **_kwargs): yield exchange_manager + ensure_ticker_channel_mock = mock.AsyncMock() with ( mock.patch.object( global_view_account_job_module.trading_exchanges, "exchange_manager_from_exchange_data", fake_exchange_manager, ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module.tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -110,37 +163,19 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_order_exchange_ids", return_value=set(), ), - mock.patch.object( - global_view_persistence_module, - "persist_global_view_refresh_result", - ), mock.patch.object( account_state_persistence_module, - "build_portfolio_history_state", - side_effect=lambda user_id, account_id, snapshot, valuation_unit, evaluation_time: ( - protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit=valuation_unit, - values=[snapshot], - ), - ) - ), - ), - mock.patch.object( - trading_api, - "get_portfolio", - return_value=_portfolio_content(), + "load_previous_open_orders", + return_value=[], ), mock.patch.object( - trading_api, - "get_current_portfolio_value", - return_value=1500.0, + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), ), mock.patch.object( - trading_api, - "get_current_crypto_currency_value", - side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + global_view_persistence_module, + "persist_global_view_refresh_result", ), ): refresh_result = await global_view_account_job_module.GlobalViewAccountJob( @@ -148,34 +183,57 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + portfolio_manager.apply_forced_portfolio.assert_called_once() assert refresh_result.portfolio_history_state is not None assert refresh_result.portfolio_history_state.history is not None assert refresh_result.portfolio_history_state.history.unit == "USDC" - assert refresh_result.portfolio_history_state.history.values[-1].total == 1500.0 + assert refresh_result.portfolio_history_state.history.values[-1].total > 0 assert refresh_result.updated_account.assets + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.updated_account.assets + for asset in assets_for_trading_type.assets + } + assert "USDT" in asset_symbols assert refresh_result.changed_order_ids == set() async def test_refresh_real_account_detects_disappeared_orders(self): - context = _exchange_account_context() - context.account.is_simulated = False + context = _exchange_account_context(has_bound_automation=True, is_simulated=False) exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) exchange_manager.exchange.get_open_orders = mock.AsyncMock( return_value=[_open_order_dict("stays-order-2")], ) exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) - exchange_manager.exchange_personal_data.portfolio_manager = None + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + _open_order_dict("gone-order-1", "BTC/USDT"), + _open_order_dict("stays-order-2", "ETH/USDT"), + ] @contextlib.asynccontextmanager async def fake_exchange_manager(*_args, **_kwargs): yield exchange_manager + ensure_ticker_channel_mock = mock.AsyncMock() with ( mock.patch.object( global_view_account_job_module.trading_exchanges, "exchange_manager_from_exchange_data", fake_exchange_manager, ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -186,37 +244,19 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_order_exchange_ids", return_value={"gone-order-1", "stays-order-2"}, ), - mock.patch.object( - global_view_persistence_module, - "persist_global_view_refresh_result", - ), mock.patch.object( account_state_persistence_module, - "build_portfolio_history_state", - side_effect=lambda user_id, account_id, snapshot, valuation_unit, evaluation_time: ( - protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit=valuation_unit, - values=[snapshot], - ), - ) - ), - ), - mock.patch.object( - trading_api, - "get_portfolio", - return_value=_portfolio_content(), + "load_previous_open_orders", + return_value=previous_open_orders, ), mock.patch.object( - trading_api, - "get_current_portfolio_value", - return_value=1500.0, + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), ), mock.patch.object( - trading_api, - "get_current_crypto_currency_value", - side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + global_view_persistence_module, + "persist_global_view_refresh_result", ), ): refresh_result = await global_view_account_job_module.GlobalViewAccountJob( @@ -224,7 +264,14 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} assert refresh_result.portfolio_history_state is not None assert refresh_result.portfolio_history_state.history.unit == commons_constants.DEFAULT_REFERENCE_MARKET assert refresh_result.updated_account.assets + assert refresh_result.portfolio_history_state.history.values[-1].total > 0 assert refresh_result.changed_order_ids == {"gone-order-1"} diff --git a/packages/flow/tests/logic/accounts/test_account_state_persistence.py b/packages/flow/tests/logic/accounts/test_account_state_persistence.py index c4769b0062..7dfd9f2351 100644 --- a/packages/flow/tests/logic/accounts/test_account_state_persistence.py +++ b/packages/flow/tests/logic/accounts/test_account_state_persistence.py @@ -8,6 +8,8 @@ import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module @@ -75,6 +77,79 @@ def test_returns_empty_set_when_no_trading_state(self): ) assert exchange_ids == set() + def test_propagates_unexpected_load_state_errors(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = RuntimeError("disk failure") + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "disk failure" + + +class TestLoadPreviousOpenOrders: + def test_returns_storage_dicts_from_persisted_orders(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[_protocol_order("order-1")], + ), + ) + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + open_orders = account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + assert len(open_orders) == 1 + inner_order = open_orders[0][trading_constants.STORAGE_ORIGIN_VALUE] + assert inner_order[trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value] == "order-1" + assert inner_order[trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value] == "BTC/USDT" + + def test_returns_empty_list_when_no_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + open_orders = account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + assert open_orders == [] + + def test_propagates_unexpected_load_state_errors(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = RuntimeError("disk failure") + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "disk failure" + class TestBuildPortfolioHistoryState: def test_merges_snapshot_into_history_state(self): @@ -100,7 +175,12 @@ def test_merges_snapshot_into_history_state(self): class TestPersistAccountTrading: def test_saves_updated_trading_state(self): trading_provider = mock.Mock() - trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) with mock.patch.object( collection_providers.AccountTradingProvider, "instance", @@ -114,3 +194,193 @@ def test_saves_updated_trading_state(self): positions=[], ) trading_provider.save_state.assert_called_once() + + def test_raises_when_trading_state_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[], + trades=[], + positions=[], + ) + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + trading_provider.save_state.assert_not_called() + + def test_fills_id_from_exchange_id_when_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + order_without_id = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "exch-order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.01, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + trading_enums.ExchangeConstantsOrderColumns.REDUCE_ONLY.value: False, + trading_enums.ExchangeConstantsOrderColumns.IS_ACTIVE.value: True, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[order_without_id], + trades=[], + positions=[], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_orders = saved_state.account_trading.orders + assert len(saved_orders) == 1 + assert saved_orders[0].id == "exch-order-1" + assert saved_orders[0].exchange_id == "exch-order-1" + + def test_persists_ccxt_order_missing_optional_fields(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + ccxt_like_order = { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.01, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[ccxt_like_order], + trades=[], + positions=[], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_order = saved_state.account_trading.orders[0] + assert saved_order.id == "order-1" + assert saved_order.trigger_above is None + assert saved_order.reduce_only is False + assert saved_order.is_active is True + + def test_raises_when_order_missing_id_and_exchange_id(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + order_without_ids = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[order_without_ids], + trades=[], + positions=[], + ) + raise AssertionError("Expected ValueError") + except ValueError as error: + assert "missing both id and exchange_id" in str(error) + trading_provider.save_state.assert_not_called() + + +class TestPersistAccountTradingFromIterationState: + def test_persists_trading_snapshot_from_automation_state(self): + import octobot_flow.entities + + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + elements = octobot_flow.entities.ExchangeAccountElements() + elements.orders.open_orders = [{"exchange_id": "order-1"}] + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + with mock.patch.object( + account_state_persistence_module, + "persist_account_trading", + ) as persist_account_trading_mock: + account_state_persistence_module.persist_account_trading_from_iteration_state( + "wallet-1", + automation_state.to_dict(include_default_values=False), + ) + persist_account_trading_mock.assert_called_once_with( + "wallet-1", + "acc-sync-1", + [{"exchange_id": "order-1"}], + [], + [], + ) + + def test_skips_when_wallet_not_registered(self): + import octobot.community.wallet_backend.errors as wallet_backend_errors_module + import octobot_flow.entities + + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + elements = octobot_flow.entities.ExchangeAccountElements() + elements.orders.open_orders = [{"exchange_id": "order-1"}] + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + with mock.patch.object( + account_state_persistence_module, + "persist_account_trading", + side_effect=wallet_backend_errors_module.WalletNotFoundError("Wallet not found"), + ): + account_state_persistence_module.persist_account_trading_from_iteration_state( + "wallet-1", + automation_state.to_dict(include_default_values=False), + ) diff --git a/packages/flow/tests/logic/configuration/test_profile_data_factory.py b/packages/flow/tests/logic/configuration/test_profile_data_factory.py index 32f88d0d8c..f1e3706532 100644 --- a/packages/flow/tests/logic/configuration/test_profile_data_factory.py +++ b/packages/flow/tests/logic/configuration/test_profile_data_factory.py @@ -49,13 +49,6 @@ def test_omits_hollaex_tentacle_when_url_missing(self): assert profile_data.get_config_by_tentacle() == {} -class TestExchangeTypeFromTradingType: - def test_maps_spot_trading_type(self): - assert profile_data_factory_module.exchange_type_from_trading_type( - protocol_models.TradingType.SPOT, - ) == trading_enums.ExchangeTypes.SPOT.value - - class TestProfileDataForAccount: def test_enables_trader_for_real_account(self): exchange_account = protocol_models.ExchangeAccount( diff --git a/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py b/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py deleted file mode 100644 index a806c9be3c..0000000000 --- a/packages/flow/tests/logic/exchange/portfolio/test_valuation_unit.py +++ /dev/null @@ -1,25 +0,0 @@ -# Drakkar-Software OctoBot-Flow - -import mock -import octobot_commons.constants as commons_constants -import octobot_trading.enums as trading_enums - -import octobot_flow.logic.exchange.portfolio.valuation_unit as valuation_unit_module - - -class TestResolvePortfolioValuationUnit: - def test_returns_exchange_default_quote_currency_when_set(self): - exchange_manager = mock.Mock() - exchange_manager.exchange.get_option_value.return_value = "USDC" - assert valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) == "USDC" - exchange_manager.exchange.get_option_value.assert_called_once_with( - trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY - ) - - def test_falls_back_to_default_reference_market_when_option_missing(self): - exchange_manager = mock.Mock() - exchange_manager.exchange.get_option_value.return_value = None - assert ( - valuation_unit_module.resolve_portfolio_valuation_unit(exchange_manager) - == commons_constants.DEFAULT_REFERENCE_MARKET - ) diff --git a/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py b/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py new file mode 100644 index 0000000000..3e0f8026bb --- /dev/null +++ b/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py @@ -0,0 +1,128 @@ +# Drakkar-Software OctoBot-Flow + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module + + +def _open_order_storage_dict( + exchange_id: str, + *, + price: float = 10000.0, + symbol: str = "BTC/USDT", + trigger_above: bool | None = None, + side: str | None = None, +) -> dict: + order_columns = trading_enums.ExchangeConstantsOrderColumns + inner_order = { + order_columns.EXCHANGE_ID.value: exchange_id, + order_columns.ID.value: exchange_id, + order_columns.SYMBOL.value: symbol, + order_columns.PRICE.value: price, + } + if trigger_above is not None: + inner_order[order_columns.TRIGGER_ABOVE.value] = trigger_above + if side is not None: + inner_order[order_columns.SIDE.value] = side + return { + trading_constants.STORAGE_ORIGIN_VALUE: inner_order, + } + + +class TestIsSimulatedOrderFilled: + def test_trigger_above_false_fills_when_market_at_or_below_order_price(self): + assert simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=9000.0, + trigger_above=False, + ) + + def test_trigger_above_false_stays_open_when_market_above_order_price(self): + assert not simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=11000.0, + trigger_above=False, + ) + + def test_trigger_above_true_fills_when_market_at_or_above_order_price(self): + assert simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=11000.0, + trigger_above=True, + ) + + def test_trigger_above_true_stays_open_when_market_below_order_price(self): + assert not simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=9000.0, + trigger_above=True, + ) + + +class TestResolveSimulatedOpenOrders: + def test_drops_filled_orders_and_keeps_open_ones(self): + open_orders = [ + _open_order_storage_dict("filled-order", price=10000.0, trigger_above=False), + _open_order_storage_dict("open-order", price=8000.0, trigger_above=False), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 9000.0}, + ) + remaining_exchange_ids = { + order[trading_constants.STORAGE_ORIGIN_VALUE][ + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + ] + for order in still_open_orders + } + assert remaining_exchange_ids == {"open-order"} + + def test_keeps_order_open_when_ticker_missing(self): + open_orders = [_open_order_storage_dict("order-1", trigger_above=False)] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {}, + ) + assert len(still_open_orders) == 1 + + def test_infers_trigger_above_from_sell_side_when_missing(self): + open_orders = [ + _open_order_storage_dict( + "filled-sell", + price=10000.0, + side=trading_enums.TradeOrderSide.SELL.value, + ), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 11000.0}, + ) + assert still_open_orders == [] + + def test_infers_trigger_above_from_buy_side_when_missing(self): + open_orders = [ + _open_order_storage_dict( + "filled-buy", + price=10000.0, + side=trading_enums.TradeOrderSide.BUY.value, + ), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 9000.0}, + ) + assert still_open_orders == [] + + +class TestSymbolsFromOpenOrders: + def test_returns_unique_sorted_symbols(self): + open_orders = [ + _open_order_storage_dict("order-1", symbol="ETH/USDT"), + _open_order_storage_dict("order-2", symbol="BTC/USDT"), + _open_order_storage_dict("order-3", symbol="BTC/USDT"), + ] + assert simulated_order_fill_detector_module.symbols_from_open_orders(open_orders) == [ + "BTC/USDT", + "ETH/USDT", + ] diff --git a/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py index 2f14dcf687..8951f74360 100644 --- a/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py +++ b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py @@ -6,7 +6,6 @@ import octobot_commons.constants as commons_constants import octobot_protocol.models as protocol_models -import octobot_trading.api as trading_api import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module @@ -42,20 +41,18 @@ def test_seeds_portfolio_from_account_assets(self): ), ), ) + portfolio_manager = mock.Mock() exchange_manager = mock.Mock() - with mock.patch.object( - trading_api, - "set_simulated_portfolio_initial_config", - ) as set_portfolio_mock: - simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) - set_portfolio_mock.assert_called_once_with( - exchange_manager, + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + portfolio_manager.apply_forced_portfolio.assert_called_once_with( { "USDT": { commons_constants.PORTFOLIO_AVAILABLE: 900.0, commons_constants.PORTFOLIO_TOTAL: 1000.0, }, }, + update_available_funds_from_open_orders=True, ) def test_skips_when_account_has_no_assets(self): @@ -71,10 +68,8 @@ def test_skips_when_account_has_no_assets(self): ), ), ) + portfolio_manager = mock.Mock() exchange_manager = mock.Mock() - with mock.patch.object( - trading_api, - "set_simulated_portfolio_initial_config", - ) as set_portfolio_mock: - simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) - set_portfolio_mock.assert_not_called() + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + portfolio_manager.apply_forced_portfolio.assert_not_called() diff --git a/packages/flow/tests/logic/global_view/portfolio_test_util.py b/packages/flow/tests/logic/global_view/portfolio_test_util.py new file mode 100644 index 0000000000..6b92440c4a --- /dev/null +++ b/packages/flow/tests/logic/global_view/portfolio_test_util.py @@ -0,0 +1,69 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import decimal + +import mock + +import octobot_commons.constants as commons_constants + + +def wire_portfolio_pipeline( + exchange_manager, + portfolio_content: dict, + *, + portfolio_total: float = 1500.0, +) -> None: + stored_portfolio = { + currency: dict(balances) + for currency, balances in portfolio_content.items() + } + portfolio = mock.Mock() + portfolio.portfolio = stored_portfolio + portfolio_manager = mock.Mock() + portfolio_manager.portfolio = portfolio + portfolio_manager.reference_market = "USDC" + portfolio_value_holder = mock.Mock() + portfolio_value_holder.portfolio_current_value = portfolio_total + portfolio_value_holder.current_crypto_currencies_values = { + "BTC": decimal.Decimal("50000"), + "USDT": decimal.Decimal("1"), + } + value_converter = mock.Mock() + + def convert_using_last_prices(amount, currency, _reference_market): + if currency == "BTC": + return decimal.Decimal("50000") * amount + return decimal.Decimal("1") * amount + + value_converter.convert_currency_value_using_last_prices = convert_using_last_prices + portfolio_value_holder.value_converter = value_converter + portfolio_manager.portfolio_value_holder = portfolio_value_holder + portfolio_manager.portfolio_history_update = contextlib.nullcontext + + def handle_balance_update(balance, is_diff_update=False): + for currency, amounts in balance.items(): + if isinstance(amounts, dict): + stored_portfolio[currency] = amounts + else: + stored_portfolio[currency] = { + commons_constants.PORTFOLIO_AVAILABLE: amounts, + commons_constants.PORTFOLIO_TOTAL: amounts, + } + return True + + portfolio_manager.handle_balance_update = mock.Mock(side_effect=handle_balance_update) + portfolio_manager.resolve_pending_portfolio_update_events_if_any = mock.AsyncMock() + exchange_personal_data = exchange_manager.exchange_personal_data + exchange_personal_data.portfolio_manager = portfolio_manager + + async def handle_portfolio_update(balance, should_notify=False, is_diff_update=False): + portfolio_manager.handle_balance_update(balance, is_diff_update=is_diff_update) + await portfolio_manager.resolve_pending_portfolio_update_events_if_any() + return True + + exchange_personal_data.handle_portfolio_update = mock.AsyncMock(side_effect=handle_portfolio_update) + exchange_personal_data.handle_portfolio_profitability_update = mock.AsyncMock() + exchange_manager.get_symbol_data = mock.Mock(return_value=mock.Mock()) + exchange_manager.client_symbols = ["BTC/USDC", "USDC/BTC", "BTC/USDT", "ETH/USDT"] + exchange_manager.symbol_exists = mock.Mock(return_value=True) diff --git a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py new file mode 100644 index 0000000000..5c777a4150 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py @@ -0,0 +1,519 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime +import decimal + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.errors as trading_errors + +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline + + +def _open_order_dict(exchange_id: str, symbol: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +def _wire_portfolio_pipeline(exchange_manager, portfolio_content: dict, **kwargs) -> None: + wire_portfolio_pipeline(exchange_manager, portfolio_content, **kwargs) + + +class TestFetchOpenOrdersForSymbols: + @pytest.mark.asyncio + async def test_bad_symbol_skips_that_symbol_and_keeps_others(self): + eth_order = _open_order_dict("stays-order-2", "ETH/USDT") + + async def get_open_orders(*, symbol=None, **_kwargs): + if symbol == "STRK/USDC": + raise trading_errors.UnSupportedSymbolError( + "ob_kraken does not have market symbol STRK/USDC" + ) + if symbol == "ETH/USDT": + return [eth_order] + raise AssertionError(f"Unexpected symbol: {symbol}") + + exchange_manager = mock.Mock() + exchange_manager.exchange.get_open_orders = mock.AsyncMock(side_effect=get_open_orders) + + open_orders = await exchange_account_refresh_module._fetch_open_orders_for_symbols( + exchange_manager, + ["STRK/USDC", "ETH/USDT"], + ) + + assert open_orders == [eth_order] + + +class TestRefreshExchangeAccountPortfolio: + @pytest.mark.asyncio + async def test_real_refresh_applies_balance_to_portfolio_manager(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, {}) + ensure_ticker_channel_mock = mock.AsyncMock() + refresh_valuation_mock = mock.AsyncMock() + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + refresh_valuation_mock, + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + exchange_manager.exchange_personal_data.handle_portfolio_update.assert_awaited_once() + applied_balance = exchange_manager.exchange_personal_data.handle_portfolio_update.await_args.args[0] + assert applied_balance["BTC"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("0.1") + assert applied_balance["USDT"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("1000.0") + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.assets + for asset in assets_for_trading_type.assets + } + assert asset_symbols == {"USDT", "BTC"} + + @pytest.mark.asyncio + async def test_real_refresh_without_handle_portfolio_update_would_fail(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, {}, portfolio_total=0.0) + exchange_manager.exchange_personal_data.handle_portfolio_update = mock.AsyncMock(return_value=False) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + assert refresh_result.assets == [] + assert refresh_result.portfolio_snapshot.total == 0.0 + + @pytest.mark.asyncio + async def test_simulated_refresh_uses_seeded_account_assets(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline( + exchange_manager, + { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + }, + portfolio_total=1000.0, + ) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio = mock.Mock( + side_effect=lambda portfolio_content, update_available_funds_from_open_orders=False: ( + portfolio_manager.handle_balance_update(portfolio_content) + ), + ) + account = protocol_models.Account( + id="sim-account-1", + name="Simulated", + is_simulated=True, + created_at=datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDC", + total=1000.0, + available=1000.0, + ), + ], + ), + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ), + ), + ) + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + fetch_tickers_mock = mock.AsyncMock(return_value={}) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + fetch_tickers_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + is_simulated=True, + previous_open_orders=[], + ) + + fetch_tickers_mock.assert_awaited_once_with(exchange_manager, []) + + portfolio_manager.apply_forced_portfolio.assert_called_once() + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.assets + for asset in assets_for_trading_type.assets + } + assert "USDC" in asset_symbols + + @pytest.mark.asyncio + async def test_real_refresh_portfolio_total_positive_with_ticker_prices(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, balance_content, portfolio_total=6000.0) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + assert refresh_result.portfolio_snapshot.total > 0 + + +class TestValuationSymbolsFromPortfolio: + def test_skips_zero_holdings_and_valuation_unit(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["BTC/USDC"] + portfolio_content = { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + "ETH": { + commons_constants.PORTFOLIO_TOTAL: 0.0, + commons_constants.PORTFOLIO_AVAILABLE: 0.0, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "USDC", + ) + assert valuation_symbols == ["BTC/USDC"] + + def test_uses_bridge_symbols_when_direct_valuation_pair_missing(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["APT/USD", "USDT/USD"] + portfolio_content = { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "APT": { + commons_constants.PORTFOLIO_TOTAL: 10.0, + commons_constants.PORTFOLIO_AVAILABLE: 10.0, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "USDT", + ) + assert set(valuation_symbols) == {"APT/USD", "USDT/USD"} + + def test_does_not_use_usd_like_bridge_when_valuation_unit_is_not_usd_like(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["APT/USD", "BTC/EUR"] + portfolio_content = { + "EUR": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "APT": { + commons_constants.PORTFOLIO_TOTAL: 10.0, + commons_constants.PORTFOLIO_AVAILABLE: 10.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "EUR", + ) + assert valuation_symbols == ["BTC/EUR"] + + +class TestRefreshPortfolioValuation: + @pytest.mark.asyncio + async def test_syncs_portfolio_value_without_profitability_update(self): + exchange_manager = mock.Mock() + portfolio_manager = mock.Mock() + portfolio_value_holder = mock.Mock() + portfolio_manager.portfolio_value_holder = portfolio_value_holder + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + exchange_manager.symbol_exists = mock.Mock(return_value=False) + with ( + mock.patch.object( + trading_api, + "get_portfolio", + return_value={ + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + }, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + return_value=[], + ), + ): + await exchange_account_refresh_module._refresh_portfolio_valuation( + exchange_manager, + "USDT", + ) + + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values.assert_called_once_with( + init_price_fetchers=False, + ) + exchange_manager.exchange_personal_data.handle_portfolio_profitability_update.assert_not_called() + portfolio_manager.handle_mark_price_update.assert_not_called() + + @pytest.mark.asyncio + async def test_applies_ticker_prices_via_update_last_price_not_handle_mark_price_update(self): + exchange_manager = mock.Mock() + portfolio_manager = mock.Mock() + portfolio_value_holder = mock.Mock() + value_converter = mock.Mock() + portfolio_value_holder.value_converter = value_converter + portfolio_manager.portfolio_value_holder = portfolio_value_holder + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + exchange_manager.symbol_exists = mock.Mock(return_value=False) + tickers = { + "BTC/USDT": { + trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 50000, + }, + } + with ( + mock.patch.object( + trading_api, + "get_portfolio", + return_value={ + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + }, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + return_value=["BTC/USDT"], + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value=tickers), + ), + ): + await exchange_account_refresh_module._refresh_portfolio_valuation( + exchange_manager, + "USDT", + ) + + value_converter.update_last_price.assert_called_once_with( + "BTC/USDT", + decimal.Decimal("50000"), + ) + portfolio_manager.handle_mark_price_update.assert_not_called() + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values.assert_called_once_with( + init_price_fetchers=False, + ) + + +class TestRefreshExchangeAccountLogging: + @pytest.mark.asyncio + async def test_logs_fetched_portfolio_once_at_info(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "binance" + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, balance_content, portfolio_total=6000.0) + logger_mock = mock.Mock() + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_get_logger", + return_value=logger_mock, + ), + ): + await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + portfolio_log_calls = [ + call_args + for call_args in logger_mock.info.call_args_list + if call_args.args and call_args.args[0] == "Fetched [%s] full [%s] portfolio: %s" + ] + assert len(portfolio_log_calls) == 1 + assert portfolio_log_calls[0].args[1] == "binance" + assert portfolio_log_calls[0].args[2] == "real" + + +class TestSimulatedTickerFetchMerge: + @pytest.mark.asyncio + async def test_fetches_order_and_valuation_symbols_once(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "bitmart" + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDT") + portfolio_content = _portfolio_content() + _wire_portfolio_pipeline(exchange_manager, portfolio_content) + open_order = _open_order_dict("order-1", "ETH/USDT") + fetch_tickers_mock = mock.AsyncMock(return_value={ + "ETH/USDT": {trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 3000}, + "BTC/USDT": {trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 50000}, + }) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=portfolio_content, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + side_effect=[["BTC/USDT"], ["BTC/USDT"]], + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + fetch_tickers_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + is_simulated=True, + previous_open_orders=[open_order], + ) + + fetch_tickers_mock.assert_awaited_once() + fetched_symbols = fetch_tickers_mock.await_args.args[1] + assert set(fetched_symbols) == {"BTC/USDT", "ETH/USDT"} diff --git a/packages/flow/tests/logic/global_view/test_global_view_account_job.py b/packages/flow/tests/logic/global_view/test_global_view_account_job.py index 30a1b7d6eb..49828e2608 100644 --- a/packages/flow/tests/logic/global_view/test_global_view_account_job.py +++ b/packages/flow/tests/logic/global_view/test_global_view_account_job.py @@ -8,7 +8,6 @@ import octobot_commons.constants as commons_constants import octobot_protocol.models as protocol_models -import octobot_trading.api as trading_api import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums import octobot_trading.exchanges.util.exchange_data as exchange_data_module @@ -16,14 +15,19 @@ import octobot_flow.entities import octobot_flow.jobs.global_view_account_job as global_view_account_job_module import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module import octobot_sync.constants as sync_constants +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline -def _open_order_dict(exchange_id: str) -> dict: +def _open_order_dict(exchange_id: str, symbol: str = "BTC/USDT") -> dict: return { trading_constants.STORAGE_ORIGIN_VALUE: { trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, } } @@ -49,6 +53,7 @@ def _exchange_account_context( *, account_id: str = "account-1", is_simulated: bool = False, + has_bound_automation: bool = False, ) -> octobot_flow.entities.GlobalViewAccountContext: exchange_account = protocol_models.ExchangeAccount( account_type=protocol_models.AccountType.EXCHANGE, @@ -80,6 +85,14 @@ def _exchange_account_context( exchange_config=exchange_config, trading_type=protocol_models.TradingType.SPOT, auth_details=auth_details, + has_bound_automation=has_bound_automation, + ) + + +def _empty_portfolio_history_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, ) @@ -88,21 +101,34 @@ class TestGlobalViewAccountJobRun: async def test_run_returns_refresh_result_shape(self): context = _exchange_account_context() exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") - exchange_manager.exchange_personal_data.portfolio_manager = None + wire_portfolio_pipeline(exchange_manager, {}) @contextlib.asynccontextmanager async def fake_exchange_manager(*_args, **_kwargs): yield exchange_manager + ensure_ticker_channel_mock = mock.AsyncMock() + persist_mock = mock.Mock() with ( mock.patch.object( global_view_account_job_module.trading_exchanges, "exchange_manager_from_exchange_data", fake_exchange_manager, ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -113,35 +139,20 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_order_exchange_ids", return_value=set(), ), - mock.patch.object( - global_view_persistence_module, - "persist_global_view_refresh_result", - ), mock.patch.object( account_state_persistence_module, - "build_portfolio_history_state", - return_value=protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit="USDC", - values=[], - ), - ), + "load_previous_open_orders", + return_value=[], ), mock.patch.object( - trading_api, - "get_portfolio", - return_value=_portfolio_content(), - ), - mock.patch.object( - trading_api, - "get_current_portfolio_value", - return_value=1500.0, + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), ), mock.patch.object( - trading_api, - "get_current_crypto_currency_value", - side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, ), ): refresh_result = await global_view_account_job_module.GlobalViewAccountJob( @@ -149,32 +160,56 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + exchange_manager.exchange.get_open_orders.assert_not_called() + persist_mock.assert_called_once() + assert persist_mock.call_args.kwargs["persist_open_orders"] is False assert isinstance(refresh_result, octobot_flow.entities.GlobalViewAccountRefreshResult) assert refresh_result.updated_account.id == "account-1" assert refresh_result.changed_order_ids == set() assert refresh_result.portfolio_history_state is not None assert refresh_result.open_orders == [] + assert refresh_result.updated_account.assets is not None + assert refresh_result.updated_account.assets[0].assets + assert refresh_result.portfolio_history_state.history.values[-1].total > 0 async def test_run_detects_disappeared_orders(self): - context = _exchange_account_context() + context = _exchange_account_context(has_bound_automation=True) exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) exchange_manager.exchange.get_open_orders = mock.AsyncMock( return_value=[_open_order_dict("stays-order-2")], ) exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) - exchange_manager.exchange_personal_data.portfolio_manager = None + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + _open_order_dict("gone-order-1", "BTC/USDT"), + _open_order_dict("stays-order-2", "ETH/USDT"), + ] @contextlib.asynccontextmanager async def fake_exchange_manager(*_args, **_kwargs): yield exchange_manager + persist_mock = mock.Mock() + ensure_ticker_channel_mock = mock.AsyncMock() with ( mock.patch.object( global_view_account_job_module.trading_exchanges, "exchange_manager_from_exchange_data", fake_exchange_manager, ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -185,35 +220,107 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_order_exchange_ids", return_value={"gone-order-1", "stays-order-2"}, ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), + ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + assert persist_mock.call_args.kwargs["persist_open_orders"] is False + assert refresh_result.changed_order_ids == {"gone-order-1"} + + async def test_unbound_fetches_open_orders_by_previous_symbols(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "gone-order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + } + }, + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "stays-order-2", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "ETH/USDT", + } + }, + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + persist_mock = mock.Mock() + ensure_ticker_channel_mock = mock.AsyncMock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), ), mock.patch.object( account_state_persistence_module, - "build_portfolio_history_state", - return_value=protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit="USDC", - values=[], - ), - ), + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, ), mock.patch.object( - trading_api, - "get_portfolio", - return_value=_portfolio_content(), + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, ), mock.patch.object( - trading_api, - "get_current_portfolio_value", - return_value=1500.0, + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), ), mock.patch.object( - trading_api, - "get_current_crypto_currency_value", - side_effect=lambda _exchange_manager, symbol: 50000.0 if symbol == "BTC" else 1.0, + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, ), ): refresh_result = await global_view_account_job_module.GlobalViewAccountJob( @@ -221,8 +328,118 @@ async def fake_exchange_manager(*_args, **_kwargs): context, ).run() + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + assert persist_mock.call_args.kwargs["persist_open_orders"] is True assert refresh_result.changed_order_ids == {"gone-order-1"} + async def test_simulated_account_uses_fill_detector_without_exchange_fetch(self): + context = _exchange_account_context(account_id="sim-account", is_simulated=True) + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock() + exchange_manager.exchange.get_open_orders = mock.AsyncMock() + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + wire_portfolio_pipeline(exchange_manager, {}, portfolio_total=0.0) + previous_open_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "filled-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + } + }, + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "open-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 8000.0, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + } + }, + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + ensure_ticker_channel_mock = mock.AsyncMock() + persist_mock = mock.Mock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + global_view_account_job_module.simulated_portfolio_seeder_module, + "seed_simulated_portfolio", + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"filled-order", "open-order"}, + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + exchange_account_refresh_module.tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value={ + "BTC/USDT": { + trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 9000.0, + }, + }), + ), + mock.patch.object( + account_state_persistence_module, + "load_portfolio_history_state", + return_value=_empty_portfolio_history_state(), + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + exchange_manager.exchange.get_balance.assert_not_called() + exchange_manager.exchange.get_open_orders.assert_not_called() + assert persist_mock.call_args.kwargs["persist_open_orders"] is True + assert refresh_result.changed_order_ids == {"filled-order"} + remaining_exchange_ids = order_change_detection_module.open_order_exchange_ids_from_open_orders( + refresh_result.open_orders + ) + assert remaining_exchange_ids == {"open-order"} + async def test_generic_account_returns_no_op_result(self): generic_account = protocol_models.Account( id="generic-1", diff --git a/packages/flow/tests/logic/global_view/test_global_view_persistence.py b/packages/flow/tests/logic/global_view/test_global_view_persistence.py new file mode 100644 index 0000000000..e2aba03495 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_global_view_persistence.py @@ -0,0 +1,189 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _refresh_result(*, open_orders: list[dict] | None = None) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account = protocol_models.Account( + id="account-1", + name="Test", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + ) + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + open_orders=open_orders or [], + portfolio_history_state=protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues(unit="USDC", values=[]), + ), + ) + + +class TestPersistGlobalViewRefreshResult: + def test_does_not_persist_trading_when_persist_open_orders_false(self): + account_provider = mock.Mock() + history_provider = mock.Mock() + with ( + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers.AccountHistoryProvider, + "instance", + return_value=history_provider, + ), + mock.patch.object( + account_state_persistence_module, + "persist_account_trading_orders", + ) as persist_orders_mock, + ): + global_view_persistence_module.persist_global_view_refresh_result( + "wallet-1", + "account-1", + _refresh_result(), + persist_open_orders=False, + ) + account_provider.update_item.assert_called_once() + history_provider.save_state.assert_called_once() + persist_orders_mock.assert_not_called() + + def test_persists_orders_only_when_persist_open_orders_true(self): + account_provider = mock.Mock() + history_provider = mock.Mock() + open_orders = [ + { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", + } + ] + with ( + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers.AccountHistoryProvider, + "instance", + return_value=history_provider, + ), + mock.patch.object( + account_state_persistence_module, + "persist_account_trading_orders", + ) as persist_orders_mock, + ): + global_view_persistence_module.persist_global_view_refresh_result( + "wallet-1", + "account-1", + _refresh_result(open_orders=open_orders), + persist_open_orders=True, + ) + persist_orders_mock.assert_called_once_with( + "wallet-1", + "account-1", + open_orders, + ) + + +class TestPersistAccountTradingOrders: + def test_replaces_orders_without_clearing_trades_or_positions(self): + trading_provider = mock.Mock() + sentinel_trade = protocol_models.Trade.model_construct( + id="trade-1", + trade_id="trade-1", + ) + sentinel_position = protocol_models.Position.model_construct( + id="pos-1", + symbol="BTC/USDT", + ) + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[ + protocol_models.Order( + id="old-order", + symbol="BTC/USDT", + price=10000.0, + quantity=0.01, + filled=0.0, + exchange_id="old-order", + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_TEST_TIMESTAMP, + ) + ], + trades=[sentinel_trade], + positions=[sentinel_position], + ), + ) + order_payload = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "new-order", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "new-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "ETH/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 2000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.5, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading_orders( + "wallet-1", + "account-1", + [order_payload], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_trading = saved_state.account_trading + assert len(saved_trading.orders) == 1 + assert saved_trading.orders[0].exchange_id == "new-order" + assert saved_trading.positions == [sentinel_position] + assert saved_trading.trades == [sentinel_trade] + + def test_raises_when_trading_state_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading_orders( + "wallet-1", + "account-1", + [], + ) + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + trading_provider.save_state.assert_not_called() diff --git a/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py b/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py deleted file mode 100644 index 2326c08875..0000000000 --- a/packages/flow/tests/logic/global_view/test_global_view_refreshed_elements.py +++ /dev/null @@ -1,47 +0,0 @@ -# Drakkar-Software OctoBot-Flow - -import octobot_trading.constants as trading_constants -import octobot_trading.enums as trading_enums -import octobot_trading.exchanges.util.exchange_data as exchange_data_module - -import octobot_flow.entities - - -def _order_dict(exchange_id: str, *, filled: float = 0) -> dict: - return { - trading_constants.STORAGE_ORIGIN_VALUE: { - trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, - trading_enums.ExchangeConstantsOrderColumns.FILLED.value: filled, - } - } - - -def _exchange_data(open_orders: list[dict], portfolio: dict[str, dict]) -> exchange_data_module.ExchangeData: - return exchange_data_module.ExchangeData( - orders_details=exchange_data_module.OrdersDetails(open_orders=open_orders), - portfolio_details=exchange_data_module.PortfolioDetails(content=portfolio), - ) - - -class TestGlobalViewRefreshedElementsConfirmedChange: - def test_detects_order_disappearance(self): - before = octobot_flow.entities.GlobalViewRefreshedElements( - _exchange_data( - [_order_dict("order-1"), _order_dict("order-2")], - {"USDT": {"total": 100.0}}, - ), - ) - after_exchange_data = _exchange_data( - [_order_dict("order-2")], - {"USDT": {"total": 100.0}}, - ) - assert before.confirmed_change(after_exchange_data) is True - - def test_no_change_when_orders_and_portfolio_match(self): - orders = [_order_dict("order-1")] - portfolio = {"USDT": {"total": 100.0}} - before = octobot_flow.entities.GlobalViewRefreshedElements( - _exchange_data(orders, portfolio), - ) - after_exchange_data = _exchange_data(orders, portfolio) - assert before.confirmed_change(after_exchange_data) is False diff --git a/packages/flow/tests/repositories/exchange/test_tickers_repository.py b/packages/flow/tests/repositories/exchange/test_tickers_repository.py new file mode 100644 index 0000000000..57cb90ae69 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_tickers_repository.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import pytest + +import octobot_trading.exchange_data as exchange_data_module + +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module + +pytestmark = pytest.mark.asyncio + + +class TestTickersRepositoryEnsureTemporaryTickerChannel: + async def test_creates_channels_and_ticker_producer_only(self): + exchange_manager = mock.Mock() + with ( + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [exchange_data_module.TickerUpdater], + start_producers=False, + ) diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index b81b68f3f1..192993d3c7 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -78,7 +78,7 @@ os.getenv("GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS", "300.0") ) GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS = float( - os.getenv("GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS", "0.1") + os.getenv("GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS", "5") ) NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID = "non-trading-generic-process-octobot-strategy" diff --git a/packages/node/octobot_node/scheduler/global_view/automation_trigger.py b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py index 034bf8df85..ffe4d84acf 100644 --- a/packages/node/octobot_node/scheduler/global_view/automation_trigger.py +++ b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py @@ -2,11 +2,10 @@ # Copyright (c) Drakkar-Software, All rights reserved. import asyncio -import logging import time -import typing import dbos +import octobot_commons.logging as octobot_commons_logging import octobot_protocol.models as protocol_models import octobot_node.constants as node_constants @@ -14,7 +13,19 @@ import octobot_node.scheduler.tasks as scheduler_tasks -logger = logging.getLogger("GlobalViewAutomationTrigger") +def _logger(): + return octobot_commons_logging.get_logger("GlobalViewAutomationTrigger") + + +async def account_has_bound_running_automation(user_id: str, account_id: str) -> bool: + automation_states = await scheduler_api.get_automation_states(user_id) + for automation_state in automation_states: + if automation_state.status != protocol_models.WorkflowStatus.RUNNING: + continue + if not automation_state.exchange_account_ids or account_id not in automation_state.exchange_account_ids: + continue + return True + return False async def trigger_account_automations( @@ -54,14 +65,30 @@ async def _matching_automations( async def _trigger_automation_and_wait(user_id: str, automation_id: str) -> None: import octobot_node.scheduler as scheduler_module + logger = _logger() + logger.info( + "Starting forced automation trigger (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) scheduler = scheduler_module.SCHEDULER active_workflow_ids_before = await scheduler.resolve_active_automation_workflow_ids_for_parent_id( user_id, automation_id, ) workflow_id_before = active_workflow_ids_before[0] if active_workflow_ids_before else None + logger.info( + "Resolved active workflow before trigger (automation_id=%s, workflow_id=%s)", + automation_id, + workflow_id_before, + ) await scheduler_tasks.send_forced_trigger_to_active_automation(automation_id, user_id) + logger.info( + "Forced trigger sent (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) if workflow_id_before is None: logger.warning( @@ -70,32 +97,39 @@ async def _trigger_automation_and_wait(user_id: str, automation_id: str) -> None user_id, ) return + logger.info( + "Waiting for automation workflow iteration (workflow_id=%s)", + workflow_id_before, + ) await _wait_for_workflow_iteration_success(workflow_id_before) + logger.info( + "Finished waiting for automation workflow iteration (workflow_id=%s)", + workflow_id_before, + ) async def _wait_for_workflow_iteration_success(workflow_id: str) -> None: deadline = time.monotonic() + node_constants.GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS - poll_interval = node_constants.GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS while time.monotonic() < deadline: workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) - if workflow_status is None: - await asyncio.sleep(poll_interval) - continue - if workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: - return - if workflow_status.status in ( - dbos.WorkflowStatusString.ERROR.value, - dbos.WorkflowStatusString.CANCELLED.value, - dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED.value, - ): - logger.error( - "Automation workflow %s ended with status %s while waiting for forced trigger iteration", - workflow_id, - workflow_status.status, - ) - return - await asyncio.sleep(poll_interval) - logger.error( + if workflow_status is not None: + if workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: + return + if workflow_status.status in ( + dbos.WorkflowStatusString.ERROR.value, + dbos.WorkflowStatusString.CANCELLED.value, + dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED.value, + ): + _logger().error( + "Automation workflow %s ended with status %s while waiting for forced trigger iteration", + workflow_id, + workflow_status.status, + ) + return + await asyncio.sleep( + node_constants.GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS + ) + _logger().error( "Timed out waiting for automation workflow %s to complete forced trigger iteration", workflow_id, ) diff --git a/packages/node/octobot_node/scheduler/global_view/global_view_executor.py b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py index a295daefa8..6e05d76207 100644 --- a/packages/node/octobot_node/scheduler/global_view/global_view_executor.py +++ b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py @@ -6,6 +6,7 @@ import octobot_protocol.models as protocol_models import octobot_node.errors as node_errors +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver @@ -46,11 +47,16 @@ async def refresh_account_global_view( trading_type, exchange_config.sandboxed, ) + has_bound_automation = await automation_trigger_module.account_has_bound_running_automation( + user_id, + account.id, + ) context = octobot_flow.entities.GlobalViewAccountContext( account=account, exchange_account=exchange_account, exchange_config=exchange_config, trading_type=trading_type, auth_details=auth_details, + has_bound_automation=has_bound_automation, ) return await global_view_account_job_module.GlobalViewAccountJob(user_id, context).run() diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py index 833aa914bd..3245162126 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py @@ -14,6 +14,9 @@ # You should have received a copy of the GNU General Public License along # with OctoBot. If not, see . +import datetime + +import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_providers as collection_providers import octobot_protocol.models as protocol_models @@ -52,4 +55,14 @@ async def _do_execute( self._user_id, checked_account, ) + collection_providers.AccountTradingProvider.instance().save_state( + self._user_id, + checked_account.id, + protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ), + ) self._mark_user_action_completed(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py index 2f4fb8723d..41765fe776 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py @@ -9,47 +9,18 @@ import octobot_trading.errors as trading_errors import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver -_TRADING_TYPE_TO_EXCHANGE_TYPE: dict[protocol_models.TradingType, trading_enums.ExchangeTypes] = { - protocol_models.TradingType.SPOT: trading_enums.ExchangeTypes.SPOT, - protocol_models.TradingType.FUTURES: trading_enums.ExchangeTypes.FUTURE, - protocol_models.TradingType.OPTIONS: trading_enums.ExchangeTypes.OPTION, - protocol_models.TradingType.MARGIN: trading_enums.ExchangeTypes.MARGIN, -} - -_API_KEY_RIGHT_TO_ACCOUNT_PERMISSION: dict[ - trading_enums.APIKeyRights, - protocol_models.AccountPermission, -] = { - trading_enums.APIKeyRights.READING: protocol_models.AccountPermission.READ, - trading_enums.APIKeyRights.SPOT_TRADING: protocol_models.AccountPermission.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING: protocol_models.AccountPermission.FUTURES_TRADING, - trading_enums.APIKeyRights.WITHDRAWALS: protocol_models.AccountPermission.WITHDRAW, -} - -_OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED: list[trading_enums.APIKeyRights] = [ - trading_enums.APIKeyRights.READING, - trading_enums.APIKeyRights.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING, -] - - -def _exchange_type_from_trading_type( - trading_type: protocol_models.TradingType, -) -> str: - return _TRADING_TYPE_TO_EXCHANGE_TYPE[trading_type].value - - async def _fetch_api_key_rights(exchange) -> list[trading_enums.APIKeyRights]: try: return await exchange.get_permissions() except trading_errors.NotSupported: - return list(_OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED) + return list(protocol_trading_mapping.OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED) def _account_permissions_from_api_key_rights( @@ -58,7 +29,9 @@ def _account_permissions_from_api_key_rights( return [ account_permission for api_key_right in api_key_rights - if (account_permission := _API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right)) is not None + if ( + account_permission := protocol_trading_mapping.API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right) + ) is not None ] @@ -115,7 +88,7 @@ def _encrypted_exchange_auth_details( ) -> exchange_data_module.ExchangeAuthDetails: if authentication is None: return exchange_data_module.ExchangeAuthDetails( - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=sandboxed, exchange_account_id=exchange_account.remote_account_id, ) @@ -127,7 +100,7 @@ def _encrypted_exchange_auth_details( api_key=fields_utils.encrypt(authentication.api_key).decode(), api_secret=fields_utils.encrypt(authentication.api_secret).decode(), api_password=api_password, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=sandboxed, exchange_account_id=exchange_account.remote_account_id, ) @@ -177,7 +150,7 @@ async def _check_exchange_account_state( exchanges=[ commons_profile_data.ExchangeData( internal_name=exchange_config.exchange, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, exchange_account_id=exchange_account.remote_account_id, sandboxed=exchange_config.sandboxed, ) @@ -186,7 +159,7 @@ async def _check_exchange_account_state( profile_data.trader.enabled = True exchange_data = exchange_data_module.exchange_data_factory( exchange_internal_name=exchange_config.exchange, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=exchange_config.sandboxed, auth_details=_encrypted_exchange_auth_details( exchange_account, diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index a9d38e5a55..f9f3ca36a5 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -20,12 +20,12 @@ import octobot_commons.logging -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_trading.errors import octobot_flow.entities import octobot_flow.enums import octobot_flow.errors +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module import octobot_flow.repositories.community.community_repository as community_repository import octobot_node.enums @@ -36,7 +36,6 @@ import octobot_node.scheduler.workflows.params as params import octobot_node.scheduler.workflows_util as workflows_util import octobot_node.errors as errors -import octobot_node.protocol.accounts_trading as accounts_trading_protocol from octobot_node.scheduler import SCHEDULER # avoid circular import @@ -255,9 +254,9 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) AutomationWorkflow.get_logger(parsed_inputs).info( f"Iteration completed, executed step: '{executed_step}', {next_actions_str}" ) - AutomationWorkflow._persist_account_trading_from_iteration_result( + account_state_persistence_module.persist_account_trading_from_iteration_state( parsed_inputs.task.user_id, - result, + result.next_actions_description.state if result.next_actions_description else None, ) else: retry_delay_seconds = max(0.0, (next_step_at or time.time()) - time.time()) @@ -304,42 +303,6 @@ async def _wait_and_trigger_on_actions_update( return recv_payload return None - @staticmethod - def _persist_account_trading_from_iteration_result( - user_id: typing.Optional[str], - job_result: octobot_flow_client.OctoBotActionsJobResult, - ) -> None: - # Temporary: persist trading snapshot locally until the global view system owns this sync. - if user_id is None or job_result.next_actions_description is None: - return - automation_state = octobot_flow.entities.AutomationState.from_dict( - job_result.next_actions_description.state - ) - exchange_account_elements = automation_state.automation.exchange_account_elements - exchange_account_details = automation_state.exchange_account_details - if exchange_account_elements is None or exchange_account_details is None: - return - exchange_account_id = exchange_account_details.exchange_details.exchange_account_id - if not exchange_account_id: - return - try: - accounts_trading_protocol.update_account_trading( - user_id, - exchange_account_id, - list(exchange_account_elements.orders.open_orders), - list(exchange_account_elements.trades), - [ - position_details.position - for position_details in exchange_account_elements.positions - ], - ) - except wallet_backend_errors.WalletNotFoundError: - # Trading collections are wallet-scoped; skip until the wallet is registered locally. - octobot_commons.logging.get_logger(AutomationWorkflow.__name__).warning( - "Skipping account trading persistence for wallet %s: wallet not registered", - user_id, - ) - @staticmethod def _parse_actions_update_envelope( actions_update: typing.Optional[dict], @@ -355,6 +318,11 @@ def _parse_actions_update_envelope( return [], [] return [], [] + @staticmethod + def _is_forced_trigger_update(actions_update: dict) -> bool: + envelope = params.AutomationWorkflowActionUpdate.from_dict(actions_update) + return envelope.actions_type == octobot_node.enums.AutomationWorkflowActionTypes.FORCED_TRIGGER.value + @staticmethod def _log_iteration_execution_intent( parsed_inputs: params.AutomationWorkflowInputs, @@ -382,6 +350,11 @@ async def _process_pending_priority_actions_and_reschedule( while new_actions_update := await AutomationWorkflow._wait_and_trigger_on_actions_update( parsed_inputs, 0 ): + if AutomationWorkflow._is_forced_trigger_update(new_actions_update): + AutomationWorkflow.get_logger(parsed_inputs).info( + "Ignoring forced_trigger received after iteration completed; skipping duplicate iteration." + ) + continue extra_iteration_inputs = AutomationWorkflow._create_next_iteration_inputs( parsed_inputs, latest_iteration_result.next_iteration_description, 0, latest_iteration_result.next_iteration_description_metadata, diff --git a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py index e2b0a9ecd2..85272ad760 100644 --- a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py @@ -39,14 +39,12 @@ async def dbos_cleanup( ) -> dict[str, typing.Any]: return await DbosCleanupWorkflow._cleanup_outdated_automation_executions( scheduled_time, - context, ) @staticmethod @SCHEDULER.INSTANCE.step(name="cleanup_outdated_automation_executions") async def _cleanup_outdated_automation_executions( scheduled_time: datetime.datetime, - context: typing.Any, ) -> dict[str, typing.Any]: logger = logging.get_logger(DbosCleanupWorkflow.__name__) if workflows_retention.should_skip_retention_cleanup_on_this_node(): diff --git a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py index 070687da0c..9de0ace250 100644 --- a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py @@ -15,17 +15,20 @@ # License along with this library. import asyncio import datetime -import logging import typing import dbos import octobot_commons.logging as octobot_commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_protocol.models as protocol_models import octobot_sync.sync.collection_providers as collection_providers +import octobot_flow.entities import octobot_node.enums +import octobot_node.errors as node_errors import octobot_node.scheduler.global_view.global_view_executor as global_view_executor_module import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver import octobot_node.scheduler.workflows_retention as workflows_retention from octobot_node.scheduler import SCHEDULER # avoid circular import @@ -35,6 +38,58 @@ SCHEDULE_CRON = "*/5 * * * *" +def _exchange_label_for_account(user_id: str, account: protocol_models.Account) -> str: + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return "n/a" + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return "n/a" + try: + return exchange_account_resolver.get_exchange_config(user_id, exchange_account).exchange + except (node_errors.InvalidUserActionPayloadError, node_errors.AmbiguousExchangeConfigError): + return "n/a" + + +def _portfolio_summary_fields( + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, +) -> tuple[str, str]: + portfolio_history_state = refresh_result.portfolio_history_state + if portfolio_history_state is None or portfolio_history_state.history is None: + return "n/a", "n/a" + history = portfolio_history_state.history + valuation_unit = history.unit if history.unit else "n/a" + history_values = history.values or [] + if not history_values: + return valuation_unit, "n/a" + return valuation_unit, str(history_values[-1].total) + + +def _log_successful_account_refresh( + user_id: str, + account: protocol_models.Account, + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, +) -> None: + valuation_unit, portfolio_total = _portfolio_summary_fields(refresh_result) + open_orders_count = len(refresh_result.open_orders or []) + changed_orders_count = len(refresh_result.changed_order_ids) + octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__).info( + "Account global view refresh succeeded: account=%s wallet=%s name=%s exchange=%s " + "simulated=%s open_orders=%s changed_orders=%s automations_triggered=%s " + "valuation_unit=%s portfolio_total=%s", + account.id, + user_id, + account.name, + _exchange_label_for_account(user_id, account), + account.is_simulated, + open_orders_count, + changed_orders_count, + bool(refresh_result.changed_order_ids), + valuation_unit, + portfolio_total, + ) + + @SCHEDULER.INSTANCE.dbos_class() class GlobalViewRefreshWorkflow: @staticmethod @@ -45,40 +100,47 @@ async def global_view_refresh( ) -> dict[str, typing.Any]: return await GlobalViewRefreshWorkflow._run_global_view_refresh( scheduled_time, - context, ) @staticmethod - @SCHEDULER.INSTANCE.step(name="run_global_view_refresh") + @SCHEDULER.INSTANCE.step( + name="run_global_view_refresh", + retries_allowed=False, + ) async def _run_global_view_refresh( scheduled_time: datetime.datetime, - context: typing.Any, ) -> dict[str, typing.Any]: logger = octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__) + logger.info("global_view_refresh started (scheduled_time=%s)", scheduled_time.isoformat()) if workflows_retention.should_skip_retention_cleanup_on_this_node(): - logger.info("global_view_refresh skipped: consumer-only node") + logger.info("global_view_refresh stopped: consumer-only node (skipped)") return {"refreshed_accounts": 0, "skipped": True} wallet_ids = collection_providers.AccountProvider.instance().list_registered_wallet_ids() refreshed_accounts_count = 0 for wallet_id in wallet_ids: refreshed_accounts_count += await GlobalViewRefreshWorkflow._refresh_wallet_accounts( wallet_id, - logger, ) - return { + result = { "refreshed_accounts": refreshed_accounts_count, "scheduled_time": scheduled_time.isoformat(), } + logger.info( + "global_view_refresh completed (refreshed_accounts=%s, scheduled_time=%s)", + refreshed_accounts_count, + scheduled_time.isoformat(), + ) + return result @staticmethod async def _refresh_wallet_accounts( user_id: str, - logger: logging.Logger, ) -> int: + logger = octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__) account_provider = collection_providers.AccountProvider.instance() try: accounts = account_provider.list_items(user_id) - except Exception as error: + except wallet_backend_errors.WalletNotFoundError as error: logger.warning( "Skipping global view refresh for wallet %s: cannot list accounts (%s)", user_id, @@ -89,7 +151,7 @@ async def _refresh_wallet_accounts( return 0 refresh_results = await asyncio.gather( *[ - GlobalViewRefreshWorkflow._refresh_single_account(user_id, account, logger) + GlobalViewRefreshWorkflow._refresh_single_account(user_id, account) for account in accounts ], return_exceptions=True, @@ -111,7 +173,6 @@ async def _refresh_wallet_accounts( async def _refresh_single_account( user_id: str, account: protocol_models.Account, - logger: logging.Logger, ) -> bool: try: refresh_result = await global_view_executor_module.refresh_account_global_view( @@ -119,7 +180,9 @@ async def _refresh_single_account( account, ) except Exception as error: - logger.exception( + octobot_commons_logging.get_logger( + GlobalViewRefreshWorkflow.__name__ + ).exception( error, True, f"Failed to refresh account {account.id} for wallet {user_id}: {error}", @@ -131,8 +194,11 @@ async def _refresh_single_account( account.id, refresh_result.changed_order_ids, ) + _log_successful_account_refresh(user_id, account, refresh_result) return True + + def get_schedule_input() -> dbos.ScheduleInput: return { @@ -140,6 +206,6 @@ def get_schedule_input() -> dbos.ScheduleInput: "workflow_fn": GlobalViewRefreshWorkflow.global_view_refresh, "schedule": SCHEDULE_CRON, "context": None, - "automatic_backfill": True, + "automatic_backfill": False, "queue_name": octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, } diff --git a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py index 17dce2c104..26346920b9 100644 --- a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py +++ b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py @@ -68,11 +68,11 @@ _COPY_INIT_USDC = 2000.0 _T_ENQUEUE_SECONDS = 5.0 -_T_GRID_SECONDS = 25.0 -_T_BOOTSTRAP_COPY_SECONDS = 25.0 -_T_POST_SHOCK_SECONDS = 55.0 +_T_GRID_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) +_T_BOOTSTRAP_COPY_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) +_T_POST_SHOCK_SECONDS = workflow_common_module.functional_timeout_seconds(55.0) _T_STOP_SEND_SECONDS = 5.0 -_T_STOP_COMPLETE_SECONDS = 15.0 +_T_STOP_COMPLETE_SECONDS = workflow_common_module.functional_timeout_seconds(15.0) _D_DECIMAL_INCREMENT = decimal.Decimal(str(grid_sim_util.GRID_INCREMENT)) @@ -629,6 +629,14 @@ def _functional_seed_strategy_for_emit_copy_test(_wallet_address, stored_item_id ): caplog.set_level(logging.INFO) + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state( + user_id, master_account_id + ) + workflow_common_module.seed_empty_account_trading_state( + user_id, copy_account_id + ) + # Step 1 — enqueue master emitting grid signals (pushes to local sync server). try: await asyncio.wait_for( diff --git a/packages/node/tests/functional_tests/test_global_view_workflow.py b/packages/node/tests/functional_tests/test_global_view_workflow.py index bf77a24a68..3f58a73fad 100644 --- a/packages/node/tests/functional_tests/test_global_view_workflow.py +++ b/packages/node/tests/functional_tests/test_global_view_workflow.py @@ -7,7 +7,6 @@ import pytest import octobot_node.protocol.accounts_history as accounts_history_protocol -import octobot_node.protocol.accounts_trading as accounts_trading_protocol import octobot_node.scheduler.api as scheduler_api_module import octobot_node.scheduler.tasks as scheduler_tasks_module @@ -41,8 +40,8 @@ async def test_global_view_workflow_updates_all_accounts( sim_account_2 = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_SIM_2_ID) global_view_workflow_util.assert_real_account_assets(real_account) - assert sim_account_1.assets is not None - assert sim_account_2.assets is not None + global_view_workflow_util.assert_simulated_account_assets(sim_account_1, expected_total=1000.0) + global_view_workflow_util.assert_simulated_account_assets(sim_account_2, expected_total=1000.0) assert real_account.updated_at is not None assert sim_account_1.updated_at is not None assert sim_account_2.updated_at is not None @@ -55,7 +54,7 @@ async def test_global_view_workflow_updates_all_accounts( history_state = accounts_history_protocol.get_portfolio_history_state(user_id, account_id) assert history_state.history is not None assert history_state.history.values - assert history_state.history.values[-1].total is not None + assert history_state.history.values[-1].total > 0 assert history_state.history.unit async def test_global_view_workflow_triggers_automation_on_filled_order( @@ -80,20 +79,24 @@ async def record_forced_trigger(automation_id: str, user_id: str) -> None: async with global_view_workflow_util.global_view_functional_environment( tmp_path, - sim_open_order_ids={ - global_view_workflow_util.ACCOUNT_SIM_1_ID: [ - global_view_workflow_util.ORDER_STAYS_OPEN_ID, - ], - }, + sim_ticker_close_by_symbol={"BTC/USDT": 9000.0}, ) as environment: user_id = environment["user_id"] global_view_workflow_util.seed_account_trading_state( environment["trading_provider"], user_id, account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, - order_exchange_ids=[ - global_view_workflow_util.ORDER_FILL_ID, - global_view_workflow_util.ORDER_STAYS_OPEN_ID, + seeded_orders=[ + { + "exchange_id": global_view_workflow_util.ORDER_FILL_ID, + "price": 10000.0, + "trigger_above": False, + }, + { + "exchange_id": global_view_workflow_util.ORDER_STAYS_OPEN_ID, + "price": 8000.0, + "trigger_above": False, + }, ], ) with ( @@ -116,18 +119,6 @@ async def record_forced_trigger(automation_id: str, user_id: str) -> None: assert workflow_result["refreshed_accounts"] == 3 assert trigger_calls == [global_view_workflow_util.AUTOMATION_FILL_ID] - trading_state = accounts_trading_protocol.get_account_trading_state( - user_id, - global_view_workflow_util.ACCOUNT_SIM_1_ID, - ) - remaining_order_ids = { - str(protocol_order.exchange_id) - for protocol_order in (trading_state.account_trading.orders or []) - if protocol_order.exchange_id - } - assert global_view_workflow_util.ORDER_FILL_ID not in remaining_order_ids - assert global_view_workflow_util.ORDER_STAYS_OPEN_ID in remaining_order_ids - account_provider = environment["account_provider"] for account_id in ( global_view_workflow_util.ACCOUNT_REAL_ID, diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py index 4a6e978f31..980a421da4 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py @@ -18,11 +18,13 @@ from .util import dag_assertions as dag_assertions_module from .util import dca_workflow as dca_sim_util +from .util import authenticator_mocks as authenticator_mocks_module from .util import price_mocks as price_mocks_module from .util import protocol_assertions as protocol_assertions_module from .util import user_action_assertions as user_action_assertions_module from .util import workflow_common as workflow_common_module +import octobot.community.authentication as community_authentication_module import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_trading.enums as trading_enums_module @@ -109,8 +111,18 @@ def _decline_per_candle(symbol: str) -> float: strategy_id=dca_sim_util.SIMULATOR_DCA_DEFAULT_STRATEGY_ID, ) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + # Patch market data and providers, then create the maximum-evaluators DCA automation. with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), mock.patch.object( octobot_flow_repositories_exchange_module.TickersRepository, "fetch_tickers", @@ -141,6 +153,8 @@ def _decline_per_candle(symbol: str) -> float: ), ), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _DCA_ACCOUNT_ID) try: await asyncio.wait_for( workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py index 5dd8e903f7..5f9cc614da 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py @@ -113,6 +113,8 @@ async def test_trigger_task_dca_simulator_entry_fill_then_stop(self, temp_dbos_s ), ), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _DCA_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE; expect a completed create result and a running workflow. try: await asyncio.wait_for( diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py index f010003c9d..7aeb5cbcc1 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py @@ -47,6 +47,35 @@ # Spawn real OctoBot child processes on fixed local ports; serialize under pytest-xdist. pytestmark = pytest.mark.xdist_group("octobot_process_functional") +_SHORT_GRACE_RECV_BEFORE_RESCHEDULE_SECONDS = 0.25 + + +@pytest.fixture(autouse=True) +def _short_grace_recv_before_reschedule(monkeypatch, temp_dbos_scheduler): + # Second arg is resume_execution_time (absolute epoch), not a delay. Reschedule path + # passes 0 => zero-timeout recv; rewrite to now+grace so stop can land before next child. + import importlib + automation_workflow_module = importlib.import_module( + "octobot_node.scheduler.workflows.automation_workflow" + ) + real_wait = automation_workflow_module.AutomationWorkflow._wait_and_trigger_on_actions_update + + async def wait_with_short_grace(parsed_inputs, resume_execution_time, *args, **kwargs): + effective_resume_execution_time = resume_execution_time + if resume_execution_time == 0: + effective_resume_execution_time = ( + time.time() + _SHORT_GRACE_RECV_BEFORE_RESCHEDULE_SECONDS + ) + return await real_wait( + parsed_inputs, effective_resume_execution_time, *args, **kwargs + ) + + monkeypatch.setattr( + automation_workflow_module.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + wait_with_short_grace, + ) + class TestStartCheckAndStopDefaultConfigOctobotProcessWorkflow: @pytest.mark.asyncio @@ -118,6 +147,10 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state( + user_id, _GENERIC_PROCESS_ACCOUNT_ID + ) try: await asyncio.wait_for( workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py index 77ad2bab5c..130b9a1d61 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py @@ -41,12 +41,12 @@ _T_ENQUEUE_SECONDS = 5.0 -_T_GRID_SECONDS = 20.0 +_T_GRID_SECONDS = workflow_common_module.functional_timeout_seconds(20.0) _T_SIGNAL_SECONDS = 5.0 _T_STOP_SEND_SECONDS = 5.0 -_T_STOP_COMPLETE_SECONDS = 10.0 +_T_STOP_COMPLETE_SECONDS = workflow_common_module.functional_timeout_seconds(10.0) _T_RESTART_ENQUEUE_SECONDS = 10.0 -_T_RESTART_RUNNING_SECONDS = 30.0 +_T_RESTART_RUNNING_SECONDS = workflow_common_module.functional_timeout_seconds(30.0) # Fast poll after stop/signal send; protocol status may flip RUNNING→COMPLETED quickly on CI. _POST_STOP_PROTOCOL_POLL_SECONDS = 0.05 @@ -129,6 +129,8 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _GRID_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE user action; expect a COMPLETED create result and a running workflow. try: await asyncio.wait_for( diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py index 6e21e04c0e..9316d52877 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py @@ -129,6 +129,8 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _GRID_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE user action; expect a COMPLETED create result and a running workflow. try: await asyncio.wait_for( diff --git a/packages/node/tests/functional_tests/util/global_view_workflow.py b/packages/node/tests/functional_tests/util/global_view_workflow.py index 5defef1c3b..feed486b04 100644 --- a/packages/node/tests/functional_tests/util/global_view_workflow.py +++ b/packages/node/tests/functional_tests/util/global_view_workflow.py @@ -5,6 +5,7 @@ import contextlib import datetime +import decimal import typing import mock @@ -16,10 +17,9 @@ import octobot_protocol.models as protocol_models import octobot_sync.server as sync_server_module import octobot_sync.sync.collection_providers as collection_providers_module -import octobot_trading.constants as trading_constants -import octobot_trading.enums as trading_enums import octobot_trading.exchanges.connectors.ccxt.ccxt_connector as ccxt_connector_module import octobot_trading.exchanges.types.rest_exchange as rest_exchange_module +import octobot_trading.enums as trading_enums import octobot_node.scheduler.workflows_retention as workflows_retention_module @@ -30,7 +30,6 @@ _FUNCTIONAL_SOL_HOLDINGS, _FUNCTIONAL_USDT_HOLDINGS, _stub_get_balance_no_network, - _stub_load_symbol_markets_no_network, ) _TEST_PRIVATE_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" @@ -49,11 +48,27 @@ WORKFLOW_RESULT_TIMEOUT_SECONDS = 120.0 +_FUNCTIONAL_VALUATION_TICKER_CLOSE_BY_SYMBOL = { + "BTC/USDT": 50000.0, + "ETH/USDT": 3000.0, + "SOL/USDT": 100.0, +} + async def _stub_get_open_orders_no_network(self, symbol=None, since=None, limit=None, **kwargs): return [] +async def _stub_load_symbol_markets_for_global_view(self, reload=False, market_filter=None): + self.client.markets = { + "BTC/USDT": {"symbol": "BTC/USDT", "active": True, "spot": True, "id": "BTCUSDT"}, + "ETH/USDT": {"symbol": "ETH/USDT", "active": True, "spot": True, "id": "ETHUSDT"}, + "SOL/USDT": {"symbol": "SOL/USDT", "active": True, "spot": True, "id": "SOLUSDT"}, + "SOL/BTC": {"symbol": "SOL/BTC", "active": True, "spot": True, "id": "SOLBTC"}, + } + self.client.symbols = list(self.client.markets.keys()) + + def derive_user_id() -> str: return sync_server_module.derive_user_id(_TEST_PRIVATE_KEY) @@ -127,11 +142,16 @@ def build_functional_authentication() -> protocol_models.AccountAuthentication: ) -def _protocol_order(exchange_id: str) -> protocol_models.Order: +def _protocol_order( + exchange_id: str, + *, + price: float = 10000.0, + trigger_above: bool = False, +) -> protocol_models.Order: return protocol_models.Order( id=exchange_id, symbol="BTC/USDT", - price=10000.0, + price=price, quantity=0.01, filled=0.0, exchange_id=exchange_id, @@ -139,37 +159,25 @@ def _protocol_order(exchange_id: str) -> protocol_models.Order: type=protocol_models.OrderType.LIMIT, status=protocol_models.OrderStatus.OPEN, created_at=_FUNCTIONAL_TIMESTAMP, + trigger_above=trigger_above, ) -def _open_order_storage_dict(exchange_id: str) -> dict: - order_columns = trading_enums.ExchangeConstantsOrderColumns - return { - trading_constants.STORAGE_ORIGIN_VALUE: { - order_columns.EXCHANGE_ID.value: exchange_id, - order_columns.ID.value: exchange_id, - order_columns.SYMBOL.value: "BTC/USDT", - order_columns.PRICE.value: 10000.0, - order_columns.AMOUNT.value: 0.01, - order_columns.FILLED.value: 0, - order_columns.SIDE.value: trading_enums.TradeOrderSide.BUY.value, - order_columns.TYPE.value: trading_enums.TradeOrderType.LIMIT.value, - order_columns.TRIGGER_ABOVE.value: False, - order_columns.REDUCE_ONLY.value: False, - order_columns.IS_ACTIVE.value: True, - order_columns.STATUS.value: trading_enums.OrderStatus.OPEN.value, - order_columns.TIMESTAMP.value: _FUNCTIONAL_TIMESTAMP.timestamp(), - } - } - - def seed_account_trading_state( trading_provider: collection_providers_module.AccountTradingProvider, user_id: str, *, account_id: str, - order_exchange_ids: list[str], + seeded_orders: list[dict[str, typing.Any]], ) -> None: + protocol_orders = [ + _protocol_order( + seeded_order["exchange_id"], + price=float(seeded_order.get("price", 10000.0)), + trigger_above=bool(seeded_order.get("trigger_above", False)), + ) + for seeded_order in seeded_orders + ] trading_provider.save_state( user_id, account_id, @@ -177,7 +185,7 @@ def seed_account_trading_state( version=collection_providers_module.AccountTradingProvider.STATE_VERSION, account_trading=protocol_models.AccountTrading( updated_at=_FUNCTIONAL_TIMESTAMP, - orders=[_protocol_order(order_id) for order_id in order_exchange_ids], + orders=protocol_orders, trades=[], positions=[], ), @@ -223,7 +231,7 @@ async def enqueue_and_await_global_view_refresh() -> dict[str, typing.Any]: async def global_view_functional_environment( tmp_path, *, - sim_open_order_ids: dict[str, list[str]] | None = None, + sim_ticker_close_by_symbol: dict[str, float] | None = None, ): user_root_provider = user_root_folder_provider_module.instance() previous_user_root = user_root_provider.get_root() @@ -243,53 +251,31 @@ async def global_view_functional_environment( trading_provider = collection_providers_module.AccountTradingProvider(base_folder=str(test_user_root)) history_provider = collection_providers_module.AccountHistoryProvider(base_folder=str(test_user_root)) - configured_sim_open_order_ids = sim_open_order_ids or {} + configured_sim_ticker_close_by_symbol = sim_ticker_close_by_symbol or {} - import octobot_flow.jobs.global_view_account_job as global_view_account_job_module + import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater_module - import octobot_trading.exchanges as trading_exchanges_module - real_exchange_manager_from_exchange_data = trading_exchanges_module.exchange_manager_from_exchange_data + async def patched_fetch_ticker_close_by_symbol(_exchange_manager, symbols): + return { + symbol: configured_sim_ticker_close_by_symbol[symbol] + for symbol in symbols + if symbol in configured_sim_ticker_close_by_symbol + } - @contextlib.asynccontextmanager - async def exchange_manager_with_simulated_open_orders( - exchange_data, - profile_data, - tentacles_setup_config, - price_fallback=None, - ): - async with real_exchange_manager_from_exchange_data( - exchange_data, - profile_data, - tentacles_setup_config, - price_fallback=price_fallback, - ) as exchange_manager: - exchange_account_id = profile_data.exchanges[0].exchange_account_id - open_order_ids_after_refresh = configured_sim_open_order_ids.get(exchange_account_id, []) - - async def patched_get_open_orders(**open_orders_kwargs): - return [ - _open_order_storage_dict(order_id) - for order_id in open_order_ids_after_refresh - ] - - exchange_manager.exchange.get_open_orders = patched_get_open_orders - - real_get_balance = exchange_manager.exchange.get_balance - - async def patched_get_balance(**balance_kwargs): - balance = await real_get_balance(**balance_kwargs) - portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager - if portfolio_manager is not None and balance is not None: - portfolio_manager.handle_balance_update(balance) - return balance - - exchange_manager.exchange.get_balance = patched_get_balance - - portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager - if portfolio_manager is not None: - portfolio_manager.handle_mark_price_update = mock.AsyncMock(return_value=None) - yield exchange_manager + async def patched_fetch_tickers(_self, symbols): + if not symbols: + return {} + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol = { + **_FUNCTIONAL_VALUATION_TICKER_CLOSE_BY_SYMBOL, + **configured_sim_ticker_close_by_symbol, + } + return { + symbol: {close_column: decimal.Decimal(str(ticker_close_by_symbol[symbol]))} + for symbol in symbols + if symbol in ticker_close_by_symbol + } with ( mock.patch.object( @@ -330,7 +316,7 @@ async def patched_get_balance(**balance_kwargs): mock.patch.object( ccxt_connector_module.CCXTConnector, "load_symbol_markets", - _stub_load_symbol_markets_no_network, + _stub_load_symbol_markets_for_global_view, ), mock.patch.object( ccxt_connector_module.CCXTConnector, @@ -343,9 +329,14 @@ async def patched_get_balance(**balance_kwargs): _stub_get_open_orders_no_network, ), mock.patch.object( - trading_exchanges_module, - "exchange_manager_from_exchange_data", - exchange_manager_with_simulated_open_orders, + tickers_repository_module.TickersRepository, + "fetch_ticker_close_by_symbol", + patched_fetch_ticker_close_by_symbol, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "fetch_tickers", + patched_fetch_tickers, ), ): account_provider.create_exchange_config(user_id, build_exchange_config()) @@ -353,13 +344,6 @@ async def patched_get_balance(**balance_kwargs): account_provider.create_item(user_id, build_real_account()) account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_1_ID, account_name="Sim 1")) account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_2_ID, account_name="Sim 2")) - for account_id, order_ids in configured_sim_open_order_ids.items(): - seed_account_trading_state( - trading_provider, - user_id, - account_id=account_id, - order_exchange_ids=order_ids, - ) try: yield { "user_id": user_id, @@ -371,6 +355,20 @@ async def patched_get_balance(**balance_kwargs): user_root_provider.set_root(previous_user_root) +def assert_simulated_account_assets( + account: protocol_models.Account, + *, + expected_total: float, +) -> None: + assert account.assets is not None + flattened_assets: list[protocol_models.DetailedAsset] = [] + for assets_for_trading_type in account.assets: + flattened_assets.extend(assets_for_trading_type.assets or []) + assets_by_symbol = {asset.symbol: asset for asset in flattened_assets} + assert "USDT" in assets_by_symbol + assert assets_by_symbol["USDT"].total == pytest.approx(expected_total) + + def assert_real_account_assets(account: protocol_models.Account) -> None: assert account.assets is not None flattened_assets: list[protocol_models.DetailedAsset] = [] diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index 9531087b49..21d2193320 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -7,6 +7,7 @@ import asyncio import datetime import json +import os import time import typing import uuid @@ -15,7 +16,9 @@ import pytest import octobot_sync.chain.evm as sync_evm_module +import octobot_sync.constants as sync_constants_module import octobot_sync.server as sync_server_module +import octobot_sync.sync.collection_providers as collection_providers_module import octobot_trading.constants as trading_constants_module import octobot_trading.enums as trading_enums_module @@ -44,6 +47,34 @@ DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS = 0.5 DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS = DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS +_XDIST_FUNCTIONAL_TIMEOUT_SCALE = 2.25 + + +def functional_timeout_seconds(base_seconds: float) -> float: + """Scale tight poll budgets when pytest-xdist runs concurrent heavy workers.""" + if os.getenv("PYTEST_XDIST_WORKER"): + return base_seconds * _XDIST_FUNCTIONAL_TIMEOUT_SCALE + return base_seconds + + +def seed_empty_account_trading_state(user_id: str, account_id: str) -> None: + """ + Mirror CreateAccountActionExecutor: AccountTradingState must exist before + automation iterations call persist_account_trading. + Call inside auth mock context so encrypted storage can resolve wallet keys. + """ + collection_providers_module.AccountTradingProvider.instance().save_state( + user_id, + account_id, + protocol_models_module.AccountTradingState( + version=sync_constants_module.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models_module.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ), + ) + + _FUNCTIONAL_PROTOCOL_ACCOUNT_TS = datetime.datetime(2026, 4, 1, 12, 0, 0, tzinfo=datetime.UTC) SIMULATOR_FUNCTIONAL_STRATEGY_VERSION = "1.0.0" diff --git a/packages/node/tests/scheduler/global_view/test_automation_trigger.py b/packages/node/tests/scheduler/global_view/test_automation_trigger.py index aafb77ec2d..35a99fcc47 100644 --- a/packages/node/tests/scheduler/global_view/test_automation_trigger.py +++ b/packages/node/tests/scheduler/global_view/test_automation_trigger.py @@ -29,6 +29,41 @@ def _automation_state( ) +@pytest.mark.asyncio +class TestAccountHasBoundRunningAutomation: + async def test_true_when_running_automation_lists_account(self): + automation = _automation_state( + "automation-1", + account_id="acc-1", + order_ids=["order-1"], + ) + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock: + scheduler_api_mock.get_automation_states = mock.AsyncMock(return_value=[automation]) + assert await automation_trigger_module.account_has_bound_running_automation( + "wallet-1", + "acc-1", + ) is True + + async def test_false_when_no_running_automation_for_account(self): + automation = _automation_state( + "automation-1", + account_id="other-acc", + order_ids=["order-1"], + ) + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock: + scheduler_api_mock.get_automation_states = mock.AsyncMock(return_value=[automation]) + assert await automation_trigger_module.account_has_bound_running_automation( + "wallet-1", + "acc-1", + ) is False + + @pytest.mark.asyncio class TestTriggerAccountAutomations: async def test_triggers_only_matching_automation_sequentially(self): @@ -64,3 +99,58 @@ async def _record_trigger(user_id: str, automation_id: str) -> None: {"gv-order-fill-1"}, ) assert trigger_calls == ["automation-gv-fill"] + + +@pytest.mark.asyncio +class TestTriggerAutomationAndWait: + async def test_logs_info_sequence_when_waiting_for_workflow(self): + import octobot_node.scheduler as scheduler_package + + mock_logger = mock.Mock() + scheduler = mock.Mock() + scheduler.resolve_active_automation_workflow_ids_for_parent_id = mock.AsyncMock( + return_value=["workflow-1"], + ) + with ( + mock.patch.object( + scheduler_package, + "SCHEDULER", + scheduler, + ), + mock.patch.object( + automation_trigger_module, + "_logger", + return_value=mock_logger, + ), + mock.patch.object( + automation_trigger_module.scheduler_tasks, + "send_forced_trigger_to_active_automation", + mock.AsyncMock(), + ) as send_trigger_mock, + mock.patch.object( + automation_trigger_module, + "_wait_for_workflow_iteration_success", + mock.AsyncMock(), + ) as wait_mock, + ): + await automation_trigger_module._trigger_automation_and_wait( + "wallet-1", + "automation-1", + ) + + send_trigger_mock.assert_awaited_once_with("automation-1", "wallet-1") + wait_mock.assert_awaited_once_with("workflow-1") + info_messages = [call.args[0] for call in mock_logger.info.call_args_list] + assert info_messages == [ + "Starting forced automation trigger (automation_id=%s, user_id=%s)", + "Resolved active workflow before trigger (automation_id=%s, workflow_id=%s)", + "Forced trigger sent (automation_id=%s, user_id=%s)", + "Waiting for automation workflow iteration (workflow_id=%s)", + "Finished waiting for automation workflow iteration (workflow_id=%s)", + ] + assert mock_logger.info.call_args_list[0].args[1:] == ("automation-1", "wallet-1") + assert mock_logger.info.call_args_list[1].args[1:] == ("automation-1", "workflow-1") + assert mock_logger.info.call_args_list[2].args[1:] == ("automation-1", "wallet-1") + assert mock_logger.info.call_args_list[3].args[1:] == ("workflow-1",) + assert mock_logger.info.call_args_list[4].args[1:] == ("workflow-1",) + mock_logger.warning.assert_not_called() diff --git a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py index ad01d52bfc..32f0628479 100644 --- a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py +++ b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py @@ -5,9 +5,21 @@ import mock import pytest +import octobot_flow.entities +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + from tests.scheduler import temp_dbos_scheduler +class TestGlobalViewScheduleInput: + def test_get_schedule_input_disables_automatic_backfill(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module + + schedule_input = global_view_workflow_module.get_schedule_input() + assert schedule_input["automatic_backfill"] is False + + @pytest.mark.asyncio class TestGlobalViewRefreshWorkflowRunGlobalViewRefresh: @pytest.fixture @@ -41,7 +53,152 @@ async def test_refreshes_all_wallets_and_accounts_in_parallel_per_wallet( account_provider_class_mock.instance.return_value = account_provider result = await global_view_workflow_module.GlobalViewRefreshWorkflow._run_global_view_refresh( datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), - None, ) assert result["refreshed_accounts"] == 2 assert refresh_mock.await_count == 2 + + +@pytest.mark.asyncio +class TestRefreshWalletAccounts: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_skips_wallet_when_not_registered(self, global_view_workflow_module): + import octobot.community.wallet_backend.errors as wallet_backend_errors_module + + account_provider = mock.Mock() + account_provider.list_items.side_effect = wallet_backend_errors_module.WalletNotFoundError( + "Wallet not found" + ) + mock_logger = mock.Mock() + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock, mock.patch.object( + global_view_workflow_module.octobot_commons_logging, + "get_logger", + return_value=mock_logger, + ): + account_provider_class_mock.instance.return_value = account_provider + refreshed_count = await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_wallet_accounts( + "wallet-missing", + ) + assert refreshed_count == 0 + mock_logger.warning.assert_called_once() + + async def test_propagates_unexpected_list_items_errors(self, global_view_workflow_module): + account_provider = mock.Mock() + account_provider.list_items.side_effect = RuntimeError("storage failure") + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock: + account_provider_class_mock.instance.return_value = account_provider + try: + await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_wallet_accounts( + "wallet-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "storage failure" + + +def _exchange_account() -> protocol_models.Account: + account_timestamp = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC) + return protocol_models.Account( + id="account-1", + name="Kraken Live", + is_simulated=False, + created_at=account_timestamp, + updated_at=account_timestamp, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote-1", + exchange_config_ids=["exchange-config-1"], + ) + ), + ) + + +@pytest.mark.asyncio +class TestRefreshSingleAccount: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_logs_success_summary_after_refresh(self, global_view_workflow_module): + account = _exchange_account() + evaluation_time = datetime.datetime(2026, 8, 11, 18, 30, tzinfo=datetime.UTC) + refresh_result = octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids={"gone-order-1"}, + open_orders=[{"exchange_id": "stays-order-2"}, {"exchange_id": "stays-order-3"}], + portfolio_history_state=protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDC", + values=[ + protocol_models.PortfolioHistoricalValue( + timestamp=evaluation_time, + total=1500.5, + assets=[], + ) + ], + ), + ), + ) + mock_logger = mock.Mock() + with ( + mock.patch.object( + global_view_workflow_module.global_view_executor_module, + "refresh_account_global_view", + mock.AsyncMock(return_value=refresh_result), + ), + mock.patch.object( + global_view_workflow_module.automation_trigger_module, + "trigger_account_automations", + mock.AsyncMock(), + ) as trigger_mock, + mock.patch.object( + global_view_workflow_module.exchange_account_resolver, + "get_exchange_config", + return_value=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="kraken-main", + exchange="kraken", + sandboxed=False, + ), + ), + mock.patch.object( + global_view_workflow_module.octobot_commons_logging, + "get_logger", + return_value=mock_logger, + ), + ): + succeeded = await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_single_account( + "wallet-1", + account, + ) + assert succeeded is True + trigger_mock.assert_awaited_once_with("wallet-1", "account-1", {"gone-order-1"}) + mock_logger.info.assert_called_once() + info_args = mock_logger.info.call_args.args + assert info_args[0].startswith("Account global view refresh succeeded:") + assert info_args[1:] == ( + "account-1", + "wallet-1", + "Kraken Live", + "kraken", + False, + 2, + 1, + True, + "USDC", + "1500.5", + ) diff --git a/packages/node/tests/scheduler/test_schedules.py b/packages/node/tests/scheduler/test_schedules.py index 48834486eb..b6c4a95996 100644 --- a/packages/node/tests/scheduler/test_schedules.py +++ b/packages/node/tests/scheduler/test_schedules.py @@ -42,6 +42,44 @@ def _matching_existing_schedule( } +def _configured_global_view_schedule_input(global_view_workflow_module) -> dict: + return global_view_workflow_module.get_schedule_input() + + +def _matching_existing_global_view_schedule(global_view_workflow_module) -> dict: + schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + return { + "schedule_id": "existing-global-view-schedule-id", + "schedule_name": global_view_workflow_module.SCHEDULE_NAME, + "workflow_name": "ignored-workflow-name", + "workflow_class_name": None, + "schedule": schedule_input["schedule"], + "status": "ACTIVE", + "context": None, + "last_fired_at": "2026-07-13T00:00:00+00:00", + "automatic_backfill": schedule_input.get("automatic_backfill", False), + "cron_timezone": schedule_input.get("cron_timezone"), + "queue_name": schedule_input.get("queue_name"), + } + + +def _get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + *, + cleanup_existing: dict | None, + global_view_existing: dict | None, +): + async def get_schedule_async(schedule_name: str): + if schedule_name == dbos_cleanup_workflow_module.SCHEDULE_NAME: + return cleanup_existing + if schedule_name == global_view_workflow_module.SCHEDULE_NAME: + return global_view_existing + return None + + return get_schedule_async + + class TestExistingScheduleMatchesConfigured: def test_returns_true_when_all_fields_match(self, temp_dbos_scheduler): import octobot_node.scheduler.schedules as schedules_module @@ -185,19 +223,35 @@ def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): yield dbos_cleanup_workflow_module_loaded - async def test_creates_schedule_when_missing(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_creates_schedule_when_missing( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + global_view_schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=None, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=None, + global_view_existing=None, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -214,40 +268,68 @@ async def test_creates_schedule_when_missing(self, dbos_cleanup_workflow_module, ) as backfill_to_thread_mock: await schedules_module.register_schedules(temp_dbos_scheduler) - create_schedule_mock.assert_awaited_once_with( - schedule_name=schedule_input["schedule_name"], - workflow_fn=schedule_input["workflow_fn"], - schedule=schedule_input["schedule"], - context=schedule_input.get("context"), - automatic_backfill=schedule_input.get("automatic_backfill", False), - cron_timezone=schedule_input.get("cron_timezone"), - queue_name=schedule_input.get("queue_name"), + assert create_schedule_mock.await_count == 2 + create_schedule_mock.assert_any_await( + schedule_name=cleanup_schedule_input["schedule_name"], + workflow_fn=cleanup_schedule_input["workflow_fn"], + schedule=cleanup_schedule_input["schedule"], + context=cleanup_schedule_input.get("context"), + automatic_backfill=cleanup_schedule_input.get("automatic_backfill", False), + cron_timezone=cleanup_schedule_input.get("cron_timezone"), + queue_name=cleanup_schedule_input.get("queue_name"), + ) + create_schedule_mock.assert_any_await( + schedule_name=global_view_schedule_input["schedule_name"], + workflow_fn=global_view_schedule_input["workflow_fn"], + schedule=global_view_schedule_input["schedule"], + context=global_view_schedule_input.get("context"), + automatic_backfill=global_view_schedule_input.get("automatic_backfill", False), + cron_timezone=global_view_schedule_input.get("cron_timezone"), + queue_name=global_view_schedule_input.get("queue_name"), ) apply_schedules_mock.assert_not_awaited() backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Creating schedule %s (%s)", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Creating schedule %s (%s)", + global_view_schedule_input["schedule_name"], + global_view_schedule_input["schedule"], ) - async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + async def test_keeps_schedule_when_config_matches( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) - existing_schedule = _matching_existing_schedule( + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + global_view_schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + existing_cleanup_schedule = _matching_existing_schedule( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_cleanup_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -267,30 +349,47 @@ async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_mo create_schedule_mock.assert_not_awaited() apply_schedules_mock.assert_not_awaited() backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Keeping existing schedule %s (%s)", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Keeping existing schedule %s (%s)", + global_view_schedule_input["schedule_name"], + global_view_schedule_input["schedule"], ) - async def test_recreates_schedule_when_config_differs(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + async def test_recreates_schedule_when_config_differs( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) - existing_schedule = _matching_existing_schedule( + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_cleanup_schedule = _matching_existing_schedule( dbos_cleanup_workflow_module, temp_dbos_scheduler, automatic_backfill=False, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_cleanup_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -308,17 +407,23 @@ async def test_recreates_schedule_when_config_differs(self, dbos_cleanup_workflo await schedules_module.register_schedules(temp_dbos_scheduler) create_schedule_mock.assert_not_awaited() - apply_schedules_mock.assert_awaited_once_with([schedule_input]) + apply_schedules_mock.assert_awaited_once_with([cleanup_schedule_input]) backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Updating schedule %s (%s): configuration changed", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Keeping existing schedule %s (%s)", + global_view_workflow_module.SCHEDULE_NAME, + _configured_global_view_schedule_input(global_view_workflow_module)["schedule"], ) async def test_backfills_when_last_fired_at_is_null( self, dbos_cleanup_workflow_module, + global_view_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -328,6 +433,9 @@ async def test_backfills_when_last_fired_at_is_null( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) workflow_id = ( @@ -348,8 +456,12 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -409,6 +521,7 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): async def test_skips_backfill_when_all_window_slots_terminal( self, dbos_cleanup_workflow_module, + global_view_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -418,6 +531,9 @@ async def test_skips_backfill_when_all_window_slots_terminal( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) backfill_end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) @@ -440,8 +556,12 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -489,6 +609,7 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_window( self, dbos_cleanup_workflow_module, + global_view_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -499,6 +620,9 @@ async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_windo dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) existing_schedule["last_fired_at"] = None existing_schedule["schedule"] = "0 * * * *" anchor = datetime.datetime(2026, 7, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) @@ -533,8 +657,12 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -589,6 +717,7 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): async def test_skips_backfill_when_last_fired_at_set( self, dbos_cleanup_workflow_module, + global_view_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -598,14 +727,21 @@ async def test_skips_backfill_when_last_fired_at_set( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -627,6 +763,7 @@ async def test_skips_backfill_when_last_fired_at_set( async def test_skips_backfill_when_automatic_backfill_false( self, dbos_cleanup_workflow_module, + global_view_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -638,6 +775,9 @@ async def test_skips_backfill_when_automatic_backfill_false( temp_dbos_scheduler, automatic_backfill=False, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) existing_schedule["last_fired_at"] = None with mock.patch( @@ -645,8 +785,12 @@ async def test_skips_backfill_when_automatic_backfill_false( return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py index 57d6ae3f4f..5a1926e327 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py @@ -17,6 +17,7 @@ import mock import pytest +import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_protocol.models as protocol_models @@ -38,11 +39,16 @@ async def test_calls_provider_create_with_wallet_and_account(self): ) user_action = protocol_models.UserAction(id="ua-create", configuration=account_executor_test_utils.wrap_configuration(inner)) provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() with ( mock.patch( "octobot_sync.sync.collection_providers.AccountProvider.instance", return_value=provider_mock, ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), mock.patch.object( account_state_updater_module, "update_account_state", @@ -52,6 +58,14 @@ async def test_calls_provider_create_with_wallet_and_account(self): executor = create_account_executor.CreateAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) provider_mock.create_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, account_model) + trading_provider_mock.save_state.assert_called_once() + saved_user_id, saved_account_id, saved_trading_state = trading_provider_mock.save_state.call_args.args + assert saved_user_id == account_executor_test_utils.WALLET_ADDRESS + assert saved_account_id == "new-acc" + assert saved_trading_state.version == sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION + assert saved_trading_state.account_trading.orders is None + assert saved_trading_state.account_trading.trades is None + assert saved_trading_state.account_trading.positions is None provider_assertions.assert_user_action_terminal_state( user_action=user_action, expected_status=protocol_models.UserActionStatus.COMPLETED, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py index 8b67437d9c..b2a280ba2c 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py @@ -550,23 +550,6 @@ async def test_returns_optimistic_rights_when_permissions_fetch_is_not_supported ] -class TestAccountStateUpdaterAccountPermissionsFromApiKeyRights: - def test_maps_known_api_key_rights_to_account_permissions(self): - account_permissions = account_state_updater_module._account_permissions_from_api_key_rights([ - trading_enums.APIKeyRights.READING, - trading_enums.APIKeyRights.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING, - trading_enums.APIKeyRights.WITHDRAWALS, - trading_enums.APIKeyRights.MARGIN_TRADING, - ]) - assert account_permissions == [ - protocol_models.AccountPermission.READ, - protocol_models.AccountPermission.SPOT_TRADING, - protocol_models.AccountPermission.FUTURES_TRADING, - protocol_models.AccountPermission.WITHDRAW, - ] - - class TestAccountStateUpdaterAssetsFromBalance: def test_maps_non_zero_holdings_to_detailed_assets(self): balance = { diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index f16b61146b..4a320c48df 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -242,6 +242,13 @@ def _trading_signal_update_envelope(signal_dicts: list[dict]) -> dict[str, typin ).to_dict(include_default_values=False) +def _forced_trigger_update_envelope() -> dict[str, typing.Any]: + return params.AutomationWorkflowActionUpdate( + actions_type=octobot_node.enums.AutomationWorkflowActionTypes.FORCED_TRIGGER.value, + actions_details=[], + ).to_dict(include_default_values=False) + + @pytest.fixture def parsed_inputs(): task = octobot_node.models.Task( @@ -772,14 +779,14 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( "octobot_node.scheduler.workflows.automation_workflow.time.time", return_value=fixed_now, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_not_called() + persist_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) assert parsed_progress_status.error == expected_error_status assert parsed_progress_status.error_message == str(run_side_effect) @@ -893,19 +900,16 @@ async def test_execute_iteration_persists_open_orders_to_account_trading( "OctoBotActionsJob", mock_octobot_actions_job_class, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_called_once_with( + persist_account_trading_mock.assert_called_once_with( task.user_id, - "acc-sync-1", - [open_order], - [], - [], + automation_state.to_dict(include_default_values=False), ) @pytest.mark.asyncio @@ -957,8 +961,8 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi "OctoBotActionsJob", mock_octobot_actions_job_class, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading", side_effect=wallet_backend_errors_module.WalletNotFoundError("Wallet not found"), ): result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( @@ -997,14 +1001,14 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( "octobot_node.scheduler.workflows.automation_workflow.time.time", return_value=fixed_now, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_not_called() + persist_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) assert parsed_progress_status.postponed_iteration is True assert parsed_progress_status.next_step_at == scheduled_to @@ -1276,6 +1280,93 @@ async def test_process_pending_schedules_next_when_no_priority_actions( mock_wait.assert_awaited_once_with(parsed_inputs, 0) mock_schedule.assert_called_once() + @pytest.mark.asyncio + async def test_process_pending_ignores_forced_trigger_after_iteration( + self, import_automation_workflow, parsed_inputs, iteration_result + ): + mock_wait = mock.AsyncMock( + side_effect=[ + _forced_trigger_update_envelope(), + None, + ] + ) + mock_schedule = mock.AsyncMock() + mock_iteration = mock.AsyncMock() + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_schedule_next_iteration", + mock_schedule, + ): + should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( + parsed_inputs, iteration_result + ) + assert should_continue is True + assert mock_wait.await_count == 2 + mock_iteration.assert_not_awaited() + mock_schedule.assert_called_once() + + @pytest.mark.asyncio + async def test_process_pending_still_processes_user_actions_after_ignoring_forced_trigger( + self, import_automation_workflow, parsed_inputs, iteration_result + ): + result_with_next = params.AutomationWorkflowIterationResult( + progress_status=params.ProgressStatus( + latest_step="step_1", + next_step="step_2", + next_step_at=0.0, + remaining_steps=1, + error=None, + should_stop=False, + ), + next_iteration_description='{"state": {"automation": {}}}', + has_next_actions=True, + ) + mock_wait = mock.AsyncMock( + side_effect=[ + _forced_trigger_update_envelope(), + _user_actions_update_envelope([{"action": "stop"}]), + None, + ] + ) + mock_iteration = mock.AsyncMock( + return_value=result_with_next.to_dict(include_default_values=False) + ) + mock_schedule = mock.AsyncMock() + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_should_continue_workflow", + mock.Mock(return_value=True), + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_schedule_next_iteration", + mock_schedule, + ): + should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( + parsed_inputs, iteration_result + ) + assert should_continue is True + assert mock_wait.await_count == 3 + mock_iteration.assert_awaited_once() + mock_schedule.assert_called_once() + @pytest.mark.asyncio async def test_process_pending_returns_false_when_should_stop(self, import_automation_workflow, parsed_inputs, iteration_result): iteration_result.progress_status.should_stop = True diff --git a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py index 61eb5832e2..f49e052ffa 100644 --- a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py @@ -38,7 +38,6 @@ async def test_delegates_to_cleanup_outdated_automation_executions(self, dbos_cl ) as cleanup_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( datetime.datetime.now(datetime.timezone.utc), - None, ) cleanup_mock.assert_awaited_once() @@ -62,7 +61,6 @@ async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module ) as skip_for_scheduled_time_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( datetime.datetime.now(datetime.timezone.utc), - None, ) cleanup_mock.assert_not_called() @@ -93,7 +91,6 @@ async def test_skips_cleanup_when_newer_execution_already_ran(self, dbos_cleanup ) as cleanup_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( scheduled_time, - None, ) cleanup_mock.assert_not_called() diff --git a/packages/trading/octobot_trading/api/__init__.py b/packages/trading/octobot_trading/api/__init__.py index 62a9ee8902..1c8f966206 100644 --- a/packages/trading/octobot_trading/api/__init__.py +++ b/packages/trading/octobot_trading/api/__init__.py @@ -168,6 +168,7 @@ get_portfolio, get_portfolio_historical_values, get_portfolio_reference_market, + resolve_portfolio_valuation_unit, get_portfolio_currency, get_origin_portfolio, set_simulated_portfolio_initial_config, @@ -380,6 +381,7 @@ "get_portfolio", "get_portfolio_historical_values", "get_portfolio_reference_market", + "resolve_portfolio_valuation_unit", "get_portfolio_currency", "get_origin_portfolio", "set_simulated_portfolio_initial_config", diff --git a/packages/trading/octobot_trading/api/portfolio.py b/packages/trading/octobot_trading/api/portfolio.py index fcd6f4da12..f396300a63 100644 --- a/packages/trading/octobot_trading/api/portfolio.py +++ b/packages/trading/octobot_trading/api/portfolio.py @@ -16,12 +16,23 @@ import decimal import typing +import octobot_commons.constants as commons_constants import octobot_commons.symbols as commons_symbols +import octobot_trading.enums as trading_enums import octobot_trading.exchange_channel as exchange_channel import octobot_trading.constants import octobot_trading.personal_data as personal_data +def resolve_portfolio_valuation_unit(exchange_manager) -> str: + quote_currency = exchange_manager.exchange.get_option_value( + trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY + ) + if quote_currency: + return str(quote_currency) + return commons_constants.DEFAULT_REFERENCE_MARKET + + def get_portfolio(exchange_manager, as_decimal=True) -> dict: return format_portfolio( exchange_manager.exchange_personal_data.portfolio_manager.portfolio.portfolio, diff --git a/packages/trading/octobot_trading/exchange_channel.py b/packages/trading/octobot_trading/exchange_channel.py index d4b8ba9934..7afbcf8417 100644 --- a/packages/trading/octobot_trading/exchange_channel.py +++ b/packages/trading/octobot_trading/exchange_channel.py @@ -76,6 +76,19 @@ async def wait_for_dependencies(self, paths, timeout): return False return True + async def pause(self) -> None: + if self.channel.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug("Pausing...") + self.is_running = False + if not self.channel.is_paused: + self.channel.is_paused = True + + async def resume(self) -> None: + if self.channel.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug("Resuming...") + if self.channel.is_paused: + self.channel.is_paused = False + class IndirectExchangeChannelProducer(ExchangeChannelProducer): """ diff --git a/packages/trading/octobot_trading/exchange_data/ticker/channel/ticker_updater.py b/packages/trading/octobot_trading/exchange_data/ticker/channel/ticker_updater.py index cacf90183f..f030ad2a0c 100644 --- a/packages/trading/octobot_trading/exchange_data/ticker/channel/ticker_updater.py +++ b/packages/trading/octobot_trading/exchange_data/ticker/channel/ticker_updater.py @@ -33,6 +33,13 @@ ttl=constants.TICKER_CACHE_TTL, maxsize=50 ) +_TICKER_FETCH_LOCKS: dict[str, asyncio.Lock] = {} + + +def _get_ticker_fetch_lock(exchange_key: str) -> asyncio.Lock: + if exchange_key not in _TICKER_FETCH_LOCKS: + _TICKER_FETCH_LOCKS[exchange_key] = asyncio.Lock() + return _TICKER_FETCH_LOCKS[exchange_key] class TickerUpdater(ticker_channel.TickerProducer): @@ -125,6 +132,7 @@ async def fetch_all_tickers(self, symbols: typing.Optional[list[str]]) -> dict[s exchange_name = self.channel.exchange_manager.exchange_name exchange_type = exchange_util.get_exchange_type(self.channel.exchange_manager).value sandboxed = self.channel.exchange_manager.is_sandboxed + exchange_key = ticker_cache.TickerCache.get_exchange_key(exchange_name, exchange_type, sandboxed) cached_tickers = _TICKER_CACHE.get_all_tickers(exchange_name, exchange_type, sandboxed) or {} if isinstance(symbols, list) and len(symbols) == 1: symbol = symbols[0] @@ -136,16 +144,20 @@ async def fetch_all_tickers(self, symbols: typing.Optional[list[str]]) -> dict[s if symbols is None: if cached_tickers: return cached_tickers - symbols_filter = None - if self.channel.exchange_manager.exchange.get_option_value( - enums.ExchangeClientOptions.REQUIRES_SYMBOLS_PARAM_TO_FETCH_TICKERS - ): - symbols_filter = symbols - tickers = await self.channel.exchange_manager.exchange.get_all_currencies_price_ticker( - symbols=symbols_filter - ) - self.set_cache(exchange_name, exchange_type, sandboxed, tickers) - return tickers + async with _get_ticker_fetch_lock(exchange_key): + cached_tickers = _TICKER_CACHE.get_all_tickers(exchange_name, exchange_type, sandboxed) or {} + if cached_tickers: + return cached_tickers + symbols_filter = None + if self.channel.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.REQUIRES_SYMBOLS_PARAM_TO_FETCH_TICKERS + ): + symbols_filter = symbols + tickers = await self.channel.exchange_manager.exchange.get_all_currencies_price_ticker( + symbols=symbols_filter + ) + self.set_cache(exchange_name, exchange_type, sandboxed, tickers) + return tickers tickers_from_cache = { symbol: cached_tickers[symbol] for symbol in symbols @@ -154,9 +166,26 @@ async def fetch_all_tickers(self, symbols: typing.Optional[list[str]]) -> dict[s missing_symbols = [symbol for symbol in symbols if symbol not in cached_tickers] if not missing_symbols: return tickers_from_cache - fetched_tickers = await self._fetch_missing_tickers(missing_symbols) - self.set_cache(exchange_name, exchange_type, sandboxed, fetched_tickers) - return {**tickers_from_cache, **fetched_tickers} + async with _get_ticker_fetch_lock(exchange_key): + cached_tickers = _TICKER_CACHE.get_all_tickers(exchange_name, exchange_type, sandboxed) or {} + tickers_from_cache = { + symbol: cached_tickers[symbol] + for symbol in symbols + if symbol in cached_tickers + } + missing_symbols = [symbol for symbol in symbols if symbol not in cached_tickers] + if not missing_symbols: + return tickers_from_cache + fetched_tickers = await self._fetch_missing_tickers(missing_symbols) + self.set_cache(exchange_name, exchange_type, sandboxed, fetched_tickers) + return { + **tickers_from_cache, + **{ + symbol: fetched_tickers[symbol] + for symbol in symbols + if symbol in fetched_tickers + }, + } async def _fetch_missing_tickers(self, missing_symbols: list[str]) -> dict[str, dict]: exchange = self.channel.exchange_manager.exchange diff --git a/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py b/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py index a5e53e36b3..749dd914e0 100644 --- a/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py +++ b/packages/trading/octobot_trading/exchange_data/ticker/ticker_cache.py @@ -65,20 +65,31 @@ def set_all_tickers( f"Refreshed {len(tickers)} ({len(tickers)}/{len(merged_tickers)}) tickers cache for {exchange_name} {exchange_type}{sandbox}" ) self._ALL_TICKERS_BY_EXCHANGE_KEY[key] = merged_tickers - self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = { - octobot_commons.symbols.parse_symbol(symbol).merged_str_symbol(market_separator=""): - octobot_commons.symbols.parse_symbol(symbol) - for symbol in merged_tickers - } + parsed_symbols_by_merged_symbol = {} + for symbol in merged_tickers: + try: + parsed_symbol = octobot_commons.symbols.parse_symbol(symbol) + parsed_symbols_by_merged_symbol[ + parsed_symbol.merged_str_symbol(market_separator="") + ] = parsed_symbol + except (AttributeError, ValueError): + continue + self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = parsed_symbols_by_merged_symbol if exchange_type == octobot_commons.constants.CONFIG_EXCHANGE_FUTURE: - self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = { - **self._ALL_PARSED_SYMBOLS_BY_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key], - **{ - octobot_commons.symbols.parse_symbol(symbol).merged_str_base_and_quote_only_symbol(market_separator=""): - octobot_commons.symbols.parse_symbol(symbol) - for symbol in merged_tickers - } + parsed_future_symbols_by_merged_symbol = { + **parsed_symbols_by_merged_symbol, } + for symbol in merged_tickers: + try: + parsed_symbol = octobot_commons.symbols.parse_symbol(symbol) + parsed_future_symbols_by_merged_symbol[ + parsed_symbol.merged_str_base_and_quote_only_symbol(market_separator="") + ] = parsed_symbol + except (AttributeError, ValueError): + continue + self._ALL_PARSED_SYMBOLS_BY_FUTURE_MERGED_SYMBOLS_BY_EXCHANGE_KEY[key] = ( + parsed_future_symbols_by_merged_symbol + ) def reset_all_tickers_cache(self): self._ALL_TICKERS_BY_EXCHANGE_KEY.clear() diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py index 1caac73514..9b7bbfc489 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_client_util.py @@ -150,12 +150,28 @@ def set_sandbox_mode(exchange_connector, is_sandboxed): exchange_connector.client.enable_demo_trading(is_sandboxed) else: exchange_connector.client.set_sandbox_mode(is_sandboxed) - except ccxt.NotSupported as e: + except ccxt.NotSupported as err: default_type = exchange_connector.client.options.get('defaultType', None) additional_info = f" in type {default_type}" if default_type else "" - exchange_connector.logger.warning(f"{exchange_connector.name} does not support sandboxing {additional_info}: {e}") + message = ( + f"{exchange_connector.name} does not support sandboxing{additional_info}: " + f"sandbox/test URLs are unavailable ({err})" + ) + exchange_connector.logger.warning(message) # raise exception to stop this exchange and prevent dealing with a real funds exchange - raise e + raise ccxt.NotSupported(message) from err + except TypeError as err: + # ccxt raises TypeError when urls['test'] is None (sandbox endpoints missing). + if is_sandboxed: + default_type = exchange_connector.client.options.get('defaultType', None) + additional_info = f" in type {default_type}" if default_type else "" + message = ( + f"{exchange_connector.name} does not support sandboxing{additional_info}: " + f"sandbox/test URLs are unavailable ({err})" + ) + exchange_connector.logger.warning(message) + # raise exception to stop this exchange and prevent dealing with a real funds exchange + raise ccxt.NotSupported(message) from err return None @@ -309,6 +325,8 @@ def converted_ccxt_common_errors(f): async def converted_ccxt_common_errors_wrapper(*args, **kwargs): try: return await f(*args, **kwargs) + except ccxt.BadSymbol as err: + raise errors.UnSupportedSymbolError(err) from err except ccxt.RateLimitExceeded as err: raise errors.RateLimitExceeded(err) from err except ccxt.NotSupported as err: diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py index f4511ad3d5..0a78905d80 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py @@ -16,6 +16,7 @@ # License along with this library. import contextlib import decimal +import asyncio import aiohttp import ccxt.async_support from ccxt.base.types import ( @@ -43,6 +44,7 @@ import octobot_trading.exchanges.config.exchange_credentials_data as exchange_credentials_data import octobot_trading.exchanges.connectors.ccxt.ccxt_adapter as ccxt_adapter import octobot_trading.exchanges.connectors.ccxt.ccxt_client_util as ccxt_client_util +import octobot_trading.exchanges.connectors.ccxt.ccxt_clients_cache as ccxt_clients_cache import octobot_trading.exchanges.connectors.ccxt.enums as ccxt_enums import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants import octobot_trading.exchanges.connectors.util as connectors_util @@ -50,6 +52,15 @@ from octobot_trading.enums import ExchangeConstantsOrderColumns as ecoc +_MARKETS_LOAD_LOCKS: dict[str, asyncio.Lock] = {} + + +def _get_markets_load_lock(client_key: str) -> asyncio.Lock: + if client_key not in _MARKETS_LOAD_LOCKS: + _MARKETS_LOAD_LOCKS[client_key] = asyncio.Lock() + return _MARKETS_LOAD_LOCKS[client_key] + + class CCXTConnector(abstract_exchange.AbstractExchange): """ CCXT library connector. Everything ccxt related should be in this connector. @@ -272,67 +283,7 @@ async def load_symbol_markets( except KeyError: force_load_markets = True if force_load_markets: - self.logger.info( - f"Loading {self.exchange_manager.exchange_name} " - f"{exchanges.get_exchange_type(self.exchange_manager).value}" - f"{' sandbox' if self.exchange_manager.is_sandboxed else ''} exchange markets ({reload=} {authenticated_cache=})" - ) - try: - await self._load_markets(self.client, reload, market_filter=market_filter) - self._persist_markets_cache() - except ccxt.async_support.OBIPWhitelistError as err: - raise octobot_trading.errors.InvalidAPIKeyIPWhitelistError( - f"Invalid IP whitelist error: {html_util.get_html_summary_if_relevant(err)}" - ) from err - except ( - ccxt.async_support.AuthenticationError, - ccxt.async_support.ArgumentsRequired, - ValueError, - binascii.Error, AssertionError, IndexError - ) as err: - self.set_first_consecutive_authentication_error_at_if_unset() - if self.force_authentication: - raise ccxt.async_support.AuthenticationError( - f"Invalid key format ({html_util.get_html_summary_if_relevant(err)})" - ) from err - # should not happen: if it does, propagate it - if self.exchange_manager.exchange.get_option_value( - enums.ExchangeClientOptions.CAN_MAKE_AUTHENTICATED_REQUESTS_WHEN_LOADING_MARKETS - ): - # can happen, just warn - self.logger.warning(f"{err.__class__.__name__} when loading markets: {err}") - else: - # unexpected: notify - self.logger.error(f"Unexpected error when loading markets: {err} ({err.__class__.__name__})") - raise - except ccxt.async_support.NetworkError as err: - raise octobot_trading.errors.NetworkError( - f"Failed to load_symbol_markets: {err.__class__.__name__} " - f"on {html_util.get_html_summary_if_relevant(err)}" - ) from err - except ccxt.async_support.ExchangeError as err: - # includes AuthenticationError but also auth error not identified as such by ccxt - if not self.force_authentication and self.is_authenticated: - self.logger.debug( - f"Credentials check enabled when fetching exchange market status, trying with " - f"unauthenticated client: {err}." - ) - # auth invalid but not required: fetch markets from another client - unauth_client = None - try: - unauth_client = self._client_factory(True)[0] - await self._load_markets(unauth_client, reload, market_filter=market_filter) - self._persist_markets_cache(unauth_client, False) - # apply markets to target client - ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter) - self.logger.debug( - f"Fetched exchange market status from unauthenticated client." - ) - finally: - if unauth_client: - await unauth_client.close() - else: - raise + await self._load_symbol_markets_under_client_lock(reload, authenticated_cache, market_filter) # markets are now loaded, trigger event commons_tree.EventProvider.instance().trigger_event( self.exchange_manager.bot_id, commons_tree.get_exchange_path( @@ -341,6 +292,85 @@ async def load_symbol_markets( ) ) + async def _load_symbol_markets_under_client_lock( + self, + reload: bool, + authenticated_cache: bool, + market_filter: typing.Optional[typing.Callable[[dict], bool]] = None, + ) -> None: + client_key = ccxt_clients_cache.get_client_key(self.client, authenticated_cache) + async with _get_markets_load_lock(client_key): + should_fetch_markets = reload + if not reload: + try: + ccxt_client_util.load_markets_from_cache( + self.client, authenticated_cache, market_filter=market_filter, + ) + except KeyError: + should_fetch_markets = True + if should_fetch_markets: + self.logger.info( + f"Loading {self.exchange_manager.exchange_name} " + f"{exchanges.get_exchange_type(self.exchange_manager).value}" + f"{' sandbox' if self.exchange_manager.is_sandboxed else ''} exchange markets ({reload=} {authenticated_cache=})" + ) + try: + await self._load_markets(self.client, reload, market_filter=market_filter) + self._persist_markets_cache() + except ccxt.async_support.OBIPWhitelistError as err: + raise octobot_trading.errors.InvalidAPIKeyIPWhitelistError( + f"Invalid IP whitelist error: {html_util.get_html_summary_if_relevant(err)}" + ) from err + except ( + ccxt.async_support.AuthenticationError, + ccxt.async_support.ArgumentsRequired, + ValueError, + binascii.Error, AssertionError, IndexError + ) as err: + self.set_first_consecutive_authentication_error_at_if_unset() + if self.force_authentication: + raise ccxt.async_support.AuthenticationError( + f"Invalid key format ({html_util.get_html_summary_if_relevant(err)})" + ) from err + # should not happen: if it does, propagate it + if self.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.CAN_MAKE_AUTHENTICATED_REQUESTS_WHEN_LOADING_MARKETS + ): + # can happen, just warn + self.logger.warning(f"{err.__class__.__name__} when loading markets: {err}") + else: + # unexpected: notify + self.logger.error(f"Unexpected error when loading markets: {err} ({err.__class__.__name__})") + raise + except ccxt.async_support.NetworkError as err: + raise octobot_trading.errors.NetworkError( + f"Failed to load_symbol_markets: {err.__class__.__name__} " + f"on {html_util.get_html_summary_if_relevant(err)}" + ) from err + except ccxt.async_support.ExchangeError as err: + # includes AuthenticationError but also auth error not identified as such by ccxt + if not self.force_authentication and self.is_authenticated: + self.logger.debug( + f"Credentials check enabled when fetching exchange market status, trying with " + f"unauthenticated client: {err}." + ) + # auth invalid but not required: fetch markets from another client + unauth_client = None + try: + unauth_client = self._client_factory(True)[0] + await self._load_markets(unauth_client, reload, market_filter=market_filter) + self._persist_markets_cache(unauth_client, False) + # apply markets to target client + ccxt_client_util.load_markets_from_cache(self.client, False, market_filter=market_filter) + self.logger.debug( + f"Fetched exchange market status from unauthenticated client." + ) + finally: + if unauth_client: + await unauth_client.close() + else: + raise + def get_client_symbols(self, active_only=True) -> set[str]: return ccxt_client_util.get_symbols(self.client, active_only) @@ -1335,9 +1365,11 @@ def get_uniform_timestamp(self, timestamp) -> float: return self.adapter.get_uniformized_timestamp(timestamp) async def stop(self) -> None: - self.logger.debug(f"Closing connection.") + if self.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug(f"Closing connection.") await ccxt_client_util.close_client(self.client) - self.logger.debug(f"Connection closed.") + if self.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug(f"Connection closed.") self.client = None self.exchange_manager = None diff --git a/packages/trading/octobot_trading/exchanges/exchange_channels.py b/packages/trading/octobot_trading/exchanges/exchange_channels.py index 5e9caa25ce..9800f906a8 100644 --- a/packages/trading/octobot_trading/exchanges/exchange_channels.py +++ b/packages/trading/octobot_trading/exchanges/exchange_channels.py @@ -146,9 +146,10 @@ async def _create_producer( producer_instance = producer(exchange_channel.get_chan(producer.CHANNEL_NAME, exchange_manager.id)) if exchanges.is_channel_managed_by_websocket(exchange_manager, producer.CHANNEL_NAME): # websocket is handling this channel: initialize data if required - exchange_manager.logger.debug( - f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel is updated by websocket feed" - ) + if exchange_manager.should_log_exchange_lifecycle_debug(): + exchange_manager.logger.debug( + f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel is updated by websocket feed" + ) start_producers = \ not exchanges.is_channel_fully_managed_by_websocket(exchange_manager, producer.CHANNEL_NAME) if exchanges.is_websocket_feed_requiring_init(exchange_manager, producer.CHANNEL_NAME): @@ -161,18 +162,20 @@ async def _create_producer( ) if start_producers: # no websocket for this channel (or channel is not fully managed by ws): start a producer - exchange_manager.logger.debug( - f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel " - f"is updated by {producer_instance.__class__.__name__}" - ) + if exchange_manager.should_log_exchange_lifecycle_debug(): + exchange_manager.logger.debug( + f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel " + f"is updated by {producer_instance.__class__.__name__}" + ) await producer_instance.run() elif ( subscribe_indirect_producers_if_not_started and isinstance(producer_instance, exchange_channel.IndirectExchangeChannelProducer) ): - exchange_manager.logger.debug( - f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel subscribing as indirect producer" - ) + if exchange_manager.should_log_exchange_lifecycle_debug(): + exchange_manager.logger.debug( + f"{exchange_manager.exchange_name} {producer.CHANNEL_NAME} channel subscribing as indirect producer" + ) await producer_instance.subscribe() else: # register producer to be able to reach it later on in modify() if needed diff --git a/packages/trading/octobot_trading/exchanges/exchange_manager.py b/packages/trading/octobot_trading/exchanges/exchange_manager.py index e5e21c5e81..57d7141939 100644 --- a/packages/trading/octobot_trading/exchanges/exchange_manager.py +++ b/packages/trading/octobot_trading/exchanges/exchange_manager.py @@ -378,6 +378,9 @@ def get_exchange_sub_account_id(self, exchange_name): config_exchange = self.config[common_constants.CONFIG_EXCHANGES][exchange_name] return config_exchange.get(common_constants.CONFIG_EXCHANGE_SUB_ACCOUNT, None) + def should_log_exchange_lifecycle_debug(self) -> bool: + return not self.exchange_only + def is_storage_enabled(self): return self.enable_storage and not self.exchange_only and self.bot_id is not None diff --git a/packages/trading/octobot_trading/exchanges/traders/trader.py b/packages/trading/octobot_trading/exchanges/traders/trader.py index 3acae21231..db43e166aa 100644 --- a/packages/trading/octobot_trading/exchanges/traders/trader.py +++ b/packages/trading/octobot_trading/exchanges/traders/trader.py @@ -115,9 +115,10 @@ async def initialize_impl(self): await self.exchange_manager.register_trader(self) if self.__class__.is_paused(self.config): self.logger.warning(f"Trading on {self.exchange_manager.exchange_name} is paused, it won't be trading") - self.logger.debug( - f"{'Enabled' if self.is_enabled else 'Disabled'} on {self.exchange_manager.exchange_name}" - ) + if self.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug( + f"{'Enabled' if self.is_enabled else 'Disabled'} on {self.exchange_manager.exchange_name}" + ) def set_is_enabled(self, enabled: bool): self.logger.info( diff --git a/packages/trading/octobot_trading/personal_data/orders/protocol.py b/packages/trading/octobot_trading/personal_data/orders/protocol.py index 8a4bd803f6..cc61621c79 100644 --- a/packages/trading/octobot_trading/personal_data/orders/protocol.py +++ b/packages/trading/octobot_trading/personal_data/orders/protocol.py @@ -35,9 +35,9 @@ def to_protocol_order( exchange_id=order_details[enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value], side=order_details[enums.ExchangeConstantsOrderColumns.SIDE.value], type=order_details[enums.ExchangeConstantsOrderColumns.TYPE.value], - trigger_above=order_details[enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value], - reduce_only=order_details[enums.ExchangeConstantsOrderColumns.REDUCE_ONLY.value], - is_active=order_details[enums.ExchangeConstantsOrderColumns.IS_ACTIVE.value], + trigger_above=order_details.get(enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value), + reduce_only=order_details.get(enums.ExchangeConstantsOrderColumns.REDUCE_ONLY.value, False), + is_active=order_details.get(enums.ExchangeConstantsOrderColumns.IS_ACTIVE.value, True), status=order_details[enums.ExchangeConstantsOrderColumns.STATUS.value], created_at=timestamp_util.utc_datetime_from_timestamp(order_details[enums.ExchangeConstantsOrderColumns.TIMESTAMP.value]), ) diff --git a/packages/trading/octobot_trading/personal_data/portfolios/portfolio.py b/packages/trading/octobot_trading/personal_data/portfolios/portfolio.py index a292a18e4f..7256f4654c 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/portfolio.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/portfolio.py @@ -74,24 +74,27 @@ def reset(self): """ self.portfolio = {} - def update_portfolio_from_balance(self, balance, force_replace=True): + def update_portfolio_from_balance(self, balance, force_replace=True, should_log_update=True): """ Update portfolio from a balance dict :param balance: the portfolio dict :param force_replace: force to update portfolio. Should be False when using deltas + :param should_log_update: when False, skip portfolio update debug logs (temporary exchanges) :return: True if the portfolio has been updated """ if force_replace: self.portfolio = { currency: self._parse_raw_currency_asset(currency=currency, raw_currency_balance=balance[currency]) for currency in balance} - self.logger.debug(f"Portfolio updated | {constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self)}") + if should_log_update: + self.logger.debug(f"Portfolio updated | {constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self)}") return True if any( self._update_raw_currency_asset(currency=currency, raw_currency_balance=balance[currency]) for currency in balance ): - self.logger.debug(f"Portfolio partially updated | {constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self)}") + if should_log_update: + self.logger.debug(f"Portfolio partially updated | {constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self)}") return True return False diff --git a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py index 919248d325..b72bfbd9dc 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/portfolio_manager.py @@ -85,7 +85,11 @@ def handle_balance_update(self, balance, is_diff_update=False): """ changed = False if self.trader.can_trade_if_not_paused() and balance is not None: - changed = self.portfolio.update_portfolio_from_balance(balance, force_replace=not is_diff_update) + changed = self.portfolio.update_portfolio_from_balance( + balance, + force_replace=not is_diff_update, + should_log_update=self.exchange_manager.should_log_exchange_lifecycle_debug(), + ) if not self._is_initialized_event_set: self._set_initialized_event() self._is_initialized_event_set = True @@ -381,9 +385,10 @@ def _load_portfolio(self, reset_from_config): self.apply_forced_portfolio() else: self._load_simulated_portfolio_from_history() - self.logger.debug( - f"{constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self.portfolio.portfolio)}" - ) + if self.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug( + f"{constants.CURRENT_PORTFOLIO_STRING} {logging.get_private_placeholder_if_necessary(self.portfolio.portfolio)}" + ) def _load_simulated_portfolio_from_history(self): portfolio_amount_dict = personal_data.parse_decimal_config_portfolio( diff --git a/packages/trading/octobot_trading/personal_data/portfolios/sub_portfolio.py b/packages/trading/octobot_trading/personal_data/portfolios/sub_portfolio.py index efe5cb14e1..f6b6c3592a 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/sub_portfolio.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/sub_portfolio.py @@ -29,8 +29,10 @@ def __init__(self, config, trader, parent_portfolio, percent, is_relative=True): super().__init__(config, trader) # overwrite parent update_portfolio_balance - def update_portfolio_from_balance(self, balance, force_replace=True): - modified = self.parent_portfolio.update_portfolio_from_balance(balance, force_replace=force_replace) + def update_portfolio_from_balance(self, balance, force_replace=True, should_log_update=True): + modified = self.parent_portfolio.update_portfolio_from_balance( + balance, force_replace=force_replace, should_log_update=should_log_update, + ) self.update_from_parent() return modified diff --git a/packages/trading/octobot_trading/personal_data/portfolios/value_converter.py b/packages/trading/octobot_trading/personal_data/portfolios/value_converter.py index ddc1b3d7c9..1d285dc636 100644 --- a/packages/trading/octobot_trading/personal_data/portfolios/value_converter.py +++ b/packages/trading/octobot_trading/personal_data/portfolios/value_converter.py @@ -82,7 +82,8 @@ def initialize_from_exchange_data( def update_last_price(self, symbol, price): if symbol not in self.last_prices_by_trading_pair: self.reset_missing_price_bridges() - self.logger.debug(f"Initialized last price for {symbol}") + if self.portfolio_manager.exchange_manager.should_log_exchange_lifecycle_debug(): + self.logger.debug(f"Initialized last price for {symbol}") self.last_prices_by_trading_pair[symbol] = price def evaluate_value(self, currency, quantity, raise_error=True, target_currency=None, init_price_fetchers=True): diff --git a/packages/trading/octobot_trading/util/__init__.py b/packages/trading/octobot_trading/util/__init__.py index 26e9eded01..46d4f21db9 100644 --- a/packages/trading/octobot_trading/util/__init__.py +++ b/packages/trading/octobot_trading/util/__init__.py @@ -26,6 +26,13 @@ from octobot_trading.util import simulator_updater_utils from octobot_trading.util import config_util +from octobot_trading.util import protocol_trading_mapping + +from octobot_trading.util.protocol_trading_mapping import ( + OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED, + TRADING_TYPE_TO_EXCHANGE_TYPE, + API_KEY_RIGHT_TO_ACCOUNT_PERMISSION, +) from octobot_trading.util.simulator_updater_utils import ( stop_and_pause, @@ -76,4 +83,7 @@ "get_current_bot_live_id", "get_config", "get_formatted_portfolio", + "TRADING_TYPE_TO_EXCHANGE_TYPE", + "API_KEY_RIGHT_TO_ACCOUNT_PERMISSION", + "OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED", ] diff --git a/packages/trading/octobot_trading/util/config_util.py b/packages/trading/octobot_trading/util/config_util.py index a22ea15ead..d162eadf35 100644 --- a/packages/trading/octobot_trading/util/config_util.py +++ b/packages/trading/octobot_trading/util/config_util.py @@ -227,7 +227,7 @@ def get_config( profile = commons_profiles.EphemeralProfile.from_profile_data(profile_data) profile_data.backtesting_context = initial_backtesting_context config.profile_by_id[profile.profile_id] = profile - config.select_profile(profile.profile_id) + config.select_profile(profile.profile_id, init_tentacles_setup=False) config.config[commons_constants.CONFIG_EXCHANGES][exchange_data.exchange_details.name] = get_exchange_config( exchange_data, tentacles_setup_config, profile_data.get_config_by_tentacle(), auth ) diff --git a/packages/trading/octobot_trading/util/protocol_trading_mapping.py b/packages/trading/octobot_trading/util/protocol_trading_mapping.py new file mode 100644 index 0000000000..a41e270c7f --- /dev/null +++ b/packages/trading/octobot_trading/util/protocol_trading_mapping.py @@ -0,0 +1,29 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums + + +TRADING_TYPE_TO_EXCHANGE_TYPE: dict[protocol_models.TradingType, trading_enums.ExchangeTypes] = { + protocol_models.TradingType.SPOT: trading_enums.ExchangeTypes.SPOT, + protocol_models.TradingType.FUTURES: trading_enums.ExchangeTypes.FUTURE, + protocol_models.TradingType.OPTIONS: trading_enums.ExchangeTypes.OPTION, + protocol_models.TradingType.MARGIN: trading_enums.ExchangeTypes.MARGIN, +} + +API_KEY_RIGHT_TO_ACCOUNT_PERMISSION: dict[ + trading_enums.APIKeyRights, + protocol_models.AccountPermission, +] = { + trading_enums.APIKeyRights.READING: protocol_models.AccountPermission.READ, + trading_enums.APIKeyRights.SPOT_TRADING: protocol_models.AccountPermission.SPOT_TRADING, + trading_enums.APIKeyRights.FUTURES_TRADING: protocol_models.AccountPermission.FUTURES_TRADING, + trading_enums.APIKeyRights.WITHDRAWALS: protocol_models.AccountPermission.WITHDRAW, +} + +OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED: list[trading_enums.APIKeyRights] = [ + trading_enums.APIKeyRights.READING, + trading_enums.APIKeyRights.SPOT_TRADING, + trading_enums.APIKeyRights.FUTURES_TRADING, +] diff --git a/packages/trading/tests/api/test_portfolio.py b/packages/trading/tests/api/test_portfolio.py index bb127e7ac4..3da4759504 100644 --- a/packages/trading/tests/api/test_portfolio.py +++ b/packages/trading/tests/api/test_portfolio.py @@ -14,7 +14,25 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. -import pytest +import mock +import octobot_commons.constants as commons_constants +import octobot_trading.api.portfolio as portfolio_api +import octobot_trading.enums as trading_enums -# All test coroutines will be treated as marked. -pytestmark = pytest.mark.asyncio + +class TestResolvePortfolioValuationUnit: + def test_returns_exchange_default_quote_currency_when_set(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value.return_value = "USDC" + assert portfolio_api.resolve_portfolio_valuation_unit(exchange_manager) == "USDC" + exchange_manager.exchange.get_option_value.assert_called_once_with( + trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY + ) + + def test_falls_back_to_default_reference_market_when_option_missing(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value.return_value = None + assert ( + portfolio_api.resolve_portfolio_valuation_unit(exchange_manager) + == commons_constants.DEFAULT_REFERENCE_MARKET + ) diff --git a/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py b/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py index 333fa1baf1..2e516b7827 100644 --- a/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py +++ b/packages/trading/tests/exchange_data/ticker/test_ticker_cache.py @@ -108,3 +108,40 @@ def test_get_exchange_key(): assert exchange_data.TickerCache.get_exchange_key("binance", "spot", False) == "binance_spot_False" assert exchange_data.TickerCache.get_exchange_key("binance", "future", False) == "binance_future_False" assert exchange_data.TickerCache.get_exchange_key("okx", "future", False) == "okx_future_False" + + +def test_set_all_tickers_skips_unparseable_symbols(ticker_cache): + tickers_with_invalid_symbol = { + **SPOT_TICKERS, + "INVALID:SYMBOL": mock.Mock(), + } + ticker_cache.set_all_tickers("binance", "spot", False, tickers_with_invalid_symbol) + assert ticker_cache.get_all_tickers("binance", "spot", False) == tickers_with_invalid_symbol + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols("binance", "spot", False) == { + "BTCUSDT": octobot_commons.symbols.parse_symbol("BTC/USDT"), + "ETHUSDT": octobot_commons.symbols.parse_symbol("ETH/USDT"), + "SOLUSDT": octobot_commons.symbols.parse_symbol("SOL/USDT"), + } + + +def test_set_all_tickers_skips_unparseable_symbols_for_futures(ticker_cache): + futures_tickers_with_invalid_symbol = { + **FUTURES_TICKERS, + "INVALID:SYMBOL": mock.Mock(), + } + ticker_cache.set_all_tickers( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False, futures_tickers_with_invalid_symbol + ) + assert ticker_cache.get_all_tickers( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False + ) == futures_tickers_with_invalid_symbol + assert ticker_cache.get_all_parsed_symbols_by_merged_symbols( + "binance", octobot_commons.constants.CONFIG_EXCHANGE_FUTURE, False + ) == { + "BTCUSDT": octobot_commons.symbols.parse_symbol("BTC/USDT:USDT"), + "BTCUSDT:USDT": octobot_commons.symbols.parse_symbol("BTC/USDT:USDT"), + "ETHUSDT": octobot_commons.symbols.parse_symbol("ETH/USDT:USDT"), + "ETHUSDT:USDT": octobot_commons.symbols.parse_symbol("ETH/USDT:USDT"), + "SOLUSD": octobot_commons.symbols.parse_symbol("SOL/USD:SOL"), + "SOLUSD:SOL": octobot_commons.symbols.parse_symbol("SOL/USD:SOL"), + } diff --git a/packages/trading/tests/exchange_data/ticker/test_ticker_updater.py b/packages/trading/tests/exchange_data/ticker/test_ticker_updater.py new file mode 100644 index 0000000000..296e4790d1 --- /dev/null +++ b/packages/trading/tests/exchange_data/ticker/test_ticker_updater.py @@ -0,0 +1,82 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest +import asyncio + +import octobot_trading.exchange_data.ticker.channel.ticker_updater as ticker_updater_module + +pytestmark = pytest.mark.asyncio + + +class TestTickerUpdaterFetchAllTickers: + async def test_returns_only_requested_symbols_when_exchange_fetches_all_tickers(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.is_sandboxed = False + exchange_manager.exchange.get_option_value = mock.Mock(return_value=False) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + updater = ticker_updater_module.TickerUpdater(channel) + updater.set_cache = mock.Mock() + all_tickers = { + "BTC/USDC": {"close": 100000}, + "ETH/USDC": {"close": 3000}, + "INVALID:SYMBOL": {"close": 1}, + } + updater._fetch_missing_tickers = mock.AsyncMock(return_value=all_tickers) + + with mock.patch.object( + ticker_updater_module._TICKER_CACHE, + "get_all_tickers", + return_value={}, + ): + tickers = await updater.fetch_all_tickers(["BTC/USDC", "ETH/USDC"]) + + assert tickers == { + "BTC/USDC": {"close": 100000}, + "ETH/USDC": {"close": 3000}, + } + updater.set_cache.assert_called_once_with("kraken", mock.ANY, False, all_tickers) + + async def test_concurrent_fetch_all_tickers_fetches_missing_symbols_once(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "bitmart" + exchange_manager.is_sandboxed = False + exchange_manager.exchange.get_option_value = mock.Mock(return_value=True) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + updater = ticker_updater_module.TickerUpdater(channel) + cached_tickers: dict[str, dict] = {} + + def set_cache_side_effect(exchange_name, exchange_type, sandboxed, tickers): + cached_tickers.update(tickers) + + updater.set_cache = mock.Mock(side_effect=set_cache_side_effect) + fetched_tickers = { + "BTC/USDT": {"close": 100000}, + "ETH/USDT": {"close": 3000}, + } + fetch_count = 0 + + async def fetch_missing_tickers(_missing_symbols): + nonlocal fetch_count + fetch_count += 1 + return fetched_tickers + + updater._fetch_missing_tickers = fetch_missing_tickers + + with mock.patch.object( + ticker_updater_module._TICKER_CACHE, + "get_all_tickers", + side_effect=lambda *_args, **_kwargs: cached_tickers, + ): + results = await asyncio.gather( + updater.fetch_all_tickers(["BTC/USDT", "ETH/USDT"]), + updater.fetch_all_tickers(["BTC/USDT", "ETH/USDT"]), + ) + + assert fetch_count == 1 + assert results[0] == fetched_tickers + assert results[1] == fetched_tickers diff --git a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_client_util.py b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_client_util.py index 98397fef35..fa4069b178 100644 --- a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_client_util.py +++ b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_client_util.py @@ -177,3 +177,48 @@ async def _exchange_with_proxy_config(proxy_config: exchanges.ExchangeProxyConfi if exchange: exchange.timeout_on_exit = 0 # avoid waiting for the exchange to close await exchange.close() + + +class TestSetSandboxMode: + def _exchange_connector(self, *, raise_type_error: bool): + client = mock.Mock() + client.options = {} + if raise_type_error: + client.set_sandbox_mode.side_effect = TypeError("'NoneType' object is not iterable") + else: + client.set_sandbox_mode = mock.Mock() + exchange_connector = mock.Mock() + exchange_connector.client = client + exchange_connector.name = "test-exchange" + exchange_connector.logger = mock.Mock() + exchange_connector.exchange_manager.exchange.uses_demo_trading_instead_of_sandbox.return_value = False + return exchange_connector + + def test_type_error_when_sandboxed_raises_not_supported(self): + exchange_connector = self._exchange_connector(raise_type_error=True) + try: + ccxt_client_util.set_sandbox_mode(exchange_connector, True) + raise AssertionError("Expected ccxt.NotSupported") + except ccxt.NotSupported as error: + assert "test-exchange" in str(error) + assert "sandbox/test URLs are unavailable" in str(error) + exchange_connector.logger.warning.assert_called_once() + warning_message = exchange_connector.logger.warning.call_args.args[0] + assert "sandbox/test URLs are unavailable" in warning_message + + def test_type_error_when_not_sandboxed_is_ignored(self): + exchange_connector = self._exchange_connector(raise_type_error=True) + result = ccxt_client_util.set_sandbox_mode(exchange_connector, False) + assert result is None + exchange_connector.logger.warning.assert_not_called() + + def test_not_supported_is_re_raised_with_clear_message(self): + exchange_connector = self._exchange_connector(raise_type_error=False) + exchange_connector.client.set_sandbox_mode.side_effect = ccxt.NotSupported("sandbox mode") + try: + ccxt_client_util.set_sandbox_mode(exchange_connector, True) + raise AssertionError("Expected ccxt.NotSupported") + except ccxt.NotSupported as error: + assert "test-exchange" in str(error) + assert "sandbox/test URLs are unavailable" in str(error) + exchange_connector.logger.warning.assert_called_once() diff --git a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py index 6b6dceded7..83672aa316 100644 --- a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py +++ b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py @@ -584,6 +584,48 @@ async def test_skips_cache_persist_when_tickers_do_not_add_symbols(self, ccxt_co persist_markets_cache_mock.assert_not_called() +class TestLoadSymbolMarketsLock: + async def test_concurrent_load_symbol_markets_fetches_once(self, exchange_manager): + ccxt_connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + ccxt_connector.client = mock.Mock() + ccxt_connector.client.__class__.__name__ = "binance" + ccxt_connector.client.urls = {"api": {"public": "https://api.binance.com"}} + ccxt_connector.client.apiKey = "test-key" + ccxt_connector.exchange_manager.exchange.requires_authentication_for_this_configuration_only = ( + mock.Mock(return_value=True) + ) + ccxt_connector._persist_markets_cache = mock.Mock() + markets_loaded = False + + def load_from_cache(*_args, **_kwargs): + if not markets_loaded: + raise KeyError() + + load_count = 0 + + async def delayed_load(*_args, **_kwargs): + nonlocal load_count, markets_loaded + load_count += 1 + markets_loaded = True + import asyncio + await asyncio.sleep(0.05) + + ccxt_connector._load_markets = delayed_load + + with mock.patch.object( + ccxt_client_util, + "load_markets_from_cache", + side_effect=load_from_cache, + ): + import asyncio + await asyncio.gather( + ccxt_connector.load_symbol_markets(reload=False), + ccxt_connector.load_symbol_markets(reload=False), + ) + + assert load_count == 1 + + def _get_fees(type, currency, rate, cost): return { enums.FeePropertyColumns.TYPE.value: type, diff --git a/packages/trading/tests/exchanges/traders/test_trader_initialize_impl_logging.py b/packages/trading/tests/exchanges/traders/test_trader_initialize_impl_logging.py new file mode 100644 index 0000000000..122bfa7a53 --- /dev/null +++ b/packages/trading/tests/exchanges/traders/test_trader_initialize_impl_logging.py @@ -0,0 +1,39 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot_trading.exchanges.traders.trader as trader_module + +pytestmark = pytest.mark.asyncio + + +class TestTraderInitializeImplLogging: + async def test_does_not_log_enabled_on_when_exchange_only(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.is_trading = False + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=False) + exchange_manager.register_trader = mock.AsyncMock() + trader = trader_module.Trader({}, exchange_manager) + trader.is_enabled = True + trader.logger = mock.Mock() + + await trader.initialize_impl() + + trader.logger.debug.assert_not_called() + + async def test_logs_enabled_on_when_lifecycle_debug_enabled(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.is_trading = False + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=True) + exchange_manager.register_trader = mock.AsyncMock() + trader = trader_module.Trader({}, exchange_manager) + trader.is_enabled = True + trader.logger = mock.Mock() + + await trader.initialize_impl() + + trader.logger.debug.assert_called_once_with("Disabled on kraken") diff --git a/packages/trading/tests/personal_data/orders/test_protocol.py b/packages/trading/tests/personal_data/orders/test_protocol.py new file mode 100644 index 0000000000..be40767b93 --- /dev/null +++ b/packages/trading/tests/personal_data/orders/test_protocol.py @@ -0,0 +1,35 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. + +import datetime + +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums +import octobot_trading.personal_data.orders.protocol as orders_protocol + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +class TestToProtocolOrder: + def test_defaults_optional_fields_when_missing_from_ccxt_order(self): + ccxt_like_order = { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.01, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + + protocol_order = orders_protocol.to_protocol_order(ccxt_like_order) + + assert protocol_order.id == "order-1" + assert protocol_order.exchange_id == "order-1" + assert protocol_order.trigger_above is None + assert protocol_order.reduce_only is False + assert protocol_order.is_active is True diff --git a/packages/trading/tests/personal_data/portfolios/test_portfolio.py b/packages/trading/tests/personal_data/portfolios/test_portfolio.py index 9ff4199236..9382e2da01 100644 --- a/packages/trading/tests/personal_data/portfolios/test_portfolio.py +++ b/packages/trading/tests/personal_data/portfolios/test_portfolio.py @@ -31,6 +31,7 @@ from octobot_trading.personal_data.orders import BuyMarketOrder from octobot_trading.personal_data.orders.types.market.sell_market_order import SellMarketOrder import octobot_trading.personal_data.orders.groups as order_groups +import octobot_trading.personal_data as personal_data from tests.test_utils.order_util import fill_market_order, fill_limit_or_stop_order from tests.exchanges import backtesting_trader, backtesting_config, backtesting_exchange_manager, fake_backtesting @@ -125,6 +126,34 @@ async def test_update_portfolio_from_balance(backtesting_trader): assert not portfolio_manager.portfolio.update_portfolio_from_balance(test_portfolio, force_replace=False) +class TestUpdatePortfolioFromBalanceLogging: + pytestmark = [] + + def test_does_not_log_when_should_log_update_false(self): + portfolio = personal_data.SpotPortfolio("binance", is_simulated=False) + portfolio.logger = mock.Mock() + balance = { + "BTC": { + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("1"), + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("1"), + }, + } + portfolio.update_portfolio_from_balance(balance, should_log_update=False) + portfolio.logger.debug.assert_not_called() + + def test_logs_when_should_log_update_true(self): + portfolio = personal_data.SpotPortfolio("binance", is_simulated=False) + portfolio.logger = mock.Mock() + balance = { + "BTC": { + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("1"), + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("1"), + }, + } + portfolio.update_portfolio_from_balance(balance, should_log_update=True) + portfolio.logger.debug.assert_called_once() + + async def test_update_portfolio_from_balance_with_deltas(backtesting_trader): config, exchange_manager, trader = backtesting_trader portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager diff --git a/packages/trading/tests/personal_data/portfolios/test_portfolio_manager.py b/packages/trading/tests/personal_data/portfolios/test_portfolio_manager.py index ec7f93000a..22687aff19 100644 --- a/packages/trading/tests/personal_data/portfolios/test_portfolio_manager.py +++ b/packages/trading/tests/personal_data/portfolios/test_portfolio_manager.py @@ -59,6 +59,38 @@ async def test_handle_balance_update(backtesting_trader): portfolio_manager.handle_balance_update({}) update_portfolio_from_balance_mock.assert_called_once() can_trade_if_not_paused_mock.assert_called_once() + call_kwargs = update_portfolio_from_balance_mock.call_args.kwargs + assert call_kwargs["should_log_update"] == exchange_manager.should_log_exchange_lifecycle_debug() + + +class TestLoadPortfolioLogging: + pytestmark = [] + + def test_does_not_log_current_portfolio_when_exchange_only(self): + portfolio_manager = mock.Mock() + portfolio_manager.trader.can_trade_if_not_paused = mock.Mock(return_value=True) + portfolio_manager.trader.simulate = True + portfolio_manager.exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=False) + portfolio_manager.historical_portfolio_value_manager = None + portfolio_manager.logger = mock.Mock() + portfolio_manager.apply_forced_portfolio = mock.Mock() + + personal_data.PortfolioManager._load_portfolio(portfolio_manager, False) + + portfolio_manager.apply_forced_portfolio.assert_called_once() + portfolio_manager.logger.debug.assert_not_called() + + def test_logs_current_portfolio_when_lifecycle_debug_enabled(self): + portfolio_manager = mock.Mock() + portfolio_manager.trader.can_trade_if_not_paused = mock.Mock(return_value=True) + portfolio_manager.trader.simulate = False + portfolio_manager.exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=True) + portfolio_manager.portfolio.portfolio = {} + portfolio_manager.logger = mock.Mock() + + personal_data.PortfolioManager._load_portfolio(portfolio_manager, False) + + portfolio_manager.logger.debug.assert_called_once() async def test_handle_balance_update_from_order(backtesting_trader): diff --git a/packages/trading/tests/personal_data/portfolios/test_value_converter.py b/packages/trading/tests/personal_data/portfolios/test_value_converter.py index 080224e57c..b60647eba7 100644 --- a/packages/trading/tests/personal_data/portfolios/test_value_converter.py +++ b/packages/trading/tests/personal_data/portfolios/test_value_converter.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import decimal +import mock import pytest import octobot_trading.constants as constants @@ -22,6 +23,7 @@ import octobot_commons.constants as commons_constants +from tests import event_loop from tests.exchanges import backtesting_trader, backtesting_config, backtesting_exchange_manager, fake_backtesting @@ -182,3 +184,33 @@ def test_can_convert_symbol_to_usd_like(): f"{commons_constants.USD_LIKE_COINS[4]}/BTC" ) is True assert trading_personal_data.ValueConverter.can_convert_symbol_to_usd_like("BTC/ETH") is False + + +class TestValueConverterUpdateLastPriceLogging: + @pytest.mark.asyncio + async def test_does_not_log_initialized_last_price_when_exchange_only(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=False) + portfolio_manager = mock.Mock() + portfolio_manager.exchange_manager = exchange_manager + value_converter = trading_personal_data.ValueConverter(portfolio_manager) + value_converter.logger = mock.Mock() + + value_converter.update_last_price("BTC/USDT", decimal.Decimal("100")) + + value_converter.logger.debug.assert_not_called() + + @pytest.mark.asyncio + async def test_logs_initialized_last_price_when_lifecycle_debug_enabled(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "kraken" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=True) + portfolio_manager = mock.Mock() + portfolio_manager.exchange_manager = exchange_manager + value_converter = trading_personal_data.ValueConverter(portfolio_manager) + value_converter.logger = mock.Mock() + + value_converter.update_last_price("BTC/USDT", decimal.Decimal("100")) + + value_converter.logger.debug.assert_called_once_with("Initialized last price for BTC/USDT") diff --git a/packages/trading/tests/test_exchange_channel_producer.py b/packages/trading/tests/test_exchange_channel_producer.py new file mode 100644 index 0000000000..1828c21118 --- /dev/null +++ b/packages/trading/tests/test_exchange_channel_producer.py @@ -0,0 +1,72 @@ +# Drakkar-Software OctoBot-Trading +# Copyright (c) Drakkar-Software, All rights reserved. + +import mock +import pytest + +import octobot_trading.exchange_channel as exchange_channel_module + +pytestmark = pytest.mark.asyncio + + +class TestExchangeChannelProducerPause: + async def test_does_not_log_when_exchange_only(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "binance" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=False) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + channel.is_paused = False + producer = exchange_channel_module.ExchangeChannelProducer(channel) + producer.logger = mock.Mock() + + await producer.pause() + + producer.logger.debug.assert_not_called() + assert producer.is_running is False + assert channel.is_paused is True + + async def test_logs_when_lifecycle_debug_enabled(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "binance" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=True) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + channel.is_paused = False + producer = exchange_channel_module.ExchangeChannelProducer(channel) + producer.logger = mock.Mock() + + await producer.pause() + + producer.logger.debug.assert_called_once_with("Pausing...") + + +class TestExchangeChannelProducerResume: + async def test_does_not_log_when_exchange_only(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "binance" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=False) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + channel.is_paused = True + producer = exchange_channel_module.ExchangeChannelProducer(channel) + producer.logger = mock.Mock() + + await producer.resume() + + producer.logger.debug.assert_not_called() + assert channel.is_paused is False + + async def test_logs_when_lifecycle_debug_enabled(self): + exchange_manager = mock.Mock() + exchange_manager.exchange_name = "binance" + exchange_manager.should_log_exchange_lifecycle_debug = mock.Mock(return_value=True) + channel = mock.Mock() + channel.exchange_manager = exchange_manager + channel.is_paused = True + producer = exchange_channel_module.ExchangeChannelProducer(channel) + producer.logger = mock.Mock() + + await producer.resume() + + producer.logger.debug.assert_called_once_with("Resuming...") diff --git a/packages/trading/tests/util/test_config_util.py b/packages/trading/tests/util/test_config_util.py index 18b34c8001..a3fa573bfe 100644 --- a/packages/trading/tests/util/test_config_util.py +++ b/packages/trading/tests/util/test_config_util.py @@ -14,8 +14,11 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import copy +import mock import pytest +import octobot_commons.profiles as commons_profiles +import octobot_commons.profiles.profile_data as profile_data_module import octobot_trading.util as util import octobot_trading.constants as trading_constants import octobot_commons.symbols as symbol_util @@ -205,3 +208,43 @@ def _replace_value_by_key(pairs_by_cypto, filtered_key, replaced_value): def _select_by_base_or_quote(pairs, base_or_quote): return [s for s in pairs if base_or_quote in symbol_util.parse_symbol(s).base_and_quote()] + + +class TestGetConfig: + def test_skips_tentacles_setup_rebuild_without_binding_shared_setup(self): + profile_data = profile_data_module.ProfileData() + profile_data.trader_simulator.enabled = True + exchange_data = mock.Mock() + exchange_data.exchange_details.name = "binance" + exchange_data.portfolio_details.content = {} + market = mock.Mock() + market.time_frame = "1h" + exchange_data.markets = [market] + exchange_data.auth_details.sandboxed = False + exchange_data.auth_details.api_key = None + exchange_data.auth_details.api_secret = None + exchange_data.auth_details.api_password = None + exchange_data.auth_details.access_token = None + exchange_data.auth_details.exchange_type = commons_constants.CONFIG_EXCHANGE_SPOT + tentacles_setup_config = mock.Mock() + tentacles_setup_config.profile = None + with ( + mock.patch( + "octobot_tentacles_manager.configuration.profile_tentacles_util.build_setup_config_from_profile_data", + ) as build_setup_mock, + mock.patch( + "octobot_trading.util.config_util.get_exchange_config", + return_value={}, + ), + ): + configuration = util.get_config( + profile_data, + exchange_data, + tentacles_setup_config, + auth=False, + ignore_symbols_in_exchange_init=False, + use_exchange_data_portfolio=True, + ) + build_setup_mock.assert_not_called() + assert tentacles_setup_config.profile is None + assert configuration.profile.tentacles_setup_config is None diff --git a/packages/trading/tests/util/test_protocol_trading_mapping.py b/packages/trading/tests/util/test_protocol_trading_mapping.py new file mode 100644 index 0000000000..9c6abddae7 --- /dev/null +++ b/packages/trading/tests/util/test_protocol_trading_mapping.py @@ -0,0 +1,45 @@ +# Drakkar-Software OctoBot-Trading + +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + + +class TestTradingTypeToExchangeType: + def test_maps_spot_trading_type(self): + assert ( + protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + protocol_models.TradingType.SPOT, + ) + == trading_enums.ExchangeTypes.SPOT + ) + + +class TestApiKeyRightToAccountPermission: + def test_maps_known_api_key_rights_to_account_permissions(self): + account_permissions = [ + protocol_trading_mapping.API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right) + for api_key_right in [ + trading_enums.APIKeyRights.READING, + trading_enums.APIKeyRights.SPOT_TRADING, + trading_enums.APIKeyRights.FUTURES_TRADING, + trading_enums.APIKeyRights.WITHDRAWALS, + trading_enums.APIKeyRights.MARGIN_TRADING, + ] + if protocol_trading_mapping.API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right) is not None + ] + assert account_permissions == [ + protocol_models.AccountPermission.READ, + protocol_models.AccountPermission.SPOT_TRADING, + protocol_models.AccountPermission.FUTURES_TRADING, + protocol_models.AccountPermission.WITHDRAW, + ] + + +class TestOptimisticApiKeyRightsWhenPermissionsUnsupported: + def test_contains_expected_rights(self): + assert protocol_trading_mapping.OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED == [ + trading_enums.APIKeyRights.READING, + trading_enums.APIKeyRights.SPOT_TRADING, + trading_enums.APIKeyRights.FUTURES_TRADING, + ] diff --git a/tests/unit_tests/community/test_authentication.py b/tests/unit_tests/community/test_authentication.py index 45679fb5d8..1213f6688a 100644 --- a/tests/unit_tests/community/test_authentication.py +++ b/tests/unit_tests/community/test_authentication.py @@ -510,6 +510,39 @@ async def test_stop(auth): auth._fetch_account_task.cancel.assert_called_once() +class TestCommunityAuthenticationStopLogging: + async def test_non_singleton_stop_does_not_log_lifecycle(self): + auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + auth._use_as_singleton = False + auth.logger = mock.Mock() + auth._fetch_account_task = None + auth.supabase_client = mock.Mock(aclose=mock.AsyncMock()) + auth._community_feed = None + auth.community_bot = mock.Mock(clear=mock.Mock()) + auth._sync_client = None + + await auth.stop() + + logged_messages = [call.args[0] for call in auth.logger.debug.call_args_list] + assert "Stopping ..." not in logged_messages + assert "Stopped" not in logged_messages + + async def test_singleton_stop_logs_lifecycle(self): + auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + auth._use_as_singleton = True + auth.logger = mock.Mock() + auth._fetch_account_task = None + auth.supabase_client = mock.Mock(aclose=mock.AsyncMock()) + auth._community_feed = None + auth.community_bot = mock.Mock(clear=mock.Mock()) + auth._sync_client = None + + await auth.stop() + + auth.logger.debug.assert_any_call("Stopping ...") + auth.logger.debug.assert_any_call("Stopped") + + def test_is_node_wallet_configured(auth): auth._wallet_backend = mock.Mock() auth._wallet_backend.list_wallets.return_value = [] From b793660569d15aa233627d205d3ce38216ffe348 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Mon, 24 Aug 2026 16:56:28 +0200 Subject: [PATCH 03/10] [Copy] handle outdated orders signals --- packages/copy/octobot_copy/constants.py | 1 + .../octobot_copy/copiers/account_copier.py | 5 + .../copy/octobot_copy/copiers/formatter.py | 77 +++++++ packages/copy/octobot_copy/errors.py | 6 + .../copy/octobot_copy/validators/__init__.py | 5 + .../reference_account_outdated_orders.py | 99 +++++++++ .../python/copiers/test_account_copier.py | 32 +++ .../tests/python/copiers/test_formatter.py | 192 ++++++++++++++++++ .../test_reference_account_outdated_orders.py | 161 +++++++++++++++ .../logic/dsl/dsl_action_execution_context.py | 5 +- 10 files changed, 582 insertions(+), 1 deletion(-) create mode 100644 packages/copy/octobot_copy/copiers/formatter.py create mode 100644 packages/copy/octobot_copy/validators/__init__.py create mode 100644 packages/copy/octobot_copy/validators/reference_account_outdated_orders.py create mode 100644 packages/copy/tests/python/copiers/test_formatter.py create mode 100644 packages/copy/tests/python/validators/test_reference_account_outdated_orders.py diff --git a/packages/copy/octobot_copy/constants.py b/packages/copy/octobot_copy/constants.py index 3942c18e19..337fb76d64 100644 --- a/packages/copy/octobot_copy/constants.py +++ b/packages/copy/octobot_copy/constants.py @@ -34,6 +34,7 @@ # Exchange / order lifecycle (seconds) FILL_ORDER_TIMEOUT = 60 OPEN_ORDER_POLL_INTERVAL = 0.5 +OUTDATED_ORDER_PRICE_MAX_THRESHOLD = 0.03 # 3% # Mirrored orphan grace: max |simulated_copier_pair_share − ref_pair_share| to allow deferral DEFAULT_MIRRORED_ORPHAN_GRACE_PAIR_RATIO_MAX_DELTA = decimal.Decimal("0.02") # 2% diff --git a/packages/copy/octobot_copy/copiers/account_copier.py b/packages/copy/octobot_copy/copiers/account_copier.py index 96bc0b069c..a66d840fb8 100644 --- a/packages/copy/octobot_copy/copiers/account_copier.py +++ b/packages/copy/octobot_copy/copiers/account_copier.py @@ -13,6 +13,7 @@ import octobot_copy.errors as copy_errors import octobot_copy.exchange as copy_exchange import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_module +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders_module import octobot_copy.rebalancing as copy_rebalancing @@ -47,6 +48,10 @@ def __init__( ) async def copy_account(self) -> copy_entities.AccountCopyResult: + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + self._reference_account, + self._copier_exchange_interface, + ) await self._resync_if_mirrored_open_order_grace_period_elapsed() rebalancer, should_rebalance, details = await self._prepare_rebalance_plan() if self._orders_synchronizer.is_mirrored_orphan_grace_invalid_no_compliant_snapshot(): diff --git a/packages/copy/octobot_copy/copiers/formatter.py b/packages/copy/octobot_copy/copiers/formatter.py new file mode 100644 index 0000000000..21bdaf5f3f --- /dev/null +++ b/packages/copy/octobot_copy/copiers/formatter.py @@ -0,0 +1,77 @@ +import typing + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +REFERENCE_ACCOUNT_SUMMARY_LIMIT = 10 + + +def format_reference_account_summary(reference_account: protocol_models.CopiedAccount) -> str: + copied_assets = reference_account.copied_assets + order_list = reference_account.orders or [] + asset_entries = [_format_copied_asset_entry(copied_asset) for copied_asset in copied_assets] + assets_summary = _format_compact_section("assets", asset_entries, len(copied_assets)) + orders_summary = _format_orders_section(order_list) + updated_at = _format_reference_account_updated_at(reference_account.updated_at) + return f"v{reference_account.version}@{updated_at} {assets_summary} {orders_summary}" + + +def _format_reference_account_updated_at(updated_at: typing.Union[float, int]) -> str: + formatted_time = timestamp_util.convert_timestamp_to_datetime( + float(updated_at), + local_timezone=False, + ) + return f"{formatted_time} UTC" + + +def _format_copied_asset_entry(asset: protocol_models.CopiedAsset) -> str: + ratio_percent = float(asset.ratio) * 100 + return f"{asset.name}:{ratio_percent:.1f}%" + + +def _format_reference_order_entry(order: protocol_models.Order) -> str: + return f"{order.side.value} {order.symbol}@{order.price}" + + +def _format_order_type_label(order: protocol_models.Order) -> str: + return f"{order.side.value}_{order.type.value}" + + +def _format_order_type_count_label(orders: list[protocol_models.Order]) -> str: + if not orders: + return "0" + order_type_counts: dict[str, int] = {} + for order in orders: + order_type_label = _format_order_type_label(order) + order_type_counts[order_type_label] = order_type_counts.get(order_type_label, 0) + 1 + return ",".join( + f"{order_type_counts[order_type_label]} {order_type_label}" + for order_type_label in sorted(order_type_counts) + ) + + +def _format_orders_section(order_list: list[protocol_models.Order]) -> str: + order_type_count_label = _format_order_type_count_label(order_list) + if not order_list: + return "orders[0]" + order_entries = [_format_reference_order_entry(order) for order in order_list] + content = _format_compact_list(order_entries, len(order_list)) + return f"orders[{order_type_count_label}]:{content}" + + +def _format_compact_list(entries: list[str], total_count: int) -> str: + if total_count == 0: + return "" + displayed_entries = entries[:REFERENCE_ACCOUNT_SUMMARY_LIMIT] + content = ",".join(displayed_entries) + remaining_count = total_count - len(displayed_entries) + if remaining_count > 0: + content = f"{content},…+{remaining_count} more" + return content + + +def _format_compact_section(section_name: str, entries: list[str], total_count: int) -> str: + if total_count == 0: + return f"{section_name}[0]" + content = _format_compact_list(entries, total_count) + return f"{section_name}[{total_count}]:{content}" diff --git a/packages/copy/octobot_copy/errors.py b/packages/copy/octobot_copy/errors.py index d0b72ce0fd..674e2dacb2 100644 --- a/packages/copy/octobot_copy/errors.py +++ b/packages/copy/octobot_copy/errors.py @@ -14,3 +14,9 @@ class RebalanceAborted(RebalanceError): """ Raised when a rebalance is aborted """ + + +class OutdatedReferenceAccountError(OctobotCopyError): + """ + Raised when the reference account is outdated + """ diff --git a/packages/copy/octobot_copy/validators/__init__.py b/packages/copy/octobot_copy/validators/__init__.py new file mode 100644 index 0000000000..f0b792805c --- /dev/null +++ b/packages/copy/octobot_copy/validators/__init__.py @@ -0,0 +1,5 @@ +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders + +__all__ = [ + "reference_account_outdated_orders", +] diff --git a/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py b/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py new file mode 100644 index 0000000000..d119448339 --- /dev/null +++ b/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py @@ -0,0 +1,99 @@ +import decimal + +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.personal_data as trading_personal_data + +import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors +import octobot_copy.exchange as copy_exchange + + +def resolve_order_trigger_above(order: protocol_models.Order) -> bool: + if order.trigger_above is not None: + return bool(order.trigger_above) + if order.side is protocol_models.Side.SELL: + return True + return False + + +def get_replicable_reference_orders( + reference_account: protocol_models.CopiedAccount, +) -> list[protocol_models.Order]: + replicable: list[protocol_models.Order] = [] + for order in reference_account.orders or []: + if order.status != protocol_models.OrderStatus.OPEN: + continue + if not order.is_active: + continue + raw = trading_personal_data.exchange_columns_dict_from_protocol_order(order) + _side, trader_order_type = trading_personal_data.parse_order_type(raw) + if trader_order_type in ( + trading_enums.TraderOrderType.BUY_MARKET, + trading_enums.TraderOrderType.SELL_MARKET, + ): + continue + replicable.append(order) + return replicable + + +def is_order_impossible_at_market_price( + order: protocol_models.Order, + market_price: decimal.Decimal, + threshold: decimal.Decimal, +) -> bool: + if market_price <= trading_constants.ZERO: + raise ValueError( + f"Market price must be positive to validate order {order.id!r} on {order.symbol!r}, got {market_price}" + ) + order_price = decimal.Decimal(str(order.price)) + if order_price <= trading_constants.ZERO: + return False + trigger_above = resolve_order_trigger_above(order) + if trigger_above: + return order_price <= market_price * (trading_constants.ONE - threshold) + return order_price >= market_price * (trading_constants.ONE + threshold) + + +def _order_price_gap_ratio( + order: protocol_models.Order, + market_price: decimal.Decimal, +) -> decimal.Decimal: + order_price = decimal.Decimal(str(order.price)) + trigger_above = resolve_order_trigger_above(order) + if trigger_above: + return (market_price - order_price) / market_price + return (order_price - market_price) / market_price + + +async def ensure_reference_account_not_outdated( + reference_account: protocol_models.CopiedAccount, + exchange_interface: copy_exchange.ExchangeInterface, +) -> None: + replicable_orders = get_replicable_reference_orders(reference_account) + if not replicable_orders: + return + threshold = decimal.Decimal(str(copy_constants.OUTDATED_ORDER_PRICE_MAX_THRESHOLD)) + market_price_by_symbol: dict[str, decimal.Decimal] = {} + for order in replicable_orders: + symbol = order.symbol + if symbol not in market_price_by_symbol: + market_price, _is_outdated = exchange_interface.market.get_potentially_outdated_price(symbol) + if market_price <= trading_constants.ZERO: + raise ValueError( + f"Missing market price for {symbol!r} while validating reference account outdated orders" + ) + market_price_by_symbol[symbol] = market_price + market_price = market_price_by_symbol[symbol] + if is_order_impossible_at_market_price(order, market_price, threshold): + gap_ratio = _order_price_gap_ratio(order, market_price) + gap_percent = float(gap_ratio * trading_constants.ONE_HUNDRED) + trigger_above = resolve_order_trigger_above(order) + raise copy_errors.OutdatedReferenceAccountError( + f"Reference account updated_at={reference_account.updated_at} is outdated: " + f"order {order.id!r} on {order.symbol!r} " + f"(trigger_above={trigger_above}, price={order.price}, market={market_price}) " + f"would instantly fill by {gap_percent:.2f}% " + f"(threshold={float(threshold * trading_constants.ONE_HUNDRED):.2f}%)" + ) diff --git a/packages/copy/tests/python/copiers/test_account_copier.py b/packages/copy/tests/python/copiers/test_account_copier.py index 672906fd8d..af09b3f3ca 100644 --- a/packages/copy/tests/python/copiers/test_account_copier.py +++ b/packages/copy/tests/python/copiers/test_account_copier.py @@ -22,6 +22,7 @@ import octobot_copy.constants as copy_constants import octobot_copy.copiers.spot_account_copier as spot_account_copier_module import octobot_copy.entities as copy_entities +import octobot_copy.errors as copy_errors pytestmark = pytest.mark.asyncio @@ -120,3 +121,34 @@ async def test_does_not_bypass_grace_when_rebalance_has_no_market_orders(self): grace_identified=True, ) synchronizer.abort_mirrored_orphan_grace.assert_not_called() + + +class TestAccountCopierCopyAccountOutdatedReference: + async def test_skips_rebalance_and_sync_when_reference_account_is_outdated(self): + reference_account = _copied_reference_account() + exchange_interface = mock.MagicMock() + exchange_interface.exchange_name = "bitmart" + copy_settings = copy_entities.AccountCopySettings() + copier = spot_account_copier_module.SpotAccountCopier( + reference_account, + exchange_interface, + copy_settings, + ) + synchronizer = mock.Mock() + copier._orders_synchronizer = synchronizer + copier._prepare_rebalance_plan = mock.AsyncMock() + copier._run_rebalance = mock.AsyncMock() + copier._resync_if_mirrored_open_order_grace_period_elapsed = mock.AsyncMock() + synchronizer.synchronize = mock.AsyncMock() + + with mock.patch( + "octobot_copy.copiers.account_copier.reference_account_outdated_orders_module.ensure_reference_account_not_outdated", + mock.AsyncMock(side_effect=copy_errors.OutdatedReferenceAccountError("stale reference")), + ): + with pytest.raises(copy_errors.OutdatedReferenceAccountError, match="stale reference"): + await copier.copy_account() + + copier._resync_if_mirrored_open_order_grace_period_elapsed.assert_not_awaited() + copier._prepare_rebalance_plan.assert_not_awaited() + copier._run_rebalance.assert_not_awaited() + synchronizer.synchronize.assert_not_awaited() diff --git a/packages/copy/tests/python/copiers/test_formatter.py b/packages/copy/tests/python/copiers/test_formatter.py new file mode 100644 index 0000000000..97b5a2cdad --- /dev/null +++ b/packages/copy/tests/python/copiers/test_formatter.py @@ -0,0 +1,192 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +import time + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_copy.copiers.formatter as copy_formatter + + +_REFERENCE_UPDATED_AT = 1710000000.0 +_REFERENCE_UPDATED_AT_LABEL = ( + f"{timestamp_util.convert_timestamp_to_datetime(_REFERENCE_UPDATED_AT, local_timezone=False)} UTC" +) + + +def _copied_asset( + *, + name: str = "BTC", + ratio: float = 1.0, +) -> protocol_models.CopiedAsset: + return protocol_models.CopiedAsset( + name=name, + total=1.0, + available=1.0, + ratio=ratio, + ) + + +def _reference_limit_order( + *, + symbol: str = "BTC/USDT", + price: float = 95000.0, + side: protocol_models.Side = protocol_models.Side.BUY, + order_type: protocol_models.OrderType = protocol_models.OrderType.LIMIT, +) -> protocol_models.Order: + return protocol_models.Order( + id="reference-order-id", + symbol=symbol, + price=price, + quantity=0.001, + filled=0.0, + exchange_id="ex", + side=side, + type=order_type, + trigger_above=False, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account( + *, + copied_assets: list[protocol_models.CopiedAsset] | None = None, + orders: list[protocol_models.Order] | None = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version="1.0.0", + updated_at=_REFERENCE_UPDATED_AT, + copied_assets=copied_assets or [], + orders=orders, + ) + + +class TestFormatReferenceAccountUpdatedAt: + def test_formats_unix_timestamp_as_utc_human_readable_string(self): + assert copy_formatter._format_reference_account_updated_at(_REFERENCE_UPDATED_AT) == ( + _REFERENCE_UPDATED_AT_LABEL + ) + + +class TestFormatCopiedAssetEntry: + def test_formats_name_and_ratio_as_percent(self): + copied_asset = _copied_asset(name="BTC", ratio=0.5038280454117718) + + entry = copy_formatter._format_copied_asset_entry(copied_asset) + + assert entry == "BTC:50.4%" + + +class TestFormatReferenceOrderEntry: + def test_formats_side_symbol_and_price_without_id(self): + order = _reference_limit_order(symbol="BTC/USDT", price=95000.0) + + entry = copy_formatter._format_reference_order_entry(order) + + assert entry == "buy BTC/USDT@95000.0" + assert "reference-order-id" not in entry + + +class TestFormatOrderTypeCountLabel: + def test_empty_when_no_orders(self): + assert copy_formatter._format_order_type_count_label([]) == "0" + + def test_mixed_buy_and_sell_limit_orders(self): + orders = [ + _reference_limit_order(side=protocol_models.Side.BUY), + _reference_limit_order(side=protocol_models.Side.SELL), + ] + + label = copy_formatter._format_order_type_count_label(orders) + + assert label == "1 buy_limit,1 sell_limit" + + def test_multiple_orders_of_same_type(self): + orders = [ + _reference_limit_order(side=protocol_models.Side.BUY), + _reference_limit_order(side=protocol_models.Side.BUY), + ] + + label = copy_formatter._format_order_type_count_label(orders) + + assert label == "2 buy_limit" + + +class TestFormatReferenceAccountSummary: + def test_assets_only(self): + reference_account = _copied_account( + copied_assets=[_copied_asset(name="BTC", ratio=1.0)], + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary == ( + f"v1.0.0@{_REFERENCE_UPDATED_AT_LABEL} assets[1]:BTC:100.0% orders[0]" + ) + + def test_assets_and_orders(self): + reference_account = _copied_account( + copied_assets=[ + _copied_asset(name="USDT", ratio=0.49617195458822816), + _copied_asset(name="BTC", ratio=0.5038280454117718), + ], + orders=[ + _reference_limit_order(symbol="BTC/USDT", price=95000.0), + _reference_limit_order( + symbol="ETH/USDT", + price=3000.0, + side=protocol_models.Side.SELL, + ), + ], + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary == ( + f"v1.0.0@{_REFERENCE_UPDATED_AT_LABEL} " + "assets[2]:USDT:49.6%,BTC:50.4% " + "orders[1 buy_limit,1 sell_limit]:buy BTC/USDT@95000.0,sell ETH/USDT@3000.0" + ) + + def test_orders_none_shows_zero_count(self): + reference_account = _copied_account( + copied_assets=[_copied_asset()], + orders=None, + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary.endswith("orders[0]") + + def test_truncates_when_more_than_summary_limit(self): + copied_assets = [ + _copied_asset(name=f"ASSET{asset_index}", ratio=0.1) + for asset_index in range(copy_formatter.REFERENCE_ACCOUNT_SUMMARY_LIMIT + 1) + ] + orders = [ + _reference_limit_order(symbol=f"SYM{order_index}/USDT", price=float(order_index)) + for order_index in range(copy_formatter.REFERENCE_ACCOUNT_SUMMARY_LIMIT + 1) + ] + reference_account = _copied_account(copied_assets=copied_assets, orders=orders) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert "assets[11]:" in summary + assert "…+1 more" in summary + assert "orders[11 buy_limit]:" in summary diff --git a/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py b/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py new file mode 100644 index 0000000000..2ae805c76e --- /dev/null +++ b/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py @@ -0,0 +1,161 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +import decimal +import time + +import mock +import pytest + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders_module + +_THRESHOLD = decimal.Decimal(str(copy_constants.OUTDATED_ORDER_PRICE_MAX_THRESHOLD)) +_BTC_USDC = "BTC/USDC" + + +def _open_limit_order( + *, + order_id: str = "reference-order-id", + symbol: str = _BTC_USDC, + price: float, + side: protocol_models.Side = protocol_models.Side.SELL, + trigger_above: bool | None = True, + status: protocol_models.OrderStatus = protocol_models.OrderStatus.OPEN, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=symbol, + price=price, + quantity=0.001, + filled=0.0, + exchange_id="ex", + side=side, + type=protocol_models.OrderType.LIMIT, + trigger_above=trigger_above, + reduce_only=False, + is_active=True, + status=status, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account( + *, + orders: list[protocol_models.Order] | None = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=1710000000.0, + copied_assets=[], + orders=orders, + ) + + +class TestResolveOrderTriggerAbove: + def test_uses_explicit_trigger_above_when_set(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.SELL, trigger_above=False) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is False + + def test_infers_true_for_sell_when_trigger_above_missing(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.SELL, trigger_above=None) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is True + + def test_infers_false_for_buy_when_trigger_above_missing(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.BUY, trigger_above=None) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is False + + +class TestIsOrderImpossibleAtMarketPrice: + def test_sell_stale_when_far_below_market(self): + order = _open_limit_order(price=67326.0, trigger_above=True) + assert reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_sell_not_stale_when_near_market(self): + order = _open_limit_order(price=107000.0, trigger_above=True) + assert not reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_buy_stale_when_far_above_market(self): + order = _open_limit_order(price=113500.0, side=protocol_models.Side.BUY, trigger_above=False) + assert reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_buy_not_stale_when_near_market(self): + order = _open_limit_order(price=113000.0, side=protocol_models.Side.BUY, trigger_above=False) + assert not reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + +class TestEnsureReferenceAccountNotOutdated: + @pytest.mark.asyncio + async def test_raises_when_order_is_impossible(self): + reference_account = _copied_account( + orders=[_open_limit_order(price=67326.0, trigger_above=True)], + ) + exchange_interface = mock.Mock() + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("110000"), False), + ) + with pytest.raises(copy_errors.OutdatedReferenceAccountError): + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) + + @pytest.mark.asyncio + async def test_no_op_when_no_orders(self): + exchange_interface = mock.Mock() + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + _copied_account(orders=[]), + exchange_interface, + ) + exchange_interface.market.get_potentially_outdated_price.assert_not_called() + + @pytest.mark.asyncio + async def test_ignores_non_open_orders(self): + reference_account = _copied_account( + orders=[ + _open_limit_order( + price=67326.0, + trigger_above=True, + status=protocol_models.OrderStatus.FILLED, + ) + ], + ) + exchange_interface = mock.Mock() + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) + exchange_interface.market.get_potentially_outdated_price.assert_not_called() + + @pytest.mark.asyncio + async def test_passes_when_orders_are_near_market(self): + reference_account = _copied_account( + orders=[_open_limit_order(price=107000.0, trigger_above=True)], + ) + exchange_interface = mock.Mock() + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("110000"), False), + ) + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py index 2a95cf6115..94245a02a2 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py @@ -1,10 +1,11 @@ import octobot_commons.dsl_interpreter import octobot_commons.errors import octobot_commons.logging -import octobot_commons.constants import octobot_trading.errors import octobot_trading.enums +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.logic.dsl.action_error_util @@ -138,6 +139,8 @@ async def _action_execution_error_handler_wrapper( if _should_postpone_recallable_trading_error(self, action): raise return _map_non_recallable_postpone_trading_error(action, err) + except copy_errors.OutdatedReferenceAccountError: + raise except Exception as err: # swallowed errors: warning: will stop the workflow octobot_commons.logging.get_logger("action_execution").exception( From 21d3f3bd9f2be9e8cd814bcdd6736a41079b47bb Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Mon, 24 Aug 2026 16:56:56 +0200 Subject: [PATCH 04/10] [Portfolio] compute portfolio history from trades and transactions --- octobot/backtesting/minimal_data_importer.py | 10 +- .../octobot_backtesting/api/importer.py | 5 +- .../collectors/data_collector.py | 4 +- .../data/data_file_manager.py | 4 +- .../octobot_backtesting/databases/__init__.py | 25 ++ .../backtesting_data_sqlite_database.py} | 78 +--- .../importers/data_importer.py | 4 +- .../importers/exchanges/exchange_importer.py | 35 +- .../importers/social/social_importer.py | 13 +- .../backtesting/tests/database_test_util.py | 23 + .../backtesting/tests/databases/__init__.py | 0 .../test_backtesting_data_sqlite_database.py} | 170 ++++---- .../tests/importers/test_exchange_importer.py | 18 +- .../octobot_commons/databases/__init__.py | 6 +- .../relational_databases/__init__.py | 10 +- .../relational_databases/sqlite/__init__.py | 13 +- .../sqlite/base_sqlite_database.py | 170 ++++++++ packages/commons/octobot_commons/errors.py | 6 + .../sqlite/test_base_sqlite_database.py | 166 +++++++ packages/flow/octobot_flow/constants.py | 3 + .../flow/octobot_flow/entities/__init__.py | 6 + .../exchange_account_refresh_result.py | 2 +- .../global_view_account_refresh_result.py | 1 - .../entities/portfolio_history/__init__.py | 7 + .../portfolio_history_account_context.py | 16 + .../portfolio_history_run_result.py | 18 + .../jobs/global_view_account_job.py | 11 + .../jobs/portfolio_history_job.py | 214 +++++++++ .../accounts/account_state_persistence.py | 12 + .../global_view/account_refresh_builder.py | 10 - .../global_view/exchange_account_refresh.py | 64 +-- .../global_view/global_view_persistence.py | 6 - .../logic/portfolio_history/__init__.py | 1 + .../daily_price_cache_updater.py | 349 +++++++++++++++ .../portfolio_value_history.py | 50 +++ .../trading_history_merge.py | 52 +++ .../exchange/orders_repository.py | 4 +- .../exchange/trades_repository.py | 29 +- .../exchange/transactions_repository.py | 35 ++ .../test_global_view_account_refresh.py | 27 +- .../octobot_process_functional_shared.py | 48 +- .../test_octobot_process_edit_config.py | 45 +- .../test_octobot_process_multi_exchange.py | 29 +- .../test_octobot_process_start.py | 91 +--- .../portfolio_history/__init__.py | 1 + .../portfolio_history_test_util.py | 266 ++++++++++++ .../test_portfolio_history_job.py | 147 +++++++ .../tests/jobs/test_portfolio_history_job.py | 187 ++++++++ .../dsl/test_dsl_action_execution_context.py | 26 ++ .../test_account_refresh_builder.py | 39 +- .../test_exchange_account_refresh.py | 19 +- .../test_global_view_account_job.py | 37 +- .../test_global_view_persistence.py | 17 - .../tests/logic/portfolio_history/__init__.py | 1 + .../test_daily_price_cache_updater.py | 410 ++++++++++++++++++ .../test_portfolio_value_history.py | 76 ++++ .../test_trading_history_merge.py | 79 ++++ .../community/test_trading_signals_channel.py | 1 - .../tests/repositories/exchange/__init__.py | 1 + .../exchange/test_trades_repository.py | 62 +++ .../exchange/test_transactions_repository.py | 67 +++ .../exchange/trades_repository_test_util.py | 22 + packages/node/octobot_node/enums.py | 1 + .../octobot_node/protocol/accounts_history.py | 158 +++++-- packages/node/octobot_node/scheduler/api.py | 2 +- .../scheduler/portfolio_history/__init__.py | 1 + .../portfolio_history_executor.py | 120 +++++ .../node/octobot_node/scheduler/scheduler.py | 6 + .../node/octobot_node/scheduler/schedules.py | 85 ++++ packages/node/octobot_node/scheduler/tasks.py | 21 + .../user_actions/user_action_post_actions.py | 4 + .../user_actions/user_action_util.py | 1 + .../user_actions_executor/__init__.py | 3 + .../update_historical_exchanges_data.py | 63 +++ .../user_action_executor_factory.py | 2 + .../util/account_authentication_resolver.py | 6 +- .../util/account_state_updater.py | 6 +- .../util/exchange_account_resolver.py | 6 +- .../scheduler/workflows/__init__.py | 1 + .../workflows/automation_workflow.py | 20 +- .../workflows/dbos_cleanup_workflow.py | 7 +- .../workflows/global_view_workflow.py | 15 +- .../scheduler/workflows/params/__init__.py | 4 + .../portfolio_history_workflow_params.py | 25 ++ .../workflows/portfolio_history_workflow.py | 113 +++++ .../workflows/user_action_workflow.py | 4 + .../scheduler/workflows_retention.py | 174 +++++++- .../test_accounts_history_compute.py | 330 ++++++++++++++ ...t_emit_and_copy_grid_automation_signals.py | 118 +---- .../test_global_view_workflow.py | 15 - ...est_outdated_reference_account_workflow.py | 199 +++++++++ .../util/accounts_history_test_util.py | 211 +++++++++ .../functional_tests/util/dca_workflow.py | 12 +- .../util/exchange_account_elements_access.py | 162 +++++++ .../util/outdated_reference_workflow.py | 170 ++++++++ .../util/protocol_assertions.py | 50 +-- .../functional_tests/util/workflow_common.py | 44 +- .../protocol/test_accounts_history_compute.py | 86 ++++ packages/node/tests/scheduler/__init__.py | 2 + .../global_view/test_global_view_workflow.py | 21 +- .../scheduler/portfolio_history/__init__.py | 1 + .../test_portfolio_history_executor.py | 36 ++ .../test_portfolio_history_workflow.py | 108 +++++ .../node/tests/scheduler/test_schedules.py | 399 ++++++++++++++++- packages/node/tests/scheduler/test_tasks.py | 49 +++ .../test_user_action_executor_factory.py | 11 + .../scheduler/test_workflows_retention.py | 256 ++++++++++- .../user_actions/test_user_action_util.py | 10 + .../test_update_historical_exchanges_data.py | 207 +++++++++ .../workflows/test_automation_workflow.py | 151 ++++++- .../workflows/test_dbos_cleanup_workflow.py | 19 +- packages/protocol/.openapi-generator/FILES | 12 + packages/protocol/docs/AccountTrading.md | 1 + packages/protocol/docs/ExchangeConfig.md | 1 + packages/protocol/docs/Fee.md | 31 ++ packages/protocol/docs/Trade.md | 1 + packages/protocol/docs/Transaction.md | 34 ++ packages/protocol/docs/TransactionType.md | 17 + ...ateHistoricalExchangesDataConfiguration.md | 31 ++ packages/protocol/docs/UserActionType.md | 2 + .../octobot_protocol/models/__init__.py | 4 + .../models/account_trading.py | 14 +- .../models/exchange_config.py | 6 +- .../protocol/octobot_protocol/models/fee.py | 90 ++++ .../protocol/octobot_protocol/models/trade.py | 10 +- .../octobot_protocol/models/transaction.py | 98 +++++ .../models/transaction_type.py | 39 ++ ...historical_exchanges_data_configuration.py | 91 ++++ .../models/user_action_configuration.py | 49 ++- .../models/user_action_type.py | 1 + packages/protocol/openapi.json | 90 ++++ .../protocol/test/test_account_trading.py | 11 + .../test/test_account_trading_state.py | 22 + .../test_account_trading_with_account_id.py | 11 + packages/protocol/test/test_accounts_state.py | 5 +- ...st_create_exchange_config_configuration.py | 10 +- packages/protocol/test/test_debug.py | 16 +- packages/protocol/test/test_debug_state.py | 15 +- ...test_edit_exchange_config_configuration.py | 5 +- .../protocol/test/test_exchange_config.py | 5 +- packages/protocol/test/test_fee.py | 54 +++ packages/protocol/test/test_trade.py | 3 + packages/protocol/test/test_transaction.py | 60 +++ .../protocol/test/test_transaction_type.py | 33 ++ ...historical_exchanges_data_configuration.py | 55 +++ .../user_account_provider.py | 19 + .../test_user_account_provider.py | 54 +++ .../bot_snapshot_with_history_collector.py | 4 +- .../tests/test_history_collector.py | 4 +- .../legacy_data_converter/legacy_converter.py | 4 +- .../copy_exchange_account_operators.py | 6 + .../src/components/Debug/DebugTabsPanel.tsx | 7 + .../components/Debug/tables/AccountsTable.tsx | 14 +- .../__tests__/user-action-templates.test.ts | 21 + .../src/lib/debug/user-action-templates.ts | 18 + .../trading/octobot_trading/api/__init__.py | 24 + .../api/exchange_data_cache.py | 85 ++++ .../trading/octobot_trading/api/portfolio.py | 14 + packages/trading/octobot_trading/constants.py | 4 +- .../exchange_data/databases/__init__.py | 11 + .../databases/market_data_sqlite_database.py | 211 +++++++++ .../exchange_data/exchange_cache_key.py | 5 + .../prices/persisted_price_cache.py | 94 ++++ .../ticker/persisted_ticker_cache.py | 37 ++ .../exchange_data/ticker/ticker_cache.py | 3 +- .../exchanges/abstract_exchange.py | 10 +- .../exchanges/adapters/abstract_adapter.py | 12 + .../connectors/ccxt/ccxt_connector.py | 26 ++ .../exchanges/types/rest_exchange.py | 8 + .../exchanges/util/exchange_util.py | 58 ++- .../orders/channel/orders_updater.py | 62 +++ ...ory_from_trades_and_transaction_builder.py | 179 ++++++++ .../trades/channel/trades_updater.py | 62 ++- .../personal_data/trades/protocol.py | 24 +- .../personal_data/transactions/protocol.py | 54 +++ .../transactions/transactions_util.py | 25 ++ .../util/test_tools/exchanges_test_tools.py | 64 +-- .../tests/exchange_data/databases/__init__.py | 0 .../test_market_data_sqlite_database.py | 309 +++++++++++++ .../tests/exchange_data/prices/__init__.py | 15 - .../prices/test_persisted_price_cache.py | 118 +++++ .../ticker/test_persisted_ticker_cache.py | 80 ++++ .../exchange_data/ticker/test_ticker_cache.py | 1 + .../connectors/ccxt/test_ccxt_connector.py | 16 + .../exchanges/util/test_exchange_util.py | 38 +- .../test_orders_updater_ensure_parsing.py | 46 ++ ...ory_from_trades_and_transaction_builder.py | 345 +++++++++++++++ .../trades/test_trades_protocol.py | 66 +++ .../trades/test_trades_updater.py | 127 ++++++ 189 files changed, 9244 insertions(+), 930 deletions(-) create mode 100644 packages/backtesting/octobot_backtesting/databases/__init__.py rename packages/{commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py => backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py} (82%) create mode 100644 packages/backtesting/tests/database_test_util.py create mode 100644 packages/backtesting/tests/databases/__init__.py rename packages/{commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py => backtesting/tests/databases/test_backtesting_data_sqlite_database.py} (63%) create mode 100644 packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py create mode 100644 packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py create mode 100644 packages/flow/octobot_flow/entities/portfolio_history/__init__.py create mode 100644 packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py create mode 100644 packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py create mode 100644 packages/flow/octobot_flow/jobs/portfolio_history_job.py create mode 100644 packages/flow/octobot_flow/logic/portfolio_history/__init__.py create mode 100644 packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py create mode 100644 packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py create mode 100644 packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py create mode 100644 packages/flow/octobot_flow/repositories/exchange/transactions_repository.py create mode 100644 packages/flow/tests/functionnal_tests/portfolio_history/__init__.py create mode 100644 packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py create mode 100644 packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py create mode 100644 packages/flow/tests/jobs/test_portfolio_history_job.py create mode 100644 packages/flow/tests/logic/portfolio_history/__init__.py create mode 100644 packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py create mode 100644 packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py create mode 100644 packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py create mode 100644 packages/flow/tests/repositories/exchange/__init__.py create mode 100644 packages/flow/tests/repositories/exchange/test_trades_repository.py create mode 100644 packages/flow/tests/repositories/exchange/test_transactions_repository.py create mode 100644 packages/flow/tests/repositories/exchange/trades_repository_test_util.py create mode 100644 packages/node/octobot_node/scheduler/portfolio_history/__init__.py create mode 100644 packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py create mode 100644 packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py create mode 100644 packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py create mode 100644 packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py create mode 100644 packages/node/tests/functional_tests/test_accounts_history_compute.py create mode 100644 packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py create mode 100644 packages/node/tests/functional_tests/util/accounts_history_test_util.py create mode 100644 packages/node/tests/functional_tests/util/exchange_account_elements_access.py create mode 100644 packages/node/tests/functional_tests/util/outdated_reference_workflow.py create mode 100644 packages/node/tests/protocol/test_accounts_history_compute.py create mode 100644 packages/node/tests/scheduler/portfolio_history/__init__.py create mode 100644 packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py create mode 100644 packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py create mode 100644 packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py create mode 100644 packages/protocol/docs/Fee.md create mode 100644 packages/protocol/docs/Transaction.md create mode 100644 packages/protocol/docs/TransactionType.md create mode 100644 packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md create mode 100644 packages/protocol/octobot_protocol/models/fee.py create mode 100644 packages/protocol/octobot_protocol/models/transaction.py create mode 100644 packages/protocol/octobot_protocol/models/transaction_type.py create mode 100644 packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py create mode 100644 packages/protocol/test/test_fee.py create mode 100644 packages/protocol/test/test_transaction.py create mode 100644 packages/protocol/test/test_transaction_type.py create mode 100644 packages/protocol/test/test_update_historical_exchanges_data_configuration.py create mode 100644 packages/sync/tests/sync/collection_providers/test_user_account_provider.py create mode 100644 packages/trading/octobot_trading/api/exchange_data_cache.py create mode 100644 packages/trading/octobot_trading/exchange_data/databases/__init__.py create mode 100644 packages/trading/octobot_trading/exchange_data/databases/market_data_sqlite_database.py create mode 100644 packages/trading/octobot_trading/exchange_data/exchange_cache_key.py create mode 100644 packages/trading/octobot_trading/exchange_data/prices/persisted_price_cache.py create mode 100644 packages/trading/octobot_trading/exchange_data/ticker/persisted_ticker_cache.py create mode 100644 packages/trading/octobot_trading/personal_data/portfolios/history/history_from_trades_and_transaction_builder.py create mode 100644 packages/trading/octobot_trading/personal_data/transactions/protocol.py create mode 100644 packages/trading/octobot_trading/personal_data/transactions/transactions_util.py create mode 100644 packages/trading/tests/exchange_data/databases/__init__.py create mode 100644 packages/trading/tests/exchange_data/databases/test_market_data_sqlite_database.py create mode 100644 packages/trading/tests/exchange_data/prices/test_persisted_price_cache.py create mode 100644 packages/trading/tests/exchange_data/ticker/test_persisted_ticker_cache.py create mode 100644 packages/trading/tests/personal_data/orders/test_orders_updater_ensure_parsing.py create mode 100644 packages/trading/tests/personal_data/portfolios/history/test_history_from_trades_and_transaction_builder.py create mode 100644 packages/trading/tests/personal_data/trades/test_trades_protocol.py create mode 100644 packages/trading/tests/personal_data/trades/test_trades_updater.py diff --git a/octobot/backtesting/minimal_data_importer.py b/octobot/backtesting/minimal_data_importer.py index 2811ef1a7e..af900dcddd 100644 --- a/octobot/backtesting/minimal_data_importer.py +++ b/octobot/backtesting/minimal_data_importer.py @@ -1,9 +1,9 @@ import octobot_commons.enums -import octobot_commons.databases as databases import octobot_commons.constants import octobot_commons.symbols import octobot_commons.time_frame_manager as time_frame_manager +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.importers import octobot_backtesting.enums @@ -60,7 +60,7 @@ async def get_data_timestamp_interval(self, time_frame=None): async def get_ohlcv(self, exchange_name=None, symbol=None, time_frame=octobot_commons.enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return await self._get_from_db( @@ -74,7 +74,7 @@ async def get_ohlcv(self, exchange_name=None, symbol=None, async def _get_from_db( self, exchange_name, symbol, table, time_frame=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -88,7 +88,7 @@ def _select_from_timestamp(self, symbol, timestamps, operations, time_frame): parsed_timestamps = [float(timestamp) for timestamp in timestamps] return [ candle - for candle in self._select(databases.SQLiteDatabase.DEFAULT_SIZE, symbol, time_frame) + for candle in self._select(backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, symbol, time_frame) if _is_valid_from_operations( candle[octobot_commons.enums.PriceIndexes.IND_PRICE_TIME.value], parsed_timestamps, operations ) @@ -110,7 +110,7 @@ def _select(self, limit, symbol, time_frame): ] for candle in candles ] - if limit != databases.SQLiteDatabase.DEFAULT_SIZE: + if limit != backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE: return candles_data[:limit] return candles_data diff --git a/packages/backtesting/octobot_backtesting/api/importer.py b/packages/backtesting/octobot_backtesting/api/importer.py index 2df98a4dde..1660fc1e5e 100644 --- a/packages/backtesting/octobot_backtesting/api/importer.py +++ b/packages/backtesting/octobot_backtesting/api/importer.py @@ -18,7 +18,8 @@ import octobot_backtesting.util as util import octobot_commons.errors as commons_errors -import octobot_commons.databases as databases + +import octobot_backtesting.databases as backtesting_databases async def create_importer(config, backtesting_file, default_importer=None): @@ -63,7 +64,7 @@ async def get_all_ohlcvs(database_path, exchange_name, symbol, time_frame, inferior_timestamp=-1, superior_timestamp=-1) -> list: timestamps, operations = importers.get_operations_from_timestamps(superior_timestamp, inferior_timestamp) try: - async with databases.new_sqlite_database(database_path) as database: + async with backtesting_databases.new_sqlite_database(database_path) as database: candles = await database.select_from_timestamp(backtesting_enums.ExchangeDataTables.OHLCV, exchange_name=exchange_name, symbol=symbol, time_frame=time_frame.value, diff --git a/packages/backtesting/octobot_backtesting/collectors/data_collector.py b/packages/backtesting/octobot_backtesting/collectors/data_collector.py index a6900fc2e9..e2dc100858 100644 --- a/packages/backtesting/octobot_backtesting/collectors/data_collector.py +++ b/packages/backtesting/octobot_backtesting/collectors/data_collector.py @@ -25,6 +25,8 @@ import octobot_commons.logging as logging import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.enums as enums import octobot_backtesting.constants as constants import octobot_backtesting.data as data @@ -83,7 +85,7 @@ def set_file_path(self) -> None: def create_database(self) -> None: if not self.database: - self.database = databases.SQLiteDatabase(self.temp_file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase(self.temp_file_path) def finalize_database(self): os.rename(self.temp_file_path, self.file_path) diff --git a/packages/backtesting/octobot_backtesting/data/data_file_manager.py b/packages/backtesting/octobot_backtesting/data/data_file_manager.py index 082b97493a..c28a17148e 100644 --- a/packages/backtesting/octobot_backtesting/data/data_file_manager.py +++ b/packages/backtesting/octobot_backtesting/data/data_file_manager.py @@ -24,6 +24,8 @@ import octobot_commons.time_frame_manager as tmf_manager import octobot_commons.errors as commons_errors +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.constants as constants import octobot_backtesting.enums as enums @@ -140,7 +142,7 @@ def _parse_list(value): async def get_file_description(database_file): database = None try: - database = databases.SQLiteDatabase(database_file) + database = backtesting_databases.BacktestingDataSQLiteDatabase(database_file) await database.initialize() description = await get_database_description(database) except (commons_errors.DatabaseNotFoundError, TypeError): diff --git a/packages/backtesting/octobot_backtesting/databases/__init__.py b/packages/backtesting/octobot_backtesting/databases/__init__.py new file mode 100644 index 0000000000..5afd071b7a --- /dev/null +++ b/packages/backtesting/octobot_backtesting/databases/__init__.py @@ -0,0 +1,25 @@ +# Drakkar-Software OctoBot-Backtesting +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_backtesting.databases.backtesting_data_sqlite_database import ( + BacktestingDataSQLiteDatabase, + new_sqlite_database, +) + +__all__ = [ + "BacktestingDataSQLiteDatabase", + "new_sqlite_database", +] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py similarity index 82% rename from packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py rename to packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py index f066002f6d..42498b137a 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py +++ b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py @@ -17,27 +17,12 @@ import contextlib import sqlite3 -import octobot_commons.logging as logging import octobot_commons.enums as enums import octobot_commons.errors as errors -import octobot_commons.databases.relational_databases.sqlite.cursor_pool as cursor_pool -import octobot_commons.constants as constants +import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database -try: - import aiosqlite -except ImportError: - if constants.USE_MINIMAL_LIBS: - # mock aiosqlite imports - class AiosqliteImportMock: - def connect(self, *args): - raise ImportError("aiosqlite not installed") - aiosqlite = AiosqliteImportMock() - else: - raise - - -class SQLiteDatabase: +class BacktestingDataSQLiteDatabase(base_sqlite_database.BaseSQLiteDatabase): TIMESTAMP_COLUMN = "timestamp" DEFAULT_ORDER_BY = TIMESTAMP_COLUMN DEFAULT_SORT = enums.DataBaseOrderBy.DESC.value @@ -46,40 +31,19 @@ class SQLiteDatabase: CACHE_SIZE = 50 def __init__(self, file_name): - self.file_name = file_name - self.logger = logging.get_logger(self.__class__.__name__) - + super().__init__(file_name) self.tables = [] self.cache = {} - self.connection = None - - # should never be used directly, use async with self.aio_cursor() as cursor: instead - self._cursor_pool = None - async def initialize(self): - try: - self.connection = await aiosqlite.connect(self.file_name) - self._cursor_pool = cursor_pool.CursorPool(self.connection) - await self.__init_tables_list() - except (sqlite3.OperationalError, sqlite3.DatabaseError) as err: - raise errors.DatabaseNotFoundError(f"{err} (file: {self.file_name})") + await super().initialize() + await self.__init_tables_list() async def create_index(self, table, columns): await self.__execute_index_creation( table, "_".join(columns), ", ".join(columns) ) - @contextlib.asynccontextmanager - async def aio_cursor(self) -> sqlite3.Cursor: - """ - Use this as a context manager to get a free database cursor - :yield: A free cursor - :return: None - """ - async with self._cursor_pool.idle_cursor() as cursor: - yield cursor.cursor - async def __execute_index_creation(self, table, name, columns): async with self.aio_cursor() as cursor: await cursor.execute( @@ -90,7 +54,6 @@ async def insert(self, table, timestamp, **kwargs): if table.value not in self.tables: await self.__create_table(table, **kwargs) - # Insert a row of data inserting_values = [f"'{value}'" for value in kwargs.values()] await self.__execute_insert( table, self.__insert_values(timestamp, ", ".join(inserting_values)) @@ -104,7 +67,6 @@ async def insert_all(self, table, timestamp, **kwargs): insert_values = [] for index, values in enumerate(timestamp): - # Insert a row of data inserting_values = [ f"'{value if not isinstance(value, list) else value[index]}'" for value in kwargs.values() @@ -116,7 +78,6 @@ async def insert_all(self, table, timestamp, **kwargs): await self.__execute_insert(table, ", ".join(insert_values)) async def update(self, table, updated_value_by_column, **kwargs): - # Update a row of data updating_values = [ f"{key} = '{value}'" for key, value in updated_value_by_column.items() ] @@ -133,7 +94,6 @@ async def __execute_insert(self, table, insert_items) -> None: async with self.aio_cursor() as cursor: await cursor.execute(f"INSERT INTO {table.value} VALUES {insert_items}") - # Save (commit) the changes await self.connection.commit() async def __execute_update(self, table, update_items, where_clauses) -> None: @@ -142,7 +102,6 @@ async def __execute_update(self, table, update_items, where_clauses) -> None: f"UPDATE {table.value} SET {update_items} WHERE {where_clauses}" ) - # Save (commit) the changes await self.connection.commit() async def select( @@ -249,13 +208,13 @@ def __where_clauses_from_operations( return " AND ".join( [ self.__where_clauses_from_operation( - keys[i], - values[i], - operations[i] if len(operations) > i else None, + keys[index], + values[index], + operations[index] if len(operations) > index else None, should_quote_value=should_quote_value, ) - for i in range(len(keys)) - if values[i] is not None + for index in range(len(keys)) + if values[index] is not None ] ) @@ -311,7 +270,6 @@ async def __execute_select( async def __execute_delete(self, table, where_clauses): async with self.aio_cursor() as cursor: await cursor.execute(f"DELETE FROM {table.value} WHERE {where_clauses} ") - # nothing to return, will raise on error async def check_table_exists(self, table) -> bool: async with self.aio_cursor() as cursor: @@ -340,10 +298,10 @@ async def __create_table( if with_index_on_timestamp: await self.create_index(table, [self.TIMESTAMP_COLUMN]) - for i in range(1, round(len(columns) / 2) + 1): + for index in range(1, round(len(columns) / 2) + 1): await self.create_index( table, - [self.TIMESTAMP_COLUMN] + [columns[u] for u in range(0, i)], + [self.TIMESTAMP_COLUMN] + [columns[column_index] for column_index in range(0, index)], ) except sqlite3.OperationalError: @@ -356,20 +314,10 @@ async def __init_tables_list(self): await cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") self.tables = [res[0] for res in await cursor.fetchall()] - async def stop(self): - try: - if self._cursor_pool is not None: - await self._cursor_pool.close() - finally: - if self.connection is not None: - conn = self.connection - self.connection = None - await conn.close() - @contextlib.asynccontextmanager async def new_sqlite_database(file_path): - local_database = SQLiteDatabase(file_path) + local_database = BacktestingDataSQLiteDatabase(file_path) try: await local_database.initialize() yield local_database diff --git a/packages/backtesting/octobot_backtesting/importers/data_importer.py b/packages/backtesting/octobot_backtesting/importers/data_importer.py index 83769be886..6756aef0f1 100644 --- a/packages/backtesting/octobot_backtesting/importers/data_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/data_importer.py @@ -18,6 +18,8 @@ import octobot_commons.logging as logging import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.constants as constants import octobot_backtesting.errors as errors @@ -57,7 +59,7 @@ def provides_accurate_price_time_frame(self) -> bool: def load_database(self) -> None: file_path = self.adapt_file_path_if_necessary() if not self.database: - self.database = databases.SQLiteDatabase(file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase(file_path) def adapt_file_path_if_necessary(self): if path.isfile(self.file_path): diff --git a/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py b/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py index 52a4cf5e6f..136152ec1d 100644 --- a/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py @@ -18,6 +18,7 @@ import octobot_commons.errors as common_errors import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.data as data import octobot_backtesting.enums as enums import octobot_backtesting.errors as errors @@ -68,12 +69,12 @@ async def get_data_timestamp_interval(self, time_frame=None): if table in self.available_data_types: try: min_timestamp = (await self.database.select_min(table, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if not minimum_timestamp or minimum_timestamp > min_timestamp: minimum_timestamp = min_timestamp max_timestamp = (await self.database.select_max(table, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if not maximum_timestamp or maximum_timestamp < max_timestamp: maximum_timestamp = max_timestamp except (IndexError, common_errors.DatabaseNotFoundError): @@ -83,7 +84,7 @@ async def get_data_timestamp_interval(self, time_frame=None): try: ohlcv_kwargs = {"time_frame": time_frame} if time_frame else {} ohlcv_min_timestamps = (await self.database.select_min(enums.ExchangeDataTables.OHLCV, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN], + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN], [common_constants.CONFIG_TIME_FRAME], group_by=common_constants.CONFIG_TIME_FRAME, **ohlcv_kwargs @@ -93,7 +94,7 @@ async def get_data_timestamp_interval(self, time_frame=None): # if the required time frame is not included in this database, ohlcv_min_timestamps is empty: ignore it min_ohlcv_timestamp = max(ohlcv_min_timestamps)[0] max_ohlcv_timestamp = (await self.database.select_max(enums.ExchangeDataTables.OHLCV, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN], + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN], **ohlcv_kwargs))[0][0] elif time_frame: raise errors.MissingTimeFrame(f"Missing time frame in data file: {time_frame}") @@ -113,7 +114,7 @@ async def _init_available_data_types(self): async def _get_from_db( self, exchange_name, symbol, table, time_frame=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -135,7 +136,7 @@ async def _get_from_db( async def get_ohlcv(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_ohlcvs(await self._get_from_db( @@ -148,7 +149,7 @@ async def get_ohlcv(self, exchange_name=None, symbol=None, async def get_ohlcv_from_timestamps(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1) -> list: """ Reads OHLCV history from database and populates a local ChronologicalReadDatabaseCache. @@ -158,7 +159,7 @@ async def get_ohlcv_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_ohlcv, limit) async def get_ticker(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_tickers(await self._get_from_db( @@ -169,7 +170,7 @@ async def get_ticker(self, exchange_name=None, symbol=None, )) async def get_ticker_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads ticker history from database and populates a local ChronologicalReadDatabaseCache. @@ -179,7 +180,7 @@ async def get_ticker_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_ticker, limit) async def get_order_book(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_order_books(await self._get_from_db( @@ -190,7 +191,7 @@ async def get_order_book(self, exchange_name=None, symbol=None, )) async def get_order_book_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads order book history from database and populates a local ChronologicalReadDatabaseCache. @@ -200,7 +201,7 @@ async def get_order_book_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_order_book, limit) async def get_recent_trades(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_recent_trades(await self._get_from_db( @@ -211,7 +212,7 @@ async def get_recent_trades(self, exchange_name=None, symbol=None, )) async def get_recent_trades_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads recent trades history from database and populates a local ChronologicalReadDatabaseCache. @@ -222,7 +223,7 @@ async def get_recent_trades_from_timestamps(self, exchange_name=None, symbol=Non async def get_kline(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_klines(await self._get_from_db( @@ -235,7 +236,7 @@ async def get_kline(self, exchange_name=None, symbol=None, async def get_kline_from_timestamps(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads kline history from database and populates a local ChronologicalReadDatabaseCache. @@ -256,9 +257,9 @@ async def _get_from_cache(self, exchange_name, symbol, time_frame, data_type, # initializer without time_frame args are not expecting the time_frame argument, remove it # ignore the limit param as it might reduce the available cache and give false later select results init_cache_method_args = \ - (exchange_name, symbol, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations) \ + (exchange_name, symbol, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations) \ if time_frame is None \ - else (exchange_name, symbol, time_frame, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations) + else (exchange_name, symbol, time_frame, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations) self.chronological_cache.set( await set_cache_method(*init_cache_method_args), 0, diff --git a/packages/backtesting/octobot_backtesting/importers/social/social_importer.py b/packages/backtesting/octobot_backtesting/importers/social/social_importer.py index c57c0da680..fd6c3fd517 100644 --- a/packages/backtesting/octobot_backtesting/importers/social/social_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/social/social_importer.py @@ -19,6 +19,7 @@ import octobot_commons.errors as common_errors import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.constants as constants import octobot_backtesting.data as data import octobot_backtesting.enums as enums @@ -103,9 +104,9 @@ async def get_data_timestamp_interval(self, time_frame=None): if enums.SocialDataTables.SOCIAL_EVENTS in self.available_data_types: try: min_timestamp = (await self.database.select_min(enums.SocialDataTables.SOCIAL_EVENTS, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] max_timestamp = (await self.database.select_max(enums.SocialDataTables.SOCIAL_EVENTS, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if min_timestamp and max_timestamp: minimum_timestamp = min_timestamp maximum_timestamp = max_timestamp @@ -123,7 +124,7 @@ async def _get_from_db( self, service_name, table, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -146,7 +147,7 @@ async def _get_from_db( ) async def get_social_events(self, service_name=None, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): """ @@ -181,7 +182,7 @@ async def get_social_events(self, service_name=None, channel=None, symbol=None, return result async def get_social_events_from_timestamps(self, service_name=None, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads social events history from database and populates a local ChronologicalReadDatabaseCache. @@ -203,7 +204,7 @@ async def _get_from_cache(self, service_name, channel, symbol, data_type, # initializer without time_frame args are not expecting the time_frame argument, remove it # ignore the limit param as it might reduce the available cache and give false later select results init_cache_method_args = ( - service_name, channel, symbol, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations + service_name, channel, symbol, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations ) self.chronological_cache.set( await set_cache_method(*init_cache_method_args), diff --git a/packages/backtesting/tests/database_test_util.py b/packages/backtesting/tests/database_test_util.py new file mode 100644 index 0000000000..b94e7c83f0 --- /dev/null +++ b/packages/backtesting/tests/database_test_util.py @@ -0,0 +1,23 @@ +import os +import shutil +import tempfile + +BACKTESTING_STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") + + +def static_database_fixture_path(file_name: str) -> str: + return os.path.join(BACKTESTING_STATIC_DIR, file_name) + + +def copy_static_database_fixture(file_name: str) -> str: + with tempfile.NamedTemporaryFile(delete=False, suffix=".data") as temp_file: + temp_database_path = temp_file.name + shutil.copy2(static_database_fixture_path(file_name), temp_database_path) + return temp_database_path + + +def remove_temp_database(database_path: str) -> None: + for suffix in ("", "-wal", "-shm"): + path = f"{database_path}{suffix}" + if os.path.isfile(path): + os.remove(path) diff --git a/packages/backtesting/tests/databases/__init__.py b/packages/backtesting/tests/databases/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py similarity index 63% rename from packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py rename to packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py index 394d01e904..863ea66ca1 100644 --- a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py +++ b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py @@ -1,18 +1,3 @@ -# Drakkar-Software OctoBot -# Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. import mock import pytest import os @@ -21,13 +6,13 @@ import contextlib import tempfile - import octobot_commons.asyncio_tools as asyncio_tools import octobot_commons.errors as errors -import octobot_commons.databases as databases import octobot_commons.enums as enums +import octobot_backtesting.databases as backtesting_databases + +import tests.database_test_util as database_test_util -# All test coroutines will be treated as marked. pytestmark = pytest.mark.asyncio DATA_FILE1 = "ExchangeHistoryDataCollector_1589740606.4862757.data" @@ -36,45 +21,46 @@ KLINE = mock.Mock(value="kline") -# use context manager instead of fixture to prevent pytest threads issues @contextlib.asynccontextmanager async def get_database(data_file=DATA_FILE1): - async with databases.new_sqlite_database(os.path.join("tests", "static", data_file)) as db: - yield db - # prevent "generator didn't stop after athrow(), see https://github.com/python-trio/trio/issues/2081" - await asyncio_tools.wait_asyncio_next_cycle() + temp_database_path = database_test_util.copy_static_database_fixture(data_file) + try: + async with backtesting_databases.new_sqlite_database(temp_database_path) as database: + yield database + await asyncio_tools.wait_asyncio_next_cycle() + finally: + database_test_util.remove_temp_database(temp_database_path) -# use context manager instead of fixture to prevent pytest threads issues @contextlib.asynccontextmanager async def get_temp_empty_database(): - database_name = "temp_empty_database" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + database_name = temp_file.name try: - async with databases.new_sqlite_database(database_name) as db: - yield db + async with backtesting_databases.new_sqlite_database(database_name) as database: + yield database finally: - # prevent "generator didn't stop after athrow(), see https://github.com/python-trio/trio/issues/2081" await asyncio_tools.wait_asyncio_next_cycle() - os.remove(database_name) + if os.path.isfile(database_name): + os.remove(database_name) async def test_invalid_file(): with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: file_name = temp_file.name - db = databases.SQLiteDatabase(file_name) + database = backtesting_databases.BacktestingDataSQLiteDatabase(file_name) try: - await db.initialize() - assert not await db.check_table_exists(KLINE) + await database.initialize() + assert not await database.check_table_exists(KLINE) with pytest.raises(sqlite3.OperationalError): - await db.check_table_not_empty(KLINE) + await database.check_table_not_empty(KLINE) finally: - await db.stop() + await database.stop() os.remove(file_name) async def test_select(): async with get_database() as database: - # default values with pytest.raises(errors.DatabaseNotFoundError): await database.select(KLINE) @@ -128,24 +114,23 @@ async def test_select_from_timestamp(): assert all(candle[0] <= 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587960000"], - operations) + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587960000"], operations + ) assert len(candles) > 0 assert all(candle[0] == 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587945600"], - operations) + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587945600"], operations + ) assert len(candles) == 15 assert all(1587945600 <= candle[0] <= 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587945600"], - operations, - symbol="xyz") + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587945600"], operations, symbol="xyz" + ) assert len(candles) == 0 @@ -153,8 +138,8 @@ async def test_gather_concurrent_select(): async with get_database() as database: timestamps_1h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] timestamps_4h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h")] - coros = [_check_select_result(database, ts, "1h") for ts in timestamps_1h] - coros += [_check_select_result(database, ts, "4h") for ts in timestamps_4h] + coros = [_check_select_result(database, timestamp, "1h") for timestamp in timestamps_1h] + coros += [_check_select_result(database, timestamp, "4h") for timestamp in timestamps_4h] await asyncio.gather(*coros) @@ -162,8 +147,9 @@ async def test_create_tasks_concurrent_selects(): async with get_database() as database: timestamps_1h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] timestamps_1m = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1m")] - timestamps_4h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h", - size=50)] + timestamps_4h = [ + ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h", size=50) + ] calls_count = len(timestamps_1h) + len(timestamps_4h) + len(timestamps_1m) failed_calls = [] @@ -173,17 +159,16 @@ async def select_task(db, timestamp, time_frame): try: await _check_select_result(db, timestamp, time_frame) success_calls.append((timestamp, time_frame)) - except Exception as e: - failed_calls.append((timestamp, time_frame, e)) + except Exception as error: + failed_calls.append((timestamp, time_frame, error)) tasks = [] - for ts in timestamps_1h: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "1h"))) - for ts in timestamps_4h: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "4h"))) - for ts in timestamps_1m: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "1m"))) - # for wait for next cycle to make previous requests end and re-use previous cursors + for timestamp in timestamps_1h: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "1h"))) + for timestamp in timestamps_4h: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "4h"))) + for timestamp in timestamps_1m: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "1m"))) await asyncio_tools.wait_asyncio_next_cycle() await asyncio.gather(*tasks) @@ -196,9 +181,11 @@ async def test_stop_while_concurrent_select(): timestamps = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] await _check_select_result(database, timestamps[0]) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps]))) - # not enough time to finish all requests, most if not all will remaining pending + asyncio.gather(*[ + _check_select_result(database, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps + ]) + )) await asyncio_tools.wait_asyncio_next_cycle() @@ -206,8 +193,8 @@ async def test_double_database(): async with get_database() as database1, get_database(DATA_FILE2) as database2: timestamps1 = [ohlcv[0] for ohlcv in await database1.select(OHLCV, time_frame="1h")] timestamps2 = [ohlcv[0] for ohlcv in await database2.select(OHLCV, time_frame="1h")] - await asyncio.gather(*[_check_select_result(database1, ts) for ts in timestamps1]) - await asyncio.gather(*[_check_select_result(database2, ts) for ts in timestamps2]) + await asyncio.gather(*[_check_select_result(database1, timestamp) for timestamp in timestamps1]) + await asyncio.gather(*[_check_select_result(database2, timestamp) for timestamp in timestamps2]) async def test_double_database_stop_while_concurrent_select(): @@ -217,12 +204,17 @@ async def test_double_database_stop_while_concurrent_select(): await _check_select_result(database1, timestamps1[0]) await _check_select_result(database2, timestamps2[0]) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database1, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps1]))) + asyncio.gather(*[ + _check_select_result(database1, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps1 + ]) + )) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database2, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps2]))) - # not enough time to finish all requests, most if not all will remaining pending + asyncio.gather(*[ + _check_select_result(database2, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps2 + ]) + )) await asyncio_tools.wait_asyncio_next_cycle() @@ -234,33 +226,38 @@ async def test_insert(): async def test_insert_all(): async with get_temp_empty_database() as temp_empty_database: - await temp_empty_database.insert_all(OHLCV, - symbol=["xyz", "abc"], - timestamp=[1, 2], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["xyz", "abc"], + timestamp=[1, 2], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] assert await temp_empty_database.select(OHLCV, date="05") == [(2, 'abc', '10', '05')] async def test_delete(): async with get_temp_empty_database() as temp_empty_database: - await temp_empty_database.insert_all(OHLCV, - symbol=["xyz", "abc"], - timestamp=[1, 2], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["xyz", "abc"], + timestamp=[1, 2], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] - # no matching row to delete await temp_empty_database.delete(OHLCV, symbol="plop") assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] await temp_empty_database.delete(OHLCV, symbol="xyz") assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05')] - await temp_empty_database.insert_all(OHLCV, - symbol=["hoho", "dd"], - timestamp=[11, 11], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["hoho", "dd"], + timestamp=[11, 11], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [ (11, 'dd', '10', '05'), (11, 'hoho', '1', '01'), (2, 'abc', '10', '05') ] @@ -273,7 +270,6 @@ async def test_delete(): async def test_create_index(): async with get_temp_empty_database() as temp_empty_database: await temp_empty_database.insert(OHLCV, 1, symbol="xyz", price="1", date="01") - # ensure no exception await temp_empty_database.create_index(OHLCV, ["symbol", "timestamp"]) assert await temp_empty_database.select(OHLCV) == [(1, 'xyz', '1', '01')] @@ -283,8 +279,8 @@ async def _check_select_result(database, timestamp, time_frame="1h", expected_ex ohlcv = await database.select(OHLCV, time_frame=time_frame, timestamp=str(timestamp)) assert len(ohlcv) == 1 assert ohlcv[0][0] == timestamp - except Exception as e: - if e.__class__ is expected_exception: + except Exception as error: + if error.__class__ is expected_exception: pass else: raise diff --git a/packages/backtesting/tests/importers/test_exchange_importer.py b/packages/backtesting/tests/importers/test_exchange_importer.py index 0736c3b6fb..ce13d74060 100644 --- a/packages/backtesting/tests/importers/test_exchange_importer.py +++ b/packages/backtesting/tests/importers/test_exchange_importer.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU Lesser General Public # License along with this library. import pytest -import os from contextlib import asynccontextmanager @@ -23,20 +22,27 @@ from octobot_backtesting.enums import ExchangeDataTables from octobot_commons.enums import TimeFrames +import tests.database_test_util as database_test_util + # All test coroutines will be treated as marked. pytestmark = pytest.mark.asyncio +EXCHANGE_HISTORY_DATA_FILE = "ExchangeHistoryDataCollector_1589740606.4862757.data" + # use context manager instead of fixture to prevent pytest threads issues @asynccontextmanager async def get_importer(): - database_file = os.path.join("tests", "static", "ExchangeHistoryDataCollector_1589740606.4862757.data") - importer = ExchangeDataImporter({}, database_file) + temp_database_path = database_test_util.copy_static_database_fixture(EXCHANGE_HISTORY_DATA_FILE) try: - await importer.initialize() - yield importer + importer = ExchangeDataImporter({}, temp_database_path) + try: + await importer.initialize() + yield importer + finally: + await importer.stop() finally: - await importer.stop() + database_test_util.remove_temp_database(temp_database_path) async def test_initialize(): diff --git a/packages/commons/octobot_commons/databases/__init__.py b/packages/commons/octobot_commons/databases/__init__.py index 22b0191ace..5d48d0b39b 100644 --- a/packages/commons/octobot_commons/databases/__init__.py +++ b/packages/commons/octobot_commons/databases/__init__.py @@ -57,8 +57,7 @@ ) from octobot_commons.databases.relational_databases import ( - SQLiteDatabase, - new_sqlite_database, + BaseSQLiteDatabase, ) from octobot_commons.databases.run_databases import ( @@ -98,8 +97,7 @@ "DBWriterReader", "CacheDatabase", "CacheTimestampDatabase", - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", "RunDatabasesIdentifier", "RunDatabasesProvider", "init_bot_storage", diff --git a/packages/commons/octobot_commons/databases/relational_databases/__init__.py b/packages/commons/octobot_commons/databases/relational_databases/__init__.py index 315bc6be37..7d7c366402 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/__init__.py +++ b/packages/commons/octobot_commons/databases/relational_databases/__init__.py @@ -18,12 +18,14 @@ from octobot_commons.databases.relational_databases import sqlite from octobot_commons.databases.relational_databases.sqlite import ( - SQLiteDatabase, - new_sqlite_database, + BaseSQLiteDatabase, + open_sqlite_database, + sqlite_database_write_lock, ) __all__ = [ - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", + "open_sqlite_database", + "sqlite_database_write_lock", ] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py b/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py index 403d6dda0c..5fbf984aa7 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py +++ b/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py @@ -16,14 +16,15 @@ # License along with this library. -from octobot_commons.databases.relational_databases.sqlite import sqlite_database -from octobot_commons.databases.relational_databases.sqlite.sqlite_database import ( - SQLiteDatabase, - new_sqlite_database, +from octobot_commons.databases.relational_databases.sqlite.base_sqlite_database import ( + BaseSQLiteDatabase, + open_sqlite_database, + sqlite_database_write_lock, ) __all__ = [ - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", + "open_sqlite_database", + "sqlite_database_write_lock", ] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py new file mode 100644 index 0000000000..7b7e641f5d --- /dev/null +++ b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py @@ -0,0 +1,170 @@ +# pylint: disable=C0116 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import contextlib +import pathlib +import sqlite3 + +import octobot_commons.logging as logging +import octobot_commons.errors as errors +import octobot_commons.databases.relational_databases.sqlite.cursor_pool as cursor_pool +import octobot_commons.constants as constants + +try: + import aiosqlite +except ImportError: + if constants.USE_MINIMAL_LIBS: + class AiosqliteImportMock: + def connect(self, *args): + raise ImportError("aiosqlite not installed") + + aiosqlite = AiosqliteImportMock() + else: + raise + +_write_locks: dict[str, asyncio.Lock] = {} + + +def _get_write_lock(file_path: str) -> asyncio.Lock: + if file_path not in _write_locks: + _write_locks[file_path] = asyncio.Lock() + return _write_locks[file_path] + + +@contextlib.asynccontextmanager +async def sqlite_database_write_lock(file_path: str): + async with _get_write_lock(file_path): + yield + + +@contextlib.asynccontextmanager +async def open_sqlite_database(database_cls, file_path: str, read_only: bool = False): + if read_only: + database = database_cls(file_path, read_only=True) + try: + await database.initialize() + yield database + finally: + await database.stop() + else: + async with sqlite_database_write_lock(file_path): + database = database_cls(file_path, read_only=False) + try: + await database.initialize() + yield database + finally: + await database.stop() + + +class BaseSQLiteDatabase: + def __init__(self, file_name, read_only: bool = False): + self.file_name = file_name + self.read_only = read_only + self.logger = logging.get_logger(self.__class__.__name__) + self.connection = None + self._cursor_pool = None + + def connection_pragmas(self) -> list[str]: + # WAL: readers (mode=ro) get a consistent snapshot while writers are active; + # uncommitted writer work stays in the WAL and is rolled back on recovery. + # synchronous=FULL: fsync on commit so committed data survives sudden process kill. + # busy_timeout: wait up to 5s when a read overlaps a writer checkpoint instead of failing immediately. + return [ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=FULL", + "PRAGMA busy_timeout=5000", + ] + + def _ensure_writable(self) -> None: + if self.read_only: + raise errors.DatabaseReadOnlyError( + f"Cannot write to read-only database (file: {self.file_name})" + ) + + async def initialize(self): + try: + if self.read_only: + database_uri = f"{pathlib.Path(self.file_name).resolve().as_uri()}?mode=ro" + self.connection = await aiosqlite.connect(database_uri, uri=True) + else: + self.connection = await aiosqlite.connect(self.file_name) + self._cursor_pool = cursor_pool.CursorPool(self.connection) + for pragma in self.connection_pragmas(): + async with self.aio_cursor() as cursor: + await cursor.execute(pragma) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as err: + raise errors.DatabaseNotFoundError(f"{err} (file: {self.file_name})") + + @contextlib.asynccontextmanager + async def aio_cursor(self) -> sqlite3.Cursor: + async with self._cursor_pool.idle_cursor() as cursor: + yield cursor.cursor + + async def execute(self, sql: str, parameters=()) -> None: + self._ensure_writable() + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + + async def executemany(self, sql: str, parameters: list) -> None: + self._ensure_writable() + async with self.aio_cursor() as cursor: + await cursor.executemany(sql, parameters) + + async def fetchall(self, sql: str, parameters=()) -> list: + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + return await cursor.fetchall() + + async def fetchone(self, sql: str, parameters=()) -> tuple | None: + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + return await cursor.fetchone() + + async def commit(self) -> None: + self._ensure_writable() + await self.connection.commit() + + async def stop(self): + try: + if self._cursor_pool is not None: + await self._cursor_pool.close() + self._cursor_pool = None + if self.connection is not None and not self.read_only: + # Normal shutdown: merge all WAL pages into the main DB and truncate the -wal file. + # TRUNCATE resets the WAL to zero length so sidecar files do not accumulate across + # open/close cycles. On SIGKILL this never runs; SQLite auto-checkpoint + next open + # still recover. + checkpoint_cursor = await self.connection.cursor() + try: + await checkpoint_cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except sqlite3.OperationalError as error: + # TRUNCATE needs exclusive access; concurrent readers (same connection + # pool or another process) can still hold locks during shutdown. + # Skipping is safe: committed pages are already on disk (synchronous=FULL); + # only the -wal file may not be truncated until the next open/checkpoint. + self.logger.debug( + "wal_checkpoint(TRUNCATE) skipped during shutdown for %s: %s", + self.file_name, + error, + ) + finally: + await checkpoint_cursor.close() + finally: + if self.connection is not None: + conn = self.connection + self.connection = None + await conn.close() diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index ff01a254c1..132356d8d8 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -99,6 +99,12 @@ class DatabaseNotFoundError(Exception): """ +class DatabaseReadOnlyError(Exception): + """ + Raised when a write operation is attempted on a read-only database connection + """ + + class MissingDataError(Exception): """ Raised when there is not enough available candles diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py b/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py new file mode 100644 index 0000000000..cdd1c63f3a --- /dev/null +++ b/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py @@ -0,0 +1,166 @@ +import os +import asyncio +import contextlib +import sqlite3 +import tempfile + +import mock +import pytest + +import octobot_commons.errors as errors +import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database_module + +pytestmark = pytest.mark.asyncio + + +class TestBaseSQLiteDatabaseConnectionPragmas: + async def test_pragma_applied_on_initialize(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + journal_mode = await database.fetchone("PRAGMA journal_mode") + synchronous_mode = await database.fetchone("PRAGMA synchronous") + assert journal_mode[0] == "wal" + assert synchronous_mode[0] == 2 # FULL + finally: + await database.stop() + os.remove(file_name) + + +class TestBaseSQLiteDatabaseInitializeStop: + async def test_initialize_and_stop(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + assert database.connection is not None + await database.stop() + assert database.connection is None + finally: + os.remove(file_name) + + async def test_stop_succeeds_when_checkpoint_would_be_busy(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + checkpoint_cursor = mock.AsyncMock() + checkpoint_cursor.execute = mock.AsyncMock( + side_effect=sqlite3.OperationalError("database table is locked"), + ) + checkpoint_cursor.close = mock.AsyncMock() + with mock.patch.object( + database.connection, + "cursor", + mock.AsyncMock(return_value=checkpoint_cursor), + ): + await database.stop() + assert database.connection is None + finally: + os.remove(file_name) + + +class TestBaseSQLiteDatabaseExecuteFetch: + @contextlib.asynccontextmanager + async def _empty_database(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + await database.initialize() + try: + await database.execute( + "CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT NOT NULL)" + ) + yield database + finally: + await database.stop() + os.remove(file_name) + + async def test_execute_fetchone_fetchall_and_commit(self): + async with self._empty_database() as database: + await database.execute( + "INSERT INTO sample (value) VALUES (?)", + ("first",), + ) + await database.commit() + row = await database.fetchone("SELECT value FROM sample WHERE id = ?", (1,)) + assert row == ("first",) + rows = await database.fetchall("SELECT value FROM sample") + assert rows == [("first",)] + + async def test_executemany(self): + async with self._empty_database() as database: + await database.executemany( + "INSERT INTO sample (value) VALUES (?)", + [("one",), ("two",)], + ) + await database.commit() + rows = await database.fetchall("SELECT value FROM sample ORDER BY id") + assert rows == [("one",), ("two",)] + + +class TestBaseSQLiteDatabaseReadOnly: + @contextlib.asynccontextmanager + async def _database_with_sample_table(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + writable_database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + await writable_database.initialize() + await writable_database.execute( + "CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT NOT NULL)" + ) + await writable_database.execute( + "INSERT INTO sample (value) VALUES (?)", + ("stored",), + ) + await writable_database.commit() + await writable_database.stop() + + read_only_database = base_sqlite_database_module.BaseSQLiteDatabase(file_name, read_only=True) + await read_only_database.initialize() + try: + yield read_only_database + finally: + await read_only_database.stop() + os.remove(file_name) + + async def test_fetchall_allowed_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + rows = await database.fetchall("SELECT value FROM sample") + assert rows == [("stored",)] + + async def test_execute_raises_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + with pytest.raises(errors.DatabaseReadOnlyError): + await database.execute("INSERT INTO sample (value) VALUES (?)", ("blocked",)) + + async def test_commit_raises_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + with pytest.raises(errors.DatabaseReadOnlyError): + await database.commit() + + +class TestSqliteDatabaseWriteLock: + async def test_concurrent_write_locks_for_same_path_serialize(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + lock_order: list[str] = [] + + async def hold_write_lock(lock_name: str): + async with base_sqlite_database_module.sqlite_database_write_lock(file_name): + lock_order.append(f"{lock_name}_start") + await asyncio.sleep(0.05) + lock_order.append(f"{lock_name}_end") + + await asyncio.gather(hold_write_lock("first"), hold_write_lock("second")) + assert lock_order.index("first_start") < lock_order.index("first_end") + assert lock_order.index("second_start") < lock_order.index("second_end") + assert ( + lock_order.index("first_end") < lock_order.index("second_start") + or lock_order.index("second_end") < lock_order.index("first_start") + ) + os.remove(file_name) diff --git a/packages/flow/octobot_flow/constants.py b/packages/flow/octobot_flow/constants.py index 8245bebb7e..77219b3b85 100644 --- a/packages/flow/octobot_flow/constants.py +++ b/packages/flow/octobot_flow/constants.py @@ -16,3 +16,6 @@ DEFAULT_COPY_TRADING_ORPHAN_GRACE_ABORT_THRESHOLD = 2 PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS = 12 * 60 * 60 + +PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS = 180 +PORTFOLIO_HISTORY_DAILY_TIMEFRAME = commons_enums.TimeFrames.ONE_DAY diff --git a/packages/flow/octobot_flow/entities/__init__.py b/packages/flow/octobot_flow/entities/__init__.py index fd61b0f372..916178bb90 100644 --- a/packages/flow/octobot_flow/entities/__init__.py +++ b/packages/flow/octobot_flow/entities/__init__.py @@ -40,6 +40,10 @@ ExchangeAccountRefreshResult, GlobalViewAccountRefreshResult, ) +from octobot_flow.entities.portfolio_history import ( + PortfolioHistoryAccountContext, + PortfolioHistoryRunResult, +) __all__ = [ "AccountElements", "ExchangeAccountElements", @@ -73,4 +77,6 @@ "GlobalViewAccountContext", "ExchangeAccountRefreshResult", "GlobalViewAccountRefreshResult", + "PortfolioHistoryAccountContext", + "PortfolioHistoryRunResult", ] diff --git a/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py index 7a835e548d..87ff39a2e6 100644 --- a/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py +++ b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py @@ -9,7 +9,7 @@ @dataclasses.dataclass class ExchangeAccountRefreshResult: assets: list[protocol_models.DetailedAssetsForTradingType] - portfolio_snapshot: protocol_models.PortfolioHistoricalValue + ticker_closes: dict[str, float] valuation_unit: str open_orders: list[dict] trades: list[dict] diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py index 279762ecf3..b63d3b5b43 100644 --- a/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py @@ -13,4 +13,3 @@ class GlobalViewAccountRefreshResult: open_orders: list[dict] | None = None trades: list[dict] | None = None positions: list[dict] | None = None - portfolio_history_state: protocol_models.PortfolioHistoricalValuesState | None = None diff --git a/packages/flow/octobot_flow/entities/portfolio_history/__init__.py b/packages/flow/octobot_flow/entities/portfolio_history/__init__.py new file mode 100644 index 0000000000..e7258f0083 --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/__init__.py @@ -0,0 +1,7 @@ +from octobot_flow.entities.portfolio_history.portfolio_history_account_context import PortfolioHistoryAccountContext +from octobot_flow.entities.portfolio_history.portfolio_history_run_result import PortfolioHistoryRunResult + +__all__ = [ + "PortfolioHistoryAccountContext", + "PortfolioHistoryRunResult", +] diff --git a/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py new file mode 100644 index 0000000000..a9e00a236f --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py @@ -0,0 +1,16 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + + +@dataclasses.dataclass +class PortfolioHistoryAccountContext: + account: protocol_models.Account + exchange_account: protocol_models.ExchangeAccount + exchange_config: protocol_models.ExchangeConfig + trading_type: protocol_models.TradingType + auth_details: exchange_data_module.ExchangeAuthDetails diff --git a/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py new file mode 100644 index 0000000000..42d97cf961 --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py @@ -0,0 +1,18 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + + +@dataclasses.dataclass +class PortfolioHistoryRunResult: + account_id: str + exchange_name: str + trades_count: int = 0 + transactions_count: int = 0 + skipped: bool = False + error: str | None = None + is_simulated: bool = False + trading_type: str = "" + duration_seconds: float | None = None + price_symbols_count: int = 0 diff --git a/packages/flow/octobot_flow/jobs/global_view_account_job.py b/packages/flow/octobot_flow/jobs/global_view_account_job.py index 16f0eeb917..bf924740c6 100644 --- a/packages/flow/octobot_flow/jobs/global_view_account_job.py +++ b/packages/flow/octobot_flow/jobs/global_view_account_job.py @@ -3,6 +3,7 @@ import octobot_protocol.models as protocol_models import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.api as trading_api import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_module import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping @@ -109,4 +110,14 @@ async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: refresh_result, persist_open_orders=persist_open_orders, ) + # Update persisted latest-tickers cache from fetched ticker closes. + if exchange_refresh_result.ticker_closes: + await trading_api.update_latest_tickers( + self.context.exchange_config.exchange, + protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + self.context.trading_type + ).value, + self.context.exchange_config.sandboxed, + exchange_refresh_result.ticker_closes, + ) return refresh_result diff --git a/packages/flow/octobot_flow/jobs/portfolio_history_job.py b/packages/flow/octobot_flow/jobs/portfolio_history_job.py new file mode 100644 index 0000000000..1f230e2695 --- /dev/null +++ b/packages/flow/octobot_flow/jobs/portfolio_history_job.py @@ -0,0 +1,214 @@ +import asyncio +import time + +import octobot_commons.constants as commons_constants +import octobot_commons.logging as commons_logging +import octobot_commons.symbols.symbol_util as symbol_util +import octobot_protocol.models as protocol_models +import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot_flow.entities +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory_module +import octobot_flow.logic.portfolio_history.trading_history_merge as trading_history_merge_module +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module +import octobot_flow.repositories.exchange.transactions_repository as transactions_repository_module + +import tentacles.Meta.Keywords.scripting_library as scripting_library + + +logger = commons_logging.get_logger("PortfolioHistoryJob") + + +class PortfolioHistoryJob: + def __init__( + self, + wallet_id: str, + contexts: list[portfolio_history_entities.PortfolioHistoryAccountContext], + data_root: str = None, + ): + self.wallet_id = wallet_id + self.contexts = contexts + self.data_root = data_root + + async def run(self) -> list[portfolio_history_entities.PortfolioHistoryRunResult]: + """Run data collection for all exchange accounts in parallel.""" + tasks = [ + self._run_for_account(context) + for context in self.contexts + ] + return list(await asyncio.gather(*tasks)) + + async def _run_for_account( + self, context: portfolio_history_entities.PortfolioHistoryAccountContext + ) -> portfolio_history_entities.PortfolioHistoryRunResult: + account = context.account + account_id = account.id + exchange_name = context.exchange_config.exchange + result_metadata = _result_metadata_from_context(context) + + # Skip unsupported account types. + specifics = account.specifics + if ( + specifics is None + or specifics.actual_instance is None + or not isinstance(specifics.actual_instance, protocol_models.ExchangeAccount) + or account.is_simulated + ): + return _skipped_run_result(account_id, exchange_name, result_metadata) + + started_at = time.monotonic() + try: + result = await self._fetch_and_persist(context) + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history job failed for account {account_id}: {error}", + ) + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account_id, + exchange_name=exchange_name, + error=str(error), + duration_seconds=time.monotonic() - started_at, + **result_metadata, + ) + result.duration_seconds = time.monotonic() - started_at + return result + + async def _fetch_and_persist( + self, context: portfolio_history_entities.PortfolioHistoryAccountContext + ) -> portfolio_history_entities.PortfolioHistoryRunResult: + account = context.account + exchange_config = context.exchange_config + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + context.trading_type + ).value + + profile_data = _build_profile_data(context) + exchange_data = exchange_data_module.exchange_data_factory( + exchange_internal_name=exchange_config.exchange, + exchange_type=exchange_type, + sandboxed=exchange_config.sandboxed, + auth_details=context.auth_details, + ) + tentacles_setup_config = tentacles_manager_api.get_full_tentacles_setup_config() + + async with trading_exchanges.exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ) as exchange_manager: + await trades_repository_module.TradesRepository.ensure_temporary_trades_channel( + exchange_manager, + ) + fetched_exchange_data = octobot_flow.entities.FetchedExchangeData() + trades_repo = trades_repository_module.TradesRepository( + exchange_manager, [], fetched_exchange_data + ) + tx_repo = transactions_repository_module.TransactionsRepository( + exchange_manager, [], fetched_exchange_data + ) + + # Fetch trades, deposits, withdrawals in parallel within this account. + symbols = list(exchange_config.historical_trade_symbols or []) + trades_task = trades_repo.fetch_trades(symbols) + deposits_task = tx_repo.fetch_deposits() + withdrawals_task = tx_repo.fetch_withdrawals() + trades, deposits, withdrawals = await asyncio.gather( + trades_task, deposits_task, withdrawals_task + ) + + all_transactions = deposits + withdrawals + + # Update daily price cache for relevant symbols. + reference_market = scripting_library.get_default_exchange_reference_market( + exchange_config.exchange, + ) + price_symbols = _derive_price_symbols(symbols, all_transactions, reference_market) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + exchange_config.exchange, + exchange_type, + exchange_config.sandboxed, + price_symbols, + self.data_root, + ) + + # Merge and persist trading history. + trading_history_merge_module.merge_and_persist_trading_history( + self.wallet_id, account.id, trades, all_transactions + ) + + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account.id, + exchange_name=exchange_config.exchange, + trades_count=len(trades), + transactions_count=len(all_transactions), + is_simulated=account.is_simulated, + trading_type=context.trading_type.value, + price_symbols_count=len(price_symbols), + ) + + +def _result_metadata_from_context( + context: portfolio_history_entities.PortfolioHistoryAccountContext, +) -> dict: + return { + "is_simulated": context.account.is_simulated, + "trading_type": context.trading_type.value, + } + + +def _skipped_run_result( + account_id: str, + exchange_name: str, + result_metadata: dict, +) -> portfolio_history_entities.PortfolioHistoryRunResult: + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account_id, + exchange_name=exchange_name, + skipped=True, + **result_metadata, + ) + + +def _build_profile_data(context: portfolio_history_entities.PortfolioHistoryAccountContext): + return profile_data_factory_module.profile_data_for_account( + context.account, + context.exchange_account, + context.exchange_config, + context.trading_type, + is_simulated=context.account.is_simulated, + ) + + +def _derive_price_symbols( + trade_symbols: list[str], + transactions: list[dict], + reference_market: str, +) -> list[str]: + """Build the set of symbols whose daily prices should be cached.""" + symbols = set(trade_symbols) + for transaction in transactions: + currency = transaction.get("currency") + if currency and currency != reference_market: + symbols.add(symbol_util.merge_currencies(currency, reference_market)) + return [ + symbol for symbol in symbols + if _is_valid_trading_symbol(symbol) + ] + + +def _is_valid_trading_symbol(symbol: str) -> bool: + if "/" not in symbol: + return False + base_currency, quote_currency = symbol.split("/", 1) + if base_currency in commons_constants.USD_LIKE_COINS: + return False + return bool(base_currency) and bool(quote_currency) and base_currency != quote_currency diff --git a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py index e3c0986c33..035db240e0 100644 --- a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py +++ b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py @@ -16,6 +16,8 @@ import octobot_trading.personal_data.positions.protocol as positions_protocol import octobot_trading.personal_data.trades.protocol as trades_protocol import octobot_trading.personal_data.trades.trades_util as trades_util +import octobot_trading.personal_data.transactions.protocol as transactions_protocol +import octobot_trading.personal_data.transactions.transactions_util as transactions_util import octobot_flow.entities import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module @@ -120,6 +122,7 @@ def persist_account_trading( orders: list[dict], trades: list[dict], positions: list[dict], + transactions: list[dict] | None = None, ) -> None: try: trading_state = collection_providers.AccountTradingProvider.instance().load_state( @@ -145,6 +148,15 @@ def persist_account_trading( account_trading.trades = [ trades_protocol.to_protocol_trade(trade_dict) for trade_dict in merged_trade_dicts ] or None + if transactions is not None: + existing_tx_dicts = [ + transactions_protocol.to_exchange_columns_dict(protocol_tx) + for protocol_tx in (account_trading.transactions or []) + ] + merged_tx_dicts = transactions_util.merge_transactions_deduped(existing_tx_dicts, transactions) + account_trading.transactions = [ + transactions_protocol.to_protocol_transaction(tx_dict) for tx_dict in merged_tx_dicts + ] or None account_trading.updated_at = datetime.datetime.now(datetime.UTC) collection_providers.AccountTradingProvider.instance().save_state( user_id, diff --git a/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py index 3034281732..7d2bf53512 100644 --- a/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py +++ b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py @@ -7,7 +7,6 @@ import octobot_protocol.models as protocol_models import octobot_flow.entities -import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module def build_updated_account( @@ -28,19 +27,10 @@ def build_global_view_account_refresh_result( exchange_refresh_result: octobot_flow.entities.ExchangeAccountRefreshResult, ) -> octobot_flow.entities.GlobalViewAccountRefreshResult: updated_account = build_updated_account(context.account, exchange_refresh_result.assets) - evaluation_time = exchange_refresh_result.portfolio_snapshot.timestamp - portfolio_history_state = account_state_persistence_module.build_portfolio_history_state( - user_id, - context.account.id, - exchange_refresh_result.portfolio_snapshot, - exchange_refresh_result.valuation_unit, - evaluation_time, - ) return octobot_flow.entities.GlobalViewAccountRefreshResult( updated_account=updated_account, changed_order_ids=exchange_refresh_result.changed_order_ids, open_orders=exchange_refresh_result.open_orders, trades=exchange_refresh_result.trades, positions=exchange_refresh_result.positions, - portfolio_history_state=portfolio_history_state, ) diff --git a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py index 01a36815dc..2ac589515f 100644 --- a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py +++ b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py @@ -6,7 +6,6 @@ import octobot_commons.constants as commons_constants import octobot_commons.logging as octobot_commons_logging -import octobot_commons.timestamp_util as timestamp_util import octobot_protocol.models as protocol_models import octobot_trading.api as trading_api import octobot_trading.constants as trading_constants @@ -89,10 +88,15 @@ async def refresh_exchange_account( valuation_symbols=simulated_valuation_symbols, ) else: - await _refresh_portfolio_valuation(exchange_manager, valuation_unit) - portfolio_total = trading_api.get_current_portfolio_value(exchange_manager) - - # Step: build protocol assets and historical snapshot payload. + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, portfolio_content, valuation_unit, + ) + tickers = await _fetch_tickers(exchange_manager, valuation_symbols) + await _refresh_portfolio_valuation( + exchange_manager, valuation_unit, tickers=tickers, valuation_symbols=valuation_symbols, + ) + # Step: build protocol assets (holdings only, no historical snapshot). portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) balance_summary = octobot_commons_logging.get_private_placeholder_if_necessary( portfolio_util_module.get_balance_summary(portfolio_content, use_exchange_format=False) @@ -104,18 +108,6 @@ async def refresh_exchange_account( balance_summary, ) detailed_assets = portfolios_protocol.to_protocol_assets(portfolio_content) - historical_assets = _historical_assets_from_portfolio( - exchange_manager, - portfolio_content, - trading_type, - valuation_unit, - ) - evaluation_time = timestamp_util.utc_now_datetime() - portfolio_snapshot = protocol_models.PortfolioHistoricalValue( - timestamp=evaluation_time, - total=portfolio_total, - assets=historical_assets, - ) assets_for_trading_type = [ protocol_models.DetailedAssetsForTradingType( trading_type=trading_type, @@ -123,6 +115,9 @@ async def refresh_exchange_account( ) ] if detailed_assets else [] + # Collect ticker close prices for the persisted latest-tickers cache. + ticker_closes = _ticker_close_by_symbol_from_tickers(tickers) if tickers else {} + # Step: detect orders that disappeared since the previous refresh. changed_order_ids = order_change_detection_module.detect_changed_order_ids( previous_open_order_exchange_ids, @@ -130,7 +125,7 @@ async def refresh_exchange_account( ) return octobot_flow.entities.ExchangeAccountRefreshResult( assets=assets_for_trading_type, - portfolio_snapshot=portfolio_snapshot, + ticker_closes=ticker_closes, valuation_unit=valuation_unit, open_orders=open_orders, trades=trades, @@ -282,36 +277,3 @@ async def _fetch_open_orders_for_symbols(exchange_manager, symbols: list[str]) - return list(orders_by_exchange_id.values()) -def _historical_assets_from_portfolio( - exchange_manager, - portfolio_content: dict, - trading_type: protocol_models.TradingType, - valuation_unit: str, -) -> list[protocol_models.HistoricalAssetsForTradingType]: - historical_asset_values: list[protocol_models.HistoricalAssetValue] = [] - for symbol, symbol_balance in portfolio_content.items(): - total_holdings = float(symbol_balance.get(commons_constants.PORTFOLIO_TOTAL) or 0) - if total_holdings == 0: - continue - try: - unit_price = float( - trading_api.get_current_crypto_currency_value(exchange_manager, symbol) - ) - except (KeyError, trading_errors.MissingPriceDataError): - unit_price = 0.0 - asset_value = unit_price * total_holdings - historical_asset_values.append( - protocol_models.HistoricalAssetValue( - symbol=str(symbol), - holdings=total_holdings, - value=asset_value, - ) - ) - if not historical_asset_values: - return [] - return [ - protocol_models.HistoricalAssetsForTradingType( - trading_type=trading_type, - assets=historical_asset_values, - ) - ] diff --git a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py index 7dd4a05e6d..aa3e4dd3fd 100644 --- a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py +++ b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py @@ -21,9 +21,3 @@ def persist_global_view_refresh_result( account_id, refresh_result.open_orders or [], ) - if refresh_result.portfolio_history_state is not None: - collection_providers.AccountHistoryProvider.instance().save_state( - user_id, - account_id, - refresh_result.portfolio_history_state, - ) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/__init__.py b/packages/flow/octobot_flow/logic/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py new file mode 100644 index 0000000000..5dcae89b17 --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py @@ -0,0 +1,349 @@ +import math +import time + +import octobot_commons.enums as commons_enums +import octobot_commons.constants as commons_constants +import octobot_commons.logging as commons_logging +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.errors as trading_errors +import octobot_trading.exchanges.util.exchange_util as exchange_util + +import octobot_flow.constants as flow_constants + +logger = commons_logging.get_logger("DailyPriceCacheUpdater") + + +async def update_daily_prices( + exchange_manager, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + symbols: list[str], + data_root: str = None, +) -> None: + """ + Fetch daily OHLCV candles for the given reference symbols and merge close prices + into the persisted daily price cache under exchange-native fetch symbols. + """ + daily_prices = await trading_api.load_daily_prices( + exchange_name, exchange_type, sandboxed, data_root + ) + + for reference_symbol in symbols: + parsed = _parse_base_quote(reference_symbol) + if parsed is None: + continue + base_asset, quote = parsed + if base_asset in commons_constants.USD_LIKE_COINS: + continue + + sticky_fetch_symbol = daily_prices.get("sources", {}).get(base_asset) + if _is_daily_cache_up_to_date(daily_prices, reference_symbol): + continue + + since_ms = _compute_fetch_since_ms(daily_prices, reference_symbol) + candidates = _build_fetch_candidates( + exchange_manager, base_asset, quote, sticky_fetch_symbol, + ) + + fetch_result = await _fetch_daily_candles( + exchange_manager, since_ms, candidates, + ) + if fetch_result is None: + logger.info( + "Skipping daily price fetch for %s on %s: no supported market found", + reference_symbol, + exchange_name, + ) + continue + + candles, fetch_symbol, is_reversed = fetch_result + if sticky_fetch_symbol and fetch_symbol != sticky_fetch_symbol: + logger.info( + "Migrating daily closes for %s on %s from %s to %s", + base_asset, + exchange_name, + sticky_fetch_symbol, + fetch_symbol, + ) + await trading_api.rename_daily_closes_symbol( + exchange_name, exchange_type, sandboxed, + sticky_fetch_symbol, fetch_symbol, data_root, + ) + _move_symbol_cache_in_memory( + daily_prices, sticky_fetch_symbol, fetch_symbol, + ) + elif fetch_symbol != reference_symbol: + logger.debug( + "Fetched %s for %s on %s", + fetch_symbol, + reference_symbol, + exchange_name, + ) + + closes_by_timestamp = _closes_from_candles(candles, is_reversed) + await trading_api.merge_daily_prices( + exchange_name, exchange_type, sandboxed, fetch_symbol, closes_by_timestamp, data_root, + ) + await trading_api.set_daily_close_source( + exchange_name, exchange_type, sandboxed, base_asset, fetch_symbol, data_root, + ) + daily_prices.setdefault("sources", {})[base_asset] = fetch_symbol + daily_prices.setdefault("symbols", {}).setdefault(fetch_symbol, {}).update(closes_by_timestamp) + + +def _parse_base_quote(symbol: str) -> tuple[str, str] | None: + if "/" not in symbol: + return None + base_asset, quote = symbol.split("/", 1) + if not base_asset or not quote or base_asset == quote: + return None + return base_asset, quote + + +def _ordered_usd_like_quotes(quote: str) -> list[str]: + quotes = [] + if "USD" in commons_constants.USD_LIKE_COINS and quote != "USD": + quotes.append("USD") + for usd_like_quote in commons_constants.USD_LIKE_COINS: + if usd_like_quote == quote or usd_like_quote in quotes: + continue + quotes.append(usd_like_quote) + return quotes + + +def _is_reversed_fetch_symbol(base_asset: str, fetch_symbol: str) -> bool: + parsed = _parse_base_quote(fetch_symbol) + if parsed is None: + return False + fetch_base, _fetch_quote = parsed + return fetch_base != base_asset + + +def _build_fetch_candidates( + exchange_manager, + base_asset: str, + quote: str, + sticky_fetch_symbol: str | None, +) -> list[tuple[str, bool]]: + candidates: list[tuple[str, bool]] = [] + seen_fetch_symbols: set[str] = set() + + def add_candidate(fetch_symbol: str | None, is_reversed: bool) -> None: + if fetch_symbol is None or fetch_symbol in seen_fetch_symbols: + return + seen_fetch_symbols.add(fetch_symbol) + candidates.append((fetch_symbol, is_reversed)) + + if sticky_fetch_symbol: + add_candidate( + sticky_fetch_symbol, + _is_reversed_fetch_symbol(base_asset, sticky_fetch_symbol), + ) + + direct_symbol, is_reversed = exchange_util.get_associated_symbol( + exchange_manager, base_asset, quote, + ) + add_candidate(direct_symbol, is_reversed) + + if quote in commons_constants.USD_LIKE_COINS: + for alt_quote in _ordered_usd_like_quotes(quote): + alt_symbol, alt_is_reversed = exchange_util.get_associated_symbol( + exchange_manager, base_asset, alt_quote, + ) + add_candidate(alt_symbol, alt_is_reversed) + + return candidates + + +async def _get_historical_daily_candles( + exchange_manager, + fetch_symbol: str, + since_ms: int | None, +) -> list: + start_time_ms, end_time_ms = _compute_fetch_time_range_ms(since_ms) + candles = [] + async for batch in exchange_util.get_historical_ohlcv( + exchange_manager, + fetch_symbol, + commons_enums.TimeFrames.ONE_DAY, + start_time_ms, + end_time_ms, + ): + candles.extend(batch) + return candles + + +async def _get_symbol_daily_candles( + exchange_manager, + fetch_symbol: str, + since_ms: int | None, +) -> list: + fetch_kwargs = {} + if since_ms is not None: + fetch_kwargs["since"] = since_ms + return await exchange_manager.exchange.get_symbol_prices( + fetch_symbol, + commons_enums.TimeFrames.ONE_DAY, + **fetch_kwargs, + ) + + +def _supports_full_candle_history(exchange_name: str) -> bool: + return exchange_name in trading_constants.FULL_CANDLE_HISTORY_EXCHANGES + + +async def _fetch_daily_candles( + exchange_manager, + since_ms: int | None, + candidates: list[tuple[str, bool]], +) -> tuple[list, str, bool] | None: + exchange_name = exchange_manager.exchange_name + supports_full_history = _supports_full_candle_history(exchange_name) + for fetch_symbol, is_reversed in candidates: + if supports_full_history: + candles = await _try_full_history_daily_candles( + exchange_manager, exchange_name, fetch_symbol, since_ms, + ) + else: + candles = await _try_limited_history_daily_candles( + exchange_manager, exchange_name, fetch_symbol, since_ms, + ) + if candles: + return candles, fetch_symbol, is_reversed + return None + + +async def _try_full_history_daily_candles( + exchange_manager, + exchange_name: str, + fetch_symbol: str, + since_ms: int | None, +) -> list | None: + try: + candles = await _get_historical_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + logger.info( + "Daily candle historical fetch failed for %s on %s, retrying with get_symbol_prices: %s", + fetch_symbol, + exchange_name, + error, + ) + else: + if candles: + return candles + logger.info( + "Daily candle historical fetch returned empty for %s on %s, retrying with get_symbol_prices", + fetch_symbol, + exchange_name, + ) + + try: + return await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + logger.exception( + error, + True, + f"Failed to fetch daily candles for {fetch_symbol}: {error}", + ) + return None + + +async def _try_limited_history_daily_candles( + exchange_manager, + exchange_name: str, + fetch_symbol: str, + since_ms: int | None, +) -> list | None: + try: + candles = await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + logger.info( + "Daily candle ideal fetch failed for %s on %s, retrying without since/limit: %s", + fetch_symbol, + exchange_name, + error, + ) + try: + candles = await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, None, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as fallback_error: + logger.error( + "Failed to fetch daily candles for %s on %s after fallback without since/limit: %s", + fetch_symbol, + exchange_name, + fallback_error, + ) + return None + if candles: + return candles + return None + + +def _closes_from_candles(candles: list, is_reversed: bool) -> dict[str, float]: + closes_by_timestamp = {} + for candle in candles: + day_ts = str(int(candle[0] / 1000)) + close_price = float(candle[4]) + if is_reversed: + close_price = 1.0 / close_price + closes_by_timestamp[day_ts] = close_price + return closes_by_timestamp + + +def _move_symbol_cache_in_memory(daily_prices: dict, old_symbol: str, new_symbol: str) -> None: + symbols = daily_prices.setdefault("symbols", {}) + old_closes = symbols.pop(old_symbol, {}) + if old_closes: + symbols.setdefault(new_symbol, {}).update(old_closes) + + +def _utc_day_start(timestamp: float) -> float: + """Return the UTC day-start (00:00:00) for the given unix timestamp.""" + return float(math.floor( + timestamp / commons_constants.DAYS_TO_SECONDS + ) * commons_constants.DAYS_TO_SECONDS) + + +def _compute_fetch_since_ms(daily_prices: dict, symbol: str) -> int | None: + newest_timestamp = trading_api.get_latest_daily_price_timestamp(daily_prices, symbol) + if newest_timestamp is None: + return None + + since_seconds = newest_timestamp - commons_constants.DAYS_TO_SECONDS + return int(since_seconds * 1000) + + +def _compute_fetch_time_range_ms(since_ms: int | None) -> tuple[int, int]: + end_time_ms = int(time.time() * commons_constants.MSECONDS_TO_SECONDS) + if since_ms is not None: + return since_ms, end_time_ms + lookback_ms = ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS + * commons_constants.DAYS_TO_SECONDS + * commons_constants.MSECONDS_TO_SECONDS + ) + return end_time_ms - lookback_ms, end_time_ms + + +def _is_daily_cache_up_to_date(daily_prices: dict, symbol: str) -> bool: + latest_cached_timestamp = trading_api.get_latest_daily_price_timestamp(daily_prices, symbol) + if latest_cached_timestamp is None: + return False + return latest_cached_timestamp >= _utc_day_start(time.time()) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py b/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py new file mode 100644 index 0000000000..e5cf8aad32 --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py @@ -0,0 +1,50 @@ +import decimal + +import octobot_commons.constants as commons_constants +import octobot_trading.api as trading_api + + +def compute_daily_portfolio_values( + daily_holdings: dict[float, dict[str, dict[str, decimal.Decimal]]], + daily_prices: dict, + latest_tickers: dict, + reference_market: str = "USDT", +) -> list[dict]: + """ + Value daily portfolio holdings using daily price cache, falling back to + latest ticker prices when a historical price is unavailable. + + Returns a list of dicts with 'timestamp' and 'value' keys, sorted ascending. + """ + valued_days = [] + for day_timestamp in sorted(daily_holdings): + holdings = daily_holdings[day_timestamp] + total_value = decimal.Decimal(0) + day_ts_str = str(int(day_timestamp)) + + for asset, amounts in holdings.items(): + asset_total = amounts.get("total", decimal.Decimal(0)) + if asset_total == 0: + continue + + if asset == reference_market: + total_value += asset_total + continue + + if asset in commons_constants.USD_LIKE_COINS: + total_value += asset_total + continue + + symbol = f"{asset}/{reference_market}" + price = trading_api.get_daily_price(daily_prices, symbol, day_ts_str) + if price is None: + price = trading_api.get_latest_ticker_close(latest_tickers, symbol) + if price is not None: + total_value += asset_total * decimal.Decimal(str(price)) + + valued_days.append({ + "timestamp": day_timestamp, + "value": float(total_value), + }) + + return valued_days diff --git a/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py b/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py new file mode 100644 index 0000000000..07988eaefa --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py @@ -0,0 +1,52 @@ +import datetime + +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.personal_data.trades.protocol as trades_protocol +import octobot_trading.personal_data.trades.trades_util as trades_util +import octobot_trading.personal_data.transactions.protocol as transactions_protocol +import octobot_trading.personal_data.transactions.transactions_util as transactions_util + + +def merge_and_persist_trading_history( + wallet_id: str, + account_id: str, + new_trades: list[dict], + new_transactions: list[dict], +) -> None: + """ + Merge new trades and transactions into AccountTrading without touching + orders or positions. + """ + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + wallet_id, + account_id, + ) + account_trading = trading_state.account_trading + + # Merge trades. + existing_trade_dicts = [ + trades_protocol.exchange_columns_dict_from_protocol_trade(protocol_trade) + for protocol_trade in (account_trading.trades or []) + ] + merged_trade_dicts = trades_util.merge_trades_deduped(existing_trade_dicts, new_trades) + account_trading.trades = [ + trades_protocol.to_protocol_trade(trade_dict) for trade_dict in merged_trade_dicts + ] or None + + # Merge transactions. + existing_tx_dicts = [ + transactions_protocol.to_exchange_columns_dict(protocol_tx) + for protocol_tx in (account_trading.transactions or []) + ] + merged_tx_dicts = transactions_util.merge_transactions_deduped(existing_tx_dicts, new_transactions) + account_trading.transactions = [ + transactions_protocol.to_protocol_transaction(tx_dict) for tx_dict in merged_tx_dicts + ] or None + + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + wallet_id, + account_id, + trading_state, + ) diff --git a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py index ba1b034ceb..37bbc78ff2 100644 --- a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py @@ -1,11 +1,9 @@ import typing import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import -import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools_import import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums import octobot_trading.storage as orders_storage -import octobot_trading.constants as trading_constants import octobot_trading.personal_data as trading_personal_data @@ -22,7 +20,7 @@ async def fetch_open_orders( ) open_orders = await updater.fetch_open_orders(symbols) return [ - exchanges_test_tools_import.parse_order_into_dict( + trading_personal_data.OrdersUpdater.ensure_parsing( self.exchange_manager, order, True, ignore_unsupported_orders ) for order in open_orders diff --git a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py index 8254ee9031..8dc6dccd5c 100644 --- a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py @@ -1,12 +1,35 @@ +import typing + +import octobot_trading.constants as trading_constants +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.personal_data as trading_personal_data + import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import -import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools_import class TradesRepository(base_exchange_repository_import.BaseExchangeRepository): + @classmethod + async def ensure_temporary_trades_channel(cls, exchange_manager) -> None: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) + async def fetch_trades(self, symbols: list[str]) -> list[dict]: if not symbols: return [] - return await exchanges_test_tools_import.get_trades( - self.exchange_manager, None, symbols=symbols + updater = typing.cast( + trading_personal_data.TradesUpdater, + self.get_channel_updater(trading_constants.TRADES_CHANNEL), ) + raw_trades = await updater.fetch_trades(symbols) + return [ + parsed_trade + for raw_trade in raw_trades + if (parsed_trade := trading_personal_data.TradesUpdater.ensure_parsing( + self.exchange_manager, raw_trade + )) + ] diff --git a/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py b/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py new file mode 100644 index 0000000000..689ea52a4d --- /dev/null +++ b/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py @@ -0,0 +1,35 @@ +import octobot_commons.logging as commons_logging +import octobot_trading.errors as trading_errors + +import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import + +logger = commons_logging.get_logger("TransactionsRepository") + + +class TransactionsRepository(base_exchange_repository_import.BaseExchangeRepository): + + async def fetch_deposits(self, since: int = None, limit: int = None) -> list[dict]: + try: + return await self.exchange_manager.exchange.get_deposits(since=since, limit=limit) + except trading_errors.NotSupported: + return [] + except trading_errors.AuthenticationError as error: + logger.warning( + "Skipping deposits fetch for %s: %s", + self.exchange_manager.exchange_name, + error, + ) + return [] + + async def fetch_withdrawals(self, since: int = None, limit: int = None) -> list[dict]: + try: + return await self.exchange_manager.exchange.get_withdrawals(since=since, limit=limit) + except trading_errors.NotSupported: + return [] + except trading_errors.AuthenticationError as error: + logger.warning( + "Skipping withdrawals fetch for %s: %s", + self.exchange_manager.exchange_name, + error, + ) + return [] diff --git a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py index 1c8baffce7..e84497ee44 100644 --- a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py +++ b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py @@ -148,6 +148,11 @@ async def fake_exchange_manager(*_args, **_kwargs): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( exchange_account_refresh_module.tickers_repository_module.TickersRepository, "fetch_tickers", @@ -168,11 +173,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_orders", return_value=[], ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", @@ -185,10 +185,6 @@ async def fake_exchange_manager(*_args, **_kwargs): ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) portfolio_manager.apply_forced_portfolio.assert_called_once() - assert refresh_result.portfolio_history_state is not None - assert refresh_result.portfolio_history_state.history is not None - assert refresh_result.portfolio_history_state.history.unit == "USDC" - assert refresh_result.portfolio_history_state.history.values[-1].total > 0 assert refresh_result.updated_account.assets asset_symbols = { asset.symbol @@ -234,6 +230,11 @@ async def fake_exchange_manager(*_args, **_kwargs): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -249,11 +250,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_orders", return_value=previous_open_orders, ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", @@ -270,8 +266,5 @@ async def fake_exchange_manager(*_args, **_kwargs): for call in exchange_manager.exchange.get_open_orders.await_args_list } assert called_symbols == {"BTC/USDT", "ETH/USDT"} - assert refresh_result.portfolio_history_state is not None - assert refresh_result.portfolio_history_state.history.unit == commons_constants.DEFAULT_REFERENCE_MARKET assert refresh_result.updated_account.assets - assert refresh_result.portfolio_history_state.history.values[-1].total > 0 assert refresh_result.changed_order_ids == {"gone-order-1"} diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index bda3b42770..d53a112d81 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -316,7 +316,13 @@ def _get_action_by_id( _FERNET_ENCRYPTED_PREFIX = "gAAAAA" -# --- Child on-disk readiness (poll after init_state_ok; PID can be up before first dump / encrypt) --- +# --- Child on-disk readiness (poll until process_bot_state.json exists) --- + + +def _child_process_state_file_ready(inner: typing.Optional[dict]) -> bool: + if not inner or not inner.get("user_root"): + return False + return os.path.isfile(_process_bot_state_path(inner)) def _process_bot_state_path(inner: dict) -> str: @@ -345,6 +351,46 @@ async def _wait_for_process_bot_state_file( ) +async def wait_for_child_process_ready( + inner: dict, + *, + timeout_sec: float = GLOBAL_START_TIMEOUT_SEC, +) -> str: + """Poll until process_bot_state.json exists; return state_path.""" + state_path = _process_bot_state_path(inner) + await _wait_for_process_bot_state_file(state_path, timeout_sec=timeout_sec) + return state_path + + +async def poll_automation_until_child_process_ready( + automation_state: dict, + *, + timeout_sec: float = GLOBAL_START_TIMEOUT_SEC, +) -> tuple[dict, dict, str]: + """ + Run automation job polls until process_bot_state.json exists on disk. + Returns (updated automation state dump, recall inner dict, state_path). + """ + deadline = time.monotonic() + timeout_sec + inner: typing.Optional[dict] = None + state = automation_state + while time.monotonic() < deadline: + poll_job = await run_automation_job_without_exchange_manager(state, [], [], {}) + _assert_run_octobot_process_recall_scheduled_to_in_dump(poll_job.dump()) + run_action_state = _get_action_by_id(poll_job, ACTION_ID_RUN_OCTOBOT) + assert run_action_state is not None + inner = _recall_inner_from_dsl_action(run_action_state) + state = poll_job.dump() + if _child_process_state_file_ready(inner): + assert inner is not None + state_path = _process_bot_state_path(inner) + return state, inner, state_path + await asyncio.sleep(SLEEP_BETWEEN_JOB_POLLS_SEC) + pytest.fail( + f"Timed out waiting for process_bot_state.json within {timeout_sec}s" + ) + + async def _assert_encrypted_exchange_credentials_in_user_config( user_root: typing.Union[pathlib.Path, str], exchange_internal_name: str, diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index 3533af4a8c..e2282b532b 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -145,54 +145,19 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None - # 2) First automation pass, then poll until the child reports init_state_ok (ready to query). - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + # 2) Poll until process_bot_state.json exists (single readiness gate). + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid") initial_spawn_count = popen_calls["count"] assert initial_spawn_count >= 1 - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) - # 3) Wait until at least four open ladder orders exist, then assert a 2×2 grid pattern. orders_deadline = time.monotonic() + octobot_process_functional_shared.GRID_ORDERS_TIMEOUT_SEC exchange_account_snapshot: typing.Optional[ diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py index bb71fd39cd..5e027e1386 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py @@ -105,34 +105,13 @@ async def test_run_octobot_process_aggregates_two_simulator_exchanges( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - while time.monotonic() < deadline: - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_action_state = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_action_state is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_action_state) - assert inner is not None - if inner.get("init_state_ok"): - break - state = poll_job.dump() - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - else: - pytest.fail( - f"Timed out waiting for init_state_ok within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state ) + ) assert popen_calls["count"] >= 1 - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) orders_deadline = time.monotonic() + DUAL_EXCHANGE_ORDERS_TIMEOUT_SEC exchange_account_snapshot: typing.Optional[ diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py index 50c1a2203b..0271626eaa 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py @@ -106,61 +106,23 @@ async def test_run_octobot_process_lifecycle_grid_trading( await init_job.run() state = init_job.dump() - # 2) Register run_octobot_process; poll job until the child reports init_state_ok (live process_bot_state). + # 2) Register run_octobot_process; poll until process_bot_state.json exists. async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as job: job.automation_state.upsert_automation_actions( functionnal_tests.resolved_actions([run_action]) ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - # Run DSL job once, then optionally poll until recall payload shows init_state_ok. - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid"), "expected child pid in ensure state" assert popen_calls["count"] >= 1 - # --- process_bot_state path: must exist before poll (child wrote at least one dump) --- - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) - # 1) Poll AutomationJob + dump() until merge yields ≥4 open orders (EAE from automation snapshot, # not from parsing full process_bot_state on disk). orders_deadline = time.monotonic() + octobot_process_functional_shared.GRID_ORDERS_TIMEOUT_SEC @@ -405,43 +367,11 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid"), "expected child pid in ensure state" @@ -482,9 +412,6 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ) assert not local_non_trading_profile_json.exists() - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) with open(state_path, encoding="utf-8") as process_state_file: file_metadata_payload = json.load(process_state_file) process_metadata = process_bot_state_import.Metadata.from_dict( diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py b/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py new file mode 100644 index 0000000000..c96852cf27 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py @@ -0,0 +1,266 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock + +import octobot.community.authentication as community_authentication +import octobot_commons.tests.test_config as test_config_module +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.exchange_manager as exchange_manager_module +import octobot_trading.exchanges.traders.trader_simulator as trader_simulator_module +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +import tests.repositories.exchange.trades_repository_test_util as trades_repository_test_util + + +TEST_WALLET_ID = "0xfunctionaltestwallet" +_TEST_WALLET_ID = TEST_WALLET_ID +_TEST_PRIVATE_KEY = "functional-test-private-key" +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) +_DEFAULT_SYMBOL = "BTC/USDT" + + +def build_portfolio_history_context( + *, + account_id: str = "functional-account-1", + symbols: list[str] | None = None, + is_simulated: bool = False, + exchange: str = "binanceus", +) -> portfolio_history_entities.PortfolioHistoryAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id=account_id, + name="Functional portfolio history account", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange=exchange, + sandboxed=False, + historical_trade_symbols=symbols or [_DEFAULT_SYMBOL], + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + api_key=account_id, + ) + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + ) + + +def sample_raw_trade( + *, + trade_id: str = "functional-trade-1", + symbol: str = _DEFAULT_SYMBOL, +) -> dict: + return { + "info": {}, + "id": trade_id, + "exchange_id": trade_id, + "exchange_trade_id": trade_id, + "timestamp": 1700000000.0, + "symbol": symbol, + "type": "limit", + "side": "buy", + "price": 30000.0, + "amount": 0.1, + "cost": 3000.0, + "status": "closed", + "fee": {"cost": 0.0, "currency": "USDT"}, + } + + +def sample_deposit( + *, + txid: str = "functional-deposit-1", + currency: str = "BTC", + amount: float = 1.0, +) -> dict: + return { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: txid, + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: currency, + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: amount, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1700001000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: ( + trading_enums.TransactionType.BLOCKCHAIN_DEPOSIT.value + ), + } + + +def sample_withdrawal( + *, + txid: str = "functional-withdrawal-1", + currency: str = "ETH", + amount: float = 0.5, +) -> dict: + return { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: txid, + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: currency, + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: amount, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1700002000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: ( + trading_enums.TransactionType.BLOCKCHAIN_WITHDRAWAL.value + ), + } + + +def sample_daily_candles( + *, + day_timestamp_ms: int = 86400000, + close_price: float = 40500.0, +) -> list[list[float]]: + return [ + [day_timestamp_ms, 40000.0, 41000.0, 39000.0, close_price, 100.0], + ] + + +async def _create_exchange_manager_with_trader() -> exchange_manager_module.ExchangeManager: + config = test_config_module.load_test_config() + exchange_manager = exchange_manager_module.ExchangeManager(config, "binanceus") + await exchange_manager.initialize(exchange_config_by_exchange=None) + trader = trader_simulator_module.TraderSimulator(config, exchange_manager) + await trader.initialize() + return exchange_manager + + +async def build_exchange_manager( + *, + raw_trades: list[dict] | None = None, + deposits: list[dict] | None = None, + withdrawals: list[dict] | None = None, + daily_candles: list[list[float]] | None = None, +) -> exchange_manager_module.ExchangeManager: + exchange_manager = await _create_exchange_manager_with_trader() + + configured_raw_trades = raw_trades or [] + configured_deposits = deposits or [] + configured_withdrawals = withdrawals or [] + configured_daily_candles = daily_candles or sample_daily_candles() + + async def get_my_recent_trades(symbol: str, limit=None): + return [ + raw_trade + for raw_trade in configured_raw_trades + if raw_trade.get(trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value) == symbol + ] + + exchange_manager.exchange.get_my_recent_trades = get_my_recent_trades + exchange_manager.exchange.get_deposits = mock.AsyncMock(return_value=configured_deposits) + exchange_manager.exchange.get_withdrawals = mock.AsyncMock(return_value=configured_withdrawals) + exchange_manager.exchange.get_symbol_prices = mock.AsyncMock(return_value=configured_daily_candles) + await trades_repository_test_util.ensure_trades_channel(exchange_manager) + return exchange_manager + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def seed_empty_account_trading( + provider: collection_providers.AccountTradingProvider, + wallet_id: str, + account_id: str, +) -> None: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + provider.save_state(wallet_id, account_id, trading_state) + + +def load_account_trading(wallet_id: str, account_id: str) -> protocol_models.AccountTrading: + trading_state = collection_providers.AccountTradingProvider.instance().load_state(wallet_id, account_id) + return trading_state.account_trading + + +async def load_daily_prices_from_root( + data_root, + exchange_name: str, + exchange_type: str, + sandboxed: bool, +) -> dict: + return await trading_api.load_daily_prices(exchange_name, exchange_type, sandboxed, data_root) + + +async def _empty_historical_ohlcv(*_args, **_kwargs): + if False: + yield [] + + +@contextlib.contextmanager +def portfolio_history_test_environment( + tmp_path, + *, + exchange_manager_by_account_id: dict[str, exchange_manager_module.ExchangeManager], + wallet_id: str = _TEST_WALLET_ID, +): + trading_provider = collection_providers.AccountTradingProvider(base_folder=str(tmp_path)) + + @contextlib.asynccontextmanager + async def fake_exchange_manager(exchange_data, *_args, **_kwargs): + account_id = exchange_data.auth_details.api_key + exchange_manager = exchange_manager_by_account_id[account_id] + yield exchange_manager + + with ( + _patch_wallet(), + mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + mock.patch.object( + portfolio_history_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + portfolio_history_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + portfolio_history_job_module.trades_repository_module.TradesRepository, + "ensure_temporary_trades_channel", + trades_repository_test_util.ensure_trades_channel, + ), + mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _empty_historical_ohlcv, + ), + ): + yield trading_provider diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py b/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py new file mode 100644 index 0000000000..4919eb6f68 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py @@ -0,0 +1,147 @@ +# Drakkar-Software OctoBot-Flow + +import pytest + +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module +from tests.functionnal_tests.portfolio_history import portfolio_history_test_util as portfolio_history_test_util + + +@pytest.mark.asyncio +class TestPortfolioHistoryJobFunctional: + async def test_collects_and_persists_trading_history(self, tmp_path): + account_id = "functional-account-1" + context = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id, + symbols=["BTC/USDT"], + ) + exchange_manager = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[portfolio_history_test_util.sample_raw_trade()], + deposits=[portfolio_history_test_util.sample_deposit()], + withdrawals=[portfolio_history_test_util.sample_withdrawal()], + daily_candles=portfolio_history_test_util.sample_daily_candles(close_price=40500.0), + ) + + with portfolio_history_test_util.portfolio_history_test_environment( + tmp_path, + exchange_manager_by_account_id={account_id: exchange_manager}, + ) as trading_provider: + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id, + ) + results = await portfolio_history_job_module.PortfolioHistoryJob( + portfolio_history_test_util.TEST_WALLET_ID, + [context], + data_root=str(tmp_path), + ).run() + + assert len(results) == 1 + result = results[0] + assert result.account_id == account_id + assert result.trades_count == 1 + assert result.transactions_count == 2 + assert not result.skipped + assert result.error is None + + account_trading = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id, + ) + trade_ids = {trade.trade_id for trade in account_trading.trades or []} + transaction_ids = {transaction.id for transaction in account_trading.transactions or []} + assert "functional-trade-1" in trade_ids + assert "functional-deposit-1" in transaction_ids + assert "functional-withdrawal-1" in transaction_ids + + daily_prices = await portfolio_history_test_util.load_daily_prices_from_root( + str(tmp_path), + "binanceus", + "spot", + False, + ) + assert daily_prices["symbols"]["BTC/USDT"]["86400"] == 40500.0 + assert "ETH/USDT" in daily_prices["symbols"] + + async def test_two_accounts_both_persist(self, tmp_path): + account_id_1 = "functional-account-1" + account_id_2 = "functional-account-2" + context_1 = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id_1, + symbols=["BTC/USDT"], + ) + context_2 = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id_2, + symbols=["ETH/USDT"], + ) + exchange_manager_1 = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[ + portfolio_history_test_util.sample_raw_trade( + trade_id="account-1-trade", + symbol="BTC/USDT", + ) + ], + deposits=[portfolio_history_test_util.sample_deposit(txid="account-1-deposit")], + withdrawals=[portfolio_history_test_util.sample_withdrawal(txid="account-1-withdrawal")], + daily_candles=portfolio_history_test_util.sample_daily_candles(close_price=41000.0), + ) + exchange_manager_2 = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[ + portfolio_history_test_util.sample_raw_trade( + trade_id="account-2-trade", + symbol="ETH/USDT", + ) + ], + deposits=[portfolio_history_test_util.sample_deposit(txid="account-2-deposit", currency="ETH")], + withdrawals=[portfolio_history_test_util.sample_withdrawal(txid="account-2-withdrawal", currency="USDT")], + daily_candles=portfolio_history_test_util.sample_daily_candles( + day_timestamp_ms=172800000, + close_price=2500.0, + ), + ) + + with portfolio_history_test_util.portfolio_history_test_environment( + tmp_path, + exchange_manager_by_account_id={ + account_id_1: exchange_manager_1, + account_id_2: exchange_manager_2, + }, + ) as trading_provider: + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id_1, + ) + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id_2, + ) + results = await portfolio_history_job_module.PortfolioHistoryJob( + portfolio_history_test_util.TEST_WALLET_ID, + [context_1, context_2], + data_root=str(tmp_path), + ).run() + + assert len(results) == 2 + assert all(not result.skipped for result in results) + assert {result.account_id for result in results} == {account_id_1, account_id_2} + + account_trading_1 = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id_1, + ) + account_trading_2 = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id_2, + ) + assert {trade.trade_id for trade in account_trading_1.trades or []} == {"account-1-trade"} + assert {trade.trade_id for trade in account_trading_2.trades or []} == {"account-2-trade"} + assert {transaction.id for transaction in account_trading_1.transactions or []} == { + "account-1-deposit", + "account-1-withdrawal", + } + assert {transaction.id for transaction in account_trading_2.transactions or []} == { + "account-2-deposit", + "account-2-withdrawal", + } diff --git a/packages/flow/tests/jobs/test_portfolio_history_job.py b/packages/flow/tests/jobs/test_portfolio_history_job.py new file mode 100644 index 0000000000..5fbdbd1329 --- /dev/null +++ b/packages/flow/tests/jobs/test_portfolio_history_job.py @@ -0,0 +1,187 @@ +import asyncio +import mock +import pytest + +import octobot_protocol.models as protocol_models + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module + + +def _make_account(account_id: str, is_simulated: bool = False, is_exchange: bool = True): + account = mock.MagicMock() + account.id = account_id + account.is_simulated = is_simulated + if is_exchange: + account.specifics.actual_instance = mock.MagicMock(spec=protocol_models.ExchangeAccount) + else: + account.specifics.actual_instance = mock.MagicMock(spec=protocol_models.GenericAccount) + return account + + +def _make_context(account, exchange_config=None): + if exchange_config is None: + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "binance" + exchange_config.sandboxed = False + exchange_config.historical_trade_symbols = ["BTC/USDT"] + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=account.specifics.actual_instance, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=mock.MagicMock(), + ) + + +class TestRunForAccountSkipsUnsupported: + @pytest.mark.asyncio + async def test_skips_generic_account(self): + account = _make_account("acc1", is_exchange=False) + context = _make_context(account) + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + assert results[0].skipped is True + assert results[0].exchange_name == "binance" + + @pytest.mark.asyncio + async def test_skips_simulated_account(self): + account = _make_account("acc1", is_simulated=True) + context = _make_context(account) + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + assert results[0].skipped is True + assert results[0].exchange_name == "binance" + + +class TestRunForAccountDoesNotPersistHistory: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_account_history_provider_not_called( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + ): + account = _make_account("acc1") + context = _make_context(account) + + mock_exchange_manager = mock.AsyncMock() + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert not results[0].skipped + assert results[0].exchange_name == "binance" + assert results[0].trading_type == protocol_models.TradingType.SPOT.value + assert results[0].duration_seconds is not None + assert results[0].duration_seconds >= 0 + assert results[0].price_symbols_count >= 0 + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel.assert_awaited_once_with( + mock_exchange_manager, + ) + # merge_and_persist_trading_history should be called, but not AccountHistoryProvider + mock_merge.merge_and_persist_trading_history.assert_called_once() + + +class TestRunParallelExchangeAccounts: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_multiple_accounts_run_in_parallel( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + ): + account1 = _make_account("acc1") + account2 = _make_account("acc2") + context1 = _make_context(account1) + context2 = _make_context(account2) + + mock_exchange_manager = mock.AsyncMock() + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context1, context2]) + results = await job.run() + + assert len(results) == 2 + assert mock_merge.merge_and_persist_trading_history.call_count == 2 + + +class TestDerivePriceSymbols: + @mock.patch( + "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_skips_reference_market_currency_from_transactions(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "USDT"}], + "USDT", + ) + assert symbols == [] + + @mock.patch( + "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_adds_transaction_currency_against_reference_market(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "BTC"}], + "USDT", + ) + assert symbols == ["BTC/USDT"] + + @mock.patch( + "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_filters_invalid_same_base_quote_symbols(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + ["BTC/BTC", "ETH/USDT"], + [], + "USDT", + ) + assert symbols == ["ETH/USDT"] + + @mock.patch( + "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_skips_usd_like_stablecoin_from_transactions(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "USDC"}], + "USDT", + ) + assert symbols == [] diff --git a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py index 0b32f94f2d..908c4ef21f 100644 --- a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py +++ b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py @@ -5,6 +5,8 @@ import octobot_trading.enums import octobot_trading.errors +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.logic.dsl.dsl_action_execution_context @@ -243,3 +245,27 @@ async def execute_action(self, action, **_kwargs): assert action.error_status == expected_error_status.value assert action.error_message == str(raised_exception) + + +class TestDslActionExecutionReraisesOutdatedReferenceAccountError: + @pytest.mark.asyncio + async def test_reraises_outdated_reference_account_error(self): + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise outdated_error + + action = octobot_flow.entities.DSLScriptActionDetails( + id="copy_1", + dsl_script="copy_exchange_account()", + ) + stub_executor = StubExecutor() + + with pytest.raises(copy_errors.OutdatedReferenceAccountError) as raised_error: + await stub_executor.execute_action(action) + + assert str(raised_error.value) == str(outdated_error) + assert action.error_status is None + assert action.error_message is None diff --git a/packages/flow/tests/logic/global_view/test_account_refresh_builder.py b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py index 5dc10e7046..506ea3b61a 100644 --- a/packages/flow/tests/logic/global_view/test_account_refresh_builder.py +++ b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py @@ -2,15 +2,11 @@ import datetime -import mock - import octobot_protocol.models as protocol_models -import octobot_sync.constants as sync_constants import octobot_trading.exchanges.util.exchange_data as exchange_data_module import octobot_trading.enums as trading_enums import octobot_flow.entities -import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module @@ -39,6 +35,7 @@ def _context() -> octobot_flow.entities.GlobalViewAccountContext: name="binance-main", exchange="binanceus", sandboxed=False, + historical_trade_symbols=["BTC/USDT"], ), trading_type=protocol_models.TradingType.SPOT, auth_details=exchange_data_module.ExchangeAuthDetails( @@ -50,44 +47,22 @@ def _context() -> octobot_flow.entities.GlobalViewAccountContext: class TestBuildGlobalViewAccountRefreshResult: - def test_builds_refresh_result_with_history_state(self): + def test_builds_refresh_result_without_history(self): context = _context() exchange_refresh_result = octobot_flow.entities.ExchangeAccountRefreshResult( assets=[], - portfolio_snapshot=protocol_models.PortfolioHistoricalValue( - timestamp=_TEST_TIMESTAMP, - total=1000.0, - ), + ticker_closes={"BTC/USDT": 65000.0}, valuation_unit="USDC", open_orders=[], trades=[], positions=[], changed_order_ids={"gone-order"}, ) - expected_history_state = protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit="USDC", - values=[exchange_refresh_result.portfolio_snapshot], - ), - ) - with mock.patch.object( - account_state_persistence_module, - "build_portfolio_history_state", - return_value=expected_history_state, - ) as build_history_mock: - refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( - "wallet-1", - context, - exchange_refresh_result, - ) - build_history_mock.assert_called_once_with( + refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( "wallet-1", - "account-1", - exchange_refresh_result.portfolio_snapshot, - "USDC", - _TEST_TIMESTAMP, + context, + exchange_refresh_result, ) assert refresh_result.updated_account.id == "account-1" assert refresh_result.changed_order_ids == {"gone-order"} - assert refresh_result.portfolio_history_state == expected_history_state + assert not hasattr(refresh_result, "portfolio_history_state") or refresh_result.__dict__.get("portfolio_history_state") is None diff --git a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py index 5c777a4150..67bde1e0fc 100644 --- a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py +++ b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py @@ -94,6 +94,11 @@ async def test_real_refresh_applies_balance_to_portfolio_manager(self): "_refresh_portfolio_valuation", refresh_valuation_mock, ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), ): refresh_result = await exchange_account_refresh_module.refresh_exchange_account( exchange_manager, @@ -142,7 +147,7 @@ async def test_real_refresh_without_handle_portfolio_update_would_fail(self): ) assert refresh_result.assets == [] - assert refresh_result.portfolio_snapshot.total == 0.0 + assert refresh_result.ticker_closes == {} @pytest.mark.asyncio async def test_simulated_refresh_uses_seeded_account_assets(self): @@ -247,6 +252,11 @@ async def test_real_refresh_portfolio_total_positive_with_ticker_prices(self): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), ): refresh_result = await exchange_account_refresh_module.refresh_exchange_account( exchange_manager, @@ -255,7 +265,7 @@ async def test_real_refresh_portfolio_total_positive_with_ticker_prices(self): fetch_open_orders=False, ) - assert refresh_result.portfolio_snapshot.total > 0 + assert isinstance(refresh_result.ticker_closes, dict) class TestValuationSymbolsFromPortfolio: @@ -443,6 +453,11 @@ async def test_logs_fetched_portfolio_once_at_info(self): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( exchange_account_refresh_module, "_get_logger", diff --git a/packages/flow/tests/logic/global_view/test_global_view_account_job.py b/packages/flow/tests/logic/global_view/test_global_view_account_job.py index 49828e2608..45e471f03a 100644 --- a/packages/flow/tests/logic/global_view/test_global_view_account_job.py +++ b/packages/flow/tests/logic/global_view/test_global_view_account_job.py @@ -129,6 +129,11 @@ async def fake_exchange_manager(*_args, **_kwargs): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -144,11 +149,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_orders", return_value=[], ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", @@ -167,11 +167,9 @@ async def fake_exchange_manager(*_args, **_kwargs): assert isinstance(refresh_result, octobot_flow.entities.GlobalViewAccountRefreshResult) assert refresh_result.updated_account.id == "account-1" assert refresh_result.changed_order_ids == set() - assert refresh_result.portfolio_history_state is not None assert refresh_result.open_orders == [] assert refresh_result.updated_account.assets is not None assert refresh_result.updated_account.assets[0].assets - assert refresh_result.portfolio_history_state.history.values[-1].total > 0 async def test_run_detects_disappeared_orders(self): context = _exchange_account_context(has_bound_automation=True) @@ -210,6 +208,11 @@ async def fake_exchange_manager(*_args, **_kwargs): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -225,11 +228,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_orders", return_value=previous_open_orders, ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", @@ -297,6 +295,11 @@ async def fake_exchange_manager(*_args, **_kwargs): "_refresh_portfolio_valuation", mock.AsyncMock(), ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), mock.patch.object( global_view_account_job_module.tentacles_manager_api, "get_full_tentacles_setup_config", @@ -312,11 +315,6 @@ async def fake_exchange_manager(*_args, **_kwargs): "load_previous_open_orders", return_value=previous_open_orders, ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", @@ -414,11 +412,6 @@ async def fake_exchange_manager(*_args, **_kwargs): }, }), ), - mock.patch.object( - account_state_persistence_module, - "load_portfolio_history_state", - return_value=_empty_portfolio_history_state(), - ), mock.patch.object( global_view_persistence_module, "persist_global_view_refresh_result", diff --git a/packages/flow/tests/logic/global_view/test_global_view_persistence.py b/packages/flow/tests/logic/global_view/test_global_view_persistence.py index e2aba03495..4b4259ac03 100644 --- a/packages/flow/tests/logic/global_view/test_global_view_persistence.py +++ b/packages/flow/tests/logic/global_view/test_global_view_persistence.py @@ -31,28 +31,18 @@ def _refresh_result(*, open_orders: list[dict] | None = None) -> octobot_flow.en updated_account=account, changed_order_ids=set(), open_orders=open_orders or [], - portfolio_history_state=protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues(unit="USDC", values=[]), - ), ) class TestPersistGlobalViewRefreshResult: def test_does_not_persist_trading_when_persist_open_orders_false(self): account_provider = mock.Mock() - history_provider = mock.Mock() with ( mock.patch.object( collection_providers.AccountProvider, "instance", return_value=account_provider, ), - mock.patch.object( - collection_providers.AccountHistoryProvider, - "instance", - return_value=history_provider, - ), mock.patch.object( account_state_persistence_module, "persist_account_trading_orders", @@ -65,12 +55,10 @@ def test_does_not_persist_trading_when_persist_open_orders_false(self): persist_open_orders=False, ) account_provider.update_item.assert_called_once() - history_provider.save_state.assert_called_once() persist_orders_mock.assert_not_called() def test_persists_orders_only_when_persist_open_orders_true(self): account_provider = mock.Mock() - history_provider = mock.Mock() open_orders = [ { trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", @@ -82,11 +70,6 @@ def test_persists_orders_only_when_persist_open_orders_true(self): "instance", return_value=account_provider, ), - mock.patch.object( - collection_providers.AccountHistoryProvider, - "instance", - return_value=history_provider, - ), mock.patch.object( account_state_persistence_module, "persist_account_trading_orders", diff --git a/packages/flow/tests/logic/portfolio_history/__init__.py b/packages/flow/tests/logic/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py new file mode 100644 index 0000000000..292108d695 --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py @@ -0,0 +1,410 @@ +import time +import mock +import pytest + +import octobot_trading.api as trading_api +import octobot_trading.errors as trading_errors + +import octobot_commons.constants as commons_constants +import octobot_flow.constants as flow_constants +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module + + +def _exchange_manager_with_symbols(symbols: list[str], exchange_name: str = "binance"): + exchange_manager = mock.AsyncMock() + exchange_manager.client_symbols = symbols + exchange_manager.exchange_name = exchange_name + return exchange_manager + + +async def _empty_historical_ohlcv(*_args, **_kwargs): + if False: + yield [] + + +def _patch_historical_ohlcv(candles=None, error=None): + async def _historical_ohlcv(*_args, **_kwargs): + if error is not None: + raise error + if candles: + yield candles + + return mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _historical_ohlcv, + ) + + +@pytest.fixture(autouse=True) +def mock_empty_historical_ohlcv(): + with mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _empty_historical_ohlcv, + ): + yield + + +class TestComputeFetchTimeRangeMs: + def test_incremental_uses_since_ms_as_start(self): + since_ms = 1_700_000_000_000 + start_time_ms, end_time_ms = daily_price_cache_updater_module._compute_fetch_time_range_ms(since_ms) + assert start_time_ms == since_ms + assert end_time_ms > since_ms + + def test_empty_cache_uses_lookback_start(self): + start_time_ms, end_time_ms = daily_price_cache_updater_module._compute_fetch_time_range_ms(None) + lookback_ms = ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS + * commons_constants.DAYS_TO_SECONDS + * commons_constants.MSECONDS_TO_SECONDS + ) + assert end_time_ms - start_time_ms == lookback_ms + + +class TestComputeFetchSinceMs: + def test_empty_cache_returns_none(self): + daily_prices = {"symbols": {}, "sources": {}} + assert daily_price_cache_updater_module._compute_fetch_since_ms( + daily_prices, "BTC/USDT", + ) is None + + def test_uses_newest_minus_one_day(self): + daily_prices = { + "symbols": {"BTC/USDT": {"1000": 42000.0, "2000": 43000.0}}, + "sources": {}, + } + since_ms = daily_price_cache_updater_module._compute_fetch_since_ms( + daily_prices, "BTC/USDT", + ) + expected_since_ms = int((2000 - commons_constants.DAYS_TO_SECONDS) * 1000) + assert since_ms == expected_since_ms + + +class TestIsDailyCacheUpToDate: + def test_cache_with_today_is_up_to_date(self): + today_start = daily_price_cache_updater_module._utc_day_start(time.time()) + daily_prices = { + "symbols": {"BTC/USDT": {str(int(today_start)): 42000.0}}, + "sources": {}, + } + assert daily_price_cache_updater_module._is_daily_cache_up_to_date(daily_prices, "BTC/USDT") is True + + def test_cache_with_yesterday_is_stale(self): + yesterday_start = daily_price_cache_updater_module._utc_day_start(time.time()) - 86400 + daily_prices = { + "symbols": {"BTC/USDT": {str(int(yesterday_start)): 42000.0}}, + "sources": {}, + } + assert daily_price_cache_updater_module._is_daily_cache_up_to_date(daily_prices, "BTC/USDT") is False + + +class TestUpdateDailyPrices: + @pytest.mark.asyncio + async def test_fetches_and_merges_closes(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + [172800000, 40500, 42000, 40000, 41000, 200], + ] + + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result["symbols"]["BTC/USDT"]["86400"] == 40500 + assert result["symbols"]["BTC/USDT"]["172800"] == 41000 + assert result["sources"]["BTC"] == "BTC/USDT" + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + + @pytest.mark.asyncio + async def test_empty_cache_fetches_without_since(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + + @pytest.mark.asyncio + async def test_empty_candles_skips(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [] + + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result == {"symbols": {}, "sources": {}} + + @pytest.mark.asyncio + async def test_skips_fetch_when_cache_has_today(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + data_root = str(tmp_path) + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + await trading_api.merge_daily_prices( + "binance", "spot", False, "BTC/USDT", {str(today_start): 42000.0}, data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_not_called() + + @pytest.mark.asyncio + async def test_still_fetches_when_cache_stale(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + ] + data_root = str(tmp_path) + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "binance", "spot", False, "BTC/USDT", {str(yesterday_start): 41000.0}, data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + expected_since_ms = int((yesterday_start - commons_constants.DAYS_TO_SECONDS) * 1000) + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] == expected_since_ms + + @pytest.mark.asyncio + async def test_stale_cache_with_long_history_uses_newest_not_oldest(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + ] + data_root = str(tmp_path) + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "binance", + "spot", + False, + "BTC/USDT", + {str(1000): 41000.0, str(yesterday_start): 42000.0}, + data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + expected_since_ms = int((yesterday_start - commons_constants.DAYS_TO_SECONDS) * 1000) + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] == expected_since_ms + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] != 1000 * 1000 + + @pytest.mark.asyncio + async def test_skips_usd_like_stablecoin_symbol(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["USDC/USDT"]) + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["USDC/USDT"], data_root, + ) + exchange_manager.exchange.get_symbol_prices.assert_not_called() + + @pytest.mark.asyncio + async def test_continues_on_failed_request(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT", "ETH/USDT"]) + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.FailedRequest("bingx range error"), + [[86400000, 40000, 41000, 39000, 40500, 100]], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + "binance", + "spot", + False, + ["BTC/USDT", "ETH/USDT"], + data_root, + ) + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result["symbols"]["ETH/USDT"]["86400"] == 40500 + assert "BTC/USDT" not in result["symbols"] + + @pytest.mark.asyncio + async def test_continues_when_symbol_is_unsupported(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + "binance", + "spot", + False, + ["USDT/USDT", "BTC/USDT"], + data_root, + ) + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result["symbols"]["BTC/USDT"]["86400"] == 40500 + assert "USDT/USDT" not in result["symbols"] + + @pytest.mark.asyncio + async def test_falls_back_to_usd_like_quote(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USD"], "kraken") + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 1.0, 1.1, 0.9, 1.05, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + result = await trading_api.load_daily_prices("kraken", "spot", False, data_root) + assert result["symbols"]["KNC/USD"]["86400"] == 1.05 + assert result["sources"]["KNC"] == "KNC/USD" + assert trading_api.get_daily_price(result, "KNC/USDT", "86400") == 1.05 + + @pytest.mark.asyncio + async def test_sticky_fetch_symbol_is_tried_first(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USD"], "kraken") + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 1.0, 1.1, 0.9, 1.05, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + exchange_manager.client_symbols = ["KNC/USD", "KNC/USDT"] + exchange_manager.exchange.get_symbol_prices.reset_mock() + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "kraken", "spot", False, "KNC/USD", {str(yesterday_start): 1.0}, data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + assert exchange_manager.exchange.get_symbol_prices.call_args[0][0] == "KNC/USD" + + @pytest.mark.asyncio + async def test_migrates_rows_when_sticky_pair_is_delisted(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USDC"], "kraken") + data_root = str(tmp_path) + await trading_api.merge_daily_prices( + "kraken", "spot", False, "KNC/USD", {"86400": 1.0, "172800": 1.1}, data_root, + ) + await trading_api.set_daily_close_source( + "kraken", "spot", False, "KNC", "KNC/USD", data_root, + ) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [259200000, 1.0, 1.1, 0.9, 1.2, 100], + ] + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.UnSupportedSymbolError("delisted"), + [[259200000, 1.0, 1.1, 0.9, 1.2, 100]], + ] + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("kraken", "spot", False, data_root) + assert "KNC/USD" not in result["symbols"] + assert result["symbols"]["KNC/USDC"]["86400"] == 1.0 + assert result["symbols"]["KNC/USDC"]["172800"] == 1.1 + assert result["symbols"]["KNC/USDC"]["259200"] == 1.2 + assert result["sources"]["KNC"] == "KNC/USDC" + + @pytest.mark.asyncio + async def test_limited_history_retries_without_since_after_failed_request(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["SOL/USDT"], "bingx") + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.FailedRequest("bingx range error"), + [[86400000, 40000, 41000, 39000, 40500, 100]], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "bingx", "spot", False, ["SOL/USDT"], data_root, + ) + + assert exchange_manager.exchange.get_symbol_prices.call_count == 2 + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args_list[1][1] + result = await trading_api.load_daily_prices("bingx", "spot", False, data_root) + assert result["symbols"]["SOL/USDT"]["86400"] == 40500 + + @pytest.mark.asyncio + async def test_limited_history_logs_error_when_both_attempts_fail(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["SOL/USDT"], "bingx") + exchange_manager.exchange.get_symbol_prices.side_effect = trading_errors.FailedRequest( + "bingx range error", + ) + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "bingx", "spot", False, ["SOL/USDT"], data_root, + ) + + assert exchange_manager.exchange.get_symbol_prices.call_count == 2 + result = await trading_api.load_daily_prices("bingx", "spot", False, data_root) + assert result == {"symbols": {}, "sources": {}} + + @pytest.mark.asyncio + async def test_full_history_uses_historical_ohlcv_first(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + candles = [[86400000, 40000, 41000, 39000, 40500, 100]] + data_root = str(tmp_path) + with _patch_historical_ohlcv(candles=candles): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_not_called() + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result["symbols"]["BTC/USDT"]["86400"] == 40500 + + @pytest.mark.asyncio + async def test_full_history_falls_back_to_get_symbol_prices(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [86400000, 40000, 41000, 39000, 40500, 100], + ] + data_root = str(tmp_path) + with _patch_historical_ohlcv(error=trading_errors.FailedRequest("historical error")): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result["symbols"]["BTC/USDT"]["86400"] == 40500 + + @pytest.mark.asyncio + async def test_full_history_does_not_use_limited_history_fallback(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.side_effect = trading_errors.FailedRequest( + "binance range error", + ) + data_root = str(tmp_path) + with _patch_historical_ohlcv(error=trading_errors.FailedRequest("historical error")): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result == {"symbols": {}, "sources": {}} diff --git a/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py b/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py new file mode 100644 index 0000000000..bc3dce091a --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py @@ -0,0 +1,76 @@ +import decimal + +import pytest + +import octobot_flow.logic.portfolio_history.portfolio_value_history as portfolio_value_history_module + + +def _portfolio(assets: dict[str, float]) -> dict[str, dict[str, decimal.Decimal]]: + return { + asset: {"total": decimal.Decimal(str(amount)), "available": decimal.Decimal(str(amount))} + for asset, amount in assets.items() + } + + +class TestComputeDailyPortfolioValues: + def test_single_day_with_daily_price(self): + daily_holdings = { + 86400.0: _portfolio({"BTC": 1.0, "USDT": 500.0}), + } + daily_prices = {"symbols": {"BTC/USDT": {"86400": 40000.0}}} + latest_tickers = {"closes": {}} + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, daily_prices, latest_tickers, + ) + assert len(result) == 1 + assert result[0]["timestamp"] == 86400.0 + assert result[0]["value"] == pytest.approx(40500.0) + + def test_fallback_to_latest_ticker(self): + daily_holdings = { + 86400.0: _portfolio({"ETH": 2.0}), + } + daily_prices = {"symbols": {}} + latest_tickers = {"closes": {"ETH/USDT": 3000.0}} + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, daily_prices, latest_tickers, + ) + assert result[0]["value"] == pytest.approx(6000.0) + + def test_reference_market_asset_counted_directly(self): + daily_holdings = { + 0.0: _portfolio({"USDT": 1000.0}), + } + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, {"symbols": {}}, {"closes": {}}, + ) + assert result[0]["value"] == pytest.approx(1000.0) + + def test_usd_like_stablecoin_valued_at_face_value(self): + daily_holdings = { + 0.0: _portfolio({"USDC": 250.0, "USDT": 500.0}), + } + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, {"symbols": {}}, {"closes": {}}, + ) + assert result[0]["value"] == pytest.approx(750.0) + + def test_missing_price_skips_asset(self): + daily_holdings = { + 0.0: _portfolio({"UNKNOWN": 100.0, "USDT": 500.0}), + } + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, {"symbols": {}}, {"closes": {}}, + ) + assert result[0]["value"] == pytest.approx(500.0) + + def test_sorted_ascending(self): + daily_holdings = { + 172800.0: _portfolio({"USDT": 200.0}), + 86400.0: _portfolio({"USDT": 100.0}), + } + result = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, {"symbols": {}}, {"closes": {}}, + ) + assert result[0]["timestamp"] == 86400.0 + assert result[1]["timestamp"] == 172800.0 diff --git a/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py b/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py new file mode 100644 index 0000000000..5d1a52c81f --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py @@ -0,0 +1,79 @@ +import datetime +import mock +import pytest + +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.portfolio_history.trading_history_merge as trading_history_merge_module + + +class TestMergeAndPersistTradingHistory: + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_merges_new_trades_deduped(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trade = mock.MagicMock() + existing_trade.trades = [] + existing_trade.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trade + mock_provider.load_state.return_value = mock_state + + new_trades = [ + { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "t1", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value: "t1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: "limit", + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: "buy", + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 1.0, + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 30000.0, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: "filled", + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: 1700000000.0, + } + ] + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", new_trades, [], + ) + mock_provider.save_state.assert_called_once() + + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_merges_new_transactions_deduped(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trading = mock.MagicMock() + existing_trading.trades = [] + existing_trading.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trading + mock_provider.load_state.return_value = mock_state + + new_txs = [ + { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: "tx1", + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: "BTC", + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: 1.0, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1000000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: "blockchain_deposit", + } + ] + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", [], new_txs, + ) + mock_provider.save_state.assert_called_once() + + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_empty_inputs_still_saves(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trading = mock.MagicMock() + existing_trading.trades = [] + existing_trading.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trading + mock_provider.load_state.return_value = mock_state + + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", [], [], + ) + mock_provider.save_state.assert_called_once() diff --git a/packages/flow/tests/repositories/community/test_trading_signals_channel.py b/packages/flow/tests/repositories/community/test_trading_signals_channel.py index 85cfd500d4..f803d55a28 100644 --- a/packages/flow/tests/repositories/community/test_trading_signals_channel.py +++ b/packages/flow/tests/repositories/community/test_trading_signals_channel.py @@ -82,4 +82,3 @@ async def test_shutdown_internal_trading_signal_channel_allows_recreate(internal new_channel = await trading_signals_channel.get_or_create_internal_trading_signal_channel() assert new_channel is not None - diff --git a/packages/flow/tests/repositories/exchange/__init__.py b/packages/flow/tests/repositories/exchange/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/repositories/exchange/test_trades_repository.py b/packages/flow/tests/repositories/exchange/test_trades_repository.py new file mode 100644 index 0000000000..e144bfc688 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_trades_repository.py @@ -0,0 +1,62 @@ +import mock +import pytest + +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module +import octobot_trading.personal_data as trading_personal_data + + +def _make_repo(exchange_manager): + fetched_data = mock.MagicMock() + return trades_repository_module.TradesRepository(exchange_manager, [], fetched_data) + + +class TestTradesRepositoryFetchTrades: + @pytest.mark.asyncio + async def test_uses_updater_fetch_and_ensure_parsing(self): + exchange_manager = mock.MagicMock() + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [{"id": "raw-trade-1"}] + with ( + mock.patch.object(repo, "get_channel_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + return_value={"id": "parsed-trade-1"}, + ) as ensure_parsing_mock, + ): + result = await repo.fetch_trades(["BTC/USDT"]) + + updater.fetch_trades.assert_awaited_once_with(["BTC/USDT"]) + ensure_parsing_mock.assert_called_once_with(exchange_manager, {"id": "raw-trade-1"}) + assert result == [{"id": "parsed-trade-1"}] + + @pytest.mark.asyncio + async def test_empty_symbols_returns_empty_list(self): + repo = _make_repo(mock.MagicMock()) + assert await repo.fetch_trades([]) == [] + + +class TestTradesRepositoryEnsureTemporaryTradesChannel: + @pytest.mark.asyncio + async def test_creates_channels_and_trades_producer_only(self): + exchange_manager = mock.Mock() + with ( + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await trades_repository_module.TradesRepository.ensure_temporary_trades_channel(exchange_manager) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) + \ No newline at end of file diff --git a/packages/flow/tests/repositories/exchange/test_transactions_repository.py b/packages/flow/tests/repositories/exchange/test_transactions_repository.py new file mode 100644 index 0000000000..37852c1f2e --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_transactions_repository.py @@ -0,0 +1,67 @@ +import mock +import pytest + +import octobot_trading.errors as trading_errors + +import octobot_flow.repositories.exchange.transactions_repository as transactions_repository_module + + +def _make_repo(exchange_manager): + fetched_data = mock.MagicMock() + return transactions_repository_module.TransactionsRepository(exchange_manager, [], fetched_data) + + +class TestFetchDeposits: + @pytest.mark.asyncio + async def test_returns_deposits_when_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_deposits.return_value = [{"id": "d1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [{"id": "d1"}] + + @pytest.mark.asyncio + async def test_returns_empty_when_not_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "binance" + exchange_manager.exchange.get_deposits.side_effect = trading_errors.NotSupported("nope") + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [] + + @pytest.mark.asyncio + async def test_returns_empty_when_authentication_fails(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "kraken" + exchange_manager.exchange.get_deposits.side_effect = trading_errors.AuthenticationError("denied") + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [] + + +class TestFetchWithdrawals: + @pytest.mark.asyncio + async def test_returns_withdrawals_when_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_withdrawals.return_value = [{"id": "w1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [{"id": "w1"}] + + @pytest.mark.asyncio + async def test_returns_empty_when_not_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "binance" + exchange_manager.exchange.get_withdrawals.side_effect = trading_errors.NotSupported("nope") + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [] + + @pytest.mark.asyncio + async def test_returns_empty_when_authentication_fails(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "kraken" + exchange_manager.exchange.get_withdrawals.side_effect = trading_errors.AuthenticationError("denied") + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [] diff --git a/packages/flow/tests/repositories/exchange/trades_repository_test_util.py b/packages/flow/tests/repositories/exchange/trades_repository_test_util.py new file mode 100644 index 0000000000..8b1e5251bf --- /dev/null +++ b/packages/flow/tests/repositories/exchange/trades_repository_test_util.py @@ -0,0 +1,22 @@ +import octobot_trading.api as trading_api_module +import octobot_trading.constants as trading_constants +import octobot_trading.exchange_channel as exchange_channel_module +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.personal_data as trading_personal_data + + +async def ensure_trades_channel(exchange_manager) -> None: + try: + trading_api_module.get_channel_updater(exchange_manager, trading_constants.TRADES_CHANNEL) + return + except KeyError: + pass + try: + exchange_channel_module.get_chan(trading_constants.TRADES_CHANNEL, exchange_manager.id) + except KeyError: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index 01d59aedce..77ec7ffa20 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -39,3 +39,4 @@ class SchedulerQueues(enum.Enum): USER_ACTION_QUEUE = "user_action_queue" DBOS_CLEANUP_QUEUE = "dbos_cleanup_queue" GLOBAL_VIEW_QUEUE = "global_view_queue" + PORTFOLIO_HISTORY_QUEUE = "portfolio_history_queue" diff --git a/packages/node/octobot_node/protocol/accounts_history.py b/packages/node/octobot_node/protocol/accounts_history.py index 9262ff0e54..e60d4ea350 100644 --- a/packages/node/octobot_node/protocol/accounts_history.py +++ b/packages/node/octobot_node/protocol/accounts_history.py @@ -15,57 +15,151 @@ # with OctoBot. If not, see . import datetime +import decimal +import octobot_commons.constants as commons_constants import octobot_protocol.models as protocol_models import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors -import octobot_sync.sync.collection_providers.user_account_history_provider as history_provider +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping -import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module +import octobot_flow.logic.portfolio_history.portfolio_value_history as portfolio_value_history_module def get_portfolio_history_state( user_id: str, account_id: str, ) -> protocol_models.PortfolioHistoricalValuesState: - try: - return history_provider.AccountHistoryProvider.instance().load_state(user_id, account_id) - except collection_errors.CollectionNoDataError: - return protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=None, - ) + """Compute portfolio history on-the-fly from persisted data.""" + # Placeholder: the async compute must be awaited from an async context. + # This synchronous wrapper returns an empty state; callers should use the async version. + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) -def save_portfolio_evaluation( +async def compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( user_id: str, account_id: str, - snapshot: protocol_models.PortfolioHistoricalValue, - valuation_unit: str, - evaluation_time: datetime.datetime, + *, + data_root: str = None, ) -> protocol_models.PortfolioHistoricalValuesState: - history_state = get_portfolio_history_state(user_id, account_id) - existing_values = ( - history_state.history.values - if history_state.history is not None and history_state.history.values - else [] + """ + Compute portfolio value history on-the-fly from Account.assets, + AccountTrading trades/transactions, and persisted price caches. + """ + # Step 1: Load account and its assets as the latest portfolio anchor. + try: + account = collection_providers.AccountProvider.instance().get_account(user_id, account_id) + except (collection_errors.CollectionNoDataError, collection_errors.ItemNotFoundError): + return _empty_state() + + portfolio = _portfolio_from_account_assets(account) + if not portfolio: + return _empty_state() + + # Step 2: Load AccountTrading trades and transactions. + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + return _empty_state() + + account_trading = trading_state.account_trading + + # Step 3: Resolve exchange info from the account's exchange config. + exchange_info = _resolve_exchange_info(user_id, account) + if exchange_info is None: + return _empty_state() + exchange_name, exchange_type, sandboxed = exchange_info + + # Step 4: Load persisted caches. + daily_prices = await trading_api.load_daily_prices(exchange_name, exchange_type, sandboxed, data_root) + latest_tickers = await trading_api.load_latest_tickers(exchange_name, exchange_type, sandboxed, data_root) + + # Step 5: Compute historical holdings via reverse replay. + daily_holdings = trading_api.compute_portfolio_historical_holdings_from_latest_portfolio_trades_and_transations( + portfolio, + account_trading.trades or [], + account_trading.transactions or [], ) - merged_values = portfolio_history_module.merge_snapshot( - existing_values, - snapshot, - evaluation_time, + + # Step 6: Daily valuation. + valuation_unit = "USDT" + valued_days = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, daily_prices, latest_tickers, reference_market=valuation_unit, ) - updated_history = protocol_models.PortfolioHistoricalValues( - unit=valuation_unit, - values=merged_values, + + # Step 7: Build and return result (not saved). + history_values = [ + protocol_models.PortfolioHistoricalValue( + timestamp=_timestamp_to_datetime(day["timestamp"]), + total=day["value"], + assets=[], + ) + for day in valued_days + ] + if not history_values: + return _empty_state() + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=history_values, + ), ) - updated_state = protocol_models.PortfolioHistoricalValuesState( + + +def _empty_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=updated_history, + history=None, + ) + + +def _portfolio_from_account_assets(account: protocol_models.Account) -> dict[str, dict[str, decimal.Decimal]]: + """Convert Account.assets to the portfolio dict format used by the builder.""" + portfolio: dict[str, dict[str, decimal.Decimal]] = {} + if not account.assets: + return portfolio + for assets_for_type in account.assets: + for asset in (assets_for_type.assets or []): + portfolio[asset.symbol] = { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal(str(asset.total)), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal(str(asset.available or asset.total)), + } + return portfolio + + +def _resolve_exchange_info( + user_id: str, account: protocol_models.Account +) -> tuple[str, str, bool] | None: + """Derive exchange identity from the account's first exchange config.""" + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return None + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return None + config_ids = exchange_account.exchange_config_ids or [] + if not config_ids: + return None + try: + exchange_config = collection_providers.AccountProvider.instance().get_exchange_config( + user_id, config_ids[0] + ) + except collection_errors.CollectionNoDataError: + return None + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + protocol_models.TradingType.SPOT ) - history_provider.AccountHistoryProvider.instance().save_state( - user_id, - account_id, - updated_state, + return ( + exchange_config.exchange, + exchange_type.value if exchange_type else "spot", + exchange_config.sandboxed, ) - return updated_state + +def _timestamp_to_datetime(timestamp: float) -> datetime.datetime: + return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) \ No newline at end of file diff --git a/packages/node/octobot_node/scheduler/api.py b/packages/node/octobot_node/scheduler/api.py index d2caf6211f..4bb71e96eb 100644 --- a/packages/node/octobot_node/scheduler/api.py +++ b/packages/node/octobot_node/scheduler/api.py @@ -28,7 +28,7 @@ import octobot_node.scheduler import octobot_node.scheduler.workflows_util as workflows_util -logger = octobot_commons_logging.get_logger("octobot_node.scheduler.api") +logger = octobot_commons_logging.get_logger("scheduler_api") def get_node_status() -> dict[str, str | int | None | uuid.UUID]: diff --git a/packages/node/octobot_node/scheduler/portfolio_history/__init__.py b/packages/node/octobot_node/scheduler/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/node/octobot_node/scheduler/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py new file mode 100644 index 0000000000..1e4b3d0938 --- /dev/null +++ b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py @@ -0,0 +1,120 @@ +import octobot_commons.logging as commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module + +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver + +logger = commons_logging.get_logger("PortfolioHistoryExecutor") + + +async def run_portfolio_history_collection( + wallet_id: str, + account_ids: list[str] | None = None, +) -> list[portfolio_history_entities.PortfolioHistoryRunResult]: + """ + Build contexts for all exchange accounts of the given wallet and run + the PortfolioHistoryJob with parallel per-exchange fetching. + """ + account_provider = collection_providers.AccountProvider.instance() + try: + accounts = account_provider.list_items(wallet_id) + except wallet_backend_errors.WalletNotFoundError as error: + logger.warning( + "Skipping portfolio history collection for wallet %s: cannot list accounts (%s)", + wallet_id, + error, + ) + return [] + + if account_ids is not None: + account_ids_set = set(account_ids) + accounts = [account for account in accounts if account.id in account_ids_set] + + contexts = [ + context + for account in accounts + if (context := _build_context_for_account(wallet_id, account)) + ] + + if not contexts: + logger.info("No exchange accounts found for wallet %s", wallet_id) + return [] + + job = portfolio_history_job_module.PortfolioHistoryJob(wallet_id, contexts) + results = await job.run() + for result in results: + if result.error: + logger.error( + "Portfolio history collection failed for account %s: %s", + result.account_id, result.error, + ) + elif result.skipped: + logger.debug("Skipped account %s", result.account_id) + else: + logger.info( + "Collected %d trades + %d transactions for [%s %s %s] account %s " + "in %.2fs, %d fetched candles symbols", + result.trades_count, + result.transactions_count, + result.exchange_name, + "simulated" if result.is_simulated else "real", + result.trading_type or commons_constants.CONFIG_EXCHANGE_SPOT, + result.account_id, + result.duration_seconds or 0.0, + result.price_symbols_count, + ) + return results + + +def _build_context_for_account( + wallet_id: str, + account: protocol_models.Account, +) -> portfolio_history_entities.PortfolioHistoryAccountContext | None: + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return None + if not isinstance(specifics.actual_instance, protocol_models.ExchangeAccount): + return None + if account.is_simulated: + return None + + exchange_account = specifics.actual_instance + trading_type = account_state_updater._trading_type_for_account_state_check(account) + try: + exchange_config = exchange_account_resolver.get_exchange_config(wallet_id, exchange_account) + except Exception as error: + logger.exception( + error, + True, + f"Could not resolve exchange config for account {account.id}: {error}", + ) + return None + try: + authentication = account_authentication_resolver.get_exchange_authentication(wallet_id, account) + except Exception as error: + logger.exception( + error, + True, + f"Could not resolve authentication for account {account.id}: {error}", + ) + return None + auth_details = account_state_updater._encrypted_exchange_auth_details( + exchange_account, + authentication, + trading_type, + exchange_config.sandboxed, + ) + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=trading_type, + auth_details=auth_details, + ) diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index a29e83b8f8..e51fc50414 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -68,6 +68,7 @@ class Scheduler: USER_ACTION_QUEUE: dbos.Queue = None # type: ignore DBOS_CLEANUP_QUEUE: dbos.Queue = None # type: ignore GLOBAL_VIEW_QUEUE: dbos.Queue = None # type: ignore + PORTFOLIO_HISTORY_QUEUE: dbos.Queue = None # type: ignore @staticmethod def _wallet_filter_queue(queue_names: typing.Optional[list[str]]) -> octobot_node.enums.SchedulerQueues: @@ -153,6 +154,7 @@ def stop(self) -> None: Scheduler.USER_ACTION_QUEUE = None Scheduler.DBOS_CLEANUP_QUEUE = None Scheduler.GLOBAL_VIEW_QUEUE = None + Scheduler.PORTFOLIO_HISTORY_QUEUE = None def create_queues(self): self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) @@ -166,6 +168,10 @@ def create_queues(self): name=octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, concurrency=1, ) + self.PORTFOLIO_HISTORY_QUEUE = dbos.Queue( + name=octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, + concurrency=1, + ) async def get_periodic_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" diff --git a/packages/node/octobot_node/scheduler/schedules.py b/packages/node/octobot_node/scheduler/schedules.py index e0d3b9049d..485afcc1cf 100644 --- a/packages/node/octobot_node/scheduler/schedules.py +++ b/packages/node/octobot_node/scheduler/schedules.py @@ -26,6 +26,7 @@ import octobot_node.scheduler.scheduler as scheduler_module import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow +import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow import octobot_node.scheduler.workflows_retention as workflows_retention @@ -115,6 +116,20 @@ async def _classify_schedule_window_slots( } +def _get_latest_schedule_trigger_time_before( + schedule_input: dbos.ScheduleInput, + before: datetime.datetime, +) -> datetime.datetime | None: + schedule_cron = schedule_input["schedule"] + cron_timezone = schedule_input.get("cron_timezone") + tz = zoneinfo.ZoneInfo(cron_timezone) if cron_timezone else datetime.timezone.utc + if before.tzinfo is None: + before = before.replace(tzinfo=datetime.timezone.utc) + before_in_tz = before.astimezone(tz) + iterator = dbos_croniter.croniter(schedule_cron, before_in_tz, second_at_beginning=True) + return iterator.get_prev(_DATETIME_CLASS) + + def _existing_schedule_matches_configured( existing: dbos.WorkflowSchedule, schedule_input: dbos.ScheduleInput, @@ -123,11 +138,65 @@ def _existing_schedule_matches_configured( existing["schedule"] == schedule_input["schedule"] and bool(existing.get("automatic_backfill")) == schedule_input.get("automatic_backfill", False) + and bool(existing.get("catch_up_once_on_startup")) + == schedule_input.get("catch_up_once_on_startup", False) and existing.get("cron_timezone") == schedule_input.get("cron_timezone") and existing.get("queue_name") == schedule_input.get("queue_name") ) +async def _maybe_catch_up_schedule_once_on_startup( + schedule_name: str, + schedule_input: dbos.ScheduleInput, +) -> None: + if not schedule_input.get("catch_up_once_on_startup", False): + return + existing_schedule = await dbos.DBOS.get_schedule_async(schedule_name) + if existing_schedule is None: + return + + logger = _get_logger() + now = datetime.datetime.now(datetime.timezone.utc) + trigger_time = _get_latest_schedule_trigger_time_before(schedule_input, now) + if trigger_time is None: + return + + workflow_id = build_scheduled_workflow_id(schedule_name, trigger_time) + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + if workflow_status is not None: + if workflows_retention.is_terminal_workflow(workflow_status): + logger.info( + "Startup catch-up not needed for schedule %s: latest slot %s already %s", + schedule_name, + workflow_id, + _workflow_status_string(workflow_status), + ) + else: + logger.info( + "Startup catch-up skipped for schedule %s: latest slot %s already %s", + schedule_name, + workflow_id, + _workflow_status_string(workflow_status), + ) + return + + logger.info( + "Startup catch-up for schedule %s: enqueuing missed latest slot %s", + schedule_name, + workflow_id, + ) + # backfill_schedule uses get_next from start; start one second before the slot + # so the target trigger_time is the first cron fire in [start, end). + backfill_start = trigger_time - datetime.timedelta(seconds=1) + backfill_end = trigger_time + datetime.timedelta(seconds=1) + await asyncio.to_thread( + dbos.DBOS.backfill_schedule, + schedule_name, + backfill_start, + backfill_end, + ) + + async def _maybe_backfill_schedule_on_startup( schedule_name: str, schedule_input: dbos.ScheduleInput, @@ -267,16 +336,32 @@ async def _ensure_schedule( ) await scheduler.INSTANCE.apply_schedules_async([schedule_input]) await _maybe_backfill_schedule_on_startup(schedule_name, schedule_input) + await _maybe_catch_up_schedule_once_on_startup(schedule_name, schedule_input) async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: schedule_inputs: list[dbos.ScheduleInput] = [ dbos_cleanup_workflow.get_schedule_input(), global_view_workflow.get_schedule_input(), + portfolio_history_workflow.get_schedule_input(), ] for schedule_input in schedule_inputs: await _ensure_schedule(scheduler, schedule_input) +async def update_cleanup_schedule_cron( + scheduler: scheduler_module.Scheduler, + cron: str, +) -> dict[str, typing.Any]: + existing_schedule = await dbos.DBOS.get_schedule_async(dbos_cleanup_workflow.SCHEDULE_NAME) + if existing_schedule is not None and existing_schedule["schedule"] == cron: + return {"changed": False, "cron": cron} + schedule_input = dbos_cleanup_workflow.get_schedule_input(cron=cron) + logger = _get_logger() + logger.info("Updating cleanup schedule cron to %s", cron) + await scheduler.INSTANCE.apply_schedules_async([schedule_input]) + return {"changed": True, "cron": cron} + + def _get_logger() -> logging.BotLogger: return logging.get_logger("schedules") diff --git a/packages/node/octobot_node/scheduler/tasks.py b/packages/node/octobot_node/scheduler/tasks.py index 188585636c..f74d2c246d 100644 --- a/packages/node/octobot_node/scheduler/tasks.py +++ b/packages/node/octobot_node/scheduler/tasks.py @@ -18,6 +18,7 @@ import typing import octobot_flow.entities +import octobot_commons.timestamp_util as timestamp_util import octobot_node.constants as node_constants import octobot_node.enums import octobot_node.errors as node_errors @@ -43,6 +44,26 @@ async def trigger_user_action_workflow( ) return handle.workflow_id + +async def trigger_portfolio_history_collection( + collection_params: typing.Optional["params.PortfolioHistoryCollectionParams"] = None, +) -> str: + import octobot_node.scheduler # avoid circular import + if not octobot_node.scheduler.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow + scheduled_time = timestamp_util.utc_now_datetime() + workflow_context = None + if collection_params is not None: + workflow_context = collection_params.to_dict(include_default_values=False) + handle = await octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE.enqueue_async( + portfolio_history_workflow.PortfolioHistoryWorkflow.portfolio_history_collection, + scheduled_time, + workflow_context, + ) + return handle.workflow_id + + async def trigger_task( task: octobot_node.models.Task, target_workflow_id: typing.Optional[str] = None ) -> str: diff --git a/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py b/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py index ad5f778b28..8426a61a4f 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py @@ -3,8 +3,12 @@ import typing import octobot_commons.dataclasses.minimizable_dataclass import octobot_node.models as models +import octobot_node.scheduler.workflows.params.portfolio_history_workflow_params as portfolio_history_workflow_params_module @dataclasses.dataclass class UserActionPostActions(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): to_create_automation_task: typing.Optional[models.Task] = None + portfolio_history_collection_params: typing.Optional[ + portfolio_history_workflow_params_module.PortfolioHistoryCollectionParams + ] = None diff --git a/packages/node/octobot_node/scheduler/user_actions/user_action_util.py b/packages/node/octobot_node/scheduler/user_actions/user_action_util.py index cdeaa09df2..dfad905f4a 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_action_util.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_action_util.py @@ -53,6 +53,7 @@ def resolve_user_action_result_type( | protocol_models.UserActionType.ACCOUNT_EDIT | protocol_models.UserActionType.ACCOUNT_DELETE | protocol_models.UserActionType.ACCOUNTS_REFRESH + | protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA ): return protocol_models.UserActionResultType.ACCOUNT case ( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py index a3b01750ec..13ffcc5e59 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py @@ -38,6 +38,7 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as user_actions_executor_strategy_base import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as user_actions_executor_restart_automation import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as user_actions_executor_stop_automation +import octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data as user_actions_executor_update_historical_exchanges_data from octobot_node.scheduler.user_actions.user_action_post_actions import UserActionPostActions from octobot_node.scheduler.user_actions.user_actions_executor.user_action_executor_factory import ( @@ -68,6 +69,7 @@ CreateAccountAuthActionExecutor = user_actions_executor_create_account_auth.CreateAccountAuthActionExecutor EditAccountAuthActionExecutor = user_actions_executor_edit_account_auth.EditAccountAuthActionExecutor DeleteAccountAuthActionExecutor = user_actions_executor_delete_account_auth.DeleteAccountAuthActionExecutor +UpdateHistoricalExchangesDataActionExecutor = user_actions_executor_update_historical_exchanges_data.UpdateHistoricalExchangesDataActionExecutor __all__ = [ "UserActionExecutor", @@ -94,6 +96,7 @@ "CreateAccountAuthActionExecutor", "EditAccountAuthActionExecutor", "DeleteAccountAuthActionExecutor", + "UpdateHistoricalExchangesDataActionExecutor", "user_action_executor_factory", "UserActionPostActions", ] diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py new file mode 100644 index 0000000000..5e8e528411 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py @@ -0,0 +1,63 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.errors as node_errors +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor + + +def _get_update_historical_exchanges_data_payload( + user_action: protocol_models.UserAction, +) -> protocol_models.UpdateHistoricalExchangesDataConfiguration: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "UserAction.configuration must wrap a concrete update-historical-exchanges-data configuration." + ) + payload = wrapper.actual_instance + if not isinstance(payload, protocol_models.UpdateHistoricalExchangesDataConfiguration): + raise node_errors.InvalidUserActionPayloadError( + "UpdateHistoricalExchangesDataActionExecutor expected " + f"UpdateHistoricalExchangesDataConfiguration, got {type(payload).__name__}" + ) + return payload + + +class UpdateHistoricalExchangesDataActionExecutor(account_user_action_executor.AccountUserActionExecutor): + async def _do_execute( + self, + user_action: protocol_models.UserAction, + ) -> None: + payload = _get_update_historical_exchanges_data_payload(user_action) + if not scheduler_module.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + account_provider = collection_providers.AccountProvider.instance() + # Client-supplied account_ids are scoped to this wallet only: get_item raises + # if an id does not belong to self._user_id (same check as refresh_accounts). + if payload.account_ids: + for account_id in payload.account_ids: + account_provider.get_item(self._user_id, account_id) + self.post_actions.portfolio_history_collection_params = ( + workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=[self._user_id], + account_ids=payload.account_ids, + ) + ) + self._mark_user_action_completed(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py index 29a32a3ff1..c06773b675 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py @@ -73,6 +73,8 @@ def user_action_executor_factory( return user_actions_executor_package.EditAccountAuthActionExecutor case protocol_models.DeleteAccountAuthConfiguration: return user_actions_executor_package.DeleteAccountAuthActionExecutor + case protocol_models.UpdateHistoricalExchangesDataConfiguration: + return user_actions_executor_package.UpdateHistoricalExchangesDataActionExecutor case _: raise node_errors.UnsupportedUserActionConfigurationTypeError( f"Unknown user action configuration type: {type(actual).__name__}" diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py index 31e4490106..403c416b54 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py @@ -22,7 +22,7 @@ def get_exchange_authentication( - user_id: str, + wallet_id: str, account: protocol_models.Account, ) -> protocol_models.AccountAuthentication | None: if account.is_simulated: @@ -43,12 +43,12 @@ def get_exchange_authentication( ) try: authentication = collection_providers.AccountAuthenticationProvider.instance().get_item( - user_id, + wallet_id, authentication_id, ) except collection_errors.ItemNotFoundError as err: raise node_errors.AccountAuthenticationNotFoundError( - f"Authentication {authentication_id!r} for account {account.id!r} not found for address {user_id!r}: {err}" + f"Authentication {authentication_id!r} for account {account.id!r} not found for address {wallet_id!r}: {err}" ) from err if not authentication.api_key or not authentication.api_secret: raise node_errors.AccountAuthenticationNotFoundError( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py index 41765fe776..66ecb6929e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py @@ -127,7 +127,7 @@ def _trading_type_for_account_state_check( async def _check_exchange_account_state( exchange_account: protocol_models.ExchangeAccount, account: protocol_models.Account, - user_id: str, + wallet_id: str, ) -> tuple[protocol_models.AccountState, list[protocol_models.DetailedAssetsForTradingType] | None]: if account.is_simulated: return ( @@ -138,11 +138,11 @@ async def _check_exchange_account_state( None, # not fetching assets for simulated accounts ) authentication = account_authentication_resolver.get_exchange_authentication( - user_id, + wallet_id, account, ) exchange_config = exchange_account_resolver.get_exchange_config( - user_id, + wallet_id, exchange_account, ) trading_type = _trading_type_for_account_state_check(account) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py index a567a10eb0..ebadb731e0 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py @@ -43,18 +43,18 @@ def get_primary_exchange_config_id( def get_exchange_config( - user_id: str, + wallet_id: str, exchange_account: protocol_models.ExchangeAccount, ) -> protocol_models.ExchangeConfig: exchange_config_id = get_primary_exchange_config_id(exchange_account) try: return collection_providers.AccountProvider.instance().get_exchange_config( - user_id, + wallet_id, exchange_config_id, ) except collection_errors.ItemNotFoundError as err: raise node_errors.InvalidUserActionPayloadError( - f"Exchange config {exchange_config_id!r} not found for address {user_id!r}: {err}" + f"Exchange config {exchange_config_id!r} not found for address {wallet_id!r}: {err}" ) from err diff --git a/packages/node/octobot_node/scheduler/workflows/__init__.py b/packages/node/octobot_node/scheduler/workflows/__init__.py index 8f489dd044..496ac1e14f 100644 --- a/packages/node/octobot_node/scheduler/workflows/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/__init__.py @@ -20,3 +20,4 @@ def register_workflows() -> None: import octobot_node.scheduler.workflows.user_action_workflow import octobot_node.scheduler.workflows.dbos_cleanup_workflow import octobot_node.scheduler.workflows.global_view_workflow + import octobot_node.scheduler.workflows.portfolio_history_workflow diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index f9f3ca36a5..06ab2cc238 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -22,6 +22,8 @@ import octobot_trading.errors +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.errors @@ -116,6 +118,7 @@ def _should_retry(error: BaseException) -> bool: # workflow stopping errors errors.WorkflowError, octobot_flow.errors.ConfigurationError, + copy_errors.OutdatedReferenceAccountError, )) @staticmethod @@ -175,11 +178,20 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) err, True, f"Retriable error while running automation job: {err}" ) raise - except octobot_flow.errors.PendingPriorityActionsSkippedError as err: + except ( + octobot_flow.errors.PendingPriorityActionsSkippedError, + copy_errors.OutdatedReferenceAccountError, + ) as err: # don't retry, just skip the iteration - AutomationWorkflow.get_logger(parsed_inputs).error( - f"Pending priority actions were skipped: {err}" - ) + workflow_logger = AutomationWorkflow.get_logger(parsed_inputs) + if isinstance(err, copy_errors.OutdatedReferenceAccountError): + workflow_logger.info( + f"Outdated reference account, skipping copy iteration: {err}" + ) + else: + workflow_logger.error( + f"Pending priority actions were skipped: {err}" + ) if action_job is None: # should never happen, but just in case raise diff --git a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py index 85272ad760..7cd212926b 100644 --- a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py @@ -59,14 +59,15 @@ async def _cleanup_outdated_automation_executions( scheduled_time.isoformat(), ) return dict(workflows_retention.EMPTY_CLEANUP_SUMMARY) - return await workflows_retention.cleanup_outdated_automation_executions(SCHEDULER) + cleanup_summary = await workflows_retention.cleanup_outdated_automation_executions(SCHEDULER) + return await workflows_retention.finalize_dbos_cleanup_run(SCHEDULER, cleanup_summary) -def get_schedule_input() -> dbos.ScheduleInput: +def get_schedule_input(cron: str | None = None) -> dbos.ScheduleInput: return { "schedule_name": SCHEDULE_NAME, "workflow_fn": DbosCleanupWorkflow.dbos_cleanup, - "schedule": SCHEDULE_CRON, + "schedule": cron or SCHEDULE_CRON, "context": None, "automatic_backfill": True, "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, diff --git a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py index 9de0ace250..e8cead645c 100644 --- a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py @@ -35,7 +35,7 @@ WORKFLOW_NAME = "global_view_refresh" SCHEDULE_NAME = "global_view_refresh_every_5m" -SCHEDULE_CRON = "*/5 * * * *" +SCHEDULE_CRON = "*/5 * * * *" # every 5 minutes def _exchange_label_for_account(user_id: str, account: protocol_models.Account) -> str: @@ -54,15 +54,8 @@ def _exchange_label_for_account(user_id: str, account: protocol_models.Account) def _portfolio_summary_fields( refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, ) -> tuple[str, str]: - portfolio_history_state = refresh_result.portfolio_history_state - if portfolio_history_state is None or portfolio_history_state.history is None: - return "n/a", "n/a" - history = portfolio_history_state.history - valuation_unit = history.unit if history.unit else "n/a" - history_values = history.values or [] - if not history_values: - return valuation_unit, "n/a" - return valuation_unit, str(history_values[-1].total) + # Portfolio history is now computed on-the-fly; summary fields are no longer available here. + return "n/a", "n/a" def _log_successful_account_refresh( @@ -115,7 +108,7 @@ async def _run_global_view_refresh( if workflows_retention.should_skip_retention_cleanup_on_this_node(): logger.info("global_view_refresh stopped: consumer-only node (skipped)") return {"refreshed_accounts": 0, "skipped": True} - wallet_ids = collection_providers.AccountProvider.instance().list_registered_wallet_ids() + wallet_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() refreshed_accounts_count = 0 for wallet_id in wallet_ids: refreshed_accounts_count += await GlobalViewRefreshWorkflow._refresh_wallet_accounts( diff --git a/packages/node/octobot_node/scheduler/workflows/params/__init__.py b/packages/node/octobot_node/scheduler/workflows/params/__init__.py index ed8efa3175..1c6158d7fa 100644 --- a/packages/node/octobot_node/scheduler/workflows/params/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/params/__init__.py @@ -27,12 +27,16 @@ UserActionWorkflowOutput, UserActionExecutionResult, ) +from .portfolio_history_workflow_params import ( + PortfolioHistoryCollectionParams, +) __all__ = [ "AutomationWorkflowActionUpdate", "AutomationWorkflowInputs", "AutomationWorkflowIterationResult", "AutomationWorkflowOutput", + "PortfolioHistoryCollectionParams", "ProgressStatus", "UserActionWorkflowInputs", "UserActionExecutionResult", diff --git a/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py b/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py new file mode 100644 index 0000000000..e08cbfd969 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py @@ -0,0 +1,25 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import dataclasses +import typing + +import octobot_commons.dataclasses.minimizable_dataclass + + +@dataclasses.dataclass +class PortfolioHistoryCollectionParams(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): + wallet_ids: typing.Optional[list[str]] = None + account_ids: typing.Optional[list[str]] = None diff --git a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py new file mode 100644 index 0000000000..2df4be5048 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py @@ -0,0 +1,113 @@ +import datetime +import typing + +import dbos +import octobot_commons.logging as commons_logging +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.scheduler.portfolio_history.portfolio_history_executor as portfolio_history_executor_module +import octobot_node.scheduler.workflows.params as workflow_params_module + +import octobot_node.enums +from octobot_node.scheduler import SCHEDULER + +WORKFLOW_NAME = "portfolio_history_collection" +SCHEDULE_NAME = "portfolio_history_daily_3am" +SCHEDULE_CRON = "0 3 * * *" # 3:00 AM every day + +logger = commons_logging.get_logger("PortfolioHistoryWorkflow") + +_EMPTY_COLLECTION_SUMMARY = { + "succeeded": 0, + "failed": 0, + "skipped": 0, +} + + +def _parse_collection_params( + context: typing.Any, +) -> workflow_params_module.PortfolioHistoryCollectionParams | None: + if context is None: + return None + if isinstance(context, workflow_params_module.PortfolioHistoryCollectionParams): + return context + if isinstance(context, dict): + return workflow_params_module.PortfolioHistoryCollectionParams.from_dict(context) + return None + + +@SCHEDULER.INSTANCE.dbos_class() +class PortfolioHistoryWorkflow: + @staticmethod + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + async def portfolio_history_collection( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + collection_params = _parse_collection_params(context) + return await PortfolioHistoryWorkflow._run_collection(scheduled_time, collection_params) + + @staticmethod + async def _run_collection( + scheduled_time: datetime.datetime, + collection_params: workflow_params_module.PortfolioHistoryCollectionParams | None = None, + ) -> dict[str, typing.Any]: + try: + logger.info("Starting portfolio history collection at %s", scheduled_time) + if collection_params and collection_params.wallet_ids: + wallet_ids = collection_params.wallet_ids + logger.info( + "Portfolio history collection scoped to %d wallet(s), account_ids=%s", + len(wallet_ids), + collection_params.account_ids, + ) + else: + wallet_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() + account_ids = collection_params.account_ids if collection_params else None + total_results = [] + + for wallet_id in wallet_ids: + try: + results = await portfolio_history_executor_module.run_portfolio_history_collection( + wallet_id, + account_ids=account_ids, + ) + total_results.extend(results) + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history collection failed for wallet {wallet_id}: {error}", + ) + + succeeded = sum(1 for result in total_results if not result.skipped and not result.error) + failed = sum(1 for result in total_results if result.error) + skipped = sum(1 for result in total_results if result.skipped) + logger.info( + "Portfolio history collection complete: %d succeeded, %d failed, %d skipped", + succeeded, failed, skipped, + ) + return { + "succeeded": succeeded, + "failed": failed, + "skipped": skipped, + } + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history collection failed: {error}", + ) + return dict(_EMPTY_COLLECTION_SUMMARY) + + +def get_schedule_input() -> dbos.ScheduleInput: + return { + "schedule_name": SCHEDULE_NAME, + "workflow_fn": PortfolioHistoryWorkflow.portfolio_history_collection, + "schedule": SCHEDULE_CRON, + "context": None, + "automatic_backfill": False, + "catch_up_once_on_startup": True, + "queue_name": octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, + } diff --git a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py index 5d5d892b19..79944386f5 100644 --- a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py @@ -97,3 +97,7 @@ async def after_user_action_execution(output: params.UserActionExecutionResult) output.post_actions.to_create_automation_task, target_workflow_id=output.post_actions.to_create_automation_task.id ) + if output.post_actions.portfolio_history_collection_params: + await scheduler_tasks.trigger_portfolio_history_collection( + output.post_actions.portfolio_history_collection_params, + ) diff --git a/packages/node/octobot_node/scheduler/workflows_retention.py b/packages/node/octobot_node/scheduler/workflows_retention.py index 01886d0e6f..4a9c9df751 100644 --- a/packages/node/octobot_node/scheduler/workflows_retention.py +++ b/packages/node/octobot_node/scheduler/workflows_retention.py @@ -15,6 +15,7 @@ # License along with this library. import datetime import os +import pathlib import time import typing @@ -29,14 +30,28 @@ if typing.TYPE_CHECKING: import octobot_node.scheduler.scheduler as scheduler_module +_GIBIBYTE = 1024 ** 3 + AUTOMATION_EXECUTION_RETENTION_SECONDS = float( os.getenv("AUTOMATION_EXECUTION_RETENTION_SECONDS", 60 * 60 * 24 * 2) ) # 2 days AUTOMATION_EXECUTIONS_TO_KEEP = 2 +DBOS_CLEANUP_SIZE_TIER_1_BYTES = int( + os.getenv("DBOS_CLEANUP_SIZE_TIER_1_BYTES", str(1 * _GIBIBYTE)) +) +DBOS_CLEANUP_SIZE_TIER_2_BYTES = int( + os.getenv("DBOS_CLEANUP_SIZE_TIER_2_BYTES", str(3 * _GIBIBYTE)) +) +DBOS_CLEANUP_CRON_DAILY = os.getenv("DBOS_CLEANUP_CRON_DAILY", "0 0 * * *") +DBOS_CLEANUP_CRON_12H = os.getenv("DBOS_CLEANUP_CRON_12H", "0 */12 * * *") +DBOS_CLEANUP_CRON_6H = os.getenv("DBOS_CLEANUP_CRON_6H", "0 */6 * * *") + EMPTY_CLEANUP_SUMMARY: dict[str, typing.Any] = { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -62,6 +77,21 @@ def _retention_cutoff_ms(*, retention_seconds: float, now_ms: int) -> int: return now_ms - int(retention_seconds * 1000) +def get_outdated_terminal_workflow_ids( + workflows: list[dbos.WorkflowStatus], + *, + retention_seconds: float, + now_ms: int, +) -> list[str]: + cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) + return [ + workflow_status.workflow_id + for workflow_status in workflows + if is_terminal_workflow(workflow_status) + and (workflow_status.updated_at or 0) < cutoff_ms + ] + + def get_outdated_automation_execution_deletions( workflows: list[dbos.WorkflowStatus], *, @@ -102,13 +132,11 @@ def get_outdated_dbos_cleanup_execution_workflow_ids( retention_seconds: float, now_ms: int, ) -> list[str]: - cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) - return [ - workflow_status.workflow_id - for workflow_status in cleanup_workflows - if is_terminal_workflow(workflow_status) - and (workflow_status.updated_at or 0) < cutoff_ms - ] + return get_outdated_terminal_workflow_ids( + cleanup_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) _TERMINAL_DELETE_WORKFLOW_STATUSES = [ @@ -137,6 +165,22 @@ async def get_workflows_to_delete( return automation_workflows + user_action_workflows +def get_scheduler_database_size_bytes(dbos_instance: dbos.DBOS) -> int | None: + if octobot_node.config.settings.SCHEDULER_POSTGRES_URL: + return _get_postgres_database_size_bytes(dbos_instance) + return _get_sqlite_database_size_bytes() + + +def select_cleanup_cron_for_database_size(database_size_bytes: int | None) -> str: + if database_size_bytes is None: + return DBOS_CLEANUP_CRON_DAILY + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_1_BYTES: + return DBOS_CLEANUP_CRON_DAILY + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_2_BYTES: + return DBOS_CLEANUP_CRON_12H + return DBOS_CLEANUP_CRON_6H + + def vacuum_dbos_system_database(dbos_instance: dbos.DBOS) -> None: logger = _get_logger() logger.info("Vacuuming database") @@ -145,12 +189,19 @@ def vacuum_dbos_system_database(dbos_instance: dbos.DBOS) -> None: logger.info("Database vacuum completed") -async def delete_workflows_and_vacuum( +async def delete_workflows( dbos_instance: dbos.DBOS, - workflow_ids: list[str] + workflow_ids: list[str], ) -> None: _get_logger().info("Deleting %s workflows", len(workflow_ids)) await dbos_instance.delete_workflows_async(workflow_ids, delete_children=False) + + +async def delete_workflows_and_vacuum( + dbos_instance: dbos.DBOS, + workflow_ids: list[str] +) -> None: + await delete_workflows(dbos_instance, workflow_ids) vacuum_dbos_system_database(dbos_instance) @@ -164,6 +215,8 @@ async def cleanup_outdated_automation_executions( retention_seconds = AUTOMATION_EXECUTION_RETENTION_SECONDS import octobot_node.scheduler.workflows.automation_workflow as automation_workflow import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow automation_workflows = await scheduler.INSTANCE.list_workflows_async( name=automation_workflow.WORKFLOW_NAME, queue_name=[octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], @@ -175,6 +228,17 @@ async def cleanup_outdated_automation_executions( load_input=False, load_output=False, ) + global_view_workflows = await scheduler.INSTANCE.list_workflows_async( + name=global_view_workflow.WORKFLOW_NAME, + queue_name=[octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value], + load_input=False, + load_output=False, + ) + portfolio_history_workflows = await scheduler.INSTANCE.list_workflows_async( + name=portfolio_history_workflow.WORKFLOW_NAME, + load_input=False, + load_output=False, + ) deletions_by_automation = get_outdated_automation_execution_deletions( automation_workflows, retention_seconds=retention_seconds, @@ -185,34 +249,82 @@ async def cleanup_outdated_automation_executions( retention_seconds=retention_seconds, now_ms=now_ms, ) + global_view_execution_ids = get_outdated_terminal_workflow_ids( + global_view_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + portfolio_history_execution_ids = get_outdated_terminal_workflow_ids( + portfolio_history_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) automation_execution_ids = [ workflow_id for workflow_ids in deletions_by_automation.values() for workflow_id in workflow_ids ] - all_ids_to_delete = automation_execution_ids + cleanup_execution_ids + all_ids_to_delete = ( + automation_execution_ids + + cleanup_execution_ids + + global_view_execution_ids + + portfolio_history_execution_ids + ) summary = { "deleted_by_automation": { parent_id: len(workflow_ids) for parent_id, workflow_ids in deletions_by_automation.items() }, "deleted_cleanup_executions": len(cleanup_execution_ids), + "deleted_global_view_executions": len(global_view_execution_ids), + "deleted_portfolio_history_executions": len(portfolio_history_execution_ids), "total_deleted": len(all_ids_to_delete), } if all_ids_to_delete: _get_logger().info( - "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs, " + "%s global view runs, %s portfolio history runs", len(all_ids_to_delete), len(deletions_by_automation), len(cleanup_execution_ids), + len(global_view_execution_ids), + len(portfolio_history_execution_ids), ) - await delete_workflows_and_vacuum( + await delete_workflows( scheduler.INSTANCE, all_ids_to_delete, ) _get_logger().info("DBOS cleanup summary: %s", summary) return summary + +async def finalize_dbos_cleanup_run( + scheduler: "scheduler_module.Scheduler", + summary: dict[str, typing.Any], +) -> dict[str, typing.Any]: + if not scheduler.INSTANCE: + return summary + database_size_bytes = get_scheduler_database_size_bytes(scheduler.INSTANCE) + finalized_summary = dict(summary) + finalized_summary["database_size_bytes"] = database_size_bytes + total_deleted = finalized_summary.get("total_deleted", 0) + if total_deleted > 0 or ( + database_size_bytes is not None + and database_size_bytes >= DBOS_CLEANUP_SIZE_TIER_2_BYTES + ): + vacuum_dbos_system_database(scheduler.INSTANCE) + desired_cron = select_cleanup_cron_for_database_size(database_size_bytes) + finalized_summary["cleanup_schedule_cron"] = desired_cron + import octobot_node.scheduler.schedules as schedules_module + schedule_update = await schedules_module.update_cleanup_schedule_cron( + scheduler, + desired_cron, + ) + finalized_summary["cleanup_schedule_updated"] = schedule_update["changed"] + _get_logger().info("DBOS cleanup finalized summary: %s", finalized_summary) + return finalized_summary + + def _get_latest_completed_cleanup_timestamp_ms( cleanup_workflows: list[dbos.WorkflowStatus], ) -> int: @@ -244,5 +356,43 @@ async def should_skip_retention_cleanup_for_scheduled_time( return latest_timestamp_ms > scheduled_timestamp_ms +def _get_sqlite_database_size_bytes() -> int | None: + sqlite_path = pathlib.Path(octobot_node.config.settings.SCHEDULER_SQLITE_FILE) + if not sqlite_path.is_file(): + missing_file_error = FileNotFoundError(f"Scheduler sqlite file not found: {sqlite_path}") + _get_logger().exception( + missing_file_error, + True, + "Scheduler sqlite file not found: %s", + sqlite_path, + ) + return None + total_size_bytes = sqlite_path.stat().st_size + for suffix in ("-wal", "-shm"): + sidecar_path = pathlib.Path(f"{sqlite_path}{suffix}") + if sidecar_path.is_file(): + total_size_bytes += sidecar_path.stat().st_size + return total_size_bytes + + +def _get_postgres_database_size_bytes(dbos_instance: dbos.DBOS) -> int | None: + try: + with dbos_instance._sys_db.engine.begin() as connection: + database_size = connection.execute( + sqlalchemy.text("SELECT pg_database_size(current_database())"), + ).scalar() + if database_size is None: + return None + return int(database_size) + except Exception as error: + _get_logger().exception( + error, + True, + "Failed to read postgres database size: %s", + error, + ) + return None + + def _get_logger() -> logging.BotLogger: return logging.get_logger("workflows_retention") diff --git a/packages/node/tests/functional_tests/test_accounts_history_compute.py b/packages/node/tests/functional_tests/test_accounts_history_compute.py new file mode 100644 index 0000000000..2cb19b0894 --- /dev/null +++ b/packages/node/tests/functional_tests/test_accounts_history_compute.py @@ -0,0 +1,330 @@ +# Drakkar-Software OctoBot-Node + +""" +Functional tests for on-the-fly portfolio history computation. + +Each test seeds persisted Account (current holdings), AccountTrading (trades / +transactions), and exchange price caches, then calls +compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions. + +The compute path reverse-replays trades and transactions from the latest +portfolio to reconstruct past daily holdings, then values each day using daily +prices (or latest tickers when daily prices are missing). +""" + +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + +import octobot_node.protocol.accounts_history as accounts_history_module +from tests.functional_tests.util import accounts_history_test_util as accounts_history_test_util + + +@pytest.mark.asyncio +class TestComputeHistoryFunctionalSimulated: + """End-to-end history compute with seeded sync providers and persisted caches.""" + + async def test_single_buy_trade_produces_two_day_history(self, tmp_path): + """ + Scenario: account holds 1 BTC and 50000 USDT now. + One historical buy of 1 BTC at 30000 USDT on day 1. + Daily price cache has BTC/USDT = 30000 on day 1 and 50000 on day 2. + Expected: day 1 portfolio had 0 BTC + 80000 USDT, day 2 has 1 BTC + 50000 USDT. + """ + # Latest portfolio anchor: what the account holds today. + account = accounts_history_test_util.make_account("a1", {"BTC": 1.0, "USDT": 50000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + # Single BUY on day 1; reverse replay will subtract 1 BTC and credit 30000 USDT. + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "BTC/USDT", + protocol_models.Side.BUY, + quantity=1.0, + price=30000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + + buy_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.BUY_TIME.timestamp()) + ) + next_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.DAY_2_TS) + ) + data_root = str(tmp_path) + + # Day 1 close price 30000; day 2 close price 50000 (current-day valuation). + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": {buy_day_str: 30000.0, next_day_str: 50000.0}}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": 50000.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + ) + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.version == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + assert result.history is not None + assert result.history.unit == "USDT" + assert len(result.history.values) >= 2 + + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + buy_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.BUY_TIME.timestamp() + ) + prior_day_start = buy_day_start - 86400 + assert prior_day_start in day_values + assert buy_day_start in day_values + # Before the buy: 0 BTC + 80000 USDT (50000 + 1 * 30000), valued at 30000/BTC → 80000. + assert day_values[prior_day_start] == pytest.approx(80000.0) + # After the buy on day 1: 1 BTC + 50000 USDT, valued at 30000/BTC → 80000. + assert day_values[buy_day_start] == pytest.approx(80000.0) + + async def test_deposit_and_trade_produce_coherent_history(self, tmp_path): + """ + Scenario: account holds 2 BTC and 20000 USDT now. + Day 1: bought 1 BTC at 30000 USDT. + Day 2: deposited 1 BTC. + Daily price for BTC/USDT: day 1 = 30000, day 2 = 35000. + """ + # Latest: 2 BTC + 20000 USDT. + account = accounts_history_test_util.make_account("a1", {"BTC": 2.0, "USDT": 20000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + # Day 1 BUY: reverse removes 1 BTC and adds 30000 USDT → 1 BTC + 20000 USDT. + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "BTC/USDT", + protocol_models.Side.BUY, + quantity=1.0, + price=30000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + # Day 2 deposit: reverse removes 1 BTC → 1 BTC + 20000 USDT before deposit. + deposit = accounts_history_test_util.make_protocol_transaction( + "tx1", + "BTC", + 1.0, + protocol_models.TransactionType.BLOCKCHAIN_DEPOSIT, + accounts_history_test_util.DEPOSIT_TIME, + ) + + buy_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.BUY_TIME.timestamp()) + ) + deposit_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.DEPOSIT_TIME.timestamp()) + ) + data_root = str(tmp_path) + + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": {buy_day_str: 30000.0, deposit_day_str: 35000.0}}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": 35000.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + transactions=[deposit], + ) + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.history is not None + assert len(result.history.values) >= 2 + + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + buy_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.BUY_TIME.timestamp() + ) + deposit_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.DEPOSIT_TIME.timestamp() + ) + prior_day_start = buy_day_start - 86400 + assert deposit_day_start in day_values + assert buy_day_start in day_values + assert prior_day_start in day_values + # After deposit on day 2: 2 BTC + 20000 USDT @ 35000 → 90000. + assert day_values[deposit_day_start] == pytest.approx(90000.0) + # After buy on day 1 (before deposit): 1 BTC + 20000 USDT @ 30000 → 50000. + assert day_values[buy_day_start] == pytest.approx(50000.0) + # Before buy: 0 BTC + 50000 USDT @ 30000 → 50000. + assert day_values[prior_day_start] == pytest.approx(50000.0) + + async def test_no_account_returns_empty_state(self, tmp_path): + """Missing account id must yield an empty history state, not an error.""" + with accounts_history_test_util.accounts_history_test_environment(tmp_path): + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "missing", + ) + + assert result.history is None + + async def test_no_trading_data_returns_empty_state(self, tmp_path): + """Account exists but AccountTrading was never seeded → no history to compute.""" + account = accounts_history_test_util.make_account("a1", {"BTC": 1.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + _trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + ) + + assert result.history is None + + async def test_ticker_fallback_when_no_daily_prices(self, tmp_path): + """ + When daily price cache is empty, latest ticker is used for all days. + + Scenario: account holds 5 ETH and 1000 USDT now. + One BUY of 5 ETH at 2000 USDT on day 1. + Ticker fallback: ETH/USDT = 2500 for every day. + """ + account = accounts_history_test_util.make_account("a1", {"ETH": 5.0, "USDT": 1000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "ETH/USDT", + protocol_models.Side.BUY, + quantity=5.0, + price=2000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + data_root = str(tmp_path) + + # Empty daily prices force valuation to use the latest ticker only. + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"ETH/USDT": 2500.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + ) + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.history is not None + buy_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.BUY_TIME.timestamp() + ) + prior_day_start = buy_day_start - 86400 + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + # Before buy: 0 ETH + 11000 USDT (1000 + 5 * 2000), all in USDT → 11000. + assert day_values[prior_day_start] == pytest.approx(11000.0) + # After buy: 5 ETH + 1000 USDT @ 2500 ticker → 12500 + 1000 = 13500. + assert day_values[buy_day_start] == pytest.approx(13500.0) diff --git a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py index 26346920b9..0c25d8d7b3 100644 --- a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py +++ b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py @@ -33,6 +33,7 @@ import starfish_server.storage.filesystem as starfish_filesystem_storage_module import starfish_sharing as starfish_sharing_module +from .util import exchange_account_elements_access as exchange_account_elements_access_module from .util import grid_workflow as grid_sim_util from .util import price_mocks as price_mocks_module from .util import user_action_assertions as user_action_assertions_module @@ -40,11 +41,9 @@ import octobot.community.authentication as community_authentication_module import octobot.community.local_authenticator as local_authenticator_module -import octobot_trading.constants as trading_constants_module import octobot_node.config import octobot_node.constants as node_constants_module import octobot_node.scheduler.workflows_util as workflows_util_module -import octobot_commons.constants as octobot_commons_constants_module import octobot_commons.os_util as commons_os_util_module import octobot_copy.constants as octobot_copy_constants_module import octobot_flow.entities as octobot_flow_entities @@ -179,72 +178,6 @@ def _account_for_id( return copy_account raise AssertionError(f"Unexpected account_id lookup: {account_id!r}") -def _d_order_price(raw: typing.Union[int, float, str, decimal.Decimal]) -> decimal.Decimal: - if isinstance(raw, decimal.Decimal): - return raw - return decimal.Decimal(str(raw)) - -def _sorted_limit_prices_from_elements( - exchange_account_elements: typing.Any, - *, - trade_order_side, -) -> list[decimal.Decimal]: - if exchange_account_elements is None: - return [] - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - return [] - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) - open_orders = open_orders or [] - - storage_origin = trading_constants_module.STORAGE_ORIGIN_VALUE - - def _open_order_payload(order_row: typing.Any) -> typing.Any: - """Exchange rows may nest ccxt fields under ``STORAGE_ORIGIN_VALUE``; protocol orders are flat.""" - if isinstance(order_row, dict): - nested = order_row.get(storage_origin) - if isinstance(nested, dict): - return nested - return order_row - nested = getattr(order_row, storage_origin, None) - if nested is not None: - return nested - return order_row - - side_key = trading_enums_module.ExchangeConstantsOrderColumns.SIDE.value - price_col = trading_enums_module.ExchangeConstantsOrderColumns.PRICE.value - type_col = trading_enums_module.ExchangeConstantsOrderColumns.TYPE.value - want_side = trade_order_side.value - limit_type = trading_enums_module.TradeOrderType.LIMIT.value - prices: list[decimal.Decimal] = [] - for order in open_orders: - payload = _open_order_payload(order) - if isinstance(payload, dict): - side = payload.get(side_key) - price_raw = payload.get(price_col) - order_type = payload.get(type_col) - else: - side = getattr(payload, side_key, None) - price_raw = getattr(payload, price_col, None) - order_type = getattr(payload, type_col, None) - if hasattr(side, "value"): - side = side.value - if side != want_side: - continue - if price_raw is None: - continue - if hasattr(order_type, "value"): - order_type = order_type.value - if order_type is not None and order_type != limit_type: - continue - prices.append(_d_order_price(price_raw)) - prices.sort() - return prices - def _assert_open_limit_prices_match_reference( reference_elements: typing.Any, follower_elements: typing.Any, @@ -253,31 +186,16 @@ def _assert_open_limit_prices_match_reference( trading_enums_module.TradeOrderSide.BUY, trading_enums_module.TradeOrderSide.SELL, ): - ref_prices = _sorted_limit_prices_from_elements(reference_elements, trade_order_side=side) - got_prices = _sorted_limit_prices_from_elements(follower_elements, trade_order_side=side) + ref_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + reference_elements, + trade_order_side=side, + ) + got_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + follower_elements, + trade_order_side=side, + ) assert ref_prices == got_prices, f"side={side!r} ref={ref_prices!s} follower={got_prices!s}" -def _portfolio_content_from_exchange_elements(exchange_account_elements: typing.Any) -> dict[str, typing.Any]: - portfolio = getattr(exchange_account_elements, "portfolio", None) - if portfolio is None and isinstance(exchange_account_elements, dict): - portfolio = exchange_account_elements.get("portfolio") - if portfolio is None: - return {} - content = getattr(portfolio, "content", None) - if content is None and isinstance(portfolio, dict): - content = portfolio.get("content") - return content if isinstance(content, dict) else {} - -def _portfolio_row_total(row: typing.Any) -> decimal.Decimal: - total_key = octobot_commons_constants_module.PORTFOLIO_TOTAL - if isinstance(row, dict): - raw = row.get(total_key, row.get("total")) - else: - raw = getattr(row, total_key, None) or getattr(row, "total", None) - if raw is None: - raise AssertionError("portfolio row has no total amount") - return raw if isinstance(raw, decimal.Decimal) else decimal.Decimal(str(raw)) - def _value_weighted_btc_usdc_shares( content: dict[str, typing.Any], *, @@ -286,8 +204,8 @@ def _value_weighted_btc_usdc_shares( """USDC notionals: ``btc_total * btc_usdc_close`` vs USDC total; shares sum to 1.""" for asset in ("BTC", "USDC"): assert asset in content, f"missing portfolio row for {asset}" - btc_total = _portfolio_row_total(content["BTC"]) - usdc_total = _portfolio_row_total(content["USDC"]) + btc_total = exchange_account_elements_access_module.portfolio_row_total(content["BTC"]) + usdc_total = exchange_account_elements_access_module.portfolio_row_total(content["USDC"]) btc_notional_usdc = btc_total * btc_usdc_close usdc_notional = usdc_total total_notional = btc_notional_usdc + usdc_notional @@ -303,8 +221,8 @@ def _assert_btc_usdc_value_shares_match_reference( *, btc_usdc_close: decimal.Decimal, ) -> None: - ref_content = _portfolio_content_from_exchange_elements(reference_elements) - follower_content = _portfolio_content_from_exchange_elements(follower_elements) + ref_content = exchange_account_elements_access_module.portfolio_content_from_elements(reference_elements) + follower_content = exchange_account_elements_access_module.portfolio_content_from_elements(follower_elements) ref_shares = _value_weighted_btc_usdc_shares(ref_content, btc_usdc_close=btc_usdc_close) follower_shares = _value_weighted_btc_usdc_shares(follower_content, btc_usdc_close=btc_usdc_close) # ~3 percentage points slack (master vs copy notionals / float portfolio totals). @@ -318,7 +236,7 @@ def _assert_btc_usdc_value_shares_match_reference( ) def _first_sell_limit_price(exchange_account_elements: typing.Any) -> decimal.Decimal: - sells = _sorted_limit_prices_from_elements( + sells = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( exchange_account_elements, trade_order_side=trading_enums_module.TradeOrderSide.SELL, ) @@ -338,8 +256,10 @@ def _sorted_limit_prices_from_trading_signal_account( *, trade_order_side, ) -> list[decimal.Decimal]: - wrapper = {"orders": {"open_orders": list(trading_signal.account.orders or [])}} - return _sorted_limit_prices_from_elements(wrapper, trade_order_side=trade_order_side) + return exchange_account_elements_access_module.sorted_open_limit_prices_from_protocol_orders( + trading_signal.account.orders, + trade_order_side=trade_order_side, + ) async def _fetch_strategy_signals_from_sync(evm_address: str) -> list[typing.Any]: async with local_authenticator_module.local_user_authenticator() as auth: @@ -360,7 +280,7 @@ def _ladder_limit_prices_match_reference( trading_enums_module.TradeOrderSide.BUY, trading_enums_module.TradeOrderSide.SELL, ): - reference_prices = _sorted_limit_prices_from_elements( + reference_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( reference_exchange_account_elements, trade_order_side=order_side, ) diff --git a/packages/node/tests/functional_tests/test_global_view_workflow.py b/packages/node/tests/functional_tests/test_global_view_workflow.py index 3f58a73fad..8c407e5934 100644 --- a/packages/node/tests/functional_tests/test_global_view_workflow.py +++ b/packages/node/tests/functional_tests/test_global_view_workflow.py @@ -6,7 +6,6 @@ import mock import pytest -import octobot_node.protocol.accounts_history as accounts_history_protocol import octobot_node.scheduler.api as scheduler_api_module import octobot_node.scheduler.tasks as scheduler_tasks_module @@ -46,17 +45,6 @@ async def test_global_view_workflow_updates_all_accounts( assert sim_account_1.updated_at is not None assert sim_account_2.updated_at is not None - for account_id in ( - global_view_workflow_util.ACCOUNT_REAL_ID, - global_view_workflow_util.ACCOUNT_SIM_1_ID, - global_view_workflow_util.ACCOUNT_SIM_2_ID, - ): - history_state = accounts_history_protocol.get_portfolio_history_state(user_id, account_id) - assert history_state.history is not None - assert history_state.history.values - assert history_state.history.values[-1].total > 0 - assert history_state.history.unit - async def test_global_view_workflow_triggers_automation_on_filled_order( self, tmp_path: pathlib.Path, @@ -127,6 +115,3 @@ async def record_forced_trigger(automation_id: str, user_id: str) -> None: ): updated_account = account_provider.get_item(user_id, account_id) assert updated_account.assets is not None - history_state = accounts_history_protocol.get_portfolio_history_state(user_id, account_id) - assert history_state.history is not None - assert history_state.history.values diff --git a/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py new file mode 100644 index 0000000000..b6f316a922 --- /dev/null +++ b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py @@ -0,0 +1,199 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +from __future__ import annotations + +import asyncio +import decimal +import logging +import time + +import mock +import pytest + +from .util import authenticator_mocks as authenticator_mocks_module +from .util import grid_workflow as grid_workflow_module +from .util import price_mocks as price_mocks_module +from .util import user_action_assertions as user_action_assertions_module +from .util import workflow_common as workflow_common_module + +import octobot.community.authentication as community_authentication_module +import octobot_flow.entities as octobot_flow_entities +import octobot_node.config +import octobot_node.scheduler +import octobot_node.scheduler.workflows_util as workflows_util_module +import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module + +from tests.scheduler import temp_dbos_scheduler + +_T_ENQUEUE_SECONDS = 5.0 +_T_POLL_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) + + +@pytest.mark.asyncio +class TestOutdatedReferenceAccountWorkflow: + async def test_skips_stale_signal_then_executes_fresh_signal_from_internal_channel( + self, + temp_dbos_scheduler, + caplog, + ): + """ + Copy automation on the simulator exchange: stale TradingSignal is rejected as outdated, + workflow postpones and waits; a fresh signal drives a real mirror limit order. + """ + import octobot_flow.repositories.community.trading_signals_channel as trading_signals_channel_module + import octobot_node.scheduler.internal_trading_signals as internal_trading_signals_module + + from .util import outdated_reference_workflow as outdated_reference_workflow_module + + fresh_order_price_decimal = decimal.Decimal(str(outdated_reference_workflow_module.FRESH_ORDER_PRICE)) + stale_order_price_decimal = decimal.Decimal(str(outdated_reference_workflow_module.STALE_ORDER_PRICE)) + + # Step 0 — Pin BTC/USDC close and mock only exchange/infra providers; DBOS + flow run for real. + patched_fetch_tickers = price_mocks_module.tickers_repository_fetch_tickers_btc_usdc_close_override( + lambda: outdated_reference_workflow_module.FRESH_MARKET_PRICE, + ) + patched_fetch_ohlcv = price_mocks_module.fetch_ohlcv_side_effect_for_close_price( + lambda: outdated_reference_workflow_module.FRESH_MARKET_PRICE, + ) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + protocol_account = workflow_common_module.protocol_account_for_functional( + account_id=outdated_reference_workflow_module.COPY_ACCOUNT_ID, + usdc_total=2000.0, + account_name="Outdated reference functional copy account", + ) + copy_user_action = outdated_reference_workflow_module.build_create_copy_follower_user_action() + automation_id = user_action_assertions_module.resolve_create_automation_metadata_id(copy_user_action) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + def _functional_seed_strategy_for_outdated_reference_test(_wallet_address, stored_item_id): + if stored_item_id == grid_workflow_module.SIMULATOR_COPY_FOLLOWER_STORED_STRATEGY_ID: + return grid_workflow_module.seeded_copy_follower_strategy_for_functional_wallet( + copy_master_strategy_id=outdated_reference_workflow_module.MASTER_STRATEGY_ID, + ) + raise AssertionError(f"unexpected strategy id for functional seed: {stored_item_id!r}") + + await internal_trading_signals_module.subscribe_internal_trading_signal_consumer() + try: + # Step 0 (continued) — Subscribe internal trading-signal consumer and seed account trading state. + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch.object( + octobot_flow_repositories_exchange_module.TickersRepository, + "fetch_tickers", + new=patched_fetch_tickers, + ), + mock.patch.object( + octobot_flow_repositories_exchange_module.OhlcvRepository, + "fetch_ohlcv", + side_effect=patched_fetch_ohlcv, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock(return_value=protocol_account), + get_exchange_config=mock.Mock( + return_value=workflow_common_module.protocol_exchange_config_for_grid_functional(), + ), + ), + ), + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock( + side_effect=_functional_seed_strategy_for_outdated_reference_test, + ), + ), + ), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), + ): + caplog.set_level(logging.INFO) + workflow_common_module.seed_empty_account_trading_state( + user_id, + outdated_reference_workflow_module.COPY_ACCOUNT_ID, + ) + + stale_signal = outdated_reference_workflow_module.build_stale_trading_signal() + fresh_signal = outdated_reference_workflow_module.build_fresh_trading_signal() + + stale_delivery_task = asyncio.create_task( + outdated_reference_workflow_module.deliver_trading_signal_when_pending( + temp_dbos_scheduler, + automation_id, + stale_signal, + _T_POLL_SECONDS, + ) + ) + + # Step 1 — Enqueue AUTOMATION_CREATE for copy follower; expect COMPLETED create result. + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + copy_user_action, + user_id, + ), + timeout=_T_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing copy workflow") from exc + + await stale_delivery_task + + await user_action_assertions_module.assert_user_action_selector_completed_automation_create( + user_id=user_id, + user_action_id=copy_user_action.id, + expected_workflow_id=None, + ) + + # Step 2 — Stale signal delivered while copy workflow was pending; expect outdated skip log. + stale_skip_deadline = time.monotonic() + _T_POLL_SECONDS + while time.monotonic() < stale_skip_deadline: + if outdated_reference_workflow_module.caplog_contains_outdated_skip(caplog): + break + await asyncio.sleep(workflow_common_module.DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + pytest.fail( + "Timed out waiting for outdated reference account skip log " + f"({outdated_reference_workflow_module.OUTDATED_SKIP_LOG_SUBSTRING!r})" + ) + + await outdated_reference_workflow_module.poll_state_reader_until( + temp_dbos_scheduler, + automation_id, + lambda reader: stale_order_price_decimal + not in outdated_reference_workflow_module.sell_limit_prices_from_reader(reader), + _T_POLL_SECONDS, + "stale sell limit not mirrored", + ) + + # Step 3 — Send fresh TradingSignal after child workflow is pending again; expect mirror at fresh price. + await outdated_reference_workflow_module.deliver_trading_signal_when_pending( + temp_dbos_scheduler, + automation_id, + fresh_signal, + _T_POLL_SECONDS, + ) + fresh_reader = await outdated_reference_workflow_module.poll_state_reader_until( + temp_dbos_scheduler, + automation_id, + lambda reader: fresh_order_price_decimal + in outdated_reference_workflow_module.sell_limit_prices_from_reader(reader), + _T_POLL_SECONDS, + "fresh sell limit mirrored", + ) + + # Step 4 — Assert stale price never mirrored, fresh order present, automation still healthy. + sell_prices = outdated_reference_workflow_module.sell_limit_prices_from_reader(fresh_reader) + assert stale_order_price_decimal not in sell_prices + assert sell_prices.count(fresh_order_price_decimal) == 1 + assert outdated_reference_workflow_module.caplog_contains_outdated_skip(caplog) + finally: + await trading_signals_channel_module.shutdown_internal_trading_signal_channel() diff --git a/packages/node/tests/functional_tests/util/accounts_history_test_util.py b/packages/node/tests/functional_tests/util/accounts_history_test_util.py new file mode 100644 index 0000000000..a4fe444586 --- /dev/null +++ b/packages/node/tests/functional_tests/util/accounts_history_test_util.py @@ -0,0 +1,211 @@ +# Drakkar-Software OctoBot-Node + +import contextlib +import datetime +import math + +import mock + +import octobot.community.authentication as community_authentication +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api + + +TEST_USER_ID = "0xaccountshistorytestwallet" +_TEST_PRIVATE_KEY = "accounts-history-test-private-key" +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + +DAY_1_TS = 1700000000.0 +DAY_2_TS = DAY_1_TS + 86400 +BUY_TIME = datetime.datetime.fromtimestamp(DAY_1_TS + 3600, tz=datetime.timezone.utc) +DEPOSIT_TIME = datetime.datetime.fromtimestamp(DAY_2_TS + 1800, tz=datetime.timezone.utc) + + +def utc_day_start(timestamp: float) -> int: + return int(math.floor(timestamp / 86400.0) * 86400) + + +def make_account( + account_id: str, + assets: dict[str, float], + exchange: str = "binance", + sandboxed: bool = False, + is_simulated: bool = False, +) -> protocol_models.Account: + detailed_assets = [ + protocol_models.DetailedAsset(symbol=symbol, total=amount, available=amount) + for symbol, amount in assets.items() + ] + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote1", + exchange_config_ids=["cfg1"], + ) + return protocol_models.Account( + id=account_id, + name="Test", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=detailed_assets, + ) + ], + ) + + +def make_exchange_config( + exchange: str = "binance", + sandboxed: bool = False, +) -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id="cfg1", + name="test", + exchange=exchange, + sandboxed=sandboxed, + historical_trade_symbols=["BTC/USDT"], + ) + + +def make_protocol_trade( + trade_id: str, + symbol: str, + side: protocol_models.Side, + quantity: float, + price: float, + executed_at: datetime.datetime, +) -> protocol_models.Trade: + return protocol_models.Trade( + id=trade_id, + trade_id=trade_id, + type=protocol_models.OrderType.LIMIT, + symbol=symbol, + side=side, + quantity=quantity, + price=price, + status=protocol_models.OrderStatus.FILLED, + executed_at=executed_at, + ) + + +def make_protocol_transaction( + tx_id: str, + asset: str, + amount: float, + tx_type: protocol_models.TransactionType, + timestamp: datetime.datetime, +) -> protocol_models.Transaction: + return protocol_models.Transaction( + id=tx_id, + timestamp=timestamp, + asset=asset, + amount=amount, + type=tx_type, + ) + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def seed_exchange_config( + account_provider: collection_providers.AccountProvider, + user_id: str, + exchange_config: protocol_models.ExchangeConfig, +) -> None: + account_provider.create_exchange_config(user_id, exchange_config) + + +def seed_account( + account_provider: collection_providers.AccountProvider, + user_id: str, + account: protocol_models.Account, +) -> None: + account_provider.create_account(user_id, account) + + +def seed_trading_state( + trading_provider: collection_providers.AccountTradingProvider, + user_id: str, + account_id: str, + *, + trades: list[protocol_models.Trade] | None = None, + transactions: list[protocol_models.Transaction] | None = None, +) -> None: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + trades=trades or None, + transactions=transactions or None, + ), + ) + trading_provider.save_state(user_id, account_id, trading_state) + + +async def write_daily_prices_cache( + data_root: str, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + prices_by_symbol: dict[str, dict[str, float]], +) -> None: + for symbol, closes_by_timestamp in prices_by_symbol.items(): + await trading_api.merge_daily_prices( + exchange_name, + exchange_type, + sandboxed, + symbol, + closes_by_timestamp, + data_root, + ) + + +async def write_latest_tickers_cache( + data_root: str, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + closes_by_symbol: dict[str, float], +) -> None: + await trading_api.update_latest_tickers( + exchange_name, + exchange_type, + sandboxed, + closes_by_symbol, + data_root, + ) + + +@contextlib.contextmanager +def accounts_history_test_environment(tmp_path): + account_provider = collection_providers.AccountProvider(base_folder=str(tmp_path)) + trading_provider = collection_providers.AccountTradingProvider(base_folder=str(tmp_path)) + + with ( + _patch_wallet(), + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + ): + yield account_provider, trading_provider diff --git a/packages/node/tests/functional_tests/util/dca_workflow.py b/packages/node/tests/functional_tests/util/dca_workflow.py index 5a146f7812..d67f211ea2 100644 --- a/packages/node/tests/functional_tests/util/dca_workflow.py +++ b/packages/node/tests/functional_tests/util/dca_workflow.py @@ -15,6 +15,7 @@ from tests.scheduler.user_actions.user_actions_executor.util import trading_tentacles_test_utils +from . import exchange_account_elements_access as exchange_account_elements_access_module from . import price_mocks as price_mocks_module from . import workflow_common as workflow_common_module @@ -62,18 +63,11 @@ def d_order_price(price: typing.Any) -> decimal.Decimal: def open_order_origins_from_exchange_elements( exchange_account_elements: typing.Any, ) -> list[dict]: - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - return [] - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) + open_orders = exchange_account_elements_access_module.open_orders_from_elements(exchange_account_elements) storage_key = trading_constants_module.STORAGE_ORIGIN_VALUE return [ order_document[storage_key] - for order_document in (open_orders or []) + for order_document in open_orders if isinstance(order_document, dict) and storage_key in order_document ] diff --git a/packages/node/tests/functional_tests/util/exchange_account_elements_access.py b/packages/node/tests/functional_tests/util/exchange_account_elements_access.py new file mode 100644 index 0000000000..ae0ec23807 --- /dev/null +++ b/packages/node/tests/functional_tests/util/exchange_account_elements_access.py @@ -0,0 +1,162 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""Typed accessors for flow exchange account elements in functional tests.""" + +from __future__ import annotations + +import decimal +import enum +import typing + +import octobot_commons.constants as commons_constants +import octobot_flow.entities.accounts.exchange_account_elements as exchange_account_elements_module +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +ExchangeAccountElementsInput = ( + exchange_account_elements_module.ExchangeAccountElements + | dict[str, typing.Any] + | None +) + + +def resolve_exchange_account_elements( + elements: ExchangeAccountElementsInput, +) -> exchange_account_elements_module.ExchangeAccountElements | None: + if elements is None: + return None + if isinstance(elements, dict): + return exchange_account_elements_module.ExchangeAccountElements.from_dict(elements) + return elements + + +def open_orders_from_elements(elements: ExchangeAccountElementsInput) -> list[dict]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return [] + return list(resolved.orders.open_orders or []) + + +def trades_from_elements(elements: ExchangeAccountElementsInput) -> list[dict]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return [] + return list(resolved.trades or []) + + +def portfolio_content_from_elements(elements: ExchangeAccountElementsInput) -> dict[str, typing.Any]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return {} + content = resolved.portfolio.content + return content if isinstance(content, dict) else {} + + +def order_storage_payload(order_row: dict) -> dict: + if not isinstance(order_row, dict): + raise TypeError(f"expected order row dict, got {type(order_row).__name__}") + storage_origin = trading_constants.STORAGE_ORIGIN_VALUE + nested = order_row.get(storage_origin) + if isinstance(nested, dict): + return nested + return order_row + + +def order_column_value(payload: dict, column: str) -> typing.Any: + return payload.get(column) + + +def _decimal_order_price(raw: typing.Union[int, float, str, decimal.Decimal]) -> decimal.Decimal: + if isinstance(raw, decimal.Decimal): + return raw + return decimal.Decimal(str(raw)) + + +def _enum_column_value(raw: typing.Any) -> typing.Any: + if isinstance(raw, enum.Enum): + return raw.value + return raw + + +normalize_order_column_value = _enum_column_value + + +def sorted_open_limit_prices_from_elements( + elements: ExchangeAccountElementsInput, + *, + trade_order_side: trading_enums.TradeOrderSide, +) -> list[decimal.Decimal]: + side_key = trading_enums.ExchangeConstantsOrderColumns.SIDE.value + price_col = trading_enums.ExchangeConstantsOrderColumns.PRICE.value + type_col = trading_enums.ExchangeConstantsOrderColumns.TYPE.value + want_side = trade_order_side.value + limit_type = trading_enums.TradeOrderType.LIMIT.value + prices: list[decimal.Decimal] = [] + for order_row in open_orders_from_elements(elements): + if not isinstance(order_row, dict): + raise TypeError(f"expected open order dict, got {type(order_row).__name__}") + payload = order_storage_payload(order_row) + side = _enum_column_value(order_column_value(payload, side_key)) + if side != want_side: + continue + price_raw = order_column_value(payload, price_col) + if price_raw is None: + continue + order_type = _enum_column_value(order_column_value(payload, type_col)) + if order_type is not None and order_type != limit_type: + continue + prices.append(_decimal_order_price(price_raw)) + prices.sort() + return prices + + +def sorted_open_limit_prices_from_protocol_orders( + orders: typing.Iterable[protocol_models.Order] | None, + *, + trade_order_side: trading_enums.TradeOrderSide, +) -> list[decimal.Decimal]: + if trade_order_side == trading_enums.TradeOrderSide.BUY: + want_side = protocol_models.Side.BUY + else: + want_side = protocol_models.Side.SELL + prices: list[decimal.Decimal] = [] + for order in orders or []: + if not isinstance(order, protocol_models.Order): + raise TypeError(f"expected protocol Order, got {type(order).__name__}") + if order.side != want_side: + continue + if order.type != protocol_models.OrderType.LIMIT: + continue + if order.price is None: + continue + prices.append(_decimal_order_price(order.price)) + prices.sort() + return prices + + +def portfolio_row_scalar(row: typing.Any, field_name: str) -> float: + if not isinstance(row, dict): + raise AssertionError(f"portfolio row must be a dict, got {type(row).__name__}: {row!r}") + if field_name == commons_constants.PORTFOLIO_AVAILABLE: + raw_value = row.get(commons_constants.PORTFOLIO_AVAILABLE) + if raw_value is None: + return portfolio_row_scalar(row, commons_constants.PORTFOLIO_TOTAL) + elif field_name == commons_constants.PORTFOLIO_TOTAL: + raw_value = row.get(commons_constants.PORTFOLIO_TOTAL) + else: + raw_value = row.get(field_name) + if raw_value is None: + raise AssertionError(f"portfolio row missing field {field_name!r}: {row!r}") + return float(raw_value) + + +def portfolio_row_total(row: typing.Any) -> decimal.Decimal: + if not isinstance(row, dict): + raise AssertionError(f"portfolio row must be a dict, got {type(row).__name__}: {row!r}") + raw_value = row.get(commons_constants.PORTFOLIO_TOTAL) + if raw_value is None: + raise AssertionError("portfolio row has no total amount") + if isinstance(raw_value, decimal.Decimal): + return raw_value + return decimal.Decimal(str(raw_value)) diff --git a/packages/node/tests/functional_tests/util/outdated_reference_workflow.py b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py new file mode 100644 index 0000000000..14e40ead65 --- /dev/null +++ b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py @@ -0,0 +1,170 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""Helpers for outdated-reference-account copy automation functional tests.""" + +from __future__ import annotations + +import asyncio +import decimal +import time +import typing + +import dbos +import pytest + +import octobot_commons.timestamp_util as timestamp_util +import octobot_copy.constants as copy_constants +import octobot_flow.entities as octobot_flow_entities +import octobot_node.scheduler.internal_trading_signals as internal_trading_signals_module +import octobot_node.scheduler.workflows_util as workflows_util_module +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums + +from . import exchange_account_elements_access as exchange_account_elements_access_module +from . import grid_workflow as grid_workflow_module +from . import workflow_common as workflow_common_module + +FRESH_MARKET_PRICE = 110_000.0 +STALE_ORDER_PRICE = 67_326.0 +FRESH_ORDER_PRICE = FRESH_MARKET_PRICE * 0.99 +MASTER_STRATEGY_ID = "functional-outdated-reference-master-strategy" +COPY_AUTOMATION_ID = "d4e5f6a7-b8c9-4901-d234-567890abcdef" +COPY_ACCOUNT_ID = "functional_outdated_reference_copy_account" +OUTDATED_SKIP_LOG_SUBSTRING = "Outdated reference account, skipping copy iteration" +_BTC_USDC = "BTC/USDC" + + +def _open_limit_order( + *, + order_id: str, + price: float, + side: protocol_models.Side = protocol_models.Side.SELL, + trigger_above: bool = True, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=_BTC_USDC, + price=price, + quantity=0.001, + filled=0.0, + exchange_id=f"exchange-{order_id}", + side=side, + type=protocol_models.OrderType.LIMIT, + trigger_above=trigger_above, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account(*, orders: list[protocol_models.Order]) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=1_710_000_000.0, + copied_assets=[ + protocol_models.CopiedAsset(name="USDC", total=1000.0, available=1000.0, ratio=0.5), + protocol_models.CopiedAsset(name="BTC", total=0.01, available=0.01, ratio=0.5), + ], + orders=orders, + ) + + +def build_stale_trading_signal() -> octobot_flow_entities.TradingSignal: + return octobot_flow_entities.TradingSignal( + strategy_id=MASTER_STRATEGY_ID, + account=_copied_account( + orders=[_open_limit_order(order_id="stale-order", price=STALE_ORDER_PRICE, trigger_above=True)], + ), + ) + + +def build_fresh_trading_signal() -> octobot_flow_entities.TradingSignal: + return octobot_flow_entities.TradingSignal( + strategy_id=MASTER_STRATEGY_ID, + account=_copied_account( + orders=[_open_limit_order(order_id="fresh-order", price=FRESH_ORDER_PRICE, trigger_above=True)], + ), + ) + + +def build_create_copy_follower_user_action() -> protocol_models.UserAction: + return grid_workflow_module.build_create_copy_follower_user_action( + automation_id=COPY_AUTOMATION_ID, + account_id=COPY_ACCOUNT_ID, + name="test_outdated_reference_copy_follower", + strategy_id=MASTER_STRATEGY_ID, + ) + + +def caplog_contains_outdated_skip(caplog) -> bool: + return any( + OUTDATED_SKIP_LOG_SUBSTRING in record.getMessage() + for record in caplog.records + ) + + +async def deliver_trading_signal_when_pending( + scheduler: typing.Any, + automation_id: str, + trading_signal: octobot_flow_entities.TradingSignal, + deadline_seconds: float, +) -> None: + poll_interval = 0.05 + poll_deadline = time.monotonic() + deadline_seconds + while time.monotonic() < poll_deadline: + pending_rows = await scheduler.INSTANCE.list_workflows_async( + status=[ + dbos.WorkflowStatusString.ENQUEUED.value, + dbos.WorkflowStatusString.PENDING.value, + ], + ) + for workflow_row in pending_rows: + if workflows_util_module.get_automation_id(workflow_row) != automation_id: + continue + copied_strategy_ids = workflows_util_module.get_automation_copied_strategy_ids(workflow_row) + if trading_signal.strategy_id not in copied_strategy_ids: + continue + await internal_trading_signals_module.send_internal_trading_signal(trading_signal) + return + await asyncio.sleep(poll_interval) + pytest.fail( + f"Timed out delivering trading signal for automation_id={automation_id!r} " + f"strategy_id={trading_signal.strategy_id!r}" + ) + + +def sell_limit_prices_from_reader(state_reader: typing.Any) -> list[decimal.Decimal]: + return exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + state_reader.state.automation.exchange_account_elements, + trade_order_side=trading_enums.TradeOrderSide.SELL, + ) + + +async def poll_state_reader_until( + scheduler: typing.Any, + automation_id: str, + predicate: typing.Callable[[typing.Any], bool], + deadline_seconds: float, + failure_label: str, +) -> typing.Any: + poll_interval = workflow_common_module.DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS + poll_deadline = time.monotonic() + deadline_seconds + last_reader: typing.Any = None + while time.monotonic() < poll_deadline: + workflow_rows = await scheduler.INSTANCE.list_workflows_async() + for workflow_row in workflow_rows: + if workflows_util_module.get_automation_id(workflow_row) != automation_id: + continue + state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + if state_reader is None: + continue + last_reader = state_reader + if predicate(state_reader): + return state_reader + await asyncio.sleep(poll_interval) + detail = "no reader" + if last_reader is not None: + sell_prices = sell_limit_prices_from_reader(last_reader) + detail = f"last sell limit prices={sell_prices!s}" + pytest.fail(f"Timed out waiting for {failure_label} ({detail})") diff --git a/packages/node/tests/functional_tests/util/protocol_assertions.py b/packages/node/tests/functional_tests/util/protocol_assertions.py index e5b2fd1dea..ff7d9cbff4 100644 --- a/packages/node/tests/functional_tests/util/protocol_assertions.py +++ b/packages/node/tests/functional_tests/util/protocol_assertions.py @@ -7,8 +7,10 @@ import math import typing +import octobot_commons.constants as commons_constants_module import octobot_protocol.models as protocol_models_module +from . import exchange_account_elements_access as exchange_account_elements_access_module from . import workflow_common as workflow_common_module buy_sell_trade_counts_from_exchange_elements = ( @@ -16,30 +18,6 @@ ) -def portfolio_content_from_exchange_elements(exchange_account_elements: typing.Any) -> dict[str, typing.Any]: - portfolio = getattr(exchange_account_elements, "portfolio", None) - if portfolio is None and isinstance(exchange_account_elements, dict): - portfolio = exchange_account_elements.get("portfolio") - if portfolio is None: - return {} - content = getattr(portfolio, "content", None) - if content is None and isinstance(portfolio, dict): - content = portfolio.get("content") - return content if isinstance(content, dict) else {} - - -def portfolio_row_scalar(row: typing.Any, field_name: str) -> float: - if isinstance(row, dict): - raw_value = row.get(field_name) - else: - raw_value = getattr(row, field_name, None) - if raw_value is None and field_name == "available": - return portfolio_row_scalar(row, "total") - if raw_value is None: - raise AssertionError(f"portfolio row missing field {field_name!r}: {row!r}") - return float(raw_value) - - def assert_protocol_automation_metadata_name( protocol_automation: protocol_models_module.AutomationState, expected_name: str, @@ -85,7 +63,9 @@ def assert_protocol_automation_matches_exchange_account_elements( assert order_summary.symbol == expected_order_symbol, ( f"unexpected OrderSummary.symbol {order_summary.symbol!r}; expected {expected_order_symbol!r}" ) - content = portfolio_content_from_exchange_elements(exchange_account_elements) + content = exchange_account_elements_access_module.portfolio_content_from_elements( + exchange_account_elements, + ) protocol_assets = protocol_automation.assets assert protocol_assets is not None, ( "expected AutomationState.assets to be set" @@ -97,8 +77,12 @@ def assert_protocol_automation_matches_exchange_account_elements( f"missing protocol Asset for portfolio symbol {symbol!r}; " f"protocol asset symbols: {sorted(assets_by_symbol)!r}" ) - expected_total = portfolio_row_scalar(row, "total") - expected_available = portfolio_row_scalar(row, "available") + expected_total = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_TOTAL, + ) + expected_available = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_AVAILABLE, + ) if not math.isclose(float(matching_asset.total), expected_total, rel_tol=1e-9, abs_tol=1e-6): raise AssertionError( f"Asset.total mismatch for {symbol!r}: protocol={matching_asset.total!r} " @@ -153,7 +137,9 @@ def assert_protocol_automation_matches_exchange_account_elements_multi_symbol( f"unexpected OrderSummary.symbol {order_summary.symbol!r}; " f"expected one of {allowed_order_symbols!r}" ) - content = portfolio_content_from_exchange_elements(exchange_account_elements) + content = exchange_account_elements_access_module.portfolio_content_from_elements( + exchange_account_elements, + ) protocol_assets = protocol_automation.assets assert protocol_assets is not None, "expected AutomationState.assets to be set" assets_by_symbol = {asset.symbol: asset for asset in protocol_assets} @@ -163,8 +149,12 @@ def assert_protocol_automation_matches_exchange_account_elements_multi_symbol( f"missing protocol Asset for portfolio symbol {symbol!r}; " f"protocol asset symbols: {sorted(assets_by_symbol)!r}" ) - expected_total = portfolio_row_scalar(row, "total") - expected_available = portfolio_row_scalar(row, "available") + expected_total = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_TOTAL, + ) + expected_available = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_AVAILABLE, + ) if not math.isclose(float(matching_asset.total), expected_total, rel_tol=1e-9, abs_tol=1e-6): raise AssertionError( f"Asset.total mismatch for {symbol!r}: protocol={matching_asset.total!r} " diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index 21d2193320..885901dab1 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -19,7 +19,6 @@ import octobot_sync.constants as sync_constants_module import octobot_sync.server as sync_server_module import octobot_sync.sync.collection_providers as collection_providers_module -import octobot_trading.constants as trading_constants_module import octobot_trading.enums as trading_enums_module import octobot_node.constants as node_constants_module @@ -27,10 +26,13 @@ import octobot_node.scheduler import octobot_node.scheduler.workflows import octobot_node.scheduler.api as scheduler_api_module +import octobot_node.scheduler.tasks as scheduler_tasks_module import octobot_node.scheduler.workflows.params as workflow_params_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_protocol.models as protocol_models_module +from . import exchange_account_elements_access as exchange_account_elements_access_module + # Passphrase for grid functional tests (WalletBackend requires length >= 8). SIMULATOR_GRID_TEST_WALLET_PASSPHRASE = "simgridPW1!" @@ -208,41 +210,29 @@ def job_description_dict_from_output( def buy_sell_trade_counts_from_exchange_elements( exchange_account_elements: typing.Any, ) -> tuple[int, int, int]: - if exchange_account_elements is None: + resolved = exchange_account_elements_access_module.resolve_exchange_account_elements( + exchange_account_elements, + ) + if resolved is None: return 0, 0, 0 - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - trades_only = getattr(exchange_account_elements, "trades", None) - if trades_only is None and isinstance(exchange_account_elements, dict): - trades_only = exchange_account_elements.get("trades", []) - return 0, 0, len(trades_only or []) - - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) - open_orders = open_orders or [] + open_orders = exchange_account_elements_access_module.open_orders_from_elements(resolved) side_key = trading_enums_module.ExchangeConstantsOrderColumns.SIDE.value - storage_key = trading_constants_module.STORAGE_ORIGIN_VALUE buy_count = 0 sell_count = 0 - for order in open_orders: - if isinstance(order, dict): - inner = order.get(storage_key, {}) - else: - inner = getattr(order, storage_key, {}) - side = inner.get(side_key) if isinstance(inner, dict) else getattr(inner, side_key, None) + for order_row in open_orders: + if not isinstance(order_row, dict): + raise TypeError(f"expected open order dict, got {type(order_row).__name__}") + inner = exchange_account_elements_access_module.order_storage_payload(order_row) + side = exchange_account_elements_access_module.normalize_order_column_value( + exchange_account_elements_access_module.order_column_value(inner, side_key), + ) if side == trading_enums_module.TradeOrderSide.BUY.value: buy_count += 1 elif side == trading_enums_module.TradeOrderSide.SELL.value: sell_count += 1 - trades = getattr(exchange_account_elements, "trades", None) - if trades is None and isinstance(exchange_account_elements, dict): - trades = exchange_account_elements.get("trades", []) - trade_count = len(trades or []) + trade_count = len(exchange_account_elements_access_module.trades_from_elements(resolved)) return buy_count, sell_count, trade_count @@ -483,8 +473,6 @@ async def enqueue_user_action_workflow_and_await_terminal_result( user_id: str, ): """``execute_user_action`` queues user actions; wait until the USER_ACTION_QUEUE workflow completes.""" - import octobot_node.scheduler.tasks as scheduler_tasks_module - workflow_identifier_encoded = await scheduler_tasks_module.trigger_user_action_workflow( user_action_bundle, user_id, diff --git a/packages/node/tests/protocol/test_accounts_history_compute.py b/packages/node/tests/protocol/test_accounts_history_compute.py new file mode 100644 index 0000000000..935480d576 --- /dev/null +++ b/packages/node/tests/protocol/test_accounts_history_compute.py @@ -0,0 +1,86 @@ +import datetime +import decimal +import json +import os +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + +import octobot_node.protocol.accounts_history as accounts_history_module + + +def _make_account( + account_id: str, + assets: dict[str, float], + exchange: str = "binance", + sandboxed: bool = False, +) -> protocol_models.Account: + detailed_assets = [ + protocol_models.DetailedAsset(symbol=symbol, total=amount, available=amount) + for symbol, amount in assets.items() + ] + exchange_config = protocol_models.ExchangeConfig( + id="cfg1", name="test", exchange=exchange, sandboxed=sandboxed, + historical_trade_symbols=["BTC/USDT"], + ) + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote1", + exchange_config_ids=["cfg1"], + ) + return protocol_models.Account( + id=account_id, + name="Test", + is_simulated=False, + created_at=datetime.datetime.now(datetime.timezone.utc), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=detailed_assets, + ) + ], + ) + + +class TestPortfolioFromAccountAssets: + def test_converts_assets_to_portfolio_dict(self): + account = _make_account("a1", {"BTC": 1.5, "USDT": 10000.0}) + portfolio = accounts_history_module._portfolio_from_account_assets(account) + assert portfolio["BTC"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("1.5") + assert portfolio["USDT"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("10000.0") + + def test_empty_assets(self): + account = mock.MagicMock(spec=protocol_models.Account) + account.assets = None + portfolio = accounts_history_module._portfolio_from_account_assets(account) + assert portfolio == {} + + +class TestComputePortfolioHistoricalValues: + @pytest.mark.asyncio + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + async def test_empty_trading_returns_empty_history(self, mock_trading_provider, mock_account_provider, tmp_path): + account = _make_account("a1", {"BTC": 1.0}) + mock_account_provider.instance.return_value.get_account.return_value = account + mock_account_provider.instance.return_value.get_exchange_config.return_value = protocol_models.ExchangeConfig( + id="cfg1", name="test", exchange="binance", sandboxed=False, + historical_trade_symbols=["BTC/USDT"], + ) + + trading_state = mock.MagicMock() + trading_state.account_trading.trades = [] + trading_state.account_trading.transactions = [] + mock_trading_provider.instance.return_value.load_state.return_value = trading_state + + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + "user1", "a1", + ) + assert result.version == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + # No trades/transactions → no history values. + if result.history is not None: + assert result.history.values is None or len(result.history.values) == 0 diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index 2825fdf5a2..e8830819ab 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -92,6 +92,7 @@ def _ensure_scheduler_queues() -> None: octobot_node_enums_module.SchedulerQueues.USER_ACTION_QUEUE.value: "USER_ACTION_QUEUE", octobot_node_enums_module.SchedulerQueues.DBOS_CLEANUP_QUEUE.value: "DBOS_CLEANUP_QUEUE", octobot_node_enums_module.SchedulerQueues.GLOBAL_VIEW_QUEUE.value: "GLOBAL_VIEW_QUEUE", + octobot_node_enums_module.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value: "PORTFOLIO_HISTORY_QUEUE", } if all(queue_name in registry.queue_info_map for queue_name in queue_bindings): for queue_name, scheduler_attribute in queue_bindings.items(): @@ -117,6 +118,7 @@ def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE = None octobot_node.scheduler.SCHEDULER.DBOS_CLEANUP_QUEUE = None octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE = None + octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE = None def init_scheduler(db_file_name: str, application_version: str | None = None): diff --git a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py index 32f0628479..c22ea60202 100644 --- a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py +++ b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py @@ -7,7 +7,6 @@ import octobot_flow.entities import octobot_protocol.models as protocol_models -import octobot_sync.constants as sync_constants from tests.scheduler import temp_dbos_scheduler @@ -35,7 +34,7 @@ async def test_refreshes_all_wallets_and_accounts_in_parallel_per_wallet( wallet_ids = ["wallet-a"] accounts = [mock.Mock(id="acc-1"), mock.Mock(id="acc-2")] account_provider = mock.Mock() - account_provider.list_registered_wallet_ids.return_value = wallet_ids + account_provider.list_collectable_wallet_ids.return_value = wallet_ids account_provider.list_items.return_value = accounts refresh_mock = mock.AsyncMock(return_value=True) with mock.patch.object( @@ -134,24 +133,10 @@ def global_view_workflow_module(self, temp_dbos_scheduler): async def test_logs_success_summary_after_refresh(self, global_view_workflow_module): account = _exchange_account() - evaluation_time = datetime.datetime(2026, 8, 11, 18, 30, tzinfo=datetime.UTC) refresh_result = octobot_flow.entities.GlobalViewAccountRefreshResult( updated_account=account, changed_order_ids={"gone-order-1"}, open_orders=[{"exchange_id": "stays-order-2"}, {"exchange_id": "stays-order-3"}], - portfolio_history_state=protocol_models.PortfolioHistoricalValuesState( - version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, - history=protocol_models.PortfolioHistoricalValues( - unit="USDC", - values=[ - protocol_models.PortfolioHistoricalValue( - timestamp=evaluation_time, - total=1500.5, - assets=[], - ) - ], - ), - ), ) mock_logger = mock.Mock() with ( @@ -199,6 +184,6 @@ async def test_logs_success_summary_after_refresh(self, global_view_workflow_mod 2, 1, True, - "USDC", - "1500.5", + "n/a", + "n/a", ) diff --git a/packages/node/tests/scheduler/portfolio_history/__init__.py b/packages/node/tests/scheduler/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py new file mode 100644 index 0000000000..35ec681508 --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py @@ -0,0 +1,36 @@ +import mock +import pytest + +import octobot_node.scheduler.portfolio_history.portfolio_history_executor as portfolio_history_executor_module + + +class TestRunPortfolioHistoryCollectionAccountFilter: + @pytest.mark.asyncio + @mock.patch.object( + portfolio_history_executor_module, + "_build_context_for_account", + return_value=None, + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_filters_accounts_when_account_ids_set( + self, mock_account_provider, mock_build_context, + ): + account_one = mock.MagicMock() + account_one.id = "acc-1" + account_two = mock.MagicMock() + account_two.id = "acc-2" + mock_account_provider.instance.return_value.list_items.return_value = [ + account_one, + account_two, + ] + + results = await portfolio_history_executor_module.run_portfolio_history_collection( + "wallet-user-1", + account_ids=["acc-2"], + ) + + assert results == [] + mock_account_provider.instance.return_value.list_items.assert_called_once_with( + "wallet-user-1", + ) + mock_build_context.assert_called_once_with("wallet-user-1", account_two) diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py new file mode 100644 index 0000000000..58ff4f2f7e --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py @@ -0,0 +1,108 @@ +import datetime +import mock +import pytest + +import octobot_node.scheduler.workflows.params as workflow_params_module + +from tests.scheduler import temp_dbos_scheduler + + +@pytest.fixture +def portfolio_history_workflow_module(temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow + return portfolio_history_workflow + + +class TestPortfolioHistoryScheduleInput: + def test_get_schedule_input_enables_catch_up_once_on_startup(self, portfolio_history_workflow_module): + schedule_input = portfolio_history_workflow_module.get_schedule_input() + assert schedule_input["automatic_backfill"] is False + assert schedule_input["catch_up_once_on_startup"] is True + assert schedule_input["queue_name"] == "portfolio_history_queue" + + +class TestPortfolioHistoryWorkflow: + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_iterates_wallets_and_delegates( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.return_value = [ + "wallet1", "wallet2", + ] + + mock_result = mock.MagicMock() + mock_result.skipped = False + mock_result.error = None + mock_executor.run_portfolio_history_collection = mock.AsyncMock(return_value=[mock_result]) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + assert mock_executor.run_portfolio_history_collection.call_count == 2 + assert result["succeeded"] == 2 + assert result["failed"] == 0 + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_error_isolation_per_wallet( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.return_value = [ + "wallet1", "wallet2", + ] + mock_executor.run_portfolio_history_collection = mock.AsyncMock( + side_effect=[Exception("fail"), [mock.MagicMock(skipped=False, error=None)]] + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + # wallet1 failed, wallet2 succeeded. + assert result["succeeded"] == 1 + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.logger") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_top_level_exception_is_logged_and_does_not_propagate( + self, mock_account_provider, mock_logger, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.side_effect = ( + AttributeError("missing method") + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + + mock_logger.exception.assert_called_once() + assert result == {"succeeded": 0, "failed": 0, "skipped": 0} + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_uses_wallet_whitelist_from_params_without_listing_all_wallets( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_result = mock.MagicMock() + mock_result.skipped = False + mock_result.error = None + mock_executor.run_portfolio_history_collection = mock.AsyncMock(return_value=[mock_result]) + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc), + collection_params, + ) + + mock_account_provider.instance.return_value.list_collectable_wallet_ids.assert_not_called() + mock_executor.run_portfolio_history_collection.assert_awaited_once_with( + "wallet-user-1", + account_ids=["acc-1"], + ) + assert result["succeeded"] == 1 diff --git a/packages/node/tests/scheduler/test_schedules.py b/packages/node/tests/scheduler/test_schedules.py index b6c4a95996..108c177553 100644 --- a/packages/node/tests/scheduler/test_schedules.py +++ b/packages/node/tests/scheduler/test_schedules.py @@ -63,18 +63,72 @@ def _matching_existing_global_view_schedule(global_view_workflow_module) -> dict } +def _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) -> dict: + return portfolio_history_workflow_module.get_schedule_input() + + +def _matching_existing_portfolio_history_schedule(portfolio_history_workflow_module) -> dict: + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + return { + "schedule_id": "existing-portfolio-history-schedule-id", + "schedule_name": portfolio_history_workflow_module.SCHEDULE_NAME, + "workflow_name": "ignored-workflow-name", + "workflow_class_name": None, + "schedule": schedule_input["schedule"], + "status": "ACTIVE", + "context": None, + "last_fired_at": "2026-07-13T00:00:00+00:00", + "automatic_backfill": schedule_input.get("automatic_backfill", False), + "catch_up_once_on_startup": schedule_input.get("catch_up_once_on_startup", False), + "cron_timezone": schedule_input.get("cron_timezone"), + "queue_name": schedule_input.get("queue_name"), + } + + +def _success_workflow_status(): + success_status = mock.Mock(spec=dbos.WorkflowStatus) + success_status.status = dbos.WorkflowStatusString.SUCCESS.value + return success_status + + +def _enqueued_workflow_status(): + enqueued_status = mock.Mock(spec=dbos.WorkflowStatus) + enqueued_status.status = dbos.WorkflowStatusString.ENQUEUED.value + return enqueued_status + + +def _workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=None, +): + portfolio_prefix = f"sched-{portfolio_history_workflow_module.SCHEDULE_NAME}-" + + async def get_workflow_status_async(workflow_id: str): + if workflow_id.startswith(portfolio_prefix): + return _success_workflow_status() + if inner_side_effect is None: + return None + return await inner_side_effect(workflow_id) + + return get_workflow_status_async + + def _get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, *, cleanup_existing: dict | None, global_view_existing: dict | None, + portfolio_history_existing: dict | None, ): async def get_schedule_async(schedule_name: str): if schedule_name == dbos_cleanup_workflow_module.SCHEDULE_NAME: return cleanup_existing if schedule_name == global_view_workflow_module.SCHEDULE_NAME: return global_view_existing + if schedule_name == portfolio_history_workflow_module.SCHEDULE_NAME: + return portfolio_history_existing return None return get_schedule_async @@ -144,6 +198,33 @@ def test_returns_false_when_automatic_backfill_differs(self, temp_dbos_scheduler schedule_input, ) is False + def test_returns_false_when_catch_up_once_on_startup_differs(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + existing_schedule = _matching_existing_portfolio_history_schedule(portfolio_history_workflow_module) + existing_schedule["catch_up_once_on_startup"] = False + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is False + + +class TestGetLatestScheduleTriggerTimeBefore: + def test_daily_cron_returns_latest_slot_before_now(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + before = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + trigger_time = schedules_module._get_latest_schedule_trigger_time_before( + schedule_input, + before, + ) + assert trigger_time == datetime.datetime(2026, 7, 15, 3, 0, 0, tzinfo=datetime.timezone.utc) + class TestGetBackfillScheduleDefaultAnchor: def test_returns_utc_now_minus_one_day(self, temp_dbos_scheduler): @@ -229,16 +310,26 @@ def global_view_workflow_module(self, temp_dbos_scheduler): yield global_view_workflow_module_loaded + @pytest.fixture + def portfolio_history_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + yield portfolio_history_workflow_module_loaded + async def test_creates_schedule_when_missing( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) global_view_schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + portfolio_history_schedule_input = _configured_portfolio_history_schedule_input( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( @@ -249,8 +340,10 @@ async def test_creates_schedule_when_missing( side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=None, global_view_existing=None, + portfolio_history_existing=None, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -268,7 +361,7 @@ async def test_creates_schedule_when_missing( ) as backfill_to_thread_mock: await schedules_module.register_schedules(temp_dbos_scheduler) - assert create_schedule_mock.await_count == 2 + assert create_schedule_mock.await_count == 3 create_schedule_mock.assert_any_await( schedule_name=cleanup_schedule_input["schedule_name"], workflow_fn=cleanup_schedule_input["workflow_fn"], @@ -287,6 +380,15 @@ async def test_creates_schedule_when_missing( cron_timezone=global_view_schedule_input.get("cron_timezone"), queue_name=global_view_schedule_input.get("queue_name"), ) + create_schedule_mock.assert_any_await( + schedule_name=portfolio_history_schedule_input["schedule_name"], + workflow_fn=portfolio_history_schedule_input["workflow_fn"], + schedule=portfolio_history_schedule_input["schedule"], + context=portfolio_history_schedule_input.get("context"), + automatic_backfill=portfolio_history_schedule_input.get("automatic_backfill", False), + cron_timezone=portfolio_history_schedule_input.get("cron_timezone"), + queue_name=portfolio_history_schedule_input.get("queue_name"), + ) apply_schedules_mock.assert_not_awaited() backfill_to_thread_mock.assert_not_awaited() mock_logger.info.assert_any_call( @@ -299,11 +401,17 @@ async def test_creates_schedule_when_missing( global_view_schedule_input["schedule_name"], global_view_schedule_input["schedule"], ) + mock_logger.info.assert_any_call( + "Creating schedule %s (%s)", + portfolio_history_schedule_input["schedule_name"], + portfolio_history_schedule_input["schedule"], + ) async def test_keeps_schedule_when_config_matches( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -317,6 +425,9 @@ async def test_keeps_schedule_when_config_matches( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( @@ -327,8 +438,10 @@ async def test_keeps_schedule_when_config_matches( side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_cleanup_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -341,6 +454,11 @@ async def test_keeps_schedule_when_config_matches( "apply_schedules_async", new_callable=mock.AsyncMock, ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), + ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, ) as backfill_to_thread_mock: @@ -364,6 +482,7 @@ async def test_recreates_schedule_when_config_differs( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -377,6 +496,9 @@ async def test_recreates_schedule_when_config_differs( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( @@ -387,8 +509,10 @@ async def test_recreates_schedule_when_config_differs( side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_cleanup_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -401,6 +525,11 @@ async def test_recreates_schedule_when_config_differs( "apply_schedules_async", new_callable=mock.AsyncMock, ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), + ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, ) as backfill_to_thread_mock: @@ -424,6 +553,7 @@ async def test_backfills_when_last_fired_at_is_null( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -436,6 +566,9 @@ async def test_backfills_when_last_fired_at_is_null( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) workflow_id = ( @@ -459,8 +592,10 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -484,8 +619,9 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - new_callable=mock.AsyncMock, - return_value=None, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", side_effect=backfill_to_thread_side_effect, @@ -522,6 +658,7 @@ async def test_skips_backfill_when_all_window_slots_terminal( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -534,6 +671,9 @@ async def test_skips_backfill_when_all_window_slots_terminal( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) backfill_end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) @@ -559,8 +699,10 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -584,7 +726,10 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - side_effect=get_workflow_status_side_effect, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=get_workflow_status_side_effect, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, @@ -610,6 +755,7 @@ async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_windo self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -623,6 +769,9 @@ async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_windo existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None existing_schedule["schedule"] = "0 * * * *" anchor = datetime.datetime(2026, 7, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) @@ -660,8 +809,10 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -685,7 +836,10 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - side_effect=get_workflow_status_side_effect, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=get_workflow_status_side_effect, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", side_effect=backfill_to_thread_side_effect, @@ -718,6 +872,7 @@ async def test_skips_backfill_when_last_fired_at_set( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -730,6 +885,9 @@ async def test_skips_backfill_when_last_fired_at_set( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", @@ -739,8 +897,10 @@ async def test_skips_backfill_when_last_fired_at_set( side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", @@ -752,6 +912,11 @@ async def test_skips_backfill_when_last_fired_at_set( temp_dbos_scheduler.INSTANCE, "apply_schedules_async", new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, @@ -764,6 +929,7 @@ async def test_skips_backfill_when_automatic_backfill_false( self, dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -778,6 +944,9 @@ async def test_skips_backfill_when_automatic_backfill_false( existing_global_view_schedule = _matching_existing_global_view_schedule( global_view_workflow_module, ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None with mock.patch( @@ -788,12 +957,18 @@ async def test_skips_backfill_when_automatic_backfill_false( side_effect=_get_schedule_async_side_effect( dbos_cleanup_workflow_module, global_view_workflow_module, + portfolio_history_workflow_module, cleanup_existing=existing_schedule, global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_success_workflow_status(), ), mock.patch( "octobot_node.scheduler.schedules._get_logger", return_value=mock.Mock(), @@ -808,3 +983,217 @@ async def test_skips_backfill_when_automatic_backfill_false( await schedules_module.register_schedules(temp_dbos_scheduler) backfill_to_thread_mock.assert_not_awaited() + + +class TestMaybeCatchUpScheduleOnceOnStartup: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def portfolio_history_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + yield portfolio_history_workflow_module_loaded + + async def test_backfills_latest_slot_when_missing(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + trigger_time = datetime.datetime(2026, 7, 15, 3, 0, 0, tzinfo=datetime.timezone.utc) + workflow_id = schedules_module.build_scheduled_workflow_id( + portfolio_history_workflow_module.SCHEDULE_NAME, + trigger_time, + ) + mock_handle = mock.Mock() + mock_handle.get_workflow_id.return_value = workflow_id + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + async def backfill_to_thread_side_effect(func, *args, **kwargs): + return func(*args, **kwargs) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=None, + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + side_effect=backfill_to_thread_side_effect, + ) as backfill_to_thread_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.backfill_schedule", + return_value=[mock_handle], + ) as backfill_schedule_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_awaited_once() + backfill_schedule_mock.assert_called_once() + schedule_name, backfill_start, backfill_end = backfill_schedule_mock.call_args[0] + assert schedule_name == portfolio_history_workflow_module.SCHEDULE_NAME + assert backfill_start == trigger_time - datetime.timedelta(seconds=1) + assert backfill_end == trigger_time + datetime.timedelta(seconds=1) + + async def test_skips_when_latest_slot_terminal(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_success_workflow_status(), + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_not_awaited() + + async def test_skips_when_latest_slot_in_progress(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_enqueued_workflow_status(), + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_not_awaited() + + async def test_skips_when_flag_disabled(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + schedule_input["catch_up_once_on_startup"] = False + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + ) as get_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + get_schedule_mock.assert_not_awaited() + backfill_to_thread_mock.assert_not_awaited() + + +class TestUpdateCleanupScheduleCron: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module_loaded + + yield dbos_cleanup_workflow_module_loaded + + async def test_applies_schedule_when_cron_changes( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + desired_cron = "0 */6 * * *" + schedule_input = dbos_cleanup_workflow_module.get_schedule_input(cron=desired_cron) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value={"schedule": "0 0 * * *"}, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + result = await schedules_module.update_cleanup_schedule_cron( + temp_dbos_scheduler, + desired_cron, + ) + + assert result == {"changed": True, "cron": desired_cron} + apply_schedules_mock.assert_awaited_once_with([schedule_input]) + + async def test_skips_apply_when_cron_unchanged( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + desired_cron = dbos_cleanup_workflow_module.SCHEDULE_CRON + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value={"schedule": desired_cron}, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + result = await schedules_module.update_cleanup_schedule_cron( + temp_dbos_scheduler, + desired_cron, + ) + + assert result == {"changed": False, "cron": desired_cron} + apply_schedules_mock.assert_not_awaited() diff --git a/packages/node/tests/scheduler/test_tasks.py b/packages/node/tests/scheduler/test_tasks.py index 9f6bdc7a02..42cbc4c0c8 100644 --- a/packages/node/tests/scheduler/test_tasks.py +++ b/packages/node/tests/scheduler/test_tasks.py @@ -189,6 +189,55 @@ async def test_enqueues_execute_user_action_workflow_with_encoded_inputs(self, t assert enqueue_keyword_arguments["inputs"] == expected_inputs_encoded +class TestTriggerPortfolioHistoryCollection: + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await octobot_node.scheduler.tasks.trigger_portfolio_history_collection() + + @pytest.mark.asyncio + async def test_enqueues_portfolio_history_collection_on_portfolio_history_queue(self, temp_dbos_scheduler): + import datetime + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + expected_workflow_id = "portfolio-history-workflow-test-id" + scheduled_time = datetime.datetime(2026, 3, 15, 12, 0, 0, tzinfo=datetime.UTC) + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + with ( + mock.patch( + "octobot_commons.timestamp_util.utc_now_datetime", + return_value=scheduled_time, + ), + mock.patch.object( + temp_dbos_scheduler.PORTFOLIO_HISTORY_QUEUE, + "enqueue_async", + mock.AsyncMock(), + ) as mock_enqueue_async_operation, + ): + mock_workflow_enqueue_handle = mock.Mock() + mock_workflow_enqueue_handle.workflow_id = expected_workflow_id + mock_enqueue_async_operation.return_value = mock_workflow_enqueue_handle + + enqueue_function_result = await octobot_node.scheduler.tasks.trigger_portfolio_history_collection( + collection_params, + ) + + assert enqueue_function_result == expected_workflow_id + mock_enqueue_async_operation.assert_awaited_once() + positional_workflow_targets, enqueue_keyword_arguments = mock_enqueue_async_operation.call_args + assert ( + positional_workflow_targets[0] + is portfolio_history_workflow_module_loaded.PortfolioHistoryWorkflow.portfolio_history_collection + ) + assert positional_workflow_targets[1] == scheduled_time + assert positional_workflow_targets[2] == collection_params.to_dict(include_default_values=False) + assert enqueue_keyword_arguments == {} + + class TestSendToActiveAutomationWorkflow: _TEST_WALLET_ADDRESS = "0xaaabbbcccddd" _TEST_PARENT_AUTOMATION_ID = "00000000-0000-4000-8000-000000000099" diff --git a/packages/node/tests/scheduler/test_user_action_executor_factory.py b/packages/node/tests/scheduler/test_user_action_executor_factory.py index 302c762ab0..73f0c2ec25 100644 --- a/packages/node/tests/scheduler/test_user_action_executor_factory.py +++ b/packages/node/tests/scheduler/test_user_action_executor_factory.py @@ -153,6 +153,17 @@ def test_returns_refresh_accounts_executor_class(self): resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) assert resolved_executor_cls is user_actions_executor_package.RefreshAccountsActionExecutor + def test_returns_update_historical_exchanges_data_executor_class(self): + configuration_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action_model = self._user_action( + action_identifier="ua-update-historical", + configuration_inner=configuration_inner, + ) + resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) + assert resolved_executor_cls is user_actions_executor_package.UpdateHistoricalExchangesDataActionExecutor + def test_returns_create_exchange_config_executor_class(self): configuration_inner = protocol_models.CreateExchangeConfigConfiguration( action_type=protocol_models.UserActionType.EXCHANGE_CONFIG_CREATE, diff --git a/packages/node/tests/scheduler/test_workflows_retention.py b/packages/node/tests/scheduler/test_workflows_retention.py index e992ad0d06..ba7c8b6aa9 100644 --- a/packages/node/tests/scheduler/test_workflows_retention.py +++ b/packages/node/tests/scheduler/test_workflows_retention.py @@ -1,4 +1,4 @@ -# Drakkar-Software OctoBot-Node +# Drakkar-Software OctoBot-Node # Copyright (c) 2025 Drakkar-Software, All rights reserved. import datetime @@ -18,6 +18,8 @@ _AUTOMATION_WORKFLOW_NAME = "execute_automation" _DBOS_CLEANUP_WORKFLOW_NAME = "dbos_cleanup" +_GLOBAL_VIEW_WORKFLOW_NAME = "global_view_refresh" +_PORTFOLIO_HISTORY_WORKFLOW_NAME = "portfolio_history_collection" _PARENT_WORKFLOW_ID_A = "741ce171-dac9-40be-83dc-b443c0eaf0e2" _PARENT_WORKFLOW_ID_B = "852df282-edb0-51cf-94ed-c554d1fbf1f3" @@ -407,7 +409,7 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s sched = scheduler_module.Scheduler() mock_instance = mock.Mock() mock_instance.list_workflows_async = mock.AsyncMock( - side_effect=[automation_workflows, cleanup_workflows] + side_effect=[automation_workflows, cleanup_workflows, [], []] ) mock_instance.delete_workflows_async = mock.AsyncMock() mock_engine = mock.Mock() @@ -431,6 +433,8 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s assert summary == { "deleted_by_automation": {_PARENT_WORKFLOW_ID_A: 1}, "deleted_cleanup_executions": 1, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 2, } mock_instance.delete_workflows_async.assert_awaited_once_with( @@ -440,12 +444,14 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s ], delete_children=False, ) - mock_connection.execute.assert_called_once() mock_logger.info.assert_any_call( - "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs, " + "%s global view runs, %s portfolio history runs", 2, 1, 1, + 0, + 0, ) mock_logger.info.assert_any_call("DBOS cleanup summary: %s", summary) @@ -453,7 +459,7 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_scheduler): sched = scheduler_module.Scheduler() mock_instance = mock.Mock() - mock_instance.list_workflows_async = mock.AsyncMock(side_effect=[[], []]) + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=[[], [], [], []]) mock_instance.delete_workflows_async = mock.AsyncMock() mock_instance._sys_db.engine = mock.Mock() sched.INSTANCE = mock_instance @@ -468,6 +474,8 @@ async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_sc assert summary == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } mock_instance.delete_workflows_async.assert_not_called() @@ -475,6 +483,7 @@ async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_sc mock_logger.info.assert_called_once_with("DBOS cleanup summary: %s", summary) + class TestShouldSkipRetentionCleanupForScheduledTime: @pytest.mark.asyncio async def test_returns_true_when_scheduler_not_initialized(self, temp_dbos_scheduler): @@ -593,3 +602,240 @@ async def test_returns_false_when_latest_terminal_cleanup_is_older_than_backfill ) assert result is False + + +class TestGetOutdatedTerminalWorkflowIds: + def test_deletes_old_terminal_workflows(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id="global-view-old", + updated_at=old_updated_at, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + _workflow_status_row( + workflow_id="global-view-recent", + updated_at=now_ms - 1, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_terminal_workflow_ids( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == ["global-view-old"] + + def test_skips_non_terminal_workflows(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id="portfolio-history-pending", + updated_at=old_updated_at, + status=dbos.WorkflowStatusString.PENDING.value, + name=_PORTFOLIO_HISTORY_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_terminal_workflow_ids( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == [] + + +class TestSelectCleanupCronForDatabaseSize: + def test_returns_daily_cron_when_size_unknown(self): + assert workflows_retention.select_cleanup_cron_for_database_size(None) == workflows_retention.DBOS_CLEANUP_CRON_DAILY + + def test_returns_daily_cron_below_first_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES - 1 + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_DAILY + + def test_returns_12h_cron_between_tiers(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_12H + + def test_returns_6h_cron_at_second_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_6H + + +class TestGetSchedulerDatabaseSizeBytes: + def test_sums_sqlite_file_and_sidecars(self, tmp_path): + sqlite_path = tmp_path / "tasks.db" + sqlite_path.write_bytes(b"x" * 10) + wal_path = tmp_path / "tasks.db-wal" + wal_path.write_bytes(b"y" * 5) + mock_dbos_instance = mock.Mock() + + with mock.patch( + "octobot_node.config.settings.SCHEDULER_SQLITE_FILE", + str(sqlite_path), + ), mock.patch( + "octobot_node.config.settings.SCHEDULER_POSTGRES_URL", + None, + ): + size_bytes = workflows_retention.get_scheduler_database_size_bytes(mock_dbos_instance) + + assert size_bytes == 15 + + def test_returns_none_when_sqlite_file_missing(self): + mock_dbos_instance = mock.Mock() + mock_logger = mock.Mock() + with mock.patch( + "octobot_node.config.settings.SCHEDULER_SQLITE_FILE", + "missing-tasks.db", + ), mock.patch( + "octobot_node.config.settings.SCHEDULER_POSTGRES_URL", + None, + ), mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + size_bytes = workflows_retention.get_scheduler_database_size_bytes(mock_dbos_instance) + + assert size_bytes is None + mock_logger.exception.assert_called_once() + + +class TestCleanupOutdatedAutomationExecutionsScheduledWorkflows: + @pytest.mark.asyncio + async def test_deletes_outdated_global_view_and_portfolio_history_workflows(self, temp_dbos_scheduler): + now_ms = 10_000_000 + retention_seconds = 100.0 + cutoff_ms = now_ms - int(retention_seconds * 1000) + global_view_workflows = [ + _workflow_status_row( + workflow_id="global-view-old", + updated_at=cutoff_ms - 1, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + ] + portfolio_history_workflows = [ + _workflow_status_row( + workflow_id="portfolio-history-old", + updated_at=cutoff_ms - 1, + name=_PORTFOLIO_HISTORY_WORKFLOW_NAME, + ), + ] + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock( + side_effect=[[], [], global_view_workflows, portfolio_history_workflows] + ) + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch.object( + workflows_retention, + "AUTOMATION_EXECUTION_RETENTION_SECONDS", + retention_seconds, + ): + summary = await workflows_retention.cleanup_outdated_automation_executions(sched) + + assert summary["deleted_global_view_executions"] == 1 + assert summary["deleted_portfolio_history_executions"] == 1 + assert summary["total_deleted"] == 2 + mock_instance.delete_workflows_async.assert_awaited_once_with( + ["global-view-old", "portfolio-history-old"], + delete_children=False, + ) + + +class TestFinalizeDbosCleanupRun: + @pytest.mark.asyncio + async def test_adds_size_and_updates_schedule(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + sched.INSTANCE = mock_instance + cleanup_summary = { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, + "total_deleted": 0, + } + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": True, "cron": workflows_retention.DBOS_CLEANUP_CRON_12H}), + ) as update_schedule_mock: + summary = await workflows_retention.finalize_dbos_cleanup_run(sched, cleanup_summary) + + assert summary["database_size_bytes"] == workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES + assert summary["cleanup_schedule_cron"] == workflows_retention.DBOS_CLEANUP_CRON_12H + assert summary["cleanup_schedule_updated"] is True + update_schedule_mock.assert_awaited_once_with( + sched, + workflows_retention.DBOS_CLEANUP_CRON_12H, + ) + + @pytest.mark.asyncio + async def test_vacuums_when_workflows_were_deleted(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=0, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": False, "cron": workflows_retention.DBOS_CLEANUP_CRON_DAILY}), + ): + await workflows_retention.finalize_dbos_cleanup_run( + sched, + {"total_deleted": 3}, + ) + + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" + + @pytest.mark.asyncio + async def test_vacuums_when_large_database_and_nothing_deleted(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": False, "cron": workflows_retention.DBOS_CLEANUP_CRON_6H}), + ): + await workflows_retention.finalize_dbos_cleanup_run( + sched, + {"total_deleted": 0}, + ) + + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" diff --git a/packages/node/tests/scheduler/user_actions/test_user_action_util.py b/packages/node/tests/scheduler/user_actions/test_user_action_util.py index 9e2fe98159..fcb5a77468 100644 --- a/packages/node/tests/scheduler/user_actions/test_user_action_util.py +++ b/packages/node/tests/scheduler/user_actions/test_user_action_util.py @@ -234,6 +234,16 @@ def test_account_auth_actions_return_account_auth(self, action_type): == protocol_models.UserActionResultType.ACCOUNT_AUTH ) + def test_update_historical_exchanges_data_returns_account(self): + configuration_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = _user_action_with_configuration(configuration_inner) + assert ( + user_action_util.resolve_user_action_result_type(user_action) + == protocol_models.UserActionResultType.ACCOUNT + ) + class TestBuildSynthesizedFailureUserActionResult: def test_automation_result(self): diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py new file mode 100644 index 0000000000..42a41cfb0d --- /dev/null +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py @@ -0,0 +1,207 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors + +from .account import account_executor_test_utils +from . import provider_assertions + +import octobot_node.errors as node_errors + +import octobot_node.scheduler.tasks as scheduler_tasks_module +import octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data as update_historical_exchanges_data_executor +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_action_post_actions as user_action_post_actions_module + +from tests.scheduler import temp_dbos_scheduler + + +class TestUpdateHistoricalExchangesDataActionExecutorExecute: + @pytest.mark.asyncio + async def test_sets_portfolio_history_collection_params_and_completes(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = protocol_models.UserAction( + id="ua-update-historical", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + collection_params = executor.post_actions.portfolio_history_collection_params + assert collection_params is not None + assert collection_params.wallet_ids == [account_executor_test_utils.WALLET_ADDRESS] + assert collection_params.account_ids is None + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="account", + expect_error_details=False, + ) + + @pytest.mark.asyncio + async def test_validates_account_ids_belong_to_user(self): + owned_account = account_executor_test_utils.minimal_exchange_account(account_id="acc-owned") + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + account_ids=["acc-owned"], + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-scoped", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + provider_mock = mock.Mock() + provider_mock.get_item.return_value = owned_account + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + provider_mock.get_item.assert_called_once_with( + account_executor_test_utils.WALLET_ADDRESS, + "acc-owned", + ) + collection_params = executor.post_actions.portfolio_history_collection_params + assert collection_params is not None + assert collection_params.account_ids == ["acc-owned"] + + @pytest.mark.asyncio + async def test_raises_when_account_id_not_owned(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + account_ids=["foreign-acc"], + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-foreign", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + provider_mock = mock.Mock() + provider_mock.get_item.side_effect = collection_errors.ItemNotFoundError("missing") + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(collection_errors.ItemNotFoundError): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.ACCOUNT_NOT_FOUND, + ) + + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-no-scheduler", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=False, + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INTERNAL_ERROR, + ) + + @pytest.mark.asyncio + async def test_raises_when_payload_type_is_wrong(self): + inner = protocol_models.DeleteAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_DELETE, + id="acc-1", + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-wrong", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(node_errors.InvalidUserActionPayloadError, match="UpdateHistoricalExchangesDataConfiguration"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + +class TestUpdateHistoricalExchangesDataAfterUserActionExecution: + @pytest.mark.asyncio + async def test_triggers_portfolio_history_collection_when_post_action_set(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.user_action_workflow as user_action_workflow_module + + trigger_mock = mock.AsyncMock(return_value="portfolio-history-workflow-id") + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + execution_result = workflow_params_module.UserActionExecutionResult( + updated_user_action=protocol_models.UserAction(id="ua-update-historical"), + post_actions=user_action_post_actions_module.UserActionPostActions( + portfolio_history_collection_params=collection_params, + ), + ) + with mock.patch.object( + scheduler_tasks_module, + "trigger_portfolio_history_collection", + trigger_mock, + ): + await user_action_workflow_module.UserActionWorkflow.after_user_action_execution(execution_result) + trigger_mock.assert_awaited_once_with(collection_params) diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index 4a320c48df..e8b295b00e 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -32,6 +32,7 @@ import octobot_commons.cryptography import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors import octobot_protocol.models as protocol_models import octobot_node.config import octobot_node.constants @@ -101,7 +102,10 @@ def _automation_state_dict_with_scheduled_to( def _octobot_actions_job_mock_class_pending_priority_skipped( *, automation_inner_state: dict[str, typing.Any], - skip_error: "octobot_flow.errors.PendingPriorityActionsSkippedError", + skip_error: typing.Union[ + "octobot_flow.errors.PendingPriorityActionsSkippedError", + copy_errors.OutdatedReferenceAccountError, + ], ) -> mock.Mock: async def run_raises(*args, **kwargs): raise skip_error @@ -1087,6 +1091,151 @@ async def test_postponed_log_uses_none_error_fields(self, import_automation_work ) +class TestExecuteIterationOutdatedReferenceAccountError: + @pytest.mark.asyncio + @required_imports + async def test_postpones_iteration_at_scheduled_to_without_degraded_state( + self, import_automation_workflow, task + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=outdated_error, + ) + fixed_now = 1000.0 + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: + result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + persist_account_trading_mock.assert_not_called() + parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) + assert parsed_progress_status.postponed_iteration is True + assert parsed_progress_status.next_step_at == scheduled_to + assert parsed_progress_status.error is None + assert parsed_progress_status.error_message is None + assert result["has_next_actions"] is True + assert result["next_iteration_description"] == task_content + next_iteration_description = json.loads(result["next_iteration_description"]) + execution = next_iteration_description["state"]["automation"].get("execution", {}) + assert "degraded_state" not in execution + + @pytest.mark.asyncio + @required_imports + async def test_logs_outdated_reference_account_info(self, import_automation_workflow, task): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=outdated_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + await automation_workflow.execute_iteration(inputs, None) + + mock_logger.info.assert_any_call( + f"Outdated reference account, skipping copy iteration: {outdated_error}" + ) + mock_logger.error.assert_not_called() + + @pytest.mark.parametrize( + "skip_error,expected_log_method,expected_log_message", + [ + pytest.param( + octobot_flow.errors.PendingPriorityActionsSkippedError("stale priority skipped"), + "error", + "Pending priority actions were skipped: stale priority skipped", + id="pending_priority_skipped", + ), + pytest.param( + copy_errors.OutdatedReferenceAccountError("reference account is outdated"), + "info", + "Outdated reference account, skipping copy iteration: reference account is outdated", + id="outdated_reference_account", + ), + ], + ) + @pytest.mark.asyncio + @required_imports + async def test_shared_skip_handler_postpones_without_degraded_state( + self, + import_automation_workflow, + task, + skip_error, + expected_log_method, + expected_log_message, + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + result = await automation_workflow.execute_iteration(inputs, None) + + if expected_log_method == "error": + getattr(mock_logger, expected_log_method).assert_called_once_with(expected_log_message) + else: + getattr(mock_logger, expected_log_method).assert_any_call(expected_log_message) + parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) + assert parsed_progress_status.postponed_iteration is True + assert parsed_progress_status.error is None + + +class TestShouldRetryOutdatedReferenceAccountError: + def test_should_retry_returns_false_for_outdated_reference_account_error( + self, import_automation_workflow + ): + assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_retry( + copy_errors.OutdatedReferenceAccountError("reference account is outdated") + ) is False + + class TestExecuteAutomationPostponedIteration: @pytest.mark.asyncio @required_imports diff --git a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py index f49e052ffa..c0802bbf7f 100644 --- a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py @@ -35,13 +35,17 @@ async def test_delegates_to_cleanup_outdated_automation_executions(self, dbos_cl ), mock.patch( "octobot_node.scheduler.workflows_retention.cleanup_outdated_automation_executions", mock.AsyncMock(return_value=expected_summary), - ) as cleanup_mock: + ) as cleanup_mock, mock.patch( + "octobot_node.scheduler.workflows_retention.finalize_dbos_cleanup_run", + mock.AsyncMock(return_value={**expected_summary, "database_size_bytes": 1024, "cleanup_schedule_cron": "0 0 * * *", "cleanup_schedule_updated": False}), + ) as finalize_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( datetime.datetime.now(datetime.timezone.utc), ) cleanup_mock.assert_awaited_once() - assert result == expected_summary + finalize_mock.assert_awaited_once() + assert result["total_deleted"] == 3 @pytest.mark.asyncio async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module): @@ -69,6 +73,8 @@ async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module assert result == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -101,6 +107,8 @@ async def test_skips_cleanup_when_newer_execution_already_ran(self, dbos_cleanup assert result == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -120,3 +128,10 @@ def test_returns_daily_schedule_input(self, temp_dbos_scheduler): "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, } + def test_accepts_custom_cron(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = dbos_cleanup_workflow_module.get_schedule_input(cron="0 */6 * * *") + + assert schedule_input["schedule"] == "0 */6 * * *" + diff --git a/packages/protocol/.openapi-generator/FILES b/packages/protocol/.openapi-generator/FILES index de6cd26a7a..260ff92df9 100644 --- a/packages/protocol/.openapi-generator/FILES +++ b/packages/protocol/.openapi-generator/FILES @@ -67,6 +67,7 @@ docs/ExchangeAccount.md docs/ExchangeConfig.md docs/ExchangeConfigActionResult.md docs/ExchangeConfigActionResultErrorMessage.md +docs/Fee.md docs/GenericAccount.md docs/GenericProcessConfiguration.md docs/GenericWorkflowConfiguration.md @@ -118,6 +119,9 @@ docs/TradingTentaclesConfiguration.md docs/TradingType.md docs/TrailingProfile.md docs/TrailingProfileType.md +docs/Transaction.md +docs/TransactionType.md +docs/UpdateHistoricalExchangesDataConfiguration.md docs/UserAction.md docs/UserActionConfiguration.md docs/UserActionResult.md @@ -199,6 +203,7 @@ octobot_protocol/models/exchange_account.py octobot_protocol/models/exchange_config.py octobot_protocol/models/exchange_config_action_result.py octobot_protocol/models/exchange_config_action_result_error_message.py +octobot_protocol/models/fee.py octobot_protocol/models/generic_account.py octobot_protocol/models/generic_process_configuration.py octobot_protocol/models/generic_workflow_configuration.py @@ -250,6 +255,9 @@ octobot_protocol/models/trading_tentacles_configuration.py octobot_protocol/models/trading_type.py octobot_protocol/models/trailing_profile.py octobot_protocol/models/trailing_profile_type.py +octobot_protocol/models/transaction.py +octobot_protocol/models/transaction_type.py +octobot_protocol/models/update_historical_exchanges_data_configuration.py octobot_protocol/models/user_action.py octobot_protocol/models/user_action_configuration.py octobot_protocol/models/user_action_result.py @@ -329,6 +337,7 @@ test/test_exchange_account.py test/test_exchange_config.py test/test_exchange_config_action_result.py test/test_exchange_config_action_result_error_message.py +test/test_fee.py test/test_generic_account.py test/test_generic_process_configuration.py test/test_generic_workflow_configuration.py @@ -380,6 +389,9 @@ test/test_trading_tentacles_configuration.py test/test_trading_type.py test/test_trailing_profile.py test/test_trailing_profile_type.py +test/test_transaction.py +test/test_transaction_type.py +test/test_update_historical_exchanges_data_configuration.py test/test_user_action.py test/test_user_action_configuration.py test/test_user_action_result.py diff --git a/packages/protocol/docs/AccountTrading.md b/packages/protocol/docs/AccountTrading.md index 44050729ca..45cc3fcddd 100644 --- a/packages/protocol/docs/AccountTrading.md +++ b/packages/protocol/docs/AccountTrading.md @@ -10,6 +10,7 @@ Name | Type | Description | Notes **orders** | [**List[Order]**](Order.md) | | [optional] **trades** | [**List[Trade]**](Trade.md) | | [optional] **positions** | [**List[Position]**](Position.md) | | [optional] +**transactions** | [**List[Transaction]**](Transaction.md) | | [optional] ## Example diff --git a/packages/protocol/docs/ExchangeConfig.md b/packages/protocol/docs/ExchangeConfig.md index cd6ce7a34f..ed57508def 100644 --- a/packages/protocol/docs/ExchangeConfig.md +++ b/packages/protocol/docs/ExchangeConfig.md @@ -11,6 +11,7 @@ Name | Type | Description | Notes **exchange** | **str** | | **sandboxed** | **bool** | | [default to False] **url** | **str** | | [optional] +**historical_trade_symbols** | **List[str]** | | [optional] ## Example diff --git a/packages/protocol/docs/Fee.md b/packages/protocol/docs/Fee.md new file mode 100644 index 0000000000..c1e39e0c61 --- /dev/null +++ b/packages/protocol/docs/Fee.md @@ -0,0 +1,31 @@ +# Fee + +TradeFee + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**amount** | **float** | | +**currency** | **str** | | + +## Example + +```python +from octobot_protocol.models.fee import Fee + +# TODO update the JSON string below +json = "{}" +# create an instance of Fee from a JSON string +fee_instance = Fee.from_json(json) +# print the JSON string representation of the object +print(Fee.to_json()) + +# convert the object into a dict +fee_dict = fee_instance.to_dict() +# create an instance of Fee from a dict +fee_from_dict = Fee.from_dict(fee_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/Trade.md b/packages/protocol/docs/Trade.md index 3e60e90eec..9c4d371846 100644 --- a/packages/protocol/docs/Trade.md +++ b/packages/protocol/docs/Trade.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **side** | [**Side**](Side.md) | | **quantity** | **float** | | **price** | **float** | | +**fee** | [**Fee**](Fee.md) | | [optional] **status** | [**OrderStatus**](OrderStatus.md) | | **executed_at** | **datetime** | | diff --git a/packages/protocol/docs/Transaction.md b/packages/protocol/docs/Transaction.md new file mode 100644 index 0000000000..e968030b81 --- /dev/null +++ b/packages/protocol/docs/Transaction.md @@ -0,0 +1,34 @@ +# Transaction + +Transaction + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**id** | **str** | | +**timestamp** | **datetime** | | +**asset** | **str** | | +**amount** | **float** | | +**type** | [**TransactionType**](TransactionType.md) | | + +## Example + +```python +from octobot_protocol.models.transaction import Transaction + +# TODO update the JSON string below +json = "{}" +# create an instance of Transaction from a JSON string +transaction_instance = Transaction.from_json(json) +# print the JSON string representation of the object +print(Transaction.to_json()) + +# convert the object into a dict +transaction_dict = transaction_instance.to_dict() +# create an instance of Transaction from a dict +transaction_from_dict = Transaction.from_dict(transaction_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/TransactionType.md b/packages/protocol/docs/TransactionType.md new file mode 100644 index 0000000000..a8f42a33eb --- /dev/null +++ b/packages/protocol/docs/TransactionType.md @@ -0,0 +1,17 @@ +# TransactionType + +TransactionType + +## Enum + +* `BLOCKCHAIN_DEPOSIT` (value: `'blockchain_deposit'`) + +* `BLOCKCHAIN_WITHDRAWAL` (value: `'blockchain_withdrawal'`) + +* `FUNDING_FEE` (value: `'funding_fee'`) + +* `TRADING_FEE` (value: `'trading_fee'`) + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md b/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md new file mode 100644 index 0000000000..50cd8fd6bd --- /dev/null +++ b/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md @@ -0,0 +1,31 @@ +# UpdateHistoricalExchangesDataConfiguration + +UpdateHistoricalExchangesDataConfiguration + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**action_type** | [**UserActionType**](UserActionType.md) | update_historical_exchanges_data | +**account_ids** | **List[str]** | | [optional] + +## Example + +```python +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration + +# TODO update the JSON string below +json = "{}" +# create an instance of UpdateHistoricalExchangesDataConfiguration from a JSON string +update_historical_exchanges_data_configuration_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json) +# print the JSON string representation of the object +print(UpdateHistoricalExchangesDataConfiguration.to_json()) + +# convert the object into a dict +update_historical_exchanges_data_configuration_dict = update_historical_exchanges_data_configuration_instance.to_dict() +# create an instance of UpdateHistoricalExchangesDataConfiguration from a dict +update_historical_exchanges_data_configuration_from_dict = UpdateHistoricalExchangesDataConfiguration.from_dict(update_historical_exchanges_data_configuration_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UserActionType.md b/packages/protocol/docs/UserActionType.md index 8d10bdec95..ee0504784d 100644 --- a/packages/protocol/docs/UserActionType.md +++ b/packages/protocol/docs/UserActionType.md @@ -28,6 +28,8 @@ UserActionType * `EXCHANGE_CONFIG_DELETE` (value: `'exchange_config_delete'`) +* `UPDATE_HISTORICAL_EXCHANGES_DATA` (value: `'update_historical_exchanges_data'`) + * `STRATEGY_CREATE` (value: `'strategy_create'`) * `STRATEGY_EDIT` (value: `'strategy_edit'`) diff --git a/packages/protocol/octobot_protocol/models/__init__.py b/packages/protocol/octobot_protocol/models/__init__.py index ab3b57bf78..6500eceee6 100644 --- a/packages/protocol/octobot_protocol/models/__init__.py +++ b/packages/protocol/octobot_protocol/models/__init__.py @@ -82,6 +82,7 @@ from octobot_protocol.models.exchange_config import ExchangeConfig from octobot_protocol.models.exchange_config_action_result import ExchangeConfigActionResult from octobot_protocol.models.exchange_config_action_result_error_message import ExchangeConfigActionResultErrorMessage +from octobot_protocol.models.fee import Fee from octobot_protocol.models.generic_account import GenericAccount from octobot_protocol.models.generic_process_configuration import GenericProcessConfiguration from octobot_protocol.models.generic_workflow_configuration import GenericWorkflowConfiguration @@ -133,6 +134,9 @@ from octobot_protocol.models.trading_type import TradingType from octobot_protocol.models.trailing_profile import TrailingProfile from octobot_protocol.models.trailing_profile_type import TrailingProfileType +from octobot_protocol.models.transaction import Transaction +from octobot_protocol.models.transaction_type import TransactionType +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration from octobot_protocol.models.user_action import UserAction from octobot_protocol.models.user_action_configuration import UserActionConfiguration from octobot_protocol.models.user_action_result import UserActionResult diff --git a/packages/protocol/octobot_protocol/models/account_trading.py b/packages/protocol/octobot_protocol/models/account_trading.py index ddc5ab2253..74f2e3b854 100644 --- a/packages/protocol/octobot_protocol/models/account_trading.py +++ b/packages/protocol/octobot_protocol/models/account_trading.py @@ -23,6 +23,7 @@ from octobot_protocol.models.order import Order from octobot_protocol.models.position import Position from octobot_protocol.models.trade import Trade +from octobot_protocol.models.transaction import Transaction from typing import Optional, Set from typing_extensions import Self from pydantic_core import to_jsonable_python @@ -35,7 +36,8 @@ class AccountTrading(BaseModel): orders: Optional[List[Order]] = None trades: Optional[List[Trade]] = None positions: Optional[List[Position]] = None - __properties: ClassVar[List[str]] = ["updated_at", "orders", "trades", "positions"] + transactions: Optional[List[Transaction]] = None + __properties: ClassVar[List[str]] = ["updated_at", "orders", "trades", "positions", "transactions"] model_config = ConfigDict( validate_by_name=True, @@ -97,6 +99,13 @@ def to_dict(self) -> Dict[str, Any]: if _item_positions: _items.append(_item_positions.to_dict()) _dict['positions'] = _items + # override the default output from pydantic by calling `to_dict()` of each item in transactions (list) + _items = [] + if self.transactions: + for _item_transactions in self.transactions: + if _item_transactions: + _items.append(_item_transactions.to_dict()) + _dict['transactions'] = _items return _dict @classmethod @@ -112,7 +121,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "updated_at": obj.get("updated_at"), "orders": [Order.from_dict(_item) for _item in obj["orders"]] if obj.get("orders") is not None else None, "trades": [Trade.from_dict(_item) for _item in obj["trades"]] if obj.get("trades") is not None else None, - "positions": [Position.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None + "positions": [Position.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None, + "transactions": [Transaction.from_dict(_item) for _item in obj["transactions"]] if obj.get("transactions") is not None else None }) return _obj diff --git a/packages/protocol/octobot_protocol/models/exchange_config.py b/packages/protocol/octobot_protocol/models/exchange_config.py index 6c585bec66..8d260a5004 100644 --- a/packages/protocol/octobot_protocol/models/exchange_config.py +++ b/packages/protocol/octobot_protocol/models/exchange_config.py @@ -32,7 +32,8 @@ class ExchangeConfig(BaseModel): exchange: StrictStr sandboxed: StrictBool url: Optional[StrictStr] = None - __properties: ClassVar[List[str]] = ["id", "name", "exchange", "sandboxed", "url"] + historical_trade_symbols: Optional[List[StrictStr]] = None + __properties: ClassVar[List[str]] = ["id", "name", "exchange", "sandboxed", "url", "historical_trade_symbols"] model_config = ConfigDict( validate_by_name=True, @@ -89,7 +90,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "name": obj.get("name"), "exchange": obj.get("exchange"), "sandboxed": obj.get("sandboxed") if obj.get("sandboxed") is not None else False, - "url": obj.get("url") + "url": obj.get("url"), + "historical_trade_symbols": obj.get("historical_trade_symbols") }) return _obj diff --git a/packages/protocol/octobot_protocol/models/fee.py b/packages/protocol/octobot_protocol/models/fee.py new file mode 100644 index 0000000000..eddb354b42 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/fee.py @@ -0,0 +1,90 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Union +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class Fee(BaseModel): + """ + TradeFee + """ # noqa: E501 + amount: Union[StrictFloat, StrictInt] + currency: StrictStr + __properties: ClassVar[List[str]] = ["amount", "currency"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of Fee from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of Fee from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "amount": obj.get("amount"), + "currency": obj.get("currency") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/trade.py b/packages/protocol/octobot_protocol/models/trade.py index e85839b871..ae9fa6df55 100644 --- a/packages/protocol/octobot_protocol/models/trade.py +++ b/packages/protocol/octobot_protocol/models/trade.py @@ -19,7 +19,8 @@ from datetime import datetime from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr -from typing import Any, ClassVar, Dict, List, Union +from typing import Any, ClassVar, Dict, List, Optional, Union +from octobot_protocol.models.fee import Fee from octobot_protocol.models.order_status import OrderStatus from octobot_protocol.models.order_type import OrderType from octobot_protocol.models.side import Side @@ -38,9 +39,10 @@ class Trade(BaseModel): side: Side quantity: Union[StrictFloat, StrictInt] price: Union[StrictFloat, StrictInt] + fee: Optional[Fee] = None status: OrderStatus executed_at: datetime - __properties: ClassVar[List[str]] = ["id", "trade_id", "type", "symbol", "side", "quantity", "price", "status", "executed_at"] + __properties: ClassVar[List[str]] = ["id", "trade_id", "type", "symbol", "side", "quantity", "price", "fee", "status", "executed_at"] model_config = ConfigDict( validate_by_name=True, @@ -81,6 +83,9 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) + # override the default output from pydantic by calling `to_dict()` of fee + if self.fee: + _dict['fee'] = self.fee.to_dict() return _dict @classmethod @@ -100,6 +105,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "side": obj.get("side"), "quantity": obj.get("quantity"), "price": obj.get("price"), + "fee": Fee.from_dict(obj["fee"]) if obj.get("fee") is not None else None, "status": obj.get("status"), "executed_at": obj.get("executed_at") }) diff --git a/packages/protocol/octobot_protocol/models/transaction.py b/packages/protocol/octobot_protocol/models/transaction.py new file mode 100644 index 0000000000..8cf4e95cda --- /dev/null +++ b/packages/protocol/octobot_protocol/models/transaction.py @@ -0,0 +1,98 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from datetime import datetime +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Union +from octobot_protocol.models.transaction_type import TransactionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class Transaction(BaseModel): + """ + Transaction + """ # noqa: E501 + id: StrictStr + timestamp: datetime + asset: StrictStr + amount: Union[StrictFloat, StrictInt] + type: TransactionType + __properties: ClassVar[List[str]] = ["id", "timestamp", "asset", "amount", "type"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of Transaction from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of Transaction from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "id": obj.get("id"), + "timestamp": obj.get("timestamp"), + "asset": obj.get("asset"), + "amount": obj.get("amount"), + "type": obj.get("type") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/transaction_type.py b/packages/protocol/octobot_protocol/models/transaction_type.py new file mode 100644 index 0000000000..c8dfd6849c --- /dev/null +++ b/packages/protocol/octobot_protocol/models/transaction_type.py @@ -0,0 +1,39 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import json +from enum import Enum +from typing_extensions import Self + + +class TransactionType(str, Enum): + """ + TransactionType + """ + + """ + allowed enum values + """ + BLOCKCHAIN_DEPOSIT = 'blockchain_deposit' + BLOCKCHAIN_WITHDRAWAL = 'blockchain_withdrawal' + FUNDING_FEE = 'funding_fee' + TRADING_FEE = 'trading_fee' + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of TransactionType from a JSON string""" + return cls(json.loads(json_str)) + + diff --git a/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py b/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py new file mode 100644 index 0000000000..d5b901bc13 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py @@ -0,0 +1,91 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List, Optional +from octobot_protocol.models.user_action_type import UserActionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class UpdateHistoricalExchangesDataConfiguration(BaseModel): + """ + UpdateHistoricalExchangesDataConfiguration + """ # noqa: E501 + action_type: UserActionType = Field(description="update_historical_exchanges_data") + account_ids: Optional[List[StrictStr]] = None + __properties: ClassVar[List[str]] = ["action_type", "account_ids"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of UpdateHistoricalExchangesDataConfiguration from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of UpdateHistoricalExchangesDataConfiguration from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "action_type": obj.get("action_type"), + "account_ids": obj.get("account_ids") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/user_action_configuration.py b/packages/protocol/octobot_protocol/models/user_action_configuration.py index f4f6866acc..1cf85ac022 100644 --- a/packages/protocol/octobot_protocol/models/user_action_configuration.py +++ b/packages/protocol/octobot_protocol/models/user_action_configuration.py @@ -35,11 +35,12 @@ from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.stop_automation_configuration import StopAutomationConfiguration +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration from pydantic import StrictStr, Field from typing import Union, List, Set, Optional, Dict from typing_extensions import Literal, Self -USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] +USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration", "UpdateHistoricalExchangesDataConfiguration"] class UserActionConfiguration(BaseModel): """ @@ -67,22 +68,24 @@ class UserActionConfiguration(BaseModel): oneof_schema_10_validator: Optional[EditExchangeConfigConfiguration] = None # data type: DeleteExchangeConfigConfiguration oneof_schema_11_validator: Optional[DeleteExchangeConfigConfiguration] = None + # data type: UpdateHistoricalExchangesDataConfiguration + oneof_schema_12_validator: Optional[UpdateHistoricalExchangesDataConfiguration] = None # data type: RefreshAccountsConfiguration - oneof_schema_12_validator: Optional[RefreshAccountsConfiguration] = None + oneof_schema_13_validator: Optional[RefreshAccountsConfiguration] = None # data type: CreateStrategyConfiguration - oneof_schema_13_validator: Optional[CreateStrategyConfiguration] = None + oneof_schema_14_validator: Optional[CreateStrategyConfiguration] = None # data type: EditStrategyConfiguration - oneof_schema_14_validator: Optional[EditStrategyConfiguration] = None + oneof_schema_15_validator: Optional[EditStrategyConfiguration] = None # data type: DeleteStrategyConfiguration - oneof_schema_15_validator: Optional[DeleteStrategyConfiguration] = None + oneof_schema_16_validator: Optional[DeleteStrategyConfiguration] = None # data type: CreateAccountAuthConfiguration - oneof_schema_16_validator: Optional[CreateAccountAuthConfiguration] = None + oneof_schema_17_validator: Optional[CreateAccountAuthConfiguration] = None # data type: EditAccountAuthConfiguration - oneof_schema_17_validator: Optional[EditAccountAuthConfiguration] = None + oneof_schema_18_validator: Optional[EditAccountAuthConfiguration] = None # data type: DeleteAccountAuthConfiguration - oneof_schema_18_validator: Optional[DeleteAccountAuthConfiguration] = None - actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None - one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } + oneof_schema_19_validator: Optional[DeleteAccountAuthConfiguration] = None + actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration]] = None + one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration", "UpdateHistoricalExchangesDataConfiguration" } model_config = ConfigDict( validate_assignment=True, @@ -163,6 +166,11 @@ def actual_instance_must_validate_oneof(cls, v): error_messages.append(f"Error! Input type `{type(v)}` is not `DeleteExchangeConfigConfiguration`") else: match += 1 + # validate data type: UpdateHistoricalExchangesDataConfiguration + if not isinstance(v, UpdateHistoricalExchangesDataConfiguration): + error_messages.append(f"Error! Input type `{type(v)}` is not `UpdateHistoricalExchangesDataConfiguration`") + else: + match += 1 # validate data type: RefreshAccountsConfiguration if not isinstance(v, RefreshAccountsConfiguration): error_messages.append(f"Error! Input type `{type(v)}` is not `RefreshAccountsConfiguration`") @@ -200,10 +208,10 @@ def actual_instance_must_validate_oneof(cls, v): match += 1 if match > 1: # more than 1 match - raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) else: return v @@ -313,6 +321,11 @@ def from_json(cls, json_str: str) -> Self: instance.actual_instance = EditStrategyConfiguration.from_json(json_str) return instance + # check if data type is `UpdateHistoricalExchangesDataConfiguration` + if _data_type == "update_historical_exchanges_data": + instance.actual_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json_str) + return instance + # deserialize data into CreateAutomationConfiguration try: instance.actual_instance = CreateAutomationConfiguration.from_json(json_str) @@ -379,6 +392,12 @@ def from_json(cls, json_str: str) -> Self: match += 1 except (ValidationError, ValueError) as e: error_messages.append(str(e)) + # deserialize data into UpdateHistoricalExchangesDataConfiguration + try: + instance.actual_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) # deserialize data into RefreshAccountsConfiguration try: instance.actual_instance = RefreshAccountsConfiguration.from_json(json_str) @@ -424,10 +443,10 @@ def from_json(cls, json_str: str) -> Self: if match > 1: # more than 1 match - raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) else: return instance @@ -441,7 +460,7 @@ def to_json(self) -> str: else: return json.dumps(self.actual_instance) - def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: + def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration]]: """Returns the dict representation of the actual instance""" if self.actual_instance is None: return None diff --git a/packages/protocol/octobot_protocol/models/user_action_type.py b/packages/protocol/octobot_protocol/models/user_action_type.py index 8836a794d0..59f1f40158 100644 --- a/packages/protocol/octobot_protocol/models/user_action_type.py +++ b/packages/protocol/octobot_protocol/models/user_action_type.py @@ -38,6 +38,7 @@ class UserActionType(str, Enum): EXCHANGE_CONFIG_CREATE = 'exchange_config_create' EXCHANGE_CONFIG_EDIT = 'exchange_config_edit' EXCHANGE_CONFIG_DELETE = 'exchange_config_delete' + UPDATE_HISTORICAL_EXCHANGES_DATA = 'update_historical_exchanges_data' STRATEGY_CREATE = 'strategy_create' STRATEGY_EDIT = 'strategy_edit' STRATEGY_DELETE = 'strategy_delete' diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index d61609cce9..832523d5f7 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -279,6 +279,22 @@ } } }, + "Fee": { + "description": "TradeFee", + "type": "object", + "required": [ + "amount", + "currency" + ], + "properties": { + "amount": { + "type": "number" + }, + "currency": { + "type": "string" + } + } + }, "Trade": { "description": "Trade", "type": "object", @@ -315,6 +331,9 @@ "price": { "type": "number" }, + "fee": { + "$ref": "#/components/schemas/Fee" + }, "status": { "$ref": "#/components/schemas/OrderStatus" }, @@ -818,6 +837,45 @@ } } }, + "TransactionType": { + "description": "TransactionType", + "type": "string", + "enum": [ + "blockchain_deposit", + "blockchain_withdrawal", + "funding_fee", + "trading_fee" + ] + }, + "Transaction": { + "description": "Transaction", + "type": "object", + "required": [ + "id", + "timestamp", + "asset", + "amount", + "type" + ], + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "asset": { + "type": "string" + }, + "amount": { + "type": "number" + }, + "type": { + "$ref": "#/components/schemas/TransactionType" + } + } + }, "AccountTrading": { "description": "AccountTrading", "type": "object", @@ -846,6 +904,12 @@ "items": { "$ref": "#/components/schemas/Position" } + }, + "transactions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Transaction" + } } } }, @@ -874,6 +938,12 @@ }, "url": { "type": "string" + }, + "historical_trade_symbols": { + "type": "array", + "items": { + "type": "string" + } } } }, @@ -1926,6 +1996,25 @@ } } }, + "UpdateHistoricalExchangesDataConfiguration": { + "description": "UpdateHistoricalExchangesDataConfiguration", + "type": "object", + "required": [ + "action_type" + ], + "properties": { + "action_type": { + "$ref": "#/components/schemas/UserActionType", + "description": "update_historical_exchanges_data" + }, + "account_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, "CreateExchangeConfigConfiguration": { "description": "CreateExchangeConfigConfiguration", "type": "object", @@ -2159,6 +2248,7 @@ "exchange_config_create", "exchange_config_edit", "exchange_config_delete", + "update_historical_exchanges_data", "strategy_create", "strategy_edit", "strategy_delete", diff --git a/packages/protocol/test/test_account_trading.py b/packages/protocol/test/test_account_trading.py index d1e022fd11..d8cac32991 100644 --- a/packages/protocol/test/test_account_trading.py +++ b/packages/protocol/test/test_account_trading.py @@ -94,6 +94,9 @@ def make_instance(self, include_optional) -> AccountTrading: side = 'buy', quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = 'pending_creation', executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -107,6 +110,14 @@ def make_instance(self, include_optional) -> AccountTrading: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ] ) else: diff --git a/packages/protocol/test/test_account_trading_state.py b/packages/protocol/test/test_account_trading_state.py index 02360cac9c..70222bc0ed 100644 --- a/packages/protocol/test/test_account_trading_state.py +++ b/packages/protocol/test/test_account_trading_state.py @@ -96,6 +96,9 @@ def make_instance(self, include_optional) -> AccountTradingState: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -109,6 +112,14 @@ def make_instance(self, include_optional) -> AccountTradingState: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ) ) else: @@ -174,6 +185,9 @@ def make_instance(self, include_optional) -> AccountTradingState: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -187,6 +201,14 @@ def make_instance(self, include_optional) -> AccountTradingState: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ), ) """ diff --git a/packages/protocol/test/test_account_trading_with_account_id.py b/packages/protocol/test/test_account_trading_with_account_id.py index 14adcdd2c6..48c970e9f2 100644 --- a/packages/protocol/test/test_account_trading_with_account_id.py +++ b/packages/protocol/test/test_account_trading_with_account_id.py @@ -96,6 +96,9 @@ def make_instance(self, include_optional) -> AccountTradingWithAccountId: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -109,6 +112,14 @@ def make_instance(self, include_optional) -> AccountTradingWithAccountId: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ) ) else: diff --git a/packages/protocol/test/test_accounts_state.py b/packages/protocol/test/test_accounts_state.py index d347396eb5..03fdfb35c7 100644 --- a/packages/protocol/test/test_accounts_state.py +++ b/packages/protocol/test/test_accounts_state.py @@ -69,7 +69,10 @@ def make_instance(self, include_optional) -> AccountsState: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ] ) else: diff --git a/packages/protocol/test/test_create_exchange_config_configuration.py b/packages/protocol/test/test_create_exchange_config_configuration.py index 18af585bee..cfa806fc48 100644 --- a/packages/protocol/test/test_create_exchange_config_configuration.py +++ b/packages/protocol/test/test_create_exchange_config_configuration.py @@ -41,7 +41,10 @@ def make_instance(self, include_optional) -> CreateExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ) else: return CreateExchangeConfigConfiguration( @@ -51,7 +54,10 @@ def make_instance(self, include_optional) -> CreateExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ), + url = '', + historical_trade_symbols = [ + '' + ], ), ) """ diff --git a/packages/protocol/test/test_debug.py b/packages/protocol/test/test_debug.py index 714cd0887c..9daad058d8 100644 --- a/packages/protocol/test/test_debug.py +++ b/packages/protocol/test/test_debug.py @@ -145,7 +145,10 @@ def make_instance(self, include_optional) -> Debug: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ], account_tradings = [ octobot_protocol.models.account_trading_with_account_id.AccountTradingWithAccountId( @@ -210,6 +213,9 @@ def make_instance(self, include_optional) -> Debug: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -223,6 +229,14 @@ def make_instance(self, include_optional) -> Debug: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ), ) ], local_strategies = [ diff --git a/packages/protocol/test/test_debug_state.py b/packages/protocol/test/test_debug_state.py index 7e9a2381b2..1bb4ab87e8 100644 --- a/packages/protocol/test/test_debug_state.py +++ b/packages/protocol/test/test_debug_state.py @@ -135,13 +135,24 @@ def make_instance(self, include_optional) -> DebugState: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ], account_tradings = [ octobot_protocol.models.account_trading_with_account_id.AccountTradingWithAccountId( account_id = '', account_trading = octobot_protocol.models.account_trading.AccountTrading( - updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ), ) + updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) + ], ), ) ], local_strategies = [ octobot_protocol.models.strategy.Strategy( diff --git a/packages/protocol/test/test_edit_exchange_config_configuration.py b/packages/protocol/test/test_edit_exchange_config_configuration.py index e241890163..ea17e3d34b 100644 --- a/packages/protocol/test/test_edit_exchange_config_configuration.py +++ b/packages/protocol/test/test_edit_exchange_config_configuration.py @@ -42,7 +42,10 @@ def make_instance(self, include_optional) -> EditExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ) else: return EditExchangeConfigConfiguration( diff --git a/packages/protocol/test/test_exchange_config.py b/packages/protocol/test/test_exchange_config.py index e8d1ee6bbd..7fde94d3cc 100644 --- a/packages/protocol/test/test_exchange_config.py +++ b/packages/protocol/test/test_exchange_config.py @@ -39,7 +39,10 @@ def make_instance(self, include_optional) -> ExchangeConfig: name = '', exchange = '', sandboxed = True, - url = '' + url = '', + historical_trade_symbols = [ + '' + ] ) else: return ExchangeConfig( diff --git a/packages/protocol/test/test_fee.py b/packages/protocol/test/test_fee.py new file mode 100644 index 0000000000..c728329c75 --- /dev/null +++ b/packages/protocol/test/test_fee.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.fee import Fee + +class TestFee(unittest.TestCase): + """Fee unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> Fee: + """Test Fee + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `Fee` + """ + model = Fee() + if include_optional: + return Fee( + amount = 1.337, + currency = '' + ) + else: + return Fee( + amount = 1.337, + currency = '', + ) + """ + + def testFee(self): + """Test Fee""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_trade.py b/packages/protocol/test/test_trade.py index 232ffcec76..3960b720ff 100644 --- a/packages/protocol/test/test_trade.py +++ b/packages/protocol/test/test_trade.py @@ -42,6 +42,9 @@ def make_instance(self, include_optional) -> Trade: side = 'buy', quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = 'pending_creation', executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f') ) diff --git a/packages/protocol/test/test_transaction.py b/packages/protocol/test/test_transaction.py new file mode 100644 index 0000000000..760003ef3c --- /dev/null +++ b/packages/protocol/test/test_transaction.py @@ -0,0 +1,60 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.transaction import Transaction + +class TestTransaction(unittest.TestCase): + """Transaction unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> Transaction: + """Test Transaction + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `Transaction` + """ + model = Transaction() + if include_optional: + return Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit' + ) + else: + return Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', + ) + """ + + def testTransaction(self): + """Test Transaction""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_transaction_type.py b/packages/protocol/test/test_transaction_type.py new file mode 100644 index 0000000000..07fe0560ae --- /dev/null +++ b/packages/protocol/test/test_transaction_type.py @@ -0,0 +1,33 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.transaction_type import TransactionType + +class TestTransactionType(unittest.TestCase): + """TransactionType unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testTransactionType(self): + """Test TransactionType""" + # inst = TransactionType() + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_update_historical_exchanges_data_configuration.py b/packages/protocol/test/test_update_historical_exchanges_data_configuration.py new file mode 100644 index 0000000000..faaaf9d2a4 --- /dev/null +++ b/packages/protocol/test/test_update_historical_exchanges_data_configuration.py @@ -0,0 +1,55 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration + +class TestUpdateHistoricalExchangesDataConfiguration(unittest.TestCase): + """UpdateHistoricalExchangesDataConfiguration unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> UpdateHistoricalExchangesDataConfiguration: + """Test UpdateHistoricalExchangesDataConfiguration + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `UpdateHistoricalExchangesDataConfiguration` + """ + model = UpdateHistoricalExchangesDataConfiguration() + if include_optional: + return UpdateHistoricalExchangesDataConfiguration( + action_type = 'automation_create', + account_ids = [ + '' + ] + ) + else: + return UpdateHistoricalExchangesDataConfiguration( + action_type = 'automation_create', + ) + """ + + def testUpdateHistoricalExchangesDataConfiguration(self): + """Test UpdateHistoricalExchangesDataConfiguration""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index 34bf81ea56..5cb37641eb 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -17,6 +17,9 @@ import typing +import octobot.community.authentication as community_authentication +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_commons.logging as commons_logging import octobot_commons.singleton.singleton_class as singleton_class import octobot_sync.constants as sync_constants import octobot_protocol.models as protocol_models @@ -130,3 +133,19 @@ def delete_exchange_config(self, address: str, config_id: str) -> None: def list_registered_wallet_ids(self) -> list[str]: return self._storage.list_wallet_storage_keys() + + def list_collectable_wallet_ids(self) -> list[str]: + logger = commons_logging.get_logger(self.__class__.__name__) + community_auth = community_authentication.CommunityAuthentication.instance() + collectable_wallet_ids = [] + for wallet_id in self.list_registered_wallet_ids(): + try: + community_auth.get_wallet_by_user_id(wallet_id) + except wallet_backend_errors.WalletNotFoundError: + logger.debug( + "Skipping wallet %s: not registered locally", + wallet_id, + ) + continue + collectable_wallet_ids.append(wallet_id) + return collectable_wallet_ids diff --git a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py new file mode 100644 index 0000000000..beaa56c092 --- /dev/null +++ b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py @@ -0,0 +1,54 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock + +import octobot.community.authentication as community_authentication +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_sync.sync.collection_providers.user_account_provider as account_provider_module + + +class TestListCollectableWalletIds: + def test_returns_only_wallets_registered_locally(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [ + "wallet-known", + "wallet-missing", + ] + community_auth = mock.Mock() + community_auth.get_wallet_by_user_id.side_effect = lambda wallet_id: ( + mock.Mock() + if wallet_id == "wallet-known" + else (_ for _ in ()).throw(wallet_backend_errors.WalletNotFoundError("missing")) + ) + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=community_auth, + ): + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == ["wallet-known"] + + def test_returns_empty_when_no_registered_wallets(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [] + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=mock.Mock(), + ): + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == [] diff --git a/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py b/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py index 00835bc156..a5e8224c2c 100644 --- a/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py +++ b/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py @@ -30,7 +30,7 @@ import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.symbols.symbol_util as symbol_util -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.data as data import octobot_trading.api as trading_api import octobot_trading.errors as trading_errors @@ -396,7 +396,7 @@ async def get_ohlcv_history(self, exchange, symbol, time_frame): async def _import_candles_from_datafile(self, exchange, symbol, time_frame): return importers.import_ohlcvs( await self.database.select(backtesting_enums.ExchangeDataTables.OHLCV, - size=databases.SQLiteDatabase.DEFAULT_SIZE, + size=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, exchange_name=exchange, symbol=symbol.symbol_str, time_frame=time_frame.value) ) diff --git a/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py b/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py index bd8c96bcae..e46b7c1266 100644 --- a/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py +++ b/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py @@ -19,7 +19,7 @@ import json import asyncio -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_commons.symbols as commons_symbols import octobot_commons.enums as commons_enums import octobot_commons.constants as commons_constants @@ -59,7 +59,7 @@ async def data_collector(exchange_name, tentacles_setup_config, symbols, time_fr @contextlib.asynccontextmanager async def collector_database(collector): - database = databases.SQLiteDatabase(collector.file_path) + database = backtesting_databases.BacktestingDataSQLiteDatabase(collector.file_path) try: await database.initialize() yield database diff --git a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py index 716f8bec7d..a530df2131 100644 --- a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py +++ b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py @@ -24,7 +24,7 @@ import octobot_backtesting.converters as converters import octobot_backtesting.data as backtesting_data import octobot_backtesting.enums as backtesting_enums -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.symbols.symbol_util as symbol_util @@ -78,7 +78,7 @@ async def can_convert(self, ) -> bool: async def convert(self) -> bool: try: - self.database = databases.SQLiteDatabase( + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( path.join(backtesting_constants.BACKTESTING_FILE_PATH, self.converted_file)) await self.database.initialize() await self._create_description() diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py index ff0dfd4496..19fe5c56b4 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py @@ -23,6 +23,7 @@ import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors as commons_errors +import octobot_commons.logging as commons_logging import octobot_commons.symbols.symbol_util as symbol_util @@ -33,6 +34,7 @@ import octobot_protocol.models as protocol_models import octobot_copy.copiers +import octobot_copy.copiers.formatter as copy_formatter import octobot_copy.entities import octobot_copy.constants @@ -198,6 +200,10 @@ async def pre_compute(self) -> None: copier_exchange_manager, copier_trading_mode, ) + commons_logging.get_logger("copy_exchange_account_operator").info( + f"Copying account for strategy {strategy_id!r}, account: " + f"{copy_formatter.format_reference_account_summary(reference_account)}" + ) copy_result = await account_copier.copy_account() self.value = self.create_re_callable_result_dict( keyword=self.get_name(), diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx index 6ecfc2e5a8..b3c66a83d0 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx @@ -26,6 +26,7 @@ import { buildAutomationStopUserActionJson, buildExchangeConfigEditUserActionJson, buildStrategyEditUserActionJson, + buildUpdateHistoricalExchangesDataUserActionJson, } from "@/lib/debug/user-action-templates" const DELETABLE_TABS = new Set(DEBUG_DELETABLE_TAB_VALUES) @@ -201,6 +202,12 @@ function DebugTabsPanelComponent({ jsonText: buildAutomationCreateUserActionJsonForAccount(account), }) } + onUpdateHistory={(account) => + onOpenExecuteAction({ + actionType: "update_historical_exchanges_data", + jsonText: buildUpdateHistoricalExchangesDataUserActionJson(account.id), + }) + } /> diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx index faeaf035b8..00242fb706 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx @@ -1,4 +1,4 @@ -import { Eye, Pencil, Play } from "lucide-react" +import { Eye, History, Pencil, Play } from "lucide-react" import { useMemo, useState } from "react" import type { @@ -49,6 +49,7 @@ type AccountsTableProps = { accountTradings: AccountTradingWithAccountId[] onEdit?: (account: Account) => void onStartAutomation?: (account: Account) => void + onUpdateHistory?: (account: Account) => void } export function AccountsTable({ @@ -57,6 +58,7 @@ export function AccountsTable({ accountTradings, onEdit, onStartAutomation, + onUpdateHistory, }: AccountsTableProps) { const [detail, setDetail] = useState(null) const [sort, setSort] = useState>({ @@ -91,7 +93,7 @@ export function AccountsTable({ ] const accountColumnCount = accountColumns.length + 1 - const actionsHeadClass = "w-24" + const actionsHeadClass = "w-32" if (rows.length === 0) { return ( @@ -295,6 +297,14 @@ export function AccountsTable({ > + + + ) : null} + getPortfolioHistoryChartPoints(state), + [state], + ) + const chartPoints = useMemo( + () => chartData.map((point) => ({ x: point.x, y: point.y })), + [chartData], + ) + const unit = state?.history?.unit ?? "USDT" + + if (isImportedMode) { + return ( +

+ History unavailable in imported snapshot. +

+ ) + } + + if (isLoading) { + return ( +
+ + Loading history… +
+ ) + } + + if (error) { + return ( +

+ Failed to load portfolio history. +

+ ) + } + + return ( + { + const historyValue = chartData[index]?.historyValue + if (!historyValue) { + return null + } + const tooltip = formatPortfolioHistoryTooltip(historyValue, unit) + return ( +
+
{tooltip.timestampLabel}
+
Total: {tooltip.totalLabel}
+ {tooltip.assetRows.length > 0 ? ( + + + + + + + + + + {tooltip.assetRows.map((assetRow) => ( + + + + + + ))} + +
AssetHoldings{tooltip.valueColumnLabel}
{assetRow.symbol}{assetRow.holdings}{assetRow.value}
+ ) : ( +

No holdings breakdown.

+ )} +
+ ) + }} + /> + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx new file mode 100644 index 0000000000..658bd09393 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx @@ -0,0 +1,95 @@ +import { useQuery } from "@tanstack/react-query" +import { useMemo } from "react" + +import type { Account, ExchangeConfig } from "@/client" +import { PortfolioHistoryChart } from "@/components/Debug/PortfolioHistoryChart" +import { CollapsibleJsonView } from "@/components/ui/collapsible-json-view" +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { getAccountHistoricalValuesQueryOptions } from "@/lib/debug/queries" +import { getNegativeHoldingsWarnings } from "@/lib/debug/portfolio-history-warnings" + +type AccountDetailDialogProps = { + account: Account | null + open: boolean + onOpenChange: (open: boolean) => void + walletQueryParam?: string + isImportedMode?: boolean + exchangeConfigs?: ExchangeConfig[] +} + +export function AccountDetailDialog({ + account, + open, + onOpenChange, + walletQueryParam, + isImportedMode = false, + exchangeConfigs = [], +}: AccountDetailDialogProps) { + const historyQuery = useQuery({ + ...getAccountHistoricalValuesQueryOptions( + account?.id, + walletQueryParam, + open && !isImportedMode, + ), + }) + + const negativeHoldingsWarnings = useMemo( + () => + getNegativeHoldingsWarnings( + historyQuery.data, + historyQuery.data?.history?.unit ?? "USDT", + account, + exchangeConfigs, + ), + [account, exchangeConfigs, historyQuery.data], + ) + + return ( + + + + {account?.name ?? "Account"} + + Portfolio history and full JSON payload + + +
+ + {negativeHoldingsWarnings.length > 0 ? ( +
+

Negative holdings detected in history

+
    + {negativeHoldingsWarnings.map((warning) => ( +
  • + {warning.symbol} has negative holdings. Add{" "} + {warning.suggestedTradeSymbol} to{" "} + historical_trade_symbols + {warning.exchangeConfigLabel + ? ` on exchange config "${warning.exchangeConfigLabel}".` + : " on the linked exchange config."} +
    • + ))} +
+
+ ) : null} + {account != null ? ( + + ) : ( +

—

+ )} +
+ + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx new file mode 100644 index 0000000000..a93aeb1a67 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx @@ -0,0 +1,139 @@ +import { useQuery } from "@tanstack/react-query" +import { useMemo, useState } from "react" + +import type { Account } from "@/client" +import { PortfolioHistoryChart } from "@/components/Debug/PortfolioHistoryChart" +import { CopyableIdCell } from "@/components/Common/Tables/CopyableIdCell" +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { getAggregatedAccountHistoricalValuesQueryOptions } from "@/lib/debug/queries" + +type AccountsHistoryDialogProps = { + accounts: Account[] + open: boolean + onOpenChange: (open: boolean) => void + walletQueryParam?: string + isImportedMode?: boolean +} + +type AccountsHistoryTabProps = { + accounts: Account[] + isSimulated: boolean + isActive: boolean + dialogOpen: boolean + walletQueryParam?: string + isImportedMode: boolean +} + +function AccountsHistoryTab({ + accounts, + isSimulated, + isActive, + dialogOpen, + walletQueryParam, + isImportedMode, +}: AccountsHistoryTabProps) { + const tabAccounts = useMemo( + () => accounts.filter((account) => account.is_simulated === isSimulated), + [accounts, isSimulated], + ) + const historyQuery = useQuery({ + ...getAggregatedAccountHistoricalValuesQueryOptions( + isSimulated, + walletQueryParam, + dialogOpen && isActive && !isImportedMode, + ), + }) + + return ( +
+ +
+

+ Accounts ({tabAccounts.length}) +

+ {tabAccounts.length === 0 ? ( +

+ No {isSimulated ? "simulated" : "real"} accounts. +

+ ) : ( +
    + {tabAccounts.map((account) => ( +
  • + {account.name} + +
    • + ))} +
+ )} +
+
+ ) +} + +export function AccountsHistoryDialog({ + accounts, + open, + onOpenChange, + walletQueryParam, + isImportedMode = false, +}: AccountsHistoryDialogProps) { + const [activeTab, setActiveTab] = useState<"real" | "simulated">("real") + + return ( + + + + Accounts history + + Aggregated portfolio value across all accounts of the selected type. + + + setActiveTab(value as "real" | "simulated")} + className="flex min-h-0 flex-1 flex-col" + > + + Real + Simulated + + + + + + + + + + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx index 00242fb706..93eece2f24 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx @@ -12,9 +12,9 @@ import { ColumnFilterInput } from "@/components/Common/Tables/ColumnFilterInput" import { CopyableIdCell } from "@/components/Common/Tables/CopyableIdCell" import { SortableTableHead } from "@/components/Common/Tables/SortableTableHead" import { AssetsPortfolioCell } from "@/components/Debug/cells/AssetsPortfolioCell" +import { AccountDetailDialog } from "@/components/Debug/dialogs/AccountDetailDialog" import { AutomationTradingCountCell } from "@/components/Debug/cells/AutomationTradingCountCell" import { DebugStatusCell } from "@/components/Debug/cells/DebugStatusCell" -import { JsonDetailDialog } from "@/components/Debug/dialogs/JsonDetailDialog" import { Table, TableBody, @@ -47,6 +47,8 @@ type AccountsTableProps = { rows: Account[] exchangeConfigs: ExchangeConfig[] accountTradings: AccountTradingWithAccountId[] + walletQueryParam?: string + isImportedMode?: boolean onEdit?: (account: Account) => void onStartAutomation?: (account: Account) => void onUpdateHistory?: (account: Account) => void @@ -56,6 +58,8 @@ export function AccountsTable({ rows, exchangeConfigs, accountTradings, + walletQueryParam, + isImportedMode = false, onEdit, onStartAutomation, onUpdateHistory, @@ -320,13 +324,15 @@ export function AccountsTable({ )} - { if (!open) setDetail(null) }} + walletQueryParam={walletQueryParam} + isImportedMode={isImportedMode} + exchangeConfigs={exchangeConfigs} /> ) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/__tests__/svg-line-chart.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/__tests__/svg-line-chart.test.ts new file mode 100644 index 0000000000..f7b6672cc5 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/__tests__/svg-line-chart.test.ts @@ -0,0 +1,90 @@ +import { describe, expect, it } from "vitest" + +import { + buildLinearTicks, + buildLinePath, + domainFromPoints, + findNearestPointIndex, + formatDefaultYTick, + getChartTooltipViewportPosition, + scaleLinear, +} from "@/lib/charts/svg-line-chart" + +describe("scaleLinear", () => { + it("maps values into the svg range", () => { + const scale = scaleLinear(0, 100, 10, 110) + expect(scale.toSvg(0)).toBe(10) + expect(scale.toSvg(100)).toBe(110) + expect(scale.toSvg(50)).toBe(60) + }) + + it("returns midpoint when domain span is zero", () => { + const scale = scaleLinear(5, 5, 0, 200) + expect(scale.toSvg(5)).toBe(100) + }) +}) + +describe("buildLinePath", () => { + it("builds an svg path from scaled points", () => { + const xScale = scaleLinear(0, 100, 0, 100) + const yScale = scaleLinear(0, 100, 100, 0) + const path = buildLinePath( + [ + { x: 0, y: 0 }, + { x: 100, y: 100 }, + ], + xScale, + yScale, + ) + expect(path).toBe("M0,100 L100,0") + }) +}) + +describe("findNearestPointIndex", () => { + it("returns the closest point by x coordinate", () => { + const points = [ + { x: 10, y: 1 }, + { x: 20, y: 2 }, + { x: 30, y: 3 }, + ] + expect(findNearestPointIndex(points, 21)).toBe(1) + expect(findNearestPointIndex(points, 8)).toBe(0) + }) +}) + +describe("domainFromPoints", () => { + it("pads a flat y domain", () => { + const domain = domainFromPoints([{ x: 1, y: 100 }], "y") + expect(domain.min).toBeLessThan(100) + expect(domain.max).toBeGreaterThan(100) + }) +}) + +describe("buildLinearTicks", () => { + it("returns evenly spaced ticks across the domain", () => { + const ticks = buildLinearTicks(0, 1000, 5) + expect(ticks.length).toBeGreaterThanOrEqual(2) + expect(ticks[0]).toBeLessThanOrEqual(0) + expect(ticks[ticks.length - 1]).toBeGreaterThanOrEqual(1000) + }) +}) + +describe("formatDefaultYTick", () => { + it("formats large values without decimals", () => { + expect(formatDefaultYTick(1500)).toMatch(/1.500/) + }) +}) + +describe("getChartTooltipViewportPosition", () => { + it("places the tooltip to the right of the anchor by default", () => { + const position = getChartTooltipViewportPosition( + { left: 100, top: 50, width: 400, height: 200 }, + 320, + 110, + 640, + 220, + ) + expect(position.left).toBeGreaterThan(100) + expect(position.top).toBeGreaterThan(50) + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/svg-line-chart.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/svg-line-chart.ts new file mode 100644 index 0000000000..5d4babb44a --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/charts/svg-line-chart.ts @@ -0,0 +1,162 @@ +export type SvgLineChartPoint = { + x: number + y: number +} + +export type LinearScale = { + min: number + max: number + toSvg: (value: number) => number +} + +export function scaleLinear( + domainMin: number, + domainMax: number, + rangeMin: number, + rangeMax: number, +): LinearScale { + const domainSpan = domainMax - domainMin + const rangeSpan = rangeMax - rangeMin + return { + min: domainMin, + max: domainMax, + toSvg: (value: number) => { + if (domainSpan === 0) { + return (rangeMin + rangeMax) / 2 + } + const ratio = (value - domainMin) / domainSpan + return rangeMin + ratio * rangeSpan + }, + } +} + +export function buildLinePath( + points: SvgLineChartPoint[], + xScale: LinearScale, + yScale: LinearScale, +): string { + if (points.length === 0) { + return "" + } + return points + .map((point, index) => { + const command = index === 0 ? "M" : "L" + const svgX = xScale.toSvg(point.x) + const svgY = yScale.toSvg(point.y) + return `${command}${svgX},${svgY}` + }) + .join(" ") +} + +export function findNearestPointIndex( + points: SvgLineChartPoint[], + targetX: number, +): number | null { + if (points.length === 0) { + return null + } + let nearestIndex = 0 + let nearestDistance = Math.abs(points[0].x - targetX) + for (let pointIndex = 1; pointIndex < points.length; pointIndex += 1) { + const distance = Math.abs(points[pointIndex].x - targetX) + if (distance < nearestDistance) { + nearestDistance = distance + nearestIndex = pointIndex + } + } + return nearestIndex +} + +export function domainFromPoints( + points: SvgLineChartPoint[], + axis: "x" | "y", +): { min: number; max: number } { + if (points.length === 0) { + return { min: 0, max: 1 } + } + const values = points.map((point) => (axis === "x" ? point.x : point.y)) + const min = Math.min(...values) + const max = Math.max(...values) + if (min === max) { + const padding = axis === "x" ? 86_400_000 : Math.max(Math.abs(min) * 0.1, 1) + return { min: min - padding, max: max + padding } + } + return { min, max } +} + +export function buildLinearTicks( + domainMin: number, + domainMax: number, + tickCount: number = 5, +): number[] { + if (tickCount < 2) { + return [domainMin, domainMax] + } + const domainSpan = domainMax - domainMin + if (domainSpan === 0) { + return [domainMin] + } + const rawStep = domainSpan / (tickCount - 1) + const magnitude = 10 ** Math.floor(Math.log10(rawStep)) + const normalizedStep = rawStep / magnitude + let niceStep = magnitude + if (normalizedStep <= 1) { + niceStep = magnitude + } else if (normalizedStep <= 2) { + niceStep = 2 * magnitude + } else if (normalizedStep <= 5) { + niceStep = 5 * magnitude + } else { + niceStep = 10 * magnitude + } + const niceMin = Math.floor(domainMin / niceStep) * niceStep + const ticks: number[] = [] + for (let tickValue = niceMin; tickValue <= domainMax + niceStep * 0.5; tickValue += niceStep) { + if (tickValue >= domainMin - niceStep * 0.5) { + ticks.push(Number(tickValue.toPrecision(12))) + } + } + return ticks.length > 0 ? ticks : [domainMin, domainMax] +} + +export function formatDefaultYTick(value: number): string { + return value.toLocaleString(undefined, { + maximumFractionDigits: value >= 1000 ? 0 : 2, + }) +} + +export type ChartTooltipViewportPosition = { + left: number + top: number + transform?: string +} + +export type SvgElementBounds = { + left: number + top: number + width: number + height: number +} + +export function getChartTooltipViewportPosition( + svgBounds: SvgElementBounds, + scaledX: number, + scaledY: number, + viewBoxWidth: number, + viewBoxHeight: number, + tooltipOffset = 12, + viewportWidth = typeof window === "undefined" ? Number.POSITIVE_INFINITY : window.innerWidth, +): ChartTooltipViewportPosition { + const anchorLeft = svgBounds.left + (scaledX / viewBoxWidth) * svgBounds.width + const anchorTop = svgBounds.top + (scaledY / viewBoxHeight) * svgBounds.height + const estimatedTooltipWidth = 280 + const placeLeft = + anchorLeft + tooltipOffset + estimatedTooltipWidth > viewportWidth + return { + left: placeLeft + ? anchorLeft - tooltipOffset + : anchorLeft + tooltipOffset, + top: anchorTop + tooltipOffset, + transform: placeLeft ? "translateX(-100%)" : undefined, + } +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-chart.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-chart.test.ts new file mode 100644 index 0000000000..a9b2ed4bc2 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-chart.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "vitest" + +import { formatPortfolioHistoryTooltip } from "@/lib/debug/portfolio-history-chart" +import type { PortfolioHistoricalValue } from "@/lib/debug/portfolio-historical-values-types" + +describe("formatPortfolioHistoryTooltip", () => { + it("puts the unit in the column header and not in value cells", () => { + const historyValue: PortfolioHistoricalValue = { + timestamp: "2024-01-01T00:00:00Z", + total: 1500.5, + assets: [ + { + trading_type: "spot", + assets: [ + { symbol: "BTC", holdings: 1, value: 1000.25 }, + { symbol: "USDT", holdings: 500, value: 500.25 }, + ], + }, + ], + } + + const tooltip = formatPortfolioHistoryTooltip(historyValue, "USDT") + + expect(tooltip.valueColumnLabel).toBe("Value (USDT)") + expect(tooltip.totalLabel).toContain("USDT") + expect(tooltip.assetRows[0].value).not.toContain("USDT") + expect(tooltip.assetRows[1].value).not.toContain("USDT") + }) + + it("orders assets by value descending", () => { + const historyValue: PortfolioHistoricalValue = { + timestamp: "2024-01-01T00:00:00Z", + total: 1500.5, + assets: [ + { + trading_type: "spot", + assets: [ + { symbol: "USDT", holdings: 500, value: 500.25 }, + { symbol: "ETH", holdings: 2, value: 750.5 }, + { symbol: "BTC", holdings: 1, value: 249.75 }, + ], + }, + ], + } + + const tooltip = formatPortfolioHistoryTooltip(historyValue, "USDT") + + expect(tooltip.assetRows.map((assetRow) => assetRow.symbol)).toEqual([ + "ETH", + "USDT", + "BTC", + ]) + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-warnings.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-warnings.test.ts new file mode 100644 index 0000000000..1cc5448ccc --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/portfolio-history-warnings.test.ts @@ -0,0 +1,390 @@ +import { describe, expect, it } from "vitest" + +import type { Account } from "@/client" +import { + formatSuggestedTradePair, + getNegativeHoldingsWarnings, +} from "@/lib/debug/portfolio-history-warnings" +import type { PortfolioHistoricalValuesState } from "@/lib/debug/portfolio-historical-values-types" + +type AssetInput = { + holdings: number + value?: number +} + +type DayInput = { + timestamp?: string + total?: number + assets: Record +} + +function makeHistoryState(days: DayInput[]): PortfolioHistoricalValuesState { + return { + version: "1.0.0", + history: { + unit: "USDT", + values: days.map((day, dayIndex) => { + const assetEntries = Object.entries(day.assets).map(([symbol, assetInput]) => { + const holdings = typeof assetInput === "number" ? assetInput : assetInput.holdings + const value = + typeof assetInput === "number" ? assetInput : (assetInput.value ?? assetInput.holdings) + return { symbol, holdings, value } + }) + const defaultTotal = assetEntries.reduce( + (runningTotal, asset) => runningTotal + Math.abs(asset.value), + 0, + ) + return { + timestamp: day.timestamp ?? `2024-01-0${dayIndex + 1}T00:00:00Z`, + total: day.total ?? defaultTotal, + assets: [ + { + trading_type: "spot", + assets: assetEntries, + }, + ], + } + }), + }, + } +} + +describe("formatSuggestedTradePair", () => { + it("puts the acquired asset first when quote went negative", () => { + expect(formatSuggestedTradePair("USDT", "SOL", "USDT")).toBe("SOL/USDT") + }) + + it("puts the sold asset first when quote increased", () => { + expect(formatSuggestedTradePair("ADA", "USDT", "USDT")).toBe("ADA/USDT") + }) + + it("puts the negative crypto first for crypto-crypto moves", () => { + expect(formatSuggestedTradePair("ETH", "BTC", "USDT")).toBe("ETH/BTC") + }) +}) + +describe("getNegativeHoldingsWarnings", () => { + it("returns deduplicated warnings for significant negative holdings", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + BTC: { holdings: 2, value: 900 }, + }, + }, + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + ETH: { holdings: -5, value: -50 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: undefined, + }, + { + symbol: "ETH", + suggestedTradeSymbol: "ETH/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("ignores zero and positive holdings", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + USDT: { holdings: 100, value: 100 }, + BTC: { holdings: 0, value: 0 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([]) + }) + + it("ignores negative holdings below the portfolio significance threshold", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -1, value: -5 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([]) + }) + + it("warns when negative holdings exceed the portfolio significance threshold", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -1, value: -50 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("infers SOL/USDC via holdings when SOL has no priced value", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + SOL: { holdings: 0, value: 0 }, + USDC: { holdings: 1000, value: 1000 }, + }, + }, + { + total: 1000, + assets: { + SOL: { holdings: 10, value: 0 }, + USDC: { holdings: -50, value: 950 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "USDC", + suggestedTradeSymbol: "SOL/USDC", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("warns with quote-only suggestion when only USDC is in the portfolio", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + USDC: { holdings: -50, value: -50 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "USDC", + suggestedTradeSymbol: "/USDC", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("suggests quote only when quote delta matching fails", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ALGO: { holdings: 100, value: 0 }, + SOL: { holdings: 10, value: 0 }, + USDC: { holdings: 1000, value: 1000 }, + }, + }, + { + total: 1000, + assets: { + ALGO: { holdings: 100, value: 0 }, + SOL: { holdings: 10, value: 0 }, + USDC: { holdings: -200, value: 800 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "USDC", + suggestedTradeSymbol: "/USDC", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("does not suggest a random pair from unchanged holdings", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + SOL: { holdings: 10, value: 0 }, + USDC: { holdings: 1000, value: 1000 }, + }, + }, + { + total: 1000, + assets: { + SOL: { holdings: 10, value: 0 }, + USDC: { holdings: -200, value: 800 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "USDC", + suggestedTradeSymbol: "/USDC", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("falls back to detected quote market for crypto without counterparty", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + USDC: { holdings: 500, value: 500 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDC", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("infers SOL/USDT when quote went negative and SOL increased", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + SOL: { holdings: 0, value: 0 }, + USDT: { holdings: 1000, value: 1000 }, + }, + }, + { + total: 1000, + assets: { + SOL: { holdings: 5, value: 200 }, + USDT: { holdings: -50, value: 800 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "USDT", + suggestedTradeSymbol: "SOL/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("infers ADA/USDT when ADA went negative and USDT increased", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: 10, value: 400 }, + USDT: { holdings: 600, value: 600 }, + }, + }, + { + total: 1000, + assets: { + ADA: { holdings: -2, value: 200 }, + USDT: { holdings: 800, value: 800 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("falls back to symbol over reference market when there is no previous day", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("falls back when no counterparty asset increased on the trigger day", () => { + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: 10, value: 1000 }, + }, + }, + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: undefined, + }, + ]) + }) + + it("includes exchange config label when account is linked", () => { + const account = { + id: "acc-1", + name: "Kraken real", + is_simulated: false, + created_at: "2024-01-01T00:00:00Z", + specifics: { + actual_instance: { + account_type: "exchange", + remote_account_id: "remote-1", + exchange_config_ids: ["cfg-kraken"], + }, + }, + } as Account + const exchangeConfigs = [ + { + id: "cfg-kraken", + name: "Kraken main", + exchange: "kraken", + sandboxed: false, + }, + ] + const state = makeHistoryState([ + { + total: 1000, + assets: { + ADA: { holdings: -10, value: -100 }, + }, + }, + ]) + expect(getNegativeHoldingsWarnings(state, "USDT", account, exchangeConfigs)).toEqual([ + { + symbol: "ADA", + suggestedTradeSymbol: "ADA/USDT", + exchangeConfigLabel: "Kraken main", + }, + ]) + }) +}) diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts index 94b69099d2..13291ad3d4 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/__tests__/user-action-templates.test.ts @@ -8,6 +8,7 @@ import { buildAutomationRestartUserActionJson, buildAutomationSignalUserActionJson, buildAutomationStopUserActionJson, + buildResetAccountTradingDataUserActionJson, buildExchangeConfigEditUserActionJson, buildStrategyEditUserActionJson, buildUpdateHistoricalExchangesDataUserActionJson, @@ -319,6 +320,15 @@ describe("buildUserActionTemplate", () => { action_type: "update_historical_exchanges_data", }) }) + + it("builds a reset account trading data template", () => { + const action = buildUserActionTemplate("reset_account_trading_data") + expect(action.id).toContain("reset_account_trading_data") + expect(action.configuration).toMatchObject({ + action_type: "reset_account_trading_data", + account_ids: [TEMPLATE_ACCOUNT_ID], + }) + }) }) describe("buildUserActionTemplateJson", () => { @@ -362,6 +372,18 @@ describe("buildUpdateHistoricalExchangesDataUserActionJson", () => { }) }) +describe("buildResetAccountTradingDataUserActionJson", () => { + it("includes the account id in account_ids", () => { + const json = JSON.parse( + buildResetAccountTradingDataUserActionJson("acc-reset-1"), + ) + expect(json.configuration).toMatchObject({ + action_type: "reset_account_trading_data", + account_ids: ["acc-reset-1"], + }) + }) +}) + describe("buildExchangeConfigEditUserActionJson", () => { it("embeds the exchange config", () => { const config: ExchangeConfig = { @@ -373,6 +395,21 @@ describe("buildExchangeConfigEditUserActionJson", () => { const json = JSON.parse(buildExchangeConfigEditUserActionJson(config)) expect(json.configuration.id).toBe("cfg-1") }) + + it("preserves historical_trade_symbols when present", () => { + const config: ExchangeConfig = { + id: "cfg-1", + name: "Kraken", + exchange: "kraken", + sandboxed: false, + historical_trade_symbols: ["BTC/USDT", "ADA/USDT"], + } + const json = JSON.parse(buildExchangeConfigEditUserActionJson(config)) + expect(json.configuration.configuration.historical_trade_symbols).toEqual([ + "BTC/USDT", + "ADA/USDT", + ]) + }) }) describe("buildStrategyEditUserActionJson", () => { diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/account-historical-values-api.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/account-historical-values-api.ts new file mode 100644 index 0000000000..03a64372e4 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/account-historical-values-api.ts @@ -0,0 +1,26 @@ +import { OpenAPI } from "@/client/core/OpenAPI" +import { request } from "@/client/core/request" + +import type { PortfolioHistoricalValuesState } from "@/lib/debug/portfolio-historical-values-types" + +/** + * Interim API helper until OpenAPI regen adds AccountsService.getAccountHistoricalValues. + */ +export function fetchAccountHistoricalValues( + accountId: string, + walletAddress?: string, +) { + return request(OpenAPI, { + method: "GET", + url: "/api/v1/accounts/{account_id}/historical-values", + path: { + account_id: accountId, + }, + query: { + wallet_address: walletAddress, + }, + errors: { + 422: "Validation Error", + }, + }) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/aggregated-account-historical-values-api.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/aggregated-account-historical-values-api.ts new file mode 100644 index 0000000000..b7888c635f --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/aggregated-account-historical-values-api.ts @@ -0,0 +1,24 @@ +import { OpenAPI } from "@/client/core/OpenAPI" +import { request } from "@/client/core/request" + +import type { PortfolioHistoricalValuesState } from "@/lib/debug/portfolio-historical-values-types" + +/** + * Interim API helper until OpenAPI regen adds aggregated historical values. + */ +export function fetchAggregatedAccountHistoricalValues( + isSimulated: boolean, + walletAddress?: string, +) { + return request(OpenAPI, { + method: "GET", + url: "/api/v1/accounts/aggregated/historical-values", + query: { + is_simulated: isSimulated, + wallet_address: walletAddress, + }, + errors: { + 422: "Validation Error", + }, + }) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-historical-values-types.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-historical-values-types.ts new file mode 100644 index 0000000000..d6dd888223 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-historical-values-types.ts @@ -0,0 +1,26 @@ +export type HistoricalAssetValue = { + symbol: string + holdings: number + value: number +} + +export type HistoricalAssetsForTradingType = { + trading_type: string + assets: HistoricalAssetValue[] +} + +export type PortfolioHistoricalValue = { + timestamp: string + total: number + assets?: HistoricalAssetsForTradingType[] | null +} + +export type PortfolioHistoricalValues = { + unit: string + values: PortfolioHistoricalValue[] +} + +export type PortfolioHistoricalValuesState = { + version: string + history?: PortfolioHistoricalValues | null +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-chart.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-chart.ts new file mode 100644 index 0000000000..f14691e1c1 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-chart.ts @@ -0,0 +1,49 @@ +import type { + PortfolioHistoricalValue, + PortfolioHistoricalValuesState, +} from "@/lib/debug/portfolio-historical-values-types" +import { formatDateTime } from "@/lib/format-datetime" + +export function portfolioHistoryToChartPoints( + historyValues: PortfolioHistoricalValue[], +) { + return historyValues.map((historyValue) => ({ + x: Date.parse(historyValue.timestamp), + y: historyValue.total, + historyValue, + })) +} + +export function formatPortfolioHistoryTooltip( + historyValue: PortfolioHistoricalValue, + unit: string, +) { + const assetRows = + historyValue.assets?.flatMap((assetsForType) => assetsForType.assets ?? []) ?? + [] + const assetsSortedByValue = [...assetRows].sort( + (leftAsset, rightAsset) => rightAsset.value - leftAsset.value, + ) + + return { + timestampLabel: formatDateTime(historyValue.timestamp), + totalLabel: `${historyValue.total.toLocaleString(undefined, { + maximumFractionDigits: 2, + })} ${unit}`, + valueColumnLabel: `Value (${unit})`, + assetRows: assetsSortedByValue.map((asset) => ({ + symbol: asset.symbol, + holdings: asset.holdings.toLocaleString(undefined, { + maximumFractionDigits: 8, + }), + value: asset.value.toLocaleString(undefined, { + maximumFractionDigits: 2, + }), + })), + } +} + +export function getPortfolioHistoryChartPoints(state?: PortfolioHistoricalValuesState) { + const historyValues = state?.history?.values ?? [] + return portfolioHistoryToChartPoints(historyValues) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-warnings.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-warnings.ts new file mode 100644 index 0000000000..d455e29348 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/portfolio-history-warnings.ts @@ -0,0 +1,297 @@ +import type { Account, ExchangeConfig } from "@/client" +import { getAccountExchangeConfigIds } from "@/lib/debug/display-utils" +import type { + HistoricalAssetValue, + PortfolioHistoricalValue, + PortfolioHistoricalValuesState, +} from "@/lib/debug/portfolio-historical-values-types" + +export const NEGATIVE_HOLDINGS_SIGNIFICANCE_RATIO = 0.01 + +const USD_LIKE_SYMBOLS = new Set(["USDC", "USD", "BUSD", "DAI"]) + +export type NegativeHoldingsWarning = { + symbol: string + suggestedTradeSymbol: string + exchangeConfigLabel?: string +} + +type AssetSnapshot = Map + +export function getNegativeHoldingsWarnings( + state: PortfolioHistoricalValuesState | undefined, + referenceMarket = "USDT", + account?: Account | null, + exchangeConfigs: ExchangeConfig[] = [], +): NegativeHoldingsWarning[] { + const historyValues = [...(state?.history?.values ?? [])].sort( + (leftHistoryValue, rightHistoryValue) => + Date.parse(leftHistoryValue.timestamp) - Date.parse(rightHistoryValue.timestamp), + ) + const exchangeConfigLabel = resolveExchangeConfigLabel(account, exchangeConfigs) + + const warnings: NegativeHoldingsWarning[] = [] + const warnedSymbols = new Set() + + for (const historyValue of historyValues) { + for (const asset of flattenAssets(historyValue)) { + if (warnedSymbols.has(asset.symbol) || !isSignificantNegative(asset, historyValue.total)) { + continue + } + warnedSymbols.add(asset.symbol) + const previousDay = findPreviousHistoryValue(historyValues, historyValue) + warnings.push({ + symbol: asset.symbol, + suggestedTradeSymbol: inferSuggestedTradeSymbol( + asset.symbol, + historyValue, + previousDay, + historyValues, + referenceMarket, + ), + exchangeConfigLabel, + }) + } + } + + return warnings.sort((leftWarning, rightWarning) => + leftWarning.symbol.localeCompare(rightWarning.symbol), + ) +} + +export function formatSuggestedTradePair( + negativeSymbol: string, + counterpartySymbol: string, + referenceMarket: string, +): string { + if (isUsdLikeSymbol(negativeSymbol, referenceMarket)) { + return `${counterpartySymbol}/${negativeSymbol}` + } + if (isUsdLikeSymbol(counterpartySymbol, referenceMarket)) { + return `${negativeSymbol}/${counterpartySymbol}` + } + return `${negativeSymbol}/${counterpartySymbol}` +} + +function flattenAssets(historyValue: PortfolioHistoricalValue): HistoricalAssetValue[] { + return historyValue.assets?.flatMap((assetsForType) => assetsForType.assets ?? []) ?? [] +} + +function buildAssetSnapshot(historyValue: PortfolioHistoricalValue): AssetSnapshot { + const snapshot: AssetSnapshot = new Map() + for (const asset of flattenAssets(historyValue)) { + snapshot.set(asset.symbol, asset) + } + return snapshot +} + +function isSignificantNegative(asset: HistoricalAssetValue, total: number): boolean { + if (asset.holdings >= 0 || total <= 0) { + return false + } + return Math.abs(asset.value) / total > NEGATIVE_HOLDINGS_SIGNIFICANCE_RATIO +} + +function findPreviousHistoryValue( + historyValues: PortfolioHistoricalValue[], + currentHistoryValue: PortfolioHistoricalValue, +): PortfolioHistoricalValue | undefined { + const currentIndex = historyValues.indexOf(currentHistoryValue) + if (currentIndex <= 0) { + return undefined + } + return historyValues[currentIndex - 1] +} + +function isUsdLikeSymbol(symbol: string, referenceMarket: string): boolean { + return symbol === referenceMarket || USD_LIKE_SYMBOLS.has(symbol) +} + +function inferSuggestedTradeSymbol( + negativeSymbol: string, + triggerDay: PortfolioHistoricalValue, + previousDay: PortfolioHistoricalValue | undefined, + historyValues: PortfolioHistoricalValue[], + referenceMarket: string, +): string { + if (previousDay) { + const previousSnapshot = buildAssetSnapshot(previousDay) + const currentSnapshot = buildAssetSnapshot(triggerDay) + const negativeAsset = currentSnapshot.get(negativeSymbol) + if (negativeAsset) { + const previousNegativeAsset = previousSnapshot.get(negativeSymbol) + const negativeValueDelta = negativeAsset.value - (previousNegativeAsset?.value ?? 0) + const counterpartySymbol = findCounterpartySymbol( + negativeSymbol, + negativeValueDelta, + referenceMarket, + previousSnapshot, + currentSnapshot, + ) + if (counterpartySymbol) { + return formatSuggestedTradePair(negativeSymbol, counterpartySymbol, referenceMarket) + } + } + } + + return resolveFallbackTradeSymbol(negativeSymbol, triggerDay, historyValues, referenceMarket) +} + +function findCounterpartySymbol( + negativeSymbol: string, + negativeValueDelta: number, + referenceMarket: string, + previousSnapshot: AssetSnapshot, + currentSnapshot: AssetSnapshot, +): string | undefined { + const counterpartyByValueDelta = findBestCounterpartyByValueDelta( + negativeSymbol, + negativeValueDelta, + previousSnapshot, + currentSnapshot, + ) + if (counterpartyByValueDelta) { + return counterpartyByValueDelta + } + + const includeSymbol = isUsdLikeSymbol(negativeSymbol, referenceMarket) + ? (symbol: string) => !isUsdLikeSymbol(symbol, referenceMarket) + : (symbol: string) => isUsdLikeSymbol(symbol, referenceMarket) + + return findLargestPositiveHoldingsDelta( + negativeSymbol, + previousSnapshot, + currentSnapshot, + includeSymbol, + ) +} + +function findBestCounterpartyByValueDelta( + negativeSymbol: string, + negativeValueDelta: number, + previousSnapshot: AssetSnapshot, + currentSnapshot: AssetSnapshot, +): string | undefined { + let bestCounterpartySymbol: string | undefined + let bestMatchDistance = Number.POSITIVE_INFINITY + + for (const [symbol, currentAsset] of currentSnapshot) { + if (symbol === negativeSymbol) { + continue + } + + const previousAsset = previousSnapshot.get(symbol) + const valueDelta = currentAsset.value - (previousAsset?.value ?? 0) + if (valueDelta <= 0) { + continue + } + + const matchDistance = Math.abs(valueDelta + negativeValueDelta) + if (matchDistance < bestMatchDistance) { + bestMatchDistance = matchDistance + bestCounterpartySymbol = symbol + } + } + + return bestCounterpartySymbol +} + +function findLargestPositiveHoldingsDelta( + negativeSymbol: string, + previousSnapshot: AssetSnapshot, + currentSnapshot: AssetSnapshot, + includeSymbol: (symbol: string) => boolean, +): string | undefined { + let bestCounterpartySymbol: string | undefined + let largestHoldingsDelta = 0 + + for (const [symbol, currentAsset] of currentSnapshot) { + if (symbol === negativeSymbol || !includeSymbol(symbol)) { + continue + } + + const previousAsset = previousSnapshot.get(symbol) + const holdingsDelta = currentAsset.holdings - (previousAsset?.holdings ?? 0) + if (holdingsDelta > largestHoldingsDelta) { + largestHoldingsDelta = holdingsDelta + bestCounterpartySymbol = symbol + } + } + + return bestCounterpartySymbol +} + +function detectPortfolioQuoteSymbol( + triggerDay: PortfolioHistoricalValue, + historyValues: PortfolioHistoricalValue[], + referenceMarket: string, +): string { + const quoteHoldingsBySymbol = new Map() + + for (const historyValue of historyValues) { + for (const asset of flattenAssets(historyValue)) { + if (!isUsdLikeSymbol(asset.symbol, referenceMarket)) { + continue + } + const currentHoldings = quoteHoldingsBySymbol.get(asset.symbol) ?? 0 + quoteHoldingsBySymbol.set(asset.symbol, currentHoldings + Math.abs(asset.holdings)) + } + } + + if (quoteHoldingsBySymbol.has(referenceMarket)) { + return referenceMarket + } + + let detectedQuote = referenceMarket + let largestQuoteHoldings = 0 + for (const [symbol, holdings] of quoteHoldingsBySymbol) { + if (holdings > largestQuoteHoldings) { + largestQuoteHoldings = holdings + detectedQuote = symbol + } + } + + if (largestQuoteHoldings === 0) { + const triggerSnapshot = buildAssetSnapshot(triggerDay) + for (const [symbol, asset] of triggerSnapshot) { + if (isUsdLikeSymbol(symbol, referenceMarket)) { + return symbol + } + } + } + + return detectedQuote +} + +function resolveFallbackTradeSymbol( + negativeSymbol: string, + triggerDay: PortfolioHistoricalValue, + historyValues: PortfolioHistoricalValue[], + referenceMarket: string, +): string { + const detectedQuote = detectPortfolioQuoteSymbol(triggerDay, historyValues, referenceMarket) + + if (isUsdLikeSymbol(negativeSymbol, referenceMarket)) { + return `/${detectedQuote}` + } + + return `${negativeSymbol}/${detectedQuote}` +} + +function resolveExchangeConfigLabel( + account: Account | null | undefined, + exchangeConfigs: ExchangeConfig[], +): string | undefined { + if (!account) { + return undefined + } + const configIds = getAccountExchangeConfigIds(account) + if (configIds.length === 0) { + return undefined + } + const exchangeConfig = exchangeConfigs.find((config) => config.id === configIds[0]) + if (!exchangeConfig) { + return configIds[0] + } + return exchangeConfig.name ?? exchangeConfig.exchange ?? exchangeConfig.id +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/queries.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/queries.ts index 951e43320a..23de22b30e 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/queries.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/queries.ts @@ -1,4 +1,6 @@ import { DebugService } from "@/client" +import { fetchAccountHistoricalValues } from "@/lib/debug/account-historical-values-api" +import { fetchAggregatedAccountHistoricalValues } from "@/lib/debug/aggregated-account-historical-values-api" export function getDebugQueryOptions(walletAddress?: string | null) { const resolved = @@ -9,3 +11,45 @@ export function getDebugQueryOptions(walletAddress?: string | null) { DebugService.getDebug(resolved ? { walletAddress: resolved } : {}), } } + +export function getAccountHistoricalValuesQueryOptions( + accountId: string | undefined, + walletAddress?: string | null, + enabled = false, +) { + const resolvedWallet = + walletAddress && walletAddress.length > 0 ? walletAddress : undefined + return { + queryKey: [ + "account-historical-values", + accountId ?? "none", + resolvedWallet ?? "current", + ] as const, + queryFn: () => { + if (!accountId) { + throw new Error("Account id is required") + } + return fetchAccountHistoricalValues(accountId, resolvedWallet) + }, + enabled: enabled && Boolean(accountId), + } +} + +export function getAggregatedAccountHistoricalValuesQueryOptions( + isSimulated: boolean, + walletAddress?: string | null, + enabled = false, +) { + const resolvedWallet = + walletAddress && walletAddress.length > 0 ? walletAddress : undefined + return { + queryKey: [ + "aggregated-account-historical-values", + isSimulated ? "simulated" : "real", + resolvedWallet ?? "current", + ] as const, + queryFn: () => + fetchAggregatedAccountHistoricalValues(isSimulated, resolvedWallet), + enabled, + } +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts index e03ab7ef8b..2437fcd28d 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/lib/debug/user-action-templates.ts @@ -28,6 +28,7 @@ import type { MarketMakingConfiguration, MarketMakingSymbolConfiguration, RefreshAccountsConfiguration, + ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, @@ -82,6 +83,7 @@ export const USER_ACTION_TEMPLATE_OPTIONS: { { value: "account_auth_delete", label: "Account auth delete" }, { value: "accounts_refresh", label: "Accounts refresh" }, { value: "update_historical_exchanges_data", label: "Update historical exchanges data" }, + { value: "reset_account_trading_data", label: "Reset account trading data" }, { value: "exchange_config_create", label: "Exchange config create" }, { value: "exchange_config_edit", label: "Exchange config edit" }, { value: "exchange_config_delete", label: "Exchange config delete" }, @@ -146,6 +148,7 @@ type DebugUserActionConfiguration = | DeleteAccountAuthConfiguration | RefreshAccountsConfiguration | UpdateHistoricalExchangesDataConfiguration + | ResetAccountTradingDataConfiguration | CreateExchangeConfigConfiguration | EditExchangeConfigConfiguration | DeleteExchangeConfigConfiguration @@ -307,6 +310,7 @@ function sampleExchangeConfig(id = TEMPLATE_EXCHANGE_CONFIG_ID): ExchangeConfig name: "binance-main", exchange: "binance", sandboxed: false, + historical_trade_symbols: ["BTC/USDT", "ETH/USDT"], } satisfies ExchangeConfig } @@ -731,6 +735,11 @@ export function buildUserActionTemplate( return userAction(id, { action_type: actionType, } satisfies UpdateHistoricalExchangesDataConfiguration) + case "reset_account_trading_data": + return userAction(id, { + action_type: actionType, + account_ids: [TEMPLATE_ACCOUNT_ID], + } satisfies ResetAccountTradingDataConfiguration) case "exchange_config_create": return userAction(id, { action_type: actionType, @@ -803,6 +812,17 @@ export function buildUpdateHistoricalExchangesDataUserActionJson( ) } +export function buildResetAccountTradingDataUserActionJson( + accountId: string, +): string { + return userActionJson( + userAction(uniqueUserActionId(`ua-reset-account-trading-${accountId}`), { + action_type: "reset_account_trading_data", + account_ids: [accountId], + } satisfies ResetAccountTradingDataConfiguration), + ) +} + export function buildExchangeConfigEditUserActionJson( config: ExchangeConfig, ): string { diff --git a/packages/tentacles/Trading/Exchange/coinbase/coinbase_exchange.py b/packages/tentacles/Trading/Exchange/coinbase/coinbase_exchange.py index 5ad4d3cea2..04c077601b 100644 --- a/packages/tentacles/Trading/Exchange/coinbase/coinbase_exchange.py +++ b/packages/tentacles/Trading/Exchange/coinbase/coinbase_exchange.py @@ -327,3 +327,90 @@ def is_market_open_for_order_type(self, symbol: str, order_type: trading_enums.T f"market_status_info: {market_status_info}" ) return True + + def _uses_single_transaction_currency_request(self, currency: str, kwargs: dict) -> bool: + return currency is not None or kwargs.get("account_id") or kwargs.get("accountId") + + async def _fetch_transactions_for_currencies( + self, + fetch_method_name: str, + currencies: list[str], + since: int = None, + limit: int = None, + kwargs: dict = None, + ) -> list[dict]: + fetch_method = getattr(self.connector, fetch_method_name) + merged_transactions = [] + seen_transaction_ids = set() + transaction_columns = trading_enums.ExchangeConstantsTransactionColumns + kwargs = kwargs or {} + + for wallet_currency in currencies: + for currency_type in (None, "crypto"): + local_kwargs = dict(kwargs) + if currency_type is not None: + local_kwargs["currencyType"] = currency_type + try: + currency_transactions = await fetch_method( + currency=wallet_currency, + since=since, + limit=limit, + **local_kwargs, + ) + except octobot_trading.errors.OctoBotTradingError as error: + self.logger.warning( + f"Skipping {self.get_name()} {fetch_method_name} for {wallet_currency} " + f"({currency_type or 'fiat'}): {error}" + ) + continue + for transaction in currency_transactions: + transaction_id = ( + transaction.get(transaction_columns.ID.value) + or transaction.get(transaction_columns.TXID.value) + ) + if transaction_id and transaction_id in seen_transaction_ids: + continue + if transaction_id: + seen_transaction_ids.add(transaction_id) + merged_transactions.append(transaction) + return merged_transactions + + async def get_deposits( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: + if self._uses_single_transaction_currency_request(currency, kwargs): + return await self.connector.get_deposits( + currency=currency, since=since, limit=limit, **kwargs + ) + if not currencies: + raise octobot_trading.errors.FailedRequest( + f"{self.get_name()} get_deposits requires a currency, account_id, or non-empty currencies list" + ) + return await self._fetch_transactions_for_currencies( + "get_deposits", currencies, since=since, limit=limit, kwargs=kwargs + ) + + async def get_withdrawals( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: + if self._uses_single_transaction_currency_request(currency, kwargs): + return await self.connector.get_withdrawals( + currency=currency, since=since, limit=limit, **kwargs + ) + if not currencies: + raise octobot_trading.errors.FailedRequest( + f"{self.get_name()} get_withdrawals requires a currency, account_id, or non-empty currencies list" + ) + return await self._fetch_transactions_for_currencies( + "get_withdrawals", currencies, since=since, limit=limit, kwargs=kwargs + ) diff --git a/packages/tentacles/Trading/Exchange/kraken/kraken_exchange.py b/packages/tentacles/Trading/Exchange/kraken/kraken_exchange.py index 9325ce841d..34c95ee8ef 100644 --- a/packages/tentacles/Trading/Exchange/kraken/kraken_exchange.py +++ b/packages/tentacles/Trading/Exchange/kraken/kraken_exchange.py @@ -17,6 +17,54 @@ class Kraken(exchanges.RestExchange): + """ + Kraken exchange connector. + + Portfolio history limitation (TradesHistory fees) + ------------------------------------------------- + Kraken TradesHistory always reports the ``fee`` field in quote currency, even + when the fee was actually deducted from base. See: + https://support.kraken.com/hc/en-us/articles/360001184886 + + CCXT ``kraken.parseTrade`` sets ``amount = vol`` and ``fee.currency = quote``. + Portfolio history reverse-replay uses ``fee.currency`` to decide which leg is + debited; when the currency is wrong, replay can show a small negative holding + for an asset before its first trade (e.g. -1.18966e-05 BTC when vol is gross + but the account was credited net of a base fee). + + Fee currency on Kraken orders (``oflags``) + ------------------------------------------ + Kraken expresses fee-currency preference per order via ``oflags`` on order + objects (not on trade history records): + + - ``fciq``: prefer fee in quote (default for buy orders) + - ``fcib``: prefer fee in base (default for sell orders) + + CCXT ``kraken.parseOrder`` reads these flags; ``parseTrade`` does not, + because TradesHistory fills only expose ``ordertxid`` — not ``oflags``. + + Why verified fees are not enriched on trades + ------------------------------------------ + Correcting ``fee.currency`` / net base quantity would require Kraken-specific + post-fetch enrichment, for example: + + - Batch ``QueryOrders`` by ``ordertxid`` to read ``oflags``, then convert the + quote-denominated ``fee`` to base when ``fcib`` applies; or + - Batch ledger lookups (``ledgers`` IDs on trades, or ``QueryLedgers``) for + authoritative credited/debited amounts. + + Tradeoffs (intentionally not implemented): + + - Extra authenticated API calls on every portfolio-history sync (rate limits, + latency, failure modes). + - ``oflags`` is a preference, not a guarantee (Kraken may fall back if the + chosen currency has insufficient balance); only ledger entries are exact. + - Enrichment is exchange-specific; replay builder stays generic. + + Latest portfolio balances remain correct; only historical pre-trade snapshots + for affected trades may be slightly wrong. + """ + @classmethod def get_name(cls): return 'kraken' diff --git a/packages/trading/octobot_trading/api/__init__.py b/packages/trading/octobot_trading/api/__init__.py index 06957d7708..2453e34577 100644 --- a/packages/trading/octobot_trading/api/__init__.py +++ b/packages/trading/octobot_trading/api/__init__.py @@ -262,9 +262,14 @@ load_latest_tickers, update_latest_tickers, get_daily_price, + get_latest_daily_close_on_or_before, get_latest_ticker_close, get_oldest_daily_price_timestamp, get_latest_daily_price_timestamp, + get_daily_close_source, + set_daily_close_source_in_memory, + merge_daily_prices_in_memory, + move_daily_prices_symbol_in_memory, ) __all__ = [ "get_symbol_data", @@ -472,7 +477,12 @@ "load_latest_tickers", "update_latest_tickers", "get_daily_price", + "get_latest_daily_close_on_or_before", "get_latest_ticker_close", "get_oldest_daily_price_timestamp", "get_latest_daily_price_timestamp", + "get_daily_close_source", + "set_daily_close_source_in_memory", + "merge_daily_prices_in_memory", + "move_daily_prices_symbol_in_memory", ] diff --git a/packages/trading/octobot_trading/api/exchange_data_cache.py b/packages/trading/octobot_trading/api/exchange_data_cache.py index 9a56986911..9d4be81676 100644 --- a/packages/trading/octobot_trading/api/exchange_data_cache.py +++ b/packages/trading/octobot_trading/api/exchange_data_cache.py @@ -1,3 +1,4 @@ +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types import octobot_trading.exchange_data.prices.persisted_price_cache as persisted_price_cache import octobot_trading.exchange_data.ticker.persisted_ticker_cache as persisted_ticker_cache @@ -7,7 +8,7 @@ async def load_daily_prices( exchange_type: str, sandboxed: bool, data_root: str = None, -) -> dict: +) -> daily_prices_cache_types.DailyPricesCache: return await persisted_price_cache.load(exchange_name, exchange_type, sandboxed, data_root) @@ -55,7 +56,7 @@ async def load_latest_tickers( exchange_type: str, sandboxed: bool, data_root: str = None, -) -> dict: +) -> daily_prices_cache_types.LatestTickersCache: return await persisted_ticker_cache.load(exchange_name, exchange_type, sandboxed, data_root) @@ -69,17 +70,71 @@ async def update_latest_tickers( await persisted_ticker_cache.update(exchange_name, exchange_type, sandboxed, closes, data_root) -def get_daily_price(data: dict, symbol: str, day_timestamp: str, default: float | None = None) -> float | None: +def get_daily_price( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + day_timestamp: str, + default: float | None = None, +) -> float | None: return persisted_price_cache.get_close(data, symbol, day_timestamp, default) -def get_latest_ticker_close(data: dict, symbol: str, default: float | None = None) -> float | None: +def get_latest_daily_close_on_or_before( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + day_timestamp: float, +) -> float | None: + return persisted_price_cache.latest_close_on_or_before(data, symbol, day_timestamp) + + +def get_latest_ticker_close( + data: daily_prices_cache_types.LatestTickersCache, + symbol: str, + default: float | None = None, +) -> float | None: return persisted_ticker_cache.get_close(data, symbol, default) -def get_oldest_daily_price_timestamp(data: dict, symbol: str) -> float | None: +def get_oldest_daily_price_timestamp( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> float | None: return persisted_price_cache.oldest_timestamp(data, symbol) -def get_latest_daily_price_timestamp(data: dict, symbol: str) -> float | None: +def get_latest_daily_price_timestamp( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> float | None: return persisted_price_cache.newest_timestamp(data, symbol) + + +def get_daily_close_source( + data: daily_prices_cache_types.DailyPricesCache, + base_asset: str, +) -> str | None: + return persisted_price_cache.get_close_source(data, base_asset) + + +def set_daily_close_source_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + base_asset: str, + fetch_symbol: str, +) -> None: + persisted_price_cache.set_close_source_in_memory(data, base_asset, fetch_symbol) + + +def merge_daily_prices_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + closes_by_timestamp: dict[str, float], +) -> None: + persisted_price_cache.merge_symbol_closes_in_memory(data, symbol, closes_by_timestamp) + + +def move_daily_prices_symbol_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + old_symbol: str, + new_symbol: str, +) -> None: + persisted_price_cache.move_symbol_closes_in_memory(data, old_symbol, new_symbol) diff --git a/packages/trading/octobot_trading/enums.py b/packages/trading/octobot_trading/enums.py index 61d405c825..c51ab30037 100644 --- a/packages/trading/octobot_trading/enums.py +++ b/packages/trading/octobot_trading/enums.py @@ -701,6 +701,16 @@ class ExchangeFeatureKeys(enum.Enum): SUPPORTED_BUNDLED_ORDERS = "supported_bundled_orders" +class DailyPricesCacheKeys(enum.StrEnum): + SYMBOLS = "symbols" + SOURCES = "sources" + + +class LatestTickersCacheKeys(enum.StrEnum): + UPDATED_AT = "updated_at" + CLOSES = "closes" + + class ExchangeClientOptions(enum.StrEnum): FIX_MARKET_STATUS = "fixMarketStatus" REMOVE_MARKET_STATUS_PRICE_LIMITS = "removeMarketStatusPriceLimits" @@ -748,6 +758,10 @@ class ExchangeClientOptions(enum.StrEnum): SUPPORTS_FORCED_SIGNING_ALL_REQUESTS = "supportsForcedSigningAllRequests" ENABLE_FORCED_SIGNING_ALL_REQUESTS = "enableForcedSigningAllRequests" SUPPORTED_ELEMENTS = "supportedElements" + MY_TRADES_SYMBOL_FILTER_IS_CLIENT_SIDE = "myTradesSymbolFilterIsClientSide" + MY_TRADES_FETCH_PAGINATION_OFFSET = "myTradesFetchPaginationOffset" + MY_TRADES_FETCH_USE_CCXT_PAGINATE = "myTradesFetchUseCcxtPaginate" + CLOSED_ORDERS_FETCH_USE_CCXT_PAGINATE = "closedOrdersFetchUseCcxtPaginate" class ExchangeSupportedElements(enum.StrEnum): @@ -853,6 +867,10 @@ class ExchangeSupportedElements(enum.StrEnum): ExchangeClientOptions.SUPPORTS_FORCED_SIGNING_ALL_REQUESTS: False, # set True when the exchange requires signing all requests (when supported) ExchangeClientOptions.ENABLE_FORCED_SIGNING_ALL_REQUESTS: False, + ExchangeClientOptions.MY_TRADES_SYMBOL_FILTER_IS_CLIENT_SIDE: False, + ExchangeClientOptions.MY_TRADES_FETCH_PAGINATION_OFFSET: None, + ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE: False, + ExchangeClientOptions.CLOSED_ORDERS_FETCH_USE_CCXT_PAGINATE: False, ExchangeClientOptions.SUPPORTED_ELEMENTS: { ExchangeSupportedElements.FUTURES: { ExchangeSupportedElements.ORDERS: [TradeOrderType.MARKET.value, TradeOrderType.LIMIT.value], diff --git a/packages/trading/octobot_trading/exchange_data/databases/market_data_sqlite_database.py b/packages/trading/octobot_trading/exchange_data/databases/market_data_sqlite_database.py index cef59e6aa0..a7821586f9 100644 --- a/packages/trading/octobot_trading/exchange_data/databases/market_data_sqlite_database.py +++ b/packages/trading/octobot_trading/exchange_data/databases/market_data_sqlite_database.py @@ -6,7 +6,9 @@ import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums import octobot_trading.exchange_data.exchange_cache_key as exchange_cache_key_module +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types MARKET_DATA_DB_FILENAME = "market_data.sqlite" SCHEMA_VERSION = 1 @@ -93,15 +95,18 @@ async def merge_daily_closes(self, symbol: str, closes_by_timestamp: dict[str, f ) await self.commit() - async def load_daily_prices_dict(self) -> dict: + async def load_daily_prices_dict(self) -> daily_prices_cache_types.DailyPricesCache: if not await self._table_exists("daily_closes"): - return {"symbols": {}, "sources": {}} + return daily_prices_cache_types.empty_daily_prices_cache() rows = await self.fetchall("SELECT symbol, day_ts, close FROM daily_closes") symbols: dict[str, dict[str, float]] = {} for symbol, day_timestamp, close in rows: symbols.setdefault(symbol, {})[str(day_timestamp)] = close sources = await self._load_daily_close_sources() - return {"symbols": symbols, "sources": sources} + return { + trading_enums.DailyPricesCacheKeys.SYMBOLS: symbols, + trading_enums.DailyPricesCacheKeys.SOURCES: sources, + } async def _load_daily_close_sources(self) -> dict[str, str]: if not await self._table_exists("daily_close_sources"): @@ -178,13 +183,16 @@ async def update_latest_tickers(self, closes: dict[str, float]) -> None: ) await self.commit() - async def load_latest_tickers_dict(self) -> dict: + async def load_latest_tickers_dict(self) -> daily_prices_cache_types.LatestTickersCache: if not await self._table_exists("latest_tickers"): - return {"updated_at": None, "closes": {}} + return daily_prices_cache_types.empty_latest_tickers_cache() rows = await self.fetchall("SELECT symbol, close, updated_at FROM latest_tickers") closes = {symbol: close for symbol, close, _updated_at in rows} updated_at = max((row[2] for row in rows), default=None) if rows else None - return {"updated_at": updated_at, "closes": closes} + return { + trading_enums.LatestTickersCacheKeys.UPDATED_AT: updated_at, + trading_enums.LatestTickersCacheKeys.CLOSES: closes, + } @contextlib.asynccontextmanager diff --git a/packages/trading/octobot_trading/exchange_data/prices/daily_prices_cache_types.py b/packages/trading/octobot_trading/exchange_data/prices/daily_prices_cache_types.py new file mode 100644 index 0000000000..1e7bb1d374 --- /dev/null +++ b/packages/trading/octobot_trading/exchange_data/prices/daily_prices_cache_types.py @@ -0,0 +1,27 @@ +import typing + +import octobot_trading.enums as trading_enums + + +class DailyPricesCache(typing.TypedDict): + symbols: dict[str, dict[str, float]] + sources: dict[str, str] + + +class LatestTickersCache(typing.TypedDict): + updated_at: typing.Optional[float] + closes: dict[str, float] + + +def empty_daily_prices_cache() -> DailyPricesCache: + return { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + + +def empty_latest_tickers_cache() -> LatestTickersCache: + return { + trading_enums.LatestTickersCacheKeys.UPDATED_AT: None, + trading_enums.LatestTickersCacheKeys.CLOSES: {}, + } diff --git a/packages/trading/octobot_trading/exchange_data/prices/persisted_price_cache.py b/packages/trading/octobot_trading/exchange_data/prices/persisted_price_cache.py index 34e7693e73..bf10f14297 100644 --- a/packages/trading/octobot_trading/exchange_data/prices/persisted_price_cache.py +++ b/packages/trading/octobot_trading/exchange_data/prices/persisted_price_cache.py @@ -1,5 +1,7 @@ import octobot_commons.errors as commons_errors +import octobot_trading.enums as trading_enums import octobot_trading.exchange_data.databases.market_data_sqlite_database as market_data_sqlite_database_module +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types async def load( @@ -7,7 +9,7 @@ async def load( exchange_type: str, sandboxed: bool, data_root: str = None, -) -> dict: +) -> daily_prices_cache_types.DailyPricesCache: """Load daily closes from market_data.sqlite; return empty structure when missing.""" try: async with market_data_sqlite_database_module.open_market_data_sqlite_database( @@ -15,7 +17,7 @@ async def load( ) as database: return await database.load_daily_prices_dict() except commons_errors.DatabaseNotFoundError: - return {"symbols": {}, "sources": {}} + return daily_prices_cache_types.empty_daily_prices_cache() async def merge_closes( @@ -63,32 +65,109 @@ async def rename_daily_closes_symbol( await database.rename_daily_closes_symbol(old_symbol, new_symbol) -def _resolve_symbol(data: dict, symbol: str) -> str: +def _resolve_symbol( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> str: if "/" not in symbol: return symbol base_asset, _quote = symbol.split("/", 1) - return data.get("sources", {}).get(base_asset, symbol) + return data[trading_enums.DailyPricesCacheKeys.SOURCES].get(base_asset, symbol) -def oldest_timestamp(data: dict, symbol: str) -> float | None: +def oldest_timestamp( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> float | None: """Return the oldest cached day timestamp for a symbol, or None.""" resolved_symbol = _resolve_symbol(data, symbol) - symbol_data = data.get("symbols", {}).get(resolved_symbol, {}) + symbol_data = data[trading_enums.DailyPricesCacheKeys.SYMBOLS].get(resolved_symbol, {}) if not symbol_data: return None return min(float(timestamp) for timestamp in symbol_data) -def newest_timestamp(data: dict, symbol: str) -> float | None: +def newest_timestamp( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> float | None: """Return the newest cached day timestamp for a symbol, or None.""" resolved_symbol = _resolve_symbol(data, symbol) - symbol_data = data.get("symbols", {}).get(resolved_symbol, {}) + symbol_data = data[trading_enums.DailyPricesCacheKeys.SYMBOLS].get(resolved_symbol, {}) if not symbol_data: return None return max(float(timestamp) for timestamp in symbol_data) -def get_close(data: dict, symbol: str, day_timestamp: str, default: float | None = None) -> float | None: +def get_close( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + day_timestamp: str, + default: float | None = None, +) -> float | None: """Look up a close price for a symbol at a specific day timestamp.""" resolved_symbol = _resolve_symbol(data, symbol) - return data.get("symbols", {}).get(resolved_symbol, {}).get(day_timestamp, default) + return data[trading_enums.DailyPricesCacheKeys.SYMBOLS].get(resolved_symbol, {}).get( + day_timestamp, default, + ) + + +def latest_close_on_or_before( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + day_timestamp: float, +) -> float | None: + """Return the latest cached close on or before the given day timestamp.""" + resolved_symbol = _resolve_symbol(data, symbol) + symbol_data = data[trading_enums.DailyPricesCacheKeys.SYMBOLS].get(resolved_symbol, {}) + if not symbol_data: + return None + day_timestamp_int = int(day_timestamp) + latest_timestamp = None + latest_close = None + for timestamp_str, close_price in symbol_data.items(): + timestamp_int = int(timestamp_str) + if timestamp_int > day_timestamp_int: + continue + if latest_timestamp is None or timestamp_int > latest_timestamp: + latest_timestamp = timestamp_int + latest_close = close_price + return latest_close + + +def get_close_source( + data: daily_prices_cache_types.DailyPricesCache, + base_asset: str, +) -> str | None: + """Return the exchange symbol used to fetch daily closes for a base asset.""" + return data[trading_enums.DailyPricesCacheKeys.SOURCES].get(base_asset) + + +def set_close_source_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + base_asset: str, + fetch_symbol: str, +) -> None: + """Record in memory which exchange symbol backs daily closes for a base asset.""" + data[trading_enums.DailyPricesCacheKeys.SOURCES][base_asset] = fetch_symbol + + +def merge_symbol_closes_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + symbol: str, + closes_by_timestamp: dict[str, float], +) -> None: + """Merge close prices for a symbol into the in-memory daily prices cache.""" + data[trading_enums.DailyPricesCacheKeys.SYMBOLS].setdefault(symbol, {}).update( + closes_by_timestamp, + ) + + +def move_symbol_closes_in_memory( + data: daily_prices_cache_types.DailyPricesCache, + old_symbol: str, + new_symbol: str, +) -> None: + old_closes = data[trading_enums.DailyPricesCacheKeys.SYMBOLS].pop(old_symbol, {}) + if old_closes: + data[trading_enums.DailyPricesCacheKeys.SYMBOLS].setdefault(new_symbol, {}).update(old_closes) diff --git a/packages/trading/octobot_trading/exchange_data/ticker/persisted_ticker_cache.py b/packages/trading/octobot_trading/exchange_data/ticker/persisted_ticker_cache.py index ba2dc990f0..29f37d0501 100644 --- a/packages/trading/octobot_trading/exchange_data/ticker/persisted_ticker_cache.py +++ b/packages/trading/octobot_trading/exchange_data/ticker/persisted_ticker_cache.py @@ -1,5 +1,7 @@ import octobot_commons.errors as commons_errors +import octobot_trading.enums as trading_enums import octobot_trading.exchange_data.databases.market_data_sqlite_database as market_data_sqlite_database_module +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types async def load( @@ -7,7 +9,7 @@ async def load( exchange_type: str, sandboxed: bool, data_root: str = None, -) -> dict: +) -> daily_prices_cache_types.LatestTickersCache: """Load latest tickers from market_data.sqlite; return empty structure when missing.""" try: async with market_data_sqlite_database_module.open_market_data_sqlite_database( @@ -15,7 +17,7 @@ async def load( ) as database: return await database.load_latest_tickers_dict() except commons_errors.DatabaseNotFoundError: - return {"updated_at": None, "closes": {}} + return daily_prices_cache_types.empty_latest_tickers_cache() async def update( @@ -32,6 +34,10 @@ async def update( await database.update_latest_tickers(closes) -def get_close(data: dict, symbol: str, default: float | None = None) -> float | None: +def get_close( + data: daily_prices_cache_types.LatestTickersCache, + symbol: str, + default: float | None = None, +) -> float | None: """Look up the latest close price for a symbol.""" - return data.get("closes", {}).get(symbol, default) + return data[trading_enums.LatestTickersCacheKeys.CLOSES].get(symbol, default) diff --git a/packages/trading/octobot_trading/exchanges/abstract_exchange.py b/packages/trading/octobot_trading/exchanges/abstract_exchange.py index 25f870e922..55d178dbc8 100644 --- a/packages/trading/octobot_trading/exchanges/abstract_exchange.py +++ b/packages/trading/octobot_trading/exchanges/abstract_exchange.py @@ -310,12 +310,24 @@ async def get_my_recent_trades(self, symbol: str = None, since: int = None, """ raise NotImplementedError("get_my_recent_trades is not implemented") - async def get_deposits(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: + async def get_deposits( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: raise NotImplementedError("get_deposits is not implemented") - async def get_withdrawals(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: + async def get_withdrawals( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: raise NotImplementedError("get_withdrawals is not implemented") async def get_user_recent_trades(self, user_id: str, symbol: str = None, since: int = None, diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_adapter.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_adapter.py index 94ea18ecab..aacd1c717e 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_adapter.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_adapter.py @@ -474,8 +474,17 @@ def parse_dex_pairs(self, fixed: list[dict], **kwargs) -> list[dict]: }) return fixed + def parse_transactions(self, fixed, **kwargs) -> list[dict]: + return [ + self.parse_transaction(transaction, **kwargs) + for transaction in fixed + ] + def parse_transaction(self, fixed: CCXTTransaction, **kwargs) -> dict: # CCXT standard transaction parsing logic + transaction_type = kwargs.get("transaction_type") + if transaction_type is None: + raise ValueError("transaction_type is required when parsing CCXT transactions") return { enums.ExchangeConstantsTransactionColumns.ID.value: fixed.get(enums.ExchangeConstantsTransactionColumns.ID.value), enums.ExchangeConstantsTransactionColumns.TXID.value: fixed.get(enums.ExchangeConstantsTransactionColumns.TXID.value), @@ -483,7 +492,7 @@ def parse_transaction(self, fixed: CCXTTransaction, **kwargs) -> dict: enums.ExchangeConstantsTransactionColumns.ADDRESS_FROM.value: fixed.get(enums.ExchangeConstantsTransactionColumns.ADDRESS_FROM.value), enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value: fixed.get(enums.ExchangeConstantsTransactionColumns.ADDRESS_TO.value), enums.ExchangeConstantsTransactionColumns.TAG.value: fixed.get(enums.ExchangeConstantsTransactionColumns.TAG.value), - enums.ExchangeConstantsTransactionColumns.TYPE.value: fixed.get(enums.ExchangeConstantsTransactionColumns.TYPE.value), + enums.ExchangeConstantsTransactionColumns.TYPE.value: transaction_type.value, enums.ExchangeConstantsTransactionColumns.AMOUNT.value: decimal.Decimal(str(fixed.get(enums.ExchangeConstantsTransactionColumns.AMOUNT.value, 0))), enums.ExchangeConstantsTransactionColumns.CURRENCY.value: fixed.get(enums.ExchangeConstantsTransactionColumns.CURRENCY.value), enums.ExchangeConstantsTransactionColumns.STATUS.value: fixed.get(enums.ExchangeConstantsTransactionColumns.STATUS.value), diff --git a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py index dbae66590e..529c555ccb 100644 --- a/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py +++ b/packages/trading/octobot_trading/exchanges/connectors/ccxt/ccxt_connector.py @@ -48,11 +48,22 @@ import octobot_trading.exchanges.connectors.ccxt.enums as ccxt_enums import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants import octobot_trading.exchanges.connectors.util as connectors_util +import octobot_trading.personal_data.trades.trades_util as trades_util_module import octobot_trading.personal_data as personal_data from octobot_trading.enums import ExchangeConstantsOrderColumns as ecoc _MARKETS_LOAD_LOCKS: dict[str, asyncio.Lock] = {} +PAGINATION_FULL_PAGE_MIN_SIZE = 10 +MAX_MY_TRADES_OFFSET_PAGINATION_REQUESTS = 200 + + +def _is_potentially_full_page(trade_count: int) -> bool: + # Exchanges (e.g. Kraken TradesHistory) return fixed-size pages (often 50) but we + # do not hardcode page size. A count that is a positive multiple of 10 suggests the + # API may have more data; stop when the page is empty, tiny (<10), or not aligned + # to 10 (e.g. 25 = last page). Avoids needing MY_TRADES_FETCH_PAGE_SIZE per exchange. + return trade_count >= PAGINATION_FULL_PAGE_MIN_SIZE and trade_count % PAGINATION_FULL_PAGE_MIN_SIZE == 0 def _get_markets_load_lock(client_key: str) -> asyncio.Lock: @@ -216,15 +227,6 @@ async def _filtered_if_necessary_load_markets( market_filter: typing.Optional[typing.Callable[[dict], bool]] ): try: - if self.exchange_manager.exchange.get_option_value( - enums.ExchangeClientOptions.ADJUST_FOR_TIME_DIFFERENCE - ): - # load time difference before loading markets in case a signature is needed to load markets - try: - await client.load_time_difference() - except Exception as err: - # don't crash when loading time difference - self.logger.error(f"Error loading time difference for {self.exchange_manager.exchange_name}: {err}") if self.exchange_manager.exchange.FETCH_MIN_EXCHANGE_MARKETS and market_filter: with ccxt_client_util.filtered_fetched_markets(client, market_filter): await client.load_markets(reload=reload) @@ -257,6 +259,30 @@ async def _load_markets( """ await self._filtered_if_necessary_load_markets(client, reload, market_filter) + async def _ensure_time_difference_synced( + self, + client, + authenticated_cache: bool, + ): + if not self.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.ADJUST_FOR_TIME_DIFFERENCE + ): + return + try: + client_key = ccxt_clients_cache.get_client_key(client, authenticated_cache) + if cached_time_difference := ccxt_clients_cache.get_exchange_time_difference(client_key): + if client.options is not None: + client.options[ccxt_constants.CCXT_TIME_DIFFERENCE] = cached_time_difference + return + await client.load_time_difference() + if time_difference := client.options.get(ccxt_constants.CCXT_TIME_DIFFERENCE): + ccxt_clients_cache.set_exchange_time_difference(client_key, time_difference) + except Exception as err: + self.logger.exception( + err, True, + f"Error loading time difference for {self.exchange_manager.exchange_name}: {err}" + ) + def set_first_consecutive_authentication_error_at_if_unset(self): if self.first_consecutive_authentication_error_at is None: self.first_consecutive_authentication_error_at = self.get_exchange_current_time() @@ -276,6 +302,8 @@ async def load_symbol_markets( reload = True self._force_next_market_reload = False authenticated_cache = self.exchange_manager.exchange.requires_authentication_for_this_configuration_only() + if self.is_authenticated: + await self._ensure_time_difference_synced(self.client, authenticated_cache) force_load_markets = reload if not force_load_markets: try: @@ -850,9 +878,17 @@ async def get_user_open_orders(self, user_id: str, symbol: str = None, since: in @ccxt_client_util.converted_ccxt_common_errors async def get_closed_orders(self, symbol: str = None, since: int = None, limit: int = None, **kwargs: dict) -> list[dict]: + exhaust_history = bool(kwargs.pop("exhaust_history", False)) + request_params = dict(kwargs) + if exhaust_history and self.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.CLOSED_ORDERS_FETCH_USE_CCXT_PAGINATE + ): + request_params["paginate"] = True with self.error_describer(True): return self.adapter.adapt_orders( - await self.client.fetch_closed_orders(symbol=symbol, since=since, limit=limit, params=kwargs), + await self.client.fetch_closed_orders( + symbol=symbol, since=since, limit=limit, params=request_params + ), symbol=symbol ) @@ -876,24 +912,79 @@ async def get_cancelled_orders(self, symbol: str = None, since: int = None, @ccxt_client_util.converted_ccxt_common_errors async def get_my_recent_trades(self, symbol: str = None, since: int = None, limit: int = None, **kwargs: dict) -> list[dict]: + exhaust_history = bool(kwargs.pop("exhaust_history", False)) if self.client.has['fetchMyTrades'] or self.client.has['fetchTrades']: with self.error_describer(True): method = self.client.fetch_my_trades if self.client.has['fetchMyTrades'] else self.client.fetch_trades - trades = self.adapter.adapt_trades(await method(symbol=symbol, since=since, limit=limit, params=kwargs)) + offset_param = self.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.MY_TRADES_FETCH_PAGINATION_OFFSET + ) + if exhaust_history and offset_param is not None: + return await self._fetch_my_recent_trades_with_offset_pagination( + method=method, + symbol=symbol, + since=since, + limit=limit, + offset_param=str(offset_param), + request_params=kwargs, + ) + request_params = dict(kwargs) + if exhaust_history and self.exchange_manager.exchange.get_option_value( + enums.ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE + ): + request_params["paginate"] = True + trades = self.adapter.adapt_trades( + await method(symbol=symbol, since=since, limit=limit, params=request_params) + ) if trades or not self.exchange_manager.exchange.get_option_value( enums.ExchangeClientOptions.ALLOW_TRADES_FROM_CLOSED_ORDERS ): return trades - # on some exchanges, recent trades are only fetching very recent trade. also try closed orders return await self.exchange_manager.exchange.get_closed_orders( symbol=symbol, since=since, limit=limit, + exhaust_history=exhaust_history, **kwargs ) else: raise octobot_trading.errors.NotSupported("This exchange doesn't support fetchMyTrades nor fetchTrades") + async def _fetch_my_recent_trades_with_offset_pagination( + self, + method, + symbol: str | None, + since: int | None, + limit: int | None, + offset_param: str, + request_params: dict, + ) -> list[dict]: + offset = 0 + merged_trades: list[dict] = [] + request_count = 0 + while request_count < MAX_MY_TRADES_OFFSET_PAGINATION_REQUESTS: + request_count += 1 + page_params = dict(request_params) + page_params[offset_param] = offset + trade_page = self.adapter.adapt_trades( + await method(symbol=symbol, since=since, limit=limit, params=page_params) + ) + merged_trades = trades_util_module.merge_trades_deduped(merged_trades, trade_page) + if not _is_potentially_full_page(len(trade_page)): + break + offset += len(trade_page) + else: + self.logger.warning( + "Stopped %s my-trades offset pagination after %d requests " + "(offset_param=%s, offset=%d, symbol=%s); exchange may have more history", + self.exchange_manager.exchange_name, + MAX_MY_TRADES_OFFSET_PAGINATION_REQUESTS, + offset_param, + offset, + symbol, + ) + return merged_trades + @ccxt_client_util.converted_ccxt_common_errors async def get_user_recent_trades(self, user_id: str, symbol: str = None, since: int = None, limit: int = None, **kwargs: dict) -> list: @@ -906,26 +997,40 @@ async def get_user_recent_trades(self, user_id: str, symbol: str = None, since: raise octobot_trading.errors.NotSupported("This exchange doesn't support fetchUserRecentTrades") @ccxt_client_util.converted_ccxt_common_errors - async def get_deposits(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: + async def get_deposits( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: try: if self.client.has['fetchDeposits']: with self.error_describer(True): return self.adapter.adapt_transactions( - await self.client.fetch_deposits(code=currency, since=since, limit=limit, params=kwargs) + await self.client.fetch_deposits(code=currency, since=since, limit=limit, params=kwargs), + transaction_type=enums.TransactionType.BLOCKCHAIN_DEPOSIT, ) raise octobot_trading.errors.NotSupported("This exchange doesn't support fetchDeposits") except NotImplementedError as error: raise octobot_trading.errors.NotSupported(str(error)) from error @ccxt_client_util.converted_ccxt_common_errors - async def get_withdrawals(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: + async def get_withdrawals( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: try: if self.client.has['fetchWithdrawals']: with self.error_describer(True): return self.adapter.adapt_transactions( - await self.client.fetch_withdrawals(code=currency, since=since, limit=limit, params=kwargs) + await self.client.fetch_withdrawals(code=currency, since=since, limit=limit, params=kwargs), + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, ) raise octobot_trading.errors.NotSupported("This exchange doesn't support fetchWithdrawals") except NotImplementedError as error: @@ -1198,7 +1303,8 @@ async def withdraw( params = params or {} params[self.exchange_manager.exchange.WITHDRAW_NETWORK_PARAM_KEY] = network return self.adapter.adapt_transaction( - await self.client.withdraw(asset, float(amount), address, tag=tag, params=params) + await self.client.withdraw(asset, float(amount), address, tag=tag, params=params), + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, ) async def get_deposit_address(self, asset: str, params: dict = None) -> dict: diff --git a/packages/trading/octobot_trading/exchanges/types/rest_exchange.py b/packages/trading/octobot_trading/exchanges/types/rest_exchange.py index b0c7b13286..5aaac079e3 100644 --- a/packages/trading/octobot_trading/exchanges/types/rest_exchange.py +++ b/packages/trading/octobot_trading/exchanges/types/rest_exchange.py @@ -861,13 +861,29 @@ async def get_my_recent_trades(self, symbol: str = None, since: int = None, limi async def get_user_recent_trades(self, user_id: str, symbol: str = None, since: int = None, limit: int = None, **kwargs: dict) -> list: return await self.connector.get_user_recent_trades(user_id=user_id, symbol=symbol, since=since, limit=limit, **kwargs) - async def get_deposits(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: - return await self.connector.get_deposits(currency=currency, since=since, limit=limit, **kwargs) + async def get_deposits( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: + return await self.connector.get_deposits( + currency=currency, since=since, limit=limit, currencies=currencies, **kwargs + ) - async def get_withdrawals(self, currency: str = None, since: int = None, - limit: int = None, **kwargs: dict) -> list[dict]: - return await self.connector.get_withdrawals(currency=currency, since=since, limit=limit, **kwargs) + async def get_withdrawals( + self, + currency: str = None, + since: int = None, + limit: int = None, + currencies: typing.Optional[list[str]] = None, + **kwargs: dict, + ) -> list[dict]: + return await self.connector.get_withdrawals( + currency=currency, since=since, limit=limit, currencies=currencies, **kwargs + ) async def cancel_all_orders(self, symbol: str = None, **kwargs: dict) -> None: return await self.connector.cancel_all_orders(symbol=symbol, **kwargs) diff --git a/packages/trading/octobot_trading/personal_data/trades/channel/trades_updater.py b/packages/trading/octobot_trading/personal_data/trades/channel/trades_updater.py index 4bbc2d5c62..d3af02942c 100644 --- a/packages/trading/octobot_trading/personal_data/trades/channel/trades_updater.py +++ b/packages/trading/octobot_trading/personal_data/trades/channel/trades_updater.py @@ -70,26 +70,48 @@ async def init_trade_history(self): except Exception as error: self.logger.error(f"Fail to initialize trade history : {html_util.get_html_summary_if_relevant(error)}") + @staticmethod + async def _fetch_trades_for_symbol( + exchange, + symbol: str, + *, + limit: int | None = None, + exhaust_history: bool = False, + ) -> list: + if exhaust_history: + return await exchange.get_my_recent_trades(symbol=symbol, exhaust_history=True) + return await exchange.get_my_recent_trades(symbol=symbol, limit=limit) + async def fetch_trades( self, symbols: list[str], limit: int = MAX_OLD_TRADES_TO_FETCH, + *, + exhaust_history: bool = False, ) -> list: """ Fetch recent trades from the exchange for the given symbols. This is the only method that calls exchange.get_my_recent_trades. + When exhaust_history is True and symbols is empty, fetches account-wide history. """ + exchange = self.channel.exchange_manager.exchange + if exhaust_history and not symbols: + trades = await exchange.get_my_recent_trades(symbol=None, exhaust_history=True) + return trades or [] if not symbols: return [] - exchange = self.channel.exchange_manager.exchange if len(symbols) == 1: - trades = await exchange.get_my_recent_trades(symbol=symbols[0], limit=limit) + trades = await TradesUpdater._fetch_trades_for_symbol( + exchange, symbols[0], limit=limit, exhaust_history=exhaust_history, + ) return trades or [] trade_batches = await asyncio_tools.gather_waiting_for_all_before_raising( *[ - exchange.get_my_recent_trades(symbol=trading_symbol, limit=limit) + TradesUpdater._fetch_trades_for_symbol( + exchange, trading_symbol, limit=limit, exhaust_history=exhaust_history, + ) for trading_symbol in symbols ] ) diff --git a/packages/trading/tests/exchange_data/databases/test_market_data_sqlite_database.py b/packages/trading/tests/exchange_data/databases/test_market_data_sqlite_database.py index f87295beaa..0beb67deac 100644 --- a/packages/trading/tests/exchange_data/databases/test_market_data_sqlite_database.py +++ b/packages/trading/tests/exchange_data/databases/test_market_data_sqlite_database.py @@ -5,6 +5,7 @@ import pytest import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database_module import octobot_trading.exchange_data.prices.persisted_price_cache as persisted_price_cache import octobot_trading.exchange_data.databases.market_data_sqlite_database as market_data_sqlite_database_module @@ -78,7 +79,7 @@ async def test_merge_and_overwrite_same_day(self, data_root): await database.merge_daily_closes("BTC/USDT", {"1000": 42000.0, "2000": 43000.0}) await database.merge_daily_closes("BTC/USDT", {"1000": 42500.0}) result = await database.load_daily_prices_dict() - assert result["symbols"]["BTC/USDT"] == {"1000": 42500.0, "2000": 43000.0} + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 42500.0, "2000": 43000.0} class TestLoadDailyPricesDict: @@ -92,7 +93,10 @@ async def test_empty_database(self, data_root): "binance", "spot", False, data_root, read_only=True ) as database: result = await database.load_daily_prices_dict() - assert result == {"symbols": {}, "sources": {}} + assert result == { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } @pytest.mark.asyncio async def test_multi_symbol_shape(self, data_root): @@ -103,8 +107,8 @@ async def test_multi_symbol_shape(self, data_root): await database.merge_daily_closes("ETH/USDT", {"1000": 2.0}) result = await database.load_daily_prices_dict() assert result == { - "symbols": {"BTC/USDT": {"1000": 1.0}, "ETH/USDT": {"1000": 2.0}}, - "sources": {}, + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1000": 1.0}, "ETH/USDT": {"1000": 2.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, } @@ -117,7 +121,7 @@ async def test_set_and_load_source(self, data_root): await database.set_daily_close_source("KNC", "KNC/USD") result = await database.load_daily_prices_dict() assert await database.get_daily_close_source("KNC") == "KNC/USD" - assert result["sources"] == {"KNC": "KNC/USD"} + assert result[trading_enums.DailyPricesCacheKeys.SOURCES] == {"KNC": "KNC/USD"} class TestRenameDailyClosesSymbol: @@ -129,8 +133,8 @@ async def test_moves_rows_to_new_symbol(self, data_root): await database.merge_daily_closes("KNC/USD", {"1000": 1.0, "2000": 1.1}) await database.rename_daily_closes_symbol("KNC/USD", "KNC/USDC") result = await database.load_daily_prices_dict() - assert "KNC/USD" not in result["symbols"] - assert result["symbols"]["KNC/USDC"] == {"1000": 1.0, "2000": 1.1} + assert "KNC/USD" not in result[trading_enums.DailyPricesCacheKeys.SYMBOLS] + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USDC"] == {"1000": 1.0, "2000": 1.1} @pytest.mark.asyncio async def test_conflict_keeps_migrated_value(self, data_root): @@ -141,7 +145,7 @@ async def test_conflict_keeps_migrated_value(self, data_root): await database.merge_daily_closes("KNC/USDC", {"1000": 2.0}) await database.rename_daily_closes_symbol("KNC/USD", "KNC/USDC") result = await database.load_daily_prices_dict() - assert result["symbols"]["KNC/USDC"] == {"1000": 1.0} + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USDC"] == {"1000": 1.0} class TestOldestNewestDayTs: @@ -165,8 +169,8 @@ async def test_merge_and_updated_at(self, data_root): ) as database: await database.update_latest_tickers({"BTC/USDT": 65000.0}) result = await database.load_latest_tickers_dict() - assert result["closes"]["BTC/USDT"] == 65000.0 - assert result["updated_at"] is not None + assert result[trading_enums.LatestTickersCacheKeys.CLOSES]["BTC/USDT"] == 65000.0 + assert result[trading_enums.LatestTickersCacheKeys.UPDATED_AT] is not None class TestPerDbPathLock: @@ -184,8 +188,8 @@ async def merge_closes(close_value: float): merge_closes(1.0), merge_closes(2.0), ) - final = first_result if first_result["symbols"] else second_result - assert final["symbols"]["BTC/USDT"]["1000"] in (1.0, 2.0) + final = first_result if first_result[trading_enums.DailyPricesCacheKeys.SYMBOLS] else second_result + assert final[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] in (1.0, 2.0) class TestConcurrentReadOnlyOpens: @@ -206,8 +210,8 @@ async def read_prices(): started_at = time.monotonic() first_result, second_result = await asyncio.gather(read_prices(), read_prices()) elapsed = time.monotonic() - started_at - assert first_result["symbols"]["BTC/USDT"]["1000"] == 42000.0 - assert second_result["symbols"]["BTC/USDT"]["1000"] == 42000.0 + assert first_result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] == 42000.0 + assert second_result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] == 42000.0 assert elapsed < _CONCURRENT_READ_MAX_ELAPSED_SECONDS @@ -226,7 +230,7 @@ async def test_committed_data_passes_integrity_check_after_reopen(self, data_roo result = await database.load_daily_prices_dict() assert integrity_row == ("ok",) - assert result["symbols"]["BTC/USDT"]["1000"] == 42000.0 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] == 42000.0 @pytest.mark.asyncio async def test_uncommitted_write_discarded_after_abrupt_connection_close(self, data_root): @@ -262,7 +266,7 @@ async def test_uncommitted_write_discarded_after_abrupt_connection_close(self, d result = await database.load_daily_prices_dict() assert integrity_row == ("ok",) - assert result["symbols"]["BTC/USDT"] == {"1000": 1.0} + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 1.0} @pytest.mark.asyncio async def test_wal_truncated_after_graceful_close(self, data_root): @@ -284,7 +288,7 @@ async def test_wal_truncated_after_graceful_close(self, data_root): result = await database.load_daily_prices_dict() assert integrity_row == ("ok",) - assert result["symbols"]["BTC/USDT"]["1000"] == 42000.0 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] == 42000.0 @pytest.mark.asyncio async def test_read_only_open_after_graceful_close(self, data_root): @@ -294,14 +298,17 @@ async def test_read_only_open_after_graceful_close(self, data_root): await database.merge_daily_closes("BTC/USDT", {"1000": 42000.0}) result = await persisted_price_cache.load("binance", "spot", False, data_root) - assert result["symbols"]["BTC/USDT"]["1000"] == 42000.0 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"]["1000"] == 42000.0 class TestReadOnlyMissingDatabase: @pytest.mark.asyncio async def test_missing_database_returns_empty_without_creating_file(self, data_root): result = await persisted_price_cache.load("binance", "spot", False, data_root) - assert result == {"symbols": {}, "sources": {}} + assert result == { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } db_path = market_data_sqlite_database_module.MarketDataSQLiteDatabase.get_db_path( "binance", "spot", False, data_root ) diff --git a/packages/trading/tests/exchange_data/prices/test_persisted_price_cache.py b/packages/trading/tests/exchange_data/prices/test_persisted_price_cache.py index 829a2ba999..366a80840d 100644 --- a/packages/trading/tests/exchange_data/prices/test_persisted_price_cache.py +++ b/packages/trading/tests/exchange_data/prices/test_persisted_price_cache.py @@ -2,10 +2,18 @@ import pytest +import octobot_trading.enums as trading_enums import octobot_trading.exchange_data.databases.market_data_sqlite_database as market_data_sqlite_database_module import octobot_trading.exchange_data.prices.persisted_price_cache as persisted_price_cache +def _empty_daily_prices(): + return { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + + @pytest.fixture def data_root(tmp_path): return str(tmp_path) @@ -30,7 +38,7 @@ class TestLoad: @pytest.mark.asyncio async def test_missing_database_returns_empty(self, exchange_name, exchange_type, sandboxed, data_root): result = await persisted_price_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result == {"symbols": {}, "sources": {}} + assert result == _empty_daily_prices() @pytest.mark.asyncio async def test_existing_data_loaded(self, exchange_name, exchange_type, sandboxed, data_root): @@ -38,7 +46,10 @@ async def test_existing_data_loaded(self, exchange_name, exchange_type, sandboxe exchange_name, exchange_type, sandboxed, "BTC/USDT", {"1704067200": 42000.0}, data_root ) result = await persisted_price_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result == {"symbols": {"BTC/USDT": {"1704067200": 42000.0}}, "sources": {}} + assert result == { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1704067200": 42000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } db_path = market_data_sqlite_database_module.MarketDataSQLiteDatabase.get_db_path( exchange_name, exchange_type, sandboxed, data_root ) @@ -53,7 +64,7 @@ async def test_merge_into_empty(self, exchange_name, exchange_type, sandboxed, d exchange_name, exchange_type, sandboxed, "BTC/USDT", {"1000": 50000.0, "2000": 51000.0}, data_root ) result = await persisted_price_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result["symbols"]["BTC/USDT"] == {"1000": 50000.0, "2000": 51000.0} + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 50000.0, "2000": 51000.0} @pytest.mark.asyncio async def test_incremental_merge(self, exchange_name, exchange_type, sandboxed, data_root): @@ -64,55 +75,137 @@ async def test_incremental_merge(self, exchange_name, exchange_type, sandboxed, exchange_name, exchange_type, sandboxed, "BTC/USDT", {"2000": 51000.0}, data_root ) result = await persisted_price_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result["symbols"]["BTC/USDT"] == {"1000": 50000.0, "2000": 51000.0} + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 50000.0, "2000": 51000.0} class TestOldestTimestamp: def test_empty_returns_none(self): - assert persisted_price_cache.oldest_timestamp({"symbols": {}}, "BTC/USDT") is None + assert persisted_price_cache.oldest_timestamp(_empty_daily_prices(), "BTC/USDT") is None def test_returns_minimum(self): - data = {"symbols": {"BTC/USDT": {"2000": 1.0, "1000": 2.0, "3000": 3.0}}} + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 1.0, "1000": 2.0, "3000": 3.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } assert persisted_price_cache.oldest_timestamp(data, "BTC/USDT") == 1000.0 class TestNewestTimestamp: def test_empty_returns_none(self): - assert persisted_price_cache.newest_timestamp({"symbols": {}}, "BTC/USDT") is None + assert persisted_price_cache.newest_timestamp(_empty_daily_prices(), "BTC/USDT") is None def test_returns_maximum(self): - data = {"symbols": {"BTC/USDT": {"2000": 1.0, "1000": 2.0, "3000": 3.0}}} + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"2000": 1.0, "1000": 2.0, "3000": 3.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } assert persisted_price_cache.newest_timestamp(data, "BTC/USDT") == 3000.0 class TestGetClose: def test_found(self): - data = {"symbols": {"BTC/USDT": {"1000": 42000.0}}, "sources": {}} + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1000": 42000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } assert persisted_price_cache.get_close(data, "BTC/USDT", "1000") == 42000.0 def test_missing_returns_default(self): - data = {"symbols": {}, "sources": {}} - assert persisted_price_cache.get_close(data, "BTC/USDT", "1000", default=-1.0) == -1.0 + assert persisted_price_cache.get_close(_empty_daily_prices(), "BTC/USDT", "1000", default=-1.0) == -1.0 def test_resolves_reference_symbol_via_source_mapping(self): data = { - "symbols": {"KNC/USD": {"1000": 1.05}}, - "sources": {"KNC": "KNC/USD"}, + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"KNC/USD": {"1000": 1.05}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USD"}, } assert persisted_price_cache.get_close(data, "KNC/USDT", "1000") == 1.05 +class TestLatestCloseOnOrBefore: + def test_returns_latest_close_on_or_before_day(self): + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: { + "BTC/USDT": { + "86400": 40000.0, + "172800": 41000.0, + } + }, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + assert persisted_price_cache.latest_close_on_or_before( + data, "BTC/USDT", 200000.0, + ) == 41000.0 + + def test_returns_none_when_no_close_on_or_before_day(self): + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"172800": 41000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + assert persisted_price_cache.latest_close_on_or_before( + data, "BTC/USDT", 86400.0, + ) is None + + +class TestMoveSymbolClosesInMemory: + def test_moves_closes_to_new_symbol(self): + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"OLD/USDT": {"1000": 1.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + persisted_price_cache.move_symbol_closes_in_memory(data, "OLD/USDT", "NEW/USDT") + assert "OLD/USDT" not in data[trading_enums.DailyPricesCacheKeys.SYMBOLS] + assert data[trading_enums.DailyPricesCacheKeys.SYMBOLS]["NEW/USDT"] == {"1000": 1.0} + + +class TestGetCloseSource: + def test_returns_none_when_missing(self): + assert persisted_price_cache.get_close_source(_empty_daily_prices(), "BTC") is None + + def test_returns_mapped_symbol(self): + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USD"}, + } + assert persisted_price_cache.get_close_source(data, "KNC") == "KNC/USD" + + +class TestSetCloseSourceInMemory: + def test_sets_source_mapping(self): + data = _empty_daily_prices() + persisted_price_cache.set_close_source_in_memory(data, "BTC", "BTC/USDT") + assert data[trading_enums.DailyPricesCacheKeys.SOURCES]["BTC"] == "BTC/USDT" + + +class TestMergeSymbolClosesInMemory: + def test_merges_closes_for_symbol(self): + data = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1000": 1.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + persisted_price_cache.merge_symbol_closes_in_memory( + data, "BTC/USDT", {"2000": 2.0}, + ) + assert data[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 1.0, "2000": 2.0} + + def test_creates_symbol_entry_when_missing(self): + data = _empty_daily_prices() + persisted_price_cache.merge_symbol_closes_in_memory( + data, "BTC/USDT", {"1000": 1.0}, + ) + assert data[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] == {"1000": 1.0} + + class TestResolveSymbolTimestamps: def test_oldest_timestamp_uses_source_mapping(self): data = { - "symbols": {"KNC/USD": {"1000": 1.0, "2000": 1.1}}, - "sources": {"KNC": "KNC/USD"}, + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"KNC/USD": {"1000": 1.0, "2000": 1.1}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USD"}, } assert persisted_price_cache.oldest_timestamp(data, "KNC/USDT") == 1000.0 def test_newest_timestamp_uses_source_mapping(self): data = { - "symbols": {"KNC/USD": {"1000": 1.0, "2000": 1.1}}, - "sources": {"KNC": "KNC/USD"}, + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"KNC/USD": {"1000": 1.0, "2000": 1.1}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {"KNC": "KNC/USD"}, } assert persisted_price_cache.newest_timestamp(data, "KNC/USDT") == 2000.0 diff --git a/packages/trading/tests/exchange_data/ticker/test_persisted_ticker_cache.py b/packages/trading/tests/exchange_data/ticker/test_persisted_ticker_cache.py index 2391eec993..b84da04956 100644 --- a/packages/trading/tests/exchange_data/ticker/test_persisted_ticker_cache.py +++ b/packages/trading/tests/exchange_data/ticker/test_persisted_ticker_cache.py @@ -2,10 +2,18 @@ import pytest +import octobot_trading.enums as trading_enums import octobot_trading.exchange_data.databases.market_data_sqlite_database as market_data_sqlite_database_module import octobot_trading.exchange_data.ticker.persisted_ticker_cache as persisted_ticker_cache +def _empty_latest_tickers(): + return { + trading_enums.LatestTickersCacheKeys.UPDATED_AT: None, + trading_enums.LatestTickersCacheKeys.CLOSES: {}, + } + + @pytest.fixture def data_root(tmp_path): return str(tmp_path) @@ -30,7 +38,7 @@ class TestLoad: @pytest.mark.asyncio async def test_missing_database_returns_empty(self, exchange_name, exchange_type, sandboxed, data_root): result = await persisted_ticker_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result == {"updated_at": None, "closes": {}} + assert result == _empty_latest_tickers() @pytest.mark.asyncio async def test_existing_data_loaded(self, exchange_name, exchange_type, sandboxed, data_root): @@ -38,8 +46,8 @@ async def test_existing_data_loaded(self, exchange_name, exchange_type, sandboxe exchange_name, exchange_type, sandboxed, {"BTC/USDT": 42000.0}, data_root ) result = await persisted_ticker_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result["closes"]["BTC/USDT"] == 42000.0 - assert result["updated_at"] is not None + assert result[trading_enums.LatestTickersCacheKeys.CLOSES]["BTC/USDT"] == 42000.0 + assert result[trading_enums.LatestTickersCacheKeys.UPDATED_AT] is not None db_path = market_data_sqlite_database_module.MarketDataSQLiteDatabase.get_db_path( exchange_name, exchange_type, sandboxed, data_root ) @@ -54,8 +62,8 @@ async def test_merge_closes_and_set_updated_at(self, exchange_name, exchange_typ exchange_name, exchange_type, sandboxed, {"BTC/USDT": 65000.0}, data_root ) result = await persisted_ticker_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result["closes"]["BTC/USDT"] == 65000.0 - assert result["updated_at"] is not None + assert result[trading_enums.LatestTickersCacheKeys.CLOSES]["BTC/USDT"] == 65000.0 + assert result[trading_enums.LatestTickersCacheKeys.UPDATED_AT] is not None @pytest.mark.asyncio async def test_incremental_update(self, exchange_name, exchange_type, sandboxed, data_root): @@ -66,15 +74,15 @@ async def test_incremental_update(self, exchange_name, exchange_type, sandboxed, exchange_name, exchange_type, sandboxed, {"ETH/USDT": 3200.0}, data_root ) result = await persisted_ticker_cache.load(exchange_name, exchange_type, sandboxed, data_root) - assert result["closes"]["BTC/USDT"] == 65000.0 - assert result["closes"]["ETH/USDT"] == 3200.0 + assert result[trading_enums.LatestTickersCacheKeys.CLOSES]["BTC/USDT"] == 65000.0 + assert result[trading_enums.LatestTickersCacheKeys.CLOSES]["ETH/USDT"] == 3200.0 class TestGetClose: def test_found(self): - data = {"closes": {"BTC/USDT": 65000.0}} + data = {trading_enums.LatestTickersCacheKeys.CLOSES: {"BTC/USDT": 65000.0}} assert persisted_ticker_cache.get_close(data, "BTC/USDT") == 65000.0 def test_missing_returns_default(self): - data = {"closes": {}} + data = {trading_enums.LatestTickersCacheKeys.CLOSES: {}} assert persisted_ticker_cache.get_close(data, "BTC/USDT", default=0.0) == 0.0 diff --git a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_adapter.py b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_adapter.py new file mode 100644 index 0000000000..cb2da38704 --- /dev/null +++ b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_adapter.py @@ -0,0 +1,112 @@ +import decimal +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_trading.enums as enums +import octobot_trading.exchanges.connectors.ccxt.ccxt_adapter as ccxt_adapter_module +import octobot_trading.personal_data.transactions.protocol as transactions_protocol +import octobot_trading.personal_data.portfolios.history.history_from_trades_and_transaction_builder as history_builder + + +DAY_SECONDS = commons_constants.DAYS_TO_SECONDS +TOTAL = commons_constants.PORTFOLIO_TOTAL + + +def _ccxt_adapter() -> ccxt_adapter_module.CCXTAdapter: + return ccxt_adapter_module.CCXTAdapter(mock.Mock()) + + +def _ccxt_transaction(**overrides) -> dict: + transaction = { + enums.ExchangeConstantsTransactionColumns.ID.value: "tx-internal-1", + enums.ExchangeConstantsTransactionColumns.TXID.value: "tx-blockchain-1", + enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1_700_002_000, + enums.ExchangeConstantsTransactionColumns.CURRENCY.value: "SOL", + enums.ExchangeConstantsTransactionColumns.AMOUNT.value: 200, + enums.ExchangeConstantsTransactionColumns.TYPE.value: "withdrawal", + enums.ExchangeConstantsTransactionColumns.STATUS.value: "ok", + } + transaction.update(overrides) + return transaction + + +class TestCcxtAdapterParseTransaction: + def test_enforces_withdrawal_type_when_provided(self): + adapter = _ccxt_adapter() + + parsed = adapter.parse_transaction( + _ccxt_transaction(), + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, + ) + + assert parsed[enums.ExchangeConstantsTransactionColumns.TYPE.value] == ( + enums.TransactionType.BLOCKCHAIN_WITHDRAWAL.value + ) + assert parsed[enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("200") + + def test_enforces_deposit_type_when_provided(self): + adapter = _ccxt_adapter() + + parsed = adapter.parse_transaction( + _ccxt_transaction(type="deposit"), + transaction_type=enums.TransactionType.BLOCKCHAIN_DEPOSIT, + ) + + assert parsed[enums.ExchangeConstantsTransactionColumns.TYPE.value] == ( + enums.TransactionType.BLOCKCHAIN_DEPOSIT.value + ) + + def test_raises_when_transaction_type_not_provided(self): + adapter = _ccxt_adapter() + + with pytest.raises(ValueError, match="transaction_type is required"): + adapter.parse_transaction(_ccxt_transaction(type="withdrawal")) + + +class TestCcxtAdapterParseTransactions: + def test_parses_each_transaction_through_parse_transaction(self): + adapter = _ccxt_adapter() + transactions = [ + _ccxt_transaction(txid="tx-1"), + _ccxt_transaction(txid="tx-2", amount=50), + ] + + parsed = adapter.parse_transactions( + transactions, + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, + ) + + assert len(parsed) == 2 + assert all( + transaction[enums.ExchangeConstantsTransactionColumns.TYPE.value] + == enums.TransactionType.BLOCKCHAIN_WITHDRAWAL.value + for transaction in parsed + ) + assert parsed[1][enums.ExchangeConstantsTransactionColumns.AMOUNT.value] == decimal.Decimal("50") + + +class TestCcxtAdapterWithdrawalHistoryReplay: + def test_enforced_withdrawal_replays_positive_holdings_before_withdraw(self): + adapter = _ccxt_adapter() + withdrawal_timestamp_ms = int((DAY_SECONDS * 4 + 200) * 1000) + parsed = adapter.parse_transaction( + _ccxt_transaction(timestamp=withdrawal_timestamp_ms), + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, + ) + protocol_transaction = transactions_protocol.to_protocol_transaction(parsed) + portfolio = { + "SOL": { + TOTAL: decimal.Decimal("0"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("0"), + } + } + + historical_holdings = history_builder.build_historical_holdings( + portfolio, + [], + [protocol_transaction], + ) + + day_before_withdraw = DAY_SECONDS * 3 + assert historical_holdings[day_before_withdraw]["SOL"][TOTAL] == decimal.Decimal("200") diff --git a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py index 7bb566599e..f753649724 100644 --- a/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py +++ b/packages/trading/tests/exchanges/connectors/ccxt/test_ccxt_connector.py @@ -24,6 +24,7 @@ from mock import patch import octobot_trading.exchanges.connectors as exchange_connectors +import octobot_trading.exchanges.connectors.ccxt.ccxt_connector as ccxt_connector_module import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants import octobot_trading.enums as enums import octobot_trading.errors @@ -626,6 +627,142 @@ async def delayed_load(*_args, **_kwargs): assert load_count == 1 +class TestEnsureTimeDifferenceSyncedOnCachedMarkets: + def _create_connector(self, exchange_manager): + ccxt_connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + ccxt_connector.client = mock.Mock() + ccxt_connector.client.__class__.__name__ = "ob_kraken" + ccxt_connector.client.urls = {"api": {"public": "https://api.kraken.com"}} + ccxt_connector.client.apiKey = "test-key" + ccxt_connector.client.options = {} + ccxt_connector.client.load_time_difference = mock.AsyncMock(return_value=0) + ccxt_connector.exchange_manager.exchange.requires_authentication_for_this_configuration_only = ( + mock.Mock(return_value=False) + ) + return ccxt_connector + + def _enable_adjust_for_time_difference(self, ccxt_connector): + original_get_option_value = ccxt_connector.exchange_manager.exchange.get_option_value + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.ADJUST_FOR_TIME_DIFFERENCE: + return True + return original_get_option_value(option_key) + + ccxt_connector.exchange_manager.exchange.get_option_value = get_option_value + + async def test_load_symbol_markets_calls_load_time_difference_on_cache_hit_when_authenticated( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + ccxt_connector.is_authenticated = True + self._enable_adjust_for_time_difference(ccxt_connector) + + with ( + mock.patch.object(ccxt_client_util, "load_markets_from_cache") as load_cache_mock, + mock.patch.object(ccxt_clients_cache, "get_exchange_time_difference", return_value=None), + ): + await ccxt_connector.load_symbol_markets(reload=False) + + load_cache_mock.assert_called_once() + ccxt_connector.client.load_time_difference.assert_called_once() + + async def test_load_symbol_markets_uses_cached_time_difference_without_api_call( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + ccxt_connector.is_authenticated = True + self._enable_adjust_for_time_difference(ccxt_connector) + cached_time_difference = 1234.0 + + with ( + mock.patch.object(ccxt_client_util, "load_markets_from_cache"), + mock.patch.object( + ccxt_clients_cache, + "get_exchange_time_difference", + return_value=cached_time_difference, + ), + ): + await ccxt_connector.load_symbol_markets(reload=False) + + ccxt_connector.client.load_time_difference.assert_not_called() + assert ccxt_connector.client.options[ccxt_constants.CCXT_TIME_DIFFERENCE] == cached_time_difference + + async def test_load_symbol_markets_does_not_crash_when_time_sync_fails( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + ccxt_connector.is_authenticated = True + self._enable_adjust_for_time_difference(ccxt_connector) + ccxt_connector.client.load_time_difference = mock.AsyncMock( + side_effect=Exception("time sync failed") + ) + + with ( + mock.patch.object(ccxt_client_util, "load_markets_from_cache"), + mock.patch.object(ccxt_clients_cache, "get_exchange_time_difference", return_value=None), + ): + await ccxt_connector.load_symbol_markets(reload=False) + + ccxt_connector.client.load_time_difference.assert_called_once() + + async def test_filtered_load_markets_does_not_call_load_time_difference( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + self._enable_adjust_for_time_difference(ccxt_connector) + ccxt_connector.client.markets = {"BTC/USDT": {"symbol": "BTC/USDT"}} + ccxt_connector.client.load_markets = mock.AsyncMock() + ccxt_connector.exchange_manager.exchange.FETCH_MIN_EXCHANGE_MARKETS = False + + await ccxt_connector._filtered_if_necessary_load_markets(ccxt_connector.client, False, None) + + ccxt_connector.client.load_time_difference.assert_not_called() + + async def test_load_symbol_markets_skips_time_sync_on_cache_hit_when_unauthenticated( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + ccxt_connector.is_authenticated = False + self._enable_adjust_for_time_difference(ccxt_connector) + + with mock.patch.object(ccxt_client_util, "load_markets_from_cache"): + await ccxt_connector.load_symbol_markets(reload=False) + + ccxt_connector.client.load_time_difference.assert_not_called() + + async def test_load_symbol_markets_syncs_time_before_markets_fetch( + self, exchange_manager + ): + ccxt_connector = self._create_connector(exchange_manager) + ccxt_connector.is_authenticated = True + ccxt_connector.exchange_manager.exchange.FETCH_MIN_EXCHANGE_MARKETS = False + ccxt_connector._persist_markets_cache = mock.Mock() + self._enable_adjust_for_time_difference(ccxt_connector) + call_order = [] + + async def load_time_difference_side_effect(*_args, **_kwargs): + call_order.append("load_time_difference") + ccxt_connector.client.options[ccxt_constants.CCXT_TIME_DIFFERENCE] = 100 + return 100 + + async def load_markets_side_effect(*_args, **_kwargs): + call_order.append("load_markets") + ccxt_connector.client.markets = {"BTC/USDT": {"symbol": "BTC/USDT"}} + + ccxt_connector.client.load_time_difference = mock.AsyncMock( + side_effect=load_time_difference_side_effect + ) + ccxt_connector.client.load_markets = mock.AsyncMock( + side_effect=load_markets_side_effect + ) + + with mock.patch.object(ccxt_clients_cache, "get_exchange_time_difference", return_value=None): + await ccxt_connector.load_symbol_markets(reload=True) + + assert call_order == ["load_time_difference", "load_markets"] + + async def test_get_deposits_raises_not_supported_on_not_implemented(ccxt_connector): ccxt_connector.client.has = {"fetchDeposits": True} ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) @@ -634,6 +771,19 @@ async def test_get_deposits_raises_not_supported_on_not_implemented(ccxt_connect await ccxt_connector.get_deposits() +async def test_get_deposits_passes_blockchain_deposit_transaction_type(ccxt_connector): + ccxt_connector.client.has = {"fetchDeposits": True} + ccxt_connector.client.fetch_deposits = mock.AsyncMock(return_value=[{"id": "d1"}]) + ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) + + await ccxt_connector.get_deposits() + + ccxt_connector.adapter.adapt_transactions.assert_called_once_with( + [{"id": "d1"}], + transaction_type=enums.TransactionType.BLOCKCHAIN_DEPOSIT, + ) + + async def test_get_withdrawals_raises_not_supported_on_not_implemented(ccxt_connector): ccxt_connector.client.has = {"fetchWithdrawals": True} ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) @@ -642,6 +792,45 @@ async def test_get_withdrawals_raises_not_supported_on_not_implemented(ccxt_conn await ccxt_connector.get_withdrawals() +async def test_get_withdrawals_passes_blockchain_withdrawal_transaction_type(ccxt_connector): + ccxt_connector.client.has = {"fetchWithdrawals": True} + ccxt_connector.client.fetch_withdrawals = mock.AsyncMock(return_value=[{"id": "w1"}]) + ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) + + await ccxt_connector.get_withdrawals() + + ccxt_connector.adapter.adapt_transactions.assert_called_once_with( + [{"id": "w1"}], + transaction_type=enums.TransactionType.BLOCKCHAIN_WITHDRAWAL, + ) + + +class TestGetDepositsCurrenciesParam: + async def test_currencies_not_passed_to_ccxt_params(self, ccxt_connector): + ccxt_connector.client.has = {"fetchDeposits": True} + ccxt_connector.client.fetch_deposits = mock.AsyncMock(return_value=[{"id": "d1"}]) + ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) + + await ccxt_connector.get_deposits(currencies=["BTC"]) + + ccxt_connector.client.fetch_deposits.assert_awaited_once_with( + code=None, since=None, limit=None, params={} + ) + + +class TestGetWithdrawalsCurrenciesParam: + async def test_currencies_not_passed_to_ccxt_params(self, ccxt_connector): + ccxt_connector.client.has = {"fetchWithdrawals": True} + ccxt_connector.client.fetch_withdrawals = mock.AsyncMock(return_value=[{"id": "w1"}]) + ccxt_connector.adapter.adapt_transactions = mock.Mock(return_value=[]) + + await ccxt_connector.get_withdrawals(currencies=["BTC"]) + + ccxt_connector.client.fetch_withdrawals.assert_awaited_once_with( + code=None, since=None, limit=None, params={} + ) + + def _get_fees(type, currency, rate, cost): return { enums.FeePropertyColumns.TYPE.value: type, @@ -650,3 +839,204 @@ def _get_fees(type, currency, rate, cost): enums.FeePropertyColumns.COST.value: decimal.Decimal(str(cost)), enums.FeePropertyColumns.IS_FROM_EXCHANGE.value: False, } + + +class TestGetMyRecentTradesOffsetPagination: + def _create_connector(self, exchange_manager, pagination_offset=None): + connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + connector.client = mock.Mock() + connector.client.has = {"fetchMyTrades": True} + connector.client.fetch_my_trades = mock.AsyncMock() + connector.adapter.adapt_trades = mock.Mock(side_effect=lambda trades: trades) + original_get_option_value = exchange_manager.exchange.get_option_value + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.MY_TRADES_FETCH_PAGINATION_OFFSET: + return pagination_offset + if option_key == enums.ExchangeClientOptions.ALLOW_TRADES_FROM_CLOSED_ORDERS: + return False + return original_get_option_value(option_key) + + exchange_manager.exchange.get_option_value = get_option_value + return connector + + async def test_paginates_with_offset_param_when_exhaust_history_true(self, exchange_manager): + connector = self._create_connector(exchange_manager, pagination_offset="ofs") + first_page = [{"id": str(trade_index)} for trade_index in range(50)] + second_page = [{"id": str(50 + trade_index)} for trade_index in range(25)] + connector.client.fetch_my_trades = mock.AsyncMock(side_effect=[first_page, second_page]) + + result = await connector.get_my_recent_trades(symbol=None, exhaust_history=True) + + assert len(result) == 75 + assert connector.client.fetch_my_trades.await_count == 2 + second_call_kwargs = connector.client.fetch_my_trades.await_args_list[1].kwargs + assert second_call_kwargs["params"]["ofs"] == 50 + + async def test_single_fetch_when_page_not_multiple_of_ten(self, exchange_manager): + connector = self._create_connector(exchange_manager, pagination_offset="ofs") + connector.client.fetch_my_trades = mock.AsyncMock( + return_value=[{"id": str(trade_index)} for trade_index in range(9)], + ) + + result = await connector.get_my_recent_trades(symbol=None, exhaust_history=True) + + assert len(result) == 9 + connector.client.fetch_my_trades.assert_awaited_once() + + async def test_single_fetch_when_offset_option_unset(self, exchange_manager): + connector = self._create_connector(exchange_manager, pagination_offset=None) + connector.client.fetch_my_trades = mock.AsyncMock(return_value=[{"id": "1"}] * 50) + + result = await connector.get_my_recent_trades(symbol=None, exhaust_history=True) + + assert len(result) == 50 + connector.client.fetch_my_trades.assert_awaited_once() + + async def test_single_fetch_when_exhaust_history_false(self, exchange_manager): + connector = self._create_connector(exchange_manager, pagination_offset="ofs") + connector.client.fetch_my_trades = mock.AsyncMock(return_value=[{"id": "1"}] * 50) + + result = await connector.get_my_recent_trades(symbol=None, exhaust_history=False) + + assert len(result) == 50 + connector.client.fetch_my_trades.assert_awaited_once() + + async def test_stops_after_max_pagination_requests(self, exchange_manager, caplog): + import logging + caplog.set_level(logging.WARNING) + connector = self._create_connector(exchange_manager, pagination_offset="ofs") + page_index = 0 + + async def infinite_full_pages(*_args, **_kwargs): + nonlocal page_index + trade_page = [ + {"id": str(page_index * 50 + trade_index)} + for trade_index in range(50) + ] + page_index += 1 + return trade_page + + connector.client.fetch_my_trades = mock.AsyncMock(side_effect=infinite_full_pages) + max_requests = ccxt_connector_module.MAX_MY_TRADES_OFFSET_PAGINATION_REQUESTS + + result = await connector.get_my_recent_trades(symbol=None, exhaust_history=True) + + assert connector.client.fetch_my_trades.await_count == max_requests + assert len(result) == 50 * max_requests + assert "Stopped" in caplog.text + assert "my-trades offset pagination" in caplog.text + + +class TestGetMyRecentTradesCcxtPaginate: + def _create_connector(self, exchange_manager, use_ccxt_paginate=False): + connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + connector.client = mock.Mock() + connector.client.has = {"fetchMyTrades": True} + connector.client.fetch_my_trades = mock.AsyncMock() + connector.adapter.adapt_trades = mock.Mock(side_effect=lambda trades: trades) + original_get_option_value = exchange_manager.exchange.get_option_value + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE: + return use_ccxt_paginate + if option_key == enums.ExchangeClientOptions.ALLOW_TRADES_FROM_CLOSED_ORDERS: + return False + return original_get_option_value(option_key) + + exchange_manager.exchange.get_option_value = get_option_value + return connector + + async def test_paginate_true_when_exhaust_history_and_option_enabled(self, exchange_manager): + connector = self._create_connector(exchange_manager, use_ccxt_paginate=True) + connector.client.fetch_my_trades = mock.AsyncMock(return_value=[{"id": "1"}]) + + await connector.get_my_recent_trades(symbol="BTC/USDT", exhaust_history=True) + + call_kwargs = connector.client.fetch_my_trades.await_args.kwargs + assert call_kwargs["params"]["paginate"] is True + + async def test_no_paginate_when_exhaust_history_false(self, exchange_manager): + connector = self._create_connector(exchange_manager, use_ccxt_paginate=True) + connector.client.fetch_my_trades = mock.AsyncMock(return_value=[{"id": "1"}]) + + await connector.get_my_recent_trades(symbol="BTC/USDT", exhaust_history=False) + + call_kwargs = connector.client.fetch_my_trades.await_args.kwargs + assert "paginate" not in call_kwargs["params"] + + async def test_single_fetch_when_option_disabled(self, exchange_manager): + connector = self._create_connector(exchange_manager, use_ccxt_paginate=False) + connector.client.fetch_my_trades = mock.AsyncMock(return_value=[{"id": "1"}] * 50) + + result = await connector.get_my_recent_trades(symbol="BTC/USDT", exhaust_history=True) + + assert len(result) == 50 + call_kwargs = connector.client.fetch_my_trades.await_args.kwargs + assert "paginate" not in call_kwargs["params"] + + +class TestGetMyRecentTradesExhaustPrecedence: + async def test_offset_pagination_wins_over_ccxt_paginate(self, exchange_manager): + connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + connector.client = mock.Mock() + connector.client.has = {"fetchMyTrades": True} + connector.client.fetch_my_trades = mock.AsyncMock( + side_effect=[ + [{"id": str(trade_index)} for trade_index in range(50)], + [{"id": str(50 + trade_index)} for trade_index in range(9)], + ], + ) + connector.adapter.adapt_trades = mock.Mock(side_effect=lambda trades: trades) + original_get_option_value = exchange_manager.exchange.get_option_value + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.MY_TRADES_FETCH_PAGINATION_OFFSET: + return "ofs" + if option_key == enums.ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE: + return True + if option_key == enums.ExchangeClientOptions.ALLOW_TRADES_FROM_CLOSED_ORDERS: + return False + return original_get_option_value(option_key) + + exchange_manager.exchange.get_option_value = get_option_value + + await connector.get_my_recent_trades(symbol=None, exhaust_history=True) + + assert connector.client.fetch_my_trades.await_count == 2 + first_call_kwargs = connector.client.fetch_my_trades.await_args_list[0].kwargs + assert "paginate" not in first_call_kwargs["params"] + assert first_call_kwargs["params"]["ofs"] == 0 + + +class TestGetClosedOrdersCcxtPaginate: + def _create_connector(self, exchange_manager, use_ccxt_paginate=False): + connector = exchange_connectors.CCXTConnector(exchange_manager.config, exchange_manager) + connector.client = mock.Mock() + connector.client.fetch_closed_orders = mock.AsyncMock(return_value=[]) + connector.adapter.adapt_orders = mock.Mock(side_effect=lambda orders, **kwargs: orders) + original_get_option_value = exchange_manager.exchange.get_option_value + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.CLOSED_ORDERS_FETCH_USE_CCXT_PAGINATE: + return use_ccxt_paginate + return original_get_option_value(option_key) + + exchange_manager.exchange.get_option_value = get_option_value + return connector + + async def test_paginate_true_when_exhaust_history_and_option_enabled(self, exchange_manager): + connector = self._create_connector(exchange_manager, use_ccxt_paginate=True) + + await connector.get_closed_orders(symbol="BTC/USDT", exhaust_history=True) + + call_kwargs = connector.client.fetch_closed_orders.await_args.kwargs + assert call_kwargs["params"]["paginate"] is True + + async def test_no_paginate_when_exhaust_history_false(self, exchange_manager): + connector = self._create_connector(exchange_manager, use_ccxt_paginate=True) + + await connector.get_closed_orders(symbol="BTC/USDT", exhaust_history=False) + + call_kwargs = connector.client.fetch_closed_orders.await_args.kwargs + assert "paginate" not in call_kwargs["params"] diff --git a/packages/trading/tests/exchanges/types/test_rest_exchange.py b/packages/trading/tests/exchanges/types/test_rest_exchange.py new file mode 100644 index 0000000000..2faac326a3 --- /dev/null +++ b/packages/trading/tests/exchanges/types/test_rest_exchange.py @@ -0,0 +1,36 @@ +import mock +import pytest + +import octobot_trading.enums as enums +import octobot_trading.exchanges.types.rest_exchange as rest_exchange_module + + +class TestRestExchangeGetMyRecentTradesExhaustHistory: + @pytest.mark.asyncio + async def test_routes_closed_orders_when_require_recent_trades_from_closed_orders(self): + exchange = mock.MagicMock() + exchange.get_closed_orders = mock.AsyncMock(return_value=[{"id": "order-1"}]) + exchange.connector = mock.MagicMock() + exchange.connector.get_my_recent_trades = mock.AsyncMock() + + def get_option_value(option_key): + if option_key == enums.ExchangeClientOptions.REQUIRE_RECENT_TRADES_FROM_CLOSED_ORDERS: + return True + return enums.DEFAULT_EXCHANGE_OPTION_VALUES.get(option_key) + + exchange.get_option_value = get_option_value + + result = await rest_exchange_module.RestExchange.get_my_recent_trades( + exchange, + symbol="SOL/USDT", + exhaust_history=True, + ) + + exchange.get_closed_orders.assert_awaited_once_with( + symbol="SOL/USDT", + since=None, + limit=None, + exhaust_history=True, + ) + exchange.connector.get_my_recent_trades.assert_not_awaited() + assert result == [{"id": "order-1"}] diff --git a/packages/trading/tests/personal_data/trades/test_trades_updater.py b/packages/trading/tests/personal_data/trades/test_trades_updater.py index 36a32125be..c7227cc462 100644 --- a/packages/trading/tests/personal_data/trades/test_trades_updater.py +++ b/packages/trading/tests/personal_data/trades/test_trades_updater.py @@ -39,6 +39,57 @@ async def test_fetch_trades_empty_symbols(self): assert result == [] +class TestTradesUpdaterFetchTradesExhaustHistory: + @pytest.mark.asyncio + async def test_fetch_trades_exhaust_history_single_symbol(self): + updater = mock.MagicMock() + updater.channel.exchange_manager.exchange.get_my_recent_trades = mock.AsyncMock( + return_value=[{"id": "trade-1"}] + ) + result = await trades_updater_module.TradesUpdater.fetch_trades( + updater, ["BTC/USDT"], exhaust_history=True, + ) + assert result == [{"id": "trade-1"}] + updater.channel.exchange_manager.exchange.get_my_recent_trades.assert_awaited_once_with( + symbol="BTC/USDT", exhaust_history=True, + ) + + @pytest.mark.asyncio + async def test_fetch_trades_exhaust_history_multiple_symbols(self): + updater = mock.MagicMock() + + async def get_my_recent_trades(symbol, exhaust_history=False, limit=None): + return [{"id": symbol}] + + updater.channel.exchange_manager.exchange.get_my_recent_trades = get_my_recent_trades + result = await trades_updater_module.TradesUpdater.fetch_trades( + updater, ["BTC/USDT", "ETH/USDT"], exhaust_history=True, + ) + assert result == [{"id": "BTC/USDT"}, {"id": "ETH/USDT"}] + + @pytest.mark.asyncio + async def test_fetch_trades_exhaust_history_account_wide(self): + updater = mock.MagicMock() + updater.channel.exchange_manager.exchange.get_my_recent_trades = mock.AsyncMock( + return_value=[{"id": "trade-1"}] + ) + result = await trades_updater_module.TradesUpdater.fetch_trades( + updater, [], exhaust_history=True, + ) + assert result == [{"id": "trade-1"}] + updater.channel.exchange_manager.exchange.get_my_recent_trades.assert_awaited_once_with( + symbol=None, exhaust_history=True, + ) + + @pytest.mark.asyncio + async def test_fetch_trades_empty_symbols_without_exhaust_returns_empty(self): + updater = mock.MagicMock() + updater.channel.exchange_manager.exchange.get_my_recent_trades = mock.AsyncMock() + result = await trades_updater_module.TradesUpdater.fetch_trades(updater, []) + assert result == [] + updater.channel.exchange_manager.exchange.get_my_recent_trades.assert_not_called() + + class TestTradesUpdaterFetchAndPush: @pytest.mark.asyncio async def test_fetch_and_push_fetches_and_pushes_per_symbol(self): From 57a00852ca2dc40745f8aebe1bcd3e9306449673 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Fri, 28 Aug 2026 16:23:55 +0200 Subject: [PATCH 07/10] CI fixees --- .../octobot_flow/jobs/exchange_account_job.py | 3 +- .../jobs/portfolio_history_job.py | 6 +- .../configuration/profile_data_factory.py | 8 +- .../parsers/actions_dag_parser.py | 92 +++++++++++++------ .../parsers/automation_state_reader.py | 4 +- .../tests/jobs/test_portfolio_history_job.py | 17 ++-- .../test_profile_data_factory.py | 13 +++ .../logic/global_view/portfolio_test_util.py | 2 + .../test_exchange_account_refresh.py | 8 +- .../scheduler/automations/__init__.py | 4 + .../automations/automation_states_loader.py | 29 +++++- .../portfolio_history_executor.py | 4 +- .../workflows/portfolio_history_workflow.py | 5 + .../test_automation_states_loader.py | 28 +++++- .../test_portfolio_history_executor.py | 42 +++++++++ .../test_portfolio_history_workflow.py | 9 ++ .../configuration/exchanges_configuration.py | 14 --- .../configuration/indexes_configuration.py | 4 +- .../trading_view_signals_trading.py | 2 +- .../trading/octobot_trading/api/exchange.py | 8 ++ .../trading/octobot_trading/api/portfolio.py | 10 +- .../exchanges/util/exchange_util.py | 20 ++++ packages/trading/tests/api/test_portfolio.py | 21 +++-- .../exchanges/util/test_exchange_util.py | 40 +++++++- .../community/test_authentication.py | 7 ++ 25 files changed, 310 insertions(+), 90 deletions(-) diff --git a/packages/flow/octobot_flow/jobs/exchange_account_job.py b/packages/flow/octobot_flow/jobs/exchange_account_job.py index 3a026a3f7b..d8071a3f1a 100644 --- a/packages/flow/octobot_flow/jobs/exchange_account_job.py +++ b/packages/flow/octobot_flow/jobs/exchange_account_job.py @@ -11,6 +11,7 @@ import octobot_trading.constants as trading_constants import octobot_trading.enums import octobot_trading.errors +import octobot_trading.api.exchange as exchange_api import octobot_trading.personal_data as personal_data import octobot_trading.personal_data.orders.order_util as order_util import octobot_trading.exchanges @@ -229,7 +230,7 @@ async def _fetch_portfolio(self, fetched_authenticated_data: octobot_flow.entiti self._update_exchange_account_portfolio(fetched_authenticated_data.portfolio) def _update_exchange_account_portfolio(self, portfolio: exchange_data_import.PortfolioDetails): - unit = scripting_library.get_default_exchange_reference_market(self._exchange_manager.exchange_name) + unit = exchange_api.get_default_exchange_reference_market(self._exchange_manager.exchange_name) self.automation_state.exchange_account_details.portfolio.content = [ octobot_flow.entities.PortfolioAssetHolding( asset, diff --git a/packages/flow/octobot_flow/jobs/portfolio_history_job.py b/packages/flow/octobot_flow/jobs/portfolio_history_job.py index 5803d0132b..c40f5e56f2 100644 --- a/packages/flow/octobot_flow/jobs/portfolio_history_job.py +++ b/packages/flow/octobot_flow/jobs/portfolio_history_job.py @@ -6,6 +6,7 @@ import octobot_commons.symbols.symbol_util as symbol_util import octobot_protocol.models as protocol_models import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.api.exchange as exchange_api import octobot_trading.enums as trading_enums import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_module @@ -23,9 +24,6 @@ import octobot_flow.repositories.exchange.trades_repository as trades_repository_module import octobot_flow.repositories.exchange.transactions_repository as transactions_repository_module -import tentacles.Meta.Keywords.scripting_library as scripting_library - - logger = commons_logging.get_logger("PortfolioHistoryJob") @@ -304,7 +302,7 @@ def _reference_market_from_account_assets( ) if usd_like_holdings: return max(usd_like_holdings, key=usd_like_holdings.get) - return scripting_library.get_default_exchange_reference_market(exchange_name) + return exchange_api.get_default_exchange_reference_market(exchange_name) def _filter_trades_on_live_markets( diff --git a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py index f66e174f9d..5eb320266c 100644 --- a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py +++ b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py @@ -4,12 +4,11 @@ import octobot_commons.constants import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums +import octobot_trading.api.exchange as exchange_api import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_flow.entities -import tentacles.Meta.Keywords.scripting_library as scripting_library - def profile_data_for_account( account: protocol_models.Account, @@ -90,7 +89,10 @@ def infer_reference_market( if exchange_account_details.portfolio.unit: # portfolio unit can be used to define the reference market return exchange_account_details.portfolio.unit - return scripting_library.get_default_exchange_reference_market(exchange_account_details.exchange_details.internal_name) + if exchange_account_details.exchange_details.internal_name: + return exchange_api.get_default_exchange_reference_market( + exchange_account_details.exchange_details.internal_name + ) return octobot_commons.constants.DEFAULT_REFERENCE_MARKET def _get_crypto_currencies(symbols: set[str]) -> list[profile_data_import.CryptoCurrencyData]: diff --git a/packages/flow/octobot_flow/parsers/actions_dag_parser.py b/packages/flow/octobot_flow/parsers/actions_dag_parser.py index 823bd785ad..0ddf9206b2 100644 --- a/packages/flow/octobot_flow/parsers/actions_dag_parser.py +++ b/packages/flow/octobot_flow/parsers/actions_dag_parser.py @@ -17,11 +17,30 @@ import octobot_flow.entities import octobot_flow.enums -import tentacles.Trading.Mode.trading_view_signals_trading_mode.actions_params as actions_params -import tentacles.Trading.Mode.trading_view_signals_trading_mode.trading_view_signals_trading as trading_view_signals_trading -import tentacles.Trading.Mode.trading_view_signals_trading_mode.tradingview_signal_to_dsl_translator as tradingview_signal_to_dsl_translator + +def _actions_params(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.actions_params as actions_params + + return actions_params + + +def _trading_view_signals_trading(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.trading_view_signals_trading as trading_view_signals_trading + + return trading_view_signals_trading + + +def _tradingview_signal_to_dsl_translator(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.tradingview_signal_to_dsl_translator as tradingview_signal_to_dsl_translator + + return tradingview_signal_to_dsl_translator + def key_val_to_dict(key_val: str) -> dict: + trading_view_signals_trading = _trading_view_signals_trading() return trading_view_signals_trading.TradingViewSignalsTradingMode.parse_signal_data(key_val, None, None, None, []) @@ -429,6 +448,8 @@ def _create_generic_action( ) def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + trading_view_signals_trading = _trading_view_signals_trading() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode self._ensure_params( ["ORDER_SYMBOL", "ORDER_AMOUNT", "ORDER_TYPE"], "trade", @@ -438,57 +459,60 @@ def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActi signal = self.params.ORDER_SIDE.lower() elif parsed_symbol.base == self.params.BLOCKCHAIN_FROM_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_TO_ASSET: # type: ignore # sell the first blockchain asset to get the second one - signal = trading_view_signals_trading.TradingViewSignalsTradingMode.SELL_SIGNAL + signal = tv_trading_mode.SELL_SIGNAL elif parsed_symbol.base == self.params.BLOCKCHAIN_TO_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_FROM_ASSET: # type: ignore # buy the second blockchain asset to get the first one - signal = trading_view_signals_trading.TradingViewSignalsTradingMode.BUY_SIGNAL + signal = tv_trading_mode.BUY_SIGNAL else: raise octobot_flow.errors.InvalidAutomationActionError( f"Invalid order symbol: {self.params.ORDER_SYMBOL}: symbol must contain the 2 " f"blockchain assets to determine the side of the order" ) order_details = { - trading_view_signals_trading.TradingViewSignalsTradingMode.EXCHANGE_KEY: self.params.get_exchange_internal_name(), - trading_view_signals_trading.TradingViewSignalsTradingMode.SYMBOL_KEY: self.params.ORDER_SYMBOL, - trading_view_signals_trading.TradingViewSignalsTradingMode.VOLUME_KEY: self.params.ORDER_AMOUNT, - trading_view_signals_trading.TradingViewSignalsTradingMode.ORDER_TYPE_SIGNAL: self.params.ORDER_TYPE, + tv_trading_mode.EXCHANGE_KEY: self.params.get_exchange_internal_name(), + tv_trading_mode.SYMBOL_KEY: self.params.ORDER_SYMBOL, + tv_trading_mode.VOLUME_KEY: self.params.ORDER_AMOUNT, + tv_trading_mode.ORDER_TYPE_SIGNAL: self.params.ORDER_TYPE, } if self.params.ORDER_PRICE: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.PRICE_KEY] = self.params.ORDER_PRICE + order_details[tv_trading_mode.PRICE_KEY] = self.params.ORDER_PRICE if self.params.ORDER_STOP_PRICE: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.STOP_PRICE_KEY] = self.params.ORDER_STOP_PRICE + order_details[tv_trading_mode.STOP_PRICE_KEY] = self.params.ORDER_STOP_PRICE if self.params.ORDER_TAG: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.TAG_KEY] = self.params.ORDER_TAG + order_details[tv_trading_mode.TAG_KEY] = self.params.ORDER_TAG if self.params.ORDER_REDUCE_ONLY: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.REDUCE_ONLY_KEY] = self.params.ORDER_REDUCE_ONLY + order_details[tv_trading_mode.REDUCE_ONLY_KEY] = self.params.ORDER_REDUCE_ONLY if self.params.ORDER_EXTRA_PARAMS: for extra_param, value in self.params.ORDER_EXTRA_PARAMS.items(): - order_details[f"{trading_view_signals_trading.TradingViewSignalsTradingMode.PARAM_PREFIX_KEY}{extra_param}"] = value + order_details[f"{tv_trading_mode.PARAM_PREFIX_KEY}{extra_param}"] = value return self.create_dsl_script_from_tv_format_action_details( f"action_trade_{index}", signal, order_details, ) def _create_cancel_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["ORDER_SYMBOL"], "cancel", ) cancel_details = { - trading_view_signals_trading.TradingViewSignalsTradingMode.SYMBOL_KEY: self.params.ORDER_SYMBOL, + tv_trading_mode.SYMBOL_KEY: self.params.ORDER_SYMBOL, } if self.params.ORDER_SIDE: - cancel_details[trading_view_signals_trading.TradingViewSignalsTradingMode.SIDE_PARAM_KEY] = self.params.ORDER_SIDE.lower() + cancel_details[tv_trading_mode.SIDE_PARAM_KEY] = self.params.ORDER_SIDE.lower() if self.params.ORDER_TAG: - cancel_details[trading_view_signals_trading.TradingViewSignalsTradingMode.TAG_KEY] = self.params.ORDER_TAG + cancel_details[tv_trading_mode.TAG_KEY] = self.params.ORDER_TAG return self.create_dsl_script_from_tv_format_action_details( f"action_cancel_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.CANCEL_SIGNAL, + tv_trading_mode.CANCEL_SIGNAL, cancel_details, ) def _create_withdraw_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_TO_ASSET", "BLOCKCHAIN_TO", "BLOCKCHAIN_TO_ADDRESS"], "withdraw", @@ -502,13 +526,15 @@ def _create_withdraw_action( withdraw_details.amount = self.params.BLOCKCHAIN_TO_AMOUNT return self.create_dsl_script_from_tv_format_action_details( f"action_withdraw_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.WITHDRAW_FUNDS_SIGNAL, + tv_trading_mode.WITHDRAW_FUNDS_SIGNAL, dataclasses.asdict(withdraw_details), ) def _create_deposit_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM_AMOUNT", "BLOCKCHAIN_FROM", "EXCHANGE_TO"], "deposit", @@ -522,7 +548,7 @@ def _create_deposit_action( ) return self.create_dsl_script_from_tv_format_action_details( f"action_deposit_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.TRANSFER_FUNDS_SIGNAL, + tv_trading_mode.TRANSFER_FUNDS_SIGNAL, dataclasses.asdict(deposit_details), ) @@ -531,6 +557,7 @@ def _wallet_init_details_for_translate( *, close_wallet_override: typing.Optional[bool] = None, ) -> dict: + actions_params = _actions_params() self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM"], "blockchain_wallet_init", @@ -555,9 +582,12 @@ def _wallet_init_details_for_translate( ) def _translate_blockchain_wallet_init_signal(self, details: dict) -> str: + trading_view_signals_trading = _trading_view_signals_trading() + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode parsed_signal = { - trading_view_signals_trading.TradingViewSignalsTradingMode.SIGNAL_KEY: - trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL, + tv_trading_mode.SIGNAL_KEY: + tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL, **details, } dsl_script = tradingview_signal_to_dsl_translator.TradingViewSignalToDSLTranslator.translate_signal( @@ -565,7 +595,7 @@ def _translate_blockchain_wallet_init_signal(self, details: dict) -> str: ) if dsl_script == tradingview_signal_to_dsl_translator.UNKNOWN_SIGNAL_RESULT: raise octobot_flow.errors.InvalidAutomationActionError( - f"Invalid signal: {trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL}" + f"Invalid signal: {tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL}" f"({details})" ) return dsl_script @@ -593,16 +623,19 @@ def _wrap_dsl_script_with_wallet_cleanup_if_error( ) def _create_blockchain_wallet_init_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode blockchain_wallet_init_details = self._wallet_init_details_for_translate() return self.create_dsl_script_from_tv_format_action_details( f"action_blockchain_wallet_init_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL, + tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL, blockchain_wallet_init_details, ) def _create_transfer_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM_AMOUNT", "BLOCKCHAIN_FROM", "BLOCKCHAIN_TO_ADDRESS"], "transfer", @@ -615,7 +648,7 @@ def _create_transfer_action( ) return self.create_dsl_script_from_tv_format_action_details( f"action_transfer_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.TRANSFER_FUNDS_SIGNAL, + tv_trading_mode.TRANSFER_FUNDS_SIGNAL, dataclasses.asdict(transfer_details), ) @@ -656,6 +689,7 @@ def _create_loop_until_order_closed_action(self, index: int) -> octobot_flow.ent return self._create_dsl_action_with_dependencies_if_any(action_id, dsl_script, params) def _create_loop_until_blockchain_balance_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() loop_interval, loop_timeout, loop_max_attempts = self._get_loop_params() amount, asset = self.params.BLOCKCHAIN_BALANCE_AMOUNT, self.params.BLOCKCHAIN_BALANCE_ASSET if not amount or not asset: @@ -758,6 +792,7 @@ def _collect_dependency_refs_from_details( Find dependency::... references in string param values. Returns (dsl_parameter_name, dependency_action_id, result_path_keys, source_literal). """ + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode refs: list[tuple[str, str, tuple[str, ...], str]] = [] for key, value in details.items(): if isinstance(value, dict): @@ -769,7 +804,7 @@ def _collect_dependency_refs_from_details( continue dep_action_id, result_path = parsed dsl_key = ( - trading_view_signals_trading.TradingViewSignalsTradingMode.TRADINGVIEW_TO_DSL_PARAM.get( + tv_trading_mode.TRADINGVIEW_TO_DSL_PARAM.get( key, key.lower() if isinstance(key, str) else str(key).lower() ) ) @@ -807,8 +842,11 @@ def _inject_dependency_placeholders_in_dsl_script( def create_dsl_script_from_tv_format_action_details( self, action_id: str, signal: str, details: dict ) -> octobot_flow.entities.DSLScriptActionDetails: + trading_view_signals_trading = _trading_view_signals_trading() + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode dsl_script = tradingview_signal_to_dsl_translator.TradingViewSignalToDSLTranslator.translate_signal( - {**{trading_view_signals_trading.TradingViewSignalsTradingMode.SIGNAL_KEY: signal}, **details} + {**{tv_trading_mode.SIGNAL_KEY: signal}, **details} ) if dsl_script == tradingview_signal_to_dsl_translator.UNKNOWN_SIGNAL_RESULT: raise octobot_flow.errors.InvalidAutomationActionError( diff --git a/packages/flow/octobot_flow/parsers/automation_state_reader.py b/packages/flow/octobot_flow/parsers/automation_state_reader.py index b0760f7abb..540ed518f4 100644 --- a/packages/flow/octobot_flow/parsers/automation_state_reader.py +++ b/packages/flow/octobot_flow/parsers/automation_state_reader.py @@ -1,6 +1,5 @@ import octobot_flow.entities import octobot_flow.logic.configuration -import octobot_flow.logic.dsl class AutomationStateReader: @@ -8,6 +7,9 @@ def __init__(self, state: octobot_flow.entities.AutomationState): self.state: octobot_flow.entities.AutomationState = state def get_automation_copied_strategy_ids(self) -> list[str]: + # avoid hard tentacles dependencies + import octobot_flow.logic.dsl + to_execute_actions = self.state.automation.actions_dag.get_executable_actions() self._resolve_dsl_scripts_for_actions(to_execute_actions) minimal_profile_data = octobot_flow.logic.configuration.create_profile_data( diff --git a/packages/flow/tests/jobs/test_portfolio_history_job.py b/packages/flow/tests/jobs/test_portfolio_history_job.py index b426da9995..6c2d1a8b5b 100644 --- a/packages/flow/tests/jobs/test_portfolio_history_job.py +++ b/packages/flow/tests/jobs/test_portfolio_history_job.py @@ -370,9 +370,8 @@ def test_uses_dominant_usd_like_holding(self): def test_falls_back_to_exchange_default_when_no_usd_like_holdings(self): account = mock.MagicMock() account.assets = [] - with mock.patch.object( - portfolio_history_job_module.scripting_library, - "get_default_exchange_reference_market", + with mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) as default_reference_market_mock: reference_market = portfolio_history_job_module._reference_market_from_account_assets( @@ -420,7 +419,7 @@ def test_merges_totals_across_trading_types(self): class TestDerivePriceSymbols: @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_skips_reference_market_currency_from_transactions(self, _mock_reference_market): @@ -432,7 +431,7 @@ def test_skips_reference_market_currency_from_transactions(self, _mock_reference assert symbols == [] @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_adds_transaction_currency_against_reference_market(self, _mock_reference_market): @@ -444,7 +443,7 @@ def test_adds_transaction_currency_against_reference_market(self, _mock_referenc assert symbols == ["BTC/USDT"] @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_filters_invalid_same_base_quote_symbols(self, _mock_reference_market): @@ -456,7 +455,7 @@ def test_filters_invalid_same_base_quote_symbols(self, _mock_reference_market): assert symbols == ["BTC/USDT", "ETH/USDT"] @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_maps_trade_symbols_to_one_reference_pair_per_base(self, _mock_reference_market): @@ -468,7 +467,7 @@ def test_maps_trade_symbols_to_one_reference_pair_per_base(self, _mock_reference assert symbols == ["DOT/USDC", "ETH/USDC"] @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_maps_transaction_currency_to_reference_pair(self, _mock_reference_market): @@ -480,7 +479,7 @@ def test_maps_transaction_currency_to_reference_pair(self, _mock_reference_marke assert symbols == ["EUR/USDC"] @mock.patch( - "octobot_flow.jobs.portfolio_history_job.scripting_library.get_default_exchange_reference_market", + "octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDT", ) def test_skips_usd_like_stablecoin_from_transactions(self, _mock_reference_market): diff --git a/packages/flow/tests/logic/configuration/test_profile_data_factory.py b/packages/flow/tests/logic/configuration/test_profile_data_factory.py index f1e3706532..0231d86f68 100644 --- a/packages/flow/tests/logic/configuration/test_profile_data_factory.py +++ b/packages/flow/tests/logic/configuration/test_profile_data_factory.py @@ -3,6 +3,7 @@ import datetime +import octobot_commons.constants as commons_constants import octobot_commons.profiles.profile_data as profile_data_module import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums @@ -110,3 +111,15 @@ def test_enables_simulator_for_simulated_account(self): ) assert profile_data.trader.enabled is False assert profile_data.trader_simulator.enabled is True + + +class TestInferReferenceMarket: + def test_returns_default_reference_market_when_internal_name_missing(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, []) + == commons_constants.DEFAULT_REFERENCE_MARKET + ) diff --git a/packages/flow/tests/logic/global_view/portfolio_test_util.py b/packages/flow/tests/logic/global_view/portfolio_test_util.py index 6b92440c4a..303ddbc332 100644 --- a/packages/flow/tests/logic/global_view/portfolio_test_util.py +++ b/packages/flow/tests/logic/global_view/portfolio_test_util.py @@ -13,6 +13,7 @@ def wire_portfolio_pipeline( portfolio_content: dict, *, portfolio_total: float = 1500.0, + exchange_name: str = "binanceus", ) -> None: stored_portfolio = { currency: dict(balances) @@ -67,3 +68,4 @@ async def handle_portfolio_update(balance, should_notify=False, is_diff_update=F exchange_manager.get_symbol_data = mock.Mock(return_value=mock.Mock()) exchange_manager.client_symbols = ["BTC/USDC", "USDC/BTC", "BTC/USDT", "ETH/USDT"] exchange_manager.symbol_exists = mock.Mock(return_value=True) + exchange_manager.exchange_name = exchange_name diff --git a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py index 67bde1e0fc..936c4091d8 100644 --- a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py +++ b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py @@ -436,11 +436,12 @@ class TestRefreshExchangeAccountLogging: async def test_logs_fetched_portfolio_once_at_info(self): balance_content = _portfolio_content() exchange_manager = mock.Mock() - exchange_manager.exchange_name = "binance" exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") exchange_manager.exchange_personal_data = mock.Mock() - _wire_portfolio_pipeline(exchange_manager, balance_content, portfolio_total=6000.0) + _wire_portfolio_pipeline( + exchange_manager, balance_content, portfolio_total=6000.0, exchange_name="binance", + ) logger_mock = mock.Mock() with ( mock.patch.object( @@ -485,10 +486,9 @@ class TestSimulatedTickerFetchMerge: @pytest.mark.asyncio async def test_fetches_order_and_valuation_symbols_once(self): exchange_manager = mock.Mock() - exchange_manager.exchange_name = "bitmart" exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDT") portfolio_content = _portfolio_content() - _wire_portfolio_pipeline(exchange_manager, portfolio_content) + _wire_portfolio_pipeline(exchange_manager, portfolio_content, exchange_name="bitmart") open_order = _open_order_dict("order-1", "ETH/USDT") fetch_tickers_mock = mock.AsyncMock(return_value={ "ETH/USDT": {trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 3000}, diff --git a/packages/node/octobot_node/scheduler/automations/__init__.py b/packages/node/octobot_node/scheduler/automations/__init__.py index 30c6641ca5..c5aca2ae6c 100644 --- a/packages/node/octobot_node/scheduler/automations/__init__.py +++ b/packages/node/octobot_node/scheduler/automations/__init__.py @@ -14,6 +14,9 @@ load_flow_automation_states_by_id = automations_automation_states_loader.load_flow_automation_states_by_id load_protocol_automation_states = automations_automation_states_loader.load_protocol_automation_states load_wallet_automation_states = automations_automation_states_loader.load_wallet_automation_states +load_wallet_automation_states_for_trade_symbols = ( + automations_automation_states_loader.load_wallet_automation_states_for_trade_symbols +) parse_flow_automation_state = automations_automation_states_loader.parse_flow_automation_state patch_task_content_degraded_state = automations_automation_states_loader.patch_task_content_degraded_state resolve_trade_symbols = automations_trade_symbols_resolver.resolve_trade_symbols @@ -36,6 +39,7 @@ "load_flow_automation_states_by_id", "load_protocol_automation_states", "load_wallet_automation_states", + "load_wallet_automation_states_for_trade_symbols", "parse_flow_automation_state", "patch_task_content_degraded_state", "resolve_trade_symbols", diff --git a/packages/node/octobot_node/scheduler/automations/automation_states_loader.py b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py index 2019ea3315..c0f0327f1c 100644 --- a/packages/node/octobot_node/scheduler/automations/automation_states_loader.py +++ b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py @@ -10,7 +10,7 @@ import octobot_protocol.models as protocol_models import octobot_flow.entities as flow_entities -import octobot_flow.parsers as flow_parsers +import octobot_flow.parsers.automation_state_reader as automation_state_reader_module import octobot_node.models as node_models import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client @@ -50,10 +50,10 @@ def get_automation_state_dict(workflow_status: dbos_lib.WorkflowStatus) -> typin def get_automation_state_reader( workflow_status: dbos_lib.WorkflowStatus, -) -> typing.Optional[flow_parsers.AutomationStateReader]: +) -> typing.Optional[automation_state_reader_module.AutomationStateReader]: """Get the resolved automation state for a workflow row (input or terminal output).""" if state_dict := get_automation_state_dict(workflow_status): - return flow_parsers.AutomationStateReader( + return automation_state_reader_module.AutomationStateReader( flow_entities.AutomationState.from_dict(state_dict) ) return None @@ -132,9 +132,17 @@ def _protocol_states_from_sources( return automations_protocol.to_protocol_automations_state(sources) +TRADE_SYMBOL_AUTOMATION_STATUSES: tuple[dbos_lib.WorkflowStatusString, ...] = ( + dbos_lib.WorkflowStatusString.ENQUEUED, + dbos_lib.WorkflowStatusString.PENDING, +) + + async def load_automation_state_sources( wallet_id: str, statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, + *, + load_output: bool = True, ) -> list["automations_protocol.AutomationStateSource"]: import octobot_node.protocol.automations as automations_protocol import octobot_node.scheduler as scheduler_module @@ -143,7 +151,7 @@ async def load_automation_state_sources( workflows = await scheduler._get_latest_workflow_for_each_automation( wallet_id, statuses, - load_output=True, + load_output=load_output, ) sources: list[automations_protocol.AutomationStateSource] = [] for workflow in workflows: @@ -186,8 +194,10 @@ async def load_flow_automation_states_by_id( async def load_wallet_automation_states( wallet_id: str, statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, + *, + load_output: bool = True, ) -> WalletAutomationStates: - sources = await load_automation_state_sources(wallet_id, statuses) + sources = await load_automation_state_sources(wallet_id, statuses, load_output=load_output) with contextlib.ExitStack() as exit_stack: for source in sources: exit_stack.enter_context(task_context_module.encrypted_task(source.task)) @@ -197,3 +207,12 @@ async def load_wallet_automation_states( protocol_states=protocol_states, flow_states_by_id=flow_states_by_id, ) + + +async def load_wallet_automation_states_for_trade_symbols(wallet_id: str) -> WalletAutomationStates: + """Load active automation states for trade-symbol resolution (inputs only, no workflow outputs).""" + return await load_wallet_automation_states( + wallet_id, + statuses=list(TRADE_SYMBOL_AUTOMATION_STATUSES), + load_output=False, + ) diff --git a/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py index 58e01b8c1e..6255b76d22 100644 --- a/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py +++ b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py @@ -49,7 +49,9 @@ async def run_portfolio_history_collection( logger.info("No exchange accounts found for wallet %s", wallet_id) return [] - wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states(wallet_id) + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states_for_trade_symbols( + wallet_id, + ) for context in contexts: context.trade_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( wallet_id, diff --git a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py index 2df4be5048..3c0e06494a 100644 --- a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py @@ -48,10 +48,15 @@ async def portfolio_history_collection( return await PortfolioHistoryWorkflow._run_collection(scheduled_time, collection_params) @staticmethod + @SCHEDULER.INSTANCE.step(name="run_portfolio_history_collection") async def _run_collection( scheduled_time: datetime.datetime, collection_params: workflow_params_module.PortfolioHistoryCollectionParams | None = None, ) -> dict[str, typing.Any]: + """ + This is a step to avoid storing results of internal dbos select statements, + which are otherwise counted as steps and have their result stored + """ try: logger.info("Starting portfolio history collection at %s", scheduled_time) if collection_params and collection_params.wallet_ids: diff --git a/packages/node/tests/scheduler/automations/test_automation_states_loader.py b/packages/node/tests/scheduler/automations/test_automation_states_loader.py index 2a66ec5e55..9a69661dba 100644 --- a/packages/node/tests/scheduler/automations/test_automation_states_loader.py +++ b/packages/node/tests/scheduler/automations/test_automation_states_loader.py @@ -322,12 +322,38 @@ async def test_loads_protocol_and_flow_states_from_single_source_fetch(self): "wallet-id", ) - load_sources_mock.assert_awaited_once_with("wallet-id", None) + load_sources_mock.assert_awaited_once_with("wallet-id", None, load_output=True) assert len(wallet_automation_states.protocol_states) == 1 assert wallet_automation_states.protocol_states[0].id == _LOADER_PARENT_ID assert wallet_automation_states.flow_states_by_id[_LOADER_PARENT_ID].automation.metadata.automation_id == "automation_1" +class TestLoadWalletAutomationStatesForTradeSymbols: + @pytest.mark.asyncio + async def test_uses_enqueued_pending_statuses_and_skips_workflow_outputs(self): + get_latest_mock = mock.AsyncMock(return_value=[]) + scheduler_mock = mock.Mock() + scheduler_mock._get_latest_workflow_for_each_automation = get_latest_mock + with mock.patch( + "octobot_node.scheduler.SCHEDULER", + scheduler_mock, + ): + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states_for_trade_symbols( + "wallet-id", + ) + + get_latest_mock.assert_awaited_once_with( + "wallet-id", + [ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ], + load_output=False, + ) + assert wallet_automation_states.protocol_states == [] + assert wallet_automation_states.flow_states_by_id == {} + + class TestGetAutomationWorkflowStatus: @pytest.mark.asyncio async def test_returns_matching_pending_workflow(self): diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py index 35ec681508..2e70d889b7 100644 --- a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py @@ -1,6 +1,7 @@ import mock import pytest +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.portfolio_history.portfolio_history_executor as portfolio_history_executor_module @@ -34,3 +35,44 @@ async def test_filters_accounts_when_account_ids_set( "wallet-user-1", ) mock_build_context.assert_called_once_with("wallet-user-1", account_two) + + +class TestRunPortfolioHistoryCollectionTradeSymbolLoader: + @pytest.mark.asyncio + @mock.patch( + "octobot_node.scheduler.portfolio_history.portfolio_history_executor.trade_symbols_resolver_module.resolve_trade_symbols", + new_callable=mock.AsyncMock, + return_value=[], + ) + @mock.patch( + "octobot_node.scheduler.portfolio_history.portfolio_history_executor.automation_states_loader_module.load_wallet_automation_states_for_trade_symbols", + new_callable=mock.AsyncMock, + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_uses_trade_symbol_loader_not_full_wallet_loader( + self, + mock_account_provider, + load_for_trade_symbols_mock, + resolve_trade_symbols_mock, + ): + account = mock.MagicMock() + account.id = "acc-1" + mock_account_provider.instance.return_value.list_items.return_value = [account] + wallet_automation_states = automation_states_loader_module.WalletAutomationStates( + protocol_states=[], + flow_states_by_id={}, + ) + load_for_trade_symbols_mock.return_value = wallet_automation_states + + with mock.patch.object( + portfolio_history_executor_module, + "_build_context_for_account", + return_value=mock.MagicMock(), + ), mock.patch( + "octobot_flow.jobs.portfolio_history_job.PortfolioHistoryJob", + ) as portfolio_history_job_class_mock: + portfolio_history_job_class_mock.return_value.run = mock.AsyncMock(return_value=[]) + await portfolio_history_executor_module.run_portfolio_history_collection("wallet-user-1") + + load_for_trade_symbols_mock.assert_awaited_once_with("wallet-user-1") + resolve_trade_symbols_mock.assert_awaited_once() diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py index 58ff4f2f7e..692f37a434 100644 --- a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py @@ -106,3 +106,12 @@ async def test_uses_wallet_whitelist_from_params_without_listing_all_wallets( account_ids=["acc-1"], ) assert result["succeeded"] == 1 + + +class TestPortfolioHistoryCollectionStepRegistration: + def test_run_collection_is_registered_as_dbos_step(self, portfolio_history_workflow_module): + import dbos._registrations as dbos_registrations_module + + run_collection = portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection + assert dbos_registrations_module.get_dbos_func_name(run_collection) == "run_portfolio_history_collection" + assert hasattr(run_collection, "dbos_function_name") diff --git a/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py index 22d9cb4563..fbe030d715 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py @@ -13,25 +13,11 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import octobot_commons.constants as constants - -# TODO later: find a way to store this in exchange tentacles instead and use exchange.get_default_reference_market -# Issue: hollaex based exchanages require an exchange configuration to be identified as such -_SPECIFIC_REFERENCE_MARKET_PER_EXCHANGE: dict[str, str] = { - "coinbase": "USDC", - "binance": "USDC", -} _EXCHANGES_WITH_DIFFERENT_PUBLIC_DATA_AFTER_AUTH = set[str]([ "mexc", "lbank", ]) -def get_default_reference_market_per_exchange(exchanges: list[str]) -> dict[str, str]: - return {exchange: get_default_exchange_reference_market(exchange) for exchange in exchanges} - -def get_default_exchange_reference_market(exchange: str) -> str: - return _SPECIFIC_REFERENCE_MARKET_PER_EXCHANGE.get(exchange, constants.DEFAULT_REFERENCE_MARKET) - def is_exchange_with_different_public_data_after_auth(exchange: str) -> bool: return exchange in _EXCHANGES_WITH_DIFFERENT_PUBLIC_DATA_AFTER_AUTH diff --git a/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py index c63b6507b6..16c2903f56 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py @@ -24,10 +24,10 @@ import octobot_evaluators.constants as evaluators_constants import octobot_trading.constants as trading_constants +import octobot_trading.api.exchange as exchange_api import tentacles.Trading.Mode.index_trading_mode.index_trading as index_trading import octobot_copy.enums as rebalancer_enums -import tentacles.Meta.Keywords.scripting_library.configuration.exchanges_configuration as exchanges_configuration def create_index_config_from_tentacles_config( @@ -36,7 +36,7 @@ def create_index_config_from_tentacles_config( ) -> commons_profiles.ProfileData: trading_mode_config = tentacles_config[0].config distribution = trading_mode_config[index_trading.IndexTradingModeProducer.INDEX_CONTENT] - reference_market = exchanges_configuration.get_default_exchange_reference_market(exchange) + reference_market = exchange_api.get_default_exchange_reference_market(exchange) # replace USD by reference market for element in distribution: if element[rebalancer_enums.DistributionKeys.NAME] == "USD": diff --git a/packages/tentacles/Trading/Mode/trading_view_signals_trading_mode/trading_view_signals_trading.py b/packages/tentacles/Trading/Mode/trading_view_signals_trading_mode/trading_view_signals_trading.py index a25a0797fe..28e72035ff 100644 --- a/packages/tentacles/Trading/Mode/trading_view_signals_trading_mode/trading_view_signals_trading.py +++ b/packages/tentacles/Trading/Mode/trading_view_signals_trading_mode/trading_view_signals_trading.py @@ -227,7 +227,7 @@ def _adapt_symbol( default_reference_market = reference_market else: # replace the generic USD stablecoin symbol with the actual stablecoin symbol for this exchange - default_reference_market = scripting_library.get_default_exchange_reference_market(exchange_name) + default_reference_market = trading_api.exchange.get_default_exchange_reference_market(exchange_name) replaced_symbol = parsed_data[cls.SYMBOL_KEY].replace(cls.GENERIC_USD_STABLECOIN_SYMBOL, default_reference_market) commons_logging.get_logger(cls.__name__).info( f"Replaced generic USD stablecoin symbol {parsed_data[cls.SYMBOL_KEY]} with {replaced_symbol} for exchange {exchange_name} in signal data: {parsed_data}" diff --git a/packages/trading/octobot_trading/api/exchange.py b/packages/trading/octobot_trading/api/exchange.py index 2c66b30f50..3c050c1236 100644 --- a/packages/trading/octobot_trading/api/exchange.py +++ b/packages/trading/octobot_trading/api/exchange.py @@ -353,6 +353,14 @@ def get_default_exchange_type(exchange_name: str) -> str: return exchanges.get_default_exchange_type(exchange_name) +def get_default_exchange_reference_market(exchange_name: str) -> str: + return exchanges.util.exchange_util.get_default_exchange_reference_market(exchange_name) + + +def get_default_reference_market_per_exchange(exchange_names: list[str]) -> dict[str, str]: + return exchanges.util.exchange_util.get_default_reference_market_per_exchange(exchange_names) + + def is_sponsoring(exchange_name: str) -> bool: return exchanges.is_broker_enabled_on_exchange(exchange_name) diff --git a/packages/trading/octobot_trading/api/portfolio.py b/packages/trading/octobot_trading/api/portfolio.py index 2941bac6a8..01074e17c9 100644 --- a/packages/trading/octobot_trading/api/portfolio.py +++ b/packages/trading/octobot_trading/api/portfolio.py @@ -17,22 +17,16 @@ import typing import octobot_protocol.models as protocol_models -import octobot_commons.constants as commons_constants import octobot_commons.symbols as commons_symbols +import octobot_trading.api.exchange as exchange_api import octobot_trading.personal_data.portfolios.history.history_from_trades_and_transaction_builder as history_builder -import octobot_trading.enums as trading_enums import octobot_trading.exchange_channel as exchange_channel import octobot_trading.constants import octobot_trading.personal_data as personal_data def resolve_portfolio_valuation_unit(exchange_manager) -> str: - quote_currency = exchange_manager.exchange.get_option_value( - trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY - ) - if quote_currency: - return str(quote_currency) - return commons_constants.DEFAULT_REFERENCE_MARKET + return exchange_api.get_default_exchange_reference_market(exchange_manager.exchange_name) def get_portfolio(exchange_manager, as_decimal=True) -> dict: diff --git a/packages/trading/octobot_trading/exchanges/util/exchange_util.py b/packages/trading/octobot_trading/exchanges/util/exchange_util.py index 5eedeb7874..317408fe74 100644 --- a/packages/trading/octobot_trading/exchanges/util/exchange_util.py +++ b/packages/trading/octobot_trading/exchanges/util/exchange_util.py @@ -527,6 +527,26 @@ def get_default_exchange_type(exchange_name): return common_constants.DEFAULT_EXCHANGE_TYPE +def get_default_exchange_reference_market(exchange_name: str) -> str: + try: + quote_currency = ccxt_client_util.get_option_value_from_new_ccxt_client( + exchange_name, + enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY, + ) + except AttributeError: + quote_currency = None + if quote_currency: + return str(quote_currency) + return common_constants.DEFAULT_REFERENCE_MARKET + + +def get_default_reference_market_per_exchange(exchange_names: list[str]) -> dict[str, str]: + return { + exchange_name: get_default_exchange_reference_market(exchange_name) + for exchange_name in exchange_names + } + + def get_supported_exchange_types(exchange_name, tentacles_setup_config, exchange_config_by_exchange=None): exchange_class = get_exchange_class_from_name( exchanges_types.RestExchange, exchange_name, tentacles_setup_config, diff --git a/packages/trading/tests/api/test_portfolio.py b/packages/trading/tests/api/test_portfolio.py index 3da4759504..089ffa3db4 100644 --- a/packages/trading/tests/api/test_portfolio.py +++ b/packages/trading/tests/api/test_portfolio.py @@ -17,22 +17,27 @@ import mock import octobot_commons.constants as commons_constants import octobot_trading.api.portfolio as portfolio_api -import octobot_trading.enums as trading_enums class TestResolvePortfolioValuationUnit: - def test_returns_exchange_default_quote_currency_when_set(self): + @mock.patch("octobot_trading.api.exchange.get_default_exchange_reference_market", return_value="USDC") + def test_returns_exchange_default_quote_currency_when_set(self, mock_get_default_reference_market): exchange_manager = mock.Mock() - exchange_manager.exchange.get_option_value.return_value = "USDC" + exchange_manager.exchange_name = "binance" assert portfolio_api.resolve_portfolio_valuation_unit(exchange_manager) == "USDC" - exchange_manager.exchange.get_option_value.assert_called_once_with( - trading_enums.ExchangeClientOptions.DEFAULT_QUOTE_CURRENCY - ) + mock_get_default_reference_market.assert_called_once_with("binance") - def test_falls_back_to_default_reference_market_when_option_missing(self): + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value=commons_constants.DEFAULT_REFERENCE_MARKET, + ) + def test_falls_back_to_default_reference_market_when_option_missing( + self, mock_get_default_reference_market, + ): exchange_manager = mock.Mock() - exchange_manager.exchange.get_option_value.return_value = None + exchange_manager.exchange_name = "kraken" assert ( portfolio_api.resolve_portfolio_valuation_unit(exchange_manager) == commons_constants.DEFAULT_REFERENCE_MARKET ) + mock_get_default_reference_market.assert_called_once_with("kraken") diff --git a/packages/trading/tests/exchanges/util/test_exchange_util.py b/packages/trading/tests/exchanges/util/test_exchange_util.py index bd54b0178f..72a8e3896b 100644 --- a/packages/trading/tests/exchanges/util/test_exchange_util.py +++ b/packages/trading/tests/exchanges/util/test_exchange_util.py @@ -487,4 +487,42 @@ async def retry_till_success(_timeout, func, *args, **kwargs): assert len(batches) == 1 assert batches[0] == static_candle logger_mock.warning.assert_called_once() - assert "start_time did not advance" in logger_mock.warning.call_args[0][0] \ No newline at end of file + assert "start_time did not advance" in logger_mock.warning.call_args[0][0] + + +class TestGetDefaultExchangeReferenceMarket: + @mock.patch.object( + exchange_util.ccxt_client_util, + "get_option_value_from_new_ccxt_client", + return_value="USDC", + ) + def test_returns_ccxt_default_quote_currency_when_set(self, _mock_get_option): + assert exchange_util.get_default_exchange_reference_market("binance") == "USDC" + + @mock.patch.object( + exchange_util.ccxt_client_util, + "get_option_value_from_new_ccxt_client", + return_value=None, + ) + def test_falls_back_to_default_reference_market_when_option_missing(self, _mock_get_option): + assert ( + exchange_util.get_default_exchange_reference_market("kraken") + == commons_constants.DEFAULT_REFERENCE_MARKET + ) + + def test_returns_default_reference_market_for_unknown_exchange(self): + assert ( + exchange_util.get_default_exchange_reference_market("????") + == commons_constants.DEFAULT_REFERENCE_MARKET + ) + + @mock.patch.object( + exchange_util, + "get_default_exchange_reference_market", + side_effect=lambda exchange_name: "USDC" if exchange_name == "binance" else "USDT", + ) + def test_get_default_reference_market_per_exchange(self, _mock_get_default): + assert exchange_util.get_default_reference_market_per_exchange(["binance", "kraken"]) == { + "binance": "USDC", + "kraken": "USDT", + } \ No newline at end of file diff --git a/tests/unit_tests/community/test_authentication.py b/tests/unit_tests/community/test_authentication.py index 1213f6688a..b0004c8b9e 100644 --- a/tests/unit_tests/community/test_authentication.py +++ b/tests/unit_tests/community/test_authentication.py @@ -513,7 +513,9 @@ async def test_stop(auth): class TestCommunityAuthenticationStopLogging: async def test_non_singleton_stop_does_not_log_lifecycle(self): auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + # __init__ is skipped via __new__; assign attributes normally set in __init__ auth._use_as_singleton = False + auth._dk_sessions = {} auth.logger = mock.Mock() auth._fetch_account_task = None auth.supabase_client = mock.Mock(aclose=mock.AsyncMock()) @@ -529,7 +531,9 @@ async def test_non_singleton_stop_does_not_log_lifecycle(self): async def test_singleton_stop_logs_lifecycle(self): auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + # __init__ is skipped via __new__; assign attributes normally set in __init__ auth._use_as_singleton = True + auth._dk_sessions = {} auth.logger = mock.Mock() auth._fetch_account_task = None auth.supabase_client = mock.Mock(aclose=mock.AsyncMock()) @@ -784,6 +788,7 @@ def test_sync_server_url_is_a_bare_origin(): def _new_auth_for_signal_session_tests(): auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + # __init__ is skipped via __new__; assign attributes normally set in __init__ auth._dk_sessions = {} auth._dk_session_lock = asyncio.Lock() return auth @@ -887,6 +892,8 @@ async def test_distinct_addresses_get_distinct_sessions(self): class TestStopClosesCachedDkSessions: async def test_stop_closes_and_clears_cached_dk_sessions(self): auth = community.CommunityAuthentication.__new__(community.CommunityAuthentication) + # __init__ is skipped via __new__; assign attributes normally set in __init__ + auth._use_as_singleton = False auth.logger = mock.Mock() auth._fetch_account_task = None auth.supabase_client = mock.Mock(aclose=mock.AsyncMock()) From 5cb036a20a267b61008f6711d722103d3ebb422e Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Sat, 29 Aug 2026 14:10:19 +0200 Subject: [PATCH 08/10] [Backtesting] improve tests and use readonly --- .../collectors/data_collector.py | 4 ++- .../backtesting_data_sqlite_database.py | 14 ++++++--- .../importers/data_importer.py | 4 ++- .../backtesting/tests/database_test_util.py | 16 ---------- .../test_backtesting_data_sqlite_database.py | 30 +++++++++++++------ .../tests/importers/test_exchange_importer.py | 22 +++++++++----- .../sqlite/base_sqlite_database.py | 2 ++ .../legacy_data_converter/legacy_converter.py | 4 ++- 8 files changed, 56 insertions(+), 40 deletions(-) diff --git a/packages/backtesting/octobot_backtesting/collectors/data_collector.py b/packages/backtesting/octobot_backtesting/collectors/data_collector.py index e2dc100858..f73dccb2ab 100644 --- a/packages/backtesting/octobot_backtesting/collectors/data_collector.py +++ b/packages/backtesting/octobot_backtesting/collectors/data_collector.py @@ -85,7 +85,9 @@ def set_file_path(self) -> None: def create_database(self) -> None: if not self.database: - self.database = backtesting_databases.BacktestingDataSQLiteDatabase(self.temp_file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( + self.temp_file_path, read_only=False + ) def finalize_database(self): os.rename(self.temp_file_path, self.file_path) diff --git a/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py index 42498b137a..9ecc620666 100644 --- a/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py +++ b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py @@ -23,6 +23,12 @@ class BacktestingDataSQLiteDatabase(base_sqlite_database.BaseSQLiteDatabase): + """SQLite store for backtesting .data files. + + Read-only by default: data importers only read committed fixtures and must not + checkpoint WAL into them. Data collectors and converters that write OHLCV / + description rows must pass read_only=False. + """ TIMESTAMP_COLUMN = "timestamp" DEFAULT_ORDER_BY = TIMESTAMP_COLUMN DEFAULT_SORT = enums.DataBaseOrderBy.DESC.value @@ -30,8 +36,8 @@ class BacktestingDataSQLiteDatabase(base_sqlite_database.BaseSQLiteDatabase): DEFAULT_SIZE = -1 CACHE_SIZE = 50 - def __init__(self, file_name): - super().__init__(file_name) + def __init__(self, file_name, read_only: bool = True): + super().__init__(file_name, read_only=read_only) self.tables = [] self.cache = {} @@ -316,8 +322,8 @@ async def __init_tables_list(self): @contextlib.asynccontextmanager -async def new_sqlite_database(file_path): - local_database = BacktestingDataSQLiteDatabase(file_path) +async def new_sqlite_database(file_path, read_only: bool = True): + local_database = BacktestingDataSQLiteDatabase(file_path, read_only=read_only) try: await local_database.initialize() yield local_database diff --git a/packages/backtesting/octobot_backtesting/importers/data_importer.py b/packages/backtesting/octobot_backtesting/importers/data_importer.py index 6756aef0f1..2be5a971f8 100644 --- a/packages/backtesting/octobot_backtesting/importers/data_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/data_importer.py @@ -59,7 +59,9 @@ def provides_accurate_price_time_frame(self) -> bool: def load_database(self) -> None: file_path = self.adapt_file_path_if_necessary() if not self.database: - self.database = backtesting_databases.BacktestingDataSQLiteDatabase(file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( + file_path, read_only=True + ) def adapt_file_path_if_necessary(self): if path.isfile(self.file_path): diff --git a/packages/backtesting/tests/database_test_util.py b/packages/backtesting/tests/database_test_util.py index b94e7c83f0..68d4c158ec 100644 --- a/packages/backtesting/tests/database_test_util.py +++ b/packages/backtesting/tests/database_test_util.py @@ -1,23 +1,7 @@ import os -import shutil -import tempfile BACKTESTING_STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") def static_database_fixture_path(file_name: str) -> str: return os.path.join(BACKTESTING_STATIC_DIR, file_name) - - -def copy_static_database_fixture(file_name: str) -> str: - with tempfile.NamedTemporaryFile(delete=False, suffix=".data") as temp_file: - temp_database_path = temp_file.name - shutil.copy2(static_database_fixture_path(file_name), temp_database_path) - return temp_database_path - - -def remove_temp_database(database_path: str) -> None: - for suffix in ("", "-wal", "-shm"): - path = f"{database_path}{suffix}" - if os.path.isfile(path): - os.remove(path) diff --git a/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py index 863ea66ca1..3e2f30faeb 100644 --- a/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py +++ b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py @@ -17,19 +17,31 @@ DATA_FILE1 = "ExchangeHistoryDataCollector_1589740606.4862757.data" DATA_FILE2 = "second_ExchangeHistoryDataCollector_1589740606.4862757.data" +STATIC_FIXTURE_PATHS = { + DATA_FILE1: database_test_util.static_database_fixture_path(DATA_FILE1), + DATA_FILE2: database_test_util.static_database_fixture_path(DATA_FILE2), +} OHLCV = mock.Mock(value="ohlcv") KLINE = mock.Mock(value="kline") +@pytest.fixture(scope="module", autouse=True) +def _static_fixtures_unchanged(): + mtimes_before = { + file_name: os.path.getmtime(fixture_path) + for file_name, fixture_path in STATIC_FIXTURE_PATHS.items() + } + yield + for file_name, fixture_path in STATIC_FIXTURE_PATHS.items(): + assert os.path.getmtime(fixture_path) == mtimes_before[file_name] + + @contextlib.asynccontextmanager async def get_database(data_file=DATA_FILE1): - temp_database_path = database_test_util.copy_static_database_fixture(data_file) - try: - async with backtesting_databases.new_sqlite_database(temp_database_path) as database: - yield database - await asyncio_tools.wait_asyncio_next_cycle() - finally: - database_test_util.remove_temp_database(temp_database_path) + fixture_path = STATIC_FIXTURE_PATHS[data_file] + async with backtesting_databases.new_sqlite_database(fixture_path) as database: + yield database + await asyncio_tools.wait_asyncio_next_cycle() @contextlib.asynccontextmanager @@ -37,7 +49,7 @@ async def get_temp_empty_database(): with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: database_name = temp_file.name try: - async with backtesting_databases.new_sqlite_database(database_name) as database: + async with backtesting_databases.new_sqlite_database(database_name, read_only=False) as database: yield database finally: await asyncio_tools.wait_asyncio_next_cycle() @@ -48,7 +60,7 @@ async def get_temp_empty_database(): async def test_invalid_file(): with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: file_name = temp_file.name - database = backtesting_databases.BacktestingDataSQLiteDatabase(file_name) + database = backtesting_databases.BacktestingDataSQLiteDatabase(file_name, read_only=False) try: await database.initialize() assert not await database.check_table_exists(KLINE) diff --git a/packages/backtesting/tests/importers/test_exchange_importer.py b/packages/backtesting/tests/importers/test_exchange_importer.py index ce13d74060..d4e8b90a42 100644 --- a/packages/backtesting/tests/importers/test_exchange_importer.py +++ b/packages/backtesting/tests/importers/test_exchange_importer.py @@ -13,6 +13,8 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. +import os + import pytest from contextlib import asynccontextmanager @@ -28,21 +30,25 @@ pytestmark = pytest.mark.asyncio EXCHANGE_HISTORY_DATA_FILE = "ExchangeHistoryDataCollector_1589740606.4862757.data" +STATIC_FIXTURE_PATH = database_test_util.static_database_fixture_path(EXCHANGE_HISTORY_DATA_FILE) + + +@pytest.fixture(scope="module", autouse=True) +def _static_fixture_unchanged(): + mtime_before = os.path.getmtime(STATIC_FIXTURE_PATH) + yield + assert os.path.getmtime(STATIC_FIXTURE_PATH) == mtime_before # use context manager instead of fixture to prevent pytest threads issues @asynccontextmanager async def get_importer(): - temp_database_path = database_test_util.copy_static_database_fixture(EXCHANGE_HISTORY_DATA_FILE) + importer = ExchangeDataImporter({}, STATIC_FIXTURE_PATH) try: - importer = ExchangeDataImporter({}, temp_database_path) - try: - await importer.initialize() - yield importer - finally: - await importer.stop() + await importer.initialize() + yield importer finally: - database_test_util.remove_temp_database(temp_database_path) + await importer.stop() async def test_initialize(): diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py index 7b7e641f5d..ed9338cc3a 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py +++ b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py @@ -83,6 +83,8 @@ def connection_pragmas(self) -> list[str]: # uncommitted writer work stays in the WAL and is rolled back on recovery. # synchronous=FULL: fsync on commit so committed data survives sudden process kill. # busy_timeout: wait up to 5s when a read overlaps a writer checkpoint instead of failing immediately. + if self.read_only: + return ["PRAGMA busy_timeout=5000"] return [ "PRAGMA journal_mode=WAL", "PRAGMA synchronous=FULL", diff --git a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py index a530df2131..4f1f9b1a04 100644 --- a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py +++ b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py @@ -79,7 +79,9 @@ async def can_convert(self, ) -> bool: async def convert(self) -> bool: try: self.database = backtesting_databases.BacktestingDataSQLiteDatabase( - path.join(backtesting_constants.BACKTESTING_FILE_PATH, self.converted_file)) + path.join(backtesting_constants.BACKTESTING_FILE_PATH, self.converted_file), + read_only=False, + ) await self.database.initialize() await self._create_description() for time_frame in self.time_frames: From 400380835a220fde310d090c6bf69c13d5121b24 Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Sat, 29 Aug 2026 14:16:31 +0200 Subject: [PATCH 09/10] [Automation] reduce stored data size --- .../tests/cryptography/test_signing.py | 7 +- .../accounts/exchange_account_elements.py | 64 ++++++ .../post_iteration_actions_details.py | 31 +++ .../accounts/account_state_persistence.py | 16 ++ .../logic/actions/actions_executor.py | 12 + ...st_post_iteration_actions_details_clear.py | 118 ++++++++++ ...t_exchange_account_elements_trim_trades.py | 76 ++++++ .../octobot_process_functional_shared.py | 34 ++- .../test_octobot_process_edit_config.py | 10 +- .../test_account_state_persistence.py | 57 ++++- .../actions/test_actions_executor_merge.py | 127 ++++++++++ packages/node/octobot_node/constants.py | 3 + .../node/octobot_node/protocol/automations.py | 23 +- .../workflows/automation_workflow.py | 5 + .../node/tests/protocol/test_automations.py | 70 ++++++ .../node/tests/scheduler/test_task_context.py | 68 ++++++ .../workflows/test_automation_workflow.py | 169 +++++++++++++- .../octobot_process_ops.py | 89 ++++++-- .../tests/test_octobot_process_ops.py | 216 +++++++++++++++++- 19 files changed, 1157 insertions(+), 38 deletions(-) create mode 100644 packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py create mode 100644 packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py diff --git a/packages/commons/tests/cryptography/test_signing.py b/packages/commons/tests/cryptography/test_signing.py index c758279e66..9df55c9780 100644 --- a/packages/commons/tests/cryptography/test_signing.py +++ b/packages/commons/tests/cryptography/test_signing.py @@ -117,8 +117,11 @@ def test_verify_signature_invalid_signature(): data = b"Test data" signature = cryptography.sign_data(data, private_key_pem) - # Corrupt the signature - corrupted_signature = signature[:-1] + b"\x00" + # Corrupt the signature (always change a byte) + corrupted_bytes = bytearray(signature) + corruption_index = len(corrupted_bytes) // 2 + corrupted_bytes[corruption_index] ^= 0x01 + corrupted_signature = bytes(corrupted_bytes) is_valid = cryptography.verify_signature(data, public_key_pem, corrupted_signature) diff --git a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py index 2909be57ce..d7d4d2c00c 100644 --- a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py +++ b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py @@ -23,6 +23,7 @@ class ExchangeAccountElements(account_elements_import.AccountElements): orders: octobot_trading.exchanges.OrdersDetails = dataclasses.field(default_factory=octobot_trading.exchanges.OrdersDetails) positions: list[octobot_trading.exchanges.PositionDetails] = dataclasses.field(default_factory=list) trades: list[dict] = dataclasses.field(default_factory=list) + trade_summaries: dict[str, list[str]] = dataclasses.field(default_factory=dict) def __post_init__(self): super().__post_init__() @@ -36,6 +37,69 @@ def __post_init__(self): self.trades = [ dict(trade) for trade in self.trades # type: ignore ] + self.trade_summaries = self._normalize_trade_summaries(self.trade_summaries) + + @staticmethod + def _normalize_trade_summaries( + trade_summaries: typing.Optional[dict[str, list[str]]], + ) -> dict[str, list[str]]: + if not trade_summaries: + return {} + normalized_summaries: dict[str, list[str]] = {} + for symbol, trade_ids in trade_summaries.items(): + symbol_text = str(symbol).strip() + if not symbol_text: + continue + normalized_ids = [ + str(trade_id) + for trade_id in (trade_ids or []) + if trade_id is not None and str(trade_id) + ] + if normalized_ids: + normalized_summaries[symbol_text] = normalized_ids + return normalized_summaries + + @staticmethod + def trade_id_and_symbol_from_trade_dict(trade: dict) -> typing.Optional[tuple[str, str]]: + order_columns = octobot_trading.enums.ExchangeConstantsOrderColumns + trade_id = trade.get(order_columns.EXCHANGE_TRADE_ID.value) or trade.get( + order_columns.EXCHANGE_ID.value + ) + symbol = trade.get(order_columns.SYMBOL.value) + if trade_id is None or symbol is None: + return None + symbol_text = str(symbol).strip() + if not symbol_text: + return None + return str(trade_id), symbol_text + + def trim_trades_to_live_window(self, max_full_trades: int) -> None: + if max_full_trades <= 0 or len(self.trades) <= max_full_trades: + return + order_columns = octobot_trading.enums.ExchangeConstantsOrderColumns + timestamp_key = order_columns.TIMESTAMP.value + + def sort_key(trade: dict) -> tuple[float, str]: + timestamp = trade.get(timestamp_key) + try: + timestamp_value = float(timestamp) if timestamp is not None else 0.0 + except (TypeError, ValueError): + timestamp_value = 0.0 + identity_key = octobot_trading.personal_data.trades.trades_util.trade_identity_key(trade) + identity_text = str(identity_key) if identity_key is not None else "" + return timestamp_value, identity_text + + sorted_trades = sorted(self.trades, key=sort_key) + evicted_trades = sorted_trades[: len(sorted_trades) - max_full_trades] + self.trades = sorted_trades[len(sorted_trades) - max_full_trades:] + for evicted_trade in evicted_trades: + trade_identity = self.trade_id_and_symbol_from_trade_dict(evicted_trade) + if trade_identity is None: + continue + trade_id, symbol = trade_identity + archived_ids = self.trade_summaries.setdefault(symbol, []) + if trade_id not in archived_ids: + archived_ids.append(trade_id) def has_pending_chained_orders(self) -> bool: for order in self.orders.missing_orders: diff --git a/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py b/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py index a5a3fd6400..10a5d8ab6f 100644 --- a/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py +++ b/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py @@ -2,6 +2,7 @@ import typing import octobot_commons.dataclasses +import octobot_commons.dsl_interpreter @dataclasses.dataclass @@ -36,6 +37,36 @@ class PostIterationActionsDetails(octobot_commons.dataclasses.MinimizableDatacla def has_automation_actions(self) -> bool: return bool(self.stop_automation) + @classmethod + def post_iteration_clear(cls, post_iteration_payload: dict) -> None: + post_iteration_payload.pop("updated_exchange_account_elements", None) + + @classmethod + def post_iteration_clear_from_action_result(cls, action_result: dict) -> None: + """ + Remove merge-consumed PostIteration fields from persisted action-result dicts. + Mutates ``action_result`` in place (top-level and recall-nested blobs). + """ + post_iter_name = cls.__name__ + # Top-level PostIteration (e.g. stop_automation / update_automation_configuration). + top_level_payload = action_result.get(post_iter_name) + if isinstance(top_level_payload, dict): + cls.post_iteration_clear(top_level_payload) + # Nested PostIteration inside a re-calling operator recall payload (e.g. run_octobot_process). + if not octobot_commons.dsl_interpreter.ReCallingOperatorResult.is_re_calling_operator_result( + action_result + ): + return + recall_wrapper = octobot_commons.dsl_interpreter.ReCallingOperatorResult.from_dict( + action_result[octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__] + ) + inner_last_result = recall_wrapper.last_execution_result + if not isinstance(inner_last_result, dict): + return + nested_payload = inner_last_result.get(post_iter_name) + if isinstance(nested_payload, dict): + cls.post_iteration_clear(nested_payload) + def should_cancel_iteration(self) -> bool: # cancelled if global view refresh is triggered, otherwise proceed # with next iteration required steps diff --git a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py index 035db240e0..f03ec2ebff 100644 --- a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py +++ b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py @@ -222,3 +222,19 @@ def persist_account_trading_from_iteration_state( "Skipping account trading persistence for wallet %s: wallet not registered", user_id, ) + + +def trim_live_trades_in_iteration_state( + iteration_state: typing.Optional[dict], + max_full_trades: int, +) -> None: + if iteration_state is None or max_full_trades <= 0: + return + automation_state = octobot_flow.entities.AutomationState.from_dict(iteration_state) + exchange_account_elements = automation_state.automation.exchange_account_elements + if exchange_account_elements is None: + return + exchange_account_elements.trim_trades_to_live_window(max_full_trades) + trimmed_state = automation_state.to_dict(include_default_values=False) + iteration_state.clear() + iteration_state.update(trimmed_state) diff --git a/packages/flow/octobot_flow/logic/actions/actions_executor.py b/packages/flow/octobot_flow/logic/actions/actions_executor.py index f70e1a0dd9..a29ef02ab8 100644 --- a/packages/flow/octobot_flow/logic/actions/actions_executor.py +++ b/packages/flow/octobot_flow/logic/actions/actions_executor.py @@ -59,6 +59,8 @@ async def execute(self): if should_stop_processing: break self._sync_after_execution(synchronized_exchange_account_elements) + if synchronized_exchange_account_elements: + self._clear_post_iteration_after_merge_from_action_results() if self._update_execution_details: await self._update_actions_history() await self._insert_execution_bot_logs(dsl_executor.pending_bot_logs) @@ -366,6 +368,16 @@ def _sync_after_execution( ) self._sync_exchange_account_elements(exchange_account_elements, new_transactions) + def _clear_post_iteration_after_merge_from_action_results(self) -> None: + for action in self._actions: + if not isinstance(action, octobot_flow.entities.DSLScriptActionDetails): + continue + for result_candidate in (action.result, action.previous_execution_result): + if isinstance(result_candidate, dict): + octobot_flow.entities.PostIterationActionsDetails.post_iteration_clear_from_action_result( + result_candidate + ) + def _get_new_transactions_from_actions_results( self, exchange_account_elements: octobot_flow.entities.ExchangeAccountElements, diff --git a/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py b/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py new file mode 100644 index 0000000000..a344bfeab3 --- /dev/null +++ b/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py @@ -0,0 +1,118 @@ +import octobot_commons.dsl_interpreter + +import octobot_flow.entities.automations.post_iteration_actions_details as post_iteration_actions_details_module + + +class TestPostIterationClear: + def test_removes_only_updated_exchange_account_elements(self): + post_iteration_payload = { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + "configuration_update": "run_octobot_process('folder')", + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear( + post_iteration_payload + ) + assert "updated_exchange_account_elements" not in post_iteration_payload + assert post_iteration_payload["stop_automation"] is False + assert post_iteration_payload["configuration_update"] == "run_octobot_process('folder')" + + +class TestPostIterationClearFromActionResult: + def test_clears_nested_recall_post_iteration(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + }, + "pid": 42, + }, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + assert inner_post_iteration["stop_automation"] is False + assert action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"]["pid"] == 42 + + def test_top_level_stop_automation_unchanged(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + post_iteration_name: { + "stop_automation": True, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result[post_iteration_name] == {"stop_automation": True} + + def test_top_level_configuration_update_unchanged(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + configuration_update = "run_octobot_process('folder')" + action_result = { + post_iteration_name: { + "configuration_update": configuration_update, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result[post_iteration_name]["configuration_update"] == configuration_update + + def test_clears_top_level_and_nested_when_both_present(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": [{"id": "top"}]}, + }, + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": [{"id": "nested"}]}, + }, + }, + }, + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert "updated_exchange_account_elements" not in action_result[post_iteration_name] + nested_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in nested_post_iteration + + def test_recall_without_post_iteration_preserves_inner_fields(self): + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + "pid": 42, + "init_state_ok": True, + }, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_last_result = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"] + assert inner_last_result["pid"] == 42 + assert inner_last_result["init_state_ok"] is True + + def test_non_recall_result_without_post_iteration_is_no_op(self): + action_result = {"pid": 42} + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result == {"pid": 42} diff --git a/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py b/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py new file mode 100644 index 0000000000..0e4c1f4b02 --- /dev/null +++ b/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py @@ -0,0 +1,76 @@ +import octobot_trading.enums as octobot_trading_enums_import + +import octobot_flow.entities as octobot_flow_entities + + +def _trade_dict(trade_id: str, symbol: str, timestamp: float) -> dict: + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + return { + order_columns.EXCHANGE_TRADE_ID.value: trade_id, + order_columns.SYMBOL.value: symbol, + order_columns.TIMESTAMP.value: timestamp, + } + + +class TestTrimTradesToLiveWindow: + def test_keeps_newest_trades_and_archives_older_ids_by_symbol(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + _trade_dict("old-1", "BTC/USDT", 1.0), + _trade_dict("new-1", "ETH/USDT", 3.0), + _trade_dict("old-2", "BTC/USDT", 2.0), + ] + ) + elements.trim_trades_to_live_window(2) + trade_id_key = octobot_trading_enums_import.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value + kept_trade_ids = [trade[trade_id_key] for trade in elements.trades] + assert kept_trade_ids == ["old-2", "new-1"] + assert elements.trade_summaries == {"BTC/USDT": ["old-1"]} + + def test_deduplicates_archived_ids_per_symbol(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + _trade_dict("old-1", "BTC/USDT", 1.0), + _trade_dict("old-2", "BTC/USDT", 2.0), + ], + trade_summaries={"BTC/USDT": ["old-1"]}, + ) + elements.trim_trades_to_live_window(1) + assert elements.trade_summaries == {"BTC/USDT": ["old-1"]} + assert len(elements.trades) == 1 + + def test_no_op_when_trade_count_within_window(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[_trade_dict("trade-1", "BTC/USDT", 1.0)], + trade_summaries={"ETH/USDT": ["archived-1"]}, + ) + elements.trim_trades_to_live_window(100) + assert len(elements.trades) == 1 + assert elements.trade_summaries == {"ETH/USDT": ["archived-1"]} + + def test_skips_evicted_trade_missing_symbol(self): + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "no-symbol", + order_columns.TIMESTAMP.value: 1.0, + }, + _trade_dict("keep-1", "BTC/USDT", 2.0), + ] + ) + elements.trim_trades_to_live_window(1) + assert len(elements.trades) == 1 + assert elements.trades[0][order_columns.EXCHANGE_TRADE_ID.value] == "keep-1" + assert elements.trade_summaries == {} + + +class TestExchangeAccountElementsTradeSummariesRoundTrip: + def test_to_dict_from_dict_preserves_trade_summaries_dict(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["archived-1", "archived-2"]}, + ) + restored = octobot_flow_entities.ExchangeAccountElements.from_dict( + elements.to_dict(include_default_values=False) + ) + assert restored.trade_summaries == {"BTC/USDT": ["archived-1", "archived-2"]} diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index d53a112d81..297c8baa3a 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -50,8 +50,10 @@ # it is fixed at import time and stays 30 unless the interpreter reloads constants. EXPECTED_PROCESS_BOT_DUMP_INTERVAL_SEC = 5.0 -# Same as `waiting_time=` in run_octobot_process(...) DSL for this file's tests. +# Same as `waiting_time=` in run_octobot_process(...) DSL for this file's tests (steady override when child is healthy). WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2 +# Init / child-not-alive recall interval (matches octobot_process_ops.FAST_PING_WAITING_TIME). +FAST_RECALL_WAITING_TIME_SEC = 5.0 RECALL_SCHEDULE_TOLERANCE_SEC = 1.5 @@ -271,6 +273,12 @@ def _recall_inner_from_dsl_action( return None +def _expected_recall_waiting_time_from_inner(inner: typing.Optional[dict]) -> float: + if isinstance(inner, dict) and inner.get("waiting_time") is not None: + return float(inner["waiting_time"]) + return FAST_RECALL_WAITING_TIME_SEC + + def _assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump: dict[str, typing.Any], *, @@ -304,6 +312,28 @@ def _assert_run_octobot_process_recall_scheduled_to_in_dump( ) +def _assert_run_octobot_process_recall_scheduled_to_from_job( + job: octobot_flow.jobs.AutomationJob, + *, + expected_waiting_time_sec: typing.Optional[float] = None, + schedule_tolerance_sec: float = RECALL_SCHEDULE_TOLERANCE_SEC, + assert_delay_matches_waiting_time: bool = True, +) -> None: + if assert_delay_matches_waiting_time and expected_waiting_time_sec is None: + run_action_state = _get_action_by_id(job, ACTION_ID_RUN_OCTOBOT) + assert run_action_state is not None + inner = _recall_inner_from_dsl_action(run_action_state) + expected_waiting_time_sec = _expected_recall_waiting_time_from_inner(inner) + elif expected_waiting_time_sec is None: + expected_waiting_time_sec = WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC + _assert_run_octobot_process_recall_scheduled_to_in_dump( + job.dump(), + expected_waiting_time_sec=expected_waiting_time_sec, + schedule_tolerance_sec=schedule_tolerance_sec, + assert_delay_matches_waiting_time=assert_delay_matches_waiting_time, + ) + + def _get_action_by_id( job: octobot_flow.jobs.AutomationJob, action_id: str ) -> typing.Optional[octobot_flow.entities.AbstractActionDetails]: @@ -376,7 +406,7 @@ async def poll_automation_until_child_process_ready( state = automation_state while time.monotonic() < deadline: poll_job = await run_automation_job_without_exchange_manager(state, [], [], {}) - _assert_run_octobot_process_recall_scheduled_to_in_dump(poll_job.dump()) + _assert_run_octobot_process_recall_scheduled_to_from_job(poll_job) run_action_state = _get_action_by_id(poll_job, ACTION_ID_RUN_OCTOBOT) assert run_action_state is not None inner = _recall_inner_from_dsl_action(run_action_state) diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index e2282b532b..a472691b40 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -218,8 +218,8 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( priority_actions = functionnal_tests.resolved_actions([update_config_priority_action]) async with octobot_flow.jobs.AutomationJob(state, priority_actions, [], {}) as refresh_phase: await refresh_phase.run() - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - refresh_phase.dump() + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_from_job( + refresh_phase ) assert popen_calls["count"] == spawn_before_refresh + 1 state = refresh_phase.dump() @@ -237,10 +237,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( six_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( state, [], [], {} ) - dump_payload = six_poll.dump() - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - dump_payload + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_from_job( + six_poll ) + dump_payload = six_poll.dump() automation_dump = dump_payload.get("automation") eae_dict = ( automation_dump.get("exchange_account_elements") diff --git a/packages/flow/tests/logic/accounts/test_account_state_persistence.py b/packages/flow/tests/logic/accounts/test_account_state_persistence.py index 7dfd9f2351..eb8b0060eb 100644 --- a/packages/flow/tests/logic/accounts/test_account_state_persistence.py +++ b/packages/flow/tests/logic/accounts/test_account_state_persistence.py @@ -4,6 +4,7 @@ import mock +import octobot.community.wallet_backend.errors as wallet_backend_errors_module import octobot_protocol.models as protocol_models import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors @@ -11,6 +12,7 @@ import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums +import octobot_flow.entities import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module @@ -331,8 +333,6 @@ def test_raises_when_order_missing_id_and_exchange_id(self): class TestPersistAccountTradingFromIterationState: def test_persists_trading_snapshot_from_automation_state(self): - import octobot_flow.entities - exchange_details = octobot_flow.entities.ExchangeAccountDetails() exchange_details.exchange_details.exchange_account_id = "acc-sync-1" elements = octobot_flow.entities.ExchangeAccountElements() @@ -361,9 +361,6 @@ def test_persists_trading_snapshot_from_automation_state(self): ) def test_skips_when_wallet_not_registered(self): - import octobot.community.wallet_backend.errors as wallet_backend_errors_module - import octobot_flow.entities - exchange_details = octobot_flow.entities.ExchangeAccountDetails() exchange_details.exchange_details.exchange_account_id = "acc-sync-1" elements = octobot_flow.entities.ExchangeAccountElements() @@ -384,3 +381,53 @@ def test_skips_when_wallet_not_registered(self): "wallet-1", automation_state.to_dict(include_default_values=False), ) + + +class TestTrimLiveTradesInIterationState: + def test_trims_exchange_account_elements_in_state_dict(self): + order_columns = trading_enums.ExchangeConstantsOrderColumns + state_dict = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=octobot_flow.entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(3) + ], + ), + ), + ).to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state(state_dict, 2) + restored_state = octobot_flow.entities.AutomationState.from_dict(state_dict) + elements = restored_state.automation.exchange_account_elements + assert elements is not None + assert len(elements.trades) == 2 + assert elements.trade_summaries == {"BTC/USDT": ["trade-0"]} + + def test_no_op_when_trade_count_within_window(self): + order_columns = trading_enums.ExchangeConstantsOrderColumns + state_dict = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=octobot_flow.entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "trade-1", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: 1.0, + }, + ], + trade_summaries={"ETH/USDT": ["archived-1"]}, + ), + ), + ).to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state(state_dict, 100) + restored_state = octobot_flow.entities.AutomationState.from_dict(state_dict) + elements = restored_state.automation.exchange_account_elements + assert elements is not None + assert len(elements.trades) == 1 + assert elements.trade_summaries == {"ETH/USDT": ["archived-1"]} diff --git a/packages/flow/tests/logic/actions/test_actions_executor_merge.py b/packages/flow/tests/logic/actions/test_actions_executor_merge.py index 18b2ff299f..e91916bd20 100644 --- a/packages/flow/tests/logic/actions/test_actions_executor_merge.py +++ b/packages/flow/tests/logic/actions/test_actions_executor_merge.py @@ -1,5 +1,6 @@ import mock +import octobot_commons.dsl_interpreter import octobot_commons.profiles as commons_profiles import octobot_trading.enums as octobot_trading_enums_import import octobot_trading.exchanges.util.exchange_data as exchange_data_import @@ -84,3 +85,129 @@ def test_merges_transactions_only_once_per_txid(self): automation.exchange_account_elements.merge_synchronized_snapshots([snap1, snap2]) txs = automation.exchange_account_elements.transactions assert [t[tx_key] for t in txs] == ["tx-existing", "tx-a", "tx-b"] + + +class TestClearPostIterationAfterMerge: + def test_clears_updated_exchange_account_elements_from_recall_wrapper(self): + post_iteration_name = octobot_flow_entities.PostIterationActionsDetails.__name__ + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + }, + "pid": 42, + }, + } + } + octobot_flow_entities.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + assert inner_post_iteration["stop_automation"] is False + + def test_clear_after_sync_clears_post_iteration_on_action_result(self): + post_iteration_name = octobot_flow_entities.PostIterationActionsDetails.__name__ + action = octobot_flow_entities.DSLScriptActionDetails( + id="process-action", + dsl_script="run_octobot_process('folder')", + ) + action.result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": { + "trades": [_trade_stub("merged-trade")], + }, + }, + }, + } + } + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + snapshot = octobot_flow_entities.ExchangeAccountElements( + trades=[_trade_stub("merged-trade")], + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [action], + False, + ) + executor_action._sync_after_execution([snapshot]) + executor_action._clear_post_iteration_after_merge_from_action_results() + inner_post_iteration = action.result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + trade_id_key = octobot_trading_enums_import.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value + assert automation.exchange_account_elements.trades[0][trade_id_key] == "merged-trade" + + def test_clear_without_post_iteration_eae_preserves_recall_fields(self): + action = octobot_flow_entities.DSLScriptActionDetails( + id="process-action", + dsl_script="run_octobot_process('folder')", + ) + action.result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + "pid": 42, + "init_state_ok": True, + }, + } + } + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [action], + False, + ) + executor_action._clear_post_iteration_after_merge_from_action_results() + inner_last = action.result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"] + assert inner_last["pid"] == 42 + assert inner_last["init_state_ok"] is True + + +class TestMergeThenTrimIntegration: + def test_large_snapshot_merge_then_trim_obey_live_window(self): + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(120) + ] + snapshot = octobot_flow_entities.ExchangeAccountElements(trades=trades) + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [], + False, + ) + executor_action._sync_after_execution([snapshot]) + automation.exchange_account_elements.trim_trades_to_live_window(100) + assert len(automation.exchange_account_elements.trades) == 100 + assert len(automation.exchange_account_elements.trade_summaries.get("BTC/USDT", [])) == 20 diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 192993d3c7..beeac2ccdc 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -51,6 +51,9 @@ os.getenv("RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", 150.0) ) +# Max full trade dicts kept on automation.exchange_account_elements.trades; older ids archived in trade_summaries. +AUTOMATION_LIVE_STATE_MAX_TRADES = 100 + TASKS_ENCRYPTION_ENV_VARS = [ "TASKS_SERVER_RSA_PRIVATE_KEY", "TASKS_SERVER_ECDSA_PRIVATE_KEY", diff --git a/packages/node/octobot_node/protocol/automations.py b/packages/node/octobot_node/protocol/automations.py index d7ba7b355d..d305b69b7b 100644 --- a/packages/node/octobot_node/protocol/automations.py +++ b/packages/node/octobot_node/protocol/automations.py @@ -403,7 +403,7 @@ def _fill_protocol_automation_state( assets = octobot_trading_portfolios_protocol.to_protocol_assets(portfolio.content) orders = _order_summaries_from_open_orders(exchange_elements.orders.open_orders) or None positions = _position_summaries(exchange_elements.positions) or None - trades = _trade_summaries(exchange_elements.trades) or None + trades = _automation_trade_summaries(exchange_elements) or None metadata = protocol_automation_state.metadata.model_copy( update={ "updated_at": _metadata_updated_at_from_execution( @@ -474,3 +474,24 @@ def _trade_summaries(trades: list[dict]) -> list[protocol_models.TradeSummary]: continue summaries.append(protocol_models.TradeSummary(id=str(trade_id), symbol=str(symbol))) return summaries + + +def _automation_trade_summaries( + exchange_elements: flow_entities.ExchangeAccountElements, +) -> list[protocol_models.TradeSummary]: + summaries_by_id: dict[str, protocol_models.TradeSummary] = {} + for symbol, trade_ids in (exchange_elements.trade_summaries or {}).items(): + symbol_text = str(symbol).strip() + if not symbol_text: + continue + for trade_id in trade_ids or []: + trade_id_text = str(trade_id).strip() + if not trade_id_text: + continue + summaries_by_id[trade_id_text] = protocol_models.TradeSummary( + id=trade_id_text, + symbol=symbol_text, + ) + for trade_summary in _trade_summaries(exchange_elements.trades): + summaries_by_id[trade_summary.id] = trade_summary + return list(summaries_by_id.values()) diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index ecb7e93d14..80dfa0e549 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -271,6 +271,11 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) parsed_inputs.task.user_id, result.next_actions_description.state if result.next_actions_description else None, ) + if result.next_actions_description is not None: + account_state_persistence_module.trim_live_trades_in_iteration_state( + result.next_actions_description.state, + constants.AUTOMATION_LIVE_STATE_MAX_TRADES, + ) else: retry_delay_seconds = max(0.0, (next_step_at or time.time()) - time.time()) AutomationWorkflow.get_logger(parsed_inputs).info( diff --git a/packages/node/tests/protocol/test_automations.py b/packages/node/tests/protocol/test_automations.py index 610b294fcc..f0a8768c2b 100644 --- a/packages/node/tests/protocol/test_automations.py +++ b/packages/node/tests/protocol/test_automations.py @@ -757,6 +757,76 @@ def test_assets_without_enrichment(self): assert filled.assets[0].available == 1.0 +class TestAutomationTradeSummariesFromExchangeElements: + def test_union_of_archived_and_full_trades(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["old-1", "old-2"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "live-1", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + summaries = automations_protocol._automation_trade_summaries(elements) + summary_ids = {summary.id for summary in summaries} + assert summary_ids == {"old-1", "old-2", "live-1"} + + def test_full_trade_overrides_archived_summary_for_same_id(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["shared-id"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "shared-id", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "shared-id" + assert summaries[0].symbol == "ETH/USDT" + + def test_only_trade_summaries_dict_populated(self): + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["archived-1"]}, + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "archived-1" + assert summaries[0].symbol == "BTC/USDT" + + def test_skips_empty_symbol_or_trade_id_entries(self): + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"": ["id-1"], "BTC/USDT": ["", "valid-id"]}, + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "valid-id" + + def test_fill_protocol_state_counts_archived_and_live_trades(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["old-1"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "live-1", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + automation = _minimal_automation_details() + automation.exchange_account_elements = elements + flow_state = flow_entities.AutomationState(automation=automation) + filled = automations_protocol._fill_protocol_automation_state(_minimal_protocol_base(), flow_state) + assert filled.trades is not None + assert len(filled.trades) == 2 + trade_ids = {trade.id for trade in filled.trades} + assert trade_ids == {"old-1", "live-1"} + + class TestFillProtocolAutomationStateEmpties: def test_no_exchange_elements_yields_empty_protocol_lists(self): flow_state = flow_entities.AutomationState(automation=_minimal_automation_details()) diff --git a/packages/node/tests/scheduler/test_task_context.py b/packages/node/tests/scheduler/test_task_context.py index 81c672df32..e82efb393a 100644 --- a/packages/node/tests/scheduler/test_task_context.py +++ b/packages/node/tests/scheduler/test_task_context.py @@ -17,6 +17,10 @@ import pytest import mock +import octobot_flow.entities as flow_entities_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_trading.enums as trading_enums_module + import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client from octobot_node.scheduler.task_context import encrypted_task from octobot_node.models import Task @@ -211,3 +215,67 @@ def test_encrypted_task_per_task_ecdsa_key_takes_precedence(self) -> None: mock_decrypt.assert_called_once_with( "encrypted_content", "metadata", user_ecdsa_public_key=b"per_task_key" ) + + +class TestEncryptedTaskTrimmedState: + def test_encrypts_trimmed_next_actions_description_state(self) -> None: + mock_settings = mock.Mock() + mock_settings.TASKS_SERVER_RSA_PRIVATE_KEY = None + mock_settings.TASKS_USER_ECDSA_PUBLIC_KEY = None + + order_columns = trading_enums_module.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(120) + ] + elements = flow_entities_module.ExchangeAccountElements(trades=trades) + automation_state = flow_entities_module.AutomationState( + automation=flow_entities_module.AutomationDetails( + metadata=flow_entities_module.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=elements, + ), + ) + job_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + auth_details={}, + params={}, + ) + to_update_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[mock.Mock()], + next_actions_description=job_description, + actions_dag=mock.Mock(), + ) + captured_state_dicts: list[dict] = [] + + def capture_encrypt(state_dict: dict) -> tuple[str, str]: + captured_state_dicts.append(state_dict) + return "encrypted_payload", "encryption_meta" + + mock_encrypt = mock.Mock(side_effect=capture_encrypt) + + with mock.patch("octobot_node.config.settings", mock_settings), \ + mock.patch( + "octobot_node.scheduler.task_context.encryption.get_next_encrypted_if_needed_content_and_metadata", + mock_encrypt, + ): + task = Task(name="test_task", content="plain") + with encrypted_task(task, to_update_result=to_update_result): + job_description.state = automation_state.to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state( + job_description.state, + 100, + ) + + assert len(captured_state_dicts) == 1 + restored_state = flow_entities_module.AutomationState.from_dict( + captured_state_dicts[0]["state"] + ) + trimmed_elements = restored_state.automation.exchange_account_elements + assert trimmed_elements is not None + assert len(trimmed_elements.trades) == 100 + assert len(trimmed_elements.trade_summaries.get("BTC/USDT", [])) == 20 + assert to_update_result.maybe_encrypted_next_actions_description == "encrypted_payload" diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index 4c83b7e29e..404dc1fed7 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -16,6 +16,7 @@ import asyncio import contextlib +import copy import importlib import json import os @@ -29,8 +30,11 @@ import octobot_trading.constants import octobot_trading.errors as octobot_trading_errors +import octobot_trading.enums as trading_enums import octobot_commons.cryptography +import octobot.community.wallet_backend.errors as wallet_backend_errors_module + import octobot_copy.constants as copy_constants import octobot_copy.errors as copy_errors import octobot_protocol.models as protocol_models @@ -99,6 +103,30 @@ def _automation_state_dict_with_scheduled_to( return state +def _automation_state_with_trade_count(trade_count: int) -> octobot_flow.entities.AutomationState: + order_columns = trading_enums.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(trade_count) + ] + elements = octobot_flow.entities.ExchangeAccountElements() + elements.trades = trades + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation_1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + return automation_state + + def _octobot_actions_job_mock_class_pending_priority_skipped( *, automation_inner_state: dict[str, typing.Any], @@ -856,8 +884,6 @@ async def test_execute_iteration_passes_trading_signals_to_octobot_actions_job( async def test_execute_iteration_persists_open_orders_to_account_trading( self, import_automation_workflow, task ): - import octobot_trading.enums as trading_enums - task.user_id = "0xwallet-trading-sync" task.content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", @@ -921,9 +947,6 @@ async def test_execute_iteration_persists_open_orders_to_account_trading( async def test_execute_iteration_continues_when_trading_persistence_wallet_missing( self, import_automation_workflow, task ): - import octobot.community.wallet_backend.errors as wallet_backend_errors_module - import octobot_trading.enums as trading_enums - task.user_id = "0xwallet-trading-sync" task.content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", @@ -977,6 +1000,142 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi assert parsed_progress_status.error is None +class TestPersistBeforeTrimTrades: + @pytest.mark.asyncio + @required_imports + async def test_persist_receives_full_trades_then_state_trimmed_to_live_window( + self, import_automation_workflow, task + ): + task.user_id = "0xwallet-trading-sync" + task.content = json.dumps({"params": {"ACTIONS": "trade"}}) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + automation_state = _automation_state_with_trade_count(120) + next_actions_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + ) + action = octobot_flow.entities.ConfiguredActionDetails(id="action_1", action="trade") + mock_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[action], + next_actions_description=next_actions_description, + has_next_actions=True, + actions_dag=None, + should_stop=False, + ) + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_on_result=lambda result_ref: _apply_octobot_actions_job_result_template(result_ref, mock_result), + ) + persisted_state_snapshots: list[dict] = [] + + def capture_persisted_state(user_id, state_dict): + if state_dict is not None: + persisted_state_snapshots.append(copy.deepcopy(state_dict)) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + task_context, + "encrypted_task", + mock.MagicMock(), + ) as mock_encrypted, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + side_effect=capture_persisted_state, + ): + mock_encrypted.return_value.__enter__ = mock.Mock(return_value=None) + mock_encrypted.return_value.__exit__ = mock.Mock(return_value=None) + await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + assert len(persisted_state_snapshots) == 1 + persisted_state = octobot_flow.entities.AutomationState.from_dict(persisted_state_snapshots[0]) + assert len(persisted_state.automation.exchange_account_elements.trades) == 120 + trimmed_state = octobot_flow.entities.AutomationState.from_dict(mock_result.next_actions_description.state) + trimmed_elements = trimmed_state.automation.exchange_account_elements + assert len(trimmed_elements.trades) == octobot_node.constants.AUTOMATION_LIVE_STATE_MAX_TRADES + assert len(trimmed_elements.trade_summaries.get("BTC/USDT", [])) == 20 + + @pytest.mark.asyncio + @required_imports + async def test_trim_runs_when_wallet_persistence_skipped(self, import_automation_workflow, task): + task.user_id = "0xwallet-trading-sync" + task.content = json.dumps({"params": {"ACTIONS": "trade"}}) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + automation_state = _automation_state_with_trade_count(120) + next_actions_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + ) + mock_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[], + next_actions_description=next_actions_description, + has_next_actions=True, + actions_dag=None, + should_stop=False, + ) + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_on_result=lambda result_ref: _apply_octobot_actions_job_result_template(result_ref, mock_result), + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + task_context, + "encrypted_task", + mock.MagicMock(), + ) as mock_encrypted, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "trim_live_trades_in_iteration_state", + ) as trim_mock: + mock_encrypted.return_value.__enter__ = mock.Mock(return_value=None) + mock_encrypted.return_value.__exit__ = mock.Mock(return_value=None) + await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + trim_mock.assert_called_once_with( + mock_result.next_actions_description.state, + octobot_node.constants.AUTOMATION_LIVE_STATE_MAX_TRADES, + ) + + @pytest.mark.asyncio + @required_imports + async def test_postponed_iteration_skips_persist_and_trim( + self, import_automation_workflow, task + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + skip_error = octobot_flow.errors.PendingPriorityActionsSkippedError("skipped") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_mock, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "trim_live_trades_in_iteration_state", + ) as trim_mock: + await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + persist_mock.assert_not_called() + trim_mock.assert_not_called() + + class TestExecuteIterationPendingPriorityActionsSkippedError: @pytest.mark.asyncio @required_imports diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index 46ff619d97..7f29b0493f 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -51,7 +51,9 @@ import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers -DEFAULT_PING_WAITING_TIME = 2.0 +DEFAULT_PING_WAITING_TIME = 30.0 +FAST_PING_WAITING_TIME = 5.0 +STEADY_PING_WAITING_TIME = 60.0 DEFAULT_ENSURE_TIMEOUT = 120.0 DEFAULT_FORCE_KILL_EXIT_WAIT_SECONDS = 5.0 DEFAULT_DSL_PROFILE_ID = "non-trading" @@ -209,6 +211,32 @@ def _in_restart_grace_period( return (now - last_updated_at) < ping_timeout +def _resolve_recall_waiting_time( + recall_state: octobot_process_state_import.OctobotProcessState, + loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], + *, + dsl_waiting_time: typing.Optional[float], + now: float, + ping_timeout: float, + stored_pid_running: bool, +) -> float: + child_confirmed_alive = _is_child_confirmed_alive(loaded_state) + is_running = stored_pid_running or child_confirmed_alive + if _in_restart_grace_period( + recall_state, + loaded_state, + now=now, + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ): + return FAST_PING_WAITING_TIME + if recall_state.init_state_ok and is_running: + if dsl_waiting_time is not None: + return float(dsl_waiting_time) + return STEADY_PING_WAITING_TIME + return FAST_PING_WAITING_TIME + + def _should_use_recall_path( recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], @@ -1130,7 +1158,7 @@ def _emit_ensure_recall( octobot_flow_entities.PostIterationActionsDetails( updated_exchange_account_elements=( parsed_process_bot_state.exchange_account_elements.to_dict( - include_default_values=True + include_default_values=False ) ), ).to_dict(include_default_values=False) @@ -1148,7 +1176,7 @@ async def _pre_compute_recall_path( last_result: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ping_timeout: float, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState] = None, ) -> None: @@ -1205,6 +1233,14 @@ async def _pre_compute_recall_path( self.pid = resolved_pid recall_state = _apply_resolved_pid_to_state(recall_state, resolved_pid) _get_logger().info("process state path (re-call path): %s", state_path) + resolved_recall_interval = _resolve_recall_waiting_time( + recall_state, + loaded_state, + dsl_waiting_time=dsl_waiting_time, + now=now, + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ) # Running: stored recall pid or child-confirmed-alive → init_state_ok, optional EAE. is_running = stored_pid_running or child_confirmed_alive if is_running: @@ -1224,7 +1260,7 @@ async def _pre_compute_recall_path( state=updated, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded_state, ) return @@ -1242,7 +1278,7 @@ async def _pre_compute_recall_path( state=recall_state.model_copy(update={"state_file_path": state_path}), last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded_state, ) @@ -1254,7 +1290,7 @@ async def _pre_compute_first_spawn( last_result: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ) -> None: # One-time (or re-) materialization, free ports, env, and `Popen` at project root. exchange_auth_overrides = params.get("exchange_auth_data") @@ -1334,6 +1370,16 @@ async def _pre_compute_first_spawn( ) # First process state check after spawn (init cap still uses `state.started_waiting_at`). loaded = await _load_process_bot_state(state_file_path) + ping_timeout = float(params.get("ping_timeout") or DEFAULT_ENSURE_TIMEOUT) + stored_pid_running = state.pid > 0 and process_util.pid_is_running(state.pid) + resolved_recall_interval = _resolve_recall_waiting_time( + state, + loaded, + dsl_waiting_time=dsl_waiting_time, + now=time.time(), + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ) is_live = loaded is not None and _is_process_state_alive(loaded) if is_live: state_pid = loaded.metadata.pid @@ -1355,11 +1401,19 @@ async def _pre_compute_first_spawn( ) ready = state.model_copy(update={"init_state_ok": True}) _release_child_listen_ports(web_port, node_port, user_folder) + resolved_recall_interval = _resolve_recall_waiting_time( + ready, + loaded, + dsl_waiting_time=dsl_waiting_time, + now=time.time(), + ping_timeout=ping_timeout, + stored_pid_running=True, + ) self._emit_ensure_recall( state=ready, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded, ) return @@ -1377,7 +1431,7 @@ async def _pre_compute_first_spawn( state=state, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded, ) @@ -1426,7 +1480,7 @@ async def _pre_compute_update_config_refresh( params: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ping_timeout: float, ) -> None: # Resolve prior child layout from re-call payload; required for stop, wait, and paths to remove. @@ -1469,7 +1523,7 @@ async def _pre_compute_update_config_refresh( params, {}, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) async def pre_compute(self) -> None: @@ -1520,7 +1574,12 @@ async def pre_compute(self) -> None: last_result = self.get_last_execution_result(params) or {} start_time = time.time() ping_timeout = float(params.get("ping_timeout") or DEFAULT_ENSURE_TIMEOUT) - recall_interval = float(params.get("waiting_time") or DEFAULT_PING_WAITING_TIME) + waiting_time_param = params.get("waiting_time") + dsl_waiting_time = None + if waiting_time_param is not None: + waiting_time_value = float(waiting_time_param) + if waiting_time_value != DEFAULT_PING_WAITING_TIME: + dsl_waiting_time = waiting_time_value if self.matches_operator_signal(dsl_interpreter.OperatorSignal.UPDATE_CONFIG.value): await self._pre_compute_update_config_refresh( last_result, @@ -1528,7 +1587,7 @@ async def pre_compute(self) -> None: working_directory, params, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ping_timeout=ping_timeout, ) return @@ -1558,7 +1617,7 @@ async def pre_compute(self) -> None: params, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) return # 3. Recall vs respawn from liveness/grace rules. @@ -1573,7 +1632,7 @@ async def pre_compute(self) -> None: recall_state, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ping_timeout=ping_timeout, loaded_state=loaded_state, ) @@ -1600,7 +1659,7 @@ async def pre_compute(self) -> None: params, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index 9f3a8473cd..b099a6a779 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -115,7 +115,7 @@ def _healthy_recall_inner( user_root = user_root or "/x/ub" state_fn = os.path.join(user_root, octobot_constants.PROCESS_BOT_STATE_FILE_NAME) return { - "waiting_time": octobot_process_ops.DEFAULT_PING_WAITING_TIME, + "waiting_time": octobot_process_ops.STEADY_PING_WAITING_TIME, "last_execution_time": 0.0, "http_base_url": "http://127.0.0.1:20050", "web_port": 20050, @@ -1957,7 +1957,7 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp assert le.get("init_state_ok") is True assert le.get("http_base_url", "").startswith("http://") assert le.get("pid") == 10001 - assert le.get("waiting_time") == octobot_process_ops.DEFAULT_PING_WAITING_TIME + assert le.get("waiting_time") == octobot_process_ops.STEADY_PING_WAITING_TIME assert octobot_flow_entities.PostIterationActionsDetails.__name__ in le post = octobot_flow_entities.PostIterationActionsDetails.from_dict( le[octobot_flow_entities.PostIterationActionsDetails.__name__] @@ -2318,7 +2318,7 @@ async def test_does_not_apply_init_timeout_after_init_state_ok(self, tmp_path): class TestEnsureOctobotProcessWaitingTimeConstantInPayload: - async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): + async def test_waiting_time_uses_fast_tier_during_init_even_with_dsl_override(self, tmp_path): start_script = tmp_path / "start.py" start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( @@ -2361,6 +2361,32 @@ async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): assert isinstance(op.value, dict) le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] assert isinstance(le, dict) + assert le.get("waiting_time") == octobot_process_ops.FAST_PING_WAITING_TIME + + async def test_dsl_waiting_time_overrides_steady_tier_when_child_is_healthy(self, tmp_path): + inner = _healthy_recall_inner(tmp_path=tmp_path, init_state_ok=True) + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + waiting_time=7.0, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == inner["pid"], + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ): + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] assert le.get("waiting_time") == 7.0 @@ -3927,3 +3953,187 @@ async def live_state_with_same_pid(*_unused): spawn_mock.assert_not_called() rebind_mock.assert_not_called() assert op.pid == 25044 + + +def _minimal_octobot_process_state(**updates) -> octobot_process_state_import.OctobotProcessState: + base_state = { + "http_base_url": "http://127.0.0.1:20050", + "web_port": 20050, + "node_port": 30050, + "user_root": "/x/ub", + "user_folder": "ub", + "log_folder": "/x/l", + "profile_id": "p", + "pid": 10001, + "state_file_path": "/x/ub/process_bot_state.json", + "executor_id": TEST_EXECUTOR_ID, + } + base_state.update(updates) + return octobot_process_state_import.OctobotProcessState.model_validate(base_state) + + +class TestTwoTierPingWaitingTime: + def test_fast_when_init_state_not_ok(self): + recall_state = _minimal_octobot_process_state(init_state_ok=False) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + def test_steady_when_init_ok_and_child_running(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=True, + ) + assert resolved_interval == octobot_process_ops.STEADY_PING_WAITING_TIME + + def test_restart_grace_window_uses_fast_tier(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True, pid=0) + loaded_state = process_bot_state_import.ProcessBotState( + metadata=process_bot_state_import.Metadata(updated_at=995.0), + ) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + loaded_state, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + def test_dsl_waiting_time_overrides_steady_only(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=7.0, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=True, + ) + assert resolved_interval == 7.0 + + def test_dsl_waiting_time_does_not_override_fast_tier(self): + recall_state = _minimal_octobot_process_state(init_state_ok=False) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=7.0, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + async def test_executor_restart_first_spawn_uses_fast_waiting_time(self, tmp_path): + (tmp_path / "start.py").write_text("#", encoding="utf-8") + inner = _healthy_recall_inner( + pid=10002, + tmp_path=tmp_path, + executor_id="old-executor-id", + ) + op = octobot_process_ops.create_octobot_process_operators( + None, TEST_EXECUTOR_ID + )[0]( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + return_value=False, + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": inner["user_root"], + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock: + spawn_mock.return_value = mock.Mock(spec=["pid"], pid=30005) + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + assert le.get("waiting_time") == octobot_process_ops.FAST_PING_WAITING_TIME + assert le.get("init_state_ok") is False + + +class TestEmitEnsureRecallEaeSerialization: + async def test_emitted_post_iteration_eae_omits_default_only_fields(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 10001, + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ): + spawn_mock.return_value.pid = 10001 + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + post_iteration_payload = le[octobot_flow_entities.PostIterationActionsDetails.__name__] + emitted_eae_dict = post_iteration_payload["updated_exchange_account_elements"] + assert "trade_summaries" not in emitted_eae_dict + From 2021eff63cc619bec17eb74c9084ef568e24af5e Mon Sep 17 00:00:00 2001 From: Guillaume De Saint Martin Date: Sat, 29 Aug 2026 16:13:29 +0200 Subject: [PATCH 10/10] [Node] fix unwanted automation stop --- deposit_withdrawal_test_results.txt | Bin 15978 -> 0 bytes .../flow/octobot_flow/jobs/automation_job.py | 3 +- .../workflows/automation_workflow.py | 16 +- .../test_automation_states_loader.py | 38 ++- .../workflows/test_automation_workflow.py | 312 +++++++++++++++++- 5 files changed, 349 insertions(+), 20 deletions(-) delete mode 100644 deposit_withdrawal_test_results.txt diff --git a/deposit_withdrawal_test_results.txt b/deposit_withdrawal_test_results.txt deleted file mode 100644 index 1538c5326ae272b8c3a7b6981d8581928a7fdf72..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15978 zcmeI3`EMFW6vyZ1O8p;Ds>+U3u(5MEuJ|E`-9||pH%Zk>p+bPML(Nr=*!=Ttzn?cu zmgPunY<7c#R_ryi@6Eg4ycz!cx8N2%GOq7xuBUTGXYP8g?%J+p&wW>RUA6V}v?zvx z`__HqI<6sDeRm?*UDp)6bKz;ZUALqAtd5L3azpiaKf=E2axSlS&UyEP+B??nN1Jtz z)UvJL&Zw>72JTeHP_XiP|G?V8-4(>PARh~3*X;|^Q=N)O=aGxeP zZ`&Wl|Calpaqr50UJ6%5SNsSP%SvC$lIWSvr+UWP&ULNm4ogFNc$lKJtEL|G9qOs1 zgHZ#8`<&VTzU&>#K}ox9I52ddOnp^1@3zlRkB%W&{06(H=Thdd!y=QZX?dxQZ8eF73k}yV50Wk1Pb6u1d4W zyRiSK(nRtTd~a7z4fXX|;u~GZbLSJaqqTf9zgO@oT+wp!EFUG0%~pbSE;5EqLBgX1 zJIQ@wBzGm>V4qiytXN&$xK%N}oJv<}${W3>jJSerW0B-)@GscwLs{%%F@24>t5~oY zx6!Tiqx?-s5Gy~D4hG%wx$b47|Aut2C9U+a_DFZ+q}XBKZ3y!0s7I6=m>y@;YaBx9 zqjw81(CuK!zz_O=AkR>fkMJHO7lRTX$ELfRqJ8dZ{*;b=e$AL~hkYW!>)R*M@u?yi zwu4P_J(c|h+xb*}PQ7K^Qh`N26rhb;=RfL6rHHJY*(bJ7w69wwgqX#AIpcmC>-R?O z&BAwz0KO@ zPt99wNdMSC7uNDnd@V9%n&M5oLIij1^mp?BVA9D z@`@;@3Xn8*);v10bZU&$XOhP6dB|{ZS1}+U9*Vg>}AG!klhPtCOdeRTgjO$?8?d z>etMPlcw3@tfnH}bR+qhJV=msx+7^Qk06Ub8 zmk$*Ad}Y^HJ@#e6QKf@Cmx_m9(ZL$AD!&inw$R3|N_cZDjVPPOvDw+O(aL9JWk)l5t?9hnoIVY0 z^&aN)Ig0n5tDKRgNog9A9rr_@m+!b=T4gDZQ@vd8`!&mSKJD`=!0d?Pf35s8m;X6Y z>7>5CYxR6~z4IT7@b-awsJ6+STj76N>~ZA3(YyL0qbm8fDx}-0#PhGI`>L7GuMbm0 F{2y*(h!Fq) diff --git a/packages/flow/octobot_flow/jobs/automation_job.py b/packages/flow/octobot_flow/jobs/automation_job.py index 93c7d3e2ad..7a9e7968ac 100644 --- a/packages/flow/octobot_flow/jobs/automation_job.py +++ b/packages/flow/octobot_flow/jobs/automation_job.py @@ -420,7 +420,8 @@ async def _execute_automation_actions( ) raise except Exception as err: - self._logger.error( + self._logger.exception( + err, True, f"Unexpected error when updating {automation_signature}: {err.__class__.__name__}: {err}" ) raise diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index 80dfa0e549..476bf548f5 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -87,7 +87,7 @@ async def execute_automation(inputs: dict) -> typing.Optional[str]: ) if not continue_workflow: AutomationWorkflow.get_logger(parsed_inputs).info( - f"Stopped workflow (remaining steps: {iteration_result.progress_status.remaining_steps})" + f"Automation stopped (remaining steps: {iteration_result.progress_status.remaining_steps})" ) final_state = iteration_result.next_iteration_description final_state_metadata = iteration_result.next_iteration_description_metadata @@ -147,7 +147,6 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) executed_step: str = "no action executed" execution_error = next_step = next_step_at = execution_error_message = None postponed_iteration = False - has_next_actions_override: typing.Optional[bool] = None next_iteration_description_override: typing.Optional[str] = None next_iteration_description_metadata_override: typing.Optional[str] = None result = octobot_flow_client.OctoBotActionsJobResult() @@ -200,7 +199,6 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) action_job.description.state ) postponed_iteration = True - has_next_actions_override = True next_iteration_description_override = parsed_inputs.task.content next_iteration_description_metadata_override = parsed_inputs.task.content_metadata except ( @@ -221,7 +219,6 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) execution_error = execution_error_status.value execution_error_message = str(err) postponed_iteration = True - has_next_actions_override = True next_iteration_description_override = automation_states_loader.patch_task_content_degraded_state( parsed_inputs.task.content, execution_error, @@ -306,7 +303,7 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) else result.next_actions_description_encryption_metadata ), has_next_actions=( - has_next_actions_override if postponed_iteration else result.has_next_actions + True if postponed_iteration else result.has_next_actions ), ).to_dict(include_default_values=False) @@ -360,7 +357,10 @@ async def _process_pending_priority_actions_and_reschedule( parsed_inputs: params.AutomationWorkflowInputs, previous_iteration_result: params.AutomationWorkflowIterationResult ) -> tuple[bool, params.AutomationWorkflowIterationResult]: - if not previous_iteration_result.has_next_actions: + if ( + not previous_iteration_result.has_next_actions + and not previous_iteration_result.progress_status.postponed_iteration + ): return False, previous_iteration_result # In case new priority actions were sent, execute them now. # Any action sent to this workflow will be lost if not processed by it. @@ -460,10 +460,12 @@ def _should_continue_workflow( progress_status: params.ProgressStatus, stop_on_error: bool ) -> bool: + if progress_status.postponed_iteration: + return True if progress_status.error: # failed iteration, return global progress where it stopped and exit workflow AutomationWorkflow.get_logger(parsed_inputs).error( - f"Failed iteration: stopping workflow, error: {progress_status.error}. " + f"Automation stopped: unrecoverable iteration error: {progress_status.error}. " f"Iteration's last step: {progress_status.latest_step}" ) return stop_on_error diff --git a/packages/node/tests/scheduler/automations/test_automation_states_loader.py b/packages/node/tests/scheduler/automations/test_automation_states_loader.py index 9a69661dba..a50653d98b 100644 --- a/packages/node/tests/scheduler/automations/test_automation_states_loader.py +++ b/packages/node/tests/scheduler/automations/test_automation_states_loader.py @@ -5,6 +5,7 @@ import dbos import mock +import octobot_flow.entities import pytest import octobot_protocol.models as protocol_models @@ -18,6 +19,36 @@ _LOADER_PARENT_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" +def _sample_postpone_dag_actions() -> list[dict]: + return [ + {"id": "action_dsl_1", "dsl_script": "1 if True else 2"}, + { + "id": "action_dsl_2", + "dsl_script": "1 if True else 2", + "dependencies": [{"action_id": "action_dsl_1"}], + }, + ] + + +def _actions_dag_from_task_content(task_content: str) -> dict: + return json.loads(task_content)["state"]["automation"]["actions_dag"] + + +def _canonical_actions_dag(actions_dag: dict) -> dict: + automation_state = octobot_flow.entities.AutomationState.from_dict({ + "automation": { + "metadata": {"automation_id": "canonical"}, + "actions_dag": actions_dag, + "execution": {}, + } + }) + return automation_state.to_dict(include_default_values=False)["automation"]["actions_dag"] + + +def _assert_actions_dag_equal(actions_dag_left: dict, actions_dag_right: dict) -> None: + assert _canonical_actions_dag(actions_dag_left) == _canonical_actions_dag(actions_dag_right) + + def _automation_task_content(*, automation_name: str, automation_id: str = "automation_1") -> str: return json.dumps( { @@ -27,7 +58,7 @@ def _automation_task_content(*, automation_name: str, automation_id: str = "auto "automation_id": automation_id, "name": automation_name, }, - "actions_dag": {"actions": []}, + "actions_dag": {"actions": _sample_postpone_dag_actions()}, "execution": {}, }, }, @@ -137,6 +168,7 @@ def test_success_workflow_state_dict_matches_output(self): class TestPatchTaskContentDegradedState: def test_persists_degraded_state_in_task_content(self): task_content = _automation_task_content(automation_name="copy-grid") + actions_dag_before = _actions_dag_from_task_content(task_content) patched_content = automation_states_loader_module.patch_task_content_degraded_state( task_content, @@ -145,6 +177,7 @@ def test_persists_degraded_state_in_task_content(self): since=1234.5, ) + _assert_actions_dag_equal(_actions_dag_from_task_content(patched_content), actions_dag_before) degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] assert degraded_state == { "since": 1234.5, @@ -154,12 +187,14 @@ def test_persists_degraded_state_in_task_content(self): def test_preserves_existing_degraded_since_on_subsequent_patch(self): task_content = _automation_task_content(automation_name="copy-grid") + actions_dag_before = _actions_dag_from_task_content(task_content) task_content = automation_states_loader_module.patch_task_content_degraded_state( task_content, "not_enough_funds", "Insufficient funds", since=1000.0, ) + _assert_actions_dag_equal(_actions_dag_from_task_content(task_content), actions_dag_before) patched_content = automation_states_loader_module.patch_task_content_degraded_state( task_content, @@ -168,6 +203,7 @@ def test_preserves_existing_degraded_since_on_subsequent_patch(self): since=2000.0, ) + _assert_actions_dag_equal(_actions_dag_from_task_content(patched_content), actions_dag_before) degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] assert degraded_state == { "since": 1000.0, diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index 404dc1fed7..f2aab64a5f 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -103,6 +103,61 @@ def _automation_state_dict_with_scheduled_to( return state +def _sample_postpone_dag_actions() -> list[dict[str, typing.Any]]: + return [ + {"id": "action_dsl_1", "dsl_script": "1 if True else 2"}, + { + "id": "action_dsl_2", + "dsl_script": "1 if True else 2", + "dependencies": [{"action_id": "action_dsl_1"}], + }, + ] + + +def _actions_dag_from_task_content(task_content: str) -> dict: + return json.loads(task_content)["state"]["automation"]["actions_dag"] + + +def _actions_dag_from_execute_iteration_result(result: dict) -> dict: + next_description = json.loads(result["next_iteration_description"]) + return next_description["state"]["automation"]["actions_dag"] + + +def _actions_dag_from_enqueued_inputs(enqueued_inputs: dict) -> dict: + return json.loads(enqueued_inputs["task"]["content"])["state"]["automation"]["actions_dag"] + + +def _canonical_actions_dag(actions_dag: dict) -> dict: + automation_state = octobot_flow.entities.AutomationState.from_dict({ + "automation": { + "metadata": {"automation_id": "canonical"}, + "actions_dag": actions_dag, + "execution": {}, + } + }) + return automation_state.to_dict(include_default_values=False)["automation"]["actions_dag"] + + +def _assert_actions_dag_equal(actions_dag_left: dict, actions_dag_right: dict) -> None: + assert _canonical_actions_dag(actions_dag_left) == _canonical_actions_dag(actions_dag_right) + + +def _assert_actions_dag_unchanged(task_content: str, result: dict) -> None: + _assert_actions_dag_equal( + _actions_dag_from_task_content(task_content), + _actions_dag_from_execute_iteration_result(result), + ) + + +def _assert_skip_postpone_preserves_state(task_content: str, result: dict) -> None: + assert result["next_iteration_description"] == task_content + _assert_actions_dag_unchanged(task_content, result) + + +def _assert_trading_postpone_preserves_dag(task_content: str, result: dict) -> None: + _assert_actions_dag_unchanged(task_content, result) + + def _automation_state_with_trade_count(trade_count: int) -> octobot_flow.entities.AutomationState: order_columns = trading_enums.ExchangeConstantsOrderColumns trades = [ @@ -795,7 +850,7 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( expected_error_status, expected_retry_delay_seconds, ): - task_content = json.dumps({"state": _automation_state_dict([])}) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( @@ -832,6 +887,42 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( assert degraded_state["error"] == expected_error_status assert degraded_state["reason"] == str(run_side_effect) assert degraded_state["since"] == fixed_now + _assert_trading_postpone_preserves_dag(task_content, result) + + @pytest.mark.asyncio + @required_imports + async def test_execute_iteration_postponed_error_round_trips_serialization( + self, + import_automation_workflow, + task, + ): + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ): + raw_result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + iteration_result = params.AutomationWorkflowIterationResult.from_dict(raw_result) + assert iteration_result.progress_status.postponed_iteration is True + assert iteration_result.has_next_actions is True + assert iteration_result.progress_status.error == ( + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value + ) + _assert_trading_postpone_preserves_dag(task_content, raw_result) @pytest.mark.asyncio @required_imports @@ -1107,7 +1198,9 @@ async def test_postponed_iteration_skips_persist_and_trim( self, import_automation_workflow, task ): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1128,12 +1221,13 @@ async def test_postponed_iteration_skips_persist_and_trim( octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, "trim_live_trades_in_iteration_state", ) as trim_mock: - await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) persist_mock.assert_not_called() trim_mock.assert_not_called() + _assert_skip_postpone_preserves_state(task_content, result) class TestExecuteIterationPendingPriorityActionsSkippedError: @@ -1143,7 +1237,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( self, import_automation_workflow, task ): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1178,7 +1274,7 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( assert parsed_progress_status.error is None assert parsed_progress_status.error_message is None assert result["has_next_actions"] is True - assert result["next_iteration_description"] == task_content + _assert_skip_postpone_preserves_state(task_content, result) next_iteration_description = json.loads(result["next_iteration_description"]) execution = next_iteration_description["state"]["automation"].get("execution", {}) assert "degraded_state" not in execution @@ -1187,7 +1283,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( @required_imports async def test_logs_pending_priority_skipped_error(self, import_automation_workflow, task): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1218,7 +1316,9 @@ async def test_logs_pending_priority_skipped_error(self, import_automation_workf @required_imports async def test_postponed_log_uses_none_error_fields(self, import_automation_workflow, task): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1257,7 +1357,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( self, import_automation_workflow, task ): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1290,7 +1392,7 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( assert parsed_progress_status.error is None assert parsed_progress_status.error_message is None assert result["has_next_actions"] is True - assert result["next_iteration_description"] == task_content + _assert_skip_postpone_preserves_state(task_content, result) next_iteration_description = json.loads(result["next_iteration_description"]) execution = next_iteration_description["state"]["automation"].get("execution", {}) assert "degraded_state" not in execution @@ -1299,7 +1401,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( @required_imports async def test_logs_outdated_reference_account_info(self, import_automation_workflow, task): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1355,7 +1459,9 @@ async def test_shared_skip_handler_postpones_without_degraded_state( expected_log_message, ): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1384,6 +1490,7 @@ async def test_shared_skip_handler_postpones_without_degraded_state( parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) assert parsed_progress_status.postponed_iteration is True assert parsed_progress_status.error is None + _assert_skip_postpone_preserves_state(task_content, result) class TestShouldRetryOutdatedReferenceAccountError: @@ -1935,6 +2042,25 @@ def test_should_continue_returns_stop_on_error_when_error(self, import_automatio parsed_inputs, progress, False ) is False + def test_should_continue_returns_true_when_postponed_iteration_despite_error( + self, import_automation_workflow, parsed_inputs + ): + progress = params.ProgressStatus( + error=octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + postponed_iteration=True, + should_stop=False, + ) + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_continue_workflow( + parsed_inputs, progress, False + ) is True + workflow_logger.error.assert_not_called() + def test_should_continue_returns_false_when_should_stop(self, import_automation_workflow, parsed_inputs): progress = params.ProgressStatus(error=None, should_stop=True) assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_continue_workflow( @@ -1954,6 +2080,170 @@ def test_should_continue_returns_true_by_no_reason_to_stop(self, import_automati ) is True +class TestExecuteAutomationPostponedFailedRequestIntegration: + def setup_method(self): + octobot_trading.constants.ALLOW_FUNDS_TRANSFER = True + self._no_encrypt_rsa = mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None) + self._no_encrypt_ecdsa = mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None) + self._no_encrypt_rsa.start() + self._no_encrypt_ecdsa.start() + + def teardown_method(self): + octobot_trading.constants.ALLOW_FUNDS_TRANSFER = False + self._no_encrypt_rsa.stop() + self._no_encrypt_ecdsa.stop() + + @pytest.mark.asyncio + @required_imports + async def test_execute_automation_reschedules_after_failed_request_via_dbos_step( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task = octobot_node.models.Task( + name="postponed_failed_request", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + fixed_now = 1000.0 + recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" + real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) + automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + with mock.patch(recv_path, mock.AsyncMock(return_value=None)), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, + "enqueue_async", + enqueue_mock, + ), mock.patch.object( + automation_wf, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + automation_wf.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + assert workflow_result is None + enqueue_mock.assert_called_once() + enqueued_inputs = enqueue_mock.call_args.kwargs["inputs"] + assert enqueued_inputs["execution_time"] == ( + fixed_now + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS + ) + workflows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + assert len(workflows) >= 2 + error_messages = [str(call) for call in workflow_logger.error.call_args_list] + assert not any("unrecoverable iteration error" in message for message in error_messages) + assert not any("Automation stopped (remaining steps:" in message for message in error_messages) + info_messages = [str(call) for call in workflow_logger.info.call_args_list] + assert any("Enqueuing next iteration" in message for message in info_messages) + _assert_actions_dag_equal( + _actions_dag_from_enqueued_inputs(enqueued_inputs), + _actions_dag_from_task_content(task_content), + ) + + @pytest.mark.asyncio + @required_imports + async def test_execute_automation_reschedules_after_postponed_iteration_with_priority_trading_signal( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, run_mock = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + trading_signal = octobot_flow.entities.TradingSignal( + account=protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[], + ), + strategy_id="test-strategy-id", + ) + trading_signal_envelope = _trading_signal_update_envelope( + [trading_signal.to_dict(include_default_values=False)] + ) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task = octobot_node.models.Task( + name="postponed_failed_request_priority_signal", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + fixed_now = 1000.0 + recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" + real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) + automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + with mock.patch( + recv_path, + mock.AsyncMock(side_effect=[None, trading_signal_envelope, None]), + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, + "enqueue_async", + enqueue_mock, + ), mock.patch.object( + automation_wf, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + automation_wf.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + assert workflow_result is None + assert run_mock.await_count == 2 + enqueue_mock.assert_called_once() + error_messages = [str(call) for call in workflow_logger.error.call_args_list] + assert not any("unrecoverable iteration error" in message for message in error_messages) + assert not any("Automation stopped (remaining steps:" in message for message in error_messages) + info_messages = [str(call) for call in workflow_logger.info.call_args_list] + assert any("Enqueuing next iteration" in message for message in info_messages) + enqueued_inputs = enqueue_mock.call_args.kwargs["inputs"] + _assert_actions_dag_equal( + _actions_dag_from_enqueued_inputs(enqueued_inputs), + _actions_dag_from_task_content(task_content), + ) + + class TestGetActionsSummary: def test_get_actions_summary_empty_returns_empty_string(self, import_automation_workflow): assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._get_actions_summary([]) == ""