diff --git a/additional_tests/exchanges_tests/.env.template b/additional_tests/exchanges_tests/.env.template index d501511cbb..c695a7729c 100644 --- a/additional_tests/exchanges_tests/.env.template +++ b/additional_tests/exchanges_tests/.env.template @@ -20,3 +20,8 @@ PHEMEX_SANDBOXED=true POLYMARKET_KEY= POLYMARKET_SECRET= POLYMARKET_PASSWORD= + +COINRABBIT_KEY= +COINRABBIT_SECRET= +COINRABBIT_JWT= # plain JWT for x-user-token +COINRABBIT_X_API_KEY= # plain OctoBot settings API key for x-api-key header diff --git a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py index 22ff5d8ba1..3bc81892cf 100644 --- a/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py +++ b/additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py @@ -37,8 +37,10 @@ import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.connectors.ccxt.constants as ccxt_constants import octobot_trading.exchanges.connectors.ccxt.ccxt_client_util as ccxt_client_util +import octobot_protocol.models as protocol_models import octobot_trading.personal_data as personal_data import octobot_trading.personal_data.orders as personal_data_orders +import octobot_trading.personal_data.transactions.protocol as transactions_protocol import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools import octobot_trading.exchanges.util.exchange_data as exchange_data_import import octobot_tentacles_manager.api as tentacles_manager_api @@ -128,6 +130,8 @@ class AbstractAuthenticatedExchangeTester: # dedicated api and have to be checked by sending an order operation EXPECTED_INVALID_ORDERS_QUANTITY = [] # orders with known invalid quantity exchange order id (usually legacy) IS_READ_ONLY_EXCHANGE = False # set True when the exchange is read only and should not be able to fetch portfolio, create or cancel orders + REQUIRES_CURRENCIES_FOR_TRANSACTIONS = False # True when get_deposits/get_withdrawals need portfolio currencies (e.g. Coinbase) + ALLOW_NO_DEPOSIT_HISTORY = False # True when empty deposit history is valid (e.g. MEXC 7-day window) CHECK_EMPTY_ACCOUNT = False # set True when the account to check has no funds. Warning: does not check order # parse/create/fill/cancel or portfolio & trades parsing IS_BROKER_ENABLED_ACCOUNT = True # set False when this test account can't generate broker fees @@ -776,6 +780,115 @@ async def inner_test_get_my_recent_trades(self): assert trades self.check_raw_trades(trades) + def _exhaust_history_symbol_kwargs(self) -> dict: + if self._get_option_value( + trading_enums.ExchangeClientOptions.MY_TRADES_SYMBOL_FILTER_IS_CLIENT_SIDE + ): + return {"symbol": None} + return {"symbol": self.SYMBOL} + + def _expects_my_trades_pagination(self) -> bool: + if self._get_option_value( + trading_enums.ExchangeClientOptions.MY_TRADES_FETCH_PAGINATION_OFFSET + ): + return True + if self._get_option_value( + trading_enums.ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE + ): + return True + if ( + self._get_option_value( + trading_enums.ExchangeClientOptions.REQUIRE_RECENT_TRADES_FROM_CLOSED_ORDERS + ) + and self._get_option_value( + trading_enums.ExchangeClientOptions.CLOSED_ORDERS_FETCH_USE_CCXT_PAGINATE + ) + ): + return True + return False + + async def test_get_my_recent_trades_exhaust_history(self): + async with self.local_exchange_manager(): + await self.inner_test_get_my_recent_trades_exhaust_history() + + @expect_not_supported_on_read_only("get_my_recent_trades") + async def inner_test_get_my_recent_trades_exhaust_history(self): + if self.CHECK_EMPTY_ACCOUNT or self.EXPECT_EMPTY_RECENT_TRADES: + return + symbol_kwargs = self._exhaust_history_symbol_kwargs() + single = await self.exchange_manager.exchange.get_my_recent_trades( + **symbol_kwargs, exhaust_history=False, + ) + exhaust = await self.exchange_manager.exchange.get_my_recent_trades( + **symbol_kwargs, exhaust_history=True, + ) + if self._expects_my_trades_pagination(): + assert len(exhaust) >= len(single) + if exhaust: + self.check_raw_trades(exhaust) + else: + assert len(exhaust) == len(single) + if exhaust: + self.check_raw_trades(exhaust) + + async def test_get_deposits(self): + async with self.local_exchange_manager(): + await self.inner_test_get_deposits() + + @expect_not_supported_on_read_only("get_deposits") + async def inner_test_get_deposits(self): + if self.CHECK_EMPTY_ACCOUNT: + deposits = await self.get_deposits() + assert deposits == [], f"{deposits} != []" + return + if not self.exchange_manager.exchange.connector.client.has["fetchDeposits"]: + with pytest.raises(trading_errors.NotSupported): + await self.get_deposits() + return + deposits_kwargs = {} + if self.REQUIRES_CURRENCIES_FOR_TRANSACTIONS: + portfolio = await self.get_portfolio() + currencies = self._portfolio_currencies_for_transactions(portfolio) + if not currencies: + return + deposits_kwargs = {"currencies": currencies} + deposits = await self.get_deposits(**deposits_kwargs) + assert isinstance(deposits, list) + if not deposits and self.ALLOW_NO_DEPOSIT_HISTORY: + return + assert deposits + self.check_raw_transactions( + deposits, trading_enums.TransactionType.BLOCKCHAIN_DEPOSIT + ) + + async def test_get_withdrawals(self): + async with self.local_exchange_manager(): + await self.inner_test_get_withdrawals() + + @expect_not_supported_on_read_only("get_withdrawals") + async def inner_test_get_withdrawals(self): + if self.CHECK_EMPTY_ACCOUNT: + withdrawals = await self.get_withdrawals() + assert withdrawals == [], f"{withdrawals} != []" + return + if not self.exchange_manager.exchange.connector.client.has["fetchWithdrawals"]: + with pytest.raises(trading_errors.NotSupported): + await self.get_withdrawals() + return + withdrawals_kwargs = {} + if self.REQUIRES_CURRENCIES_FOR_TRANSACTIONS: + portfolio = await self.get_portfolio() + currencies = self._portfolio_currencies_for_transactions(portfolio) + if not currencies: + return + withdrawals_kwargs = {"currencies": currencies} + withdrawals = await self.get_withdrawals(**withdrawals_kwargs) + assert isinstance(withdrawals, list) + if withdrawals: + self.check_raw_transactions( + withdrawals, trading_enums.TransactionType.BLOCKCHAIN_WITHDRAWAL + ) + async def test_get_closed_orders(self): async with self.local_exchange_manager(): await self.inner_test_get_closed_orders() @@ -1050,6 +1163,20 @@ async def get_my_recent_trades(self, exchange_data=None): exchange_data = exchange_data or self.get_exchange_data() return await exchanges_test_tools.get_trades(self.exchange_manager, exchange_data) + async def get_deposits(self, **kwargs): + return await self.exchange_manager.exchange.get_deposits(**kwargs) + + async def get_withdrawals(self, **kwargs): + return await self.exchange_manager.exchange.get_withdrawals(**kwargs) + + def _portfolio_currencies_for_transactions(self, portfolio, max_count=3): + min_holdings_threshold = decimal.Decimal("1e-8") + funded_currencies = sorted([ + asset for asset, values in portfolio.items() + if values[trading_constants.CONFIG_PORTFOLIO_TOTAL] > min_holdings_threshold + ]) + return funded_currencies[:max_count] + async def get_closed_orders(self, symbol=None): return await self.exchange_manager.exchange.get_closed_orders(symbol or self.SYMBOL) @@ -1205,6 +1332,65 @@ def check_parsed_trade(self, trade: personal_data.Trade): f"is FALSE: Trade: {trade.to_dict()}" ) + def check_duplicate_transactions(self, transactions): + transaction_columns = trading_enums.ExchangeConstantsTransactionColumns + unique_transaction_keys = { + f"{transaction.get(transaction_columns.ID.value)}" + f"{transaction.get(transaction_columns.TXID.value)}" + f"{transaction[transaction_columns.TIMESTAMP.value]}" + f"{transaction[transaction_columns.AMOUNT.value]}" + f"{transaction[transaction_columns.CURRENCY.value]}" + for transaction in transactions + } + assert len(unique_transaction_keys) == len(transactions) + + def check_raw_transactions(self, transactions, expected_type: trading_enums.TransactionType): + self.check_duplicate_transactions(transactions) + for transaction in transactions: + self.check_parsed_transaction(transaction, expected_type) + + def check_parsed_transaction( + self, transaction: dict, expected_type: trading_enums.TransactionType + ): + transaction_columns = trading_enums.ExchangeConstantsTransactionColumns + assert transaction[transaction_columns.TYPE.value] == expected_type.value + currency = transaction[transaction_columns.CURRENCY.value] + assert currency + assert isinstance(currency, str) + amount = transaction[transaction_columns.AMOUNT.value] + assert isinstance(amount, decimal.Decimal) + assert amount > trading_constants.ZERO + timestamp = transaction[transaction_columns.TIMESTAMP.value] + assert timestamp is not None + assert int(timestamp) > 0 + transaction_id = transaction.get(transaction_columns.ID.value) + transaction_txid = transaction.get(transaction_columns.TXID.value) + assert transaction_id or transaction_txid + status = transaction.get(transaction_columns.STATUS.value) + if status is not None: + assert status + assert isinstance(status, str) + protocol_raw = transaction + if not transaction_txid and transaction_id: + protocol_raw = dict(transaction) + protocol_raw[transaction_columns.TXID.value] = transaction_id + protocol_transaction = transactions_protocol.to_protocol_transaction(protocol_raw) + assert protocol_transaction.asset == currency + assert protocol_transaction.amount == float(amount) + expected_protocol_type = { + trading_enums.TransactionType.BLOCKCHAIN_DEPOSIT: ( + protocol_models.TransactionType.BLOCKCHAIN_DEPOSIT + ), + trading_enums.TransactionType.BLOCKCHAIN_WITHDRAWAL: ( + protocol_models.TransactionType.BLOCKCHAIN_WITHDRAWAL + ), + }[expected_type] + assert protocol_transaction.type == expected_protocol_type + if transaction_txid: + assert protocol_transaction.id == transaction_txid + else: + assert protocol_transaction.id == transaction_id + def check_theoretical_cost(self, symbol, quantity, price, cost): theoretical_cost = quantity * price assert theoretical_cost * decimal.Decimal("0.8") <= cost <= theoretical_cost * decimal.Decimal("1.2") diff --git a/additional_tests/exchanges_tests/test_binance.py b/additional_tests/exchanges_tests/test_binance.py index 06c9e9f58f..506227a11b 100644 --- a/additional_tests/exchanges_tests/test_binance.py +++ b/additional_tests/exchanges_tests/test_binance.py @@ -125,6 +125,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_binance_futures.py b/additional_tests/exchanges_tests/test_binance_futures.py index dabb2fab91..7b75e2adf5 100644 --- a/additional_tests/exchanges_tests/test_binance_futures.py +++ b/additional_tests/exchanges_tests/test_binance_futures.py @@ -165,6 +165,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_bingx.py b/additional_tests/exchanges_tests/test_bingx.py index 76762dfc79..58a1eb4abd 100644 --- a/additional_tests/exchanges_tests/test_bingx.py +++ b/additional_tests/exchanges_tests/test_bingx.py @@ -120,6 +120,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_bitget.py b/additional_tests/exchanges_tests/test_bitget.py index 2a876bc3d4..2a58179ce5 100644 --- a/additional_tests/exchanges_tests/test_bitget.py +++ b/additional_tests/exchanges_tests/test_bitget.py @@ -86,6 +86,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_bitmart.py b/additional_tests/exchanges_tests/test_bitmart.py index 0bfd6f223f..d8e074eac1 100644 --- a/additional_tests/exchanges_tests/test_bitmart.py +++ b/additional_tests/exchanges_tests/test_bitmart.py @@ -84,6 +84,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_bybit.py b/additional_tests/exchanges_tests/test_bybit.py index 314f456a87..a587ce70c1 100644 --- a/additional_tests/exchanges_tests/test_bybit.py +++ b/additional_tests/exchanges_tests/test_bybit.py @@ -87,6 +87,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_bybit_futures.py b/additional_tests/exchanges_tests/test_bybit_futures.py index bd623b30d6..10371242be 100644 --- a/additional_tests/exchanges_tests/test_bybit_futures.py +++ b/additional_tests/exchanges_tests/test_bybit_futures.py @@ -100,6 +100,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_coinbase.py b/additional_tests/exchanges_tests/test_coinbase.py index 1f1c807e65..79bda65ca4 100644 --- a/additional_tests/exchanges_tests/test_coinbase.py +++ b/additional_tests/exchanges_tests/test_coinbase.py @@ -33,12 +33,13 @@ class TestCoinbaseAuthenticatedExchange( SETTLEMENT_CURRENCY = "USDC" SYMBOL = f"{ORDER_CURRENCY}/{SETTLEMENT_CURRENCY}" ORDER_SIZE = 70 # % of portfolio to include in test orders - MIN_TRADE_USD_VALUE = decimal.Decimal("0.0004") + MIN_TRADE_USD_VALUE = decimal.Decimal("0.000003") CONVERTS_ORDER_SIZE_BEFORE_PUSHING_TO_EXCHANGES = True VALID_ORDER_ID = "8bb80a81-27f7-4415-aa50-911ea46d841c" USE_ORDER_OPERATION_TO_CHECK_API_KEY_RIGHTS = True # set True when api key rights can't be checked using a EXPECT_MISSING_FEE_IN_CANCELLED_ORDERS = False IS_AUTHENTICATED_REQUEST_CHECK_AVAILABLE = True # set True when is_authenticated_request is implemented + REQUIRES_CURRENCIES_FOR_TRANSACTIONS = True SLEEP_SECONDS_BEFORE_CHECKING_PORTFOLIO = 8 SPECIAL_ORDER_TYPES_BY_EXCHANGE_ID: dict[ @@ -128,6 +129,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_coinex.py b/additional_tests/exchanges_tests/test_coinex.py index bb0b808fbc..e41fb24d30 100644 --- a/additional_tests/exchanges_tests/test_coinex.py +++ b/additional_tests/exchanges_tests/test_coinex.py @@ -85,6 +85,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_coingecko.py b/additional_tests/exchanges_tests/test_coingecko.py index ea1d394f54..fb65a1a2b5 100644 --- a/additional_tests/exchanges_tests/test_coingecko.py +++ b/additional_tests/exchanges_tests/test_coingecko.py @@ -54,6 +54,15 @@ async def test_get_my_recent_trades(self): # ensure not supported for read only exchanges await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): # ensure not supported for read only exchanges await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_cryptocom.py b/additional_tests/exchanges_tests/test_cryptocom.py index 7fed7f917b..08f11acca2 100644 --- a/additional_tests/exchanges_tests/test_cryptocom.py +++ b/additional_tests/exchanges_tests/test_cryptocom.py @@ -83,6 +83,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_gate.py b/additional_tests/exchanges_tests/test_gate.py index d4d437fbdf..e4d47ac28c 100644 --- a/additional_tests/exchanges_tests/test_gate.py +++ b/additional_tests/exchanges_tests/test_gate.py @@ -87,6 +87,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_hollaex.py b/additional_tests/exchanges_tests/test_hollaex.py index 87e16bad47..9d8129febf 100644 --- a/additional_tests/exchanges_tests/test_hollaex.py +++ b/additional_tests/exchanges_tests/test_hollaex.py @@ -90,6 +90,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_htx.py b/additional_tests/exchanges_tests/test_htx.py index 2a25722c16..a6ea48679f 100644 --- a/additional_tests/exchanges_tests/test_htx.py +++ b/additional_tests/exchanges_tests/test_htx.py @@ -85,6 +85,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_hyperliquid.py b/additional_tests/exchanges_tests/test_hyperliquid.py index db9f634797..fedc36c91c 100644 --- a/additional_tests/exchanges_tests/test_hyperliquid.py +++ b/additional_tests/exchanges_tests/test_hyperliquid.py @@ -85,6 +85,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_kraken.py b/additional_tests/exchanges_tests/test_kraken.py index 856945d9cf..e54506e4fa 100644 --- a/additional_tests/exchanges_tests/test_kraken.py +++ b/additional_tests/exchanges_tests/test_kraken.py @@ -14,6 +14,7 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . import pytest +import decimal from additional_tests.exchanges_tests import abstract_authenticated_exchange_tester @@ -34,6 +35,7 @@ class TestKrakenAuthenticatedExchange( EXPECTED_GENERATED_ACCOUNT_ID = True IS_BROKER_ENABLED_ACCOUNT = False # set False when this test account can't generate broker fees EXPECT_MISSING_FEE_IN_CANCELLED_ORDERS = False + MAX_TRADE_USD_VALUE = decimal.Decimal(10000) async def test_get_portfolio(self): @@ -84,6 +86,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_kucoin.py b/additional_tests/exchanges_tests/test_kucoin.py index 84b4488bf5..d34c6f7301 100644 --- a/additional_tests/exchanges_tests/test_kucoin.py +++ b/additional_tests/exchanges_tests/test_kucoin.py @@ -121,6 +121,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_kucoin_futures.py b/additional_tests/exchanges_tests/test_kucoin_futures.py index 24b07821c5..8409889cea 100644 --- a/additional_tests/exchanges_tests/test_kucoin_futures.py +++ b/additional_tests/exchanges_tests/test_kucoin_futures.py @@ -130,6 +130,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_lbank.py b/additional_tests/exchanges_tests/test_lbank.py index afd23bb49e..347ce76940 100644 --- a/additional_tests/exchanges_tests/test_lbank.py +++ b/additional_tests/exchanges_tests/test_lbank.py @@ -98,6 +98,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_mexc.py b/additional_tests/exchanges_tests/test_mexc.py index b3a0e3f8fc..d25622dbc6 100644 --- a/additional_tests/exchanges_tests/test_mexc.py +++ b/additional_tests/exchanges_tests/test_mexc.py @@ -38,6 +38,7 @@ class TestMEXCAuthenticatedExchange( IS_AUTHENTICATED_REQUEST_CHECK_AVAILABLE = True # set True when is_authenticated_request is implemented USE_ORDER_OPERATION_TO_CHECK_API_KEY_RIGHTS = True USED_TO_HAVE_UNTRADABLE_SYMBOL = True + ALLOW_NO_DEPOSIT_HISTORY = True async def test_get_portfolio(self): await super().test_get_portfolio() @@ -88,6 +89,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_okx.py b/additional_tests/exchanges_tests/test_okx.py index 8f9062f03a..1d39d28f82 100644 --- a/additional_tests/exchanges_tests/test_okx.py +++ b/additional_tests/exchanges_tests/test_okx.py @@ -84,6 +84,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_okx_futures.py b/additional_tests/exchanges_tests/test_okx_futures.py index 1d0e2ae3d2..3b58f50c38 100644 --- a/additional_tests/exchanges_tests/test_okx_futures.py +++ b/additional_tests/exchanges_tests/test_okx_futures.py @@ -94,6 +94,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/additional_tests/exchanges_tests/test_phemex.py b/additional_tests/exchanges_tests/test_phemex.py index 61b761a59c..d568d06f3c 100644 --- a/additional_tests/exchanges_tests/test_phemex.py +++ b/additional_tests/exchanges_tests/test_phemex.py @@ -82,6 +82,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): # fails due to each trade being duplicated in privateGetExchangeSpotOrder resp # 23/05/25 diff --git a/additional_tests/exchanges_tests/test_polymarket.py b/additional_tests/exchanges_tests/test_polymarket.py index c5ad48492d..2b3c464b52 100644 --- a/additional_tests/exchanges_tests/test_polymarket.py +++ b/additional_tests/exchanges_tests/test_polymarket.py @@ -101,6 +101,17 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + # untested yet + await super().test_get_deposits() + + async def test_get_withdrawals(self): + # untested yet + await super().test_get_withdrawals() + async def test_get_closed_orders(self): # pass if not implemented pass diff --git a/additional_tests/exchanges_tests/test_weex.py b/additional_tests/exchanges_tests/test_weex.py index fe856946c3..5ed7846c60 100644 --- a/additional_tests/exchanges_tests/test_weex.py +++ b/additional_tests/exchanges_tests/test_weex.py @@ -82,6 +82,15 @@ async def test_create_and_fill_market_orders(self): async def test_get_my_recent_trades(self): await super().test_get_my_recent_trades() + async def test_get_my_recent_trades_exhaust_history(self): + await super().test_get_my_recent_trades_exhaust_history() + + async def test_get_deposits(self): + await super().test_get_deposits() + + async def test_get_withdrawals(self): + await super().test_get_withdrawals() + async def test_get_closed_orders(self): await super().test_get_closed_orders() diff --git a/octobot/backtesting/minimal_data_importer.py b/octobot/backtesting/minimal_data_importer.py index 2811ef1a7e..af900dcddd 100644 --- a/octobot/backtesting/minimal_data_importer.py +++ b/octobot/backtesting/minimal_data_importer.py @@ -1,9 +1,9 @@ import octobot_commons.enums -import octobot_commons.databases as databases import octobot_commons.constants import octobot_commons.symbols import octobot_commons.time_frame_manager as time_frame_manager +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.importers import octobot_backtesting.enums @@ -60,7 +60,7 @@ async def get_data_timestamp_interval(self, time_frame=None): async def get_ohlcv(self, exchange_name=None, symbol=None, time_frame=octobot_commons.enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return await self._get_from_db( @@ -74,7 +74,7 @@ async def get_ohlcv(self, exchange_name=None, symbol=None, async def _get_from_db( self, exchange_name, symbol, table, time_frame=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -88,7 +88,7 @@ def _select_from_timestamp(self, symbol, timestamps, operations, time_frame): parsed_timestamps = [float(timestamp) for timestamp in timestamps] return [ candle - for candle in self._select(databases.SQLiteDatabase.DEFAULT_SIZE, symbol, time_frame) + for candle in self._select(backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, symbol, time_frame) if _is_valid_from_operations( candle[octobot_commons.enums.PriceIndexes.IND_PRICE_TIME.value], parsed_timestamps, operations ) @@ -110,7 +110,7 @@ def _select(self, limit, symbol, time_frame): ] for candle in candles ] - if limit != databases.SQLiteDatabase.DEFAULT_SIZE: + if limit != backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE: return candles_data[:limit] return candles_data diff --git a/octobot/community/authentication.py b/octobot/community/authentication.py index add3efcce5..4964ab2ef3 100644 --- a/octobot/community/authentication.py +++ b/octobot/community/authentication.py @@ -114,6 +114,7 @@ class CommunityAuthentication(authentication.Authenticator): def __init__(self, config=None, backend_url=None, backend_key=None, use_as_singleton=True): super().__init__(use_as_singleton=use_as_singleton) + self._use_as_singleton = use_as_singleton self.config: typing.Optional[commons_configuration.Configuration] = config self.backend_url: str = backend_url or identifiers_provider.IdentifiersProvider.BACKEND_URL self.backend_key: str = backend_key or identifiers_provider.IdentifiersProvider.BACKEND_KEY @@ -600,7 +601,8 @@ def clear_local_data_if_necessary(self): self._clear_bot_scoped_config() async def stop(self): - self.logger.debug("Stopping ...") + if self._use_as_singleton: + self.logger.debug("Stopping ...") if self._fetch_account_task is not None and not self._fetch_account_task.done(): self._fetch_account_task.cancel() await self.supabase_client.aclose() @@ -615,7 +617,8 @@ async def stop(self): await session.content_client.close() await session.account_client.close() self._dk_sessions.clear() - self.logger.debug("Stopped") + if self._use_as_singleton: + self.logger.debug("Stopped") def _update_supports(self, resp_status, json_data): if resp_status == 200: diff --git a/packages/backtesting/octobot_backtesting/api/importer.py b/packages/backtesting/octobot_backtesting/api/importer.py index 2df98a4dde..1660fc1e5e 100644 --- a/packages/backtesting/octobot_backtesting/api/importer.py +++ b/packages/backtesting/octobot_backtesting/api/importer.py @@ -18,7 +18,8 @@ import octobot_backtesting.util as util import octobot_commons.errors as commons_errors -import octobot_commons.databases as databases + +import octobot_backtesting.databases as backtesting_databases async def create_importer(config, backtesting_file, default_importer=None): @@ -63,7 +64,7 @@ async def get_all_ohlcvs(database_path, exchange_name, symbol, time_frame, inferior_timestamp=-1, superior_timestamp=-1) -> list: timestamps, operations = importers.get_operations_from_timestamps(superior_timestamp, inferior_timestamp) try: - async with databases.new_sqlite_database(database_path) as database: + async with backtesting_databases.new_sqlite_database(database_path) as database: candles = await database.select_from_timestamp(backtesting_enums.ExchangeDataTables.OHLCV, exchange_name=exchange_name, symbol=symbol, time_frame=time_frame.value, diff --git a/packages/backtesting/octobot_backtesting/collectors/data_collector.py b/packages/backtesting/octobot_backtesting/collectors/data_collector.py index a6900fc2e9..f73dccb2ab 100644 --- a/packages/backtesting/octobot_backtesting/collectors/data_collector.py +++ b/packages/backtesting/octobot_backtesting/collectors/data_collector.py @@ -25,6 +25,8 @@ import octobot_commons.logging as logging import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.enums as enums import octobot_backtesting.constants as constants import octobot_backtesting.data as data @@ -83,7 +85,9 @@ def set_file_path(self) -> None: def create_database(self) -> None: if not self.database: - self.database = databases.SQLiteDatabase(self.temp_file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( + self.temp_file_path, read_only=False + ) def finalize_database(self): os.rename(self.temp_file_path, self.file_path) diff --git a/packages/backtesting/octobot_backtesting/data/data_file_manager.py b/packages/backtesting/octobot_backtesting/data/data_file_manager.py index 082b97493a..c28a17148e 100644 --- a/packages/backtesting/octobot_backtesting/data/data_file_manager.py +++ b/packages/backtesting/octobot_backtesting/data/data_file_manager.py @@ -24,6 +24,8 @@ import octobot_commons.time_frame_manager as tmf_manager import octobot_commons.errors as commons_errors +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.constants as constants import octobot_backtesting.enums as enums @@ -140,7 +142,7 @@ def _parse_list(value): async def get_file_description(database_file): database = None try: - database = databases.SQLiteDatabase(database_file) + database = backtesting_databases.BacktestingDataSQLiteDatabase(database_file) await database.initialize() description = await get_database_description(database) except (commons_errors.DatabaseNotFoundError, TypeError): diff --git a/packages/backtesting/octobot_backtesting/databases/__init__.py b/packages/backtesting/octobot_backtesting/databases/__init__.py new file mode 100644 index 0000000000..5afd071b7a --- /dev/null +++ b/packages/backtesting/octobot_backtesting/databases/__init__.py @@ -0,0 +1,25 @@ +# Drakkar-Software OctoBot-Backtesting +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +from octobot_backtesting.databases.backtesting_data_sqlite_database import ( + BacktestingDataSQLiteDatabase, + new_sqlite_database, +) + +__all__ = [ + "BacktestingDataSQLiteDatabase", + "new_sqlite_database", +] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py similarity index 81% rename from packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py rename to packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py index f066002f6d..9ecc620666 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/sqlite/sqlite_database.py +++ b/packages/backtesting/octobot_backtesting/databases/backtesting_data_sqlite_database.py @@ -17,27 +17,18 @@ import contextlib import sqlite3 -import octobot_commons.logging as logging import octobot_commons.enums as enums import octobot_commons.errors as errors -import octobot_commons.databases.relational_databases.sqlite.cursor_pool as cursor_pool -import octobot_commons.constants as constants +import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database -try: - import aiosqlite -except ImportError: - if constants.USE_MINIMAL_LIBS: - # mock aiosqlite imports - class AiosqliteImportMock: - def connect(self, *args): - raise ImportError("aiosqlite not installed") - aiosqlite = AiosqliteImportMock() - else: - raise +class BacktestingDataSQLiteDatabase(base_sqlite_database.BaseSQLiteDatabase): + """SQLite store for backtesting .data files. - -class SQLiteDatabase: + Read-only by default: data importers only read committed fixtures and must not + checkpoint WAL into them. Data collectors and converters that write OHLCV / + description rows must pass read_only=False. + """ TIMESTAMP_COLUMN = "timestamp" DEFAULT_ORDER_BY = TIMESTAMP_COLUMN DEFAULT_SORT = enums.DataBaseOrderBy.DESC.value @@ -45,41 +36,20 @@ class SQLiteDatabase: DEFAULT_SIZE = -1 CACHE_SIZE = 50 - def __init__(self, file_name): - self.file_name = file_name - self.logger = logging.get_logger(self.__class__.__name__) - + def __init__(self, file_name, read_only: bool = True): + super().__init__(file_name, read_only=read_only) self.tables = [] self.cache = {} - self.connection = None - - # should never be used directly, use async with self.aio_cursor() as cursor: instead - self._cursor_pool = None - async def initialize(self): - try: - self.connection = await aiosqlite.connect(self.file_name) - self._cursor_pool = cursor_pool.CursorPool(self.connection) - await self.__init_tables_list() - except (sqlite3.OperationalError, sqlite3.DatabaseError) as err: - raise errors.DatabaseNotFoundError(f"{err} (file: {self.file_name})") + await super().initialize() + await self.__init_tables_list() async def create_index(self, table, columns): await self.__execute_index_creation( table, "_".join(columns), ", ".join(columns) ) - @contextlib.asynccontextmanager - async def aio_cursor(self) -> sqlite3.Cursor: - """ - Use this as a context manager to get a free database cursor - :yield: A free cursor - :return: None - """ - async with self._cursor_pool.idle_cursor() as cursor: - yield cursor.cursor - async def __execute_index_creation(self, table, name, columns): async with self.aio_cursor() as cursor: await cursor.execute( @@ -90,7 +60,6 @@ async def insert(self, table, timestamp, **kwargs): if table.value not in self.tables: await self.__create_table(table, **kwargs) - # Insert a row of data inserting_values = [f"'{value}'" for value in kwargs.values()] await self.__execute_insert( table, self.__insert_values(timestamp, ", ".join(inserting_values)) @@ -104,7 +73,6 @@ async def insert_all(self, table, timestamp, **kwargs): insert_values = [] for index, values in enumerate(timestamp): - # Insert a row of data inserting_values = [ f"'{value if not isinstance(value, list) else value[index]}'" for value in kwargs.values() @@ -116,7 +84,6 @@ async def insert_all(self, table, timestamp, **kwargs): await self.__execute_insert(table, ", ".join(insert_values)) async def update(self, table, updated_value_by_column, **kwargs): - # Update a row of data updating_values = [ f"{key} = '{value}'" for key, value in updated_value_by_column.items() ] @@ -133,7 +100,6 @@ async def __execute_insert(self, table, insert_items) -> None: async with self.aio_cursor() as cursor: await cursor.execute(f"INSERT INTO {table.value} VALUES {insert_items}") - # Save (commit) the changes await self.connection.commit() async def __execute_update(self, table, update_items, where_clauses) -> None: @@ -142,7 +108,6 @@ async def __execute_update(self, table, update_items, where_clauses) -> None: f"UPDATE {table.value} SET {update_items} WHERE {where_clauses}" ) - # Save (commit) the changes await self.connection.commit() async def select( @@ -249,13 +214,13 @@ def __where_clauses_from_operations( return " AND ".join( [ self.__where_clauses_from_operation( - keys[i], - values[i], - operations[i] if len(operations) > i else None, + keys[index], + values[index], + operations[index] if len(operations) > index else None, should_quote_value=should_quote_value, ) - for i in range(len(keys)) - if values[i] is not None + for index in range(len(keys)) + if values[index] is not None ] ) @@ -311,7 +276,6 @@ async def __execute_select( async def __execute_delete(self, table, where_clauses): async with self.aio_cursor() as cursor: await cursor.execute(f"DELETE FROM {table.value} WHERE {where_clauses} ") - # nothing to return, will raise on error async def check_table_exists(self, table) -> bool: async with self.aio_cursor() as cursor: @@ -340,10 +304,10 @@ async def __create_table( if with_index_on_timestamp: await self.create_index(table, [self.TIMESTAMP_COLUMN]) - for i in range(1, round(len(columns) / 2) + 1): + for index in range(1, round(len(columns) / 2) + 1): await self.create_index( table, - [self.TIMESTAMP_COLUMN] + [columns[u] for u in range(0, i)], + [self.TIMESTAMP_COLUMN] + [columns[column_index] for column_index in range(0, index)], ) except sqlite3.OperationalError: @@ -356,20 +320,10 @@ async def __init_tables_list(self): await cursor.execute("SELECT name FROM sqlite_master WHERE type='table'") self.tables = [res[0] for res in await cursor.fetchall()] - async def stop(self): - try: - if self._cursor_pool is not None: - await self._cursor_pool.close() - finally: - if self.connection is not None: - conn = self.connection - self.connection = None - await conn.close() - @contextlib.asynccontextmanager -async def new_sqlite_database(file_path): - local_database = SQLiteDatabase(file_path) +async def new_sqlite_database(file_path, read_only: bool = True): + local_database = BacktestingDataSQLiteDatabase(file_path, read_only=read_only) try: await local_database.initialize() yield local_database diff --git a/packages/backtesting/octobot_backtesting/importers/data_importer.py b/packages/backtesting/octobot_backtesting/importers/data_importer.py index 83769be886..2be5a971f8 100644 --- a/packages/backtesting/octobot_backtesting/importers/data_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/data_importer.py @@ -18,6 +18,8 @@ import octobot_commons.logging as logging import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases + import octobot_backtesting.constants as constants import octobot_backtesting.errors as errors @@ -57,7 +59,9 @@ def provides_accurate_price_time_frame(self) -> bool: def load_database(self) -> None: file_path = self.adapt_file_path_if_necessary() if not self.database: - self.database = databases.SQLiteDatabase(file_path) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( + file_path, read_only=True + ) def adapt_file_path_if_necessary(self): if path.isfile(self.file_path): diff --git a/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py b/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py index 52a4cf5e6f..136152ec1d 100644 --- a/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/exchanges/exchange_importer.py @@ -18,6 +18,7 @@ import octobot_commons.errors as common_errors import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.data as data import octobot_backtesting.enums as enums import octobot_backtesting.errors as errors @@ -68,12 +69,12 @@ async def get_data_timestamp_interval(self, time_frame=None): if table in self.available_data_types: try: min_timestamp = (await self.database.select_min(table, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if not minimum_timestamp or minimum_timestamp > min_timestamp: minimum_timestamp = min_timestamp max_timestamp = (await self.database.select_max(table, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if not maximum_timestamp or maximum_timestamp < max_timestamp: maximum_timestamp = max_timestamp except (IndexError, common_errors.DatabaseNotFoundError): @@ -83,7 +84,7 @@ async def get_data_timestamp_interval(self, time_frame=None): try: ohlcv_kwargs = {"time_frame": time_frame} if time_frame else {} ohlcv_min_timestamps = (await self.database.select_min(enums.ExchangeDataTables.OHLCV, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN], + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN], [common_constants.CONFIG_TIME_FRAME], group_by=common_constants.CONFIG_TIME_FRAME, **ohlcv_kwargs @@ -93,7 +94,7 @@ async def get_data_timestamp_interval(self, time_frame=None): # if the required time frame is not included in this database, ohlcv_min_timestamps is empty: ignore it min_ohlcv_timestamp = max(ohlcv_min_timestamps)[0] max_ohlcv_timestamp = (await self.database.select_max(enums.ExchangeDataTables.OHLCV, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN], + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN], **ohlcv_kwargs))[0][0] elif time_frame: raise errors.MissingTimeFrame(f"Missing time frame in data file: {time_frame}") @@ -113,7 +114,7 @@ async def _init_available_data_types(self): async def _get_from_db( self, exchange_name, symbol, table, time_frame=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -135,7 +136,7 @@ async def _get_from_db( async def get_ohlcv(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_ohlcvs(await self._get_from_db( @@ -148,7 +149,7 @@ async def get_ohlcv(self, exchange_name=None, symbol=None, async def get_ohlcv_from_timestamps(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1) -> list: """ Reads OHLCV history from database and populates a local ChronologicalReadDatabaseCache. @@ -158,7 +159,7 @@ async def get_ohlcv_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_ohlcv, limit) async def get_ticker(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_tickers(await self._get_from_db( @@ -169,7 +170,7 @@ async def get_ticker(self, exchange_name=None, symbol=None, )) async def get_ticker_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads ticker history from database and populates a local ChronologicalReadDatabaseCache. @@ -179,7 +180,7 @@ async def get_ticker_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_ticker, limit) async def get_order_book(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_order_books(await self._get_from_db( @@ -190,7 +191,7 @@ async def get_order_book(self, exchange_name=None, symbol=None, )) async def get_order_book_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads order book history from database and populates a local ChronologicalReadDatabaseCache. @@ -200,7 +201,7 @@ async def get_order_book_from_timestamps(self, exchange_name=None, symbol=None, inferior_timestamp, superior_timestamp, self.get_order_book, limit) async def get_recent_trades(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_recent_trades(await self._get_from_db( @@ -211,7 +212,7 @@ async def get_recent_trades(self, exchange_name=None, symbol=None, )) async def get_recent_trades_from_timestamps(self, exchange_name=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads recent trades history from database and populates a local ChronologicalReadDatabaseCache. @@ -222,7 +223,7 @@ async def get_recent_trades_from_timestamps(self, exchange_name=None, symbol=Non async def get_kline(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): return importers.import_klines(await self._get_from_db( @@ -235,7 +236,7 @@ async def get_kline(self, exchange_name=None, symbol=None, async def get_kline_from_timestamps(self, exchange_name=None, symbol=None, time_frame=common_enums.TimeFrames.ONE_HOUR, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads kline history from database and populates a local ChronologicalReadDatabaseCache. @@ -256,9 +257,9 @@ async def _get_from_cache(self, exchange_name, symbol, time_frame, data_type, # initializer without time_frame args are not expecting the time_frame argument, remove it # ignore the limit param as it might reduce the available cache and give false later select results init_cache_method_args = \ - (exchange_name, symbol, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations) \ + (exchange_name, symbol, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations) \ if time_frame is None \ - else (exchange_name, symbol, time_frame, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations) + else (exchange_name, symbol, time_frame, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations) self.chronological_cache.set( await set_cache_method(*init_cache_method_args), 0, diff --git a/packages/backtesting/octobot_backtesting/importers/social/social_importer.py b/packages/backtesting/octobot_backtesting/importers/social/social_importer.py index c57c0da680..fd6c3fd517 100644 --- a/packages/backtesting/octobot_backtesting/importers/social/social_importer.py +++ b/packages/backtesting/octobot_backtesting/importers/social/social_importer.py @@ -19,6 +19,7 @@ import octobot_commons.errors as common_errors import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.constants as constants import octobot_backtesting.data as data import octobot_backtesting.enums as enums @@ -103,9 +104,9 @@ async def get_data_timestamp_interval(self, time_frame=None): if enums.SocialDataTables.SOCIAL_EVENTS in self.available_data_types: try: min_timestamp = (await self.database.select_min(enums.SocialDataTables.SOCIAL_EVENTS, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] max_timestamp = (await self.database.select_max(enums.SocialDataTables.SOCIAL_EVENTS, - [databases.SQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] + [backtesting_databases.BacktestingDataSQLiteDatabase.TIMESTAMP_COLUMN]))[0][0] if min_timestamp and max_timestamp: minimum_timestamp = min_timestamp maximum_timestamp = max_timestamp @@ -123,7 +124,7 @@ async def _get_from_db( self, service_name, table, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None ): @@ -146,7 +147,7 @@ async def _get_from_db( ) async def get_social_events(self, service_name=None, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps=None, operations=None): """ @@ -181,7 +182,7 @@ async def get_social_events(self, service_name=None, channel=None, symbol=None, return result async def get_social_events_from_timestamps(self, service_name=None, channel=None, symbol=None, - limit=databases.SQLiteDatabase.DEFAULT_SIZE, + limit=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, inferior_timestamp=-1, superior_timestamp=-1): """ Reads social events history from database and populates a local ChronologicalReadDatabaseCache. @@ -203,7 +204,7 @@ async def _get_from_cache(self, service_name, channel, symbol, data_type, # initializer without time_frame args are not expecting the time_frame argument, remove it # ignore the limit param as it might reduce the available cache and give false later select results init_cache_method_args = ( - service_name, channel, symbol, databases.SQLiteDatabase.DEFAULT_SIZE, timestamps, operations + service_name, channel, symbol, backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, timestamps, operations ) self.chronological_cache.set( await set_cache_method(*init_cache_method_args), diff --git a/packages/backtesting/tests/database_test_util.py b/packages/backtesting/tests/database_test_util.py new file mode 100644 index 0000000000..68d4c158ec --- /dev/null +++ b/packages/backtesting/tests/database_test_util.py @@ -0,0 +1,7 @@ +import os + +BACKTESTING_STATIC_DIR = os.path.join(os.path.dirname(__file__), "static") + + +def static_database_fixture_path(file_name: str) -> str: + return os.path.join(BACKTESTING_STATIC_DIR, file_name) diff --git a/packages/backtesting/tests/databases/__init__.py b/packages/backtesting/tests/databases/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py similarity index 63% rename from packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py rename to packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py index 394d01e904..3e2f30faeb 100644 --- a/packages/commons/tests/databases/relational_databases/sqlite/test_sqlite_database.py +++ b/packages/backtesting/tests/databases/test_backtesting_data_sqlite_database.py @@ -1,18 +1,3 @@ -# Drakkar-Software OctoBot -# Copyright (c) Drakkar-Software, All rights reserved. -# -# This library is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 3.0 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. import mock import pytest import os @@ -21,60 +6,73 @@ import contextlib import tempfile - import octobot_commons.asyncio_tools as asyncio_tools import octobot_commons.errors as errors -import octobot_commons.databases as databases import octobot_commons.enums as enums +import octobot_backtesting.databases as backtesting_databases + +import tests.database_test_util as database_test_util -# All test coroutines will be treated as marked. pytestmark = pytest.mark.asyncio DATA_FILE1 = "ExchangeHistoryDataCollector_1589740606.4862757.data" DATA_FILE2 = "second_ExchangeHistoryDataCollector_1589740606.4862757.data" +STATIC_FIXTURE_PATHS = { + DATA_FILE1: database_test_util.static_database_fixture_path(DATA_FILE1), + DATA_FILE2: database_test_util.static_database_fixture_path(DATA_FILE2), +} OHLCV = mock.Mock(value="ohlcv") KLINE = mock.Mock(value="kline") -# use context manager instead of fixture to prevent pytest threads issues +@pytest.fixture(scope="module", autouse=True) +def _static_fixtures_unchanged(): + mtimes_before = { + file_name: os.path.getmtime(fixture_path) + for file_name, fixture_path in STATIC_FIXTURE_PATHS.items() + } + yield + for file_name, fixture_path in STATIC_FIXTURE_PATHS.items(): + assert os.path.getmtime(fixture_path) == mtimes_before[file_name] + + @contextlib.asynccontextmanager async def get_database(data_file=DATA_FILE1): - async with databases.new_sqlite_database(os.path.join("tests", "static", data_file)) as db: - yield db - # prevent "generator didn't stop after athrow(), see https://github.com/python-trio/trio/issues/2081" + fixture_path = STATIC_FIXTURE_PATHS[data_file] + async with backtesting_databases.new_sqlite_database(fixture_path) as database: + yield database await asyncio_tools.wait_asyncio_next_cycle() -# use context manager instead of fixture to prevent pytest threads issues @contextlib.asynccontextmanager async def get_temp_empty_database(): - database_name = "temp_empty_database" + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + database_name = temp_file.name try: - async with databases.new_sqlite_database(database_name) as db: - yield db + async with backtesting_databases.new_sqlite_database(database_name, read_only=False) as database: + yield database finally: - # prevent "generator didn't stop after athrow(), see https://github.com/python-trio/trio/issues/2081" await asyncio_tools.wait_asyncio_next_cycle() - os.remove(database_name) + if os.path.isfile(database_name): + os.remove(database_name) async def test_invalid_file(): with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: file_name = temp_file.name - db = databases.SQLiteDatabase(file_name) + database = backtesting_databases.BacktestingDataSQLiteDatabase(file_name, read_only=False) try: - await db.initialize() - assert not await db.check_table_exists(KLINE) + await database.initialize() + assert not await database.check_table_exists(KLINE) with pytest.raises(sqlite3.OperationalError): - await db.check_table_not_empty(KLINE) + await database.check_table_not_empty(KLINE) finally: - await db.stop() + await database.stop() os.remove(file_name) async def test_select(): async with get_database() as database: - # default values with pytest.raises(errors.DatabaseNotFoundError): await database.select(KLINE) @@ -128,24 +126,23 @@ async def test_select_from_timestamp(): assert all(candle[0] <= 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587960000"], - operations) + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587960000"], operations + ) assert len(candles) > 0 assert all(candle[0] == 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587945600"], - operations) + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587945600"], operations + ) assert len(candles) == 15 assert all(1587945600 <= candle[0] <= 1587960000 for candle in candles) operations = [enums.DataBaseOperations.INF_EQUALS.value, enums.DataBaseOperations.SUP_EQUALS.value] - candles = await database.select_from_timestamp(OHLCV, - ["1587960000", "1587945600"], - operations, - symbol="xyz") + candles = await database.select_from_timestamp( + OHLCV, ["1587960000", "1587945600"], operations, symbol="xyz" + ) assert len(candles) == 0 @@ -153,8 +150,8 @@ async def test_gather_concurrent_select(): async with get_database() as database: timestamps_1h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] timestamps_4h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h")] - coros = [_check_select_result(database, ts, "1h") for ts in timestamps_1h] - coros += [_check_select_result(database, ts, "4h") for ts in timestamps_4h] + coros = [_check_select_result(database, timestamp, "1h") for timestamp in timestamps_1h] + coros += [_check_select_result(database, timestamp, "4h") for timestamp in timestamps_4h] await asyncio.gather(*coros) @@ -162,8 +159,9 @@ async def test_create_tasks_concurrent_selects(): async with get_database() as database: timestamps_1h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] timestamps_1m = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1m")] - timestamps_4h = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h", - size=50)] + timestamps_4h = [ + ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="4h", size=50) + ] calls_count = len(timestamps_1h) + len(timestamps_4h) + len(timestamps_1m) failed_calls = [] @@ -173,17 +171,16 @@ async def select_task(db, timestamp, time_frame): try: await _check_select_result(db, timestamp, time_frame) success_calls.append((timestamp, time_frame)) - except Exception as e: - failed_calls.append((timestamp, time_frame, e)) + except Exception as error: + failed_calls.append((timestamp, time_frame, error)) tasks = [] - for ts in timestamps_1h: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "1h"))) - for ts in timestamps_4h: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "4h"))) - for ts in timestamps_1m: - tasks.append(asyncio.get_event_loop().create_task(select_task(database, ts, "1m"))) - # for wait for next cycle to make previous requests end and re-use previous cursors + for timestamp in timestamps_1h: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "1h"))) + for timestamp in timestamps_4h: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "4h"))) + for timestamp in timestamps_1m: + tasks.append(asyncio.get_event_loop().create_task(select_task(database, timestamp, "1m"))) await asyncio_tools.wait_asyncio_next_cycle() await asyncio.gather(*tasks) @@ -196,9 +193,11 @@ async def test_stop_while_concurrent_select(): timestamps = [ohlcv[0] for ohlcv in await database.select(OHLCV, time_frame="1h")] await _check_select_result(database, timestamps[0]) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps]))) - # not enough time to finish all requests, most if not all will remaining pending + asyncio.gather(*[ + _check_select_result(database, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps + ]) + )) await asyncio_tools.wait_asyncio_next_cycle() @@ -206,8 +205,8 @@ async def test_double_database(): async with get_database() as database1, get_database(DATA_FILE2) as database2: timestamps1 = [ohlcv[0] for ohlcv in await database1.select(OHLCV, time_frame="1h")] timestamps2 = [ohlcv[0] for ohlcv in await database2.select(OHLCV, time_frame="1h")] - await asyncio.gather(*[_check_select_result(database1, ts) for ts in timestamps1]) - await asyncio.gather(*[_check_select_result(database2, ts) for ts in timestamps2]) + await asyncio.gather(*[_check_select_result(database1, timestamp) for timestamp in timestamps1]) + await asyncio.gather(*[_check_select_result(database2, timestamp) for timestamp in timestamps2]) async def test_double_database_stop_while_concurrent_select(): @@ -217,12 +216,17 @@ async def test_double_database_stop_while_concurrent_select(): await _check_select_result(database1, timestamps1[0]) await _check_select_result(database2, timestamps2[0]) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database1, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps1]))) + asyncio.gather(*[ + _check_select_result(database1, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps1 + ]) + )) asyncio.create_task(asyncio.wait( - asyncio.gather(*[_check_select_result(database2, ts, expected_exception=sqlite3.ProgrammingError) - for ts in timestamps2]))) - # not enough time to finish all requests, most if not all will remaining pending + asyncio.gather(*[ + _check_select_result(database2, timestamp, expected_exception=sqlite3.ProgrammingError) + for timestamp in timestamps2 + ]) + )) await asyncio_tools.wait_asyncio_next_cycle() @@ -234,33 +238,38 @@ async def test_insert(): async def test_insert_all(): async with get_temp_empty_database() as temp_empty_database: - await temp_empty_database.insert_all(OHLCV, - symbol=["xyz", "abc"], - timestamp=[1, 2], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["xyz", "abc"], + timestamp=[1, 2], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] assert await temp_empty_database.select(OHLCV, date="05") == [(2, 'abc', '10', '05')] async def test_delete(): async with get_temp_empty_database() as temp_empty_database: - await temp_empty_database.insert_all(OHLCV, - symbol=["xyz", "abc"], - timestamp=[1, 2], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["xyz", "abc"], + timestamp=[1, 2], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] - # no matching row to delete await temp_empty_database.delete(OHLCV, symbol="plop") assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05'), (1, 'xyz', '1', '01')] await temp_empty_database.delete(OHLCV, symbol="xyz") assert await temp_empty_database.select(OHLCV) == [(2, 'abc', '10', '05')] - await temp_empty_database.insert_all(OHLCV, - symbol=["hoho", "dd"], - timestamp=[11, 11], - price=[1, 10], - date=["01", "05"]) + await temp_empty_database.insert_all( + OHLCV, + symbol=["hoho", "dd"], + timestamp=[11, 11], + price=[1, 10], + date=["01", "05"], + ) assert await temp_empty_database.select(OHLCV) == [ (11, 'dd', '10', '05'), (11, 'hoho', '1', '01'), (2, 'abc', '10', '05') ] @@ -273,7 +282,6 @@ async def test_delete(): async def test_create_index(): async with get_temp_empty_database() as temp_empty_database: await temp_empty_database.insert(OHLCV, 1, symbol="xyz", price="1", date="01") - # ensure no exception await temp_empty_database.create_index(OHLCV, ["symbol", "timestamp"]) assert await temp_empty_database.select(OHLCV) == [(1, 'xyz', '1', '01')] @@ -283,8 +291,8 @@ async def _check_select_result(database, timestamp, time_frame="1h", expected_ex ohlcv = await database.select(OHLCV, time_frame=time_frame, timestamp=str(timestamp)) assert len(ohlcv) == 1 assert ohlcv[0][0] == timestamp - except Exception as e: - if e.__class__ is expected_exception: + except Exception as error: + if error.__class__ is expected_exception: pass else: raise diff --git a/packages/backtesting/tests/importers/test_exchange_importer.py b/packages/backtesting/tests/importers/test_exchange_importer.py index 0736c3b6fb..d4e8b90a42 100644 --- a/packages/backtesting/tests/importers/test_exchange_importer.py +++ b/packages/backtesting/tests/importers/test_exchange_importer.py @@ -13,8 +13,9 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import pytest import os + +import pytest from contextlib import asynccontextmanager @@ -23,15 +24,26 @@ from octobot_backtesting.enums import ExchangeDataTables from octobot_commons.enums import TimeFrames +import tests.database_test_util as database_test_util + # All test coroutines will be treated as marked. pytestmark = pytest.mark.asyncio +EXCHANGE_HISTORY_DATA_FILE = "ExchangeHistoryDataCollector_1589740606.4862757.data" +STATIC_FIXTURE_PATH = database_test_util.static_database_fixture_path(EXCHANGE_HISTORY_DATA_FILE) + + +@pytest.fixture(scope="module", autouse=True) +def _static_fixture_unchanged(): + mtime_before = os.path.getmtime(STATIC_FIXTURE_PATH) + yield + assert os.path.getmtime(STATIC_FIXTURE_PATH) == mtime_before + # use context manager instead of fixture to prevent pytest threads issues @asynccontextmanager async def get_importer(): - database_file = os.path.join("tests", "static", "ExchangeHistoryDataCollector_1589740606.4862757.data") - importer = ExchangeDataImporter({}, database_file) + importer = ExchangeDataImporter({}, STATIC_FIXTURE_PATH) try: await importer.initialize() yield importer diff --git a/packages/commons/octobot_commons/configuration/configuration.py b/packages/commons/octobot_commons/configuration/configuration.py index 405ee81065..bf42dce383 100644 --- a/packages/commons/octobot_commons/configuration/configuration.py +++ b/packages/commons/octobot_commons/configuration/configuration.py @@ -109,15 +109,16 @@ def load_profiles_if_possible_and_necessary(self) -> None: self.load_profiles() self.select_profile(self._get_selected_profile()) - def select_profile(self, profile_id) -> None: + def select_profile(self, profile_id, *, init_tentacles_setup: bool = True) -> None: """ Sets self.profile using its profile_id :param profile_id: id of the profile to select + :param init_tentacles_setup: when False, skip tentacles setup initialization on activation :return: None """ self.config[commons_constants.CONFIG_PROFILE] = profile_id self.profile = self.profile_by_id[profile_id] - self.profile_storage.activate_profile(self.profile) + self.profile_storage.activate_profile(self.profile, init_tentacles_setup=init_tentacles_setup) self.logger.info(f"Using {self.profile.name} profile.") self._generate_config_from_user_config_and_profile() diff --git a/packages/commons/octobot_commons/databases/__init__.py b/packages/commons/octobot_commons/databases/__init__.py index 22b0191ace..5d48d0b39b 100644 --- a/packages/commons/octobot_commons/databases/__init__.py +++ b/packages/commons/octobot_commons/databases/__init__.py @@ -57,8 +57,7 @@ ) from octobot_commons.databases.relational_databases import ( - SQLiteDatabase, - new_sqlite_database, + BaseSQLiteDatabase, ) from octobot_commons.databases.run_databases import ( @@ -98,8 +97,7 @@ "DBWriterReader", "CacheDatabase", "CacheTimestampDatabase", - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", "RunDatabasesIdentifier", "RunDatabasesProvider", "init_bot_storage", diff --git a/packages/commons/octobot_commons/databases/relational_databases/__init__.py b/packages/commons/octobot_commons/databases/relational_databases/__init__.py index 315bc6be37..7d7c366402 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/__init__.py +++ b/packages/commons/octobot_commons/databases/relational_databases/__init__.py @@ -18,12 +18,14 @@ from octobot_commons.databases.relational_databases import sqlite from octobot_commons.databases.relational_databases.sqlite import ( - SQLiteDatabase, - new_sqlite_database, + BaseSQLiteDatabase, + open_sqlite_database, + sqlite_database_write_lock, ) __all__ = [ - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", + "open_sqlite_database", + "sqlite_database_write_lock", ] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py b/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py index 403d6dda0c..5fbf984aa7 100644 --- a/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py +++ b/packages/commons/octobot_commons/databases/relational_databases/sqlite/__init__.py @@ -16,14 +16,15 @@ # License along with this library. -from octobot_commons.databases.relational_databases.sqlite import sqlite_database -from octobot_commons.databases.relational_databases.sqlite.sqlite_database import ( - SQLiteDatabase, - new_sqlite_database, +from octobot_commons.databases.relational_databases.sqlite.base_sqlite_database import ( + BaseSQLiteDatabase, + open_sqlite_database, + sqlite_database_write_lock, ) __all__ = [ - "SQLiteDatabase", - "new_sqlite_database", + "BaseSQLiteDatabase", + "open_sqlite_database", + "sqlite_database_write_lock", ] diff --git a/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py new file mode 100644 index 0000000000..ed9338cc3a --- /dev/null +++ b/packages/commons/octobot_commons/databases/relational_databases/sqlite/base_sqlite_database.py @@ -0,0 +1,172 @@ +# pylint: disable=C0116 +# Drakkar-Software OctoBot-Commons +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import contextlib +import pathlib +import sqlite3 + +import octobot_commons.logging as logging +import octobot_commons.errors as errors +import octobot_commons.databases.relational_databases.sqlite.cursor_pool as cursor_pool +import octobot_commons.constants as constants + +try: + import aiosqlite +except ImportError: + if constants.USE_MINIMAL_LIBS: + class AiosqliteImportMock: + def connect(self, *args): + raise ImportError("aiosqlite not installed") + + aiosqlite = AiosqliteImportMock() + else: + raise + +_write_locks: dict[str, asyncio.Lock] = {} + + +def _get_write_lock(file_path: str) -> asyncio.Lock: + if file_path not in _write_locks: + _write_locks[file_path] = asyncio.Lock() + return _write_locks[file_path] + + +@contextlib.asynccontextmanager +async def sqlite_database_write_lock(file_path: str): + async with _get_write_lock(file_path): + yield + + +@contextlib.asynccontextmanager +async def open_sqlite_database(database_cls, file_path: str, read_only: bool = False): + if read_only: + database = database_cls(file_path, read_only=True) + try: + await database.initialize() + yield database + finally: + await database.stop() + else: + async with sqlite_database_write_lock(file_path): + database = database_cls(file_path, read_only=False) + try: + await database.initialize() + yield database + finally: + await database.stop() + + +class BaseSQLiteDatabase: + def __init__(self, file_name, read_only: bool = False): + self.file_name = file_name + self.read_only = read_only + self.logger = logging.get_logger(self.__class__.__name__) + self.connection = None + self._cursor_pool = None + + def connection_pragmas(self) -> list[str]: + # WAL: readers (mode=ro) get a consistent snapshot while writers are active; + # uncommitted writer work stays in the WAL and is rolled back on recovery. + # synchronous=FULL: fsync on commit so committed data survives sudden process kill. + # busy_timeout: wait up to 5s when a read overlaps a writer checkpoint instead of failing immediately. + if self.read_only: + return ["PRAGMA busy_timeout=5000"] + return [ + "PRAGMA journal_mode=WAL", + "PRAGMA synchronous=FULL", + "PRAGMA busy_timeout=5000", + ] + + def _ensure_writable(self) -> None: + if self.read_only: + raise errors.DatabaseReadOnlyError( + f"Cannot write to read-only database (file: {self.file_name})" + ) + + async def initialize(self): + try: + if self.read_only: + database_uri = f"{pathlib.Path(self.file_name).resolve().as_uri()}?mode=ro" + self.connection = await aiosqlite.connect(database_uri, uri=True) + else: + self.connection = await aiosqlite.connect(self.file_name) + self._cursor_pool = cursor_pool.CursorPool(self.connection) + for pragma in self.connection_pragmas(): + async with self.aio_cursor() as cursor: + await cursor.execute(pragma) + except (sqlite3.OperationalError, sqlite3.DatabaseError) as err: + raise errors.DatabaseNotFoundError(f"{err} (file: {self.file_name})") + + @contextlib.asynccontextmanager + async def aio_cursor(self) -> sqlite3.Cursor: + async with self._cursor_pool.idle_cursor() as cursor: + yield cursor.cursor + + async def execute(self, sql: str, parameters=()) -> None: + self._ensure_writable() + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + + async def executemany(self, sql: str, parameters: list) -> None: + self._ensure_writable() + async with self.aio_cursor() as cursor: + await cursor.executemany(sql, parameters) + + async def fetchall(self, sql: str, parameters=()) -> list: + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + return await cursor.fetchall() + + async def fetchone(self, sql: str, parameters=()) -> tuple | None: + async with self.aio_cursor() as cursor: + await cursor.execute(sql, parameters) + return await cursor.fetchone() + + async def commit(self) -> None: + self._ensure_writable() + await self.connection.commit() + + async def stop(self): + try: + if self._cursor_pool is not None: + await self._cursor_pool.close() + self._cursor_pool = None + if self.connection is not None and not self.read_only: + # Normal shutdown: merge all WAL pages into the main DB and truncate the -wal file. + # TRUNCATE resets the WAL to zero length so sidecar files do not accumulate across + # open/close cycles. On SIGKILL this never runs; SQLite auto-checkpoint + next open + # still recover. + checkpoint_cursor = await self.connection.cursor() + try: + await checkpoint_cursor.execute("PRAGMA wal_checkpoint(TRUNCATE)") + except sqlite3.OperationalError as error: + # TRUNCATE needs exclusive access; concurrent readers (same connection + # pool or another process) can still hold locks during shutdown. + # Skipping is safe: committed pages are already on disk (synchronous=FULL); + # only the -wal file may not be truncated until the next open/checkpoint. + self.logger.debug( + "wal_checkpoint(TRUNCATE) skipped during shutdown for %s: %s", + self.file_name, + error, + ) + finally: + await checkpoint_cursor.close() + finally: + if self.connection is not None: + conn = self.connection + self.connection = None + await conn.close() diff --git a/packages/commons/octobot_commons/errors.py b/packages/commons/octobot_commons/errors.py index ff01a254c1..132356d8d8 100644 --- a/packages/commons/octobot_commons/errors.py +++ b/packages/commons/octobot_commons/errors.py @@ -99,6 +99,12 @@ class DatabaseNotFoundError(Exception): """ +class DatabaseReadOnlyError(Exception): + """ + Raised when a write operation is attempted on a read-only database connection + """ + + class MissingDataError(Exception): """ Raised when there is not enough available candles diff --git a/packages/commons/octobot_commons/profiles/profile_storage.py b/packages/commons/octobot_commons/profiles/profile_storage.py index 364b1f3841..99384174f0 100644 --- a/packages/commons/octobot_commons/profiles/profile_storage.py +++ b/packages/commons/octobot_commons/profiles/profile_storage.py @@ -208,10 +208,16 @@ def duplicate_profile( clone.bind_profile_storage(self) return clone - def activate_profile(self, profile: profile_module.Profile) -> None: + def activate_profile( + self, + profile: profile_module.Profile, + *, + init_tentacles_setup: bool = True, + ) -> None: profile.bind_profile_storage(self) self._apply_child_overlay_config(profile) - profile.init_tentacles_setup_config() + if init_tentacles_setup: + profile.init_tentacles_setup_config() def save_active_profile( self, diff --git a/packages/commons/tests/cryptography/test_signing.py b/packages/commons/tests/cryptography/test_signing.py index c758279e66..9df55c9780 100644 --- a/packages/commons/tests/cryptography/test_signing.py +++ b/packages/commons/tests/cryptography/test_signing.py @@ -117,8 +117,11 @@ def test_verify_signature_invalid_signature(): data = b"Test data" signature = cryptography.sign_data(data, private_key_pem) - # Corrupt the signature - corrupted_signature = signature[:-1] + b"\x00" + # Corrupt the signature (always change a byte) + corrupted_bytes = bytearray(signature) + corruption_index = len(corrupted_bytes) // 2 + corrupted_bytes[corruption_index] ^= 0x01 + corrupted_signature = bytes(corrupted_bytes) is_valid = cryptography.verify_signature(data, public_key_pem, corrupted_signature) diff --git a/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py b/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py new file mode 100644 index 0000000000..cdd1c63f3a --- /dev/null +++ b/packages/commons/tests/databases/relational_databases/sqlite/test_base_sqlite_database.py @@ -0,0 +1,166 @@ +import os +import asyncio +import contextlib +import sqlite3 +import tempfile + +import mock +import pytest + +import octobot_commons.errors as errors +import octobot_commons.databases.relational_databases.sqlite.base_sqlite_database as base_sqlite_database_module + +pytestmark = pytest.mark.asyncio + + +class TestBaseSQLiteDatabaseConnectionPragmas: + async def test_pragma_applied_on_initialize(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + journal_mode = await database.fetchone("PRAGMA journal_mode") + synchronous_mode = await database.fetchone("PRAGMA synchronous") + assert journal_mode[0] == "wal" + assert synchronous_mode[0] == 2 # FULL + finally: + await database.stop() + os.remove(file_name) + + +class TestBaseSQLiteDatabaseInitializeStop: + async def test_initialize_and_stop(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + assert database.connection is not None + await database.stop() + assert database.connection is None + finally: + os.remove(file_name) + + async def test_stop_succeeds_when_checkpoint_would_be_busy(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + try: + await database.initialize() + checkpoint_cursor = mock.AsyncMock() + checkpoint_cursor.execute = mock.AsyncMock( + side_effect=sqlite3.OperationalError("database table is locked"), + ) + checkpoint_cursor.close = mock.AsyncMock() + with mock.patch.object( + database.connection, + "cursor", + mock.AsyncMock(return_value=checkpoint_cursor), + ): + await database.stop() + assert database.connection is None + finally: + os.remove(file_name) + + +class TestBaseSQLiteDatabaseExecuteFetch: + @contextlib.asynccontextmanager + async def _empty_database(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + await database.initialize() + try: + await database.execute( + "CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT NOT NULL)" + ) + yield database + finally: + await database.stop() + os.remove(file_name) + + async def test_execute_fetchone_fetchall_and_commit(self): + async with self._empty_database() as database: + await database.execute( + "INSERT INTO sample (value) VALUES (?)", + ("first",), + ) + await database.commit() + row = await database.fetchone("SELECT value FROM sample WHERE id = ?", (1,)) + assert row == ("first",) + rows = await database.fetchall("SELECT value FROM sample") + assert rows == [("first",)] + + async def test_executemany(self): + async with self._empty_database() as database: + await database.executemany( + "INSERT INTO sample (value) VALUES (?)", + [("one",), ("two",)], + ) + await database.commit() + rows = await database.fetchall("SELECT value FROM sample ORDER BY id") + assert rows == [("one",), ("two",)] + + +class TestBaseSQLiteDatabaseReadOnly: + @contextlib.asynccontextmanager + async def _database_with_sample_table(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + writable_database = base_sqlite_database_module.BaseSQLiteDatabase(file_name) + await writable_database.initialize() + await writable_database.execute( + "CREATE TABLE sample (id INTEGER PRIMARY KEY, value TEXT NOT NULL)" + ) + await writable_database.execute( + "INSERT INTO sample (value) VALUES (?)", + ("stored",), + ) + await writable_database.commit() + await writable_database.stop() + + read_only_database = base_sqlite_database_module.BaseSQLiteDatabase(file_name, read_only=True) + await read_only_database.initialize() + try: + yield read_only_database + finally: + await read_only_database.stop() + os.remove(file_name) + + async def test_fetchall_allowed_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + rows = await database.fetchall("SELECT value FROM sample") + assert rows == [("stored",)] + + async def test_execute_raises_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + with pytest.raises(errors.DatabaseReadOnlyError): + await database.execute("INSERT INTO sample (value) VALUES (?)", ("blocked",)) + + async def test_commit_raises_on_read_only_connection(self): + async with self._database_with_sample_table() as database: + with pytest.raises(errors.DatabaseReadOnlyError): + await database.commit() + + +class TestSqliteDatabaseWriteLock: + async def test_concurrent_write_locks_for_same_path_serialize(self): + with tempfile.NamedTemporaryFile(delete=False, suffix=".db") as temp_file: + file_name = temp_file.name + lock_order: list[str] = [] + + async def hold_write_lock(lock_name: str): + async with base_sqlite_database_module.sqlite_database_write_lock(file_name): + lock_order.append(f"{lock_name}_start") + await asyncio.sleep(0.05) + lock_order.append(f"{lock_name}_end") + + await asyncio.gather(hold_write_lock("first"), hold_write_lock("second")) + assert lock_order.index("first_start") < lock_order.index("first_end") + assert lock_order.index("second_start") < lock_order.index("second_end") + assert ( + lock_order.index("first_end") < lock_order.index("second_start") + or lock_order.index("second_end") < lock_order.index("first_start") + ) + os.remove(file_name) diff --git a/packages/commons/tests/profiles/test_profile_storage.py b/packages/commons/tests/profiles/test_profile_storage.py index 9bb5142c03..e54663cdf5 100644 --- a/packages/commons/tests/profiles/test_profile_storage.py +++ b/packages/commons/tests/profiles/test_profile_storage.py @@ -17,6 +17,7 @@ import octobot_commons.profiles.backends as profile_backends_module import octobot_commons.profiles.profile_data as profile_data_module import octobot_commons.profiles.profile_storage as profile_storage_module +import octobot_commons.profiles.profile_types.ephemeral_profile as ephemeral_profile_module import octobot_commons.profiles.profile_types.sync_profile as sync_profile_module import octobot_commons.profiles.profile_data_import as profile_data_import_module import octobot_sync.sync.collection_backend.errors as collection_errors @@ -802,3 +803,26 @@ def _assert_imported_profile_metadata( profile_data_import_module.IMPORTED_PROFILES_DEFAULT_EXTRA_BACKTESTING_TIMEFRAME ] assert profile.profile_id + + +class TestProfileStorageActivateProfileInitTentaclesSetup: + def test_init_tentacles_setup_runs_when_flag_true_even_if_setup_pre_bound(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + pre_bound_setup = mock.Mock() + profile.bind_tentacles_setup_config(pre_bound_setup) + profile_storage = profile_storage_module.ProfileStorage(None, None) + with mock.patch.object(profile, "init_tentacles_setup_config") as init_mock: + profile_storage.activate_profile(profile, init_tentacles_setup=True) + init_mock.assert_called_once() + + def test_init_tentacles_setup_skipped_when_flag_false(self): + profile_data = profile_data_module.ProfileData() + profile = ephemeral_profile_module.EphemeralProfile.from_profile_data(profile_data) + pre_bound_setup = mock.Mock() + profile.bind_tentacles_setup_config(pre_bound_setup) + profile_storage = profile_storage_module.ProfileStorage(None, None) + with mock.patch.object(profile, "init_tentacles_setup_config") as init_mock: + profile_storage.activate_profile(profile, init_tentacles_setup=False) + init_mock.assert_not_called() + assert profile.tentacles_setup_config is pre_bound_setup diff --git a/packages/copy/octobot_copy/constants.py b/packages/copy/octobot_copy/constants.py index 3942c18e19..337fb76d64 100644 --- a/packages/copy/octobot_copy/constants.py +++ b/packages/copy/octobot_copy/constants.py @@ -34,6 +34,7 @@ # Exchange / order lifecycle (seconds) FILL_ORDER_TIMEOUT = 60 OPEN_ORDER_POLL_INTERVAL = 0.5 +OUTDATED_ORDER_PRICE_MAX_THRESHOLD = 0.03 # 3% # Mirrored orphan grace: max |simulated_copier_pair_share − ref_pair_share| to allow deferral DEFAULT_MIRRORED_ORPHAN_GRACE_PAIR_RATIO_MAX_DELTA = decimal.Decimal("0.02") # 2% diff --git a/packages/copy/octobot_copy/copiers/account_copier.py b/packages/copy/octobot_copy/copiers/account_copier.py index 96bc0b069c..a66d840fb8 100644 --- a/packages/copy/octobot_copy/copiers/account_copier.py +++ b/packages/copy/octobot_copy/copiers/account_copier.py @@ -13,6 +13,7 @@ import octobot_copy.errors as copy_errors import octobot_copy.exchange as copy_exchange import octobot_copy.orders_mirroring.orders_synchronizer as orders_synchronizer_module +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders_module import octobot_copy.rebalancing as copy_rebalancing @@ -47,6 +48,10 @@ def __init__( ) async def copy_account(self) -> copy_entities.AccountCopyResult: + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + self._reference_account, + self._copier_exchange_interface, + ) await self._resync_if_mirrored_open_order_grace_period_elapsed() rebalancer, should_rebalance, details = await self._prepare_rebalance_plan() if self._orders_synchronizer.is_mirrored_orphan_grace_invalid_no_compliant_snapshot(): diff --git a/packages/copy/octobot_copy/copiers/formatter.py b/packages/copy/octobot_copy/copiers/formatter.py new file mode 100644 index 0000000000..21bdaf5f3f --- /dev/null +++ b/packages/copy/octobot_copy/copiers/formatter.py @@ -0,0 +1,77 @@ +import typing + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +REFERENCE_ACCOUNT_SUMMARY_LIMIT = 10 + + +def format_reference_account_summary(reference_account: protocol_models.CopiedAccount) -> str: + copied_assets = reference_account.copied_assets + order_list = reference_account.orders or [] + asset_entries = [_format_copied_asset_entry(copied_asset) for copied_asset in copied_assets] + assets_summary = _format_compact_section("assets", asset_entries, len(copied_assets)) + orders_summary = _format_orders_section(order_list) + updated_at = _format_reference_account_updated_at(reference_account.updated_at) + return f"v{reference_account.version}@{updated_at} {assets_summary} {orders_summary}" + + +def _format_reference_account_updated_at(updated_at: typing.Union[float, int]) -> str: + formatted_time = timestamp_util.convert_timestamp_to_datetime( + float(updated_at), + local_timezone=False, + ) + return f"{formatted_time} UTC" + + +def _format_copied_asset_entry(asset: protocol_models.CopiedAsset) -> str: + ratio_percent = float(asset.ratio) * 100 + return f"{asset.name}:{ratio_percent:.1f}%" + + +def _format_reference_order_entry(order: protocol_models.Order) -> str: + return f"{order.side.value} {order.symbol}@{order.price}" + + +def _format_order_type_label(order: protocol_models.Order) -> str: + return f"{order.side.value}_{order.type.value}" + + +def _format_order_type_count_label(orders: list[protocol_models.Order]) -> str: + if not orders: + return "0" + order_type_counts: dict[str, int] = {} + for order in orders: + order_type_label = _format_order_type_label(order) + order_type_counts[order_type_label] = order_type_counts.get(order_type_label, 0) + 1 + return ",".join( + f"{order_type_counts[order_type_label]} {order_type_label}" + for order_type_label in sorted(order_type_counts) + ) + + +def _format_orders_section(order_list: list[protocol_models.Order]) -> str: + order_type_count_label = _format_order_type_count_label(order_list) + if not order_list: + return "orders[0]" + order_entries = [_format_reference_order_entry(order) for order in order_list] + content = _format_compact_list(order_entries, len(order_list)) + return f"orders[{order_type_count_label}]:{content}" + + +def _format_compact_list(entries: list[str], total_count: int) -> str: + if total_count == 0: + return "" + displayed_entries = entries[:REFERENCE_ACCOUNT_SUMMARY_LIMIT] + content = ",".join(displayed_entries) + remaining_count = total_count - len(displayed_entries) + if remaining_count > 0: + content = f"{content},…+{remaining_count} more" + return content + + +def _format_compact_section(section_name: str, entries: list[str], total_count: int) -> str: + if total_count == 0: + return f"{section_name}[0]" + content = _format_compact_list(entries, total_count) + return f"{section_name}[{total_count}]:{content}" diff --git a/packages/copy/octobot_copy/errors.py b/packages/copy/octobot_copy/errors.py index d0b72ce0fd..674e2dacb2 100644 --- a/packages/copy/octobot_copy/errors.py +++ b/packages/copy/octobot_copy/errors.py @@ -14,3 +14,9 @@ class RebalanceAborted(RebalanceError): """ Raised when a rebalance is aborted """ + + +class OutdatedReferenceAccountError(OctobotCopyError): + """ + Raised when the reference account is outdated + """ diff --git a/packages/copy/octobot_copy/validators/__init__.py b/packages/copy/octobot_copy/validators/__init__.py new file mode 100644 index 0000000000..f0b792805c --- /dev/null +++ b/packages/copy/octobot_copy/validators/__init__.py @@ -0,0 +1,5 @@ +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders + +__all__ = [ + "reference_account_outdated_orders", +] diff --git a/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py b/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py new file mode 100644 index 0000000000..d119448339 --- /dev/null +++ b/packages/copy/octobot_copy/validators/reference_account_outdated_orders.py @@ -0,0 +1,99 @@ +import decimal + +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.personal_data as trading_personal_data + +import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors +import octobot_copy.exchange as copy_exchange + + +def resolve_order_trigger_above(order: protocol_models.Order) -> bool: + if order.trigger_above is not None: + return bool(order.trigger_above) + if order.side is protocol_models.Side.SELL: + return True + return False + + +def get_replicable_reference_orders( + reference_account: protocol_models.CopiedAccount, +) -> list[protocol_models.Order]: + replicable: list[protocol_models.Order] = [] + for order in reference_account.orders or []: + if order.status != protocol_models.OrderStatus.OPEN: + continue + if not order.is_active: + continue + raw = trading_personal_data.exchange_columns_dict_from_protocol_order(order) + _side, trader_order_type = trading_personal_data.parse_order_type(raw) + if trader_order_type in ( + trading_enums.TraderOrderType.BUY_MARKET, + trading_enums.TraderOrderType.SELL_MARKET, + ): + continue + replicable.append(order) + return replicable + + +def is_order_impossible_at_market_price( + order: protocol_models.Order, + market_price: decimal.Decimal, + threshold: decimal.Decimal, +) -> bool: + if market_price <= trading_constants.ZERO: + raise ValueError( + f"Market price must be positive to validate order {order.id!r} on {order.symbol!r}, got {market_price}" + ) + order_price = decimal.Decimal(str(order.price)) + if order_price <= trading_constants.ZERO: + return False + trigger_above = resolve_order_trigger_above(order) + if trigger_above: + return order_price <= market_price * (trading_constants.ONE - threshold) + return order_price >= market_price * (trading_constants.ONE + threshold) + + +def _order_price_gap_ratio( + order: protocol_models.Order, + market_price: decimal.Decimal, +) -> decimal.Decimal: + order_price = decimal.Decimal(str(order.price)) + trigger_above = resolve_order_trigger_above(order) + if trigger_above: + return (market_price - order_price) / market_price + return (order_price - market_price) / market_price + + +async def ensure_reference_account_not_outdated( + reference_account: protocol_models.CopiedAccount, + exchange_interface: copy_exchange.ExchangeInterface, +) -> None: + replicable_orders = get_replicable_reference_orders(reference_account) + if not replicable_orders: + return + threshold = decimal.Decimal(str(copy_constants.OUTDATED_ORDER_PRICE_MAX_THRESHOLD)) + market_price_by_symbol: dict[str, decimal.Decimal] = {} + for order in replicable_orders: + symbol = order.symbol + if symbol not in market_price_by_symbol: + market_price, _is_outdated = exchange_interface.market.get_potentially_outdated_price(symbol) + if market_price <= trading_constants.ZERO: + raise ValueError( + f"Missing market price for {symbol!r} while validating reference account outdated orders" + ) + market_price_by_symbol[symbol] = market_price + market_price = market_price_by_symbol[symbol] + if is_order_impossible_at_market_price(order, market_price, threshold): + gap_ratio = _order_price_gap_ratio(order, market_price) + gap_percent = float(gap_ratio * trading_constants.ONE_HUNDRED) + trigger_above = resolve_order_trigger_above(order) + raise copy_errors.OutdatedReferenceAccountError( + f"Reference account updated_at={reference_account.updated_at} is outdated: " + f"order {order.id!r} on {order.symbol!r} " + f"(trigger_above={trigger_above}, price={order.price}, market={market_price}) " + f"would instantly fill by {gap_percent:.2f}% " + f"(threshold={float(threshold * trading_constants.ONE_HUNDRED):.2f}%)" + ) diff --git a/packages/copy/tests/python/copiers/test_account_copier.py b/packages/copy/tests/python/copiers/test_account_copier.py index 672906fd8d..af09b3f3ca 100644 --- a/packages/copy/tests/python/copiers/test_account_copier.py +++ b/packages/copy/tests/python/copiers/test_account_copier.py @@ -22,6 +22,7 @@ import octobot_copy.constants as copy_constants import octobot_copy.copiers.spot_account_copier as spot_account_copier_module import octobot_copy.entities as copy_entities +import octobot_copy.errors as copy_errors pytestmark = pytest.mark.asyncio @@ -120,3 +121,34 @@ async def test_does_not_bypass_grace_when_rebalance_has_no_market_orders(self): grace_identified=True, ) synchronizer.abort_mirrored_orphan_grace.assert_not_called() + + +class TestAccountCopierCopyAccountOutdatedReference: + async def test_skips_rebalance_and_sync_when_reference_account_is_outdated(self): + reference_account = _copied_reference_account() + exchange_interface = mock.MagicMock() + exchange_interface.exchange_name = "bitmart" + copy_settings = copy_entities.AccountCopySettings() + copier = spot_account_copier_module.SpotAccountCopier( + reference_account, + exchange_interface, + copy_settings, + ) + synchronizer = mock.Mock() + copier._orders_synchronizer = synchronizer + copier._prepare_rebalance_plan = mock.AsyncMock() + copier._run_rebalance = mock.AsyncMock() + copier._resync_if_mirrored_open_order_grace_period_elapsed = mock.AsyncMock() + synchronizer.synchronize = mock.AsyncMock() + + with mock.patch( + "octobot_copy.copiers.account_copier.reference_account_outdated_orders_module.ensure_reference_account_not_outdated", + mock.AsyncMock(side_effect=copy_errors.OutdatedReferenceAccountError("stale reference")), + ): + with pytest.raises(copy_errors.OutdatedReferenceAccountError, match="stale reference"): + await copier.copy_account() + + copier._resync_if_mirrored_open_order_grace_period_elapsed.assert_not_awaited() + copier._prepare_rebalance_plan.assert_not_awaited() + copier._run_rebalance.assert_not_awaited() + synchronizer.synchronize.assert_not_awaited() diff --git a/packages/copy/tests/python/copiers/test_formatter.py b/packages/copy/tests/python/copiers/test_formatter.py new file mode 100644 index 0000000000..97b5a2cdad --- /dev/null +++ b/packages/copy/tests/python/copiers/test_formatter.py @@ -0,0 +1,192 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or +# (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# OctoBot. If not, see . +import time + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_copy.copiers.formatter as copy_formatter + + +_REFERENCE_UPDATED_AT = 1710000000.0 +_REFERENCE_UPDATED_AT_LABEL = ( + f"{timestamp_util.convert_timestamp_to_datetime(_REFERENCE_UPDATED_AT, local_timezone=False)} UTC" +) + + +def _copied_asset( + *, + name: str = "BTC", + ratio: float = 1.0, +) -> protocol_models.CopiedAsset: + return protocol_models.CopiedAsset( + name=name, + total=1.0, + available=1.0, + ratio=ratio, + ) + + +def _reference_limit_order( + *, + symbol: str = "BTC/USDT", + price: float = 95000.0, + side: protocol_models.Side = protocol_models.Side.BUY, + order_type: protocol_models.OrderType = protocol_models.OrderType.LIMIT, +) -> protocol_models.Order: + return protocol_models.Order( + id="reference-order-id", + symbol=symbol, + price=price, + quantity=0.001, + filled=0.0, + exchange_id="ex", + side=side, + type=order_type, + trigger_above=False, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account( + *, + copied_assets: list[protocol_models.CopiedAsset] | None = None, + orders: list[protocol_models.Order] | None = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version="1.0.0", + updated_at=_REFERENCE_UPDATED_AT, + copied_assets=copied_assets or [], + orders=orders, + ) + + +class TestFormatReferenceAccountUpdatedAt: + def test_formats_unix_timestamp_as_utc_human_readable_string(self): + assert copy_formatter._format_reference_account_updated_at(_REFERENCE_UPDATED_AT) == ( + _REFERENCE_UPDATED_AT_LABEL + ) + + +class TestFormatCopiedAssetEntry: + def test_formats_name_and_ratio_as_percent(self): + copied_asset = _copied_asset(name="BTC", ratio=0.5038280454117718) + + entry = copy_formatter._format_copied_asset_entry(copied_asset) + + assert entry == "BTC:50.4%" + + +class TestFormatReferenceOrderEntry: + def test_formats_side_symbol_and_price_without_id(self): + order = _reference_limit_order(symbol="BTC/USDT", price=95000.0) + + entry = copy_formatter._format_reference_order_entry(order) + + assert entry == "buy BTC/USDT@95000.0" + assert "reference-order-id" not in entry + + +class TestFormatOrderTypeCountLabel: + def test_empty_when_no_orders(self): + assert copy_formatter._format_order_type_count_label([]) == "0" + + def test_mixed_buy_and_sell_limit_orders(self): + orders = [ + _reference_limit_order(side=protocol_models.Side.BUY), + _reference_limit_order(side=protocol_models.Side.SELL), + ] + + label = copy_formatter._format_order_type_count_label(orders) + + assert label == "1 buy_limit,1 sell_limit" + + def test_multiple_orders_of_same_type(self): + orders = [ + _reference_limit_order(side=protocol_models.Side.BUY), + _reference_limit_order(side=protocol_models.Side.BUY), + ] + + label = copy_formatter._format_order_type_count_label(orders) + + assert label == "2 buy_limit" + + +class TestFormatReferenceAccountSummary: + def test_assets_only(self): + reference_account = _copied_account( + copied_assets=[_copied_asset(name="BTC", ratio=1.0)], + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary == ( + f"v1.0.0@{_REFERENCE_UPDATED_AT_LABEL} assets[1]:BTC:100.0% orders[0]" + ) + + def test_assets_and_orders(self): + reference_account = _copied_account( + copied_assets=[ + _copied_asset(name="USDT", ratio=0.49617195458822816), + _copied_asset(name="BTC", ratio=0.5038280454117718), + ], + orders=[ + _reference_limit_order(symbol="BTC/USDT", price=95000.0), + _reference_limit_order( + symbol="ETH/USDT", + price=3000.0, + side=protocol_models.Side.SELL, + ), + ], + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary == ( + f"v1.0.0@{_REFERENCE_UPDATED_AT_LABEL} " + "assets[2]:USDT:49.6%,BTC:50.4% " + "orders[1 buy_limit,1 sell_limit]:buy BTC/USDT@95000.0,sell ETH/USDT@3000.0" + ) + + def test_orders_none_shows_zero_count(self): + reference_account = _copied_account( + copied_assets=[_copied_asset()], + orders=None, + ) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert summary.endswith("orders[0]") + + def test_truncates_when_more_than_summary_limit(self): + copied_assets = [ + _copied_asset(name=f"ASSET{asset_index}", ratio=0.1) + for asset_index in range(copy_formatter.REFERENCE_ACCOUNT_SUMMARY_LIMIT + 1) + ] + orders = [ + _reference_limit_order(symbol=f"SYM{order_index}/USDT", price=float(order_index)) + for order_index in range(copy_formatter.REFERENCE_ACCOUNT_SUMMARY_LIMIT + 1) + ] + reference_account = _copied_account(copied_assets=copied_assets, orders=orders) + + summary = copy_formatter.format_reference_account_summary(reference_account) + + assert "assets[11]:" in summary + assert "…+1 more" in summary + assert "orders[11 buy_limit]:" in summary diff --git a/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py b/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py new file mode 100644 index 0000000000..2ae805c76e --- /dev/null +++ b/packages/copy/tests/python/validators/test_reference_account_outdated_orders.py @@ -0,0 +1,161 @@ +# This file is part of OctoBot (https://github.com/Drakkar-Software/OctoBot) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +import decimal +import time + +import mock +import pytest + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors +import octobot_copy.validators.reference_account_outdated_orders as reference_account_outdated_orders_module + +_THRESHOLD = decimal.Decimal(str(copy_constants.OUTDATED_ORDER_PRICE_MAX_THRESHOLD)) +_BTC_USDC = "BTC/USDC" + + +def _open_limit_order( + *, + order_id: str = "reference-order-id", + symbol: str = _BTC_USDC, + price: float, + side: protocol_models.Side = protocol_models.Side.SELL, + trigger_above: bool | None = True, + status: protocol_models.OrderStatus = protocol_models.OrderStatus.OPEN, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=symbol, + price=price, + quantity=0.001, + filled=0.0, + exchange_id="ex", + side=side, + type=protocol_models.OrderType.LIMIT, + trigger_above=trigger_above, + reduce_only=False, + is_active=True, + status=status, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account( + *, + orders: list[protocol_models.Order] | None = None, +) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=1710000000.0, + copied_assets=[], + orders=orders, + ) + + +class TestResolveOrderTriggerAbove: + def test_uses_explicit_trigger_above_when_set(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.SELL, trigger_above=False) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is False + + def test_infers_true_for_sell_when_trigger_above_missing(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.SELL, trigger_above=None) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is True + + def test_infers_false_for_buy_when_trigger_above_missing(self): + order = _open_limit_order(price=95000.0, side=protocol_models.Side.BUY, trigger_above=None) + assert reference_account_outdated_orders_module.resolve_order_trigger_above(order) is False + + +class TestIsOrderImpossibleAtMarketPrice: + def test_sell_stale_when_far_below_market(self): + order = _open_limit_order(price=67326.0, trigger_above=True) + assert reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_sell_not_stale_when_near_market(self): + order = _open_limit_order(price=107000.0, trigger_above=True) + assert not reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_buy_stale_when_far_above_market(self): + order = _open_limit_order(price=113500.0, side=protocol_models.Side.BUY, trigger_above=False) + assert reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + def test_buy_not_stale_when_near_market(self): + order = _open_limit_order(price=113000.0, side=protocol_models.Side.BUY, trigger_above=False) + assert not reference_account_outdated_orders_module.is_order_impossible_at_market_price( + order, + decimal.Decimal("110000"), + _THRESHOLD, + ) + + +class TestEnsureReferenceAccountNotOutdated: + @pytest.mark.asyncio + async def test_raises_when_order_is_impossible(self): + reference_account = _copied_account( + orders=[_open_limit_order(price=67326.0, trigger_above=True)], + ) + exchange_interface = mock.Mock() + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("110000"), False), + ) + with pytest.raises(copy_errors.OutdatedReferenceAccountError): + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) + + @pytest.mark.asyncio + async def test_no_op_when_no_orders(self): + exchange_interface = mock.Mock() + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + _copied_account(orders=[]), + exchange_interface, + ) + exchange_interface.market.get_potentially_outdated_price.assert_not_called() + + @pytest.mark.asyncio + async def test_ignores_non_open_orders(self): + reference_account = _copied_account( + orders=[ + _open_limit_order( + price=67326.0, + trigger_above=True, + status=protocol_models.OrderStatus.FILLED, + ) + ], + ) + exchange_interface = mock.Mock() + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) + exchange_interface.market.get_potentially_outdated_price.assert_not_called() + + @pytest.mark.asyncio + async def test_passes_when_orders_are_near_market(self): + reference_account = _copied_account( + orders=[_open_limit_order(price=107000.0, trigger_above=True)], + ) + exchange_interface = mock.Mock() + exchange_interface.market.get_potentially_outdated_price = mock.Mock( + return_value=(decimal.Decimal("110000"), False), + ) + await reference_account_outdated_orders_module.ensure_reference_account_not_outdated( + reference_account, + exchange_interface, + ) diff --git a/packages/flow/octobot_flow/constants.py b/packages/flow/octobot_flow/constants.py index a88c766704..77219b3b85 100644 --- a/packages/flow/octobot_flow/constants.py +++ b/packages/flow/octobot_flow/constants.py @@ -14,3 +14,8 @@ # Copy-trading mirrored open-order grace (aligned with octobot_copy fill timeout by default) DEFAULT_COPY_TRADING_ORPHAN_CANCEL_GRACE_SECONDS = float(copy_constants.FILL_ORDER_TIMEOUT) DEFAULT_COPY_TRADING_ORPHAN_GRACE_ABORT_THRESHOLD = 2 + +PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS = 12 * 60 * 60 + +PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS = 180 +PORTFOLIO_HISTORY_DAILY_TIMEFRAME = commons_enums.TimeFrames.ONE_DAY diff --git a/packages/flow/octobot_flow/entities/__init__.py b/packages/flow/octobot_flow/entities/__init__.py index 372ebf2d53..916178bb90 100644 --- a/packages/flow/octobot_flow/entities/__init__.py +++ b/packages/flow/octobot_flow/entities/__init__.py @@ -35,6 +35,15 @@ UserAuthentication, TradingSignal, ) +from octobot_flow.entities.global_view import ( + GlobalViewAccountContext, + ExchangeAccountRefreshResult, + GlobalViewAccountRefreshResult, +) +from octobot_flow.entities.portfolio_history import ( + PortfolioHistoryAccountContext, + PortfolioHistoryRunResult, +) __all__ = [ "AccountElements", "ExchangeAccountElements", @@ -65,4 +74,9 @@ "AdditionalActions", "UserAuthentication", "TradingSignal", + "GlobalViewAccountContext", + "ExchangeAccountRefreshResult", + "GlobalViewAccountRefreshResult", + "PortfolioHistoryAccountContext", + "PortfolioHistoryRunResult", ] diff --git a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py index 2909be57ce..d7d4d2c00c 100644 --- a/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py +++ b/packages/flow/octobot_flow/entities/accounts/exchange_account_elements.py @@ -23,6 +23,7 @@ class ExchangeAccountElements(account_elements_import.AccountElements): orders: octobot_trading.exchanges.OrdersDetails = dataclasses.field(default_factory=octobot_trading.exchanges.OrdersDetails) positions: list[octobot_trading.exchanges.PositionDetails] = dataclasses.field(default_factory=list) trades: list[dict] = dataclasses.field(default_factory=list) + trade_summaries: dict[str, list[str]] = dataclasses.field(default_factory=dict) def __post_init__(self): super().__post_init__() @@ -36,6 +37,69 @@ def __post_init__(self): self.trades = [ dict(trade) for trade in self.trades # type: ignore ] + self.trade_summaries = self._normalize_trade_summaries(self.trade_summaries) + + @staticmethod + def _normalize_trade_summaries( + trade_summaries: typing.Optional[dict[str, list[str]]], + ) -> dict[str, list[str]]: + if not trade_summaries: + return {} + normalized_summaries: dict[str, list[str]] = {} + for symbol, trade_ids in trade_summaries.items(): + symbol_text = str(symbol).strip() + if not symbol_text: + continue + normalized_ids = [ + str(trade_id) + for trade_id in (trade_ids or []) + if trade_id is not None and str(trade_id) + ] + if normalized_ids: + normalized_summaries[symbol_text] = normalized_ids + return normalized_summaries + + @staticmethod + def trade_id_and_symbol_from_trade_dict(trade: dict) -> typing.Optional[tuple[str, str]]: + order_columns = octobot_trading.enums.ExchangeConstantsOrderColumns + trade_id = trade.get(order_columns.EXCHANGE_TRADE_ID.value) or trade.get( + order_columns.EXCHANGE_ID.value + ) + symbol = trade.get(order_columns.SYMBOL.value) + if trade_id is None or symbol is None: + return None + symbol_text = str(symbol).strip() + if not symbol_text: + return None + return str(trade_id), symbol_text + + def trim_trades_to_live_window(self, max_full_trades: int) -> None: + if max_full_trades <= 0 or len(self.trades) <= max_full_trades: + return + order_columns = octobot_trading.enums.ExchangeConstantsOrderColumns + timestamp_key = order_columns.TIMESTAMP.value + + def sort_key(trade: dict) -> tuple[float, str]: + timestamp = trade.get(timestamp_key) + try: + timestamp_value = float(timestamp) if timestamp is not None else 0.0 + except (TypeError, ValueError): + timestamp_value = 0.0 + identity_key = octobot_trading.personal_data.trades.trades_util.trade_identity_key(trade) + identity_text = str(identity_key) if identity_key is not None else "" + return timestamp_value, identity_text + + sorted_trades = sorted(self.trades, key=sort_key) + evicted_trades = sorted_trades[: len(sorted_trades) - max_full_trades] + self.trades = sorted_trades[len(sorted_trades) - max_full_trades:] + for evicted_trade in evicted_trades: + trade_identity = self.trade_id_and_symbol_from_trade_dict(evicted_trade) + if trade_identity is None: + continue + trade_id, symbol = trade_identity + archived_ids = self.trade_summaries.setdefault(symbol, []) + if trade_id not in archived_ids: + archived_ids.append(trade_id) def has_pending_chained_orders(self) -> bool: for order in self.orders.missing_orders: diff --git a/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py b/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py index a5a3fd6400..10a5d8ab6f 100644 --- a/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py +++ b/packages/flow/octobot_flow/entities/automations/post_iteration_actions_details.py @@ -2,6 +2,7 @@ import typing import octobot_commons.dataclasses +import octobot_commons.dsl_interpreter @dataclasses.dataclass @@ -36,6 +37,36 @@ class PostIterationActionsDetails(octobot_commons.dataclasses.MinimizableDatacla def has_automation_actions(self) -> bool: return bool(self.stop_automation) + @classmethod + def post_iteration_clear(cls, post_iteration_payload: dict) -> None: + post_iteration_payload.pop("updated_exchange_account_elements", None) + + @classmethod + def post_iteration_clear_from_action_result(cls, action_result: dict) -> None: + """ + Remove merge-consumed PostIteration fields from persisted action-result dicts. + Mutates ``action_result`` in place (top-level and recall-nested blobs). + """ + post_iter_name = cls.__name__ + # Top-level PostIteration (e.g. stop_automation / update_automation_configuration). + top_level_payload = action_result.get(post_iter_name) + if isinstance(top_level_payload, dict): + cls.post_iteration_clear(top_level_payload) + # Nested PostIteration inside a re-calling operator recall payload (e.g. run_octobot_process). + if not octobot_commons.dsl_interpreter.ReCallingOperatorResult.is_re_calling_operator_result( + action_result + ): + return + recall_wrapper = octobot_commons.dsl_interpreter.ReCallingOperatorResult.from_dict( + action_result[octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__] + ) + inner_last_result = recall_wrapper.last_execution_result + if not isinstance(inner_last_result, dict): + return + nested_payload = inner_last_result.get(post_iter_name) + if isinstance(nested_payload, dict): + cls.post_iteration_clear(nested_payload) + def should_cancel_iteration(self) -> bool: # cancelled if global view refresh is triggered, otherwise proceed # with next iteration required steps diff --git a/packages/flow/octobot_flow/entities/global_view/__init__.py b/packages/flow/octobot_flow/entities/global_view/__init__.py new file mode 100644 index 0000000000..f4f14072a9 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/__init__.py @@ -0,0 +1,9 @@ +from octobot_flow.entities.global_view.global_view_account_context import GlobalViewAccountContext +from octobot_flow.entities.global_view.exchange_account_refresh_result import ExchangeAccountRefreshResult +from octobot_flow.entities.global_view.global_view_account_refresh_result import GlobalViewAccountRefreshResult + +__all__ = [ + "GlobalViewAccountContext", + "ExchangeAccountRefreshResult", + "GlobalViewAccountRefreshResult", +] diff --git a/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py new file mode 100644 index 0000000000..87ff39a2e6 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/exchange_account_refresh_result.py @@ -0,0 +1,17 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models + + +@dataclasses.dataclass +class ExchangeAccountRefreshResult: + assets: list[protocol_models.DetailedAssetsForTradingType] + ticker_closes: dict[str, float] + valuation_unit: str + open_orders: list[dict] + trades: list[dict] + positions: list[dict] + changed_order_ids: set[str] diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py new file mode 100644 index 0000000000..09d797f278 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_context.py @@ -0,0 +1,17 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + + +@dataclasses.dataclass +class GlobalViewAccountContext: + account: protocol_models.Account + exchange_account: protocol_models.ExchangeAccount + exchange_config: protocol_models.ExchangeConfig + trading_type: protocol_models.TradingType + auth_details: exchange_data_module.ExchangeAuthDetails + has_bound_automation: bool = False diff --git a/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py new file mode 100644 index 0000000000..b63d3b5b43 --- /dev/null +++ b/packages/flow/octobot_flow/entities/global_view/global_view_account_refresh_result.py @@ -0,0 +1,15 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models + + +@dataclasses.dataclass +class GlobalViewAccountRefreshResult: + updated_account: protocol_models.Account + changed_order_ids: set[str] + open_orders: list[dict] | None = None + trades: list[dict] | None = None + positions: list[dict] | None = None diff --git a/packages/flow/octobot_flow/entities/portfolio_history/__init__.py b/packages/flow/octobot_flow/entities/portfolio_history/__init__.py new file mode 100644 index 0000000000..e7258f0083 --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/__init__.py @@ -0,0 +1,7 @@ +from octobot_flow.entities.portfolio_history.portfolio_history_account_context import PortfolioHistoryAccountContext +from octobot_flow.entities.portfolio_history.portfolio_history_run_result import PortfolioHistoryRunResult + +__all__ = [ + "PortfolioHistoryAccountContext", + "PortfolioHistoryRunResult", +] diff --git a/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py new file mode 100644 index 0000000000..29434c3735 --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_account_context.py @@ -0,0 +1,17 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + +import octobot_protocol.models as protocol_models +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + + +@dataclasses.dataclass +class PortfolioHistoryAccountContext: + account: protocol_models.Account + exchange_account: protocol_models.ExchangeAccount + exchange_config: protocol_models.ExchangeConfig + trading_type: protocol_models.TradingType + auth_details: exchange_data_module.ExchangeAuthDetails + trade_symbols: list[str] = dataclasses.field(default_factory=list) diff --git a/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py new file mode 100644 index 0000000000..37b8ef249f --- /dev/null +++ b/packages/flow/octobot_flow/entities/portfolio_history/portfolio_history_run_result.py @@ -0,0 +1,19 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import dataclasses + + +@dataclasses.dataclass +class PortfolioHistoryRunResult: + account_id: str + exchange_name: str + trades_count: int = 0 + transactions_count: int = 0 + skipped: bool = False + error: str | None = None + is_simulated: bool = False + trading_type: str = "" + duration_seconds: float | None = None + price_symbols_count: int = 0 + trade_symbols_count: int = 0 diff --git a/packages/flow/octobot_flow/errors.py b/packages/flow/octobot_flow/errors.py index 708d93f470..a0c0bb17c4 100644 --- a/packages/flow/octobot_flow/errors.py +++ b/packages/flow/octobot_flow/errors.py @@ -98,3 +98,7 @@ class PriorityActionError(AutomationActionError): class PendingPriorityActionsSkippedError(PriorityActionError): """raise when supplied priority actions were not executed and DAG actions would run instead""" + + +class GlobalViewUnsupportedAccountError(ConfigurationError): + """raise when global view refresh cannot run for an account type""" diff --git a/packages/flow/octobot_flow/jobs/__init__.py b/packages/flow/octobot_flow/jobs/__init__.py index c30109c891..14002b53e0 100644 --- a/packages/flow/octobot_flow/jobs/__init__.py +++ b/packages/flow/octobot_flow/jobs/__init__.py @@ -1,9 +1,11 @@ from octobot_flow.jobs.automation_job import AutomationJob from octobot_flow.jobs.automation_runner_job import AutomationRunnerJob from octobot_flow.jobs.exchange_account_job import ExchangeAccountJob +from octobot_flow.jobs.global_view_account_job import GlobalViewAccountJob __all__ = [ "AutomationJob", "AutomationRunnerJob", "ExchangeAccountJob", + "GlobalViewAccountJob", ] diff --git a/packages/flow/octobot_flow/jobs/automation_job.py b/packages/flow/octobot_flow/jobs/automation_job.py index 93c7d3e2ad..7a9e7968ac 100644 --- a/packages/flow/octobot_flow/jobs/automation_job.py +++ b/packages/flow/octobot_flow/jobs/automation_job.py @@ -420,7 +420,8 @@ async def _execute_automation_actions( ) raise except Exception as err: - self._logger.error( + self._logger.exception( + err, True, f"Unexpected error when updating {automation_signature}: {err.__class__.__name__}: {err}" ) raise diff --git a/packages/flow/octobot_flow/jobs/exchange_account_job.py b/packages/flow/octobot_flow/jobs/exchange_account_job.py index 3a026a3f7b..d8071a3f1a 100644 --- a/packages/flow/octobot_flow/jobs/exchange_account_job.py +++ b/packages/flow/octobot_flow/jobs/exchange_account_job.py @@ -11,6 +11,7 @@ import octobot_trading.constants as trading_constants import octobot_trading.enums import octobot_trading.errors +import octobot_trading.api.exchange as exchange_api import octobot_trading.personal_data as personal_data import octobot_trading.personal_data.orders.order_util as order_util import octobot_trading.exchanges @@ -229,7 +230,7 @@ async def _fetch_portfolio(self, fetched_authenticated_data: octobot_flow.entiti self._update_exchange_account_portfolio(fetched_authenticated_data.portfolio) def _update_exchange_account_portfolio(self, portfolio: exchange_data_import.PortfolioDetails): - unit = scripting_library.get_default_exchange_reference_market(self._exchange_manager.exchange_name) + unit = exchange_api.get_default_exchange_reference_market(self._exchange_manager.exchange_name) self.automation_state.exchange_account_details.portfolio.content = [ octobot_flow.entities.PortfolioAssetHolding( asset, diff --git a/packages/flow/octobot_flow/jobs/global_view_account_job.py b/packages/flow/octobot_flow/jobs/global_view_account_job.py new file mode 100644 index 0000000000..bf924740c6 --- /dev/null +++ b/packages/flow/octobot_flow/jobs/global_view_account_job.py @@ -0,0 +1,123 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.api as trading_api +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot_flow.entities +import octobot_flow.errors +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory_module +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module +import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module + + +class GlobalViewAccountJob: + def __init__(self, user_id: str, context: octobot_flow.entities.GlobalViewAccountContext): + self.user_id = user_id + self.context = context + + async def run(self) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account = self.context.account + account_specifics = account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + "Account.specifics.actual_instance is required for global view refresh." + ) + account_specifics_instance = account_specifics.actual_instance + if isinstance(account_specifics_instance, protocol_models.GenericAccount): + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + ) + if isinstance(account_specifics_instance, protocol_models.BlockchainAccount): + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + "Blockchain accounts are not supported yet." + ) + if not isinstance(account_specifics_instance, protocol_models.ExchangeAccount): + raise octobot_flow.errors.GlobalViewUnsupportedAccountError( + f"Unsupported account specifics type: {type(account_specifics_instance).__name__}." + ) + + previous_open_order_exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + self.user_id, + account.id, + ) + previous_open_orders = account_state_persistence_module.load_previous_open_orders( + self.user_id, + account.id, + ) + persist_open_orders = ( + not self.context.has_bound_automation + and bool(previous_open_orders) + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + self.context.exchange_account, + self.context.exchange_config, + self.context.trading_type, + is_simulated=account.is_simulated, + ) + exchange_data = exchange_data_module.exchange_data_factory( + exchange_internal_name=self.context.exchange_config.exchange, + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(self.context.trading_type).value, + sandboxed=self.context.exchange_config.sandboxed, + auth_details=self.context.auth_details, + ) + tentacles_setup_config = tentacles_manager_api.get_full_tentacles_setup_config() + async with trading_exchanges.exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ) as exchange_manager: + if account.is_simulated: + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + self.context.trading_type, + previous_open_order_exchange_ids, + is_simulated=True, + previous_open_orders=previous_open_orders, + ) + else: + open_order_symbols = simulated_order_fill_detector_module.symbols_from_open_orders( + previous_open_orders, + ) + exchange_refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + self.context.trading_type, + previous_open_order_exchange_ids, + fetch_open_orders=bool(open_order_symbols), + open_order_symbols=open_order_symbols, + ) + + refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( + self.user_id, + self.context, + exchange_refresh_result, + ) + global_view_persistence_module.persist_global_view_refresh_result( + self.user_id, + account.id, + refresh_result, + persist_open_orders=persist_open_orders, + ) + # Update persisted latest-tickers cache from fetched ticker closes. + if exchange_refresh_result.ticker_closes: + await trading_api.update_latest_tickers( + self.context.exchange_config.exchange, + protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + self.context.trading_type + ).value, + self.context.exchange_config.sandboxed, + exchange_refresh_result.ticker_closes, + ) + return refresh_result diff --git a/packages/flow/octobot_flow/jobs/portfolio_history_job.py b/packages/flow/octobot_flow/jobs/portfolio_history_job.py new file mode 100644 index 0000000000..c40f5e56f2 --- /dev/null +++ b/packages/flow/octobot_flow/jobs/portfolio_history_job.py @@ -0,0 +1,375 @@ +import asyncio +import time + +import octobot_commons.constants as commons_constants +import octobot_commons.logging as commons_logging +import octobot_commons.symbols.symbol_util as symbol_util +import octobot_protocol.models as protocol_models +import octobot_tentacles_manager.api as tentacles_manager_api +import octobot_trading.api.exchange as exchange_api +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot_flow.entities +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.logic.configuration.profile_data_factory as profile_data_factory_module +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.logic.portfolio_history.trading_history_merge as trading_history_merge_module +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +import octobot_flow.logic.portfolio_history.trade_symbols_discovery as trade_symbols_discovery_module +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module +import octobot_flow.repositories.exchange.transactions_repository as transactions_repository_module + +logger = commons_logging.get_logger("PortfolioHistoryJob") + + +class PortfolioHistoryJob: + def __init__( + self, + wallet_id: str, + contexts: list[portfolio_history_entities.PortfolioHistoryAccountContext], + data_root: str = None, + ): + self.wallet_id = wallet_id + self.contexts = contexts + self.data_root = data_root + + async def run(self) -> list[portfolio_history_entities.PortfolioHistoryRunResult]: + """Run data collection for all exchange accounts in parallel.""" + tasks = [ + self._run_for_account(context) + for context in self.contexts + ] + return list(await asyncio.gather(*tasks)) + + async def _run_for_account( + self, context: portfolio_history_entities.PortfolioHistoryAccountContext + ) -> portfolio_history_entities.PortfolioHistoryRunResult: + account = context.account + account_id = account.id + exchange_name = context.exchange_config.exchange + result_metadata = _result_metadata_from_context(context) + + # Skip unsupported account types. + specifics = account.specifics + if ( + specifics is None + or specifics.actual_instance is None + or not isinstance(specifics.actual_instance, protocol_models.ExchangeAccount) + or account.is_simulated + ): + return _skipped_run_result(account_id, exchange_name, result_metadata) + + started_at = time.monotonic() + try: + result = await self._fetch_and_persist(context) + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history job failed for account {account_id}: {error}", + ) + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account_id, + exchange_name=exchange_name, + error=str(error), + duration_seconds=time.monotonic() - started_at, + **result_metadata, + ) + result.duration_seconds = time.monotonic() - started_at + return result + + async def _fetch_and_persist( + self, context: portfolio_history_entities.PortfolioHistoryAccountContext + ) -> portfolio_history_entities.PortfolioHistoryRunResult: + account = context.account + exchange_config = context.exchange_config + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + context.trading_type + ).value + + profile_data = _build_profile_data(context) + exchange_data = exchange_data_module.exchange_data_factory( + exchange_internal_name=exchange_config.exchange, + exchange_type=exchange_type, + sandboxed=exchange_config.sandboxed, + auth_details=context.auth_details, + ) + tentacles_setup_config = tentacles_manager_api.get_full_tentacles_setup_config() + account_trading = _load_account_trading(self.wallet_id, account.id) + existing_config_symbols = set(exchange_config.historical_trade_symbols or []) + seed_symbols = list(context.trade_symbols) + + async with trading_exchanges.exchange_manager_from_exchange_data( + exchange_data, + profile_data, + tentacles_setup_config, + price_fallback=None, + ) as exchange_manager: + await trades_repository_module.TradesRepository.ensure_temporary_trades_channel( + exchange_manager, + ) + fetched_exchange_data = octobot_flow.entities.FetchedExchangeData() + trades_repo = trades_repository_module.TradesRepository( + exchange_manager, [], fetched_exchange_data + ) + tx_repo = transactions_repository_module.TransactionsRepository( + exchange_manager, [], fetched_exchange_data + ) + + currencies_with_balance = _currencies_with_balance_from_account(account) + if currencies_with_balance: + deposits = await tx_repo.fetch_deposits(currencies=currencies_with_balance) + withdrawals = await tx_repo.fetch_withdrawals(currencies=currencies_with_balance) + else: + deposits = [] + withdrawals = [] + all_transactions = deposits + withdrawals + + reference_market = _reference_market_from_account_assets( + account, + exchange_config.exchange, + ) + discovered_symbols = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=seed_symbols, + account=account, + account_trading=account_trading, + fresh_transactions=all_transactions, + reference_market=reference_market, + ) + fetched_trades_count = 0 + trades = await trades_repo.fetch_trades_paginated( + discovered_symbols, + existing_config_symbols=existing_config_symbols, + exchange_name=exchange_config.exchange, + account_id=account.id, + exchange_config_id=exchange_config.id, + exchange_config_name=exchange_config.name, + ) + fetched_trades_count = len(trades) + trades, dropped_trade_symbols = _filter_trades_on_live_markets( + exchange_manager, + trades, + ) + if dropped_trade_symbols: + logger.info( + "Dropped %d trades on delisted/unknown markets for %s account %s: %s", + fetched_trades_count - len(trades), + exchange_config.exchange, + account.id, + ", ".join(sorted(dropped_trade_symbols)), + ) + + live_discovered_symbols = _filter_symbols_on_live_markets( + exchange_manager, + discovered_symbols, + ) + price_symbols = _derive_price_symbols( + live_discovered_symbols, + all_transactions, + reference_market, + ) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + exchange_config.exchange, + exchange_type, + exchange_config.sandboxed, + price_symbols, + self.data_root, + ) + + # Merge and persist trading history. + trading_history_merge_module.merge_and_persist_trading_history( + self.wallet_id, account.id, trades, all_transactions + ) + + trade_confirmed_symbols = trade_symbols_discovery_module.trade_confirmed_symbols_from_fetched_trades( + trades, + ) + new_trade_symbols = trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + self.wallet_id, + exchange_config, + trade_confirmed_symbols, + ) + if new_trade_symbols: + logger.info( + "Added trade-confirmed symbols to historical_trade_symbols on %s config %s " + "(account %s): %s", + exchange_config.exchange, + exchange_config.name or exchange_config.id, + account.id, + ", ".join(new_trade_symbols), + ) + + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account.id, + exchange_name=exchange_config.exchange, + trades_count=len(trades), + transactions_count=len(all_transactions), + is_simulated=account.is_simulated, + trading_type=context.trading_type.value, + price_symbols_count=len(price_symbols), + trade_symbols_count=len(discovered_symbols), + ) + + +def _load_account_trading( + wallet_id: str, + account_id: str, +) -> protocol_models.AccountTrading | None: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + wallet_id, + account_id, + ) + return trading_state.account_trading + except collection_errors.CollectionNoDataError: + return None + + +def _result_metadata_from_context( + context: portfolio_history_entities.PortfolioHistoryAccountContext, +) -> dict: + return { + "is_simulated": context.account.is_simulated, + "trading_type": context.trading_type.value, + } + + +def _skipped_run_result( + account_id: str, + exchange_name: str, + result_metadata: dict, +) -> portfolio_history_entities.PortfolioHistoryRunResult: + return portfolio_history_entities.PortfolioHistoryRunResult( + account_id=account_id, + exchange_name=exchange_name, + skipped=True, + **result_metadata, + ) + + +def _build_profile_data(context: portfolio_history_entities.PortfolioHistoryAccountContext): + return profile_data_factory_module.profile_data_for_account( + context.account, + context.exchange_account, + context.exchange_config, + context.trading_type, + is_simulated=context.account.is_simulated, + ) + + +def _currencies_with_balance_from_account( + account: protocol_models.Account, + min_holdings_threshold: float = 1e-8, +) -> list[str]: + currency_totals: dict[str, float] = {} + if account.assets: + for assets_for_trading_type in account.assets: + for asset in assets_for_trading_type.assets or []: + asset_total = float(asset.total or 0) + if asset_total <= min_holdings_threshold: + continue + currency_totals[asset.symbol] = max( + currency_totals.get(asset.symbol, 0), + asset_total, + ) + return sorted(currency_totals.keys()) + + +def _reference_market_from_account_assets( + account: protocol_models.Account, + exchange_name: str, + min_holdings_threshold: float = 1e-8, +) -> str: + usd_like_holdings: dict[str, float] = {} + if account.assets: + for assets_for_trading_type in account.assets: + for asset in assets_for_trading_type.assets or []: + if asset.symbol not in commons_constants.USD_LIKE_COINS: + continue + asset_total = float(asset.total or 0) + if asset_total <= min_holdings_threshold: + continue + usd_like_holdings[asset.symbol] = max( + usd_like_holdings.get(asset.symbol, 0), + asset_total, + ) + if usd_like_holdings: + return max(usd_like_holdings, key=usd_like_holdings.get) + return exchange_api.get_default_exchange_reference_market(exchange_name) + + +def _filter_trades_on_live_markets( + exchange_manager, + trades: list[dict], +) -> tuple[list[dict], set[str]]: + order_columns = trading_enums.ExchangeConstantsOrderColumns + live_symbols = set(exchange_manager.client_symbols or []) + kept_trades: list[dict] = [] + dropped_symbols: set[str] = set() + for trade in trades: + trade_symbol = trade.get(order_columns.SYMBOL.value) + if not trade_symbol or trade_symbol not in live_symbols: + if trade_symbol: + dropped_symbols.add(trade_symbol) + continue + kept_trades.append(trade) + return kept_trades, dropped_symbols + + +def _filter_symbols_on_live_markets( + exchange_manager, + symbols: list[str], +) -> list[str]: + live_symbols = set(exchange_manager.client_symbols or []) + return sorted(symbol for symbol in symbols if symbol in live_symbols) + + +def _derive_price_symbols( + trade_symbols: list[str], + transactions: list[dict], + reference_market: str, +) -> list[str]: + """Build the set of symbols whose daily prices should be cached.""" + base_assets: set[str] = set() + for trading_symbol in trade_symbols: + if "/" not in trading_symbol: + continue + base_currency, _quote_currency = trading_symbol.split("/", 1) + if base_currency in commons_constants.USD_LIKE_COINS: + continue + if base_currency and base_currency != reference_market: + base_assets.add(base_currency) + for transaction in transactions: + transaction_currency = transaction.get("currency") + if ( + not transaction_currency + or transaction_currency == reference_market + or transaction_currency in commons_constants.USD_LIKE_COINS + ): + continue + base_assets.add(transaction_currency) + valuation_symbols = [ + symbol_util.merge_currencies(base_asset, reference_market) + for base_asset in base_assets + ] + return sorted( + valuation_symbol + for valuation_symbol in valuation_symbols + if _is_valid_trading_symbol(valuation_symbol) + ) + + +def _is_valid_trading_symbol(symbol: str) -> bool: + if "/" not in symbol: + return False + base_currency, quote_currency = symbol.split("/", 1) + if base_currency in commons_constants.USD_LIKE_COINS: + return False + return bool(base_currency) and bool(quote_currency) and base_currency != quote_currency diff --git a/packages/flow/octobot_flow/logic/accounts/__init__.py b/packages/flow/octobot_flow/logic/accounts/__init__.py new file mode 100644 index 0000000000..05166f7e2e --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/__init__.py @@ -0,0 +1,17 @@ +from octobot_flow.logic.accounts.portfolio_history import merge_snapshot +from octobot_flow.logic.accounts.account_state_persistence import ( + build_portfolio_history_state, + load_portfolio_history_state, + load_previous_open_order_exchange_ids, + persist_account_trading, + persist_account_trading_orders, +) + +__all__ = [ + "merge_snapshot", + "build_portfolio_history_state", + "load_portfolio_history_state", + "load_previous_open_order_exchange_ids", + "persist_account_trading", + "persist_account_trading_orders", +] diff --git a/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py new file mode 100644 index 0000000000..f03ec2ebff --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/account_state_persistence.py @@ -0,0 +1,240 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import datetime +import typing + +import octobot_commons.logging as octobot_commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.personal_data.orders.protocol as orders_protocol +import octobot_trading.personal_data.positions.protocol as positions_protocol +import octobot_trading.personal_data.trades.protocol as trades_protocol +import octobot_trading.personal_data.trades.trades_util as trades_util +import octobot_trading.personal_data.transactions.protocol as transactions_protocol +import octobot_trading.personal_data.transactions.transactions_util as transactions_util + +import octobot_flow.entities +import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module + + +def load_previous_open_order_exchange_ids(user_id: str, account_id: str) -> set[str]: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + return set() + account_trading = trading_state.account_trading + if account_trading is None or account_trading.orders is None: + return set() + return order_change_detection_module.open_order_exchange_ids_from_protocol_orders( + account_trading.orders + ) + + +def load_previous_open_orders(user_id: str, account_id: str) -> list[dict]: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + return [] + account_trading = trading_state.account_trading + if account_trading is None or account_trading.orders is None: + return [] + return [ + { + trading_constants.STORAGE_ORIGIN_VALUE: orders_protocol.exchange_columns_dict_from_protocol_order( + protocol_order + ), + } + for protocol_order in account_trading.orders + ] + + +def load_portfolio_history_state( + user_id: str, + account_id: str, +) -> protocol_models.PortfolioHistoricalValuesState: + try: + return collection_providers.AccountHistoryProvider.instance().load_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def build_portfolio_history_state( + user_id: str, + account_id: str, + snapshot: protocol_models.PortfolioHistoricalValue, + valuation_unit: str, + evaluation_time: datetime.datetime, +) -> protocol_models.PortfolioHistoricalValuesState: + history_state = load_portfolio_history_state(user_id, account_id) + existing_values = ( + history_state.history.values + if history_state.history is not None and history_state.history.values + else [] + ) + merged_values = portfolio_history_module.merge_snapshot( + existing_values, + snapshot, + evaluation_time, + ) + updated_history = protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=merged_values, + ) + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=updated_history, + ) + + +def _normalize_order_for_protocol(order: dict) -> dict: + order_details = order.get(trading_constants.STORAGE_ORIGIN_VALUE, order) + order_id = order_details.get(trading_enums.ExchangeConstantsOrderColumns.ID.value) + exchange_id = order_details.get(trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value) + if not order_id: + if exchange_id: + order_details[trading_enums.ExchangeConstantsOrderColumns.ID.value] = exchange_id + else: + raise ValueError( + f"Order is missing both id and exchange_id: {sorted(order_details.keys())}" + ) + return order + + +def persist_account_trading( + user_id: str, + account_id: str, + orders: list[dict], + trades: list[dict], + positions: list[dict], + transactions: list[dict] | None = None, +) -> None: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + # Abnormal: AccountTradingState must be created with the account, not invent here. + raise + account_trading = trading_state.account_trading + account_trading.orders = [ + orders_protocol.to_protocol_order(_normalize_order_for_protocol(order)) + for order in orders + ] or None + account_trading.positions = [ + positions_protocol.to_protocol_position(position) for position in positions + ] or None + existing_trade_dicts = [ + trades_protocol.exchange_columns_dict_from_protocol_trade(protocol_trade) + for protocol_trade in (account_trading.trades or []) + ] + merged_trade_dicts = trades_util.merge_trades_deduped(existing_trade_dicts, trades) + account_trading.trades = [ + trades_protocol.to_protocol_trade(trade_dict) for trade_dict in merged_trade_dicts + ] or None + if transactions is not None: + existing_tx_dicts = [ + transactions_protocol.to_exchange_columns_dict(protocol_tx) + for protocol_tx in (account_trading.transactions or []) + ] + merged_tx_dicts = transactions_util.merge_transactions_deduped(existing_tx_dicts, transactions) + account_trading.transactions = [ + transactions_protocol.to_protocol_transaction(tx_dict) for tx_dict in merged_tx_dicts + ] or None + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + user_id, + account_id, + trading_state, + ) + + +def persist_account_trading_orders( + user_id: str, + account_id: str, + orders: list[dict], +) -> None: + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + user_id, + account_id, + ) + except collection_errors.CollectionNoDataError: + # Abnormal: AccountTradingState must be created with the account, not invent here. + raise + account_trading = trading_state.account_trading + account_trading.orders = [ + orders_protocol.to_protocol_order(_normalize_order_for_protocol(order)) + for order in orders + ] or None + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + user_id, + account_id, + trading_state, + ) + + +def persist_account_trading_from_iteration_state( + user_id: typing.Optional[str], + iteration_state: typing.Optional[dict], +) -> None: + if user_id is None or iteration_state is None: + return + automation_state = octobot_flow.entities.AutomationState.from_dict(iteration_state) + exchange_account_elements = automation_state.automation.exchange_account_elements + exchange_account_details = automation_state.exchange_account_details + if exchange_account_elements is None or exchange_account_details is None: + return + exchange_account_id = exchange_account_details.exchange_details.exchange_account_id + if not exchange_account_id: + return + try: + persist_account_trading( + user_id, + exchange_account_id, + list(exchange_account_elements.orders.open_orders), + list(exchange_account_elements.trades), + [ + position_details.position + for position_details in exchange_account_elements.positions + ], + ) + except wallet_backend_errors.WalletNotFoundError: + # Trading collections are wallet-scoped; skip until the wallet is registered locally. + octobot_commons_logging.get_logger(__name__).warning( + "Skipping account trading persistence for wallet %s: wallet not registered", + user_id, + ) + + +def trim_live_trades_in_iteration_state( + iteration_state: typing.Optional[dict], + max_full_trades: int, +) -> None: + if iteration_state is None or max_full_trades <= 0: + return + automation_state = octobot_flow.entities.AutomationState.from_dict(iteration_state) + exchange_account_elements = automation_state.automation.exchange_account_elements + if exchange_account_elements is None: + return + exchange_account_elements.trim_trades_to_live_window(max_full_trades) + trimmed_state = automation_state.to_dict(include_default_values=False) + iteration_state.clear() + iteration_state.update(trimmed_state) diff --git a/packages/flow/octobot_flow/logic/accounts/portfolio_history.py b/packages/flow/octobot_flow/logic/accounts/portfolio_history.py new file mode 100644 index 0000000000..549fbe6e34 --- /dev/null +++ b/packages/flow/octobot_flow/logic/accounts/portfolio_history.py @@ -0,0 +1,34 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import datetime + +import octobot_flow.constants as flow_constants +import octobot_protocol.models as protocol_models + + +def merge_snapshot( + existing_values: list[protocol_models.PortfolioHistoricalValue], + new_snapshot: protocol_models.PortfolioHistoricalValue, + evaluation_time: datetime.datetime, +) -> list[protocol_models.PortfolioHistoricalValue]: + """ + Keep 12h-spaced snapshots plus a always-updated latest entry. + Persisted shape: twelve_hour_snapshots + [latest]. + """ + if not existing_values: + return [new_snapshot] + + twelve_hour_snapshots = ( + [existing_values[0]] + if len(existing_values) == 1 + else list(existing_values[:-1]) + ) + interval_seconds = flow_constants.PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS + last_twelve_hour_snapshot = twelve_hour_snapshots[-1] + elapsed_since_last_twelve_hour_snapshot = ( + evaluation_time - last_twelve_hour_snapshot.timestamp + ).total_seconds() + if elapsed_since_last_twelve_hour_snapshot >= interval_seconds: + twelve_hour_snapshots.append(new_snapshot) + return twelve_hour_snapshots + [new_snapshot] diff --git a/packages/flow/octobot_flow/logic/actions/actions_executor.py b/packages/flow/octobot_flow/logic/actions/actions_executor.py index f70e1a0dd9..a29ef02ab8 100644 --- a/packages/flow/octobot_flow/logic/actions/actions_executor.py +++ b/packages/flow/octobot_flow/logic/actions/actions_executor.py @@ -59,6 +59,8 @@ async def execute(self): if should_stop_processing: break self._sync_after_execution(synchronized_exchange_account_elements) + if synchronized_exchange_account_elements: + self._clear_post_iteration_after_merge_from_action_results() if self._update_execution_details: await self._update_actions_history() await self._insert_execution_bot_logs(dsl_executor.pending_bot_logs) @@ -366,6 +368,16 @@ def _sync_after_execution( ) self._sync_exchange_account_elements(exchange_account_elements, new_transactions) + def _clear_post_iteration_after_merge_from_action_results(self) -> None: + for action in self._actions: + if not isinstance(action, octobot_flow.entities.DSLScriptActionDetails): + continue + for result_candidate in (action.result, action.previous_execution_result): + if isinstance(result_candidate, dict): + octobot_flow.entities.PostIterationActionsDetails.post_iteration_clear_from_action_result( + result_candidate + ) + def _get_new_transactions_from_actions_results( self, exchange_account_elements: octobot_flow.entities.ExchangeAccountElements, diff --git a/packages/flow/octobot_flow/logic/configuration/__init__.py b/packages/flow/octobot_flow/logic/configuration/__init__.py index 3d5f5bd80f..64a2108727 100644 --- a/packages/flow/octobot_flow/logic/configuration/__init__.py +++ b/packages/flow/octobot_flow/logic/configuration/__init__.py @@ -3,6 +3,7 @@ from octobot_flow.logic.configuration.profile_data_factory import ( create_profile_data, infer_reference_market, + profile_data_for_account, ) __all__ = [ @@ -10,4 +11,5 @@ "AutomationConfigurationUpdater", "create_profile_data", "infer_reference_market", + "profile_data_for_account", ] diff --git a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py index 89bbdadb46..5eb320266c 100644 --- a/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py +++ b/packages/flow/octobot_flow/logic/configuration/profile_data_factory.py @@ -2,11 +2,35 @@ import octobot_commons.profiles.profile_data as profile_data_import import octobot_commons.constants +import octobot_protocol.models as protocol_models import octobot_trading.enums as trading_enums +import octobot_trading.api.exchange as exchange_api +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_flow.entities -import tentacles.Meta.Keywords.scripting_library as scripting_library + +def profile_data_for_account( + account: protocol_models.Account, + exchange_account: protocol_models.ExchangeAccount, + exchange_config: protocol_models.ExchangeConfig, + trading_type: protocol_models.TradingType, + *, + is_simulated: bool, +) -> profile_data_import.ProfileData: + profile_data = profile_data_import.ProfileData( + exchanges=[ + profile_data_import.ExchangeData( + internal_name=exchange_config.exchange, + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, + exchange_account_id=exchange_account.remote_account_id or account.id, + sandboxed=exchange_config.sandboxed, + ) + ] + ) + profile_data.trader.enabled = not is_simulated + profile_data.trader_simulator.enabled = is_simulated + return profile_data def _tentacles_for_exchange_account_details( @@ -65,7 +89,10 @@ def infer_reference_market( if exchange_account_details.portfolio.unit: # portfolio unit can be used to define the reference market return exchange_account_details.portfolio.unit - return scripting_library.get_default_exchange_reference_market(exchange_account_details.exchange_details.internal_name) + if exchange_account_details.exchange_details.internal_name: + return exchange_api.get_default_exchange_reference_market( + exchange_account_details.exchange_details.internal_name + ) return octobot_commons.constants.DEFAULT_REFERENCE_MARKET def _get_crypto_currencies(symbols: set[str]) -> list[profile_data_import.CryptoCurrencyData]: diff --git a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py index 2a95cf6115..94245a02a2 100644 --- a/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py +++ b/packages/flow/octobot_flow/logic/dsl/dsl_action_execution_context.py @@ -1,10 +1,11 @@ import octobot_commons.dsl_interpreter import octobot_commons.errors import octobot_commons.logging -import octobot_commons.constants import octobot_trading.errors import octobot_trading.enums +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.logic.dsl.action_error_util @@ -138,6 +139,8 @@ async def _action_execution_error_handler_wrapper( if _should_postpone_recallable_trading_error(self, action): raise return _map_non_recallable_postpone_trading_error(action, err) + except copy_errors.OutdatedReferenceAccountError: + raise except Exception as err: # swallowed errors: warning: will stop the workflow octobot_commons.logging.get_logger("action_execution").exception( diff --git a/packages/flow/octobot_flow/logic/exchange/orders/__init__.py b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py new file mode 100644 index 0000000000..2a048c3f01 --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/orders/__init__.py @@ -0,0 +1,11 @@ +from octobot_flow.logic.exchange.orders.order_change_detection import ( + detect_changed_order_ids, + open_order_exchange_ids_from_open_orders, + open_order_exchange_ids_from_protocol_orders, +) + +__all__ = [ + "detect_changed_order_ids", + "open_order_exchange_ids_from_open_orders", + "open_order_exchange_ids_from_protocol_orders", +] diff --git a/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py new file mode 100644 index 0000000000..60bfa4985f --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/orders/order_change_detection.py @@ -0,0 +1,41 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + + +def detect_changed_order_ids( + previous_open_order_exchange_ids: set[str], + current_open_orders: list[dict], +) -> set[str]: + if not previous_open_order_exchange_ids: + return set() + current_open_order_exchange_ids = open_order_exchange_ids_from_open_orders(current_open_orders) + return previous_open_order_exchange_ids - current_open_order_exchange_ids + + +def open_order_exchange_ids_from_protocol_orders( + protocol_orders: list[protocol_models.Order] | None, +) -> set[str]: + if not protocol_orders: + return set() + return { + str(protocol_order.exchange_id) + for protocol_order in protocol_orders + if protocol_order.exchange_id + } + + +def open_order_exchange_ids_from_open_orders(open_orders: list[dict]) -> set[str]: + exchange_ids: set[str] = set() + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + exchange_id = inner_order.get(order_columns.EXCHANGE_ID.value) or inner_order.get( + order_columns.ID.value + ) + if exchange_id is not None: + exchange_ids.add(str(exchange_id)) + return exchange_ids diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py b/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py index 70957ed759..d51b37ff4b 100644 --- a/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py +++ b/packages/flow/octobot_flow/logic/exchange/simulator/__init__.py @@ -1,7 +1,9 @@ from octobot_flow.logic.exchange.simulator.simulated_exchange_account_resolver import SimulatedExchangeAccountResolver from octobot_flow.logic.exchange.simulator.simulated_price_events_factory import SimulatedPriceEventsFactory +from octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder import seed_simulated_portfolio __all__ = [ "SimulatedExchangeAccountResolver", "SimulatedPriceEventsFactory", + "seed_simulated_portfolio", ] diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py new file mode 100644 index 0000000000..b54da9b52e --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_order_fill_detector.py @@ -0,0 +1,61 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.logging as commons_logging +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + + +def is_simulated_order_filled(order_price: float, market_price: float, trigger_above: bool) -> bool: + if trigger_above: + return market_price >= order_price + return market_price <= order_price + + +def resolve_trigger_above(inner_order: dict) -> bool: + order_columns = trading_enums.ExchangeConstantsOrderColumns + if order_columns.TRIGGER_ABOVE.value in inner_order: + return bool(inner_order[order_columns.TRIGGER_ABOVE.value]) + order_side = inner_order.get(order_columns.SIDE.value) + if order_side == trading_enums.TradeOrderSide.SELL.value: + return True + return False + + +def resolve_simulated_open_orders( + open_orders: list[dict], + ticker_close_by_symbol: dict[str, float], +) -> list[dict]: + still_open_orders: list[dict] = [] + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + symbol = inner_order.get(order_columns.SYMBOL.value) + order_price = inner_order.get(order_columns.PRICE.value) + if symbol is None or order_price is None: + still_open_orders.append(open_order) + continue + if symbol not in ticker_close_by_symbol: + commons_logging.get_logger(__name__).warning( + "No ticker close for %s, keeping simulated order open", + symbol, + ) + still_open_orders.append(open_order) + continue + trigger_above = resolve_trigger_above(inner_order) + market_price = ticker_close_by_symbol[symbol] + if is_simulated_order_filled(float(order_price), market_price, trigger_above): + continue + still_open_orders.append(open_order) + return still_open_orders + + +def symbols_from_open_orders(open_orders: list[dict]) -> list[str]: + order_columns = trading_enums.ExchangeConstantsOrderColumns + symbols: set[str] = set() + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + symbol = inner_order.get(order_columns.SYMBOL.value) + if symbol is not None: + symbols.add(str(symbol)) + return sorted(symbols) diff --git a/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py new file mode 100644 index 0000000000..f6f5618b8f --- /dev/null +++ b/packages/flow/octobot_flow/logic/exchange/simulator/simulated_portfolio_seeder.py @@ -0,0 +1,22 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models + + +def seed_simulated_portfolio(exchange_manager, account: protocol_models.Account) -> None: + portfolio_content: dict[str, dict[str, float]] = {} + if account.assets: + for assets_for_trading_type in account.assets: + for detailed_asset in assets_for_trading_type.assets: + portfolio_content[detailed_asset.symbol] = { + commons_constants.PORTFOLIO_AVAILABLE: float(detailed_asset.available), + commons_constants.PORTFOLIO_TOTAL: float(detailed_asset.total), + } + if portfolio_content: + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio( + portfolio_content, + update_available_funds_from_open_orders=True, + ) diff --git a/packages/flow/octobot_flow/logic/global_view/__init__.py b/packages/flow/octobot_flow/logic/global_view/__init__.py new file mode 100644 index 0000000000..c78ffe6b00 --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/__init__.py @@ -0,0 +1,2 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. diff --git a/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py new file mode 100644 index 0000000000..7d2bf53512 --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/account_refresh_builder.py @@ -0,0 +1,36 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import typing + +import octobot_commons.timestamp_util as timestamp_util +import octobot_protocol.models as protocol_models + +import octobot_flow.entities + + +def build_updated_account( + account: protocol_models.Account, + assets: list[protocol_models.DetailedAssetsForTradingType], +) -> protocol_models.Account: + account_updates: dict[str, typing.Any] = { + "updated_at": timestamp_util.utc_now_datetime(), + } + if assets: + account_updates["assets"] = assets + return account.model_copy(update=account_updates) + + +def build_global_view_account_refresh_result( + user_id: str, + context: octobot_flow.entities.GlobalViewAccountContext, + exchange_refresh_result: octobot_flow.entities.ExchangeAccountRefreshResult, +) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + updated_account = build_updated_account(context.account, exchange_refresh_result.assets) + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=updated_account, + changed_order_ids=exchange_refresh_result.changed_order_ids, + open_orders=exchange_refresh_result.open_orders, + trades=exchange_refresh_result.trades, + positions=exchange_refresh_result.positions, + ) diff --git a/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py new file mode 100644 index 0000000000..2ac589515f --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/exchange_account_refresh.py @@ -0,0 +1,279 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import asyncio +import decimal + +import octobot_commons.constants as commons_constants +import octobot_commons.logging as octobot_commons_logging +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.errors as trading_errors +import octobot_trading.exchanges.util.exchange_util as exchange_util_module +import octobot_trading.personal_data.portfolios.portfolio_util as portfolio_util_module +import octobot_trading.personal_data.portfolios.protocol as portfolios_protocol + +import octobot_flow.entities +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module + +def _get_logger() -> octobot_commons_logging.BotLogger: + return octobot_commons_logging.get_logger("ExchangeAccountRefresh") + + +async def refresh_exchange_account( + exchange_manager, + trading_type: protocol_models.TradingType, + previous_open_order_exchange_ids: set[str], + *, + is_simulated: bool = False, + previous_open_orders: list[dict] | None = None, + fetch_open_orders: bool = True, + open_order_symbols: list[str] | None = None, +) -> octobot_flow.entities.ExchangeAccountRefreshResult: + await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) + + # Step: fetch balance and open orders from the exchange (real accounts only). + tickers: dict[str, dict] | None = None + if is_simulated: + if previous_open_orders is None: + raise ValueError("previous_open_orders is required for simulated account refresh.") + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_unit = trading_api.resolve_portfolio_valuation_unit(exchange_manager) + order_symbols = simulated_order_fill_detector_module.symbols_from_open_orders(previous_open_orders) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + symbols_to_fetch = sorted(set(order_symbols) | set(valuation_symbols)) + tickers = await _fetch_tickers(exchange_manager, symbols_to_fetch) + ticker_close_by_symbol = _ticker_close_by_symbol_from_tickers(tickers) + open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + previous_open_orders, + ticker_close_by_symbol, + ) + elif not fetch_open_orders: + await _fetch_and_apply_exchange_balance(exchange_manager) + open_orders = [] + else: + await _fetch_and_apply_exchange_balance(exchange_manager) + open_orders = await _fetch_open_orders_for_symbols( + exchange_manager, + open_order_symbols or [], + ) + trades: list[dict] = [] + positions: list[dict] = [] + + # Step: resolve valuation unit and portfolio total in that currency. + if not is_simulated: + valuation_unit = trading_api.resolve_portfolio_valuation_unit(exchange_manager) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is not None: + portfolio_manager.reference_market = valuation_unit + if is_simulated: + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + simulated_valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + await _refresh_portfolio_valuation( + exchange_manager, + valuation_unit, + tickers=tickers, + valuation_symbols=simulated_valuation_symbols, + ) + else: + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, portfolio_content, valuation_unit, + ) + tickers = await _fetch_tickers(exchange_manager, valuation_symbols) + await _refresh_portfolio_valuation( + exchange_manager, valuation_unit, tickers=tickers, valuation_symbols=valuation_symbols, + ) + # Step: build protocol assets (holdings only, no historical snapshot). + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + balance_summary = octobot_commons_logging.get_private_placeholder_if_necessary( + portfolio_util_module.get_balance_summary(portfolio_content, use_exchange_format=False) + ) + _get_logger().info( + "Fetched [%s] full [%s] portfolio: %s", + exchange_manager.exchange_name, + "simulated" if is_simulated else "real", + balance_summary, + ) + detailed_assets = portfolios_protocol.to_protocol_assets(portfolio_content) + assets_for_trading_type = [ + protocol_models.DetailedAssetsForTradingType( + trading_type=trading_type, + assets=detailed_assets, + ) + ] if detailed_assets else [] + + # Collect ticker close prices for the persisted latest-tickers cache. + ticker_closes = _ticker_close_by_symbol_from_tickers(tickers) if tickers else {} + + # Step: detect orders that disappeared since the previous refresh. + changed_order_ids = order_change_detection_module.detect_changed_order_ids( + previous_open_order_exchange_ids, + open_orders, + ) + return octobot_flow.entities.ExchangeAccountRefreshResult( + assets=assets_for_trading_type, + ticker_closes=ticker_closes, + valuation_unit=valuation_unit, + open_orders=open_orders, + trades=trades, + positions=positions, + changed_order_ids=changed_order_ids, + ) + + +async def _fetch_and_apply_exchange_balance(exchange_manager) -> None: + balance = await exchange_manager.exchange.get_balance() + non_empty_balance = portfolio_util_module.filter_empty_values(balance) + decimal_balance = portfolio_util_module.parse_decimal_portfolio(non_empty_balance) + await exchange_manager.exchange_personal_data.handle_portfolio_update( + decimal_balance, + should_notify=False, + ) + + +async def _fetch_tickers(exchange_manager, symbols: list[str]) -> dict[str, dict]: + if not symbols: + return {} + tickers_repository = tickers_repository_module.TickersRepository( + exchange_manager, + known_automations=[], + fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), + ) + return await tickers_repository.fetch_tickers(symbols) + + +def _ticker_close_by_symbol_from_tickers(tickers: dict[str, dict]) -> dict[str, float]: + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol: dict[str, float] = {} + for symbol, ticker in tickers.items(): + close_price = ticker.get(close_column) + if close_price is not None: + ticker_close_by_symbol[symbol] = float(close_price) + return ticker_close_by_symbol + + +async def _refresh_portfolio_valuation( + exchange_manager, + valuation_unit: str, + *, + tickers: dict[str, dict] | None = None, + valuation_symbols: list[str] | None = None, +) -> None: + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + if portfolio_manager is None: + return + if valuation_symbols is None: + portfolio_content = trading_api.get_portfolio(exchange_manager, as_decimal=False) + valuation_symbols = _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + valuation_unit, + ) + if tickers is None and valuation_symbols: + tickers = await _fetch_tickers(exchange_manager, valuation_symbols) + if tickers: + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + symbols_to_apply = valuation_symbols if valuation_symbols is not None else list(tickers) + for symbol in symbols_to_apply: + ticker = tickers.get(symbol) + if not ticker: + continue + close_price = ticker.get(close_column) + if close_price is None: + continue + mark_price = decimal.Decimal(str(close_price)) + portfolio_manager.portfolio_value_holder.value_converter.update_last_price(symbol, mark_price) + if exchange_manager.symbol_exists(symbol): + exchange_manager.get_symbol_data(symbol).handle_ticker_update(ticker) + portfolio_manager.portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values( + init_price_fetchers=False, + ) + + +def _valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content: dict, + valuation_unit: str, +) -> list[str]: + valuation_symbols: set[str] = set() + bridge_quote_currencies = [] + if valuation_unit in commons_constants.USD_LIKE_COINS: + bridge_quote_currencies = [ + quote_currency + for quote_currency in commons_constants.USD_LIKE_COINS + if quote_currency != valuation_unit + ] + for currency, symbol_balance in portfolio_content.items(): + total_holdings = float(symbol_balance.get(commons_constants.PORTFOLIO_TOTAL) or 0) + if total_holdings == 0 or currency == valuation_unit: + continue + direct_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + currency, + valuation_unit, + ) + if direct_symbol is not None: + valuation_symbols.add(direct_symbol) + continue + for bridge_quote in bridge_quote_currencies: + asset_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + currency, + bridge_quote, + ) + if asset_symbol is None: + continue + valuation_symbols.add(asset_symbol) + if bridge_quote != valuation_unit: + bridge_symbol, _is_reversed_bridge_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + bridge_quote, + valuation_unit, + ) + if bridge_symbol is not None: + valuation_symbols.add(bridge_symbol) + break + return list(valuation_symbols) + + +async def _get_open_orders_for_symbol(exchange_manager, symbol: str) -> list[dict]: + try: + return await exchange_manager.exchange.get_open_orders(symbol=symbol) + except trading_errors.UnSupportedSymbolError: + return [] + + +async def _fetch_open_orders_for_symbols(exchange_manager, symbols: list[str]) -> list[dict]: + if not symbols: + return [] + open_orders_by_symbol = await asyncio.gather(*[ + _get_open_orders_for_symbol(exchange_manager, symbol) + for symbol in symbols + ]) + orders_by_exchange_id: dict[str, dict] = {} + order_columns = trading_enums.ExchangeConstantsOrderColumns + for open_orders in open_orders_by_symbol: + for open_order in open_orders: + inner_order = open_order.get(trading_constants.STORAGE_ORIGIN_VALUE, open_order) + exchange_id = inner_order.get(order_columns.EXCHANGE_ID.value) or inner_order.get( + order_columns.ID.value + ) + if exchange_id is None: + continue + orders_by_exchange_id[str(exchange_id)] = open_order + return list(orders_by_exchange_id.values()) + + diff --git a/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py new file mode 100644 index 0000000000..aa3e4dd3fd --- /dev/null +++ b/packages/flow/octobot_flow/logic/global_view/global_view_persistence.py @@ -0,0 +1,23 @@ +# Drakkar-Software OctoBot-Flow +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module + + +def persist_global_view_refresh_result( + user_id: str, + account_id: str, + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, + *, + persist_open_orders: bool = False, +) -> None: + collection_providers.AccountProvider.instance().update_item(user_id, refresh_result.updated_account) + if persist_open_orders: + account_state_persistence_module.persist_account_trading_orders( + user_id, + account_id, + refresh_result.open_orders or [], + ) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/__init__.py b/packages/flow/octobot_flow/logic/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py new file mode 100644 index 0000000000..9e466d03bf --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/daily_price_cache_updater.py @@ -0,0 +1,387 @@ +import math +import time + +import octobot_commons.enums as commons_enums +import octobot_commons.constants as commons_constants +import octobot_commons.logging as commons_logging +import octobot_trading.api as trading_api +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types +import octobot_trading.constants as trading_constants +import octobot_trading.errors as trading_errors +import octobot_trading.exchanges.util.exchange_util as exchange_util + +import octobot_flow.constants as flow_constants + +logger = commons_logging.get_logger("DailyPriceCacheUpdater") + + +async def update_daily_prices( + exchange_manager, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + symbols: list[str], + data_root: str = None, +) -> None: + """ + Fetch daily OHLCV candles for the given reference symbols and merge close prices + into the persisted daily price cache under exchange-native fetch symbols. + """ + daily_prices = await trading_api.load_daily_prices( + exchange_name, exchange_type, sandboxed, data_root + ) + + for reference_symbol in symbols: + parsed = _parse_base_quote(reference_symbol) + if parsed is None: + continue + base_asset, quote = parsed + if base_asset in commons_constants.USD_LIKE_COINS: + continue + + sticky_fetch_symbol = trading_api.get_daily_close_source(daily_prices, base_asset) + if _is_daily_cache_up_to_date(daily_prices, reference_symbol): + continue + + since_ms = _compute_fetch_since_ms(daily_prices, reference_symbol) + candidates = _build_fetch_candidates( + exchange_manager, base_asset, quote, sticky_fetch_symbol, + ) + + fetch_result = await _fetch_daily_candles( + exchange_manager, since_ms, candidates, + ) + if fetch_result is None: + logger.info( + "Skipping daily price fetch for %s on %s: no supported market found", + reference_symbol, + exchange_name, + ) + continue + + candles, fetch_symbol, is_reversed = fetch_result + if sticky_fetch_symbol and fetch_symbol != sticky_fetch_symbol: + logger.info( + "Migrating daily closes for %s on %s from %s to %s", + base_asset, + exchange_name, + sticky_fetch_symbol, + fetch_symbol, + ) + await trading_api.rename_daily_closes_symbol( + exchange_name, exchange_type, sandboxed, + sticky_fetch_symbol, fetch_symbol, data_root, + ) + trading_api.move_daily_prices_symbol_in_memory( + daily_prices, sticky_fetch_symbol, fetch_symbol, + ) + elif fetch_symbol != reference_symbol: + logger.debug( + "Fetched %s for %s on %s", + fetch_symbol, + reference_symbol, + exchange_name, + ) + + closes_by_timestamp = _closes_from_candles(candles, is_reversed) + closes_by_timestamp = _filter_closes_for_merge( + daily_prices, fetch_symbol, closes_by_timestamp, + ) + if not closes_by_timestamp: + continue + + await trading_api.merge_daily_prices( + exchange_name, exchange_type, sandboxed, fetch_symbol, closes_by_timestamp, data_root, + ) + await trading_api.set_daily_close_source( + exchange_name, exchange_type, sandboxed, base_asset, fetch_symbol, data_root, + ) + trading_api.set_daily_close_source_in_memory(daily_prices, base_asset, fetch_symbol) + trading_api.merge_daily_prices_in_memory(daily_prices, fetch_symbol, closes_by_timestamp) + + +def _parse_base_quote(symbol: str) -> tuple[str, str] | None: + if "/" not in symbol: + return None + base_asset, quote = symbol.split("/", 1) + if not base_asset or not quote or base_asset == quote: + return None + return base_asset, quote + + +def _ordered_usd_like_quotes(quote: str) -> list[str]: + quotes = [] + if "USD" in commons_constants.USD_LIKE_COINS and quote != "USD": + quotes.append("USD") + for usd_like_quote in commons_constants.USD_LIKE_COINS: + if usd_like_quote == quote or usd_like_quote in quotes: + continue + quotes.append(usd_like_quote) + return quotes + + +def _is_reversed_fetch_symbol(base_asset: str, fetch_symbol: str) -> bool: + parsed = _parse_base_quote(fetch_symbol) + if parsed is None: + return False + fetch_base, _fetch_quote = parsed + return fetch_base != base_asset + + +def _build_fetch_candidates( + exchange_manager, + base_asset: str, + quote: str, + sticky_fetch_symbol: str | None, +) -> list[tuple[str, bool]]: + candidates: list[tuple[str, bool]] = [] + seen_fetch_symbols: set[str] = set() + + def add_candidate(fetch_symbol: str | None, is_reversed: bool) -> None: + if fetch_symbol is None or fetch_symbol in seen_fetch_symbols: + return + seen_fetch_symbols.add(fetch_symbol) + candidates.append((fetch_symbol, is_reversed)) + + if sticky_fetch_symbol: + add_candidate( + sticky_fetch_symbol, + _is_reversed_fetch_symbol(base_asset, sticky_fetch_symbol), + ) + + direct_symbol, is_reversed = exchange_util.get_associated_symbol( + exchange_manager, base_asset, quote, + ) + add_candidate(direct_symbol, is_reversed) + + if quote in commons_constants.USD_LIKE_COINS: + for alt_quote in _ordered_usd_like_quotes(quote): + alt_symbol, alt_is_reversed = exchange_util.get_associated_symbol( + exchange_manager, base_asset, alt_quote, + ) + add_candidate(alt_symbol, alt_is_reversed) + + return candidates + + +async def _get_historical_daily_candles( + exchange_manager, + fetch_symbol: str, + since_ms: int | None, +) -> list: + start_time_ms, end_time_ms = _compute_fetch_time_range_ms(since_ms) + candles = [] + async for batch in exchange_util.get_historical_ohlcv( + exchange_manager, + fetch_symbol, + commons_enums.TimeFrames.ONE_DAY, + start_time_ms, + end_time_ms, + ): + candles.extend(batch) + return candles + + +async def _get_symbol_daily_candles( + exchange_manager, + fetch_symbol: str, + since_ms: int | None, +) -> list: + fetch_kwargs = {} + if since_ms is not None: + fetch_kwargs["since"] = since_ms + return await exchange_manager.exchange.get_symbol_prices( + fetch_symbol, + commons_enums.TimeFrames.ONE_DAY, + **fetch_kwargs, + ) + + +def _supports_full_candle_history(exchange_name: str) -> bool: + return exchange_name in trading_constants.FULL_CANDLE_HISTORY_EXCHANGES + + +async def _fetch_daily_candles( + exchange_manager, + since_ms: int | None, + candidates: list[tuple[str, bool]], +) -> tuple[list, str, bool] | None: + exchange_name = exchange_manager.exchange_name + supports_full_history = _supports_full_candle_history(exchange_name) + for fetch_symbol, is_reversed in candidates: + if supports_full_history: + candles = await _try_full_history_daily_candles( + exchange_manager, exchange_name, fetch_symbol, since_ms, + ) + else: + candles = await _try_limited_history_daily_candles( + exchange_manager, exchange_name, fetch_symbol, since_ms, + ) + if candles: + return candles, fetch_symbol, is_reversed + return None + + +async def _try_full_history_daily_candles( + exchange_manager, + exchange_name: str, + fetch_symbol: str, + since_ms: int | None, +) -> list | None: + try: + candles = await _get_historical_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + logger.info( + "Daily candle historical fetch failed for %s on %s, retrying with get_symbol_prices: %s", + fetch_symbol, + exchange_name, + error, + ) + else: + if candles: + return candles + logger.info( + "Daily candle historical fetch returned empty for %s on %s, retrying with get_symbol_prices", + fetch_symbol, + exchange_name, + ) + + try: + return await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + logger.exception( + error, + True, + f"Failed to fetch daily candles for {fetch_symbol}: {error}", + ) + return None + + +async def _try_limited_history_daily_candles( + exchange_manager, + exchange_name: str, + fetch_symbol: str, + since_ms: int | None, +) -> list | None: + try: + candles = await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, since_ms, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as error: + if since_ms is not None: + logger.info( + "Daily candle incremental fetch failed for %s on %s, skipping fallback without since: %s", + fetch_symbol, + exchange_name, + error, + ) + return None + logger.info( + "Daily candle ideal fetch failed for %s on %s, retrying without since/limit: %s", + fetch_symbol, + exchange_name, + error, + ) + try: + candles = await _get_symbol_daily_candles( + exchange_manager, fetch_symbol, None, + ) + except trading_errors.UnSupportedSymbolError: + return None + except trading_errors.FailedRequest as fallback_error: + logger.error( + "Failed to fetch daily candles for %s on %s after fallback without since/limit: %s", + fetch_symbol, + exchange_name, + fallback_error, + ) + return None + if candles: + return candles + return None + + +def _closes_from_candles(candles: list, is_reversed: bool) -> dict[str, float]: + closes_by_timestamp = {} + for candle in candles: + day_ts = str(int(candle[0])) + close_price = float(candle[4]) + if is_reversed: + close_price = 1.0 / close_price + closes_by_timestamp[day_ts] = close_price + return closes_by_timestamp + + +def _filter_closes_for_merge( + daily_prices: daily_prices_cache_types.DailyPricesCache, + fetch_symbol: str, + closes_by_timestamp: dict[str, float], +) -> dict[str, float]: + oldest_cached = trading_api.get_oldest_daily_price_timestamp(daily_prices, fetch_symbol) + if oldest_cached is not None: + oldest_cached_int = int(oldest_cached) + return { + day_ts: close + for day_ts, close in closes_by_timestamp.items() + if int(day_ts) >= oldest_cached_int + } + lookback_floor = _utc_day_start(time.time()) - ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS * commons_constants.DAYS_TO_SECONDS + ) + lookback_floor_int = int(lookback_floor) + return { + day_ts: close + for day_ts, close in closes_by_timestamp.items() + if int(day_ts) >= lookback_floor_int + } + + +def _utc_day_start(timestamp: float) -> float: + """Return the UTC day-start (00:00:00) for the given unix timestamp.""" + return float(math.floor( + timestamp / commons_constants.DAYS_TO_SECONDS + ) * commons_constants.DAYS_TO_SECONDS) + + +def _compute_fetch_since_ms( + daily_prices: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> int | None: + newest_timestamp = trading_api.get_latest_daily_price_timestamp(daily_prices, symbol) + if newest_timestamp is None: + return None + + since_seconds = newest_timestamp - commons_constants.DAYS_TO_SECONDS + return int(since_seconds * 1000) + + +def _compute_fetch_time_range_ms(since_ms: int | None) -> tuple[int, int]: + end_time_ms = int(time.time() * commons_constants.MSECONDS_TO_SECONDS) + if since_ms is not None: + return since_ms, end_time_ms + lookback_ms = ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS + * commons_constants.DAYS_TO_SECONDS + * commons_constants.MSECONDS_TO_SECONDS + ) + return end_time_ms - lookback_ms, end_time_ms + + +def _is_daily_cache_up_to_date( + daily_prices: daily_prices_cache_types.DailyPricesCache, + symbol: str, +) -> bool: + latest_cached_timestamp = trading_api.get_latest_daily_price_timestamp(daily_prices, symbol) + if latest_cached_timestamp is None: + return False + return latest_cached_timestamp >= _utc_day_start(time.time()) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py b/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py new file mode 100644 index 0000000000..3466540d6c --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/portfolio_value_history.py @@ -0,0 +1,225 @@ +import copy +import datetime +import decimal +import math +import time + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types + + +def _timestamp_to_datetime(timestamp: float) -> datetime.datetime: + return datetime.datetime.fromtimestamp(timestamp, tz=datetime.timezone.utc) + + +def _resolve_asset_unit_price( + asset: str, + day_timestamp: float, + day_ts_str: str, + daily_prices: daily_prices_cache_types.DailyPricesCache, + latest_tickers: daily_prices_cache_types.LatestTickersCache, + reference_market: str, +) -> decimal.Decimal | None: + if asset == reference_market: + return decimal.Decimal(1) + if asset in commons_constants.USD_LIKE_COINS: + return decimal.Decimal(1) + + symbol = f"{asset}/{reference_market}" + exact_price = trading_api.get_daily_price(daily_prices, symbol, day_ts_str) + if exact_price is not None: + return decimal.Decimal(str(exact_price)) + + historical_price = trading_api.get_latest_daily_close_on_or_before( + daily_prices, symbol, day_timestamp, + ) + if historical_price is not None: + return decimal.Decimal(str(historical_price)) + + if trading_api.get_oldest_daily_price_timestamp(daily_prices, symbol) is not None: + return None + + ticker_price = trading_api.get_latest_ticker_close(latest_tickers, symbol) + if ticker_price is not None: + return decimal.Decimal(str(ticker_price)) + return None + + +def _value_asset_holding( + asset: str, + asset_total: decimal.Decimal, + day_timestamp: float, + day_ts_str: str, + daily_prices: daily_prices_cache_types.DailyPricesCache, + latest_tickers: daily_prices_cache_types.LatestTickersCache, + reference_market: str, +) -> decimal.Decimal: + unit_price = _resolve_asset_unit_price( + asset, + day_timestamp, + day_ts_str, + daily_prices, + latest_tickers, + reference_market, + ) + if unit_price is None: + return decimal.Decimal(0) + return asset_total * unit_price + + +def _collect_required_valuation_symbols( + daily_holdings: dict[float, dict[str, dict[str, decimal.Decimal]]], + reference_market: str, +) -> set[str]: + required_symbols: set[str] = set() + for holdings in daily_holdings.values(): + for asset, amounts in holdings.items(): + asset_total = amounts.get("total", decimal.Decimal(0)) + if asset_total == 0: + continue + if asset == reference_market or asset in commons_constants.USD_LIKE_COINS: + continue + required_symbols.add(f"{asset}/{reference_market}") + return required_symbols + + +def _earliest_valuation_timestamp( + daily_prices: daily_prices_cache_types.DailyPricesCache, + required_symbols: set[str], +) -> float: + if not required_symbols: + return 0.0 + oldest_timestamps = [] + for symbol in required_symbols: + oldest_timestamp = trading_api.get_oldest_daily_price_timestamp(daily_prices, symbol) + if oldest_timestamp is not None: + oldest_timestamps.append(oldest_timestamp) + if not oldest_timestamps: + return 0.0 + return max(oldest_timestamps) + + +def _utc_day_start(timestamp: float) -> float: + return float(math.floor( + timestamp / commons_constants.DAYS_TO_SECONDS + ) * commons_constants.DAYS_TO_SECONDS) + + +def _expand_sparse_daily_holdings( + daily_holdings: dict[float, dict[str, dict[str, decimal.Decimal]]], + start_day_timestamp: float, + end_day_timestamp: float, +) -> dict[float, dict[str, dict[str, decimal.Decimal]]]: + if start_day_timestamp > end_day_timestamp or not daily_holdings: + return {} + + sparse_days = sorted(daily_holdings) + dense_holdings: dict[float, dict[str, dict[str, decimal.Decimal]]] = {} + sparse_day_index = 0 + current_holdings = None + + day_timestamp = start_day_timestamp + while day_timestamp <= end_day_timestamp: + while ( + sparse_day_index < len(sparse_days) + and sparse_days[sparse_day_index] <= day_timestamp + ): + current_holdings = copy.deepcopy(daily_holdings[sparse_days[sparse_day_index]]) + sparse_day_index += 1 + if current_holdings is not None: + dense_holdings[day_timestamp] = current_holdings + day_timestamp += commons_constants.DAYS_TO_SECONDS + + return dense_holdings + + +def _build_day_assets_protocol( + day_assets: list[protocol_models.HistoricalAssetValue], +) -> list[protocol_models.HistoricalAssetsForTradingType] | None: + if not day_assets: + return None + return [ + protocol_models.HistoricalAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=day_assets, + ) + ] + + +def compute_daily_portfolio_values( + daily_holdings: dict[float, dict[str, dict[str, decimal.Decimal]]], + daily_prices: daily_prices_cache_types.DailyPricesCache, + latest_tickers: daily_prices_cache_types.LatestTickersCache, + reference_market: str = "USDT", +) -> list[protocol_models.PortfolioHistoricalValue]: + """ + Value daily portfolio holdings using daily price cache with historical + fallback, then latest ticker only when no historical closes exist. + + Returns one valued point per UTC calendar day within the candle coverage + window. Sparse trade-replay holdings are forward-filled on quiet days so + daily price moves are reflected even without transactions. + + Days before the global candle coverage window are omitted. Assets that + cannot be priced contribute value 0 but remain in the breakdown. + + Returns protocol models sorted ascending by timestamp. + """ + if not daily_holdings: + return [] + + required_symbols = _collect_required_valuation_symbols(daily_holdings, reference_market) + earliest_valuation_timestamp = _earliest_valuation_timestamp(daily_prices, required_symbols) + + end_day_timestamp = _utc_day_start(time.time()) + valuation_start_timestamp = earliest_valuation_timestamp + if valuation_start_timestamp == 0.0: + valuation_start_timestamp = min(daily_holdings) + valuation_start_timestamp = _utc_day_start(valuation_start_timestamp) + dense_holdings = _expand_sparse_daily_holdings( + daily_holdings, + valuation_start_timestamp, + end_day_timestamp, + ) + + valued_days: list[protocol_models.PortfolioHistoricalValue] = [] + for day_timestamp in sorted(dense_holdings): + holdings = dense_holdings[day_timestamp] + total_value = decimal.Decimal(0) + day_assets: list[protocol_models.HistoricalAssetValue] = [] + day_ts_str = str(int(day_timestamp)) + + for asset, amounts in holdings.items(): + asset_total = amounts.get("total", decimal.Decimal(0)) + if asset_total == 0: + continue + + asset_value = _value_asset_holding( + asset, + asset_total, + day_timestamp, + day_ts_str, + daily_prices, + latest_tickers, + reference_market, + ) + total_value += asset_value + day_assets.append( + protocol_models.HistoricalAssetValue( + symbol=asset, + holdings=float(asset_total), + value=float(asset_value), + ) + ) + + valued_days.append( + protocol_models.PortfolioHistoricalValue( + timestamp=_timestamp_to_datetime(day_timestamp), + total=float(total_value), + assets=_build_day_assets_protocol(day_assets), + ) + ) + + return valued_days diff --git a/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py b/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py new file mode 100644 index 0000000000..1dad618fa7 --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/trade_symbols_discovery.py @@ -0,0 +1,177 @@ +import octobot_commons.constants as commons_constants +import octobot_commons.logging as commons_logging +import octobot_commons.symbols.symbol_util as symbol_util +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_util as exchange_util_module + +logger = commons_logging.get_logger("PortfolioHistoryJob") + +DEFAULT_MIN_HOLDINGS_THRESHOLD = 1e-8 + + +def discover_trade_symbols( + exchange_manager, + *, + seed_symbols: list[str], + account: protocol_models.Account, + account_trading: protocol_models.AccountTrading | None, + fresh_transactions: list[dict], + reference_market: str, + min_holdings_threshold: float = DEFAULT_MIN_HOLDINGS_THRESHOLD, +) -> list[str]: + discovered_symbols: set[str] = set(seed_symbols or []) + + if account_trading is not None: + _add_symbols_from_persisted_trades(discovered_symbols, account_trading) + _add_symbols_from_persisted_transactions( + discovered_symbols, + exchange_manager, + account_trading, + reference_market, + ) + + _add_symbols_from_fresh_transactions( + discovered_symbols, + exchange_manager, + fresh_transactions, + reference_market, + ) + _add_symbols_from_portfolio_holdings( + discovered_symbols, + exchange_manager, + account, + reference_market, + min_holdings_threshold, + ) + return sorted(discovered_symbols) + + +def trade_confirmed_symbols_from_fetched_trades(fetched_trades: list[dict]) -> set[str]: + order_columns = trading_enums.ExchangeConstantsOrderColumns + trade_confirmed_symbols: set[str] = set() + for trade in fetched_trades: + trade_symbol = trade.get(order_columns.SYMBOL.value) + if trade_symbol: + trade_confirmed_symbols.add(trade_symbol) + return trade_confirmed_symbols + + +def persist_trade_confirmed_symbols_to_exchange_config( + wallet_id: str, + exchange_config: protocol_models.ExchangeConfig, + trade_confirmed_symbols: set[str], +) -> list[str]: + existing_config_symbols = set(exchange_config.historical_trade_symbols or []) + new_trade_symbols = sorted(trade_confirmed_symbols - existing_config_symbols) + if not new_trade_symbols: + return [] + + account_provider = collection_providers.AccountProvider.instance() + updated_config = account_provider.get_exchange_config(wallet_id, exchange_config.id) + updated_config.historical_trade_symbols = sorted( + existing_config_symbols | trade_confirmed_symbols + ) + account_provider.update_exchange_config(wallet_id, updated_config) + return new_trade_symbols + + +def _add_symbols_from_persisted_trades( + discovered_symbols: set[str], + account_trading: protocol_models.AccountTrading, +) -> None: + for trade in account_trading.trades or []: + if trade.symbol: + discovered_symbols.add(trade.symbol) + + +def _add_symbols_from_persisted_transactions( + discovered_symbols: set[str], + exchange_manager, + account_trading: protocol_models.AccountTrading, + reference_market: str, +) -> None: + for transaction in account_trading.transactions or []: + _add_market_pair_for_currency( + discovered_symbols, + exchange_manager, + transaction.asset, + reference_market, + ) + + +def _add_symbols_from_fresh_transactions( + discovered_symbols: set[str], + exchange_manager, + fresh_transactions: list[dict], + reference_market: str, +) -> None: + transaction_columns = trading_enums.ExchangeConstantsTransactionColumns + for transaction in fresh_transactions: + transaction_currency = transaction.get(transaction_columns.CURRENCY.value) + _add_market_pair_for_currency( + discovered_symbols, + exchange_manager, + transaction_currency, + reference_market, + ) + + +def _add_symbols_from_portfolio_holdings( + discovered_symbols: set[str], + exchange_manager, + account: protocol_models.Account, + reference_market: str, + min_holdings_threshold: float, +) -> None: + portfolio_content = _portfolio_content_from_account(account, min_holdings_threshold) + if not portfolio_content: + return + for base_currency in portfolio_content: + _add_market_pair_for_currency( + discovered_symbols, + exchange_manager, + base_currency, + reference_market, + ) + + +def _portfolio_content_from_account( + account: protocol_models.Account, + min_holdings_threshold: float, +) -> dict[str, dict[str, float]]: + portfolio_content: dict[str, dict[str, float]] = {} + if not account.assets: + return portfolio_content + for assets_for_trading_type in account.assets: + for asset in assets_for_trading_type.assets or []: + asset_total = float(asset.total or 0) + if abs(asset_total) <= min_holdings_threshold: + continue + portfolio_content[asset.symbol] = { + commons_constants.PORTFOLIO_TOTAL: asset_total, + commons_constants.PORTFOLIO_AVAILABLE: float(asset.available or asset.total or 0), + } + return portfolio_content + + +def _add_market_pair_for_currency( + discovered_symbols: set[str], + exchange_manager, + currency: str | None, + reference_market: str, +) -> None: + if not currency or currency == reference_market or currency in commons_constants.USD_LIKE_COINS: + return + direct_symbol, _is_reversed_symbol = exchange_util_module.get_associated_symbol( + exchange_manager, + currency, + reference_market, + ) + if direct_symbol is not None: + discovered_symbols.add(direct_symbol) + return + merged_symbol = symbol_util.merge_currencies(currency, reference_market) + if exchange_manager.symbol_exists(merged_symbol): + discovered_symbols.add(merged_symbol) diff --git a/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py b/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py new file mode 100644 index 0000000000..07988eaefa --- /dev/null +++ b/packages/flow/octobot_flow/logic/portfolio_history/trading_history_merge.py @@ -0,0 +1,52 @@ +import datetime + +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.personal_data.trades.protocol as trades_protocol +import octobot_trading.personal_data.trades.trades_util as trades_util +import octobot_trading.personal_data.transactions.protocol as transactions_protocol +import octobot_trading.personal_data.transactions.transactions_util as transactions_util + + +def merge_and_persist_trading_history( + wallet_id: str, + account_id: str, + new_trades: list[dict], + new_transactions: list[dict], +) -> None: + """ + Merge new trades and transactions into AccountTrading without touching + orders or positions. + """ + trading_state = collection_providers.AccountTradingProvider.instance().load_state( + wallet_id, + account_id, + ) + account_trading = trading_state.account_trading + + # Merge trades. + existing_trade_dicts = [ + trades_protocol.exchange_columns_dict_from_protocol_trade(protocol_trade) + for protocol_trade in (account_trading.trades or []) + ] + merged_trade_dicts = trades_util.merge_trades_deduped(existing_trade_dicts, new_trades) + account_trading.trades = [ + trades_protocol.to_protocol_trade(trade_dict) for trade_dict in merged_trade_dicts + ] or None + + # Merge transactions. + existing_tx_dicts = [ + transactions_protocol.to_exchange_columns_dict(protocol_tx) + for protocol_tx in (account_trading.transactions or []) + ] + merged_tx_dicts = transactions_util.merge_transactions_deduped(existing_tx_dicts, new_transactions) + account_trading.transactions = [ + transactions_protocol.to_protocol_transaction(tx_dict) for tx_dict in merged_tx_dicts + ] or None + + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + collection_providers.AccountTradingProvider.instance().save_state( + wallet_id, + account_id, + trading_state, + ) diff --git a/packages/flow/octobot_flow/parsers/actions_dag_parser.py b/packages/flow/octobot_flow/parsers/actions_dag_parser.py index 823bd785ad..0ddf9206b2 100644 --- a/packages/flow/octobot_flow/parsers/actions_dag_parser.py +++ b/packages/flow/octobot_flow/parsers/actions_dag_parser.py @@ -17,11 +17,30 @@ import octobot_flow.entities import octobot_flow.enums -import tentacles.Trading.Mode.trading_view_signals_trading_mode.actions_params as actions_params -import tentacles.Trading.Mode.trading_view_signals_trading_mode.trading_view_signals_trading as trading_view_signals_trading -import tentacles.Trading.Mode.trading_view_signals_trading_mode.tradingview_signal_to_dsl_translator as tradingview_signal_to_dsl_translator + +def _actions_params(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.actions_params as actions_params + + return actions_params + + +def _trading_view_signals_trading(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.trading_view_signals_trading as trading_view_signals_trading + + return trading_view_signals_trading + + +def _tradingview_signal_to_dsl_translator(): + # avoid hard tentacles dependencies + import tentacles.Trading.Mode.trading_view_signals_trading_mode.tradingview_signal_to_dsl_translator as tradingview_signal_to_dsl_translator + + return tradingview_signal_to_dsl_translator + def key_val_to_dict(key_val: str) -> dict: + trading_view_signals_trading = _trading_view_signals_trading() return trading_view_signals_trading.TradingViewSignalsTradingMode.parse_signal_data(key_val, None, None, None, []) @@ -429,6 +448,8 @@ def _create_generic_action( ) def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + trading_view_signals_trading = _trading_view_signals_trading() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode self._ensure_params( ["ORDER_SYMBOL", "ORDER_AMOUNT", "ORDER_TYPE"], "trade", @@ -438,57 +459,60 @@ def _create_order_action(self, index: int) -> octobot_flow.entities.AbstractActi signal = self.params.ORDER_SIDE.lower() elif parsed_symbol.base == self.params.BLOCKCHAIN_FROM_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_TO_ASSET: # type: ignore # sell the first blockchain asset to get the second one - signal = trading_view_signals_trading.TradingViewSignalsTradingMode.SELL_SIGNAL + signal = tv_trading_mode.SELL_SIGNAL elif parsed_symbol.base == self.params.BLOCKCHAIN_TO_ASSET and parsed_symbol.quote == self.params.BLOCKCHAIN_FROM_ASSET: # type: ignore # buy the second blockchain asset to get the first one - signal = trading_view_signals_trading.TradingViewSignalsTradingMode.BUY_SIGNAL + signal = tv_trading_mode.BUY_SIGNAL else: raise octobot_flow.errors.InvalidAutomationActionError( f"Invalid order symbol: {self.params.ORDER_SYMBOL}: symbol must contain the 2 " f"blockchain assets to determine the side of the order" ) order_details = { - trading_view_signals_trading.TradingViewSignalsTradingMode.EXCHANGE_KEY: self.params.get_exchange_internal_name(), - trading_view_signals_trading.TradingViewSignalsTradingMode.SYMBOL_KEY: self.params.ORDER_SYMBOL, - trading_view_signals_trading.TradingViewSignalsTradingMode.VOLUME_KEY: self.params.ORDER_AMOUNT, - trading_view_signals_trading.TradingViewSignalsTradingMode.ORDER_TYPE_SIGNAL: self.params.ORDER_TYPE, + tv_trading_mode.EXCHANGE_KEY: self.params.get_exchange_internal_name(), + tv_trading_mode.SYMBOL_KEY: self.params.ORDER_SYMBOL, + tv_trading_mode.VOLUME_KEY: self.params.ORDER_AMOUNT, + tv_trading_mode.ORDER_TYPE_SIGNAL: self.params.ORDER_TYPE, } if self.params.ORDER_PRICE: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.PRICE_KEY] = self.params.ORDER_PRICE + order_details[tv_trading_mode.PRICE_KEY] = self.params.ORDER_PRICE if self.params.ORDER_STOP_PRICE: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.STOP_PRICE_KEY] = self.params.ORDER_STOP_PRICE + order_details[tv_trading_mode.STOP_PRICE_KEY] = self.params.ORDER_STOP_PRICE if self.params.ORDER_TAG: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.TAG_KEY] = self.params.ORDER_TAG + order_details[tv_trading_mode.TAG_KEY] = self.params.ORDER_TAG if self.params.ORDER_REDUCE_ONLY: - order_details[trading_view_signals_trading.TradingViewSignalsTradingMode.REDUCE_ONLY_KEY] = self.params.ORDER_REDUCE_ONLY + order_details[tv_trading_mode.REDUCE_ONLY_KEY] = self.params.ORDER_REDUCE_ONLY if self.params.ORDER_EXTRA_PARAMS: for extra_param, value in self.params.ORDER_EXTRA_PARAMS.items(): - order_details[f"{trading_view_signals_trading.TradingViewSignalsTradingMode.PARAM_PREFIX_KEY}{extra_param}"] = value + order_details[f"{tv_trading_mode.PARAM_PREFIX_KEY}{extra_param}"] = value return self.create_dsl_script_from_tv_format_action_details( f"action_trade_{index}", signal, order_details, ) def _create_cancel_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["ORDER_SYMBOL"], "cancel", ) cancel_details = { - trading_view_signals_trading.TradingViewSignalsTradingMode.SYMBOL_KEY: self.params.ORDER_SYMBOL, + tv_trading_mode.SYMBOL_KEY: self.params.ORDER_SYMBOL, } if self.params.ORDER_SIDE: - cancel_details[trading_view_signals_trading.TradingViewSignalsTradingMode.SIDE_PARAM_KEY] = self.params.ORDER_SIDE.lower() + cancel_details[tv_trading_mode.SIDE_PARAM_KEY] = self.params.ORDER_SIDE.lower() if self.params.ORDER_TAG: - cancel_details[trading_view_signals_trading.TradingViewSignalsTradingMode.TAG_KEY] = self.params.ORDER_TAG + cancel_details[tv_trading_mode.TAG_KEY] = self.params.ORDER_TAG return self.create_dsl_script_from_tv_format_action_details( f"action_cancel_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.CANCEL_SIGNAL, + tv_trading_mode.CANCEL_SIGNAL, cancel_details, ) def _create_withdraw_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_TO_ASSET", "BLOCKCHAIN_TO", "BLOCKCHAIN_TO_ADDRESS"], "withdraw", @@ -502,13 +526,15 @@ def _create_withdraw_action( withdraw_details.amount = self.params.BLOCKCHAIN_TO_AMOUNT return self.create_dsl_script_from_tv_format_action_details( f"action_withdraw_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.WITHDRAW_FUNDS_SIGNAL, + tv_trading_mode.WITHDRAW_FUNDS_SIGNAL, dataclasses.asdict(withdraw_details), ) def _create_deposit_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM_AMOUNT", "BLOCKCHAIN_FROM", "EXCHANGE_TO"], "deposit", @@ -522,7 +548,7 @@ def _create_deposit_action( ) return self.create_dsl_script_from_tv_format_action_details( f"action_deposit_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.TRANSFER_FUNDS_SIGNAL, + tv_trading_mode.TRANSFER_FUNDS_SIGNAL, dataclasses.asdict(deposit_details), ) @@ -531,6 +557,7 @@ def _wallet_init_details_for_translate( *, close_wallet_override: typing.Optional[bool] = None, ) -> dict: + actions_params = _actions_params() self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM"], "blockchain_wallet_init", @@ -555,9 +582,12 @@ def _wallet_init_details_for_translate( ) def _translate_blockchain_wallet_init_signal(self, details: dict) -> str: + trading_view_signals_trading = _trading_view_signals_trading() + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode parsed_signal = { - trading_view_signals_trading.TradingViewSignalsTradingMode.SIGNAL_KEY: - trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL, + tv_trading_mode.SIGNAL_KEY: + tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL, **details, } dsl_script = tradingview_signal_to_dsl_translator.TradingViewSignalToDSLTranslator.translate_signal( @@ -565,7 +595,7 @@ def _translate_blockchain_wallet_init_signal(self, details: dict) -> str: ) if dsl_script == tradingview_signal_to_dsl_translator.UNKNOWN_SIGNAL_RESULT: raise octobot_flow.errors.InvalidAutomationActionError( - f"Invalid signal: {trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL}" + f"Invalid signal: {tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL}" f"({details})" ) return dsl_script @@ -593,16 +623,19 @@ def _wrap_dsl_script_with_wallet_cleanup_if_error( ) def _create_blockchain_wallet_init_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode blockchain_wallet_init_details = self._wallet_init_details_for_translate() return self.create_dsl_script_from_tv_format_action_details( f"action_blockchain_wallet_init_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.BLOCKCHAIN_WALLET_INIT_SIGNAL, + tv_trading_mode.BLOCKCHAIN_WALLET_INIT_SIGNAL, blockchain_wallet_init_details, ) def _create_transfer_action( self, index: int ) -> octobot_flow.entities.AbstractActionDetails: + actions_params = _actions_params() + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode self._ensure_params( ["BLOCKCHAIN_FROM_ASSET", "BLOCKCHAIN_FROM_AMOUNT", "BLOCKCHAIN_FROM", "BLOCKCHAIN_TO_ADDRESS"], "transfer", @@ -615,7 +648,7 @@ def _create_transfer_action( ) return self.create_dsl_script_from_tv_format_action_details( f"action_transfer_{index}", - trading_view_signals_trading.TradingViewSignalsTradingMode.TRANSFER_FUNDS_SIGNAL, + tv_trading_mode.TRANSFER_FUNDS_SIGNAL, dataclasses.asdict(transfer_details), ) @@ -656,6 +689,7 @@ def _create_loop_until_order_closed_action(self, index: int) -> octobot_flow.ent return self._create_dsl_action_with_dependencies_if_any(action_id, dsl_script, params) def _create_loop_until_blockchain_balance_action(self, index: int) -> octobot_flow.entities.AbstractActionDetails: + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() loop_interval, loop_timeout, loop_max_attempts = self._get_loop_params() amount, asset = self.params.BLOCKCHAIN_BALANCE_AMOUNT, self.params.BLOCKCHAIN_BALANCE_ASSET if not amount or not asset: @@ -758,6 +792,7 @@ def _collect_dependency_refs_from_details( Find dependency::... references in string param values. Returns (dsl_parameter_name, dependency_action_id, result_path_keys, source_literal). """ + tv_trading_mode = _trading_view_signals_trading().TradingViewSignalsTradingMode refs: list[tuple[str, str, tuple[str, ...], str]] = [] for key, value in details.items(): if isinstance(value, dict): @@ -769,7 +804,7 @@ def _collect_dependency_refs_from_details( continue dep_action_id, result_path = parsed dsl_key = ( - trading_view_signals_trading.TradingViewSignalsTradingMode.TRADINGVIEW_TO_DSL_PARAM.get( + tv_trading_mode.TRADINGVIEW_TO_DSL_PARAM.get( key, key.lower() if isinstance(key, str) else str(key).lower() ) ) @@ -807,8 +842,11 @@ def _inject_dependency_placeholders_in_dsl_script( def create_dsl_script_from_tv_format_action_details( self, action_id: str, signal: str, details: dict ) -> octobot_flow.entities.DSLScriptActionDetails: + trading_view_signals_trading = _trading_view_signals_trading() + tradingview_signal_to_dsl_translator = _tradingview_signal_to_dsl_translator() + tv_trading_mode = trading_view_signals_trading.TradingViewSignalsTradingMode dsl_script = tradingview_signal_to_dsl_translator.TradingViewSignalToDSLTranslator.translate_signal( - {**{trading_view_signals_trading.TradingViewSignalsTradingMode.SIGNAL_KEY: signal}, **details} + {**{tv_trading_mode.SIGNAL_KEY: signal}, **details} ) if dsl_script == tradingview_signal_to_dsl_translator.UNKNOWN_SIGNAL_RESULT: raise octobot_flow.errors.InvalidAutomationActionError( diff --git a/packages/flow/octobot_flow/parsers/automation_state_reader.py b/packages/flow/octobot_flow/parsers/automation_state_reader.py index b0760f7abb..540ed518f4 100644 --- a/packages/flow/octobot_flow/parsers/automation_state_reader.py +++ b/packages/flow/octobot_flow/parsers/automation_state_reader.py @@ -1,6 +1,5 @@ import octobot_flow.entities import octobot_flow.logic.configuration -import octobot_flow.logic.dsl class AutomationStateReader: @@ -8,6 +7,9 @@ def __init__(self, state: octobot_flow.entities.AutomationState): self.state: octobot_flow.entities.AutomationState = state def get_automation_copied_strategy_ids(self) -> list[str]: + # avoid hard tentacles dependencies + import octobot_flow.logic.dsl + to_execute_actions = self.state.automation.actions_dag.get_executable_actions() self._resolve_dsl_scripts_for_actions(to_execute_actions) minimal_profile_data = octobot_flow.logic.configuration.create_profile_data( diff --git a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py index ba1b034ceb..37bbc78ff2 100644 --- a/packages/flow/octobot_flow/repositories/exchange/orders_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/orders_repository.py @@ -1,11 +1,9 @@ import typing import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import -import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools_import import octobot_trading.constants as trading_constants import octobot_trading.enums as trading_enums import octobot_trading.storage as orders_storage -import octobot_trading.constants as trading_constants import octobot_trading.personal_data as trading_personal_data @@ -22,7 +20,7 @@ async def fetch_open_orders( ) open_orders = await updater.fetch_open_orders(symbols) return [ - exchanges_test_tools_import.parse_order_into_dict( + trading_personal_data.OrdersUpdater.ensure_parsing( self.exchange_manager, order, True, ignore_unsupported_orders ) for order in open_orders diff --git a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py index f8f73e90d1..9a58b3855b 100644 --- a/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/tickers_repository.py @@ -2,14 +2,25 @@ import octobot_trading.exchange_data import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_import +import octobot_flow.entities import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import import octobot_trading.constants as trading_constants class TickersRepository(base_exchange_repository_import.BaseExchangeRepository): + @classmethod + async def ensure_temporary_ticker_channel(cls, exchange_manager) -> None: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [octobot_trading.exchange_data.TickerUpdater], + start_producers=False, + ) + async def fetch_tickers(self, symbols: typing.Optional[list[str]]) -> dict[str, dict]: updater = typing.cast( octobot_trading.exchange_data.TickerUpdater, @@ -17,6 +28,28 @@ async def fetch_tickers(self, symbols: typing.Optional[list[str]]) -> dict[str, ) return await updater.fetch_all_tickers(symbols) + @classmethod + async def fetch_ticker_close_by_symbol( + cls, + exchange_manager, + symbols: list[str], + ) -> dict[str, float]: + if not symbols: + return {} + tickers_repository = cls( + exchange_manager, + known_automations=[], + fetched_exchange_data=octobot_flow.entities.FetchedExchangeData(), + ) + tickers = await tickers_repository.fetch_tickers(symbols) + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol: dict[str, float] = {} + for symbol, ticker in tickers.items(): + close_price = ticker.get(close_column) + if close_price is not None: + ticker_close_by_symbol[symbol] = float(close_price) + return ticker_close_by_symbol + @staticmethod def get_cached_market_price(exchange_internal_name, exchange_type, sandboxed: bool, symbol: str) -> float: try: diff --git a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py index 8254ee9031..234db69578 100644 --- a/packages/flow/octobot_flow/repositories/exchange/trades_repository.py +++ b/packages/flow/octobot_flow/repositories/exchange/trades_repository.py @@ -1,12 +1,223 @@ +import dataclasses +import typing + +import octobot_commons.logging as commons_logging +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.personal_data as trading_personal_data + import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import -import octobot_trading.util.test_tools.exchanges_test_tools as exchanges_test_tools_import + +logger = commons_logging.get_logger("PortfolioHistoryJob") + +_order_columns = trading_enums.ExchangeConstantsOrderColumns + + +@dataclasses.dataclass(frozen=True, slots=True) +class _TradeFetchContext: + exchange_name: str + account_id: str + exchange_config_id: str + exchange_config_name: str + + @property + def config_label(self) -> str: + return self.exchange_config_name or self.exchange_config_id + + +def _raw_trade_symbol(raw_trade: dict) -> str | None: + trade_symbol = raw_trade.get(_order_columns.SYMBOL.value) + if trade_symbol is None: + return None + return str(trade_symbol) + + +def _is_live_market_trade(exchange_manager, raw_trade: dict) -> bool: + trade_symbol = _raw_trade_symbol(raw_trade) + if not trade_symbol: + return False + return exchange_manager.symbol_exists(trade_symbol) + + +def _parse_raw_trades( + exchange_manager, + raw_trades: list[dict], +) -> tuple[list[dict], set[str], int]: + parsed_trades: list[dict] = [] + skipped_symbols: set[str] = set() + skipped_trade_count = 0 + for raw_trade in raw_trades: + if not _is_live_market_trade(exchange_manager, raw_trade): + skipped_trade_count += 1 + trade_symbol = _raw_trade_symbol(raw_trade) + if trade_symbol: + skipped_symbols.add(trade_symbol) + continue + if parsed_trade := trading_personal_data.TradesUpdater.ensure_parsing( + exchange_manager, + raw_trade, + ): + parsed_trades.append(parsed_trade) + return parsed_trades, skipped_symbols, skipped_trade_count + + +def _log_skipped_delisted_trades( + skipped_trade_count: int, + skipped_symbols: set[str], + context: _TradeFetchContext, +) -> None: + if not skipped_trade_count: + return + logger.info( + "Skipped %d trades on delisted/unknown markets before parsing for %s account %s: %s", + skipped_trade_count, + context.exchange_name, + context.account_id, + ", ".join(sorted(skipped_symbols)), + ) + + +def _log_new_symbol_fetches( + symbols: list[str], + existing_config_symbols: set[str], + context: _TradeFetchContext, +) -> None: + for trading_symbol in symbols: + if trading_symbol not in existing_config_symbols: + logger.info( + "Fetching trade history for new symbol %s on %s " + "(account %s, config %s)", + trading_symbol, + context.exchange_name, + context.account_id, + context.config_label, + ) + + +def _log_fetched_trade_count( + trading_symbol: str, + count: int, + context: _TradeFetchContext, +) -> None: + logger.info( + "Fetched %d trades for %s on %s (account %s, config %s)", + count, + trading_symbol, + context.exchange_name, + context.account_id, + context.config_label, + ) class TradesRepository(base_exchange_repository_import.BaseExchangeRepository): + @classmethod + async def ensure_temporary_trades_channel(cls, exchange_manager) -> None: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) + async def fetch_trades(self, symbols: list[str]) -> list[dict]: if not symbols: return [] - return await exchanges_test_tools_import.get_trades( - self.exchange_manager, None, symbols=symbols + raw_trades = await self._fetch_raw_trades(symbols) + parsed_trades, _, _ = _parse_raw_trades(self.exchange_manager, raw_trades) + return parsed_trades + + async def fetch_trades_paginated( + self, + symbols: list[str], + *, + existing_config_symbols: set[str], + exchange_name: str, + account_id: str, + exchange_config_id: str, + exchange_config_name: str, + ) -> list[dict]: + if not symbols: + return [] + + context = _TradeFetchContext( + exchange_name=exchange_name, + account_id=account_id, + exchange_config_id=exchange_config_id, + exchange_config_name=exchange_config_name, ) + + if self.exchange_manager.exchange.get_option_value( + trading_enums.ExchangeClientOptions.MY_TRADES_SYMBOL_FILTER_IS_CLIENT_SIDE + ): + return await self._fetch_all_trades_account_wide( + symbols, + existing_config_symbols=existing_config_symbols, + context=context, + ) + + _log_new_symbol_fetches(symbols, existing_config_symbols, context) + parsed_trades = await self._fetch_and_parse_raw_trades(symbols, context) + self._log_fetched_trade_counts_per_symbol(parsed_trades, symbols, context) + return parsed_trades + + def _get_trades_updater(self) -> trading_personal_data.TradesUpdater: + return typing.cast( + trading_personal_data.TradesUpdater, + self.get_channel_updater(trading_constants.TRADES_CHANNEL), + ) + + async def _fetch_raw_trades( + self, + symbols: list[str], + *, + exhaust_history: bool = False, + ) -> list[dict]: + return await self._get_trades_updater().fetch_trades( + symbols, + exhaust_history=exhaust_history, + ) + + async def _fetch_all_trades_account_wide( + self, + symbols: list[str], + *, + existing_config_symbols: set[str], + context: _TradeFetchContext, + ) -> list[dict]: + _log_new_symbol_fetches(symbols, existing_config_symbols, context) + parsed_trades = await self._fetch_and_parse_raw_trades([], context) + self._log_fetched_trade_counts_per_symbol(parsed_trades, symbols, context) + return parsed_trades + + async def _fetch_and_parse_raw_trades( + self, + symbols: list[str], + context: _TradeFetchContext, + ) -> list[dict]: + raw_trades = await self._fetch_raw_trades(symbols, exhaust_history=True) + parsed_trades, skipped_symbols, skipped_trade_count = _parse_raw_trades( + self.exchange_manager, + raw_trades, + ) + _log_skipped_delisted_trades(skipped_trade_count, skipped_symbols, context) + return parsed_trades + + def _log_fetched_trade_counts_per_symbol( + self, + parsed_trades: list[dict], + symbols: list[str], + context: _TradeFetchContext, + ) -> None: + trades_by_symbol: dict[str, int] = {} + for trade in parsed_trades: + trade_symbol = trade.get(_order_columns.SYMBOL.value) + if trade_symbol: + trades_by_symbol[trade_symbol] = trades_by_symbol.get(trade_symbol, 0) + 1 + for trading_symbol in symbols: + _log_fetched_trade_count( + trading_symbol, + trades_by_symbol.get(trading_symbol, 0), + context, + ) diff --git a/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py b/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py new file mode 100644 index 0000000000..cbae6e9c49 --- /dev/null +++ b/packages/flow/octobot_flow/repositories/exchange/transactions_repository.py @@ -0,0 +1,49 @@ +import octobot_commons.logging as commons_logging +import octobot_trading.errors as trading_errors + +import octobot_flow.repositories.exchange.base_exchange_repository as base_exchange_repository_import + +logger = commons_logging.get_logger("TransactionsRepository") + + +class TransactionsRepository(base_exchange_repository_import.BaseExchangeRepository): + + async def fetch_deposits( + self, + since: int = None, + limit: int = None, + currencies: list[str] = None, + ) -> list[dict]: + try: + return await self.exchange_manager.exchange.get_deposits( + since=since, limit=limit, currencies=currencies + ) + except trading_errors.NotSupported: + return [] + except trading_errors.AuthenticationError as error: + logger.warning( + "Skipping deposits fetch for %s: %s", + self.exchange_manager.exchange_name, + error, + ) + return [] + + async def fetch_withdrawals( + self, + since: int = None, + limit: int = None, + currencies: list[str] = None, + ) -> list[dict]: + try: + return await self.exchange_manager.exchange.get_withdrawals( + since=since, limit=limit, currencies=currencies + ) + except trading_errors.NotSupported: + return [] + except trading_errors.AuthenticationError as error: + logger.warning( + "Skipping withdrawals fetch for %s: %s", + self.exchange_manager.exchange_name, + error, + ) + return [] diff --git a/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py b/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py new file mode 100644 index 0000000000..a344bfeab3 --- /dev/null +++ b/packages/flow/tests/entities/automations/test_post_iteration_actions_details_clear.py @@ -0,0 +1,118 @@ +import octobot_commons.dsl_interpreter + +import octobot_flow.entities.automations.post_iteration_actions_details as post_iteration_actions_details_module + + +class TestPostIterationClear: + def test_removes_only_updated_exchange_account_elements(self): + post_iteration_payload = { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + "configuration_update": "run_octobot_process('folder')", + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear( + post_iteration_payload + ) + assert "updated_exchange_account_elements" not in post_iteration_payload + assert post_iteration_payload["stop_automation"] is False + assert post_iteration_payload["configuration_update"] == "run_octobot_process('folder')" + + +class TestPostIterationClearFromActionResult: + def test_clears_nested_recall_post_iteration(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + }, + "pid": 42, + }, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + assert inner_post_iteration["stop_automation"] is False + assert action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"]["pid"] == 42 + + def test_top_level_stop_automation_unchanged(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + post_iteration_name: { + "stop_automation": True, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result[post_iteration_name] == {"stop_automation": True} + + def test_top_level_configuration_update_unchanged(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + configuration_update = "run_octobot_process('folder')" + action_result = { + post_iteration_name: { + "configuration_update": configuration_update, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result[post_iteration_name]["configuration_update"] == configuration_update + + def test_clears_top_level_and_nested_when_both_present(self): + post_iteration_name = post_iteration_actions_details_module.PostIterationActionsDetails.__name__ + action_result = { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": [{"id": "top"}]}, + }, + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": [{"id": "nested"}]}, + }, + }, + }, + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert "updated_exchange_account_elements" not in action_result[post_iteration_name] + nested_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in nested_post_iteration + + def test_recall_without_post_iteration_preserves_inner_fields(self): + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + "pid": 42, + "init_state_ok": True, + }, + } + } + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_last_result = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"] + assert inner_last_result["pid"] == 42 + assert inner_last_result["init_state_ok"] is True + + def test_non_recall_result_without_post_iteration_is_no_op(self): + action_result = {"pid": 42} + post_iteration_actions_details_module.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + assert action_result == {"pid": 42} diff --git a/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py b/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py new file mode 100644 index 0000000000..0e4c1f4b02 --- /dev/null +++ b/packages/flow/tests/entities/test_exchange_account_elements_trim_trades.py @@ -0,0 +1,76 @@ +import octobot_trading.enums as octobot_trading_enums_import + +import octobot_flow.entities as octobot_flow_entities + + +def _trade_dict(trade_id: str, symbol: str, timestamp: float) -> dict: + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + return { + order_columns.EXCHANGE_TRADE_ID.value: trade_id, + order_columns.SYMBOL.value: symbol, + order_columns.TIMESTAMP.value: timestamp, + } + + +class TestTrimTradesToLiveWindow: + def test_keeps_newest_trades_and_archives_older_ids_by_symbol(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + _trade_dict("old-1", "BTC/USDT", 1.0), + _trade_dict("new-1", "ETH/USDT", 3.0), + _trade_dict("old-2", "BTC/USDT", 2.0), + ] + ) + elements.trim_trades_to_live_window(2) + trade_id_key = octobot_trading_enums_import.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value + kept_trade_ids = [trade[trade_id_key] for trade in elements.trades] + assert kept_trade_ids == ["old-2", "new-1"] + assert elements.trade_summaries == {"BTC/USDT": ["old-1"]} + + def test_deduplicates_archived_ids_per_symbol(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + _trade_dict("old-1", "BTC/USDT", 1.0), + _trade_dict("old-2", "BTC/USDT", 2.0), + ], + trade_summaries={"BTC/USDT": ["old-1"]}, + ) + elements.trim_trades_to_live_window(1) + assert elements.trade_summaries == {"BTC/USDT": ["old-1"]} + assert len(elements.trades) == 1 + + def test_no_op_when_trade_count_within_window(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[_trade_dict("trade-1", "BTC/USDT", 1.0)], + trade_summaries={"ETH/USDT": ["archived-1"]}, + ) + elements.trim_trades_to_live_window(100) + assert len(elements.trades) == 1 + assert elements.trade_summaries == {"ETH/USDT": ["archived-1"]} + + def test_skips_evicted_trade_missing_symbol(self): + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + elements = octobot_flow_entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "no-symbol", + order_columns.TIMESTAMP.value: 1.0, + }, + _trade_dict("keep-1", "BTC/USDT", 2.0), + ] + ) + elements.trim_trades_to_live_window(1) + assert len(elements.trades) == 1 + assert elements.trades[0][order_columns.EXCHANGE_TRADE_ID.value] == "keep-1" + assert elements.trade_summaries == {} + + +class TestExchangeAccountElementsTradeSummariesRoundTrip: + def test_to_dict_from_dict_preserves_trade_summaries_dict(self): + elements = octobot_flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["archived-1", "archived-2"]}, + ) + restored = octobot_flow_entities.ExchangeAccountElements.from_dict( + elements.to_dict(include_default_values=False) + ) + assert restored.trade_summaries == {"BTC/USDT": ["archived-1", "archived-2"]} diff --git a/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py new file mode 100644 index 0000000000..e84497ee44 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/global_view/test_global_view_account_refresh.py @@ -0,0 +1,270 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module +import octobot_sync.constants as sync_constants +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline + + +def _open_order_dict(exchange_id: str, symbol: str = "BTC/USDT") -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _empty_portfolio_history_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def _exchange_account_context( + *, + has_bound_automation: bool = False, + is_simulated: bool = True, +) -> octobot_flow.entities.GlobalViewAccountContext: + account_id = "functional-account-1" + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account_assets = None + if is_simulated: + account_assets = [ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=1000.0, + available=1000.0, + ), + ], + ), + ] + account = protocol_models.Account( + id=account_id, + name="Functional account", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + assets=account_assets, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + has_bound_automation=has_bound_automation, + ) + + +@pytest.mark.asyncio +class TestGlobalViewAccountJobFunctional: + async def test_refresh_simulated_account(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + wire_portfolio_pipeline(exchange_manager, {}) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio = mock.Mock( + side_effect=lambda portfolio_content, update_available_funds_from_open_orders=False: ( + portfolio_manager.handle_balance_update(portfolio_content) + ), + ) + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + ensure_ticker_channel_mock = mock.AsyncMock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + exchange_account_refresh_module.tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value=set(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=[], + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + portfolio_manager.apply_forced_portfolio.assert_called_once() + assert refresh_result.updated_account.assets + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.updated_account.assets + for asset in assets_for_trading_type.assets + } + assert "USDT" in asset_symbols + assert refresh_result.changed_order_ids == set() + + async def test_refresh_real_account_detects_disappeared_orders(self): + context = _exchange_account_context(has_bound_automation=True, is_simulated=False) + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + _open_order_dict("gone-order-1", "BTC/USDT"), + _open_order_dict("stays-order-2", "ETH/USDT"), + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + ensure_ticker_channel_mock = mock.AsyncMock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} + assert refresh_result.updated_account.assets + assert refresh_result.changed_order_ids == {"gone-order-1"} diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py index bda3b42770..297c8baa3a 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/octobot_process_functional_shared.py @@ -50,8 +50,10 @@ # it is fixed at import time and stays 30 unless the interpreter reloads constants. EXPECTED_PROCESS_BOT_DUMP_INTERVAL_SEC = 5.0 -# Same as `waiting_time=` in run_octobot_process(...) DSL for this file's tests. +# Same as `waiting_time=` in run_octobot_process(...) DSL for this file's tests (steady override when child is healthy). WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC = 2 +# Init / child-not-alive recall interval (matches octobot_process_ops.FAST_PING_WAITING_TIME). +FAST_RECALL_WAITING_TIME_SEC = 5.0 RECALL_SCHEDULE_TOLERANCE_SEC = 1.5 @@ -271,6 +273,12 @@ def _recall_inner_from_dsl_action( return None +def _expected_recall_waiting_time_from_inner(inner: typing.Optional[dict]) -> float: + if isinstance(inner, dict) and inner.get("waiting_time") is not None: + return float(inner["waiting_time"]) + return FAST_RECALL_WAITING_TIME_SEC + + def _assert_run_octobot_process_recall_scheduled_to_in_dump( job_dump: dict[str, typing.Any], *, @@ -304,6 +312,28 @@ def _assert_run_octobot_process_recall_scheduled_to_in_dump( ) +def _assert_run_octobot_process_recall_scheduled_to_from_job( + job: octobot_flow.jobs.AutomationJob, + *, + expected_waiting_time_sec: typing.Optional[float] = None, + schedule_tolerance_sec: float = RECALL_SCHEDULE_TOLERANCE_SEC, + assert_delay_matches_waiting_time: bool = True, +) -> None: + if assert_delay_matches_waiting_time and expected_waiting_time_sec is None: + run_action_state = _get_action_by_id(job, ACTION_ID_RUN_OCTOBOT) + assert run_action_state is not None + inner = _recall_inner_from_dsl_action(run_action_state) + expected_waiting_time_sec = _expected_recall_waiting_time_from_inner(inner) + elif expected_waiting_time_sec is None: + expected_waiting_time_sec = WAITING_TIME_RUN_OCTOBOT_PROCESS_SEC + _assert_run_octobot_process_recall_scheduled_to_in_dump( + job.dump(), + expected_waiting_time_sec=expected_waiting_time_sec, + schedule_tolerance_sec=schedule_tolerance_sec, + assert_delay_matches_waiting_time=assert_delay_matches_waiting_time, + ) + + def _get_action_by_id( job: octobot_flow.jobs.AutomationJob, action_id: str ) -> typing.Optional[octobot_flow.entities.AbstractActionDetails]: @@ -316,7 +346,13 @@ def _get_action_by_id( _FERNET_ENCRYPTED_PREFIX = "gAAAAA" -# --- Child on-disk readiness (poll after init_state_ok; PID can be up before first dump / encrypt) --- +# --- Child on-disk readiness (poll until process_bot_state.json exists) --- + + +def _child_process_state_file_ready(inner: typing.Optional[dict]) -> bool: + if not inner or not inner.get("user_root"): + return False + return os.path.isfile(_process_bot_state_path(inner)) def _process_bot_state_path(inner: dict) -> str: @@ -345,6 +381,46 @@ async def _wait_for_process_bot_state_file( ) +async def wait_for_child_process_ready( + inner: dict, + *, + timeout_sec: float = GLOBAL_START_TIMEOUT_SEC, +) -> str: + """Poll until process_bot_state.json exists; return state_path.""" + state_path = _process_bot_state_path(inner) + await _wait_for_process_bot_state_file(state_path, timeout_sec=timeout_sec) + return state_path + + +async def poll_automation_until_child_process_ready( + automation_state: dict, + *, + timeout_sec: float = GLOBAL_START_TIMEOUT_SEC, +) -> tuple[dict, dict, str]: + """ + Run automation job polls until process_bot_state.json exists on disk. + Returns (updated automation state dump, recall inner dict, state_path). + """ + deadline = time.monotonic() + timeout_sec + inner: typing.Optional[dict] = None + state = automation_state + while time.monotonic() < deadline: + poll_job = await run_automation_job_without_exchange_manager(state, [], [], {}) + _assert_run_octobot_process_recall_scheduled_to_from_job(poll_job) + run_action_state = _get_action_by_id(poll_job, ACTION_ID_RUN_OCTOBOT) + assert run_action_state is not None + inner = _recall_inner_from_dsl_action(run_action_state) + state = poll_job.dump() + if _child_process_state_file_ready(inner): + assert inner is not None + state_path = _process_bot_state_path(inner) + return state, inner, state_path + await asyncio.sleep(SLEEP_BETWEEN_JOB_POLLS_SEC) + pytest.fail( + f"Timed out waiting for process_bot_state.json within {timeout_sec}s" + ) + + async def _assert_encrypted_exchange_credentials_in_user_config( user_root: typing.Union[pathlib.Path, str], exchange_internal_name: str, diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py index 3533af4a8c..a472691b40 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_edit_config.py @@ -145,54 +145,19 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC inner: typing.Optional[dict] = None - # 2) First automation pass, then poll until the child reports init_state_ok (ready to query). - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + # 2) Poll until process_bot_state.json exists (single readiness gate). + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid") initial_spawn_count = popen_calls["count"] assert initial_spawn_count >= 1 - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) - # 3) Wait until at least four open ladder orders exist, then assert a 2×2 grid pattern. orders_deadline = time.monotonic() + octobot_process_functional_shared.GRID_ORDERS_TIMEOUT_SEC exchange_account_snapshot: typing.Optional[ @@ -253,8 +218,8 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( priority_actions = functionnal_tests.resolved_actions([update_config_priority_action]) async with octobot_flow.jobs.AutomationJob(state, priority_actions, [], {}) as refresh_phase: await refresh_phase.run() - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - refresh_phase.dump() + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_from_job( + refresh_phase ) assert popen_calls["count"] == spawn_before_refresh + 1 state = refresh_phase.dump() @@ -272,10 +237,10 @@ async def test_run_octobot_process_grid_refresh_four_to_six_orders( six_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( state, [], [], {} ) - dump_payload = six_poll.dump() - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - dump_payload + octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_from_job( + six_poll ) + dump_payload = six_poll.dump() automation_dump = dump_payload.get("automation") eae_dict = ( automation_dump.get("exchange_account_elements") diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py index bb71fd39cd..5e027e1386 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_multi_exchange.py @@ -105,34 +105,13 @@ async def test_run_octobot_process_aggregates_two_simulator_exchanges( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - while time.monotonic() < deadline: - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_action_state = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_action_state is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_action_state) - assert inner is not None - if inner.get("init_state_ok"): - break - state = poll_job.dump() - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - else: - pytest.fail( - f"Timed out waiting for init_state_ok within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state ) + ) assert popen_calls["count"] >= 1 - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) orders_deadline = time.monotonic() + DUAL_EXCHANGE_ORDERS_TIMEOUT_SEC exchange_account_snapshot: typing.Optional[ diff --git a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py index 50c1a2203b..0271626eaa 100644 --- a/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py +++ b/packages/flow/tests/functionnal_tests/octobot_process_actions/test_octobot_process_start.py @@ -106,61 +106,23 @@ async def test_run_octobot_process_lifecycle_grid_trading( await init_job.run() state = init_job.dump() - # 2) Register run_octobot_process; poll job until the child reports init_state_ok (live process_bot_state). + # 2) Register run_octobot_process; poll until process_bot_state.json exists. async with octobot_flow.jobs.AutomationJob(state, [], [], {}) as job: job.automation_state.upsert_automation_actions( functionnal_tests.resolved_actions([run_action]) ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - # Run DSL job once, then optionally poll until recall payload shows init_state_ok. - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid"), "expected child pid in ensure state" assert popen_calls["count"] >= 1 - # --- process_bot_state path: must exist before poll (child wrote at least one dump) --- - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) - # 1) Poll AutomationJob + dump() until merge yields ≥4 open orders (EAE from automation snapshot, # not from parsing full process_bot_state on disk). orders_deadline = time.monotonic() + octobot_process_functional_shared.GRID_ORDERS_TIMEOUT_SEC @@ -405,43 +367,11 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ) state = job.dump() - deadline = time.monotonic() + octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC - inner: typing.Optional[dict] = None - first_poll = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - first_poll.dump() - ) - first_run = octobot_process_functional_shared._get_action_by_id( - first_poll, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT + state, inner, state_path = ( + await octobot_process_functional_shared.poll_automation_until_child_process_ready( + state + ) ) - assert first_run is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(first_run) - state = first_poll.dump() - if not (inner and inner.get("init_state_ok") is True): - while time.monotonic() < deadline: - await asyncio.sleep(octobot_process_functional_shared.SLEEP_BETWEEN_JOB_POLLS_SEC) - poll_job = await octobot_process_functional_shared.run_automation_job_without_exchange_manager( - state, [], [], {} - ) - octobot_process_functional_shared._assert_run_octobot_process_recall_scheduled_to_in_dump( - poll_job.dump() - ) - run_details = octobot_process_functional_shared._get_action_by_id( - poll_job, octobot_process_functional_shared.ACTION_ID_RUN_OCTOBOT - ) - assert run_details is not None - inner = octobot_process_functional_shared._recall_inner_from_dsl_action(run_details) - if inner and inner.get("init_state_ok") is True: - state = poll_job.dump() - break - state = poll_job.dump() - else: - pytest.fail( - f"OctoBot did not become ready (init_state_ok) within " - f"{octobot_process_functional_shared.GLOBAL_START_TIMEOUT_SEC}s" - ) assert inner is not None assert inner.get("pid"), "expected child pid in ensure state" @@ -482,9 +412,6 @@ async def test_run_octobot_process_lifecycle_default_config_no_profile_data( ) assert not local_non_trading_profile_json.exists() - # First process_bot_state dump can lag init_state_ok (see shared wait helper). - state_path = octobot_process_functional_shared._process_bot_state_path(inner) - await octobot_process_functional_shared._wait_for_process_bot_state_file(state_path) with open(state_path, encoding="utf-8") as process_state_file: file_metadata_payload = json.load(process_state_file) process_metadata = process_bot_state_import.Metadata.from_dict( diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py b/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py new file mode 100644 index 0000000000..89ba38b40f --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/portfolio_history_test_util.py @@ -0,0 +1,305 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock + +import octobot.community.authentication as community_authentication +import octobot_commons.tests.test_config as test_config_module +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.exchange_manager as exchange_manager_module +import octobot_trading.exchanges.traders.trader_simulator as trader_simulator_module +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +import tests.repositories.exchange.trades_repository_test_util as trades_repository_test_util + + +TEST_WALLET_ID = "0xfunctionaltestwallet" +_TEST_WALLET_ID = TEST_WALLET_ID +_TEST_PRIVATE_KEY = "functional-test-private-key" +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) +_DEFAULT_SYMBOL = "BTC/USDT" +_DEFAULT_ACCOUNT_ASSET_SYMBOLS = ["BTC", "ETH", "USDT"] + + +def default_spot_account_assets( + symbols: list[str] | None = None, +) -> list[protocol_models.DetailedAssetsForTradingType]: + resolved_symbols = symbols or _DEFAULT_ACCOUNT_ASSET_SYMBOLS + return [ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol=symbol, + total=1.0, + available=1.0, + ) + for symbol in resolved_symbols + ], + ) + ] + + +def build_portfolio_history_context( + *, + account_id: str = "functional-account-1", + symbols: list[str] | None = None, + trade_symbols: list[str] | None = None, + is_simulated: bool = False, + exchange: str = "binanceus", + assets: list[protocol_models.DetailedAssetsForTradingType] | None = None, +) -> portfolio_history_entities.PortfolioHistoryAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id=account_id, + name="Functional portfolio history account", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=default_spot_account_assets() if assets is None else assets, + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange=exchange, + sandboxed=False, + historical_trade_symbols=symbols or [_DEFAULT_SYMBOL], + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + api_key=account_id, + ) + resolved_trade_symbols = ( + trade_symbols + if trade_symbols is not None + else (symbols or [_DEFAULT_SYMBOL]) + ) + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + trade_symbols=list(resolved_trade_symbols), + ) + + +def sample_raw_trade( + *, + trade_id: str = "functional-trade-1", + symbol: str = _DEFAULT_SYMBOL, +) -> dict: + return { + "info": {}, + "id": trade_id, + "exchange_id": trade_id, + "exchange_trade_id": trade_id, + "timestamp": 1700000000.0, + "symbol": symbol, + "type": "limit", + "side": "buy", + "price": 30000.0, + "amount": 0.1, + "cost": 3000.0, + "status": "closed", + "fee": {"cost": 0.0, "currency": "USDT"}, + } + + +def sample_deposit( + *, + txid: str = "functional-deposit-1", + currency: str = "BTC", + amount: float = 1.0, +) -> dict: + return { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: txid, + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: currency, + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: amount, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1700001000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: ( + trading_enums.TransactionType.BLOCKCHAIN_DEPOSIT.value + ), + } + + +def sample_withdrawal( + *, + txid: str = "functional-withdrawal-1", + currency: str = "ETH", + amount: float = 0.5, +) -> dict: + return { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: txid, + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: currency, + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: amount, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1700002000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: ( + trading_enums.TransactionType.BLOCKCHAIN_WITHDRAWAL.value + ), + } + + +def sample_daily_candles( + *, + day_timestamp: int = 86400, + close_price: float = 40500.0, +) -> list[list[float]]: + return [ + [day_timestamp, 40000.0, 41000.0, 39000.0, close_price, 100.0], + ] + + +async def _create_exchange_manager_with_trader() -> exchange_manager_module.ExchangeManager: + config = test_config_module.load_test_config() + exchange_manager = exchange_manager_module.ExchangeManager(config, "binanceus") + await exchange_manager.initialize(exchange_config_by_exchange=None) + trader = trader_simulator_module.TraderSimulator(config, exchange_manager) + await trader.initialize() + return exchange_manager + + +async def build_exchange_manager( + *, + raw_trades: list[dict] | None = None, + deposits: list[dict] | None = None, + withdrawals: list[dict] | None = None, + daily_candles: list[list[float]] | None = None, +) -> exchange_manager_module.ExchangeManager: + exchange_manager = await _create_exchange_manager_with_trader() + + configured_raw_trades = raw_trades or [] + configured_deposits = deposits or [] + configured_withdrawals = withdrawals or [] + configured_daily_candles = daily_candles or sample_daily_candles() + + async def get_my_recent_trades(symbol: str, limit=None, since=None, **_kwargs): + matched_trades = [ + raw_trade + for raw_trade in configured_raw_trades + if raw_trade.get(trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value) == symbol + ] + if since is not None: + matched_trades = [ + raw_trade + for raw_trade in matched_trades + if int(float(raw_trade.get(trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value, 0)) * 1000) + >= since + ] + if limit is not None: + matched_trades = matched_trades[:limit] + return matched_trades + + exchange_manager.exchange.get_my_recent_trades = get_my_recent_trades + exchange_manager.exchange.get_deposits = mock.AsyncMock(return_value=configured_deposits) + exchange_manager.exchange.get_withdrawals = mock.AsyncMock(return_value=configured_withdrawals) + exchange_manager.exchange.get_symbol_prices = mock.AsyncMock(return_value=configured_daily_candles) + await trades_repository_test_util.ensure_trades_channel(exchange_manager) + return exchange_manager + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def seed_empty_account_trading( + provider: collection_providers.AccountTradingProvider, + wallet_id: str, + account_id: str, +) -> None: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + provider.save_state(wallet_id, account_id, trading_state) + + +def load_account_trading(wallet_id: str, account_id: str) -> protocol_models.AccountTrading: + trading_state = collection_providers.AccountTradingProvider.instance().load_state(wallet_id, account_id) + return trading_state.account_trading + + +async def load_daily_prices_from_root( + data_root, + exchange_name: str, + exchange_type: str, + sandboxed: bool, +) -> dict: + return await trading_api.load_daily_prices(exchange_name, exchange_type, sandboxed, data_root) + + +async def _empty_historical_ohlcv(*_args, **_kwargs): + if False: + yield [] + + +@contextlib.contextmanager +def portfolio_history_test_environment( + tmp_path, + *, + exchange_manager_by_account_id: dict[str, exchange_manager_module.ExchangeManager], + wallet_id: str = _TEST_WALLET_ID, +): + trading_provider = collection_providers.AccountTradingProvider(base_folder=str(tmp_path)) + + @contextlib.asynccontextmanager + async def fake_exchange_manager(exchange_data, *_args, **_kwargs): + account_id = exchange_data.auth_details.api_key + exchange_manager = exchange_manager_by_account_id[account_id] + yield exchange_manager + + with ( + _patch_wallet(), + mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + mock.patch.object( + portfolio_history_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + portfolio_history_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + portfolio_history_job_module.trades_repository_module.TradesRepository, + "ensure_temporary_trades_channel", + trades_repository_test_util.ensure_trades_channel, + ), + mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _empty_historical_ohlcv, + ), + ): + yield trading_provider diff --git a/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py b/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py new file mode 100644 index 0000000000..576dc03601 --- /dev/null +++ b/packages/flow/tests/functionnal_tests/portfolio_history/test_portfolio_history_job.py @@ -0,0 +1,166 @@ +# Drakkar-Software OctoBot-Flow + +import time + +import pytest + +import octobot_commons.constants as commons_constants +import octobot_trading.enums as trading_enums +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module +from tests.functionnal_tests.portfolio_history import portfolio_history_test_util as portfolio_history_test_util + + +def _recent_day_timestamp(days_ago: int = 1) -> int: + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + return today_start - days_ago * commons_constants.DAYS_TO_SECONDS + + +@pytest.mark.asyncio +class TestPortfolioHistoryJobFunctional: + async def test_collects_and_persists_trading_history(self, tmp_path): + account_id = "functional-account-1" + context = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id, + symbols=["BTC/USDT"], + ) + btc_day = _recent_day_timestamp(1) + exchange_manager = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[portfolio_history_test_util.sample_raw_trade()], + deposits=[portfolio_history_test_util.sample_deposit()], + withdrawals=[portfolio_history_test_util.sample_withdrawal()], + daily_candles=portfolio_history_test_util.sample_daily_candles( + day_timestamp=btc_day, + close_price=40500.0, + ), + ) + + with portfolio_history_test_util.portfolio_history_test_environment( + tmp_path, + exchange_manager_by_account_id={account_id: exchange_manager}, + ) as trading_provider: + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id, + ) + results = await portfolio_history_job_module.PortfolioHistoryJob( + portfolio_history_test_util.TEST_WALLET_ID, + [context], + data_root=str(tmp_path), + ).run() + + assert len(results) == 1 + result = results[0] + assert result.account_id == account_id + assert result.trades_count == 1 + assert result.transactions_count == 2 + assert not result.skipped + assert result.error is None + + account_trading = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id, + ) + trade_ids = {trade.trade_id for trade in account_trading.trades or []} + transaction_ids = {transaction.id for transaction in account_trading.transactions or []} + assert "functional-trade-1" in trade_ids + assert "functional-deposit-1" in transaction_ids + assert "functional-withdrawal-1" in transaction_ids + + daily_prices = await portfolio_history_test_util.load_daily_prices_from_root( + str(tmp_path), + "binanceus", + "spot", + False, + ) + assert daily_prices[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(btc_day)] == 40500.0 + assert "ETH/USDT" in daily_prices[trading_enums.DailyPricesCacheKeys.SYMBOLS] + + async def test_two_accounts_both_persist(self, tmp_path): + account_id_1 = "functional-account-1" + account_id_2 = "functional-account-2" + context_1 = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id_1, + symbols=["BTC/USDT"], + ) + context_2 = portfolio_history_test_util.build_portfolio_history_context( + account_id=account_id_2, + symbols=["ETH/USDT"], + ) + btc_day = _recent_day_timestamp(1) + eth_day = _recent_day_timestamp(2) + exchange_manager_1 = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[ + portfolio_history_test_util.sample_raw_trade( + trade_id="account-1-trade", + symbol="BTC/USDT", + ) + ], + deposits=[portfolio_history_test_util.sample_deposit(txid="account-1-deposit")], + withdrawals=[portfolio_history_test_util.sample_withdrawal(txid="account-1-withdrawal")], + daily_candles=portfolio_history_test_util.sample_daily_candles( + day_timestamp=btc_day, + close_price=41000.0, + ), + ) + exchange_manager_2 = await portfolio_history_test_util.build_exchange_manager( + raw_trades=[ + portfolio_history_test_util.sample_raw_trade( + trade_id="account-2-trade", + symbol="ETH/USDT", + ) + ], + deposits=[portfolio_history_test_util.sample_deposit(txid="account-2-deposit", currency="ETH")], + withdrawals=[portfolio_history_test_util.sample_withdrawal(txid="account-2-withdrawal", currency="USDT")], + daily_candles=portfolio_history_test_util.sample_daily_candles( + day_timestamp=eth_day, + close_price=2500.0, + ), + ) + + with portfolio_history_test_util.portfolio_history_test_environment( + tmp_path, + exchange_manager_by_account_id={ + account_id_1: exchange_manager_1, + account_id_2: exchange_manager_2, + }, + ) as trading_provider: + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id_1, + ) + portfolio_history_test_util.seed_empty_account_trading( + trading_provider, + portfolio_history_test_util.TEST_WALLET_ID, + account_id_2, + ) + results = await portfolio_history_job_module.PortfolioHistoryJob( + portfolio_history_test_util.TEST_WALLET_ID, + [context_1, context_2], + data_root=str(tmp_path), + ).run() + + assert len(results) == 2 + assert all(not result.skipped for result in results) + assert {result.account_id for result in results} == {account_id_1, account_id_2} + + account_trading_1 = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id_1, + ) + account_trading_2 = portfolio_history_test_util.load_account_trading( + portfolio_history_test_util.TEST_WALLET_ID, + account_id_2, + ) + assert {trade.trade_id for trade in account_trading_1.trades or []} == {"account-1-trade"} + assert {trade.trade_id for trade in account_trading_2.trades or []} == {"account-2-trade"} + assert {transaction.id for transaction in account_trading_1.transactions or []} == { + "account-1-deposit", + "account-1-withdrawal", + } + assert {transaction.id for transaction in account_trading_2.transactions or []} == { + "account-2-deposit", + "account-2-withdrawal", + } diff --git a/packages/flow/tests/jobs/test_portfolio_history_job.py b/packages/flow/tests/jobs/test_portfolio_history_job.py new file mode 100644 index 0000000000..6c2d1a8b5b --- /dev/null +++ b/packages/flow/tests/jobs/test_portfolio_history_job.py @@ -0,0 +1,1010 @@ +import asyncio +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module + +def _make_account(account_id: str, is_simulated: bool = False, is_exchange: bool = True): + account = mock.MagicMock() + account.id = account_id + account.is_simulated = is_simulated + if is_exchange: + account.specifics.actual_instance = mock.MagicMock(spec=protocol_models.ExchangeAccount) + else: + account.specifics.actual_instance = mock.MagicMock(spec=protocol_models.GenericAccount) + return account + + +def _make_context(account, exchange_config=None, trade_symbols=None): + if exchange_config is None: + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "binance" + exchange_config.sandboxed = False + exchange_config.id = "cfg-1" + exchange_config.name = "binance-main" + exchange_config.historical_trade_symbols = ["BTC/USDT"] + if trade_symbols is None: + trade_symbols = ["BTC/USDT"] + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=account.specifics.actual_instance, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=mock.MagicMock(), + trade_symbols=list(trade_symbols), + ) + + +class TestRunForAccountSkipsUnsupported: + @pytest.mark.asyncio + async def test_skips_generic_account(self): + account = _make_account("acc1", is_exchange=False) + context = _make_context(account) + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + assert results[0].skipped is True + assert results[0].exchange_name == "binance" + + @pytest.mark.asyncio + async def test_skips_simulated_account(self): + account = _make_account("acc1", is_simulated=True) + context = _make_context(account) + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + assert results[0].skipped is True + assert results[0].exchange_name == "binance" + + +class TestRunForAccountDoesNotPersistHistory: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_account_history_provider_not_called( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert not results[0].skipped + assert results[0].exchange_name == "binance" + assert results[0].trading_type == protocol_models.TradingType.SPOT.value + assert results[0].duration_seconds is not None + assert results[0].duration_seconds >= 0 + assert results[0].price_symbols_count >= 0 + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel.assert_awaited_once_with( + mock_exchange_manager, + ) + # merge_and_persist_trading_history should be called, but not AccountHistoryProvider + mock_merge.merge_and_persist_trading_history.assert_called_once() + + +class TestFetchAndPersistTransactionCurrencies: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_passes_currencies_with_balance_to_tx_repo( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + btc_asset = mock.MagicMock(symbol="BTC", total=1.0) + eth_asset = mock.MagicMock(symbol="ETH", total=0.5) + account.assets = [mock.MagicMock(assets=[btc_asset, eth_asset])] + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert not results[0].skipped + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits.assert_awaited_once_with( + currencies=["BTC", "ETH"] + ) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals.assert_awaited_once_with( + currencies=["BTC", "ETH"] + ) + + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_skips_tx_fetch_when_no_balance( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + account.assets = [] + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert not results[0].skipped + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits.assert_not_awaited() + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals.assert_not_awaited() + + +class TestFetchTradesUsesContextTradeSymbols: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_fetch_trades_uses_context_trade_symbols( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + context = _make_context(account, trade_symbols=["ETH/USDT"]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["ETH/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + fetch_trades_paginated_mock.assert_awaited_once() + assert fetch_trades_paginated_mock.await_args.args[0] == ["ETH/USDT"] + assert results[0].trade_symbols_count == 1 + + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_empty_trade_symbols_skips_trade_fetch( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + context = _make_context(account, trade_symbols=[]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = [] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + fetch_trades_paginated_mock.assert_awaited_once() + assert fetch_trades_paginated_mock.await_args.args[0] == [] + assert results[0].trade_symbols_count == 0 + + +class TestRunParallelExchangeAccounts: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_multiple_accounts_run_in_parallel( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account1 = _make_account("acc1") + account2 = _make_account("acc2") + context1 = _make_context(account1) + context2 = _make_context(account2) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context1, context2]) + results = await job.run() + + assert len(results) == 2 + assert mock_merge.merge_and_persist_trading_history.call_count == 2 + + +class TestReferenceMarketFromAccountAssets: + def test_uses_dominant_usd_like_holding(self): + account = mock.MagicMock() + usdc_asset = mock.MagicMock(symbol="USDC", total=402.0) + usdt_asset = mock.MagicMock(symbol="USDT", total=10.0) + account.assets = [mock.MagicMock(assets=[usdc_asset, usdt_asset])] + + reference_market = portfolio_history_job_module._reference_market_from_account_assets( + account, + "kraken", + ) + + assert reference_market == "USDC" + + def test_falls_back_to_exchange_default_when_no_usd_like_holdings(self): + account = mock.MagicMock() + account.assets = [] + with mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) as default_reference_market_mock: + reference_market = portfolio_history_job_module._reference_market_from_account_assets( + account, + "kraken", + ) + + default_reference_market_mock.assert_called_once_with("kraken") + assert reference_market == "USDT" + + +class TestCurrenciesWithBalanceFromAccount: + def test_returns_sorted_symbols_above_threshold(self): + account = mock.MagicMock() + btc_asset = mock.MagicMock(symbol="BTC", total=1.0) + eth_asset = mock.MagicMock(symbol="ETH", total=0.5) + usdc_dust_asset = mock.MagicMock(symbol="USDC", total=1e-10) + account.assets = [mock.MagicMock(assets=[btc_asset, eth_asset, usdc_dust_asset])] + + currencies = portfolio_history_job_module._currencies_with_balance_from_account(account) + + assert currencies == ["BTC", "ETH"] + + def test_returns_empty_when_no_assets(self): + account = mock.MagicMock() + account.assets = None + + currencies = portfolio_history_job_module._currencies_with_balance_from_account(account) + + assert currencies == [] + + def test_merges_totals_across_trading_types(self): + account = mock.MagicMock() + btc_spot_asset = mock.MagicMock(symbol="BTC", total=0.5) + btc_margin_asset = mock.MagicMock(symbol="BTC", total=1.5) + account.assets = [ + mock.MagicMock(assets=[btc_spot_asset]), + mock.MagicMock(assets=[btc_margin_asset]), + ] + + currencies = portfolio_history_job_module._currencies_with_balance_from_account(account) + + assert currencies == ["BTC"] + + +class TestDerivePriceSymbols: + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_skips_reference_market_currency_from_transactions(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "USDT"}], + "USDT", + ) + assert symbols == [] + + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_adds_transaction_currency_against_reference_market(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "BTC"}], + "USDT", + ) + assert symbols == ["BTC/USDT"] + + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_filters_invalid_same_base_quote_symbols(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + ["BTC/BTC", "ETH/USDT"], + [], + "USDT", + ) + assert symbols == ["BTC/USDT", "ETH/USDT"] + + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_maps_trade_symbols_to_one_reference_pair_per_base(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + ["DOT/BTC", "DOT/EUR", "DOT/USDC", "ETH/BTC"], + [], + "USDC", + ) + assert symbols == ["DOT/USDC", "ETH/USDC"] + + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_maps_transaction_currency_to_reference_pair(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "EUR"}], + "USDC", + ) + assert symbols == ["EUR/USDC"] + + @mock.patch( + "octobot_trading.api.exchange.get_default_exchange_reference_market", + return_value="USDT", + ) + def test_skips_usd_like_stablecoin_from_transactions(self, _mock_reference_market): + symbols = portfolio_history_job_module._derive_price_symbols( + [], + [{"currency": "USDC"}], + "USDT", + ) + assert symbols == [] + + +class TestFilterTradesOnLiveMarkets: + def test_keeps_trades_on_live_markets(self): + exchange_manager = mock.MagicMock() + exchange_manager.client_symbols = ["BTC/USDT", "ETH/USDT"] + trades = [ + {"symbol": "BTC/USDT"}, + {"symbol": "ETH/USDT"}, + ] + + kept_trades, dropped_symbols = portfolio_history_job_module._filter_trades_on_live_markets( + exchange_manager, + trades, + ) + + assert kept_trades == trades + assert dropped_symbols == set() + + def test_drops_trades_on_unknown_or_delisted_markets(self): + exchange_manager = mock.MagicMock() + exchange_manager.client_symbols = ["BTC/USDT"] + trades = [ + {"symbol": "BTC/USDT"}, + {"symbol": "MATICXBT"}, + {"symbol": "FTMEUR"}, + ] + + kept_trades, dropped_symbols = portfolio_history_job_module._filter_trades_on_live_markets( + exchange_manager, + trades, + ) + + assert kept_trades == [{"symbol": "BTC/USDT"}] + assert dropped_symbols == {"FTMEUR", "MATICXBT"} + + def test_drops_trades_with_missing_symbol(self): + exchange_manager = mock.MagicMock() + exchange_manager.client_symbols = ["BTC/USDT"] + trades = [ + {"symbol": "BTC/USDT"}, + {"side": "buy"}, + ] + + kept_trades, dropped_symbols = portfolio_history_job_module._filter_trades_on_live_markets( + exchange_manager, + trades, + ) + + assert kept_trades == [{"symbol": "BTC/USDT"}] + assert dropped_symbols == set() + + +class TestFilterSymbolsOnLiveMarkets: + def test_keeps_only_symbols_present_in_client_markets(self): + exchange_manager = mock.MagicMock() + exchange_manager.client_symbols = ["BTC/USDT", "ETH/USDT"] + + filtered_symbols = portfolio_history_job_module._filter_symbols_on_live_markets( + exchange_manager, + ["BTC/USDT", "MATICXBT", "ETH/USDT", "FTMEUR"], + ) + + assert filtered_symbols == ["BTC/USDT", "ETH/USDT"] + + +class TestDropDelistedTradesFromJob: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.logger") + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_drops_delisted_trades_and_logs_symbols( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, mock_logger, + ): + account = _make_account("acc1") + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "kraken" + exchange_config.sandboxed = False + exchange_config.id = "kraken-main" + exchange_config.name = "kraken-main" + exchange_config.historical_trade_symbols = [] + context = _make_context(account, exchange_config=exchange_config, trade_symbols=[]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = [ + "BTC/USDC", "MATICXBT", "FTMEUR", + ] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = {"BTC/USDC"} + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = ["BTC/USDC"] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDC"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock( + return_value=[ + {"symbol": "BTC/USDC"}, + {"symbol": "MATICXBT"}, + {"symbol": "FTMEUR"}, + ], + ) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + mock_merge.merge_and_persist_trading_history.assert_called_once_with( + "wallet1", + account.id, + [{"symbol": "BTC/USDC"}], + [], + ) + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.assert_called_once_with( + "wallet1", + exchange_config, + {"BTC/USDC"}, + ) + info_messages = " ".join(str(argument) for call in mock_logger.info.call_args_list for argument in call.args) + assert "Dropped" in info_messages + assert "delisted/unknown markets" in info_messages + assert "2" in info_messages + assert "FTMEUR" in info_messages + assert "MATICXBT" in info_messages + + +class TestDiscoverAndFetchTradeSymbols: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_discovers_portfolio_symbols_beyond_config_seeds( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "kraken" + exchange_config.sandboxed = False + exchange_config.id = "kraken-spot-config" + exchange_config.name = "kraken-spot-config" + exchange_config.historical_trade_symbols = ["SOL/USDC"] + context = _make_context(account, exchange_config=exchange_config, trade_symbols=["SOL/USDC"]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["ALGO/USDC", "KNC/USDC", "SOL/USDC"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + fetch_trades_paginated_mock = mock.AsyncMock(return_value=[]) + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated_mock + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert set(fetch_trades_paginated_mock.await_args.args[0]) == { + "ALGO/USDC", "KNC/USDC", "SOL/USDC", + } + assert results[0].trade_symbols_count == 3 + + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_fetches_deposits_before_trades( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + btc_asset = mock.MagicMock(symbol="BTC", total=1.0) + account.assets = [mock.MagicMock(assets=[btc_asset])] + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + call_order: list[str] = [] + + async def fetch_deposits(**_kwargs): + call_order.append("deposits") + return [] + + async def fetch_withdrawals(**_kwargs): + call_order.append("withdrawals") + return [] + + async def fetch_trades_paginated(*_args, **_kwargs): + call_order.append("trades") + return [] + + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = fetch_trades_paginated + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = fetch_deposits + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = fetch_withdrawals + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + assert call_order.index("deposits") < call_order.index("trades") + assert call_order.index("withdrawals") < call_order.index("trades") + + @pytest.mark.asyncio + @mock.patch("octobot_flow.repositories.exchange.trades_repository.logger") + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_logs_new_symbol_fetch_at_info( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, mock_trades_logger, + ): + account = _make_account("acc1") + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "kraken" + exchange_config.sandboxed = False + exchange_config.id = "kraken-spot-config" + exchange_config.name = "kraken-spot-config" + exchange_config.historical_trade_symbols = [] + context = _make_context(account, exchange_config=exchange_config, trade_symbols=[]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["ALGO/USDC"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.MagicMock() + mock_exchange_manager.client_symbols = ["ALGO/USDC"] + mock_exchange_manager.exchange.get_my_recent_trades = mock.AsyncMock(return_value=[]) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.side_effect = ( + lambda exchange_manager, *_args, **_kwargs: ( + trades_repository_module.TradesRepository(exchange_manager, [], mock.MagicMock()) + ) + ) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + info_messages = " ".join(str(argument) for call in mock_trades_logger.info.call_args_list for argument in call.args) + assert "Fetching trade history for new symbol" in info_messages + assert "ALGO/USDC" in info_messages + + @pytest.mark.asyncio + @mock.patch("octobot_flow.repositories.exchange.trades_repository.logger") + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_does_not_log_new_symbol_for_configured_symbol( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, mock_trades_logger, + ): + account = _make_account("acc1") + context = _make_context(account, trade_symbols=["BTC/USDT"]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["BTC/USDT"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = set() + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [] + + mock_exchange_manager = mock.MagicMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchange_manager.exchange.get_my_recent_trades = mock.AsyncMock(return_value=[]) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.side_effect = ( + lambda exchange_manager, *_args, **_kwargs: ( + trades_repository_module.TradesRepository(exchange_manager, [], mock.MagicMock()) + ) + ) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + info_messages = " ".join(str(call.args[0]) for call in mock_trades_logger.info.call_args_list) + assert "Fetching trade history for new symbol" not in info_messages + + +class TestPersistTradeConfirmedConfigFromJob: + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_persists_only_trade_confirmed_symbols( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + exchange_config = mock.MagicMock(spec=protocol_models.ExchangeConfig) + exchange_config.exchange = "kraken" + exchange_config.sandboxed = False + exchange_config.id = "kraken-spot-config" + exchange_config.name = "kraken-spot-config" + exchange_config.historical_trade_symbols = [] + context = _make_context(account, exchange_config=exchange_config, trade_symbols=["SOL/USDC"]) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["ALGO/USDC", "GLMR/USDC", "SOL/USDC"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = {"ALGO/USDC"} + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = ["ALGO/USDC"] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["ALGO/USDC", "GLMR/USDC", "SOL/USDC"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock( + return_value=[{"symbol": "ALGO/USDC"}], + ) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.assert_called_once_with( + "wallet1", + exchange_config, + {"ALGO/USDC"}, + ) + + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.logger") + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_logs_added_symbols_at_info( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, mock_logger, + ): + account = _make_account("acc1") + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = ["ALGO/USDC"] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = {"ALGO/USDC"} + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = ["ALGO/USDC"] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + await job.run() + + info_messages = " ".join(str(argument) for call in mock_logger.info.call_args_list for argument in call.args) + assert "Added trade-confirmed symbols" in info_messages + assert "ALGO/USDC" in info_messages + + @pytest.mark.asyncio + @mock.patch("octobot_flow.jobs.portfolio_history_job.collection_providers") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trade_symbols_discovery_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_exchanges") + @mock.patch("octobot_flow.jobs.portfolio_history_job.tentacles_manager_api") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trading_history_merge_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.daily_price_cache_updater_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.trades_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.transactions_repository_module") + @mock.patch("octobot_flow.jobs.portfolio_history_job.profile_data_factory_module") + async def test_trade_symbols_count_reflects_discovered_set( + self, mock_profile, mock_tx_repo, mock_trades_repo, + mock_daily_cache, mock_merge, mock_tentacles, mock_exchanges, + mock_discovery, mock_collection_providers, + ): + account = _make_account("acc1") + context = _make_context(account) + mock_collection_providers.AccountTradingProvider.instance.return_value.load_state.side_effect = ( + collection_errors.CollectionNoDataError + ) + mock_discovery.discover_trade_symbols.return_value = [ + "ALGO/USDC", "GLMR/USDC", "KNC/USDC", "SOL/USDC", "STRK/USDC", + ] + mock_discovery.trade_confirmed_symbols_from_fetched_trades.return_value = { + "ALGO/USDC", "KNC/USDC", "SOL/USDC", + } + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.return_value = [ + "ALGO/USDC", "KNC/USDC", "SOL/USDC", + ] + + mock_exchange_manager = mock.AsyncMock() + mock_exchange_manager.client_symbols = ["BTC/USDT"] + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aenter__ = mock.AsyncMock( + return_value=mock_exchange_manager + ) + mock_exchanges.exchange_manager_from_exchange_data.return_value.__aexit__ = mock.AsyncMock( + return_value=False + ) + mock_trades_repo.TradesRepository.ensure_temporary_trades_channel = mock.AsyncMock() + mock_trades_repo.TradesRepository.return_value.fetch_trades_paginated = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_deposits = mock.AsyncMock(return_value=[]) + mock_tx_repo.TransactionsRepository.return_value.fetch_withdrawals = mock.AsyncMock(return_value=[]) + mock_daily_cache.update_daily_prices = mock.AsyncMock() + + job = portfolio_history_job_module.PortfolioHistoryJob("wallet1", [context]) + results = await job.run() + + assert results[0].trade_symbols_count == 5 + mock_discovery.persist_trade_confirmed_symbols_to_exchange_config.assert_called_once_with( + "wallet1", + context.exchange_config, + {"ALGO/USDC", "KNC/USDC", "SOL/USDC"}, + ) diff --git a/packages/flow/tests/logic/accounts/test_account_state_persistence.py b/packages/flow/tests/logic/accounts/test_account_state_persistence.py new file mode 100644 index 0000000000..eb8b0060eb --- /dev/null +++ b/packages/flow/tests/logic/accounts/test_account_state_persistence.py @@ -0,0 +1,433 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot.community.wallet_backend.errors as wallet_backend_errors_module +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _portfolio_snapshot(total: float) -> protocol_models.PortfolioHistoricalValue: + return protocol_models.PortfolioHistoricalValue( + timestamp=_TEST_TIMESTAMP, + total=total, + ) + + +def _protocol_order(exchange_id: str) -> protocol_models.Order: + return protocol_models.Order( + id=exchange_id, + symbol="BTC/USDT", + price=10000.0, + quantity=0.01, + filled=0.0, + exchange_id=exchange_id, + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_TEST_TIMESTAMP, + ) + + +class TestLoadPreviousOpenOrderExchangeIds: + def test_returns_exchange_ids_from_persisted_orders(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[ + _protocol_order("order-1"), + _protocol_order("order-2"), + ], + ), + ) + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + assert exchange_ids == {"order-1", "order-2"} + + def test_returns_empty_set_when_no_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + exchange_ids = account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + assert exchange_ids == set() + + def test_propagates_unexpected_load_state_errors(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = RuntimeError("disk failure") + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.load_previous_open_order_exchange_ids( + "wallet-1", + "account-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "disk failure" + + +class TestLoadPreviousOpenOrders: + def test_returns_storage_dicts_from_persisted_orders(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[_protocol_order("order-1")], + ), + ) + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + open_orders = account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + assert len(open_orders) == 1 + inner_order = open_orders[0][trading_constants.STORAGE_ORIGIN_VALUE] + assert inner_order[trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value] == "order-1" + assert inner_order[trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value] == "BTC/USDT" + + def test_returns_empty_list_when_no_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + open_orders = account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + assert open_orders == [] + + def test_propagates_unexpected_load_state_errors(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = RuntimeError("disk failure") + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.load_previous_open_orders( + "wallet-1", + "account-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "disk failure" + + +class TestBuildPortfolioHistoryState: + def test_merges_snapshot_into_history_state(self): + history_provider = mock.Mock() + history_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountHistoryProvider, + "instance", + return_value=history_provider, + ): + history_state = account_state_persistence_module.build_portfolio_history_state( + "wallet-1", + "account-1", + _portfolio_snapshot(1500.0), + "USDC", + _TEST_TIMESTAMP, + ) + assert history_state.history is not None + assert history_state.history.unit == "USDC" + assert history_state.history.values[-1].total == 1500.0 + + +class TestPersistAccountTrading: + def test_saves_updated_trading_state(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[], + trades=[], + positions=[], + ) + trading_provider.save_state.assert_called_once() + + def test_raises_when_trading_state_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[], + trades=[], + positions=[], + ) + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + trading_provider.save_state.assert_not_called() + + def test_fills_id_from_exchange_id_when_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + order_without_id = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "exch-order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.01, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + trading_enums.ExchangeConstantsOrderColumns.REDUCE_ONLY.value: False, + trading_enums.ExchangeConstantsOrderColumns.IS_ACTIVE.value: True, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[order_without_id], + trades=[], + positions=[], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_orders = saved_state.account_trading.orders + assert len(saved_orders) == 1 + assert saved_orders[0].id == "exch-order-1" + assert saved_orders[0].exchange_id == "exch-order-1" + + def test_persists_ccxt_order_missing_optional_fields(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + ccxt_like_order = { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.01, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[ccxt_like_order], + trades=[], + positions=[], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_order = saved_state.account_trading.orders[0] + assert saved_order.id == "order-1" + assert saved_order.trigger_above is None + assert saved_order.reduce_only is False + assert saved_order.is_active is True + + def test_raises_when_order_missing_id_and_exchange_id(self): + trading_provider = mock.Mock() + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + ), + ) + order_without_ids = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading( + "wallet-1", + "account-1", + orders=[order_without_ids], + trades=[], + positions=[], + ) + raise AssertionError("Expected ValueError") + except ValueError as error: + assert "missing both id and exchange_id" in str(error) + trading_provider.save_state.assert_not_called() + + +class TestPersistAccountTradingFromIterationState: + def test_persists_trading_snapshot_from_automation_state(self): + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + elements = octobot_flow.entities.ExchangeAccountElements() + elements.orders.open_orders = [{"exchange_id": "order-1"}] + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + with mock.patch.object( + account_state_persistence_module, + "persist_account_trading", + ) as persist_account_trading_mock: + account_state_persistence_module.persist_account_trading_from_iteration_state( + "wallet-1", + automation_state.to_dict(include_default_values=False), + ) + persist_account_trading_mock.assert_called_once_with( + "wallet-1", + "acc-sync-1", + [{"exchange_id": "order-1"}], + [], + [], + ) + + def test_skips_when_wallet_not_registered(self): + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + elements = octobot_flow.entities.ExchangeAccountElements() + elements.orders.open_orders = [{"exchange_id": "order-1"}] + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + with mock.patch.object( + account_state_persistence_module, + "persist_account_trading", + side_effect=wallet_backend_errors_module.WalletNotFoundError("Wallet not found"), + ): + account_state_persistence_module.persist_account_trading_from_iteration_state( + "wallet-1", + automation_state.to_dict(include_default_values=False), + ) + + +class TestTrimLiveTradesInIterationState: + def test_trims_exchange_account_elements_in_state_dict(self): + order_columns = trading_enums.ExchangeConstantsOrderColumns + state_dict = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=octobot_flow.entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(3) + ], + ), + ), + ).to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state(state_dict, 2) + restored_state = octobot_flow.entities.AutomationState.from_dict(state_dict) + elements = restored_state.automation.exchange_account_elements + assert elements is not None + assert len(elements.trades) == 2 + assert elements.trade_summaries == {"BTC/USDT": ["trade-0"]} + + def test_no_op_when_trade_count_within_window(self): + order_columns = trading_enums.ExchangeConstantsOrderColumns + state_dict = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=octobot_flow.entities.ExchangeAccountElements( + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "trade-1", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: 1.0, + }, + ], + trade_summaries={"ETH/USDT": ["archived-1"]}, + ), + ), + ).to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state(state_dict, 100) + restored_state = octobot_flow.entities.AutomationState.from_dict(state_dict) + elements = restored_state.automation.exchange_account_elements + assert elements is not None + assert len(elements.trades) == 1 + assert elements.trade_summaries == {"ETH/USDT": ["archived-1"]} diff --git a/packages/flow/tests/logic/accounts/test_portfolio_history.py b/packages/flow/tests/logic/accounts/test_portfolio_history.py new file mode 100644 index 0000000000..aab9ad056d --- /dev/null +++ b/packages/flow/tests/logic/accounts/test_portfolio_history.py @@ -0,0 +1,70 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import octobot_flow.constants as flow_constants +import octobot_flow.logic.accounts.portfolio_history as portfolio_history_module +import octobot_protocol.models as protocol_models + + +def _snapshot(timestamp: datetime.datetime, total: float) -> protocol_models.PortfolioHistoricalValue: + return protocol_models.PortfolioHistoricalValue( + timestamp=timestamp, + total=total, + ) + + +class TestMergeSnapshotLatest: + def test_every_call_replaces_latest_entry(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 1, 12, 5, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 110.0), + second_time, + ) + assert len(merged_values) == 2 + assert merged_values[-1].total == 110.0 + assert merged_values[0].total == 100.0 + + +class TestMergeSnapshot12hCadence: + def test_refresh_within_12h_does_not_add_second_twelve_hour_entry(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 1, 15, 0, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 105.0), + second_time, + ) + assert len(merged_values) == 2 + assert merged_values[0].timestamp == first_time + assert merged_values[-1].timestamp == second_time + + +class TestMergeSnapshot12hElapsed: + def test_appends_new_twelve_hour_snapshot_after_interval(self): + first_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + second_time = first_time + datetime.timedelta( + seconds=flow_constants.PORTFOLIO_HISTORY_SNAPSHOT_INTERVAL_SECONDS + ) + merged_values = portfolio_history_module.merge_snapshot( + [_snapshot(first_time, 100.0)], + _snapshot(second_time, 120.0), + second_time, + ) + assert len(merged_values) == 3 + assert merged_values[0].timestamp == first_time + assert merged_values[1].timestamp == second_time + assert merged_values[-1].timestamp == second_time + + +class TestMergeSnapshotUnit: + def test_unit_preserved_by_caller_not_merge_snapshot(self): + evaluation_time = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + merged_values = portfolio_history_module.merge_snapshot( + [], + _snapshot(evaluation_time, 50.0), + evaluation_time, + ) + assert merged_values[0].total == 50.0 diff --git a/packages/flow/tests/logic/actions/test_actions_executor_merge.py b/packages/flow/tests/logic/actions/test_actions_executor_merge.py index 18b2ff299f..e91916bd20 100644 --- a/packages/flow/tests/logic/actions/test_actions_executor_merge.py +++ b/packages/flow/tests/logic/actions/test_actions_executor_merge.py @@ -1,5 +1,6 @@ import mock +import octobot_commons.dsl_interpreter import octobot_commons.profiles as commons_profiles import octobot_trading.enums as octobot_trading_enums_import import octobot_trading.exchanges.util.exchange_data as exchange_data_import @@ -84,3 +85,129 @@ def test_merges_transactions_only_once_per_txid(self): automation.exchange_account_elements.merge_synchronized_snapshots([snap1, snap2]) txs = automation.exchange_account_elements.transactions assert [t[tx_key] for t in txs] == ["tx-existing", "tx-a", "tx-b"] + + +class TestClearPostIterationAfterMerge: + def test_clears_updated_exchange_account_elements_from_recall_wrapper(self): + post_iteration_name = octobot_flow_entities.PostIterationActionsDetails.__name__ + action_result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": {"trades": []}, + "stop_automation": False, + }, + "pid": 42, + }, + } + } + octobot_flow_entities.PostIterationActionsDetails.post_iteration_clear_from_action_result( + action_result + ) + inner_post_iteration = action_result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + assert inner_post_iteration["stop_automation"] is False + + def test_clear_after_sync_clears_post_iteration_on_action_result(self): + post_iteration_name = octobot_flow_entities.PostIterationActionsDetails.__name__ + action = octobot_flow_entities.DSLScriptActionDetails( + id="process-action", + dsl_script="run_octobot_process('folder')", + ) + action.result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + post_iteration_name: { + "updated_exchange_account_elements": { + "trades": [_trade_stub("merged-trade")], + }, + }, + }, + } + } + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + snapshot = octobot_flow_entities.ExchangeAccountElements( + trades=[_trade_stub("merged-trade")], + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [action], + False, + ) + executor_action._sync_after_execution([snapshot]) + executor_action._clear_post_iteration_after_merge_from_action_results() + inner_post_iteration = action.result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"][post_iteration_name] + assert "updated_exchange_account_elements" not in inner_post_iteration + trade_id_key = octobot_trading_enums_import.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value + assert automation.exchange_account_elements.trades[0][trade_id_key] == "merged-trade" + + def test_clear_without_post_iteration_eae_preserves_recall_fields(self): + action = octobot_flow_entities.DSLScriptActionDetails( + id="process-action", + dsl_script="run_octobot_process('folder')", + ) + action.result = { + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__: { + "last_execution_result": { + "pid": 42, + "init_state_ok": True, + }, + } + } + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [action], + False, + ) + executor_action._clear_post_iteration_after_merge_from_action_results() + inner_last = action.result[ + octobot_commons.dsl_interpreter.ReCallingOperatorResult.__name__ + ]["last_execution_result"] + assert inner_last["pid"] == 42 + assert inner_last["init_state_ok"] is True + + +class TestMergeThenTrimIntegration: + def test_large_snapshot_merge_then_trim_obey_live_window(self): + order_columns = octobot_trading_enums_import.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(120) + ] + snapshot = octobot_flow_entities.ExchangeAccountElements(trades=trades) + automation = octobot_flow_entities.AutomationDetails( + metadata=octobot_flow_entities.AutomationMetadata(automation_id="aid"), + exchange_account_elements=octobot_flow_entities.ExchangeAccountElements(), + ) + executor_action = actions_executor_import.ActionsExecutor( + None, + None, + commons_profiles.ProfileData(), + automation, + [], + False, + ) + executor_action._sync_after_execution([snapshot]) + automation.exchange_account_elements.trim_trades_to_live_window(100) + assert len(automation.exchange_account_elements.trades) == 100 + assert len(automation.exchange_account_elements.trade_summaries.get("BTC/USDT", [])) == 20 diff --git a/packages/flow/tests/logic/configuration/test_profile_data_factory.py b/packages/flow/tests/logic/configuration/test_profile_data_factory.py index 396e84ec88..0231d86f68 100644 --- a/packages/flow/tests/logic/configuration/test_profile_data_factory.py +++ b/packages/flow/tests/logic/configuration/test_profile_data_factory.py @@ -1,7 +1,12 @@ # Drakkar-Software OctoBot-Flow # Copyright (c) Drakkar-Software, All rights reserved. +import datetime + +import octobot_commons.constants as commons_constants import octobot_commons.profiles.profile_data as profile_data_module +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums import octobot_trading.exchanges.util.exchange_data as exchange_data_module import octobot_flow.entities.accounts.exchange_account_details as exchange_account_details_module @@ -43,3 +48,78 @@ def test_omits_hollaex_tentacle_when_url_missing(self): as_simulator=True, ) assert profile_data.get_config_by_tentacle() == {} + + +class TestProfileDataForAccount: + def test_enables_trader_for_real_account(self): + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="account-1", + name="Real account", + is_simulated=False, + created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + exchange_account, + exchange_config, + protocol_models.TradingType.SPOT, + is_simulated=False, + ) + assert profile_data.trader.enabled is True + assert profile_data.trader_simulator.enabled is False + assert profile_data.exchanges[0].internal_name == "binanceus" + + def test_enables_simulator_for_simulated_account(self): + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="sim-account-1", + name="Sim account", + is_simulated=True, + created_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + profile_data = profile_data_factory_module.profile_data_for_account( + account, + exchange_account, + exchange_config, + protocol_models.TradingType.SPOT, + is_simulated=True, + ) + assert profile_data.trader.enabled is False + assert profile_data.trader_simulator.enabled is True + + +class TestInferReferenceMarket: + def test_returns_default_reference_market_when_internal_name_missing(self): + exchange_account_details = exchange_account_details_module.ExchangeAccountDetails( + exchange_details=profile_data_module.ExchangeData(), + auth_details=exchange_data_module.ExchangeAuthDetails(), + ) + assert ( + profile_data_factory_module.infer_reference_market(exchange_account_details, []) + == commons_constants.DEFAULT_REFERENCE_MARKET + ) diff --git a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py index 0b32f94f2d..908c4ef21f 100644 --- a/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py +++ b/packages/flow/tests/logic/dsl/test_dsl_action_execution_context.py @@ -5,6 +5,8 @@ import octobot_trading.enums import octobot_trading.errors +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.logic.dsl.dsl_action_execution_context @@ -243,3 +245,27 @@ async def execute_action(self, action, **_kwargs): assert action.error_status == expected_error_status.value assert action.error_message == str(raised_exception) + + +class TestDslActionExecutionReraisesOutdatedReferenceAccountError: + @pytest.mark.asyncio + async def test_reraises_outdated_reference_account_error(self): + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + + class StubExecutor: + @octobot_flow.logic.dsl.dsl_action_execution_context.dsl_action_execution + async def execute_action(self, action, **_kwargs): + raise outdated_error + + action = octobot_flow.entities.DSLScriptActionDetails( + id="copy_1", + dsl_script="copy_exchange_account()", + ) + stub_executor = StubExecutor() + + with pytest.raises(copy_errors.OutdatedReferenceAccountError) as raised_error: + await stub_executor.execute_action(action) + + assert str(raised_error.value) == str(outdated_error) + assert action.error_status is None + assert action.error_message is None diff --git a/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py b/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py new file mode 100644 index 0000000000..2ead3c6c33 --- /dev/null +++ b/packages/flow/tests/logic/exchange/orders/test_order_change_detection.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Flow + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module + + +def _order_dict(exchange_id: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +class TestDetectChangedOrderIds: + def test_returns_disappeared_order_ids(self): + previous_ids = {"order-1", "order-2"} + current_orders = [_order_dict("order-2")] + changed_ids = order_change_detection_module.detect_changed_order_ids( + previous_ids, + current_orders, + ) + assert changed_ids == {"order-1"} + + def test_returns_empty_when_no_previous_orders(self): + changed_ids = order_change_detection_module.detect_changed_order_ids( + set(), + [_order_dict("order-1")], + ) + assert changed_ids == set() diff --git a/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py b/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py new file mode 100644 index 0000000000..3e0f8026bb --- /dev/null +++ b/packages/flow/tests/logic/exchange/simulator/test_simulated_order_fill_detector.py @@ -0,0 +1,128 @@ +# Drakkar-Software OctoBot-Flow + +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.exchange.simulator.simulated_order_fill_detector as simulated_order_fill_detector_module + + +def _open_order_storage_dict( + exchange_id: str, + *, + price: float = 10000.0, + symbol: str = "BTC/USDT", + trigger_above: bool | None = None, + side: str | None = None, +) -> dict: + order_columns = trading_enums.ExchangeConstantsOrderColumns + inner_order = { + order_columns.EXCHANGE_ID.value: exchange_id, + order_columns.ID.value: exchange_id, + order_columns.SYMBOL.value: symbol, + order_columns.PRICE.value: price, + } + if trigger_above is not None: + inner_order[order_columns.TRIGGER_ABOVE.value] = trigger_above + if side is not None: + inner_order[order_columns.SIDE.value] = side + return { + trading_constants.STORAGE_ORIGIN_VALUE: inner_order, + } + + +class TestIsSimulatedOrderFilled: + def test_trigger_above_false_fills_when_market_at_or_below_order_price(self): + assert simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=9000.0, + trigger_above=False, + ) + + def test_trigger_above_false_stays_open_when_market_above_order_price(self): + assert not simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=11000.0, + trigger_above=False, + ) + + def test_trigger_above_true_fills_when_market_at_or_above_order_price(self): + assert simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=11000.0, + trigger_above=True, + ) + + def test_trigger_above_true_stays_open_when_market_below_order_price(self): + assert not simulated_order_fill_detector_module.is_simulated_order_filled( + order_price=10000.0, + market_price=9000.0, + trigger_above=True, + ) + + +class TestResolveSimulatedOpenOrders: + def test_drops_filled_orders_and_keeps_open_ones(self): + open_orders = [ + _open_order_storage_dict("filled-order", price=10000.0, trigger_above=False), + _open_order_storage_dict("open-order", price=8000.0, trigger_above=False), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 9000.0}, + ) + remaining_exchange_ids = { + order[trading_constants.STORAGE_ORIGIN_VALUE][ + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value + ] + for order in still_open_orders + } + assert remaining_exchange_ids == {"open-order"} + + def test_keeps_order_open_when_ticker_missing(self): + open_orders = [_open_order_storage_dict("order-1", trigger_above=False)] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {}, + ) + assert len(still_open_orders) == 1 + + def test_infers_trigger_above_from_sell_side_when_missing(self): + open_orders = [ + _open_order_storage_dict( + "filled-sell", + price=10000.0, + side=trading_enums.TradeOrderSide.SELL.value, + ), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 11000.0}, + ) + assert still_open_orders == [] + + def test_infers_trigger_above_from_buy_side_when_missing(self): + open_orders = [ + _open_order_storage_dict( + "filled-buy", + price=10000.0, + side=trading_enums.TradeOrderSide.BUY.value, + ), + ] + still_open_orders = simulated_order_fill_detector_module.resolve_simulated_open_orders( + open_orders, + {"BTC/USDT": 9000.0}, + ) + assert still_open_orders == [] + + +class TestSymbolsFromOpenOrders: + def test_returns_unique_sorted_symbols(self): + open_orders = [ + _open_order_storage_dict("order-1", symbol="ETH/USDT"), + _open_order_storage_dict("order-2", symbol="BTC/USDT"), + _open_order_storage_dict("order-3", symbol="BTC/USDT"), + ] + assert simulated_order_fill_detector_module.symbols_from_open_orders(open_orders) == [ + "BTC/USDT", + "ETH/USDT", + ] diff --git a/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py new file mode 100644 index 0000000000..8951f74360 --- /dev/null +++ b/packages/flow/tests/logic/exchange/simulator/test_simulated_portfolio_seeder.py @@ -0,0 +1,75 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models + +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +class TestSeedSimulatedPortfolio: + def test_seeds_portfolio_from_account_assets(self): + account = protocol_models.Account( + id="sim-account-1", + name="Simulated", + is_simulated=True, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=1000.0, + available=900.0, + ), + ], + ), + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ), + ), + ) + portfolio_manager = mock.Mock() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + portfolio_manager.apply_forced_portfolio.assert_called_once_with( + { + "USDT": { + commons_constants.PORTFOLIO_AVAILABLE: 900.0, + commons_constants.PORTFOLIO_TOTAL: 1000.0, + }, + }, + update_available_funds_from_open_orders=True, + ) + + def test_skips_when_account_has_no_assets(self): + account = protocol_models.Account( + id="sim-account-2", + name="Simulated empty", + is_simulated=True, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.GenericAccount( + account_type=protocol_models.AccountType.GENERIC, + ), + ), + ) + portfolio_manager = mock.Mock() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + portfolio_manager.apply_forced_portfolio.assert_not_called() diff --git a/packages/flow/tests/logic/global_view/portfolio_test_util.py b/packages/flow/tests/logic/global_view/portfolio_test_util.py new file mode 100644 index 0000000000..303ddbc332 --- /dev/null +++ b/packages/flow/tests/logic/global_view/portfolio_test_util.py @@ -0,0 +1,71 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import decimal + +import mock + +import octobot_commons.constants as commons_constants + + +def wire_portfolio_pipeline( + exchange_manager, + portfolio_content: dict, + *, + portfolio_total: float = 1500.0, + exchange_name: str = "binanceus", +) -> None: + stored_portfolio = { + currency: dict(balances) + for currency, balances in portfolio_content.items() + } + portfolio = mock.Mock() + portfolio.portfolio = stored_portfolio + portfolio_manager = mock.Mock() + portfolio_manager.portfolio = portfolio + portfolio_manager.reference_market = "USDC" + portfolio_value_holder = mock.Mock() + portfolio_value_holder.portfolio_current_value = portfolio_total + portfolio_value_holder.current_crypto_currencies_values = { + "BTC": decimal.Decimal("50000"), + "USDT": decimal.Decimal("1"), + } + value_converter = mock.Mock() + + def convert_using_last_prices(amount, currency, _reference_market): + if currency == "BTC": + return decimal.Decimal("50000") * amount + return decimal.Decimal("1") * amount + + value_converter.convert_currency_value_using_last_prices = convert_using_last_prices + portfolio_value_holder.value_converter = value_converter + portfolio_manager.portfolio_value_holder = portfolio_value_holder + portfolio_manager.portfolio_history_update = contextlib.nullcontext + + def handle_balance_update(balance, is_diff_update=False): + for currency, amounts in balance.items(): + if isinstance(amounts, dict): + stored_portfolio[currency] = amounts + else: + stored_portfolio[currency] = { + commons_constants.PORTFOLIO_AVAILABLE: amounts, + commons_constants.PORTFOLIO_TOTAL: amounts, + } + return True + + portfolio_manager.handle_balance_update = mock.Mock(side_effect=handle_balance_update) + portfolio_manager.resolve_pending_portfolio_update_events_if_any = mock.AsyncMock() + exchange_personal_data = exchange_manager.exchange_personal_data + exchange_personal_data.portfolio_manager = portfolio_manager + + async def handle_portfolio_update(balance, should_notify=False, is_diff_update=False): + portfolio_manager.handle_balance_update(balance, is_diff_update=is_diff_update) + await portfolio_manager.resolve_pending_portfolio_update_events_if_any() + return True + + exchange_personal_data.handle_portfolio_update = mock.AsyncMock(side_effect=handle_portfolio_update) + exchange_personal_data.handle_portfolio_profitability_update = mock.AsyncMock() + exchange_manager.get_symbol_data = mock.Mock(return_value=mock.Mock()) + exchange_manager.client_symbols = ["BTC/USDC", "USDC/BTC", "BTC/USDT", "ETH/USDT"] + exchange_manager.symbol_exists = mock.Mock(return_value=True) + exchange_manager.exchange_name = exchange_name diff --git a/packages/flow/tests/logic/global_view/test_account_refresh_builder.py b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py new file mode 100644 index 0000000000..506ea3b61a --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_account_refresh_builder.py @@ -0,0 +1,68 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import octobot_protocol.models as protocol_models +import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.enums as trading_enums + +import octobot_flow.entities +import octobot_flow.logic.global_view.account_refresh_builder as account_refresh_builder_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _context() -> octobot_flow.entities.GlobalViewAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="account-1", + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id="account-1", + name="Test account", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + historical_trade_symbols=["BTC/USDT"], + ), + trading_type=protocol_models.TradingType.SPOT, + auth_details=exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id="account-1", + ), + ) + + +class TestBuildGlobalViewAccountRefreshResult: + def test_builds_refresh_result_without_history(self): + context = _context() + exchange_refresh_result = octobot_flow.entities.ExchangeAccountRefreshResult( + assets=[], + ticker_closes={"BTC/USDT": 65000.0}, + valuation_unit="USDC", + open_orders=[], + trades=[], + positions=[], + changed_order_ids={"gone-order"}, + ) + refresh_result = account_refresh_builder_module.build_global_view_account_refresh_result( + "wallet-1", + context, + exchange_refresh_result, + ) + assert refresh_result.updated_account.id == "account-1" + assert refresh_result.changed_order_ids == {"gone-order"} + assert not hasattr(refresh_result, "portfolio_history_state") or refresh_result.__dict__.get("portfolio_history_state") is None diff --git a/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py new file mode 100644 index 0000000000..936c4091d8 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_exchange_account_refresh.py @@ -0,0 +1,534 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime +import decimal + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.api as trading_api +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.errors as trading_errors + +import octobot_flow.logic.exchange.simulator.simulated_portfolio_seeder as simulated_portfolio_seeder_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline + + +def _open_order_dict(exchange_id: str, symbol: str) -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +def _wire_portfolio_pipeline(exchange_manager, portfolio_content: dict, **kwargs) -> None: + wire_portfolio_pipeline(exchange_manager, portfolio_content, **kwargs) + + +class TestFetchOpenOrdersForSymbols: + @pytest.mark.asyncio + async def test_bad_symbol_skips_that_symbol_and_keeps_others(self): + eth_order = _open_order_dict("stays-order-2", "ETH/USDT") + + async def get_open_orders(*, symbol=None, **_kwargs): + if symbol == "STRK/USDC": + raise trading_errors.UnSupportedSymbolError( + "ob_kraken does not have market symbol STRK/USDC" + ) + if symbol == "ETH/USDT": + return [eth_order] + raise AssertionError(f"Unexpected symbol: {symbol}") + + exchange_manager = mock.Mock() + exchange_manager.exchange.get_open_orders = mock.AsyncMock(side_effect=get_open_orders) + + open_orders = await exchange_account_refresh_module._fetch_open_orders_for_symbols( + exchange_manager, + ["STRK/USDC", "ETH/USDT"], + ) + + assert open_orders == [eth_order] + + +class TestRefreshExchangeAccountPortfolio: + @pytest.mark.asyncio + async def test_real_refresh_applies_balance_to_portfolio_manager(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, {}) + ensure_ticker_channel_mock = mock.AsyncMock() + refresh_valuation_mock = mock.AsyncMock() + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + refresh_valuation_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + exchange_manager.exchange_personal_data.handle_portfolio_update.assert_awaited_once() + applied_balance = exchange_manager.exchange_personal_data.handle_portfolio_update.await_args.args[0] + assert applied_balance["BTC"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("0.1") + assert applied_balance["USDT"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("1000.0") + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.assets + for asset in assets_for_trading_type.assets + } + assert asset_symbols == {"USDT", "BTC"} + + @pytest.mark.asyncio + async def test_real_refresh_without_handle_portfolio_update_would_fail(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, {}, portfolio_total=0.0) + exchange_manager.exchange_personal_data.handle_portfolio_update = mock.AsyncMock(return_value=False) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + assert refresh_result.assets == [] + assert refresh_result.ticker_closes == {} + + @pytest.mark.asyncio + async def test_simulated_refresh_uses_seeded_account_assets(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline( + exchange_manager, + { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + }, + portfolio_total=1000.0, + ) + portfolio_manager = exchange_manager.exchange_personal_data.portfolio_manager + portfolio_manager.apply_forced_portfolio = mock.Mock( + side_effect=lambda portfolio_content, update_available_funds_from_open_orders=False: ( + portfolio_manager.handle_balance_update(portfolio_content) + ), + ) + account = protocol_models.Account( + id="sim-account-1", + name="Simulated", + is_simulated=True, + created_at=datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC), + updated_at=datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDC", + total=1000.0, + available=1000.0, + ), + ], + ), + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="sim-account-1", + exchange_config_ids=["exchange-config-1"], + ), + ), + ) + simulated_portfolio_seeder_module.seed_simulated_portfolio(exchange_manager, account) + fetch_tickers_mock = mock.AsyncMock(return_value={}) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + fetch_tickers_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + is_simulated=True, + previous_open_orders=[], + ) + + fetch_tickers_mock.assert_awaited_once_with(exchange_manager, []) + + portfolio_manager.apply_forced_portfolio.assert_called_once() + asset_symbols = { + asset.symbol + for assets_for_trading_type in refresh_result.assets + for asset in assets_for_trading_type.assets + } + assert "USDC" in asset_symbols + + @pytest.mark.asyncio + async def test_real_refresh_portfolio_total_positive_with_ticker_prices(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline(exchange_manager, balance_content, portfolio_total=6000.0) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + ): + refresh_result = await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + assert isinstance(refresh_result.ticker_closes, dict) + + +class TestValuationSymbolsFromPortfolio: + def test_skips_zero_holdings_and_valuation_unit(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["BTC/USDC"] + portfolio_content = { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + "ETH": { + commons_constants.PORTFOLIO_TOTAL: 0.0, + commons_constants.PORTFOLIO_AVAILABLE: 0.0, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "USDC", + ) + assert valuation_symbols == ["BTC/USDC"] + + def test_uses_bridge_symbols_when_direct_valuation_pair_missing(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["APT/USD", "USDT/USD"] + portfolio_content = { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "APT": { + commons_constants.PORTFOLIO_TOTAL: 10.0, + commons_constants.PORTFOLIO_AVAILABLE: 10.0, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "USDT", + ) + assert set(valuation_symbols) == {"APT/USD", "USDT/USD"} + + def test_does_not_use_usd_like_bridge_when_valuation_unit_is_not_usd_like(self): + exchange_manager = mock.Mock() + exchange_manager.client_symbols = ["APT/USD", "BTC/EUR"] + portfolio_content = { + "EUR": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "APT": { + commons_constants.PORTFOLIO_TOTAL: 10.0, + commons_constants.PORTFOLIO_AVAILABLE: 10.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + valuation_symbols = exchange_account_refresh_module._valuation_symbols_from_portfolio( + exchange_manager, + portfolio_content, + "EUR", + ) + assert valuation_symbols == ["BTC/EUR"] + + +class TestRefreshPortfolioValuation: + @pytest.mark.asyncio + async def test_syncs_portfolio_value_without_profitability_update(self): + exchange_manager = mock.Mock() + portfolio_manager = mock.Mock() + portfolio_value_holder = mock.Mock() + portfolio_manager.portfolio_value_holder = portfolio_value_holder + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + exchange_manager.symbol_exists = mock.Mock(return_value=False) + with ( + mock.patch.object( + trading_api, + "get_portfolio", + return_value={ + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + }, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + return_value=[], + ), + ): + await exchange_account_refresh_module._refresh_portfolio_valuation( + exchange_manager, + "USDT", + ) + + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values.assert_called_once_with( + init_price_fetchers=False, + ) + exchange_manager.exchange_personal_data.handle_portfolio_profitability_update.assert_not_called() + portfolio_manager.handle_mark_price_update.assert_not_called() + + @pytest.mark.asyncio + async def test_applies_ticker_prices_via_update_last_price_not_handle_mark_price_update(self): + exchange_manager = mock.Mock() + portfolio_manager = mock.Mock() + portfolio_value_holder = mock.Mock() + value_converter = mock.Mock() + portfolio_value_holder.value_converter = value_converter + portfolio_manager.portfolio_value_holder = portfolio_value_holder + exchange_manager.exchange_personal_data.portfolio_manager = portfolio_manager + exchange_manager.symbol_exists = mock.Mock(return_value=False) + tickers = { + "BTC/USDT": { + trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 50000, + }, + } + with ( + mock.patch.object( + trading_api, + "get_portfolio", + return_value={ + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + }, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + return_value=["BTC/USDT"], + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value=tickers), + ), + ): + await exchange_account_refresh_module._refresh_portfolio_valuation( + exchange_manager, + "USDT", + ) + + value_converter.update_last_price.assert_called_once_with( + "BTC/USDT", + decimal.Decimal("50000"), + ) + portfolio_manager.handle_mark_price_update.assert_not_called() + portfolio_value_holder._sync_portfolio_current_value_using_available_currencies_values.assert_called_once_with( + init_price_fetchers=False, + ) + + +class TestRefreshExchangeAccountLogging: + @pytest.mark.asyncio + async def test_logs_fetched_portfolio_once_at_info(self): + balance_content = _portfolio_content() + exchange_manager = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=balance_content) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + exchange_manager.exchange_personal_data = mock.Mock() + _wire_portfolio_pipeline( + exchange_manager, balance_content, portfolio_total=6000.0, exchange_name="binance", + ) + logger_mock = mock.Mock() + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + exchange_account_refresh_module, + "_get_logger", + return_value=logger_mock, + ), + ): + await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + fetch_open_orders=False, + ) + + portfolio_log_calls = [ + call_args + for call_args in logger_mock.info.call_args_list + if call_args.args and call_args.args[0] == "Fetched [%s] full [%s] portfolio: %s" + ] + assert len(portfolio_log_calls) == 1 + assert portfolio_log_calls[0].args[1] == "binance" + assert portfolio_log_calls[0].args[2] == "real" + + +class TestSimulatedTickerFetchMerge: + @pytest.mark.asyncio + async def test_fetches_order_and_valuation_symbols_once(self): + exchange_manager = mock.Mock() + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDT") + portfolio_content = _portfolio_content() + _wire_portfolio_pipeline(exchange_manager, portfolio_content, exchange_name="bitmart") + open_order = _open_order_dict("order-1", "ETH/USDT") + fetch_tickers_mock = mock.AsyncMock(return_value={ + "ETH/USDT": {trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 3000}, + "BTC/USDT": {trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 50000}, + }) + with ( + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + mock.AsyncMock(), + ), + mock.patch.object( + trading_api, + "get_portfolio", + return_value=portfolio_content, + ), + mock.patch.object( + exchange_account_refresh_module, + "_valuation_symbols_from_portfolio", + side_effect=[["BTC/USDT"], ["BTC/USDT"]], + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + fetch_tickers_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + ): + await exchange_account_refresh_module.refresh_exchange_account( + exchange_manager, + protocol_models.TradingType.SPOT, + set(), + is_simulated=True, + previous_open_orders=[open_order], + ) + + fetch_tickers_mock.assert_awaited_once() + fetched_symbols = fetch_tickers_mock.await_args.args[1] + assert set(fetched_symbols) == {"BTC/USDT", "ETH/USDT"} diff --git a/packages/flow/tests/logic/global_view/test_global_view_account_job.py b/packages/flow/tests/logic/global_view/test_global_view_account_job.py new file mode 100644 index 0000000000..45e471f03a --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_global_view_account_job.py @@ -0,0 +1,473 @@ +# Drakkar-Software OctoBot-Flow + +import contextlib +import datetime + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_data as exchange_data_module + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.exchange.orders.order_change_detection as order_change_detection_module +import octobot_flow.logic.global_view.exchange_account_refresh as exchange_account_refresh_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module +import octobot_sync.constants as sync_constants +from tests.logic.global_view.portfolio_test_util import wire_portfolio_pipeline + + +def _open_order_dict(exchange_id: str, symbol: str = "BTC/USDT") -> dict: + return { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: exchange_id, + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: symbol, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0, + } + } + + +def _portfolio_content() -> dict: + return { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: 1000.0, + commons_constants.PORTFOLIO_AVAILABLE: 1000.0, + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: 0.1, + commons_constants.PORTFOLIO_AVAILABLE: 0.1, + }, + } + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _exchange_account_context( + *, + account_id: str = "account-1", + is_simulated: bool = False, + has_bound_automation: bool = False, +) -> octobot_flow.entities.GlobalViewAccountContext: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=["exchange-config-1"], + ) + account = protocol_models.Account( + id=account_id, + name="Test account", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + exchange_config = protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + auth_details = exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + exchange_account_id=account_id, + ) + return octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=protocol_models.TradingType.SPOT, + auth_details=auth_details, + has_bound_automation=has_bound_automation, + ) + + +def _empty_portfolio_history_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +@pytest.mark.asyncio +class TestGlobalViewAccountJobRun: + async def test_run_returns_refresh_result_shape(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock(return_value=[]) + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + wire_portfolio_pipeline(exchange_manager, {}) + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + ensure_ticker_channel_mock = mock.AsyncMock() + persist_mock = mock.Mock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value=set(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=[], + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + exchange_manager.exchange.get_open_orders.assert_not_called() + persist_mock.assert_called_once() + assert persist_mock.call_args.kwargs["persist_open_orders"] is False + assert isinstance(refresh_result, octobot_flow.entities.GlobalViewAccountRefreshResult) + assert refresh_result.updated_account.id == "account-1" + assert refresh_result.changed_order_ids == set() + assert refresh_result.open_orders == [] + assert refresh_result.updated_account.assets is not None + assert refresh_result.updated_account.assets[0].assets + + async def test_run_detects_disappeared_orders(self): + context = _exchange_account_context(has_bound_automation=True) + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + _open_order_dict("gone-order-1", "BTC/USDT"), + _open_order_dict("stays-order-2", "ETH/USDT"), + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + persist_mock = mock.Mock() + ensure_ticker_channel_mock = mock.AsyncMock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + assert persist_mock.call_args.kwargs["persist_open_orders"] is False + assert refresh_result.changed_order_ids == {"gone-order-1"} + + async def test_unbound_fetches_open_orders_by_previous_symbols(self): + context = _exchange_account_context() + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock(return_value=_portfolio_content()) + exchange_manager.exchange.get_open_orders = mock.AsyncMock( + return_value=[_open_order_dict("stays-order-2")], + ) + exchange_manager.exchange.get_option_value = mock.Mock(return_value=None) + wire_portfolio_pipeline(exchange_manager, {}) + previous_open_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "gone-order-1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + } + }, + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "stays-order-2", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "ETH/USDT", + } + }, + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + persist_mock = mock.Mock() + ensure_ticker_channel_mock = mock.AsyncMock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + exchange_account_refresh_module, + "_fetch_tickers", + mock.AsyncMock(return_value={}), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"gone-order-1", "stays-order-2"}, + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + called_symbols = { + call.kwargs.get("symbol") + for call in exchange_manager.exchange.get_open_orders.await_args_list + } + assert called_symbols == {"BTC/USDT", "ETH/USDT"} + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + assert persist_mock.call_args.kwargs["persist_open_orders"] is True + assert refresh_result.changed_order_ids == {"gone-order-1"} + + async def test_simulated_account_uses_fill_detector_without_exchange_fetch(self): + context = _exchange_account_context(account_id="sim-account", is_simulated=True) + exchange_manager = mock.Mock() + exchange_manager.exchange_personal_data = mock.Mock() + exchange_manager.exchange.get_balance = mock.AsyncMock() + exchange_manager.exchange.get_open_orders = mock.AsyncMock() + exchange_manager.exchange.get_option_value = mock.Mock(return_value="USDC") + wire_portfolio_pipeline(exchange_manager, {}, portfolio_total=0.0) + previous_open_orders = [ + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "filled-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 10000.0, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + } + }, + { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "open-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 8000.0, + trading_enums.ExchangeConstantsOrderColumns.TRIGGER_ABOVE.value: False, + } + }, + ] + + @contextlib.asynccontextmanager + async def fake_exchange_manager(*_args, **_kwargs): + yield exchange_manager + + ensure_ticker_channel_mock = mock.AsyncMock() + persist_mock = mock.Mock() + with ( + mock.patch.object( + global_view_account_job_module.trading_exchanges, + "exchange_manager_from_exchange_data", + fake_exchange_manager, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "ensure_temporary_ticker_channel", + ensure_ticker_channel_mock, + ), + mock.patch.object( + exchange_account_refresh_module, + "_refresh_portfolio_valuation", + mock.AsyncMock(), + ), + mock.patch.object( + global_view_account_job_module.tentacles_manager_api, + "get_full_tentacles_setup_config", + return_value=mock.Mock(), + ), + mock.patch.object( + global_view_account_job_module.simulated_portfolio_seeder_module, + "seed_simulated_portfolio", + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_order_exchange_ids", + return_value={"filled-order", "open-order"}, + ), + mock.patch.object( + account_state_persistence_module, + "load_previous_open_orders", + return_value=previous_open_orders, + ), + mock.patch.object( + exchange_account_refresh_module.tickers_repository_module.TickersRepository, + "fetch_tickers", + mock.AsyncMock(return_value={ + "BTC/USDT": { + trading_enums.ExchangeConstantsTickersColumns.CLOSE.value: 9000.0, + }, + }), + ), + mock.patch.object( + global_view_persistence_module, + "persist_global_view_refresh_result", + persist_mock, + ), + ): + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + + ensure_ticker_channel_mock.assert_awaited_once_with(exchange_manager) + exchange_manager.exchange.get_balance.assert_not_called() + exchange_manager.exchange.get_open_orders.assert_not_called() + assert persist_mock.call_args.kwargs["persist_open_orders"] is True + assert refresh_result.changed_order_ids == {"filled-order"} + remaining_exchange_ids = order_change_detection_module.open_order_exchange_ids_from_open_orders( + refresh_result.open_orders + ) + assert remaining_exchange_ids == {"open-order"} + + async def test_generic_account_returns_no_op_result(self): + generic_account = protocol_models.Account( + id="generic-1", + name="Generic", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.GenericAccount( + account_type=protocol_models.AccountType.GENERIC, + ), + ), + ) + context = octobot_flow.entities.GlobalViewAccountContext( + account=generic_account, + exchange_account=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="unused", + exchange_config_ids=["exchange-config-1"], + ), + exchange_config=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="binance-main", + exchange="binanceus", + sandboxed=False, + ), + trading_type=protocol_models.TradingType.SPOT, + auth_details=exchange_data_module.ExchangeAuthDetails( + exchange_type=trading_enums.ExchangeTypes.SPOT.value, + sandboxed=False, + ), + ) + refresh_result = await global_view_account_job_module.GlobalViewAccountJob( + "wallet-1", + context, + ).run() + assert refresh_result.updated_account.id == "generic-1" + assert refresh_result.changed_order_ids == set() diff --git a/packages/flow/tests/logic/global_view/test_global_view_persistence.py b/packages/flow/tests/logic/global_view/test_global_view_persistence.py new file mode 100644 index 0000000000..4b4259ac03 --- /dev/null +++ b/packages/flow/tests/logic/global_view/test_global_view_persistence.py @@ -0,0 +1,172 @@ +# Drakkar-Software OctoBot-Flow + +import datetime + +import mock + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +import octobot_flow.entities +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_flow.logic.global_view.global_view_persistence as global_view_persistence_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _refresh_result(*, open_orders: list[dict] | None = None) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account = protocol_models.Account( + id="account-1", + name="Test", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + ) + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + open_orders=open_orders or [], + ) + + +class TestPersistGlobalViewRefreshResult: + def test_does_not_persist_trading_when_persist_open_orders_false(self): + account_provider = mock.Mock() + with ( + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + account_state_persistence_module, + "persist_account_trading_orders", + ) as persist_orders_mock, + ): + global_view_persistence_module.persist_global_view_refresh_result( + "wallet-1", + "account-1", + _refresh_result(), + persist_open_orders=False, + ) + account_provider.update_item.assert_called_once() + persist_orders_mock.assert_not_called() + + def test_persists_orders_only_when_persist_open_orders_true(self): + account_provider = mock.Mock() + open_orders = [ + { + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "order-1", + } + ] + with ( + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + account_state_persistence_module, + "persist_account_trading_orders", + ) as persist_orders_mock, + ): + global_view_persistence_module.persist_global_view_refresh_result( + "wallet-1", + "account-1", + _refresh_result(open_orders=open_orders), + persist_open_orders=True, + ) + persist_orders_mock.assert_called_once_with( + "wallet-1", + "account-1", + open_orders, + ) + + +class TestPersistAccountTradingOrders: + def test_replaces_orders_without_clearing_trades_or_positions(self): + trading_provider = mock.Mock() + sentinel_trade = protocol_models.Trade.model_construct( + id="trade-1", + trade_id="trade-1", + ) + sentinel_position = protocol_models.Position.model_construct( + id="pos-1", + symbol="BTC/USDT", + ) + trading_provider.load_state.return_value = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + orders=[ + protocol_models.Order( + id="old-order", + symbol="BTC/USDT", + price=10000.0, + quantity=0.01, + filled=0.0, + exchange_id="old-order", + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_TEST_TIMESTAMP, + ) + ], + trades=[sentinel_trade], + positions=[sentinel_position], + ), + ) + order_payload = { + trading_constants.STORAGE_ORIGIN_VALUE: { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "new-order", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_ID.value: "new-order", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "ETH/USDT", + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 2000.0, + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 0.5, + trading_enums.ExchangeConstantsOrderColumns.FILLED.value: 0.0, + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: protocol_models.Side.BUY.value, + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: protocol_models.OrderType.LIMIT.value, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: protocol_models.OrderStatus.OPEN.value, + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: _TEST_TIMESTAMP.timestamp(), + } + } + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + account_state_persistence_module.persist_account_trading_orders( + "wallet-1", + "account-1", + [order_payload], + ) + saved_state = trading_provider.save_state.call_args.args[2] + saved_trading = saved_state.account_trading + assert len(saved_trading.orders) == 1 + assert saved_trading.orders[0].exchange_id == "new-order" + assert saved_trading.positions == [sentinel_position] + assert saved_trading.trades == [sentinel_trade] + + def test_raises_when_trading_state_missing(self): + trading_provider = mock.Mock() + trading_provider.load_state.side_effect = collection_errors.CollectionNoDataError() + with mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ): + try: + account_state_persistence_module.persist_account_trading_orders( + "wallet-1", + "account-1", + [], + ) + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + trading_provider.save_state.assert_not_called() diff --git a/packages/flow/tests/logic/portfolio_history/__init__.py b/packages/flow/tests/logic/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py new file mode 100644 index 0000000000..5f6d9c3fd4 --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_daily_price_cache_updater.py @@ -0,0 +1,560 @@ +import time +import mock +import pytest + +import octobot_trading.api as trading_api +import octobot_trading.enums as trading_enums +import octobot_trading.errors as trading_errors + +import octobot_commons.constants as commons_constants +import octobot_flow.constants as flow_constants +import octobot_flow.logic.portfolio_history.daily_price_cache_updater as daily_price_cache_updater_module + + +def _empty_daily_prices(): + return { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + + +def _exchange_manager_with_symbols(symbols: list[str], exchange_name: str = "binance"): + exchange_manager = mock.AsyncMock() + exchange_manager.client_symbols = symbols + exchange_manager.exchange_name = exchange_name + return exchange_manager + + +def _utc_day_start(days_ago: int = 0) -> int: + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + return today_start - days_ago * commons_constants.DAYS_TO_SECONDS + + +def _sample_candle(day_timestamp: int, close_price: float = 40500.0) -> list: + return [day_timestamp, 40000, 41000, 39000, close_price, 100] + + +async def _empty_historical_ohlcv(*_args, **_kwargs): + if False: + yield [] + + +def _patch_historical_ohlcv(candles=None, error=None): + async def _historical_ohlcv(*_args, **_kwargs): + if error is not None: + raise error + if candles: + yield candles + + return mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _historical_ohlcv, + ) + + +@pytest.fixture(autouse=True) +def mock_empty_historical_ohlcv(): + with mock.patch.object( + daily_price_cache_updater_module.exchange_util, + "get_historical_ohlcv", + _empty_historical_ohlcv, + ): + yield + + +class TestComputeFetchTimeRangeMs: + def test_incremental_uses_since_ms_as_start(self): + since_ms = 1_700_000_000_000 + start_time_ms, end_time_ms = daily_price_cache_updater_module._compute_fetch_time_range_ms(since_ms) + assert start_time_ms == since_ms + assert end_time_ms > since_ms + + def test_empty_cache_uses_lookback_start(self): + start_time_ms, end_time_ms = daily_price_cache_updater_module._compute_fetch_time_range_ms(None) + lookback_ms = ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS + * commons_constants.DAYS_TO_SECONDS + * commons_constants.MSECONDS_TO_SECONDS + ) + assert end_time_ms - start_time_ms == lookback_ms + + +class TestComputeFetchSinceMs: + def test_empty_cache_returns_none(self): + daily_prices = _empty_daily_prices() + assert daily_price_cache_updater_module._compute_fetch_since_ms( + daily_prices, "BTC/USDT", + ) is None + + def test_uses_newest_minus_one_day(self): + daily_prices = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"1000": 42000.0, "2000": 43000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + since_ms = daily_price_cache_updater_module._compute_fetch_since_ms( + daily_prices, "BTC/USDT", + ) + expected_since_ms = int((2000 - commons_constants.DAYS_TO_SECONDS) * 1000) + assert since_ms == expected_since_ms + + +class TestFilterClosesForMerge: + def test_initial_seed_keeps_only_lookback_window(self): + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + lookback_floor = today_start - ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS * commons_constants.DAYS_TO_SECONDS + ) + too_old_day = lookback_floor - commons_constants.DAYS_TO_SECONDS + recent_day = today_start - commons_constants.DAYS_TO_SECONDS + daily_prices = _empty_daily_prices() + incoming_closes = { + str(too_old_day): 1.0, + str(recent_day): 2.0, + str(today_start): 3.0, + } + + filtered_closes = daily_price_cache_updater_module._filter_closes_for_merge( + daily_prices, "BTC/USDT", incoming_closes, + ) + + assert str(too_old_day) not in filtered_closes + assert filtered_closes[str(recent_day)] == 2.0 + assert filtered_closes[str(today_start)] == 3.0 + + def test_incremental_fill_drops_older_than_cached_floor(self): + daily_prices = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {"172800": 41000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + incoming_closes = { + "86400": 40000.0, + "259200": 42000.0, + } + + filtered_closes = daily_price_cache_updater_module._filter_closes_for_merge( + daily_prices, "BTC/USDT", incoming_closes, + ) + + assert filtered_closes == {"259200": 42000.0} + + +class TestIsDailyCacheUpToDate: + def test_cache_with_today_is_up_to_date(self): + today_start = daily_price_cache_updater_module._utc_day_start(time.time()) + daily_prices = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {str(int(today_start)): 42000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + assert daily_price_cache_updater_module._is_daily_cache_up_to_date(daily_prices, "BTC/USDT") is True + + def test_cache_with_yesterday_is_stale(self): + yesterday_start = daily_price_cache_updater_module._utc_day_start(time.time()) - 86400 + daily_prices = { + trading_enums.DailyPricesCacheKeys.SYMBOLS: {"BTC/USDT": {str(int(yesterday_start)): 42000.0}}, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + assert daily_price_cache_updater_module._is_daily_cache_up_to_date(daily_prices, "BTC/USDT") is False + + +class TestUpdateDailyPrices: + @pytest.mark.asyncio + async def test_fetches_and_merges_closes(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + day_one = _utc_day_start(1) + day_two = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(day_one, 40500), + _sample_candle(day_two, 41000), + ] + + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(day_one)] == 40500 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(day_two)] == 41000 + assert result[trading_enums.DailyPricesCacheKeys.SOURCES]["BTC"] == "BTC/USDT" + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + + @pytest.mark.asyncio + async def test_empty_cache_fetches_without_since(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(_utc_day_start(0)), + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + + @pytest.mark.asyncio + async def test_empty_candles_skips(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.return_value = [] + + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result == _empty_daily_prices() + + @pytest.mark.asyncio + async def test_skips_fetch_when_cache_has_today(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + data_root = str(tmp_path) + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + await trading_api.merge_daily_prices( + "binance", "spot", False, "BTC/USDT", {str(today_start): 42000.0}, data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_not_called() + + @pytest.mark.asyncio + async def test_still_fetches_when_cache_stale(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + fresh_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(fresh_day), + ] + data_root = str(tmp_path) + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "binance", "spot", False, "BTC/USDT", {str(yesterday_start): 41000.0}, data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + expected_since_ms = int((yesterday_start - commons_constants.DAYS_TO_SECONDS) * 1000) + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] == expected_since_ms + + @pytest.mark.asyncio + async def test_stale_cache_with_long_history_uses_newest_not_oldest(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + fresh_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(fresh_day), + ] + data_root = str(tmp_path) + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "binance", + "spot", + False, + "BTC/USDT", + {str(1000): 41000.0, str(yesterday_start): 42000.0}, + data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + expected_since_ms = int((yesterday_start - commons_constants.DAYS_TO_SECONDS) * 1000) + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] == expected_since_ms + assert exchange_manager.exchange.get_symbol_prices.call_args[1]["since"] != 1000 * 1000 + + @pytest.mark.asyncio + async def test_skips_usd_like_stablecoin_symbol(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["USDC/USDT"]) + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["USDC/USDT"], data_root, + ) + exchange_manager.exchange.get_symbol_prices.assert_not_called() + + @pytest.mark.asyncio + async def test_continues_on_failed_request(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT", "ETH/USDT"]) + eth_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.FailedRequest("bingx range error"), + [_sample_candle(eth_day)], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + "binance", + "spot", + False, + ["BTC/USDT", "ETH/USDT"], + data_root, + ) + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["ETH/USDT"][str(eth_day)] == 40500 + assert "BTC/USDT" not in result[trading_enums.DailyPricesCacheKeys.SYMBOLS] + + @pytest.mark.asyncio + async def test_continues_when_symbol_is_unsupported(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + btc_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(btc_day), + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, + "binance", + "spot", + False, + ["USDT/USDT", "BTC/USDT"], + data_root, + ) + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(btc_day)] == 40500 + assert "USDT/USDT" not in result[trading_enums.DailyPricesCacheKeys.SYMBOLS] + + @pytest.mark.asyncio + async def test_falls_back_to_usd_like_quote(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USD"], "kraken") + knc_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [knc_day, 1.0, 1.1, 0.9, 1.05, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + result = await trading_api.load_daily_prices("kraken", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USD"][str(knc_day)] == 1.05 + assert result[trading_enums.DailyPricesCacheKeys.SOURCES]["KNC"] == "KNC/USD" + assert trading_api.get_daily_price(result, "KNC/USDT", str(knc_day)) == 1.05 + + @pytest.mark.asyncio + async def test_sticky_fetch_symbol_is_tried_first(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USD"], "kraken") + yesterday_start = _utc_day_start(1) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [yesterday_start, 1.0, 1.1, 0.9, 1.05, 100], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + exchange_manager.client_symbols = ["KNC/USD", "KNC/USDT"] + exchange_manager.exchange.get_symbol_prices.reset_mock() + today_start = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + [today_start, 1.0, 1.1, 0.9, 1.06, 100], + ] + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + assert exchange_manager.exchange.get_symbol_prices.call_args[0][0] == "KNC/USD" + + @pytest.mark.asyncio + async def test_migrates_rows_when_sticky_pair_is_delisted(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["KNC/USDC"], "kraken") + data_root = str(tmp_path) + seed_day_one = _utc_day_start(2) + seed_day_two = _utc_day_start(1) + new_day = _utc_day_start(0) + await trading_api.merge_daily_prices( + "kraken", "spot", False, "KNC/USD", {str(seed_day_one): 1.0, str(seed_day_two): 1.1}, data_root, + ) + await trading_api.set_daily_close_source( + "kraken", "spot", False, "KNC", "KNC/USD", data_root, + ) + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.UnSupportedSymbolError("delisted"), + [[new_day, 1.0, 1.1, 0.9, 1.2, 100]], + ] + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "kraken", "spot", False, ["KNC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("kraken", "spot", False, data_root) + assert "KNC/USD" not in result[trading_enums.DailyPricesCacheKeys.SYMBOLS] + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USDC"][str(seed_day_one)] == 1.0 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USDC"][str(seed_day_two)] == 1.1 + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["KNC/USDC"][str(new_day)] == 1.2 + assert result[trading_enums.DailyPricesCacheKeys.SOURCES]["KNC"] == "KNC/USDC" + + @pytest.mark.asyncio + async def test_limited_history_retries_without_since_after_failed_request(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["SOL/USDT"], "bingx") + sol_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.side_effect = [ + trading_errors.FailedRequest("bingx range error"), + [_sample_candle(sol_day)], + ] + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "bingx", "spot", False, ["SOL/USDT"], data_root, + ) + + assert exchange_manager.exchange.get_symbol_prices.call_count == 2 + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args_list[1][1] + result = await trading_api.load_daily_prices("bingx", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["SOL/USDT"][str(sol_day)] == 40500 + + @pytest.mark.asyncio + async def test_limited_history_logs_error_when_both_attempts_fail(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["SOL/USDT"], "bingx") + exchange_manager.exchange.get_symbol_prices.side_effect = trading_errors.FailedRequest( + "bingx range error", + ) + data_root = str(tmp_path) + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "bingx", "spot", False, ["SOL/USDT"], data_root, + ) + + assert exchange_manager.exchange.get_symbol_prices.call_count == 2 + result = await trading_api.load_daily_prices("bingx", "spot", False, data_root) + assert result == _empty_daily_prices() + + @pytest.mark.asyncio + async def test_full_history_uses_historical_ohlcv_first(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + btc_day = _utc_day_start(0) + candles = [_sample_candle(btc_day)] + data_root = str(tmp_path) + with _patch_historical_ohlcv(candles=candles): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_not_called() + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(btc_day)] == 40500 + + @pytest.mark.asyncio + async def test_full_history_falls_back_to_get_symbol_prices(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + btc_day = _utc_day_start(0) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(btc_day), + ] + data_root = str(tmp_path) + with _patch_historical_ohlcv(error=trading_errors.FailedRequest("historical error")): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + assert "since" not in exchange_manager.exchange.get_symbol_prices.call_args[1] + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"][str(btc_day)] == 40500 + + @pytest.mark.asyncio + async def test_full_history_does_not_use_limited_history_fallback(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + exchange_manager.exchange.get_symbol_prices.side_effect = trading_errors.FailedRequest( + "binance range error", + ) + data_root = str(tmp_path) + with _patch_historical_ohlcv(error=trading_errors.FailedRequest("historical error")): + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + assert result == _empty_daily_prices() + + @pytest.mark.asyncio + async def test_initial_seed_trims_candles_beyond_lookback(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + today_start = int(daily_price_cache_updater_module._utc_day_start(time.time())) + lookback_floor = today_start - ( + flow_constants.PORTFOLIO_HISTORY_DAILY_LOOKBACK_DAYS * commons_constants.DAYS_TO_SECONDS + ) + too_old_day = lookback_floor - commons_constants.DAYS_TO_SECONDS + recent_day = today_start - commons_constants.DAYS_TO_SECONDS + exchange_manager.exchange.get_symbol_prices.return_value = [ + [too_old_day, 39000, 39500, 38500, 39200, 100], + [recent_day, 40000, 41000, 39000, 40500, 100], + [today_start, 40500, 42000, 40000, 41000, 200], + ] + data_root = str(tmp_path) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + symbol_closes = result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] + assert str(too_old_day) not in symbol_closes + assert symbol_closes[str(recent_day)] == 40500 + assert symbol_closes[str(today_start)] == 41000 + oldest_timestamp = min(int(day_ts) for day_ts in symbol_closes) + assert oldest_timestamp >= lookback_floor + + @pytest.mark.asyncio + async def test_incremental_fill_does_not_extend_cache_backward(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["BTC/USDT"]) + data_root = str(tmp_path) + seed_day = _utc_day_start(2) + older_day = _utc_day_start(5) + newer_day = _utc_day_start(0) + await trading_api.merge_daily_prices( + "binance", "spot", False, "BTC/USDT", {str(seed_day): 41000.0}, data_root, + ) + await trading_api.set_daily_close_source( + "binance", "spot", False, "BTC", "BTC/USDT", data_root, + ) + exchange_manager.exchange.get_symbol_prices.return_value = [ + _sample_candle(older_day, 39200), + _sample_candle(seed_day, 40500), + _sample_candle(newer_day, 42000), + ] + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "binance", "spot", False, ["BTC/USDT"], data_root, + ) + + result = await trading_api.load_daily_prices("binance", "spot", False, data_root) + symbol_closes = result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["BTC/USDT"] + assert str(older_day) not in symbol_closes + assert symbol_closes[str(seed_day)] == 40500 + assert symbol_closes[str(newer_day)] == 42000 + assert min(int(day_ts) for day_ts in symbol_closes) == seed_day + + @pytest.mark.asyncio + async def test_limited_history_incremental_failure_skips_without_since_fallback(self, tmp_path): + exchange_manager = _exchange_manager_with_symbols(["SOL/USDT"], "bingx") + data_root = str(tmp_path) + yesterday_start = int(daily_price_cache_updater_module._utc_day_start(time.time()) - 86400) + await trading_api.merge_daily_prices( + "bingx", "spot", False, "SOL/USDT", {str(yesterday_start): 100.0}, data_root, + ) + await trading_api.set_daily_close_source( + "bingx", "spot", False, "SOL", "SOL/USDT", data_root, + ) + exchange_manager.exchange.get_symbol_prices.side_effect = trading_errors.FailedRequest( + "bingx range error", + ) + + await daily_price_cache_updater_module.update_daily_prices( + exchange_manager, "bingx", "spot", False, ["SOL/USDT"], data_root, + ) + + exchange_manager.exchange.get_symbol_prices.assert_called_once() + assert "since" in exchange_manager.exchange.get_symbol_prices.call_args[1] + result = await trading_api.load_daily_prices("bingx", "spot", False, data_root) + assert result[trading_enums.DailyPricesCacheKeys.SYMBOLS]["SOL/USDT"] == {str(yesterday_start): 100.0} diff --git a/packages/flow/tests/logic/portfolio_history/test_portfolio_history_replay.py b/packages/flow/tests/logic/portfolio_history/test_portfolio_history_replay.py new file mode 100644 index 0000000000..2d1bc37a26 --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_portfolio_history_replay.py @@ -0,0 +1,217 @@ +import datetime +import decimal + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.personal_data.portfolios.history.history_from_trades_and_transaction_builder as history_builder_module + + +DAY_1_TS = 1700000000.0 +DAY_2_TS = DAY_1_TS + 86400 +DAY_3_TS = DAY_2_TS + 86400 + + +def _make_trade( + trade_id: str, + symbol: str, + side: protocol_models.Side, + quantity: float, + price: float, + executed_at: datetime.datetime, +) -> protocol_models.Trade: + return protocol_models.Trade( + id=trade_id, + trade_id=trade_id, + type=protocol_models.OrderType.LIMIT, + symbol=symbol, + side=side, + quantity=quantity, + price=price, + status=protocol_models.OrderStatus.FILLED, + executed_at=executed_at, + ) + + +def _latest_portfolio() -> dict[str, dict[str, decimal.Decimal]]: + return { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("402"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("402"), + }, + "ALGO": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("100"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("100"), + }, + "KNC": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("50"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("50"), + }, + "SOL": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("2"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("2"), + }, + } + + +def _all_pair_trades() -> list[protocol_models.Trade]: + return [ + _make_trade( + "sol-buy", + "SOL/USDC", + protocol_models.Side.BUY, + quantity=2.0, + price=50.0, + executed_at=datetime.datetime.fromtimestamp(DAY_3_TS + 3600, tz=datetime.timezone.utc), + ), + _make_trade( + "knc-buy", + "KNC/USDC", + protocol_models.Side.BUY, + quantity=50.0, + price=2.0, + executed_at=datetime.datetime.fromtimestamp(DAY_2_TS + 3600, tz=datetime.timezone.utc), + ), + _make_trade( + "algo-buy", + "ALGO/USDC", + protocol_models.Side.BUY, + quantity=100.0, + price=2.07, + executed_at=datetime.datetime.fromtimestamp(DAY_1_TS + 3600, tz=datetime.timezone.utc), + ), + ] + + +def _sell_without_buy_trades() -> list[protocol_models.Trade]: + return [ + _make_trade( + "btc-sell", + "BTC/USDC", + protocol_models.Side.SELL, + quantity=1.0, + price=609.0, + executed_at=datetime.datetime.fromtimestamp(DAY_2_TS + 3600, tz=datetime.timezone.utc), + ), + ] + + +def _usdc_holdings_by_day( + trades: list[protocol_models.Trade], + *, + latest_portfolio: dict[str, dict[str, decimal.Decimal]] | None = None, +) -> dict[float, decimal.Decimal]: + daily_holdings = history_builder_module.build_historical_holdings( + latest_portfolio or _latest_portfolio(), + trades, + [], + ) + usdc_by_day: dict[float, decimal.Decimal] = {} + for day_timestamp, holdings in daily_holdings.items(): + usdc_amounts = holdings.get("USDC", {}) + usdc_by_day[day_timestamp] = usdc_amounts.get( + commons_constants.PORTFOLIO_TOTAL, + decimal.Decimal(0), + ) + return usdc_by_day + + +class TestMultiPairReplayUsdcNonNegative: + def test_usdc_non_negative_when_all_pair_trades_present(self): + usdc_by_day = _usdc_holdings_by_day(_all_pair_trades()) + assert usdc_by_day + assert all(usdc_total >= 0 for usdc_total in usdc_by_day.values()) + + def test_usdc_goes_negative_when_pair_trades_missing(self): + latest_portfolio = { + "USDC": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("402"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("402"), + }, + "BTC": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("0"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("0"), + }, + } + usdc_by_day = _usdc_holdings_by_day( + _sell_without_buy_trades(), + latest_portfolio=latest_portfolio, + ) + assert any(usdc_total < 0 for usdc_total in usdc_by_day.values()) + + +class TestBingxMultiPageBalancedTradesReplay: + def test_holdings_non_negative_when_balanced_buy_sell_pairs_present(self): + latest_portfolio = { + "USDT": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("7"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("7"), + }, + "TRX": { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal("1.656"), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal("1.656"), + }, + } + trades = [ + _make_trade( + "sol-buy-1", + "SOL/USDT", + protocol_models.Side.BUY, + quantity=36.5, + price=50.0, + executed_at=datetime.datetime.fromtimestamp(DAY_1_TS + 3600, tz=datetime.timezone.utc), + ), + _make_trade( + "sol-sell-1", + "SOL/USDT", + protocol_models.Side.SELL, + quantity=36.5, + price=55.0, + executed_at=datetime.datetime.fromtimestamp(DAY_2_TS + 3600, tz=datetime.timezone.utc), + ), + _make_trade( + "trx-buy-1", + "TRX/USDT", + protocol_models.Side.BUY, + quantity=500.0, + price=0.1, + executed_at=datetime.datetime.fromtimestamp(DAY_1_TS + 7200, tz=datetime.timezone.utc), + ), + _make_trade( + "trx-sell-1", + "TRX/USDT", + protocol_models.Side.SELL, + quantity=498.344, + price=0.11, + executed_at=datetime.datetime.fromtimestamp(DAY_2_TS + 7200, tz=datetime.timezone.utc), + ), + _make_trade( + "trx-buy-2", + "TRX/USDT", + protocol_models.Side.BUY, + quantity=200.0, + price=0.1, + executed_at=datetime.datetime.fromtimestamp(DAY_3_TS + 3600, tz=datetime.timezone.utc), + ), + _make_trade( + "trx-sell-2", + "TRX/USDT", + protocol_models.Side.SELL, + quantity=200.0, + price=0.12, + executed_at=datetime.datetime.fromtimestamp(DAY_3_TS + 7200, tz=datetime.timezone.utc), + ), + ] + daily_holdings = history_builder_module.build_historical_holdings( + latest_portfolio, + trades, + [], + ) + assert daily_holdings + for day_holdings in daily_holdings.values(): + for asset_symbol in ("SOL", "TRX"): + asset_amounts = day_holdings.get(asset_symbol, {}) + total = asset_amounts.get( + commons_constants.PORTFOLIO_TOTAL, + decimal.Decimal(0), + ) + assert total >= 0 diff --git a/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py b/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py new file mode 100644 index 0000000000..44d8e115dc --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_portfolio_value_history.py @@ -0,0 +1,325 @@ +import datetime +import decimal +from contextlib import contextmanager + +import mock +import pytest + +import octobot_flow.logic.portfolio_history.portfolio_value_history as portfolio_value_history_module +import octobot_trading.enums as trading_enums +import octobot_trading.exchange_data.prices.daily_prices_cache_types as daily_prices_cache_types + + +def _portfolio(assets: dict[str, float]) -> dict[str, dict[str, decimal.Decimal]]: + return { + asset: {"total": decimal.Decimal(str(amount)), "available": decimal.Decimal(str(amount))} + for asset, amount in assets.items() + } + + +def _empty_tickers() -> daily_prices_cache_types.LatestTickersCache: + return daily_prices_cache_types.empty_latest_tickers_cache() + + +def _daily_prices(symbols: dict[str, dict[str, float]]) -> daily_prices_cache_types.DailyPricesCache: + return { + trading_enums.DailyPricesCacheKeys.SYMBOLS: symbols, + trading_enums.DailyPricesCacheKeys.SOURCES: {}, + } + + +def _tickers(closes: dict[str, float]) -> daily_prices_cache_types.LatestTickersCache: + return { + trading_enums.LatestTickersCacheKeys.UPDATED_AT: None, + trading_enums.LatestTickersCacheKeys.CLOSES: closes, + } + + +@contextmanager +def _with_end_day(end_day_timestamp: float): + with mock.patch.object( + portfolio_value_history_module.time, + "time", + return_value=end_day_timestamp, + ): + yield + + +def _compute_values( + daily_holdings, + daily_prices, + latest_tickers, + *, + end_day_timestamp: float, + reference_market: str = "USDT", +): + with _with_end_day(end_day_timestamp): + return portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, + daily_prices, + latest_tickers, + reference_market=reference_market, + ) + + +def _spot_assets(history_value) -> list: + assert history_value.assets is not None + return history_value.assets[0].assets + + +def _asset_by_symbol(day_assets, symbol: str): + for asset_entry in day_assets: + if asset_entry.symbol == symbol: + return asset_entry + raise AssertionError(f"Asset {symbol} not found in {day_assets}") + + +class TestExpandSparseDailyHoldings: + def test_forward_fills_holdings_between_sparse_days(self): + sparse_holdings = { + 86400.0: _portfolio({"BTC": 1.0}), + 259200.0: _portfolio({"BTC": 2.0}), + } + + dense_holdings = portfolio_value_history_module._expand_sparse_daily_holdings( + sparse_holdings, 86400.0, 259200.0, + ) + + assert list(dense_holdings) == [86400.0, 172800.0, 259200.0] + assert dense_holdings[172800.0]["BTC"]["total"] == decimal.Decimal("1") + + +class TestEarliestValuationTimestamp: + def test_returns_max_oldest_timestamp_across_symbols(self): + daily_prices = _daily_prices({ + "BTC/USDT": {"86400": 1.0}, + "ETH/USDT": {"172800": 1.0}, + }) + required_symbols = {"BTC/USDT", "ETH/USDT"} + result = portfolio_value_history_module._earliest_valuation_timestamp( + daily_prices, required_symbols, + ) + assert result == 172800.0 + + def test_returns_zero_when_no_required_symbols(self): + result = portfolio_value_history_module._earliest_valuation_timestamp( + _daily_prices({}), set(), + ) + assert result == 0.0 + + +class TestResolveAssetUnitPrice: + def test_uses_ticker_only_when_no_historical_closes_exist(self): + daily_prices = _daily_prices({}) + latest_tickers = _tickers({"ETH/USDT": 3000.0}) + result = portfolio_value_history_module._resolve_asset_unit_price( + "ETH", 86400.0, "86400", daily_prices, latest_tickers, "USDT", + ) + assert result == decimal.Decimal("3000") + + def test_does_not_use_ticker_when_historical_exists_but_not_on_day(self): + daily_prices = _daily_prices({"ETH/USDT": {"172800": 2500.0}}) + latest_tickers = _tickers({"ETH/USDT": 3000.0}) + result = portfolio_value_history_module._resolve_asset_unit_price( + "ETH", 86400.0, "86400", daily_prices, latest_tickers, "USDT", + ) + assert result is None + + +class TestComputeDailyPortfolioValues: + def test_single_day_with_daily_price(self): + daily_holdings = { + 86400.0: _portfolio({"BTC": 1.0, "USDT": 500.0}), + } + daily_prices = _daily_prices({"BTC/USDT": {"86400": 40000.0}}) + result = _compute_values( + daily_holdings, daily_prices, _empty_tickers(), end_day_timestamp=86400.0, + ) + assert len(result) == 1 + assert result[0].timestamp == datetime.datetime.fromtimestamp(86400.0, tz=datetime.timezone.utc) + assert result[0].total == pytest.approx(40500.0) + + def test_fallback_to_latest_ticker_when_no_historical_closes(self): + daily_holdings = { + 86400.0: _portfolio({"ETH": 2.0}), + } + result = _compute_values( + daily_holdings, + _daily_prices({}), + _tickers({"ETH/USDT": 3000.0}), + end_day_timestamp=86400.0, + ) + assert result[0].total == pytest.approx(6000.0) + + def test_reference_market_asset_counted_directly(self): + daily_holdings = { + 0.0: _portfolio({"USDT": 1000.0}), + } + result = _compute_values( + daily_holdings, _daily_prices({}), _empty_tickers(), end_day_timestamp=0.0, + ) + assert result[0].total == pytest.approx(1000.0) + + def test_usd_like_stablecoin_valued_at_face_value(self): + daily_holdings = { + 0.0: _portfolio({"USDC": 250.0, "USDT": 500.0}), + } + result = _compute_values( + daily_holdings, _daily_prices({}), _empty_tickers(), end_day_timestamp=0.0, + ) + assert result[0].total == pytest.approx(750.0) + + def test_unpriced_asset_has_zero_value_but_remains_in_breakdown(self): + daily_holdings = { + 0.0: _portfolio({"UNKNOWN": 100.0, "USDT": 500.0}), + } + result = _compute_values( + daily_holdings, _daily_prices({}), _empty_tickers(), end_day_timestamp=0.0, + ) + assert result[0].total == pytest.approx(500.0) + + def test_sorted_ascending(self): + daily_holdings = { + 172800.0: _portfolio({"USDT": 200.0}), + 86400.0: _portfolio({"USDT": 100.0}), + } + result = _compute_values( + daily_holdings, _daily_prices({}), _empty_tickers(), end_day_timestamp=172800.0, + ) + assert len(result) == 2 + assert result[0].timestamp == datetime.datetime.fromtimestamp(86400.0, tz=datetime.timezone.utc) + assert result[1].timestamp == datetime.datetime.fromtimestamp(172800.0, tz=datetime.timezone.utc) + + def test_zero_holding_asset_omitted_from_assets(self): + daily_holdings = { + 0.0: _portfolio({"BTC": 0.0, "USDT": 100.0}), + } + result = _compute_values( + daily_holdings, _daily_prices({}), _empty_tickers(), end_day_timestamp=0.0, + ) + assert {asset_entry.symbol for asset_entry in _spot_assets(result[0])} == {"USDT"} + + def test_excludes_days_before_global_candle_cutoff(self): + daily_holdings = { + 86400.0: _portfolio({"BTC": 1.0}), + 172800.0: _portfolio({"BTC": 1.0}), + } + daily_prices = _daily_prices({"BTC/USDT": {"172800": 50000.0}}) + result = _compute_values( + daily_holdings, daily_prices, _empty_tickers(), end_day_timestamp=172800.0, + ) + assert len(result) == 1 + assert result[0].timestamp == datetime.datetime.fromtimestamp(172800.0, tz=datetime.timezone.utc) + + def test_uses_historical_close_when_exact_day_missing(self): + daily_holdings = { + 172800.0: _portfolio({"ETH": 2.0}), + } + daily_prices = _daily_prices({"ETH/USDT": {"86400": 2000.0}}) + result = _compute_values( + daily_holdings, + daily_prices, + _tickers({"ETH/USDT": 9999.0}), + end_day_timestamp=172800.0, + ) + assert result[0].total == pytest.approx(4000.0) + + def test_forward_fills_quiet_days_and_reprices_daily(self): + day_a = 86400.0 + day_b = 172800.0 + day_c = 259200.0 + daily_holdings = { + day_a: _portfolio({"BTC": 1.0}), + day_c: _portfolio({"BTC": 1.0}), + } + daily_prices = _daily_prices({ + "BTC/USDT": { + str(int(day_a)): 40000.0, + str(int(day_b)): 45000.0, + str(int(day_c)): 50000.0, + }, + }) + result = _compute_values( + daily_holdings, daily_prices, _empty_tickers(), end_day_timestamp=day_c, + ) + assert len(result) == 3 + assert result[1].total == pytest.approx(45000.0) + + def test_returns_empty_for_empty_sparse_holdings(self): + result = _compute_values( + {}, _daily_prices({"BTC/USDT": {"86400": 1.0}}), _empty_tickers(), end_day_timestamp=86400.0, + ) + assert result == [] + + def test_excludes_ancient_day_even_when_ticker_exists(self): + daily_holdings = { + 86400.0: _portfolio({"BTC": 1.0}), + 259200.0: _portfolio({"BTC": 1.0}), + } + daily_prices = _daily_prices({"BTC/USDT": {"172800": 42000.0}}) + result = _compute_values( + daily_holdings, + daily_prices, + _tickers({"BTC/USDT": 90000.0}), + end_day_timestamp=259200.0, + ) + assert len(result) == 2 + assert result[0].timestamp == datetime.datetime.fromtimestamp(172800.0, tz=datetime.timezone.utc) + assert result[0].total == pytest.approx(42000.0) + assert result[1].timestamp == datetime.datetime.fromtimestamp(259200.0, tz=datetime.timezone.utc) + assert result[1].total == pytest.approx(42000.0) + + def test_excludes_pre_candle_trade_days_using_dynamic_cutoff(self): + ancient_trade_day = 1546300800.0 # 2019-01-01 UTC + candle_oldest_day = 1725321600.0 # 2024-09-03 UTC + recent_day = candle_oldest_day + 86400.0 + daily_holdings = { + ancient_trade_day: _portfolio({"BTC": 1.0}), + candle_oldest_day: _portfolio({"BTC": 1.0}), + recent_day: _portfolio({"BTC": 1.0}), + } + daily_prices = _daily_prices({ + "BTC/USDT": { + str(int(candle_oldest_day)): 50000.0, + str(int(recent_day)): 51000.0, + }, + }) + result = _compute_values( + daily_holdings, daily_prices, _empty_tickers(), end_day_timestamp=recent_day, + ) + assert len(result) == 2 + assert result[0].timestamp == datetime.datetime.fromtimestamp( + candle_oldest_day, tz=datetime.timezone.utc, + ) + assert result[1].timestamp == datetime.datetime.fromtimestamp( + recent_day, tz=datetime.timezone.utc, + ) + + def test_emits_utc_midnight_timestamps_when_candle_cache_starts_mid_day(self): + day_one_midnight = 86400.0 + day_one_sixteen_hundred = day_one_midnight + 57600.0 + day_two_midnight = 172800.0 + daily_holdings = { + day_one_midnight: _portfolio({"BTC": 1.0}), + day_two_midnight: _portfolio({"BTC": 1.0}), + } + daily_prices = _daily_prices({ + "BTC/USDT": { + str(int(day_one_sixteen_hundred)): 40000.0, + str(int(day_two_midnight)): 50000.0, + }, + }) + result = _compute_values( + daily_holdings, + daily_prices, + _empty_tickers(), + end_day_timestamp=day_two_midnight, + ) + assert len(result) == 2 + assert result[0].timestamp == datetime.datetime.fromtimestamp( + day_one_midnight, tz=datetime.timezone.utc, + ) + assert result[1].timestamp == datetime.datetime.fromtimestamp( + day_two_midnight, tz=datetime.timezone.utc, + ) + assert result[1].total == pytest.approx(50000.0) diff --git a/packages/flow/tests/logic/portfolio_history/test_trade_symbols_config_persistence.py b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_config_persistence.py new file mode 100644 index 0000000000..305e15bc8a --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_config_persistence.py @@ -0,0 +1,131 @@ +import datetime + +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.portfolio_history.trade_symbols_discovery as trade_symbols_discovery_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _exchange_trade(symbol: str, trade_id: str = "trade-1") -> dict: + order_columns = trading_enums.ExchangeConstantsOrderColumns + return { + order_columns.SYMBOL.value: symbol, + order_columns.ID.value: trade_id, + } + + +class TestTradeConfirmedSymbolsFromFetchedTrades: + def test_returns_unique_symbols_with_trades(self): + trades = [ + _exchange_trade("ALGO/USDC", "trade-1"), + _exchange_trade("ALGO/USDC", "trade-2"), + _exchange_trade("SOL/USDC", "trade-3"), + ] + result = trade_symbols_discovery_module.trade_confirmed_symbols_from_fetched_trades(trades) + assert result == {"ALGO/USDC", "SOL/USDC"} + + def test_returns_empty_set_for_no_trades(self): + assert trade_symbols_discovery_module.trade_confirmed_symbols_from_fetched_trades([]) == set() + + def test_ignores_trades_without_symbol(self): + order_columns = trading_enums.ExchangeConstantsOrderColumns + trades = [{order_columns.ID.value: "trade-1"}] + assert trade_symbols_discovery_module.trade_confirmed_symbols_from_fetched_trades(trades) == set() + + +class TestPersistTradeConfirmedSymbolsToExchangeConfig: + def _exchange_config(self, historical_trade_symbols: list[str] | None) -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id="cfg-1", + name="kraken-spot-config", + exchange="kraken", + sandboxed=False, + historical_trade_symbols=historical_trade_symbols, + ) + + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + def test_updates_config_when_new_trade_symbols(self, account_provider_class): + account_provider = mock.MagicMock() + account_provider_class.instance.return_value = account_provider + existing_config = self._exchange_config(["SOL/USDC"]) + account_provider.get_exchange_config.return_value = existing_config + + result = trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + "wallet-1", + existing_config, + {"SOL/USDC", "ALGO/USDC"}, + ) + + account_provider.update_exchange_config.assert_called_once() + updated_config = account_provider.update_exchange_config.call_args.args[1] + assert updated_config.historical_trade_symbols == ["ALGO/USDC", "SOL/USDC"] + assert result == ["ALGO/USDC"] + + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + def test_skips_update_when_no_new_symbols(self, account_provider_class): + account_provider = mock.MagicMock() + account_provider_class.instance.return_value = account_provider + existing_config = self._exchange_config(["SOL/USDC"]) + + result = trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + "wallet-1", + existing_config, + {"SOL/USDC"}, + ) + + account_provider.get_exchange_config.assert_not_called() + account_provider.update_exchange_config.assert_not_called() + assert result == [] + + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + def test_does_not_add_portfolio_only_symbol_without_trades(self, account_provider_class): + account_provider = mock.MagicMock() + account_provider_class.instance.return_value = account_provider + existing_config = self._exchange_config([]) + + result = trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + "wallet-1", + existing_config, + set(), + ) + + account_provider.update_exchange_config.assert_not_called() + assert result == [] + + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + def test_reloads_config_before_update(self, account_provider_class): + account_provider = mock.MagicMock() + account_provider_class.instance.return_value = account_provider + existing_config = self._exchange_config([]) + account_provider.get_exchange_config.return_value = existing_config + + trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + "wallet-1", + existing_config, + {"ALGO/USDC"}, + ) + + account_provider.get_exchange_config.assert_called_once_with("wallet-1", "cfg-1") + account_provider.update_exchange_config.assert_called_once() + + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + def test_merges_sorted_unique_historical_trade_symbols(self, account_provider_class): + account_provider = mock.MagicMock() + account_provider_class.instance.return_value = account_provider + existing_config = self._exchange_config(["BTC/USDC"]) + account_provider.get_exchange_config.return_value = existing_config + + trade_symbols_discovery_module.persist_trade_confirmed_symbols_to_exchange_config( + "wallet-1", + existing_config, + {"ALGO/USDC", "BTC/USDC"}, + ) + + updated_config = account_provider.update_exchange_config.call_args.args[1] + assert updated_config.historical_trade_symbols == ["ALGO/USDC", "BTC/USDC"] diff --git a/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py new file mode 100644 index 0000000000..df250262ee --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_trade_symbols_discovery.py @@ -0,0 +1,347 @@ +import datetime + +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums +import octobot_trading.exchanges.util.exchange_util as exchange_util_module + +import octobot_flow.logic.portfolio_history.trade_symbols_discovery as trade_symbols_discovery_module + + +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + + +def _make_account(assets: dict[str, float]) -> protocol_models.Account: + detailed_assets = [ + protocol_models.DetailedAsset(symbol=symbol, total=amount, available=amount) + for symbol, amount in assets.items() + ] + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote1", + exchange_config_ids=["cfg1"], + ) + return protocol_models.Account( + id="acc1", + name="Test", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=detailed_assets, + ) + ], + ) + + +def _make_trade(symbol: str) -> protocol_models.Trade: + return protocol_models.Trade( + id=f"trade-{symbol}", + trade_id=f"trade-{symbol}", + type=protocol_models.OrderType.LIMIT, + symbol=symbol, + side=protocol_models.Side.BUY, + quantity=1.0, + price=1.0, + status=protocol_models.OrderStatus.FILLED, + executed_at=_TEST_TIMESTAMP, + ) + + +def _make_transaction(asset: str) -> protocol_models.Transaction: + return protocol_models.Transaction( + id=f"tx-{asset}", + timestamp=_TEST_TIMESTAMP, + asset=asset, + amount=1.0, + type=protocol_models.TransactionType.BLOCKCHAIN_DEPOSIT, + ) + + +def _exchange_manager_with_markets(valid_currencies: set[str]): + exchange_manager = mock.MagicMock() + + def get_associated_symbol(_exchange_manager, currency, reference_market): + if currency in valid_currencies: + return f"{currency}/{reference_market}", False + return None, False + + exchange_manager.symbol_exists.side_effect = lambda symbol: "/" in symbol + return exchange_manager, get_associated_symbol + + +class TestDiscoverTradeSymbolsSeeds: + def test_returns_sorted_union_of_seed_symbols(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=["SOL/USDC", "BTC/USDC"], + account=_make_account({}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert result == ["BTC/USDC", "SOL/USDC"] + + +class TestDiscoverTradeSymbolsPersisted: + def test_includes_symbols_from_persisted_trades(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + account_trading = protocol_models.AccountTrading( + trades=[_make_trade("ALGO/USDC")], + updated_at=_TEST_TIMESTAMP, + ) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({}), + account_trading=account_trading, + fresh_transactions=[], + reference_market="USDC", + ) + assert "ALGO/USDC" in result + + def test_maps_persisted_transaction_currency_to_market_pair(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ALGO"}) + account_trading = protocol_models.AccountTrading( + transactions=[_make_transaction("ALGO")], + updated_at=_TEST_TIMESTAMP, + ) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({}), + account_trading=account_trading, + fresh_transactions=[], + reference_market="USDC", + ) + assert "ALGO/USDC" in result + + +class TestDiscoverTradeSymbolsFreshTransactions: + def test_includes_fresh_deposit_currency_with_valid_market(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ETH"}) + fresh_deposit = { + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: "ETH", + } + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({}), + account_trading=None, + fresh_transactions=[fresh_deposit], + reference_market="USDC", + ) + assert "ETH/USDC" in result + + def test_skips_reference_market_deposit_currency(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + fresh_deposit = { + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: "USDC", + } + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({}), + account_trading=None, + fresh_transactions=[fresh_deposit], + reference_market="USDC", + ) + assert "USDC/USDC" not in result + + +class TestDiscoverTradeSymbolsPortfolio: + def test_includes_meaningful_holding_with_valid_market(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ALGO"}) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"ALGO": 100.0, "USDC": 402.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert "ALGO/USDC" in result + + def test_skips_dust_holding_below_threshold(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"BTC"}) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"BTC": 1e-8}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert "BTC/USDC" not in result + + def test_skips_holding_when_no_market_exists(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + exchange_manager.symbol_exists.side_effect = lambda symbol: False + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"UNKNOWN": 100.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert not any("UNKNOWN" in symbol for symbol in result) + + def test_skips_usd_like_and_reference_market_assets(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets(set()) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=["SOL/USDC"], + account=_make_account({"USDC": 402.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert result == ["SOL/USDC"] + + def test_excludes_pair_when_quote_not_held(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ALGO"}) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"ALGO": 100.0, "USDC": 402.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert "ALGO/USDC" in result + assert "ALGO/USDT" not in result + + def test_uses_reference_market_pair_only_per_held_asset(self): + exchange_manager = mock.MagicMock() + exchange_manager.symbol_exists.side_effect = lambda symbol: symbol in { + "DOT/USDC", + "BTC/USDC", + "ETH/USDC", + "EUR/USDC", + } + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=lambda _exchange_manager, currency, reference_market: (None, False), + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({ + "DOT": 1.0, + "BTC": 0.001, + "ETH": 0.01, + "EUR": 10.0, + "USDC": 402.0, + }), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert "DOT/USDC" in result + assert "BTC/USDC" in result + assert "ETH/USDC" in result + assert "EUR/USDC" in result + assert "DOT/BTC" not in result + assert "DOT/EUR" not in result + assert "DOT/ETH" not in result + + def test_includes_only_reference_market_pair_per_held_base(self): + exchange_manager = mock.MagicMock() + exchange_manager.symbol_exists.side_effect = lambda symbol: symbol in {"ALGO/USDC", "ALGO/USDT"} + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=lambda _exchange_manager, currency, reference_market: (None, False), + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=[], + account=_make_account({"ALGO": 100.0, "USDC": 402.0, "USDT": 50.0}), + account_trading=None, + fresh_transactions=[], + reference_market="USDC", + ) + assert "ALGO/USDC" in result + assert "ALGO/USDT" not in result + + +class TestDiscoverTradeSymbolsDedup: + def test_deduplicates_across_all_sources(self): + exchange_manager, get_associated_symbol = _exchange_manager_with_markets({"ALGO"}) + account_trading = protocol_models.AccountTrading( + trades=[_make_trade("ALGO/USDC")], + updated_at=_TEST_TIMESTAMP, + ) + with mock.patch.object( + exchange_util_module, + "get_associated_symbol", + side_effect=get_associated_symbol, + ): + result = trade_symbols_discovery_module.discover_trade_symbols( + exchange_manager, + seed_symbols=["ALGO/USDC"], + account=_make_account({"ALGO": 100.0}), + account_trading=account_trading, + fresh_transactions=[], + reference_market="USDC", + ) + assert result.count("ALGO/USDC") == 1 diff --git a/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py b/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py new file mode 100644 index 0000000000..5d1a52c81f --- /dev/null +++ b/packages/flow/tests/logic/portfolio_history/test_trading_history_merge.py @@ -0,0 +1,79 @@ +import datetime +import mock +import pytest + +import octobot_trading.enums as trading_enums + +import octobot_flow.logic.portfolio_history.trading_history_merge as trading_history_merge_module + + +class TestMergeAndPersistTradingHistory: + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_merges_new_trades_deduped(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trade = mock.MagicMock() + existing_trade.trades = [] + existing_trade.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trade + mock_provider.load_state.return_value = mock_state + + new_trades = [ + { + trading_enums.ExchangeConstantsOrderColumns.ID.value: "t1", + trading_enums.ExchangeConstantsOrderColumns.EXCHANGE_TRADE_ID.value: "t1", + trading_enums.ExchangeConstantsOrderColumns.SYMBOL.value: "BTC/USDT", + trading_enums.ExchangeConstantsOrderColumns.TYPE.value: "limit", + trading_enums.ExchangeConstantsOrderColumns.SIDE.value: "buy", + trading_enums.ExchangeConstantsOrderColumns.AMOUNT.value: 1.0, + trading_enums.ExchangeConstantsOrderColumns.PRICE.value: 30000.0, + trading_enums.ExchangeConstantsOrderColumns.STATUS.value: "filled", + trading_enums.ExchangeConstantsOrderColumns.TIMESTAMP.value: 1700000000.0, + } + ] + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", new_trades, [], + ) + mock_provider.save_state.assert_called_once() + + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_merges_new_transactions_deduped(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trading = mock.MagicMock() + existing_trading.trades = [] + existing_trading.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trading + mock_provider.load_state.return_value = mock_state + + new_txs = [ + { + trading_enums.ExchangeConstantsTransactionColumns.TXID.value: "tx1", + trading_enums.ExchangeConstantsTransactionColumns.CURRENCY.value: "BTC", + trading_enums.ExchangeConstantsTransactionColumns.AMOUNT.value: 1.0, + trading_enums.ExchangeConstantsTransactionColumns.TIMESTAMP.value: 1000000, + trading_enums.ExchangeConstantsTransactionColumns.TYPE.value: "blockchain_deposit", + } + ] + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", [], new_txs, + ) + mock_provider.save_state.assert_called_once() + + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + def test_empty_inputs_still_saves(self, mock_provider_cls): + mock_provider = mock.MagicMock() + mock_provider_cls.instance.return_value = mock_provider + existing_trading = mock.MagicMock() + existing_trading.trades = [] + existing_trading.transactions = [] + mock_state = mock.MagicMock() + mock_state.account_trading = existing_trading + mock_provider.load_state.return_value = mock_state + + trading_history_merge_module.merge_and_persist_trading_history( + "wallet1", "acc1", [], [], + ) + mock_provider.save_state.assert_called_once() diff --git a/packages/flow/tests/repositories/community/test_trading_signals_channel.py b/packages/flow/tests/repositories/community/test_trading_signals_channel.py index 85cfd500d4..f803d55a28 100644 --- a/packages/flow/tests/repositories/community/test_trading_signals_channel.py +++ b/packages/flow/tests/repositories/community/test_trading_signals_channel.py @@ -82,4 +82,3 @@ async def test_shutdown_internal_trading_signal_channel_allows_recreate(internal new_channel = await trading_signals_channel.get_or_create_internal_trading_signal_channel() assert new_channel is not None - diff --git a/packages/flow/tests/repositories/exchange/__init__.py b/packages/flow/tests/repositories/exchange/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/flow/tests/repositories/exchange/test_tickers_repository.py b/packages/flow/tests/repositories/exchange/test_tickers_repository.py new file mode 100644 index 0000000000..57cb90ae69 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_tickers_repository.py @@ -0,0 +1,33 @@ +# Drakkar-Software OctoBot-Flow + +import mock +import pytest + +import octobot_trading.exchange_data as exchange_data_module + +import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module + +pytestmark = pytest.mark.asyncio + + +class TestTickersRepositoryEnsureTemporaryTickerChannel: + async def test_creates_channels_and_ticker_producer_only(self): + exchange_manager = mock.Mock() + with ( + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await tickers_repository_module.TickersRepository.ensure_temporary_ticker_channel(exchange_manager) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [exchange_data_module.TickerUpdater], + start_producers=False, + ) diff --git a/packages/flow/tests/repositories/exchange/test_trades_repository.py b/packages/flow/tests/repositories/exchange/test_trades_repository.py new file mode 100644 index 0000000000..ed527d4846 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_trades_repository.py @@ -0,0 +1,321 @@ +import mock +import pytest + +import octobot_flow.repositories.exchange.trades_repository as trades_repository_module +import octobot_trading.enums as trading_enums +import octobot_trading.personal_data as trading_personal_data + + +def _make_repo(exchange_manager): + fetched_data = mock.MagicMock() + return trades_repository_module.TradesRepository(exchange_manager, [], fetched_data) + + +def _allow_all_symbols(exchange_manager): + exchange_manager.symbol_exists.return_value = True + return exchange_manager + + +class TestTradesRepositoryFetchTrades: + @pytest.mark.asyncio + async def test_uses_updater_fetch_and_ensure_parsing(self): + exchange_manager = _allow_all_symbols(mock.MagicMock()) + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [{"id": "raw-trade-1", "symbol": "BTC/USDT"}] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + return_value={"id": "parsed-trade-1"}, + ) as ensure_parsing_mock, + ): + result = await repo.fetch_trades(["BTC/USDT"]) + + updater.fetch_trades.assert_awaited_once_with(["BTC/USDT"], exhaust_history=False) + ensure_parsing_mock.assert_called_once_with( + exchange_manager, + {"id": "raw-trade-1", "symbol": "BTC/USDT"}, + ) + assert result == [{"id": "parsed-trade-1"}] + + @pytest.mark.asyncio + async def test_empty_symbols_returns_empty_list(self): + repo = _make_repo(mock.MagicMock()) + assert await repo.fetch_trades([]) == [] + + +class TestFetchTradesPaginatedClientSideFilter: + @pytest.mark.asyncio + async def test_bulk_fetch_single_call_when_client_side_filter_enabled(self): + exchange_manager = _allow_all_symbols(mock.MagicMock()) + exchange_manager.exchange.get_option_value.return_value = True + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "symbol": "ALGO/USDC", "timestamp": 1700000000.0}, + {"id": "trade-2", "symbol": "SOL/USDC", "timestamp": 1700000001.0}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + result = await repo.fetch_trades_paginated( + ["ALGO/USDC", "SOL/USDC"], + existing_config_symbols=set(), + exchange_name="kraken", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + updater.fetch_trades.assert_awaited_once_with([], exhaust_history=True) + assert len(result) == 2 + + @pytest.mark.asyncio + async def test_bulk_fetch_logs_per_symbol_counts(self, caplog): + import logging + caplog.set_level(logging.INFO) + exchange_manager = _allow_all_symbols(mock.MagicMock()) + exchange_manager.exchange.get_option_value.return_value = True + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "symbol": "ALGO/USDC", "timestamp": 1700000000.0}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + await repo.fetch_trades_paginated( + ["ALGO/USDC", "SOL/USDC"], + existing_config_symbols={"ALGO/USDC"}, + exchange_name="kraken", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + assert "Fetched 1 trades for ALGO/USDC on kraken" in caplog.text + assert "Fetched 0 trades for SOL/USDC on kraken" in caplog.text + + +class TestFetchTradesPaginatedPerSymbol: + @pytest.mark.asyncio + async def test_fetches_one_call_per_symbol_when_client_side_filter_disabled(self): + exchange_manager = _allow_all_symbols(mock.MagicMock()) + exchange_manager.exchange.get_option_value.return_value = False + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "timestamp": 1700000000.0, "symbol": "BTC/USDT"}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + result = await repo.fetch_trades_paginated( + ["BTC/USDT"], + existing_config_symbols=set(), + exchange_name="binance", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + updater.fetch_trades.assert_awaited_once_with( + ["BTC/USDT"], + exhaust_history=True, + ) + assert len(result) == 1 + + @pytest.mark.asyncio + async def test_multiple_symbols_single_batch_fetch(self): + exchange_manager = _allow_all_symbols(mock.MagicMock()) + exchange_manager.exchange.get_option_value.return_value = False + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + await repo.fetch_trades_paginated( + ["BTC/USDT", "ETH/USDT"], + existing_config_symbols={"BTC/USDT"}, + exchange_name="binance", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + updater.fetch_trades.assert_awaited_once_with( + ["BTC/USDT", "ETH/USDT"], + exhaust_history=True, + ) + + @pytest.mark.asyncio + async def test_exhaust_history_when_ccxt_paginate_option_enabled(self): + exchange_manager = _allow_all_symbols(mock.MagicMock()) + + def get_option_value(option_key): + if option_key == trading_enums.ExchangeClientOptions.MY_TRADES_SYMBOL_FILTER_IS_CLIENT_SIDE: + return False + if option_key == trading_enums.ExchangeClientOptions.MY_TRADES_FETCH_USE_CCXT_PAGINATE: + return True + return trading_enums.DEFAULT_EXCHANGE_OPTION_VALUES.get(option_key) + + exchange_manager.exchange.get_option_value = get_option_value + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "timestamp": 1700000000.0, "symbol": "BTC/USDT"}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + await repo.fetch_trades_paginated( + ["BTC/USDT"], + existing_config_symbols=set(), + exchange_name="binance", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + updater.fetch_trades.assert_awaited_once_with( + ["BTC/USDT"], + exhaust_history=True, + ) + + @pytest.mark.asyncio + async def test_returns_empty_for_empty_symbol_list(self): + repo = _make_repo(mock.MagicMock()) + result = await repo.fetch_trades_paginated( + [], + existing_config_symbols=set(), + exchange_name="binance", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + assert result == [] + + +class TestTradesRepositorySkipsDelistedBeforeParsing: + @pytest.mark.asyncio + async def test_bulk_fetch_skips_parsing_for_delisted_symbols(self): + exchange_manager = mock.MagicMock() + exchange_manager.symbol_exists.side_effect = lambda symbol: symbol == "ALGO/USDC" + exchange_manager.exchange.get_option_value.return_value = True + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "symbol": "ALGO/USDC", "timestamp": 1700000000.0}, + {"id": "trade-2", "symbol": "MATICXBT", "timestamp": 1700000001.0}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ) as ensure_parsing_mock, + ): + result = await repo.fetch_trades_paginated( + ["ALGO/USDC"], + existing_config_symbols=set(), + exchange_name="kraken", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + ensure_parsing_mock.assert_called_once_with( + exchange_manager, + {"id": "trade-1", "symbol": "ALGO/USDC", "timestamp": 1700000000.0}, + ) + assert len(result) == 1 + assert result[0]["symbol"] == "ALGO/USDC" + + @pytest.mark.asyncio + async def test_bulk_fetch_logs_skipped_delisted_symbols(self, caplog): + import logging + + caplog.set_level(logging.INFO) + exchange_manager = mock.MagicMock() + exchange_manager.symbol_exists.side_effect = lambda symbol: symbol == "ALGO/USDC" + exchange_manager.exchange.get_option_value.return_value = True + repo = _make_repo(exchange_manager) + updater = mock.AsyncMock() + updater.fetch_trades.return_value = [ + {"id": "trade-1", "symbol": "ALGO/USDC", "timestamp": 1700000000.0}, + {"id": "trade-2", "symbol": "MATICXBT", "timestamp": 1700000001.0}, + {"id": "trade-3", "symbol": "FTMEUR", "timestamp": 1700000002.0}, + ] + with ( + mock.patch.object(repo, "_get_trades_updater", return_value=updater), + mock.patch.object( + trading_personal_data.TradesUpdater, + "ensure_parsing", + side_effect=lambda _exchange_manager, raw_trade: raw_trade, + ), + ): + await repo.fetch_trades_paginated( + ["ALGO/USDC"], + existing_config_symbols=set(), + exchange_name="kraken", + account_id="acc-1", + exchange_config_id="cfg-1", + exchange_config_name="cfg-name", + ) + + assert "Skipped 2 trades on delisted/unknown markets before parsing" in caplog.text + assert "MATICXBT" in caplog.text + assert "FTMEUR" in caplog.text + + +class TestTradesRepositoryEnsureTemporaryTradesChannel: + @pytest.mark.asyncio + async def test_creates_channels_and_trades_producer_only(self): + exchange_manager = mock.Mock() + with ( + mock.patch( + "octobot_trading.exchanges.create_exchange_channels", + mock.AsyncMock(), + ) as create_exchange_channels_mock, + mock.patch( + "octobot_trading.exchanges.create_producers", + mock.AsyncMock(), + ) as create_producers_mock, + ): + await trades_repository_module.TradesRepository.ensure_temporary_trades_channel(exchange_manager) + + create_exchange_channels_mock.assert_awaited_once_with(exchange_manager) + create_producers_mock.assert_awaited_once_with( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) diff --git a/packages/flow/tests/repositories/exchange/test_transactions_repository.py b/packages/flow/tests/repositories/exchange/test_transactions_repository.py new file mode 100644 index 0000000000..6a2f31dfc8 --- /dev/null +++ b/packages/flow/tests/repositories/exchange/test_transactions_repository.py @@ -0,0 +1,89 @@ +import mock +import pytest + +import octobot_trading.errors as trading_errors + +import octobot_flow.repositories.exchange.transactions_repository as transactions_repository_module + + +def _make_repo(exchange_manager): + fetched_data = mock.MagicMock() + return transactions_repository_module.TransactionsRepository(exchange_manager, [], fetched_data) + + +class TestFetchDeposits: + @pytest.mark.asyncio + async def test_returns_deposits_when_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_deposits.return_value = [{"id": "d1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [{"id": "d1"}] + + @pytest.mark.asyncio + async def test_returns_empty_when_not_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "binance" + exchange_manager.exchange.get_deposits.side_effect = trading_errors.NotSupported("nope") + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [] + + @pytest.mark.asyncio + async def test_returns_empty_when_authentication_fails(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "kraken" + exchange_manager.exchange.get_deposits.side_effect = trading_errors.AuthenticationError("denied") + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits() + assert result == [] + + @pytest.mark.asyncio + async def test_forwards_currencies_to_exchange_get_deposits(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_deposits.return_value = [{"id": "d1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_deposits(currencies=["BTC", "ETH"]) + assert result == [{"id": "d1"}] + exchange_manager.exchange.get_deposits.assert_awaited_once_with( + since=None, limit=None, currencies=["BTC", "ETH"] + ) + + +class TestFetchWithdrawals: + @pytest.mark.asyncio + async def test_returns_withdrawals_when_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_withdrawals.return_value = [{"id": "w1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [{"id": "w1"}] + + @pytest.mark.asyncio + async def test_returns_empty_when_not_supported(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "binance" + exchange_manager.exchange.get_withdrawals.side_effect = trading_errors.NotSupported("nope") + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [] + + @pytest.mark.asyncio + async def test_returns_empty_when_authentication_fails(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange_name = "kraken" + exchange_manager.exchange.get_withdrawals.side_effect = trading_errors.AuthenticationError("denied") + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals() + assert result == [] + + @pytest.mark.asyncio + async def test_forwards_currencies_to_exchange_get_withdrawals(self): + exchange_manager = mock.AsyncMock() + exchange_manager.exchange.get_withdrawals.return_value = [{"id": "w1"}] + repo = _make_repo(exchange_manager) + result = await repo.fetch_withdrawals(currencies=["BTC", "ETH"]) + assert result == [{"id": "w1"}] + exchange_manager.exchange.get_withdrawals.assert_awaited_once_with( + since=None, limit=None, currencies=["BTC", "ETH"] + ) diff --git a/packages/flow/tests/repositories/exchange/trades_repository_test_util.py b/packages/flow/tests/repositories/exchange/trades_repository_test_util.py new file mode 100644 index 0000000000..8b1e5251bf --- /dev/null +++ b/packages/flow/tests/repositories/exchange/trades_repository_test_util.py @@ -0,0 +1,22 @@ +import octobot_trading.api as trading_api_module +import octobot_trading.constants as trading_constants +import octobot_trading.exchange_channel as exchange_channel_module +import octobot_trading.exchanges as trading_exchanges +import octobot_trading.personal_data as trading_personal_data + + +async def ensure_trades_channel(exchange_manager) -> None: + try: + trading_api_module.get_channel_updater(exchange_manager, trading_constants.TRADES_CHANNEL) + return + except KeyError: + pass + try: + exchange_channel_module.get_chan(trading_constants.TRADES_CHANNEL, exchange_manager.id) + except KeyError: + await trading_exchanges.create_exchange_channels(exchange_manager) + await trading_exchanges.create_producers( + exchange_manager, + [trading_personal_data.TradesUpdater], + start_producers=False, + ) diff --git a/packages/node/octobot_node/constants.py b/packages/node/octobot_node/constants.py index 081b3a972b..beeac2ccdc 100644 --- a/packages/node/octobot_node/constants.py +++ b/packages/node/octobot_node/constants.py @@ -51,6 +51,9 @@ os.getenv("RUN_OCTOBOT_PROCESS_PING_TIMEOUT_SECONDS", 150.0) ) +# Max full trade dicts kept on automation.exchange_account_elements.trades; older ids archived in trade_summaries. +AUTOMATION_LIVE_STATE_MAX_TRADES = 100 + TASKS_ENCRYPTION_ENV_VARS = [ "TASKS_SERVER_RSA_PRIVATE_KEY", "TASKS_SERVER_ECDSA_PRIVATE_KEY", @@ -74,6 +77,13 @@ DEFAULT_PORTFOLIO_VALUATION_UNIT = "USDT" +GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS = float( + os.getenv("GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS", "300.0") +) +GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS = float( + os.getenv("GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS", "5") +) + NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_ID = "non-trading-generic-process-octobot-strategy" NON_TRADING_GENERIC_PROCESS_OCTOBOT_STRATEGY_VERSION = "1.0.0" diff --git a/packages/node/octobot_node/enums.py b/packages/node/octobot_node/enums.py index f9cc05cf4b..77ec7ffa20 100644 --- a/packages/node/octobot_node/enums.py +++ b/packages/node/octobot_node/enums.py @@ -38,3 +38,5 @@ class SchedulerQueues(enum.Enum): AUTOMATION_WORKFLOW_QUEUE = "automation_workflow_queue" USER_ACTION_QUEUE = "user_action_queue" DBOS_CLEANUP_QUEUE = "dbos_cleanup_queue" + GLOBAL_VIEW_QUEUE = "global_view_queue" + PORTFOLIO_HISTORY_QUEUE = "portfolio_history_queue" diff --git a/packages/node/octobot_node/protocol/accounts_history.py b/packages/node/octobot_node/protocol/accounts_history.py new file mode 100644 index 0000000000..67c1f027c0 --- /dev/null +++ b/packages/node/octobot_node/protocol/accounts_history.py @@ -0,0 +1,264 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import decimal +import datetime + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping + +import octobot_flow.logic.portfolio_history.portfolio_value_history as portfolio_value_history_module + + +def get_portfolio_history_state( + user_id: str, + account_id: str, +) -> protocol_models.PortfolioHistoricalValuesState: + """Compute portfolio history on-the-fly from persisted data.""" + # Placeholder: the async compute must be awaited from an async context. + # This synchronous wrapper returns an empty state; callers should use the async version. + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +async def compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + user_id: str, + account_id: str, + *, + data_root: str = None, +) -> protocol_models.PortfolioHistoricalValuesState: + """ + Compute portfolio value history on-the-fly from Account.assets, + AccountTrading trades/transactions, and persisted price caches. + """ + # Step 1: Load account and its assets as the latest portfolio anchor. + try: + account = collection_providers.AccountProvider.instance().get_account(user_id, account_id) + except (collection_errors.CollectionNoDataError, collection_errors.ItemNotFoundError): + return _empty_state() + + portfolio = _portfolio_from_account_assets(account) + if not portfolio: + return _empty_state() + + # Step 2: Load AccountTrading trades and transactions. + try: + trading_state = collection_providers.AccountTradingProvider.instance().load_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + return _empty_state() + + account_trading = trading_state.account_trading + + # Step 3: Resolve exchange info from the account's exchange config. + exchange_info = _resolve_exchange_info(user_id, account) + if exchange_info is None: + return _empty_state() + exchange_name, exchange_type, sandboxed = exchange_info + + # Step 4: Load persisted caches. + daily_prices = await trading_api.load_daily_prices(exchange_name, exchange_type, sandboxed, data_root) + latest_tickers = await trading_api.load_latest_tickers(exchange_name, exchange_type, sandboxed, data_root) + + # Step 5: Compute historical holdings via reverse replay. + daily_holdings = trading_api.compute_portfolio_historical_holdings_from_latest_portfolio_trades_and_transations( + portfolio, + account_trading.trades or [], + account_trading.transactions or [], + ) + + # Step 6: Daily valuation. + valuation_unit = "USDT" + history_values = portfolio_value_history_module.compute_daily_portfolio_values( + daily_holdings, daily_prices, latest_tickers, reference_market=valuation_unit, + ) + + # Step 7: Build and return result (not saved). + if not history_values: + return _empty_state() + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=history_values, + ), + ) + + +async def compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + user_id: str, + *, + is_simulated: bool, + data_root: str = None, +) -> protocol_models.PortfolioHistoricalValuesState: + """ + Compute aggregated portfolio value history across all accounts matching is_simulated. + Each account is valued independently, then daily totals are summed by timestamp. + """ + accounts = [ + account + for account in collection_providers.AccountProvider.instance().list_accounts(user_id) + if account.is_simulated == is_simulated + ] + if not accounts: + return _empty_state() + + account_histories: list[list[protocol_models.PortfolioHistoricalValue]] = [] + for account in accounts: + account_state = await compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + user_id, + account.id, + data_root=data_root, + ) + if account_state.history is None or not account_state.history.values: + continue + account_histories.append(account_state.history.values) + + if not account_histories: + return _empty_state() + + valuation_unit = "USDT" + aggregated_values = _aggregate_portfolio_historical_values(account_histories) + if not aggregated_values: + return _empty_state() + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit=valuation_unit, + values=aggregated_values, + ), + ) + + +def _iter_historical_assets( + history_value: protocol_models.PortfolioHistoricalValue, +): + if not history_value.assets: + return + for assets_for_type in history_value.assets: + for asset in assets_for_type.assets or []: + yield assets_for_type.trading_type, asset + + +def _aggregate_portfolio_historical_values( + account_histories: list[list[protocol_models.PortfolioHistoricalValue]], +) -> list[protocol_models.PortfolioHistoricalValue]: + totals_by_day: dict[float, float] = {} + assets_by_day: dict[float, dict[protocol_models.TradingType, dict[str, list[float]]]] = {} + for history_values in account_histories: + for history_value in history_values: + day_key = portfolio_value_history_module._utc_day_start( + history_value.timestamp.timestamp(), + ) + totals_by_day[day_key] = totals_by_day.get(day_key, 0.0) + float(history_value.total) + day_assets = assets_by_day.setdefault(day_key, {}) + for trading_type, asset in _iter_historical_assets(history_value): + symbol_totals = day_assets.setdefault(trading_type, {}) + holdings_sum, value_sum = symbol_totals.get(asset.symbol, [0.0, 0.0]) + symbol_totals[asset.symbol] = [ + holdings_sum + float(asset.holdings), + value_sum + float(asset.value), + ] + + aggregated_values: list[protocol_models.PortfolioHistoricalValue] = [] + for day_key in sorted(totals_by_day): + assets_for_day = assets_by_day.get(day_key, {}) + assets_by_trading_type: list[protocol_models.HistoricalAssetsForTradingType] = [] + for trading_type in sorted(assets_for_day, key=lambda trading_type_value: trading_type_value.value): + symbol_totals = assets_for_day[trading_type] + day_asset_values: list[protocol_models.HistoricalAssetValue] = [] + for symbol, holdings_and_value in sorted(symbol_totals.items()): + holdings_sum, value_sum = holdings_and_value + if holdings_sum == 0: + continue + day_asset_values.append( + protocol_models.HistoricalAssetValue( + symbol=symbol, + holdings=holdings_sum, + value=value_sum, + ) + ) + if day_asset_values: + assets_by_trading_type.append( + protocol_models.HistoricalAssetsForTradingType( + trading_type=trading_type, + assets=day_asset_values, + ) + ) + aggregated_values.append( + protocol_models.PortfolioHistoricalValue( + timestamp=datetime.datetime.fromtimestamp(day_key, tz=datetime.timezone.utc), + total=totals_by_day[day_key], + assets=assets_by_trading_type or None, + ) + ) + return aggregated_values + + +def _empty_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=None, + ) + + +def _portfolio_from_account_assets(account: protocol_models.Account) -> dict[str, dict[str, decimal.Decimal]]: + """Convert Account.assets to the portfolio dict format used by the builder.""" + portfolio: dict[str, dict[str, decimal.Decimal]] = {} + if not account.assets: + return portfolio + for assets_for_type in account.assets: + for asset in (assets_for_type.assets or []): + portfolio[asset.symbol] = { + commons_constants.PORTFOLIO_TOTAL: decimal.Decimal(str(asset.total)), + commons_constants.PORTFOLIO_AVAILABLE: decimal.Decimal(str(asset.available or asset.total)), + } + return portfolio + + +def _resolve_exchange_info( + user_id: str, account: protocol_models.Account +) -> tuple[str, str, bool] | None: + """Derive exchange identity from the account's first exchange config.""" + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return None + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return None + config_ids = exchange_account.exchange_config_ids or [] + if not config_ids: + return None + try: + exchange_config = collection_providers.AccountProvider.instance().get_exchange_config( + user_id, config_ids[0] + ) + except collection_errors.CollectionNoDataError: + return None + exchange_type = protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get( + protocol_models.TradingType.SPOT + ) + return ( + exchange_config.exchange, + exchange_type.value if exchange_type else "spot", + exchange_config.sandboxed, + ) \ No newline at end of file diff --git a/packages/node/octobot_node/protocol/accounts_trading.py b/packages/node/octobot_node/protocol/accounts_trading.py index e9ad04ba79..86f9d534e8 100644 --- a/packages/node/octobot_node/protocol/accounts_trading.py +++ b/packages/node/octobot_node/protocol/accounts_trading.py @@ -101,3 +101,22 @@ def update_account_trading( account_trading.updated_at = datetime.datetime.now(datetime.UTC) # Step: persist the updated trading state for this account. trading_provider.AccountTradingProvider.instance().save_state(user_id, account_id, trading_state) + + +def reset_account_trading_data(user_id: str, account_id: str) -> None: + try: + trading_state = get_account_trading_state(user_id, account_id) + except collection_errors.CollectionNoDataError: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ) + account_trading = trading_state.account_trading + account_trading.orders = None + account_trading.trades = None + account_trading.transactions = None + account_trading.positions = None + account_trading.updated_at = datetime.datetime.now(datetime.UTC) + trading_provider.AccountTradingProvider.instance().save_state(user_id, account_id, trading_state) diff --git a/packages/node/octobot_node/protocol/automations.py b/packages/node/octobot_node/protocol/automations.py index 67afc32720..d305b69b7b 100644 --- a/packages/node/octobot_node/protocol/automations.py +++ b/packages/node/octobot_node/protocol/automations.py @@ -26,7 +26,6 @@ import octobot_flow.entities as flow_entities import octobot_flow.entities.automations.octobot_process_state as octobot_process_state_module import octobot_node.models as node_models -import octobot_node.scheduler.octobot_flow_client as octobot_flow_client import octobot_node.scheduler.workflows.params as workflow_params import octobot_protocol.models as protocol_models import octobot_trading.constants as octobot_trading_constants @@ -195,11 +194,11 @@ def _to_protocol_automation_state( workflow_error: typing.Optional[str] = None, ) -> protocol_models.AutomationState: content_missing = _task_content_is_missing(task.content) - flow_automation_state = ( - _empty_flow_automation_state() - if content_missing - else _parse_automation_state(task) - ) + if content_missing: + flow_automation_state = _empty_flow_automation_state() + else: + import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader + flow_automation_state = automation_states_loader.parse_flow_automation_state(task) filled = _fill_protocol_automation_state( _base_protocol_automation_state(task), flow_automation_state ) @@ -213,14 +212,6 @@ def _to_protocol_automation_state( ) -def _parse_automation_state(task: node_models.Task) -> flow_entities.AutomationState: - parsed_description = octobot_flow_client.OctoBotActionsJobDescription.parse_task_description(task.content) - automation_state: octobot_flow_client.OctoBotActionsJobDescription = octobot_flow_client.OctoBotActionsJobDescription.from_dict( - parsed_description - ) - return flow_entities.AutomationState.from_dict(automation_state.state) - - def _flow_action_reports_error(flow_action: flow_entities.AbstractActionDetails) -> bool: error_status = flow_action.error_status if error_status is None: @@ -412,7 +403,7 @@ def _fill_protocol_automation_state( assets = octobot_trading_portfolios_protocol.to_protocol_assets(portfolio.content) orders = _order_summaries_from_open_orders(exchange_elements.orders.open_orders) or None positions = _position_summaries(exchange_elements.positions) or None - trades = _trade_summaries(exchange_elements.trades) or None + trades = _automation_trade_summaries(exchange_elements) or None metadata = protocol_automation_state.metadata.model_copy( update={ "updated_at": _metadata_updated_at_from_execution( @@ -483,3 +474,24 @@ def _trade_summaries(trades: list[dict]) -> list[protocol_models.TradeSummary]: continue summaries.append(protocol_models.TradeSummary(id=str(trade_id), symbol=str(symbol))) return summaries + + +def _automation_trade_summaries( + exchange_elements: flow_entities.ExchangeAccountElements, +) -> list[protocol_models.TradeSummary]: + summaries_by_id: dict[str, protocol_models.TradeSummary] = {} + for symbol, trade_ids in (exchange_elements.trade_summaries or {}).items(): + symbol_text = str(symbol).strip() + if not symbol_text: + continue + for trade_id in trade_ids or []: + trade_id_text = str(trade_id).strip() + if not trade_id_text: + continue + summaries_by_id[trade_id_text] = protocol_models.TradeSummary( + id=trade_id_text, + symbol=symbol_text, + ) + for trade_summary in _trade_summaries(exchange_elements.trades): + summaries_by_id[trade_summary.id] = trade_summary + return list(summaries_by_id.values()) diff --git a/packages/node/octobot_node/scheduler/api.py b/packages/node/octobot_node/scheduler/api.py index d2caf6211f..4bb71e96eb 100644 --- a/packages/node/octobot_node/scheduler/api.py +++ b/packages/node/octobot_node/scheduler/api.py @@ -28,7 +28,7 @@ import octobot_node.scheduler import octobot_node.scheduler.workflows_util as workflows_util -logger = octobot_commons_logging.get_logger("octobot_node.scheduler.api") +logger = octobot_commons_logging.get_logger("scheduler_api") def get_node_status() -> dict[str, str | int | None | uuid.UUID]: diff --git a/packages/node/octobot_node/scheduler/automations/__init__.py b/packages/node/octobot_node/scheduler/automations/__init__.py new file mode 100644 index 0000000000..c5aca2ae6c --- /dev/null +++ b/packages/node/octobot_node/scheduler/automations/__init__.py @@ -0,0 +1,46 @@ +import octobot_node.scheduler.automations.automation_states_loader as automations_automation_states_loader +import octobot_node.scheduler.automations.generic_process_octobot as automations_generic_process_octobot +import octobot_node.scheduler.automations.octobot_flow_client as automations_octobot_flow_client +import octobot_node.scheduler.automations.trade_symbols_resolver as automations_trade_symbols_resolver + +STATE_KEY = automations_automation_states_loader.STATE_KEY +create_generic_process_bot = automations_generic_process_octobot.create_generic_process_bot +get_automation_copied_strategy_ids = automations_automation_states_loader.get_automation_copied_strategy_ids +get_automation_dict = automations_automation_states_loader.get_automation_dict +get_automation_id = automations_automation_states_loader.get_automation_id +get_automation_state_dict = automations_automation_states_loader.get_automation_state_dict +get_automation_state_reader = automations_automation_states_loader.get_automation_state_reader +get_automation_workflow_status = automations_automation_states_loader.get_automation_workflow_status +load_flow_automation_states_by_id = automations_automation_states_loader.load_flow_automation_states_by_id +load_protocol_automation_states = automations_automation_states_loader.load_protocol_automation_states +load_wallet_automation_states = automations_automation_states_loader.load_wallet_automation_states +load_wallet_automation_states_for_trade_symbols = ( + automations_automation_states_loader.load_wallet_automation_states_for_trade_symbols +) +parse_flow_automation_state = automations_automation_states_loader.parse_flow_automation_state +patch_task_content_degraded_state = automations_automation_states_loader.patch_task_content_degraded_state +resolve_trade_symbols = automations_trade_symbols_resolver.resolve_trade_symbols +OctoBotActionsJob = automations_octobot_flow_client.OctoBotActionsJob +OctoBotActionsJobDescription = automations_octobot_flow_client.OctoBotActionsJobDescription +OctoBotActionsJobResult = automations_octobot_flow_client.OctoBotActionsJobResult + +__all__ = [ + "OctoBotActionsJob", + "OctoBotActionsJobDescription", + "OctoBotActionsJobResult", + "STATE_KEY", + "create_generic_process_bot", + "get_automation_copied_strategy_ids", + "get_automation_dict", + "get_automation_id", + "get_automation_state_dict", + "get_automation_state_reader", + "get_automation_workflow_status", + "load_flow_automation_states_by_id", + "load_protocol_automation_states", + "load_wallet_automation_states", + "load_wallet_automation_states_for_trade_symbols", + "parse_flow_automation_state", + "patch_task_content_degraded_state", + "resolve_trade_symbols", +] diff --git a/packages/node/octobot_node/scheduler/automations/automation_states_loader.py b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py new file mode 100644 index 0000000000..c0f0327f1c --- /dev/null +++ b/packages/node/octobot_node/scheduler/automations/automation_states_loader.py @@ -0,0 +1,218 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import contextlib +import dataclasses +import json +import typing + +import dbos as dbos_lib +import octobot_protocol.models as protocol_models + +import octobot_flow.entities as flow_entities +import octobot_flow.parsers.automation_state_reader as automation_state_reader_module + +import octobot_node.models as node_models +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client +import octobot_node.scheduler.task_context as task_context_module +import octobot_node.scheduler.workflows_util as workflows_util_module + +if typing.TYPE_CHECKING: + import octobot_node.protocol.automations as automations_protocol + +STATE_KEY = "state" + + +@dataclasses.dataclass +class WalletAutomationStates: + protocol_states: list[protocol_models.AutomationState] + flow_states_by_id: dict[str, flow_entities.AutomationState] + + +def get_automation_dict(description: typing.Union[str, dict]) -> dict: + if isinstance(description, str): + description = json.loads(description) + if isinstance(description, dict) and (state := description.get(STATE_KEY)) and isinstance(state, dict): + return description + raise ValueError("No automation state found in description") + + +def get_automation_state_dict(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[dict]: + resolved_task = workflows_util_module.get_resolved_automation_task(workflow_status) + if resolved_task is None: + return None + with task_context_module.encrypted_task(resolved_task): + try: + return get_automation_dict(resolved_task.content)[STATE_KEY] + except ValueError: + return None + + +def get_automation_state_reader( + workflow_status: dbos_lib.WorkflowStatus, +) -> typing.Optional[automation_state_reader_module.AutomationStateReader]: + """Get the resolved automation state for a workflow row (input or terminal output).""" + if state_dict := get_automation_state_dict(workflow_status): + return automation_state_reader_module.AutomationStateReader( + flow_entities.AutomationState.from_dict(state_dict) + ) + return None + + +def get_automation_id(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[str]: + if state_dict := get_automation_state_dict(workflow_status): + return state_dict.get("automation", {}).get("metadata", {}).get("automation_id") + return None + + +def get_automation_copied_strategy_ids(workflow_status: dbos_lib.WorkflowStatus) -> list[str]: + if reader := get_automation_state_reader(workflow_status): + return reader.get_automation_copied_strategy_ids() + return [] + + +def patch_task_content_degraded_state( + task_content: str, + error_status: str, + error_message: str, + *, + since: float, +) -> str: + description = get_automation_dict(task_content) + automation_state = flow_entities.AutomationState.from_dict(description[STATE_KEY]) + existing_degraded_state = automation_state.automation.execution.degraded_state + degraded_since = ( + existing_degraded_state.since + if existing_degraded_state.since > 0 + else since + ) + automation_state.automation.execution.degraded_state = flow_entities.DegradedStateDetails( + since=degraded_since, + error=error_status, + reason=error_message, + ) + description[STATE_KEY] = automation_state.to_dict(include_default_values=False) + return json.dumps(description) + + +async def get_automation_workflow_status(automation_id: str) -> dbos_lib.WorkflowStatus: + for workflow_status in await dbos_lib.DBOS.list_workflows_async(status=[ + dbos_lib.WorkflowStatusString.PENDING.value, dbos_lib.WorkflowStatusString.ENQUEUED.value + ]): + if get_automation_id(workflow_status) == automation_id: + return workflow_status + raise ValueError(f"No automation workflow found for automation_id: {automation_id}") + + +def parse_flow_automation_state(task: node_models.Task) -> flow_entities.AutomationState: + parsed_description = octobot_flow_client.OctoBotActionsJobDescription.parse_task_description(task.content) + automation_state: octobot_flow_client.OctoBotActionsJobDescription = ( + octobot_flow_client.OctoBotActionsJobDescription.from_dict(parsed_description) + ) + return flow_entities.AutomationState.from_dict(automation_state.state) + + +def _flow_states_by_id_from_sources( + sources: list["automations_protocol.AutomationStateSource"], +) -> dict[str, flow_entities.AutomationState]: + flow_automation_states_by_id: dict[str, flow_entities.AutomationState] = {} + for source in sources: + try: + flow_automation_states_by_id[source.task.id] = parse_flow_automation_state(source.task) + except Exception: + continue + return flow_automation_states_by_id + + +def _protocol_states_from_sources( + sources: list["automations_protocol.AutomationStateSource"], +) -> list[protocol_models.AutomationState]: + import octobot_node.protocol.automations as automations_protocol + + return automations_protocol.to_protocol_automations_state(sources) + + +TRADE_SYMBOL_AUTOMATION_STATUSES: tuple[dbos_lib.WorkflowStatusString, ...] = ( + dbos_lib.WorkflowStatusString.ENQUEUED, + dbos_lib.WorkflowStatusString.PENDING, +) + + +async def load_automation_state_sources( + wallet_id: str, + statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, + *, + load_output: bool = True, +) -> list["automations_protocol.AutomationStateSource"]: + import octobot_node.protocol.automations as automations_protocol + import octobot_node.scheduler as scheduler_module + + scheduler = scheduler_module.SCHEDULER + workflows = await scheduler._get_latest_workflow_for_each_automation( + wallet_id, + statuses, + load_output=load_output, + ) + sources: list[automations_protocol.AutomationStateSource] = [] + for workflow in workflows: + workflow_output = workflows_util_module.parse_automation_workflow_output(workflow) + task = workflows_util_module.get_resolved_automation_task(workflow) + if task is None: + continue + task.id = workflows_util_module.normalize_parent_automation_id(workflow.workflow_id) + sources.append(automations_protocol.AutomationStateSource( + task=task, + workflow_status=workflow.status, + workflow_output=workflow_output, + workflow_error=str(workflow.error) if workflow.error else None, + )) + return sources + + +async def load_protocol_automation_states( + wallet_id: str, + statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, +) -> list[protocol_models.AutomationState]: + sources = await load_automation_state_sources(wallet_id, statuses) + with contextlib.ExitStack() as exit_stack: + for source in sources: + exit_stack.enter_context(task_context_module.encrypted_task(source.task)) + return _protocol_states_from_sources(sources) + + +async def load_flow_automation_states_by_id( + wallet_id: str, + statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, +) -> dict[str, flow_entities.AutomationState]: + sources = await load_automation_state_sources(wallet_id, statuses) + with contextlib.ExitStack() as exit_stack: + for source in sources: + exit_stack.enter_context(task_context_module.encrypted_task(source.task)) + return _flow_states_by_id_from_sources(sources) + + +async def load_wallet_automation_states( + wallet_id: str, + statuses: typing.Optional[list[dbos_lib.WorkflowStatusString]] = None, + *, + load_output: bool = True, +) -> WalletAutomationStates: + sources = await load_automation_state_sources(wallet_id, statuses, load_output=load_output) + with contextlib.ExitStack() as exit_stack: + for source in sources: + exit_stack.enter_context(task_context_module.encrypted_task(source.task)) + protocol_states = _protocol_states_from_sources(sources) + flow_states_by_id = _flow_states_by_id_from_sources(sources) + return WalletAutomationStates( + protocol_states=protocol_states, + flow_states_by_id=flow_states_by_id, + ) + + +async def load_wallet_automation_states_for_trade_symbols(wallet_id: str) -> WalletAutomationStates: + """Load active automation states for trade-symbol resolution (inputs only, no workflow outputs).""" + return await load_wallet_automation_states( + wallet_id, + statuses=list(TRADE_SYMBOL_AUTOMATION_STATUSES), + load_output=False, + ) diff --git a/packages/node/octobot_node/scheduler/generic_process_octobot.py b/packages/node/octobot_node/scheduler/automations/generic_process_octobot.py similarity index 100% rename from packages/node/octobot_node/scheduler/generic_process_octobot.py rename to packages/node/octobot_node/scheduler/automations/generic_process_octobot.py diff --git a/packages/node/octobot_node/scheduler/octobot_flow_client.py b/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py similarity index 97% rename from packages/node/octobot_node/scheduler/octobot_flow_client.py rename to packages/node/octobot_node/scheduler/automations/octobot_flow_client.py index c6566c6820..17e2036e73 100644 --- a/packages/node/octobot_node/scheduler/octobot_flow_client.py +++ b/packages/node/octobot_node/scheduler/automations/octobot_flow_client.py @@ -22,7 +22,7 @@ import octobot_node.constants import octobot_node.errors as errors -import octobot_node.scheduler.workflows_util as workflows_util +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader try: import octobot_flow.environment @@ -55,7 +55,7 @@ def __post_init__(self): @staticmethod def parse_task_description(description: typing.Union[str, dict]) -> dict: try: - parsed_description = workflows_util.get_automation_dict(description) + parsed_description = automation_states_loader.get_automation_dict(description) except ValueError: if isinstance(description, dict): parsed_description = description diff --git a/packages/node/octobot_node/scheduler/automations/trade_symbols_resolver.py b/packages/node/octobot_node/scheduler/automations/trade_symbols_resolver.py new file mode 100644 index 0000000000..3cdb060b11 --- /dev/null +++ b/packages/node/octobot_node/scheduler/automations/trade_symbols_resolver.py @@ -0,0 +1,71 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities as flow_entities + +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver + + +async def resolve_trade_symbols( + wallet_id: str, + account: protocol_models.Account, + exchange_config: protocol_models.ExchangeConfig, + *, + automation_states: list[protocol_models.AutomationState] | None = None, + flow_automation_states_by_id: dict[str, flow_entities.AutomationState] | None = None, +) -> list[str]: + """Union explicit config symbols with running automation order and strategy pairs.""" + symbols: set[str] = set(exchange_config.historical_trade_symbols or []) + if automation_states is None or flow_automation_states_by_id is None: + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states(wallet_id) + if automation_states is None: + automation_states = wallet_automation_states.protocol_states + if flow_automation_states_by_id is None: + flow_automation_states_by_id = wallet_automation_states.flow_states_by_id + for automation_state in automation_states: + if automation_state.status != protocol_models.WorkflowStatus.RUNNING: + continue + if account.id not in (automation_state.exchange_account_ids or []): + continue + for order in automation_state.orders or []: + if order.symbol: + symbols.add(order.symbol) + flow_automation_state = flow_automation_states_by_id.get(automation_state.id) + if flow_automation_state is None: + continue + symbols.update( + _strategy_symbols_from_flow_automation(wallet_id, flow_automation_state) + ) + return sorted(symbols) + + +def _strategy_symbols_from_flow_automation( + wallet_id: str, + flow_automation_state: flow_entities.AutomationState, +) -> list[str]: + strategy_id = flow_automation_state.automation.metadata.strategy_id + if not strategy_id: + return [] + try: + stored_strategy = collection_providers.StrategyProvider.instance().get_item( + wallet_id, + strategy_id, + ) + except collection_errors.ItemNotFoundError: + return [] + configuration_wrapper = stored_strategy.configuration + if configuration_wrapper is None or configuration_wrapper.actual_instance is None: + return [] + inner_configuration = configuration_wrapper.actual_instance + try: + return exchange_account_resolver._strategy_traded_symbols( + inner_configuration, + reference_market=stored_strategy.reference_market, + ) + except Exception: + return [] diff --git a/packages/node/octobot_node/scheduler/global_view/__init__.py b/packages/node/octobot_node/scheduler/global_view/__init__.py new file mode 100644 index 0000000000..be14b9abe2 --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/__init__.py @@ -0,0 +1,2 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. diff --git a/packages/node/octobot_node/scheduler/global_view/automation_trigger.py b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py new file mode 100644 index 0000000000..ffe4d84acf --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/automation_trigger.py @@ -0,0 +1,135 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. + +import asyncio +import time + +import dbos +import octobot_commons.logging as octobot_commons_logging +import octobot_protocol.models as protocol_models + +import octobot_node.constants as node_constants +import octobot_node.scheduler.api as scheduler_api +import octobot_node.scheduler.tasks as scheduler_tasks + + +def _logger(): + return octobot_commons_logging.get_logger("GlobalViewAutomationTrigger") + + +async def account_has_bound_running_automation(user_id: str, account_id: str) -> bool: + automation_states = await scheduler_api.get_automation_states(user_id) + for automation_state in automation_states: + if automation_state.status != protocol_models.WorkflowStatus.RUNNING: + continue + if not automation_state.exchange_account_ids or account_id not in automation_state.exchange_account_ids: + continue + return True + return False + + +async def trigger_account_automations( + user_id: str, + account_id: str, + changed_order_ids: set[str], +) -> None: + if not changed_order_ids: + return + matching_automations = await _matching_automations(user_id, account_id, changed_order_ids) + for automation_state in matching_automations: + await _trigger_automation_and_wait(user_id, automation_state.id) + + +async def _matching_automations( + user_id: str, + account_id: str, + changed_order_ids: set[str], +) -> list[protocol_models.AutomationState]: + automation_states = await scheduler_api.get_automation_states(user_id) + matching_automations: list[protocol_models.AutomationState] = [] + for automation_state in automation_states: + if automation_state.status != protocol_models.WorkflowStatus.RUNNING: + continue + if not automation_state.exchange_account_ids or account_id not in automation_state.exchange_account_ids: + continue + automation_order_ids = { + str(order_summary.id) + for order_summary in (automation_state.orders or []) + if order_summary.id + } + if automation_order_ids.intersection(changed_order_ids): + matching_automations.append(automation_state) + return matching_automations + + +async def _trigger_automation_and_wait(user_id: str, automation_id: str) -> None: + import octobot_node.scheduler as scheduler_module + + logger = _logger() + logger.info( + "Starting forced automation trigger (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) + scheduler = scheduler_module.SCHEDULER + active_workflow_ids_before = await scheduler.resolve_active_automation_workflow_ids_for_parent_id( + user_id, + automation_id, + ) + workflow_id_before = active_workflow_ids_before[0] if active_workflow_ids_before else None + logger.info( + "Resolved active workflow before trigger (automation_id=%s, workflow_id=%s)", + automation_id, + workflow_id_before, + ) + + await scheduler_tasks.send_forced_trigger_to_active_automation(automation_id, user_id) + logger.info( + "Forced trigger sent (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) + + if workflow_id_before is None: + logger.warning( + "No active workflow to wait for after forced trigger (automation_id=%s, user_id=%s)", + automation_id, + user_id, + ) + return + logger.info( + "Waiting for automation workflow iteration (workflow_id=%s)", + workflow_id_before, + ) + await _wait_for_workflow_iteration_success(workflow_id_before) + logger.info( + "Finished waiting for automation workflow iteration (workflow_id=%s)", + workflow_id_before, + ) + + +async def _wait_for_workflow_iteration_success(workflow_id: str) -> None: + deadline = time.monotonic() + node_constants.GLOBAL_VIEW_AUTOMATION_TRIGGER_TIMEOUT_SECONDS + while time.monotonic() < deadline: + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + if workflow_status is not None: + if workflow_status.status == dbos.WorkflowStatusString.SUCCESS.value: + return + if workflow_status.status in ( + dbos.WorkflowStatusString.ERROR.value, + dbos.WorkflowStatusString.CANCELLED.value, + dbos.WorkflowStatusString.MAX_RECOVERY_ATTEMPTS_EXCEEDED.value, + ): + _logger().error( + "Automation workflow %s ended with status %s while waiting for forced trigger iteration", + workflow_id, + workflow_status.status, + ) + return + await asyncio.sleep( + node_constants.GLOBAL_VIEW_WORKFLOW_POLL_INTERVAL_SECONDS + ) + _logger().error( + "Timed out waiting for automation workflow %s to complete forced trigger iteration", + workflow_id, + ) diff --git a/packages/node/octobot_node/scheduler/global_view/global_view_executor.py b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py new file mode 100644 index 0000000000..6e05d76207 --- /dev/null +++ b/packages/node/octobot_node/scheduler/global_view/global_view_executor.py @@ -0,0 +1,62 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. + +import octobot_flow.entities +import octobot_flow.jobs.global_view_account_job as global_view_account_job_module +import octobot_protocol.models as protocol_models + +import octobot_node.errors as node_errors +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver + + +async def refresh_account_global_view( + user_id: str, + account: protocol_models.Account, +) -> octobot_flow.entities.GlobalViewAccountRefreshResult: + account_specifics = account.specifics + if account_specifics is None or account_specifics.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "Account.specifics.actual_instance is required for global view refresh." + ) + account_specifics_instance = account_specifics.actual_instance + if isinstance(account_specifics_instance, protocol_models.GenericAccount): + return octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids=set(), + ) + if isinstance(account_specifics_instance, protocol_models.BlockchainAccount): + raise node_errors.InvalidUserActionPayloadError("Blockchain accounts are not supported yet.") + if not isinstance(account_specifics_instance, protocol_models.ExchangeAccount): + raise node_errors.InvalidUserActionPayloadError( + f"Unsupported account specifics type: {type(account_specifics_instance).__name__}." + ) + + exchange_account = account_specifics_instance + trading_type = account_state_updater._trading_type_for_account_state_check(account) + exchange_config = exchange_account_resolver.get_exchange_config(user_id, exchange_account) + authentication = None if account.is_simulated else account_authentication_resolver.get_exchange_authentication( + user_id, + account, + ) + auth_details = account_state_updater._encrypted_exchange_auth_details( + exchange_account, + authentication, + trading_type, + exchange_config.sandboxed, + ) + has_bound_automation = await automation_trigger_module.account_has_bound_running_automation( + user_id, + account.id, + ) + context = octobot_flow.entities.GlobalViewAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=trading_type, + auth_details=auth_details, + has_bound_automation=has_bound_automation, + ) + return await global_view_account_job_module.GlobalViewAccountJob(user_id, context).run() diff --git a/packages/node/octobot_node/scheduler/internal_trading_signals.py b/packages/node/octobot_node/scheduler/internal_trading_signals.py index aecc8e5921..e02743838c 100644 --- a/packages/node/octobot_node/scheduler/internal_trading_signals.py +++ b/packages/node/octobot_node/scheduler/internal_trading_signals.py @@ -3,7 +3,7 @@ import octobot_commons.logging import octobot_flow.entities import octobot_flow.repositories.community as trading_signals_channel -import octobot_node.scheduler.workflows_util as workflows_util +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.tasks as tasks @@ -38,7 +38,7 @@ async def _trigger_copier_automation(trading_signal: octobot_flow.entities.Tradi ) for pending_workflow_status in pending_workflow_statuses: if ( - trading_signal.strategy_id in workflows_util.get_automation_copied_strategy_ids(pending_workflow_status) + trading_signal.strategy_id in automation_states_loader.get_automation_copied_strategy_ids(pending_workflow_status) ): octobot_commons.logging.get_logger("internal_trading_signals").info( f"Triggering copier automation {pending_workflow_status.workflow_id} with trading signal {trading_signal.strategy_id}" diff --git a/packages/node/octobot_node/scheduler/portfolio_history/__init__.py b/packages/node/octobot_node/scheduler/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/node/octobot_node/scheduler/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py new file mode 100644 index 0000000000..6255b76d22 --- /dev/null +++ b/packages/node/octobot_node/scheduler/portfolio_history/portfolio_history_executor.py @@ -0,0 +1,135 @@ +import octobot_commons.logging as commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities.portfolio_history as portfolio_history_entities +import octobot_flow.jobs.portfolio_history_job as portfolio_history_job_module + +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver +import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.automations.trade_symbols_resolver as trade_symbols_resolver_module + +logger = commons_logging.get_logger("PortfolioHistoryExecutor") + + +async def run_portfolio_history_collection( + wallet_id: str, + account_ids: list[str] | None = None, +) -> list[portfolio_history_entities.PortfolioHistoryRunResult]: + """ + Build contexts for all exchange accounts of the given wallet and run + the PortfolioHistoryJob with parallel per-exchange fetching. + """ + account_provider = collection_providers.AccountProvider.instance() + try: + accounts = account_provider.list_items(wallet_id) + except wallet_backend_errors.WalletNotFoundError as error: + logger.warning( + "Skipping portfolio history collection for wallet %s: cannot list accounts (%s)", + wallet_id, + error, + ) + return [] + + if account_ids is not None: + account_ids_set = set(account_ids) + accounts = [account for account in accounts if account.id in account_ids_set] + + contexts = [ + context + for account in accounts + if (context := _build_context_for_account(wallet_id, account)) + ] + + if not contexts: + logger.info("No exchange accounts found for wallet %s", wallet_id) + return [] + + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states_for_trade_symbols( + wallet_id, + ) + for context in contexts: + context.trade_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + wallet_id, + context.account, + context.exchange_config, + automation_states=wallet_automation_states.protocol_states, + flow_automation_states_by_id=wallet_automation_states.flow_states_by_id, + ) + + job = portfolio_history_job_module.PortfolioHistoryJob(wallet_id, contexts) + results = await job.run() + for result in results: + if result.error: + logger.error( + "Portfolio history collection failed for account %s: %s", + result.account_id, result.error, + ) + elif result.skipped: + logger.debug("Skipped account %s", result.account_id) + else: + logger.info( + "Collected %d trades + %d transactions for [%s %s %s] account %s " + "in %.2fs, %d trade symbols, %d fetched candles symbols", + result.trades_count, + result.transactions_count, + result.exchange_name, + "simulated" if result.is_simulated else "real", + result.trading_type or commons_constants.CONFIG_EXCHANGE_SPOT, + result.account_id, + result.duration_seconds or 0.0, + result.trade_symbols_count, + result.price_symbols_count, + ) + return results + + +def _build_context_for_account( + wallet_id: str, + account: protocol_models.Account, +) -> portfolio_history_entities.PortfolioHistoryAccountContext | None: + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return None + if not isinstance(specifics.actual_instance, protocol_models.ExchangeAccount): + return None + if account.is_simulated: + return None + + exchange_account = specifics.actual_instance + trading_type = account_state_updater._trading_type_for_account_state_check(account) + try: + exchange_config = exchange_account_resolver.get_exchange_config(wallet_id, exchange_account) + except Exception as error: + logger.exception( + error, + True, + f"Could not resolve exchange config for account {account.id}: {error}", + ) + return None + try: + authentication = account_authentication_resolver.get_exchange_authentication(wallet_id, account) + except Exception as error: + logger.exception( + error, + True, + f"Could not resolve authentication for account {account.id}: {error}", + ) + return None + auth_details = account_state_updater._encrypted_exchange_auth_details( + exchange_account, + authentication, + trading_type, + exchange_config.sandboxed, + ) + return portfolio_history_entities.PortfolioHistoryAccountContext( + account=account, + exchange_account=exchange_account, + exchange_config=exchange_config, + trading_type=trading_type, + auth_details=auth_details, + ) diff --git a/packages/node/octobot_node/scheduler/scheduler.py b/packages/node/octobot_node/scheduler/scheduler.py index e253ba9da7..05bb307d68 100644 --- a/packages/node/octobot_node/scheduler/scheduler.py +++ b/packages/node/octobot_node/scheduler/scheduler.py @@ -14,7 +14,6 @@ # You should have received a copy of the GNU General Public # License along with OctoBot. If not, see . -import contextlib import datetime import asyncio import dbos @@ -31,13 +30,13 @@ import octobot_node.enums import octobot_node.models import octobot_node.constants +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util import octobot_node.scheduler.workflows_retention as workflows_retention import octobot_node.scheduler.workflows.params as workflow_params import octobot_node.scheduler.user_actions.user_action_util as user_action_util import octobot_node.scheduler.encryption as encryption import octobot_node.scheduler.task_context as task_context -import octobot_node.protocol.automations as automations_protocol import octobot_node.protocol.util.privacy_filter as privacy_filter DEFAULT_NAME = "octobot_node" @@ -67,6 +66,8 @@ class Scheduler: AUTOMATION_WORKFLOW_QUEUE: dbos.Queue = None # type: ignore USER_ACTION_QUEUE: dbos.Queue = None # type: ignore DBOS_CLEANUP_QUEUE: dbos.Queue = None # type: ignore + GLOBAL_VIEW_QUEUE: dbos.Queue = None # type: ignore + PORTFOLIO_HISTORY_QUEUE: dbos.Queue = None # type: ignore @staticmethod def _wallet_filter_queue(queue_names: typing.Optional[list[str]]) -> octobot_node.enums.SchedulerQueues: @@ -151,6 +152,8 @@ def stop(self) -> None: Scheduler.AUTOMATION_WORKFLOW_QUEUE = None Scheduler.USER_ACTION_QUEUE = None Scheduler.DBOS_CLEANUP_QUEUE = None + Scheduler.GLOBAL_VIEW_QUEUE = None + Scheduler.PORTFOLIO_HISTORY_QUEUE = None def create_queues(self): self.AUTOMATION_WORKFLOW_QUEUE = dbos.Queue(name=octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value) @@ -160,6 +163,14 @@ def create_queues(self): # only one cleanup workflow can run at a time concurrency=1, ) + self.GLOBAL_VIEW_QUEUE = dbos.Queue( + name=octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, + concurrency=1, + ) + self.PORTFOLIO_HISTORY_QUEUE = dbos.Queue( + name=octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, + concurrency=1, + ) async def get_periodic_tasks(self, user_id: typing.Optional[str] = None) -> list[octobot_node.models.Execution]: """DBOS scheduled workflows are not easily introspectable; return empty list.""" @@ -181,7 +192,7 @@ async def get_pending_tasks(self, user_id: typing.Optional[str] = None) -> list[ for pending_workflow_status in pending_workflow_statuses: try: task = workflows_util.get_automation_input_task(pending_workflow_status) - if reader := workflows_util.get_automation_state_reader(pending_workflow_status): + if reader := automation_states_loader.get_automation_state_reader(pending_workflow_status): next_step = ", ".join([ action.get_summary() for action in reader.get_executable_actions() @@ -624,25 +635,7 @@ async def get_automation_states( user_id: typing.Optional[str], statuses: typing.Optional[list[dbos.WorkflowStatusString]] = None, ) -> list[protocol_models.AutomationState]: - workflows = await self._get_latest_workflow_for_each_automation( - user_id, statuses, load_output=True - ) - sources: list[automations_protocol.AutomationStateSource] = [] - for workflow in workflows: - workflow_output = workflows_util.parse_automation_workflow_output(workflow) - task = workflows_util.get_resolved_automation_task(workflow) - if task: - task.id = workflows_util.normalize_parent_automation_id(workflow.workflow_id) - sources.append(automations_protocol.AutomationStateSource( - task=task, - workflow_status=workflow.status, - workflow_output=workflow_output, - workflow_error=str(workflow.error) if workflow.error else None, - )) - with contextlib.ExitStack() as exit_stack: - for source in sources: - exit_stack.enter_context(task_context.encrypted_task(source.task)) - return automations_protocol.to_protocol_automations_state(sources) + return await automation_states_loader.load_protocol_automation_states(user_id, statuses) @staticmethod def _user_action_list_sort_key( diff --git a/packages/node/octobot_node/scheduler/schedules.py b/packages/node/octobot_node/scheduler/schedules.py index 6246ffafe1..485afcc1cf 100644 --- a/packages/node/octobot_node/scheduler/schedules.py +++ b/packages/node/octobot_node/scheduler/schedules.py @@ -25,6 +25,8 @@ import octobot_node.constants as constants import octobot_node.scheduler.scheduler as scheduler_module import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow +import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow +import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow import octobot_node.scheduler.workflows_retention as workflows_retention @@ -114,6 +116,20 @@ async def _classify_schedule_window_slots( } +def _get_latest_schedule_trigger_time_before( + schedule_input: dbos.ScheduleInput, + before: datetime.datetime, +) -> datetime.datetime | None: + schedule_cron = schedule_input["schedule"] + cron_timezone = schedule_input.get("cron_timezone") + tz = zoneinfo.ZoneInfo(cron_timezone) if cron_timezone else datetime.timezone.utc + if before.tzinfo is None: + before = before.replace(tzinfo=datetime.timezone.utc) + before_in_tz = before.astimezone(tz) + iterator = dbos_croniter.croniter(schedule_cron, before_in_tz, second_at_beginning=True) + return iterator.get_prev(_DATETIME_CLASS) + + def _existing_schedule_matches_configured( existing: dbos.WorkflowSchedule, schedule_input: dbos.ScheduleInput, @@ -122,11 +138,65 @@ def _existing_schedule_matches_configured( existing["schedule"] == schedule_input["schedule"] and bool(existing.get("automatic_backfill")) == schedule_input.get("automatic_backfill", False) + and bool(existing.get("catch_up_once_on_startup")) + == schedule_input.get("catch_up_once_on_startup", False) and existing.get("cron_timezone") == schedule_input.get("cron_timezone") and existing.get("queue_name") == schedule_input.get("queue_name") ) +async def _maybe_catch_up_schedule_once_on_startup( + schedule_name: str, + schedule_input: dbos.ScheduleInput, +) -> None: + if not schedule_input.get("catch_up_once_on_startup", False): + return + existing_schedule = await dbos.DBOS.get_schedule_async(schedule_name) + if existing_schedule is None: + return + + logger = _get_logger() + now = datetime.datetime.now(datetime.timezone.utc) + trigger_time = _get_latest_schedule_trigger_time_before(schedule_input, now) + if trigger_time is None: + return + + workflow_id = build_scheduled_workflow_id(schedule_name, trigger_time) + workflow_status = await dbos.DBOS.get_workflow_status_async(workflow_id) + if workflow_status is not None: + if workflows_retention.is_terminal_workflow(workflow_status): + logger.info( + "Startup catch-up not needed for schedule %s: latest slot %s already %s", + schedule_name, + workflow_id, + _workflow_status_string(workflow_status), + ) + else: + logger.info( + "Startup catch-up skipped for schedule %s: latest slot %s already %s", + schedule_name, + workflow_id, + _workflow_status_string(workflow_status), + ) + return + + logger.info( + "Startup catch-up for schedule %s: enqueuing missed latest slot %s", + schedule_name, + workflow_id, + ) + # backfill_schedule uses get_next from start; start one second before the slot + # so the target trigger_time is the first cron fire in [start, end). + backfill_start = trigger_time - datetime.timedelta(seconds=1) + backfill_end = trigger_time + datetime.timedelta(seconds=1) + await asyncio.to_thread( + dbos.DBOS.backfill_schedule, + schedule_name, + backfill_start, + backfill_end, + ) + + async def _maybe_backfill_schedule_on_startup( schedule_name: str, schedule_input: dbos.ScheduleInput, @@ -266,15 +336,32 @@ async def _ensure_schedule( ) await scheduler.INSTANCE.apply_schedules_async([schedule_input]) await _maybe_backfill_schedule_on_startup(schedule_name, schedule_input) + await _maybe_catch_up_schedule_once_on_startup(schedule_name, schedule_input) async def register_schedules(scheduler: scheduler_module.Scheduler) -> None: schedule_inputs: list[dbos.ScheduleInput] = [ dbos_cleanup_workflow.get_schedule_input(), + global_view_workflow.get_schedule_input(), + portfolio_history_workflow.get_schedule_input(), ] for schedule_input in schedule_inputs: await _ensure_schedule(scheduler, schedule_input) +async def update_cleanup_schedule_cron( + scheduler: scheduler_module.Scheduler, + cron: str, +) -> dict[str, typing.Any]: + existing_schedule = await dbos.DBOS.get_schedule_async(dbos_cleanup_workflow.SCHEDULE_NAME) + if existing_schedule is not None and existing_schedule["schedule"] == cron: + return {"changed": False, "cron": cron} + schedule_input = dbos_cleanup_workflow.get_schedule_input(cron=cron) + logger = _get_logger() + logger.info("Updating cleanup schedule cron to %s", cron) + await scheduler.INSTANCE.apply_schedules_async([schedule_input]) + return {"changed": True, "cron": cron} + + def _get_logger() -> logging.BotLogger: return logging.get_logger("schedules") diff --git a/packages/node/octobot_node/scheduler/task_context.py b/packages/node/octobot_node/scheduler/task_context.py index f0ec89de74..10d1e99a5b 100644 --- a/packages/node/octobot_node/scheduler/task_context.py +++ b/packages/node/octobot_node/scheduler/task_context.py @@ -21,7 +21,7 @@ import octobot_node.config import octobot_node.models import octobot_node.scheduler.encryption as encryption -import octobot_node.scheduler.octobot_flow_client +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client logger = logging.getLogger(__name__) @@ -29,7 +29,7 @@ @contextlib.contextmanager def encrypted_task( task: octobot_node.models.Task, - to_update_result: typing.Optional["octobot_node.scheduler.octobot_flow_client.OctoBotActionsJobResult"] = None + to_update_result: typing.Optional["octobot_flow_client.OctoBotActionsJobResult"] = None ): """ Context manager for automatically decrypting task content. @@ -58,7 +58,7 @@ def encrypted_task( # ensure maybe_encrypted_next_actions_description is encrypted if needed if isinstance( to_update_result.next_actions_description, - octobot_node.scheduler.octobot_flow_client.OctoBotActionsJobDescription + octobot_flow_client.OctoBotActionsJobDescription ): maybe_encrypted_next_actions_description, next_actions_description_encryption_metadata = encryption.get_next_encrypted_if_needed_content_and_metadata( to_update_result.next_actions_description.to_dict(include_default_values=False) diff --git a/packages/node/octobot_node/scheduler/tasks.py b/packages/node/octobot_node/scheduler/tasks.py index 188585636c..09ff564fa8 100644 --- a/packages/node/octobot_node/scheduler/tasks.py +++ b/packages/node/octobot_node/scheduler/tasks.py @@ -18,10 +18,12 @@ import typing import octobot_flow.entities +import octobot_commons.timestamp_util as timestamp_util import octobot_node.constants as node_constants import octobot_node.enums import octobot_node.errors as node_errors import octobot_node.models +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util import octobot_node.scheduler.workflows.params as params import octobot_protocol.models as protocol_models @@ -43,6 +45,26 @@ async def trigger_user_action_workflow( ) return handle.workflow_id + +async def trigger_portfolio_history_collection( + collection_params: typing.Optional["params.PortfolioHistoryCollectionParams"] = None, +) -> str: + import octobot_node.scheduler # avoid circular import + if not octobot_node.scheduler.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow + scheduled_time = timestamp_util.utc_now_datetime() + workflow_context = None + if collection_params is not None: + workflow_context = collection_params.to_dict(include_default_values=False) + handle = await octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE.enqueue_async( + portfolio_history_workflow.PortfolioHistoryWorkflow.portfolio_history_collection, + scheduled_time, + workflow_context, + ) + return handle.workflow_id + + async def trigger_task( task: octobot_node.models.Task, target_workflow_id: typing.Optional[str] = None ) -> str: @@ -71,7 +93,7 @@ async def trigger_task( async def send_actions_to_automation(actions: list[dict], automation_id: str): - workflow_status = await workflows_util.get_automation_workflow_status(automation_id) + workflow_status = await automation_states_loader.get_automation_workflow_status(automation_id) await send_actions_to_automation_workflow(actions, workflow_status.workflow_id) @@ -89,7 +111,7 @@ async def trigger_copier_automation(automation_id: str, trading_signal: octobot_ async def send_forced_trigger_to_automation(automation_id: str): - workflow_status = await workflows_util.get_automation_workflow_status(automation_id) + workflow_status = await automation_states_loader.get_automation_workflow_status(automation_id) await send_forced_trigger_to_automation_workflow(workflow_status.workflow_id) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py b/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py index ad5f778b28..8426a61a4f 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_action_post_actions.py @@ -3,8 +3,12 @@ import typing import octobot_commons.dataclasses.minimizable_dataclass import octobot_node.models as models +import octobot_node.scheduler.workflows.params.portfolio_history_workflow_params as portfolio_history_workflow_params_module @dataclasses.dataclass class UserActionPostActions(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): to_create_automation_task: typing.Optional[models.Task] = None + portfolio_history_collection_params: typing.Optional[ + portfolio_history_workflow_params_module.PortfolioHistoryCollectionParams + ] = None diff --git a/packages/node/octobot_node/scheduler/user_actions/user_action_util.py b/packages/node/octobot_node/scheduler/user_actions/user_action_util.py index cdeaa09df2..1f3b855f31 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_action_util.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_action_util.py @@ -53,6 +53,8 @@ def resolve_user_action_result_type( | protocol_models.UserActionType.ACCOUNT_EDIT | protocol_models.UserActionType.ACCOUNT_DELETE | protocol_models.UserActionType.ACCOUNTS_REFRESH + | protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA + | protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA ): return protocol_models.UserActionResultType.ACCOUNT case ( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py index a3b01750ec..6626142870 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/__init__.py @@ -38,6 +38,8 @@ import octobot_node.scheduler.user_actions.user_actions_executor.strategy.strategy_user_action_executor as user_actions_executor_strategy_base import octobot_node.scheduler.user_actions.user_actions_executor.automation.restart_automation as user_actions_executor_restart_automation import octobot_node.scheduler.user_actions.user_actions_executor.automation.stop_automation as user_actions_executor_stop_automation +import octobot_node.scheduler.user_actions.user_actions_executor.account.reset_account_trading_data as user_actions_executor_reset_account_trading_data +import octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data as user_actions_executor_update_historical_exchanges_data from octobot_node.scheduler.user_actions.user_action_post_actions import UserActionPostActions from octobot_node.scheduler.user_actions.user_actions_executor.user_action_executor_factory import ( @@ -68,6 +70,8 @@ CreateAccountAuthActionExecutor = user_actions_executor_create_account_auth.CreateAccountAuthActionExecutor EditAccountAuthActionExecutor = user_actions_executor_edit_account_auth.EditAccountAuthActionExecutor DeleteAccountAuthActionExecutor = user_actions_executor_delete_account_auth.DeleteAccountAuthActionExecutor +UpdateHistoricalExchangesDataActionExecutor = user_actions_executor_update_historical_exchanges_data.UpdateHistoricalExchangesDataActionExecutor +ResetAccountTradingDataActionExecutor = user_actions_executor_reset_account_trading_data.ResetAccountTradingDataActionExecutor __all__ = [ "UserActionExecutor", @@ -94,6 +98,8 @@ "CreateAccountAuthActionExecutor", "EditAccountAuthActionExecutor", "DeleteAccountAuthActionExecutor", + "UpdateHistoricalExchangesDataActionExecutor", + "ResetAccountTradingDataActionExecutor", "user_action_executor_factory", "UserActionPostActions", ] diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py index 833aa914bd..3245162126 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/create_account.py @@ -14,6 +14,9 @@ # You should have received a copy of the GNU General Public License along # with OctoBot. If not, see . +import datetime + +import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_providers as collection_providers import octobot_protocol.models as protocol_models @@ -52,4 +55,14 @@ async def _do_execute( self._user_id, checked_account, ) + collection_providers.AccountTradingProvider.instance().save_state( + self._user_id, + checked_account.id, + protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ), + ) self._mark_user_action_completed(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/reset_account_trading_data.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/reset_account_trading_data.py new file mode 100644 index 0000000000..1b26370001 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/account/reset_account_trading_data.py @@ -0,0 +1,56 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.errors as node_errors +import octobot_node.protocol.accounts_trading as accounts_trading_module +import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor + + +def _get_reset_account_trading_data_payload( + user_action: protocol_models.UserAction, +) -> protocol_models.ResetAccountTradingDataConfiguration: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "UserAction.configuration must wrap a concrete reset-account-trading-data configuration." + ) + payload = wrapper.actual_instance + if not isinstance(payload, protocol_models.ResetAccountTradingDataConfiguration): + raise node_errors.InvalidUserActionPayloadError( + "ResetAccountTradingDataActionExecutor expected " + f"ResetAccountTradingDataConfiguration, got {type(payload).__name__}" + ) + return payload + + +class ResetAccountTradingDataActionExecutor(account_user_action_executor.AccountUserActionExecutor): + async def _do_execute( + self, + user_action: protocol_models.UserAction, + ) -> None: + payload = _get_reset_account_trading_data_payload(user_action) + if not payload.account_ids: + raise node_errors.InvalidUserActionPayloadError( + "ResetAccountTradingDataConfiguration.account_ids must contain at least one account id." + ) + account_provider = collection_providers.AccountProvider.instance() + for account_id in payload.account_ids: + account_provider.get_item(self._user_id, account_id) + accounts_trading_module.reset_account_trading_data(self._user_id, account_id) + self._mark_user_action_completed(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py index 9626fffe2f..6c56455a60 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/create_automation.py @@ -68,7 +68,7 @@ def _execute_actions_task_content_json( ) -> str: """ Build Task.content for EXECUTE_ACTIONS: JSON envelope with automation state and actions DAG, - matching octobot_node.scheduler.workflows_util.get_automation_dict / functional workflow tests. + matching octobot_node.scheduler.automations.automation_states_loader.get_automation_dict / functional workflow tests. """ automation_state = flow_entities.AutomationState( automation=flow_entities.AutomationDetails( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py index 6847fcfab5..a8385c8b57 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/automation/restart_automation.py @@ -26,6 +26,7 @@ import octobot_node.scheduler.task_context as task_context import octobot_node.scheduler.user_actions.user_actions_executor.automation.automation_user_action_executor as automation_user_action_executor import octobot_node.scheduler.user_actions.user_actions_executor.util.action_details_factory as action_details_factory +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util @@ -126,8 +127,8 @@ async def _build_restart_task(self, parent_automation_id: str) -> models.Task: content_metadata=workflow_output.state_metadata, ) ): - automation_state_dict = workflows_util.get_automation_dict(workflow_output.state)[ - workflows_util.STATE_KEY + automation_state_dict = automation_states_loader.get_automation_dict(workflow_output.state)[ + automation_states_loader.STATE_KEY ] automation_state = flow_entities.AutomationState.from_dict(automation_state_dict) prepared_state = prepare_automation_state_for_restart(automation_state) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py new file mode 100644 index 0000000000..5e8e528411 --- /dev/null +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/historical_data/update_historical_exchanges_data.py @@ -0,0 +1,63 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot Node is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 3.0 of the License, or (at +# your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with OctoBot. If not, see . + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.errors as node_errors +import octobot_node.scheduler as scheduler_module +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_actions_executor.account.account_user_action_executor as account_user_action_executor + + +def _get_update_historical_exchanges_data_payload( + user_action: protocol_models.UserAction, +) -> protocol_models.UpdateHistoricalExchangesDataConfiguration: + wrapper = user_action.configuration + if wrapper is None or wrapper.actual_instance is None: + raise node_errors.InvalidUserActionPayloadError( + "UserAction.configuration must wrap a concrete update-historical-exchanges-data configuration." + ) + payload = wrapper.actual_instance + if not isinstance(payload, protocol_models.UpdateHistoricalExchangesDataConfiguration): + raise node_errors.InvalidUserActionPayloadError( + "UpdateHistoricalExchangesDataActionExecutor expected " + f"UpdateHistoricalExchangesDataConfiguration, got {type(payload).__name__}" + ) + return payload + + +class UpdateHistoricalExchangesDataActionExecutor(account_user_action_executor.AccountUserActionExecutor): + async def _do_execute( + self, + user_action: protocol_models.UserAction, + ) -> None: + payload = _get_update_historical_exchanges_data_payload(user_action) + if not scheduler_module.is_initialized(): + raise RuntimeError("Scheduler is not initialized") + account_provider = collection_providers.AccountProvider.instance() + # Client-supplied account_ids are scoped to this wallet only: get_item raises + # if an id does not belong to self._user_id (same check as refresh_accounts). + if payload.account_ids: + for account_id in payload.account_ids: + account_provider.get_item(self._user_id, account_id) + self.post_actions.portfolio_history_collection_params = ( + workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=[self._user_id], + account_ids=payload.account_ids, + ) + ) + self._mark_user_action_completed(user_action) diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py index 29a32a3ff1..4c4b1d1b22 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/user_action_executor_factory.py @@ -73,6 +73,10 @@ def user_action_executor_factory( return user_actions_executor_package.EditAccountAuthActionExecutor case protocol_models.DeleteAccountAuthConfiguration: return user_actions_executor_package.DeleteAccountAuthActionExecutor + case protocol_models.UpdateHistoricalExchangesDataConfiguration: + return user_actions_executor_package.UpdateHistoricalExchangesDataActionExecutor + case protocol_models.ResetAccountTradingDataConfiguration: + return user_actions_executor_package.ResetAccountTradingDataActionExecutor case _: raise node_errors.UnsupportedUserActionConfigurationTypeError( f"Unknown user action configuration type: {type(actual).__name__}" diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py index 31e4490106..403c416b54 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_authentication_resolver.py @@ -22,7 +22,7 @@ def get_exchange_authentication( - user_id: str, + wallet_id: str, account: protocol_models.Account, ) -> protocol_models.AccountAuthentication | None: if account.is_simulated: @@ -43,12 +43,12 @@ def get_exchange_authentication( ) try: authentication = collection_providers.AccountAuthenticationProvider.instance().get_item( - user_id, + wallet_id, authentication_id, ) except collection_errors.ItemNotFoundError as err: raise node_errors.AccountAuthenticationNotFoundError( - f"Authentication {authentication_id!r} for account {account.id!r} not found for address {user_id!r}: {err}" + f"Authentication {authentication_id!r} for account {account.id!r} not found for address {wallet_id!r}: {err}" ) from err if not authentication.api_key or not authentication.api_secret: raise node_errors.AccountAuthenticationNotFoundError( diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py index 2f4fb8723d..66ecb6929e 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/account_state_updater.py @@ -9,47 +9,18 @@ import octobot_trading.errors as trading_errors import octobot_trading.exchanges as trading_exchanges import octobot_trading.exchanges.util.exchange_data as exchange_data_module +import octobot_trading.util.protocol_trading_mapping as protocol_trading_mapping import octobot_node.errors as node_errors import octobot_node.scheduler.user_actions.user_actions_executor.util.account_authentication_resolver as account_authentication_resolver import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver -_TRADING_TYPE_TO_EXCHANGE_TYPE: dict[protocol_models.TradingType, trading_enums.ExchangeTypes] = { - protocol_models.TradingType.SPOT: trading_enums.ExchangeTypes.SPOT, - protocol_models.TradingType.FUTURES: trading_enums.ExchangeTypes.FUTURE, - protocol_models.TradingType.OPTIONS: trading_enums.ExchangeTypes.OPTION, - protocol_models.TradingType.MARGIN: trading_enums.ExchangeTypes.MARGIN, -} - -_API_KEY_RIGHT_TO_ACCOUNT_PERMISSION: dict[ - trading_enums.APIKeyRights, - protocol_models.AccountPermission, -] = { - trading_enums.APIKeyRights.READING: protocol_models.AccountPermission.READ, - trading_enums.APIKeyRights.SPOT_TRADING: protocol_models.AccountPermission.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING: protocol_models.AccountPermission.FUTURES_TRADING, - trading_enums.APIKeyRights.WITHDRAWALS: protocol_models.AccountPermission.WITHDRAW, -} - -_OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED: list[trading_enums.APIKeyRights] = [ - trading_enums.APIKeyRights.READING, - trading_enums.APIKeyRights.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING, -] - - -def _exchange_type_from_trading_type( - trading_type: protocol_models.TradingType, -) -> str: - return _TRADING_TYPE_TO_EXCHANGE_TYPE[trading_type].value - - async def _fetch_api_key_rights(exchange) -> list[trading_enums.APIKeyRights]: try: return await exchange.get_permissions() except trading_errors.NotSupported: - return list(_OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED) + return list(protocol_trading_mapping.OPTIMISTIC_API_KEY_RIGHTS_WHEN_PERMISSIONS_UNSUPPORTED) def _account_permissions_from_api_key_rights( @@ -58,7 +29,9 @@ def _account_permissions_from_api_key_rights( return [ account_permission for api_key_right in api_key_rights - if (account_permission := _API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right)) is not None + if ( + account_permission := protocol_trading_mapping.API_KEY_RIGHT_TO_ACCOUNT_PERMISSION.get(api_key_right) + ) is not None ] @@ -115,7 +88,7 @@ def _encrypted_exchange_auth_details( ) -> exchange_data_module.ExchangeAuthDetails: if authentication is None: return exchange_data_module.ExchangeAuthDetails( - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=sandboxed, exchange_account_id=exchange_account.remote_account_id, ) @@ -127,7 +100,7 @@ def _encrypted_exchange_auth_details( api_key=fields_utils.encrypt(authentication.api_key).decode(), api_secret=fields_utils.encrypt(authentication.api_secret).decode(), api_password=api_password, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=sandboxed, exchange_account_id=exchange_account.remote_account_id, ) @@ -154,7 +127,7 @@ def _trading_type_for_account_state_check( async def _check_exchange_account_state( exchange_account: protocol_models.ExchangeAccount, account: protocol_models.Account, - user_id: str, + wallet_id: str, ) -> tuple[protocol_models.AccountState, list[protocol_models.DetailedAssetsForTradingType] | None]: if account.is_simulated: return ( @@ -165,11 +138,11 @@ async def _check_exchange_account_state( None, # not fetching assets for simulated accounts ) authentication = account_authentication_resolver.get_exchange_authentication( - user_id, + wallet_id, account, ) exchange_config = exchange_account_resolver.get_exchange_config( - user_id, + wallet_id, exchange_account, ) trading_type = _trading_type_for_account_state_check(account) @@ -177,7 +150,7 @@ async def _check_exchange_account_state( exchanges=[ commons_profile_data.ExchangeData( internal_name=exchange_config.exchange, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, exchange_account_id=exchange_account.remote_account_id, sandboxed=exchange_config.sandboxed, ) @@ -186,7 +159,7 @@ async def _check_exchange_account_state( profile_data.trader.enabled = True exchange_data = exchange_data_module.exchange_data_factory( exchange_internal_name=exchange_config.exchange, - exchange_type=_exchange_type_from_trading_type(trading_type), + exchange_type=protocol_trading_mapping.TRADING_TYPE_TO_EXCHANGE_TYPE.get(trading_type).value, sandboxed=exchange_config.sandboxed, auth_details=_encrypted_exchange_auth_details( exchange_account, diff --git a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py index a567a10eb0..ebadb731e0 100644 --- a/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py +++ b/packages/node/octobot_node/scheduler/user_actions/user_actions_executor/util/exchange_account_resolver.py @@ -43,18 +43,18 @@ def get_primary_exchange_config_id( def get_exchange_config( - user_id: str, + wallet_id: str, exchange_account: protocol_models.ExchangeAccount, ) -> protocol_models.ExchangeConfig: exchange_config_id = get_primary_exchange_config_id(exchange_account) try: return collection_providers.AccountProvider.instance().get_exchange_config( - user_id, + wallet_id, exchange_config_id, ) except collection_errors.ItemNotFoundError as err: raise node_errors.InvalidUserActionPayloadError( - f"Exchange config {exchange_config_id!r} not found for address {user_id!r}: {err}" + f"Exchange config {exchange_config_id!r} not found for address {wallet_id!r}: {err}" ) from err diff --git a/packages/node/octobot_node/scheduler/workflows/__init__.py b/packages/node/octobot_node/scheduler/workflows/__init__.py index 624a4184a5..496ac1e14f 100644 --- a/packages/node/octobot_node/scheduler/workflows/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/__init__.py @@ -19,3 +19,5 @@ def register_workflows() -> None: import octobot_node.scheduler.workflows.automation_workflow import octobot_node.scheduler.workflows.user_action_workflow import octobot_node.scheduler.workflows.dbos_cleanup_workflow + import octobot_node.scheduler.workflows.global_view_workflow + import octobot_node.scheduler.workflows.portfolio_history_workflow diff --git a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py index a9d38e5a55..476bf548f5 100644 --- a/packages/node/octobot_node/scheduler/workflows/automation_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/automation_workflow.py @@ -20,23 +20,25 @@ import octobot_commons.logging -import octobot.community.wallet_backend.errors as wallet_backend_errors import octobot_trading.errors +import octobot_copy.errors as copy_errors + import octobot_flow.entities import octobot_flow.enums import octobot_flow.errors +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module import octobot_flow.repositories.community.community_repository as community_repository import octobot_node.enums import octobot_node.models -import octobot_node.scheduler.octobot_flow_client as octobot_flow_client +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client import octobot_node.scheduler.task_context import octobot_node.constants as constants import octobot_node.scheduler.workflows.params as params +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader import octobot_node.scheduler.workflows_util as workflows_util import octobot_node.errors as errors -import octobot_node.protocol.accounts_trading as accounts_trading_protocol from octobot_node.scheduler import SCHEDULER # avoid circular import @@ -85,7 +87,7 @@ async def execute_automation(inputs: dict) -> typing.Optional[str]: ) if not continue_workflow: AutomationWorkflow.get_logger(parsed_inputs).info( - f"Stopped workflow (remaining steps: {iteration_result.progress_status.remaining_steps})" + f"Automation stopped (remaining steps: {iteration_result.progress_status.remaining_steps})" ) final_state = iteration_result.next_iteration_description final_state_metadata = iteration_result.next_iteration_description_metadata @@ -117,6 +119,7 @@ def _should_retry(error: BaseException) -> bool: # workflow stopping errors errors.WorkflowError, octobot_flow.errors.ConfigurationError, + copy_errors.OutdatedReferenceAccountError, )) @staticmethod @@ -144,7 +147,6 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) executed_step: str = "no action executed" execution_error = next_step = next_step_at = execution_error_message = None postponed_iteration = False - has_next_actions_override: typing.Optional[bool] = None next_iteration_description_override: typing.Optional[str] = None next_iteration_description_metadata_override: typing.Optional[str] = None result = octobot_flow_client.OctoBotActionsJobResult() @@ -176,11 +178,20 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) err, True, f"Retriable error while running automation job: {err}" ) raise - except octobot_flow.errors.PendingPriorityActionsSkippedError as err: + except ( + octobot_flow.errors.PendingPriorityActionsSkippedError, + copy_errors.OutdatedReferenceAccountError, + ) as err: # don't retry, just skip the iteration - AutomationWorkflow.get_logger(parsed_inputs).error( - f"Pending priority actions were skipped: {err}" - ) + workflow_logger = AutomationWorkflow.get_logger(parsed_inputs) + if isinstance(err, copy_errors.OutdatedReferenceAccountError): + workflow_logger.info( + f"Outdated reference account, skipping copy iteration: {err}" + ) + else: + workflow_logger.error( + f"Pending priority actions were skipped: {err}" + ) if action_job is None: # should never happen, but just in case raise @@ -188,7 +199,6 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) action_job.description.state ) postponed_iteration = True - has_next_actions_override = True next_iteration_description_override = parsed_inputs.task.content next_iteration_description_metadata_override = parsed_inputs.task.content_metadata except ( @@ -209,8 +219,7 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) execution_error = execution_error_status.value execution_error_message = str(err) postponed_iteration = True - has_next_actions_override = True - next_iteration_description_override = workflows_util.patch_task_content_degraded_state( + next_iteration_description_override = automation_states_loader.patch_task_content_degraded_state( parsed_inputs.task.content, execution_error, execution_error_message, @@ -255,10 +264,15 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) AutomationWorkflow.get_logger(parsed_inputs).info( f"Iteration completed, executed step: '{executed_step}', {next_actions_str}" ) - AutomationWorkflow._persist_account_trading_from_iteration_result( + account_state_persistence_module.persist_account_trading_from_iteration_state( parsed_inputs.task.user_id, - result, + result.next_actions_description.state if result.next_actions_description else None, ) + if result.next_actions_description is not None: + account_state_persistence_module.trim_live_trades_in_iteration_state( + result.next_actions_description.state, + constants.AUTOMATION_LIVE_STATE_MAX_TRADES, + ) else: retry_delay_seconds = max(0.0, (next_step_at or time.time()) - time.time()) AutomationWorkflow.get_logger(parsed_inputs).info( @@ -289,7 +303,7 @@ async def execute_iteration(inputs: dict, actions_update: typing.Optional[dict]) else result.next_actions_description_encryption_metadata ), has_next_actions=( - has_next_actions_override if postponed_iteration else result.has_next_actions + True if postponed_iteration else result.has_next_actions ), ).to_dict(include_default_values=False) @@ -304,42 +318,6 @@ async def _wait_and_trigger_on_actions_update( return recv_payload return None - @staticmethod - def _persist_account_trading_from_iteration_result( - user_id: typing.Optional[str], - job_result: octobot_flow_client.OctoBotActionsJobResult, - ) -> None: - # Temporary: persist trading snapshot locally until the global view system owns this sync. - if user_id is None or job_result.next_actions_description is None: - return - automation_state = octobot_flow.entities.AutomationState.from_dict( - job_result.next_actions_description.state - ) - exchange_account_elements = automation_state.automation.exchange_account_elements - exchange_account_details = automation_state.exchange_account_details - if exchange_account_elements is None or exchange_account_details is None: - return - exchange_account_id = exchange_account_details.exchange_details.exchange_account_id - if not exchange_account_id: - return - try: - accounts_trading_protocol.update_account_trading( - user_id, - exchange_account_id, - list(exchange_account_elements.orders.open_orders), - list(exchange_account_elements.trades), - [ - position_details.position - for position_details in exchange_account_elements.positions - ], - ) - except wallet_backend_errors.WalletNotFoundError: - # Trading collections are wallet-scoped; skip until the wallet is registered locally. - octobot_commons.logging.get_logger(AutomationWorkflow.__name__).warning( - "Skipping account trading persistence for wallet %s: wallet not registered", - user_id, - ) - @staticmethod def _parse_actions_update_envelope( actions_update: typing.Optional[dict], @@ -355,6 +333,11 @@ def _parse_actions_update_envelope( return [], [] return [], [] + @staticmethod + def _is_forced_trigger_update(actions_update: dict) -> bool: + envelope = params.AutomationWorkflowActionUpdate.from_dict(actions_update) + return envelope.actions_type == octobot_node.enums.AutomationWorkflowActionTypes.FORCED_TRIGGER.value + @staticmethod def _log_iteration_execution_intent( parsed_inputs: params.AutomationWorkflowInputs, @@ -374,7 +357,10 @@ async def _process_pending_priority_actions_and_reschedule( parsed_inputs: params.AutomationWorkflowInputs, previous_iteration_result: params.AutomationWorkflowIterationResult ) -> tuple[bool, params.AutomationWorkflowIterationResult]: - if not previous_iteration_result.has_next_actions: + if ( + not previous_iteration_result.has_next_actions + and not previous_iteration_result.progress_status.postponed_iteration + ): return False, previous_iteration_result # In case new priority actions were sent, execute them now. # Any action sent to this workflow will be lost if not processed by it. @@ -382,6 +368,11 @@ async def _process_pending_priority_actions_and_reschedule( while new_actions_update := await AutomationWorkflow._wait_and_trigger_on_actions_update( parsed_inputs, 0 ): + if AutomationWorkflow._is_forced_trigger_update(new_actions_update): + AutomationWorkflow.get_logger(parsed_inputs).info( + "Ignoring forced_trigger received after iteration completed; skipping duplicate iteration." + ) + continue extra_iteration_inputs = AutomationWorkflow._create_next_iteration_inputs( parsed_inputs, latest_iteration_result.next_iteration_description, 0, latest_iteration_result.next_iteration_description_metadata, @@ -469,10 +460,12 @@ def _should_continue_workflow( progress_status: params.ProgressStatus, stop_on_error: bool ) -> bool: + if progress_status.postponed_iteration: + return True if progress_status.error: # failed iteration, return global progress where it stopped and exit workflow AutomationWorkflow.get_logger(parsed_inputs).error( - f"Failed iteration: stopping workflow, error: {progress_status.error}. " + f"Automation stopped: unrecoverable iteration error: {progress_status.error}. " f"Iteration's last step: {progress_status.latest_step}" ) return stop_on_error diff --git a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py index e2b0a9ecd2..7cd212926b 100644 --- a/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/dbos_cleanup_workflow.py @@ -39,14 +39,12 @@ async def dbos_cleanup( ) -> dict[str, typing.Any]: return await DbosCleanupWorkflow._cleanup_outdated_automation_executions( scheduled_time, - context, ) @staticmethod @SCHEDULER.INSTANCE.step(name="cleanup_outdated_automation_executions") async def _cleanup_outdated_automation_executions( scheduled_time: datetime.datetime, - context: typing.Any, ) -> dict[str, typing.Any]: logger = logging.get_logger(DbosCleanupWorkflow.__name__) if workflows_retention.should_skip_retention_cleanup_on_this_node(): @@ -61,14 +59,15 @@ async def _cleanup_outdated_automation_executions( scheduled_time.isoformat(), ) return dict(workflows_retention.EMPTY_CLEANUP_SUMMARY) - return await workflows_retention.cleanup_outdated_automation_executions(SCHEDULER) + cleanup_summary = await workflows_retention.cleanup_outdated_automation_executions(SCHEDULER) + return await workflows_retention.finalize_dbos_cleanup_run(SCHEDULER, cleanup_summary) -def get_schedule_input() -> dbos.ScheduleInput: +def get_schedule_input(cron: str | None = None) -> dbos.ScheduleInput: return { "schedule_name": SCHEDULE_NAME, "workflow_fn": DbosCleanupWorkflow.dbos_cleanup, - "schedule": SCHEDULE_CRON, + "schedule": cron or SCHEDULE_CRON, "context": None, "automatic_backfill": True, "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, diff --git a/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py new file mode 100644 index 0000000000..e8cead645c --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/global_view_workflow.py @@ -0,0 +1,204 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import asyncio +import datetime +import typing + +import dbos +import octobot_commons.logging as octobot_commons_logging +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_flow.entities +import octobot_node.enums +import octobot_node.errors as node_errors +import octobot_node.scheduler.global_view.global_view_executor as global_view_executor_module +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module +import octobot_node.scheduler.user_actions.user_actions_executor.util.exchange_account_resolver as exchange_account_resolver +import octobot_node.scheduler.workflows_retention as workflows_retention + +from octobot_node.scheduler import SCHEDULER # avoid circular import + +WORKFLOW_NAME = "global_view_refresh" +SCHEDULE_NAME = "global_view_refresh_every_5m" +SCHEDULE_CRON = "*/5 * * * *" # every 5 minutes + + +def _exchange_label_for_account(user_id: str, account: protocol_models.Account) -> str: + specifics = account.specifics + if specifics is None or specifics.actual_instance is None: + return "n/a" + exchange_account = specifics.actual_instance + if not isinstance(exchange_account, protocol_models.ExchangeAccount): + return "n/a" + try: + return exchange_account_resolver.get_exchange_config(user_id, exchange_account).exchange + except (node_errors.InvalidUserActionPayloadError, node_errors.AmbiguousExchangeConfigError): + return "n/a" + + +def _portfolio_summary_fields( + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, +) -> tuple[str, str]: + # Portfolio history is now computed on-the-fly; summary fields are no longer available here. + return "n/a", "n/a" + + +def _log_successful_account_refresh( + user_id: str, + account: protocol_models.Account, + refresh_result: octobot_flow.entities.GlobalViewAccountRefreshResult, +) -> None: + valuation_unit, portfolio_total = _portfolio_summary_fields(refresh_result) + open_orders_count = len(refresh_result.open_orders or []) + changed_orders_count = len(refresh_result.changed_order_ids) + octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__).info( + "Account global view refresh succeeded: account=%s wallet=%s name=%s exchange=%s " + "simulated=%s open_orders=%s changed_orders=%s automations_triggered=%s " + "valuation_unit=%s portfolio_total=%s", + account.id, + user_id, + account.name, + _exchange_label_for_account(user_id, account), + account.is_simulated, + open_orders_count, + changed_orders_count, + bool(refresh_result.changed_order_ids), + valuation_unit, + portfolio_total, + ) + + +@SCHEDULER.INSTANCE.dbos_class() +class GlobalViewRefreshWorkflow: + @staticmethod + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + async def global_view_refresh( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + return await GlobalViewRefreshWorkflow._run_global_view_refresh( + scheduled_time, + ) + + @staticmethod + @SCHEDULER.INSTANCE.step( + name="run_global_view_refresh", + retries_allowed=False, + ) + async def _run_global_view_refresh( + scheduled_time: datetime.datetime, + ) -> dict[str, typing.Any]: + logger = octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__) + logger.info("global_view_refresh started (scheduled_time=%s)", scheduled_time.isoformat()) + if workflows_retention.should_skip_retention_cleanup_on_this_node(): + logger.info("global_view_refresh stopped: consumer-only node (skipped)") + return {"refreshed_accounts": 0, "skipped": True} + wallet_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() + refreshed_accounts_count = 0 + for wallet_id in wallet_ids: + refreshed_accounts_count += await GlobalViewRefreshWorkflow._refresh_wallet_accounts( + wallet_id, + ) + result = { + "refreshed_accounts": refreshed_accounts_count, + "scheduled_time": scheduled_time.isoformat(), + } + logger.info( + "global_view_refresh completed (refreshed_accounts=%s, scheduled_time=%s)", + refreshed_accounts_count, + scheduled_time.isoformat(), + ) + return result + + @staticmethod + async def _refresh_wallet_accounts( + user_id: str, + ) -> int: + logger = octobot_commons_logging.get_logger(GlobalViewRefreshWorkflow.__name__) + account_provider = collection_providers.AccountProvider.instance() + try: + accounts = account_provider.list_items(user_id) + except wallet_backend_errors.WalletNotFoundError as error: + logger.warning( + "Skipping global view refresh for wallet %s: cannot list accounts (%s)", + user_id, + error, + ) + return 0 + if not accounts: + return 0 + refresh_results = await asyncio.gather( + *[ + GlobalViewRefreshWorkflow._refresh_single_account(user_id, account) + for account in accounts + ], + return_exceptions=True, + ) + refreshed_count = 0 + for refresh_result in refresh_results: + if isinstance(refresh_result, Exception): + logger.exception( + refresh_result, + True, + f"Account global view refresh failed: {refresh_result}", + ) + continue + if refresh_result: + refreshed_count += 1 + return refreshed_count + + @staticmethod + async def _refresh_single_account( + user_id: str, + account: protocol_models.Account, + ) -> bool: + try: + refresh_result = await global_view_executor_module.refresh_account_global_view( + user_id, + account, + ) + except Exception as error: + octobot_commons_logging.get_logger( + GlobalViewRefreshWorkflow.__name__ + ).exception( + error, + True, + f"Failed to refresh account {account.id} for wallet {user_id}: {error}", + ) + raise + if refresh_result.changed_order_ids: + await automation_trigger_module.trigger_account_automations( + user_id, + account.id, + refresh_result.changed_order_ids, + ) + _log_successful_account_refresh(user_id, account, refresh_result) + return True + + + + +def get_schedule_input() -> dbos.ScheduleInput: + return { + "schedule_name": SCHEDULE_NAME, + "workflow_fn": GlobalViewRefreshWorkflow.global_view_refresh, + "schedule": SCHEDULE_CRON, + "context": None, + "automatic_backfill": False, + "queue_name": octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value, + } diff --git a/packages/node/octobot_node/scheduler/workflows/params/__init__.py b/packages/node/octobot_node/scheduler/workflows/params/__init__.py index ed8efa3175..1c6158d7fa 100644 --- a/packages/node/octobot_node/scheduler/workflows/params/__init__.py +++ b/packages/node/octobot_node/scheduler/workflows/params/__init__.py @@ -27,12 +27,16 @@ UserActionWorkflowOutput, UserActionExecutionResult, ) +from .portfolio_history_workflow_params import ( + PortfolioHistoryCollectionParams, +) __all__ = [ "AutomationWorkflowActionUpdate", "AutomationWorkflowInputs", "AutomationWorkflowIterationResult", "AutomationWorkflowOutput", + "PortfolioHistoryCollectionParams", "ProgressStatus", "UserActionWorkflowInputs", "UserActionExecutionResult", diff --git a/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py b/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py new file mode 100644 index 0000000000..e08cbfd969 --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/params/portfolio_history_workflow_params.py @@ -0,0 +1,25 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. +import dataclasses +import typing + +import octobot_commons.dataclasses.minimizable_dataclass + + +@dataclasses.dataclass +class PortfolioHistoryCollectionParams(octobot_commons.dataclasses.minimizable_dataclass.MinimizableDataclass): + wallet_ids: typing.Optional[list[str]] = None + account_ids: typing.Optional[list[str]] = None diff --git a/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py new file mode 100644 index 0000000000..3c0e06494a --- /dev/null +++ b/packages/node/octobot_node/scheduler/workflows/portfolio_history_workflow.py @@ -0,0 +1,118 @@ +import datetime +import typing + +import dbos +import octobot_commons.logging as commons_logging +import octobot_sync.sync.collection_providers as collection_providers + +import octobot_node.scheduler.portfolio_history.portfolio_history_executor as portfolio_history_executor_module +import octobot_node.scheduler.workflows.params as workflow_params_module + +import octobot_node.enums +from octobot_node.scheduler import SCHEDULER + +WORKFLOW_NAME = "portfolio_history_collection" +SCHEDULE_NAME = "portfolio_history_daily_3am" +SCHEDULE_CRON = "0 3 * * *" # 3:00 AM every day + +logger = commons_logging.get_logger("PortfolioHistoryWorkflow") + +_EMPTY_COLLECTION_SUMMARY = { + "succeeded": 0, + "failed": 0, + "skipped": 0, +} + + +def _parse_collection_params( + context: typing.Any, +) -> workflow_params_module.PortfolioHistoryCollectionParams | None: + if context is None: + return None + if isinstance(context, workflow_params_module.PortfolioHistoryCollectionParams): + return context + if isinstance(context, dict): + return workflow_params_module.PortfolioHistoryCollectionParams.from_dict(context) + return None + + +@SCHEDULER.INSTANCE.dbos_class() +class PortfolioHistoryWorkflow: + @staticmethod + @SCHEDULER.INSTANCE.workflow(name=WORKFLOW_NAME) + async def portfolio_history_collection( + scheduled_time: datetime.datetime, + context: typing.Any, + ) -> dict[str, typing.Any]: + collection_params = _parse_collection_params(context) + return await PortfolioHistoryWorkflow._run_collection(scheduled_time, collection_params) + + @staticmethod + @SCHEDULER.INSTANCE.step(name="run_portfolio_history_collection") + async def _run_collection( + scheduled_time: datetime.datetime, + collection_params: workflow_params_module.PortfolioHistoryCollectionParams | None = None, + ) -> dict[str, typing.Any]: + """ + This is a step to avoid storing results of internal dbos select statements, + which are otherwise counted as steps and have their result stored + """ + try: + logger.info("Starting portfolio history collection at %s", scheduled_time) + if collection_params and collection_params.wallet_ids: + wallet_ids = collection_params.wallet_ids + logger.info( + "Portfolio history collection scoped to %d wallet(s), account_ids=%s", + len(wallet_ids), + collection_params.account_ids, + ) + else: + wallet_ids = collection_providers.AccountProvider.instance().list_collectable_wallet_ids() + account_ids = collection_params.account_ids if collection_params else None + total_results = [] + + for wallet_id in wallet_ids: + try: + results = await portfolio_history_executor_module.run_portfolio_history_collection( + wallet_id, + account_ids=account_ids, + ) + total_results.extend(results) + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history collection failed for wallet {wallet_id}: {error}", + ) + + succeeded = sum(1 for result in total_results if not result.skipped and not result.error) + failed = sum(1 for result in total_results if result.error) + skipped = sum(1 for result in total_results if result.skipped) + logger.info( + "Portfolio history collection complete: %d succeeded, %d failed, %d skipped", + succeeded, failed, skipped, + ) + return { + "succeeded": succeeded, + "failed": failed, + "skipped": skipped, + } + except Exception as error: + logger.exception( + error, + True, + f"Portfolio history collection failed: {error}", + ) + return dict(_EMPTY_COLLECTION_SUMMARY) + + +def get_schedule_input() -> dbos.ScheduleInput: + return { + "schedule_name": SCHEDULE_NAME, + "workflow_fn": PortfolioHistoryWorkflow.portfolio_history_collection, + "schedule": SCHEDULE_CRON, + "context": None, + "automatic_backfill": False, + "catch_up_once_on_startup": True, + "queue_name": octobot_node.enums.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value, + } diff --git a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py index 5d5d892b19..79944386f5 100644 --- a/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py +++ b/packages/node/octobot_node/scheduler/workflows/user_action_workflow.py @@ -97,3 +97,7 @@ async def after_user_action_execution(output: params.UserActionExecutionResult) output.post_actions.to_create_automation_task, target_workflow_id=output.post_actions.to_create_automation_task.id ) + if output.post_actions.portfolio_history_collection_params: + await scheduler_tasks.trigger_portfolio_history_collection( + output.post_actions.portfolio_history_collection_params, + ) diff --git a/packages/node/octobot_node/scheduler/workflows_retention.py b/packages/node/octobot_node/scheduler/workflows_retention.py index 01886d0e6f..7e8251fe86 100644 --- a/packages/node/octobot_node/scheduler/workflows_retention.py +++ b/packages/node/octobot_node/scheduler/workflows_retention.py @@ -15,6 +15,7 @@ # License along with this library. import datetime import os +import pathlib import time import typing @@ -29,14 +30,38 @@ if typing.TYPE_CHECKING: import octobot_node.scheduler.scheduler as scheduler_module +_GIBIBYTE = 1024 ** 3 +_DEFAULT_RETENTION_2_DAYS_SECONDS = 60 * 60 * 24 * 2 + AUTOMATION_EXECUTION_RETENTION_SECONDS = float( - os.getenv("AUTOMATION_EXECUTION_RETENTION_SECONDS", 60 * 60 * 24 * 2) + os.getenv("AUTOMATION_EXECUTION_RETENTION_SECONDS", _DEFAULT_RETENTION_2_DAYS_SECONDS) ) # 2 days +RETENTION_SECONDS_2_DAYS = float( + os.getenv("DBOS_RETENTION_SECONDS_2_DAYS", str(AUTOMATION_EXECUTION_RETENTION_SECONDS)) +) +RETENTION_SECONDS_1_DAY = float( + os.getenv("DBOS_RETENTION_SECONDS_1_DAY", 60 * 60 * 24) +) +RETENTION_SECONDS_6_HOURS = float( + os.getenv("DBOS_RETENTION_SECONDS_6_HOURS", 60 * 60 * 6) +) AUTOMATION_EXECUTIONS_TO_KEEP = 2 +DBOS_CLEANUP_SIZE_TIER_1_BYTES = int( + os.getenv("DBOS_CLEANUP_SIZE_TIER_1_BYTES", str(1 * _GIBIBYTE)) +) +DBOS_CLEANUP_SIZE_TIER_2_BYTES = int( + os.getenv("DBOS_CLEANUP_SIZE_TIER_2_BYTES", str(3 * _GIBIBYTE)) +) +DBOS_CLEANUP_CRON_DAILY = os.getenv("DBOS_CLEANUP_CRON_DAILY", "0 0 * * *") +DBOS_CLEANUP_CRON_12H = os.getenv("DBOS_CLEANUP_CRON_12H", "0 */12 * * *") +DBOS_CLEANUP_CRON_6H = os.getenv("DBOS_CLEANUP_CRON_6H", "0 */6 * * *") + EMPTY_CLEANUP_SUMMARY: dict[str, typing.Any] = { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -62,6 +87,21 @@ def _retention_cutoff_ms(*, retention_seconds: float, now_ms: int) -> int: return now_ms - int(retention_seconds * 1000) +def get_outdated_terminal_workflow_ids( + workflows: list[dbos.WorkflowStatus], + *, + retention_seconds: float, + now_ms: int, +) -> list[str]: + cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) + return [ + workflow_status.workflow_id + for workflow_status in workflows + if is_terminal_workflow(workflow_status) + and (workflow_status.updated_at or 0) < cutoff_ms + ] + + def get_outdated_automation_execution_deletions( workflows: list[dbos.WorkflowStatus], *, @@ -102,13 +142,11 @@ def get_outdated_dbos_cleanup_execution_workflow_ids( retention_seconds: float, now_ms: int, ) -> list[str]: - cutoff_ms = _retention_cutoff_ms(retention_seconds=retention_seconds, now_ms=now_ms) - return [ - workflow_status.workflow_id - for workflow_status in cleanup_workflows - if is_terminal_workflow(workflow_status) - and (workflow_status.updated_at or 0) < cutoff_ms - ] + return get_outdated_terminal_workflow_ids( + cleanup_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) _TERMINAL_DELETE_WORKFLOW_STATUSES = [ @@ -137,6 +175,32 @@ async def get_workflows_to_delete( return automation_workflows + user_action_workflows +def get_scheduler_database_size_bytes(dbos_instance: dbos.DBOS) -> int | None: + if octobot_node.config.settings.SCHEDULER_POSTGRES_URL: + return _get_postgres_database_size_bytes(dbos_instance) + return _get_sqlite_database_size_bytes() + + +def select_cleanup_cron_for_database_size(database_size_bytes: int | None) -> str: + if database_size_bytes is None: + return DBOS_CLEANUP_CRON_DAILY + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_1_BYTES: + return DBOS_CLEANUP_CRON_DAILY + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_2_BYTES: + return DBOS_CLEANUP_CRON_12H + return DBOS_CLEANUP_CRON_6H + + +def select_retention_seconds_for_database_size(database_size_bytes: int | None) -> float: + if database_size_bytes is None: + return RETENTION_SECONDS_2_DAYS + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_1_BYTES: + return RETENTION_SECONDS_2_DAYS + if database_size_bytes < DBOS_CLEANUP_SIZE_TIER_2_BYTES: + return RETENTION_SECONDS_1_DAY + return RETENTION_SECONDS_6_HOURS + + def vacuum_dbos_system_database(dbos_instance: dbos.DBOS) -> None: logger = _get_logger() logger.info("Vacuuming database") @@ -145,12 +209,19 @@ def vacuum_dbos_system_database(dbos_instance: dbos.DBOS) -> None: logger.info("Database vacuum completed") -async def delete_workflows_and_vacuum( +async def delete_workflows( dbos_instance: dbos.DBOS, - workflow_ids: list[str] + workflow_ids: list[str], ) -> None: _get_logger().info("Deleting %s workflows", len(workflow_ids)) await dbos_instance.delete_workflows_async(workflow_ids, delete_children=False) + + +async def delete_workflows_and_vacuum( + dbos_instance: dbos.DBOS, + workflow_ids: list[str] +) -> None: + await delete_workflows(dbos_instance, workflow_ids) vacuum_dbos_system_database(dbos_instance) @@ -161,9 +232,18 @@ async def cleanup_outdated_automation_executions( _get_logger().warning("Scheduler not initialized, skipping cleanup") return dict(EMPTY_CLEANUP_SUMMARY) now_ms = int(time.time() * 1000) - retention_seconds = AUTOMATION_EXECUTION_RETENTION_SECONDS + database_size_bytes = get_scheduler_database_size_bytes(scheduler.INSTANCE) + retention_seconds = select_retention_seconds_for_database_size(database_size_bytes) + if retention_seconds < RETENTION_SECONDS_2_DAYS: + _get_logger().info( + "Using retention %s s for database size %s bytes", + retention_seconds, + database_size_bytes, + ) import octobot_node.scheduler.workflows.automation_workflow as automation_workflow import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow automation_workflows = await scheduler.INSTANCE.list_workflows_async( name=automation_workflow.WORKFLOW_NAME, queue_name=[octobot_node.enums.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value], @@ -175,6 +255,17 @@ async def cleanup_outdated_automation_executions( load_input=False, load_output=False, ) + global_view_workflows = await scheduler.INSTANCE.list_workflows_async( + name=global_view_workflow.WORKFLOW_NAME, + queue_name=[octobot_node.enums.SchedulerQueues.GLOBAL_VIEW_QUEUE.value], + load_input=False, + load_output=False, + ) + portfolio_history_workflows = await scheduler.INSTANCE.list_workflows_async( + name=portfolio_history_workflow.WORKFLOW_NAME, + load_input=False, + load_output=False, + ) deletions_by_automation = get_outdated_automation_execution_deletions( automation_workflows, retention_seconds=retention_seconds, @@ -185,34 +276,84 @@ async def cleanup_outdated_automation_executions( retention_seconds=retention_seconds, now_ms=now_ms, ) + global_view_execution_ids = get_outdated_terminal_workflow_ids( + global_view_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + portfolio_history_execution_ids = get_outdated_terminal_workflow_ids( + portfolio_history_workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) automation_execution_ids = [ workflow_id for workflow_ids in deletions_by_automation.values() for workflow_id in workflow_ids ] - all_ids_to_delete = automation_execution_ids + cleanup_execution_ids + all_ids_to_delete = ( + automation_execution_ids + + cleanup_execution_ids + + global_view_execution_ids + + portfolio_history_execution_ids + ) summary = { "deleted_by_automation": { parent_id: len(workflow_ids) for parent_id, workflow_ids in deletions_by_automation.items() }, "deleted_cleanup_executions": len(cleanup_execution_ids), + "deleted_global_view_executions": len(global_view_execution_ids), + "deleted_portfolio_history_executions": len(portfolio_history_execution_ids), "total_deleted": len(all_ids_to_delete), + "database_size_bytes": database_size_bytes, + "retention_seconds": retention_seconds, } if all_ids_to_delete: _get_logger().info( - "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs, " + "%s global view runs, %s portfolio history runs", len(all_ids_to_delete), len(deletions_by_automation), len(cleanup_execution_ids), + len(global_view_execution_ids), + len(portfolio_history_execution_ids), ) - await delete_workflows_and_vacuum( + await delete_workflows( scheduler.INSTANCE, all_ids_to_delete, ) _get_logger().info("DBOS cleanup summary: %s", summary) return summary + +async def finalize_dbos_cleanup_run( + scheduler: "scheduler_module.Scheduler", + summary: dict[str, typing.Any], +) -> dict[str, typing.Any]: + if not scheduler.INSTANCE: + return summary + database_size_bytes = get_scheduler_database_size_bytes(scheduler.INSTANCE) + finalized_summary = dict(summary) + finalized_summary["database_size_bytes"] = database_size_bytes + total_deleted = finalized_summary.get("total_deleted", 0) + if total_deleted > 0 or ( + database_size_bytes is not None + and database_size_bytes >= DBOS_CLEANUP_SIZE_TIER_2_BYTES + ): + vacuum_dbos_system_database(scheduler.INSTANCE) + desired_cron = select_cleanup_cron_for_database_size(database_size_bytes) + finalized_summary["cleanup_schedule_cron"] = desired_cron + import octobot_node.scheduler.schedules as schedules_module + schedule_update = await schedules_module.update_cleanup_schedule_cron( + scheduler, + desired_cron, + ) + finalized_summary["cleanup_schedule_updated"] = schedule_update["changed"] + _get_logger().info("DBOS cleanup finalized summary: %s", finalized_summary) + return finalized_summary + + def _get_latest_completed_cleanup_timestamp_ms( cleanup_workflows: list[dbos.WorkflowStatus], ) -> int: @@ -244,5 +385,43 @@ async def should_skip_retention_cleanup_for_scheduled_time( return latest_timestamp_ms > scheduled_timestamp_ms +def _get_sqlite_database_size_bytes() -> int | None: + sqlite_path = pathlib.Path(octobot_node.config.settings.SCHEDULER_SQLITE_FILE) + if not sqlite_path.is_file(): + missing_file_error = FileNotFoundError(f"Scheduler sqlite file not found: {sqlite_path}") + _get_logger().exception( + missing_file_error, + True, + "Scheduler sqlite file not found: %s", + sqlite_path, + ) + return None + total_size_bytes = sqlite_path.stat().st_size + for suffix in ("-wal", "-shm"): + sidecar_path = pathlib.Path(f"{sqlite_path}{suffix}") + if sidecar_path.is_file(): + total_size_bytes += sidecar_path.stat().st_size + return total_size_bytes + + +def _get_postgres_database_size_bytes(dbos_instance: dbos.DBOS) -> int | None: + try: + with dbos_instance._sys_db.engine.begin() as connection: + database_size = connection.execute( + sqlalchemy.text("SELECT pg_database_size(current_database())"), + ).scalar() + if database_size is None: + return None + return int(database_size) + except Exception as error: + _get_logger().exception( + error, + True, + "Failed to read postgres database size: %s", + error, + ) + return None + + def _get_logger() -> logging.BotLogger: return logging.get_logger("workflows_retention") diff --git a/packages/node/octobot_node/scheduler/workflows_util.py b/packages/node/octobot_node/scheduler/workflows_util.py index 46a1730faf..0b95fd57cc 100644 --- a/packages/node/octobot_node/scheduler/workflows_util.py +++ b/packages/node/octobot_node/scheduler/workflows_util.py @@ -20,27 +20,13 @@ import octobot_commons.logging import octobot_protocol.models as protocol_models -import octobot_node.config import octobot_node.constants import octobot_node.enums as octobot_node_enums import octobot_node.models as models -import octobot_node.scheduler.task_context as task_context import octobot_node.scheduler.workflows.params as params - -try: - import octobot_flow.entities - import octobot_flow.parsers -except ImportError: - octobot_flow = None # type: ignore - octobot_commons.logging.get_logger("octobot_node.scheduler.workflows_util").warning( - "octobot_flow is not installed, workflows utilities will not be available" - ) - logger = octobot_commons.logging.get_logger("octobot_node.scheduler.workflows_util") -STATE_KEY = "state" - _USER_ACTION_TERMINAL_WORKFLOW_STATUSES = ( dbos_lib.WorkflowStatusString.SUCCESS, dbos_lib.WorkflowStatusString.ERROR, @@ -250,12 +236,6 @@ def get_latest_workflow( return get_latest_child_workflow(workflows) -def get_automation_copied_strategy_ids(workflow_status: dbos_lib.WorkflowStatus) -> list[str]: - if reader := get_automation_state_reader(workflow_status): - return reader.get_automation_copied_strategy_ids() - return [] - - def get_workflows_by_parent_id( workflows: list[dbos_lib.WorkflowStatus] ) -> dict[str, list[dbos_lib.WorkflowStatus]]: @@ -266,26 +246,6 @@ def get_workflows_by_parent_id( return by_parent -def get_automation_state_reader(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional["octobot_flow.parsers.AutomationStateReader"]: - """Get the resolved automation state for a workflow row (input or terminal output).""" - try: - import octobot_flow.entities - import octobot_flow.parsers - except ImportError: - return None - if state_dict := get_automation_state_dict(workflow_status): - return octobot_flow.parsers.AutomationStateReader( - octobot_flow.entities.AutomationState.from_dict(state_dict) - ) - return None - - -def get_automation_id(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[str]: - if state_dict := get_automation_state_dict(workflow_status): - return state_dict.get("automation", {}).get("metadata", {}).get("automation_id") - return None - - def parse_automation_workflow_output( workflow_status: dbos_lib.WorkflowStatus, ) -> typing.Optional[params.AutomationWorkflowOutput]: @@ -334,17 +294,6 @@ def get_resolved_automation_task(workflow_status: dbos_lib.WorkflowStatus) -> ty return input_task -def get_automation_state_dict(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[dict]: - resolved_task = get_resolved_automation_task(workflow_status) - if resolved_task is None: - return None - with task_context.encrypted_task(resolved_task): - try: - return get_automation_dict(resolved_task.content)[STATE_KEY] - except ValueError: - return None - - def get_automation_input_task(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[models.Task]: if inputs := get_automation_workflow_inputs(workflow_status): return inputs.task @@ -368,46 +317,3 @@ def get_automation_workflow_inputs(workflow_status: dbos_lib.WorkflowStatus) -> def get_user_action_workflow_inputs(workflow_status: dbos_lib.WorkflowStatus) -> typing.Optional[params.UserActionWorkflowInputs]: resolved = resolve_user_action_workflow_inputs(workflow_status) return resolved.inputs - - -def get_automation_dict(description: typing.Union[str, dict]) -> dict: - if isinstance(description, str): - description = json.loads(description) - if isinstance(description, dict) and (state := description.get(STATE_KEY)) and isinstance(state, dict): - return description - raise ValueError("No automation state found in description") - - -def patch_task_content_degraded_state( - task_content: str, - error_status: str, - error_message: str, - *, - since: float, -) -> str: - if octobot_flow is None: - raise RuntimeError("octobot_flow is required to patch automation degraded state") - description = get_automation_dict(task_content) - automation_state = octobot_flow.entities.AutomationState.from_dict(description[STATE_KEY]) - existing_degraded_state = automation_state.automation.execution.degraded_state - degraded_since = ( - existing_degraded_state.since - if existing_degraded_state.since > 0 - else since - ) - automation_state.automation.execution.degraded_state = octobot_flow.entities.DegradedStateDetails( - since=degraded_since, - error=error_status, - reason=error_message, - ) - description[STATE_KEY] = automation_state.to_dict(include_default_values=False) - return json.dumps(description) - - -async def get_automation_workflow_status(automation_id: str) -> dbos_lib.WorkflowStatus: - for workflow_status in await dbos_lib.DBOS.list_workflows_async(status=[ - dbos_lib.WorkflowStatusString.PENDING.value, dbos_lib.WorkflowStatusString.ENQUEUED.value - ]): - if get_automation_id(workflow_status) == automation_id: - return workflow_status - raise ValueError(f"No automation workflow found for automation_id: {automation_id}") diff --git a/packages/node/tests/functional_tests/test_accounts_history_compute.py b/packages/node/tests/functional_tests/test_accounts_history_compute.py new file mode 100644 index 0000000000..812c56db57 --- /dev/null +++ b/packages/node/tests/functional_tests/test_accounts_history_compute.py @@ -0,0 +1,359 @@ +# Drakkar-Software OctoBot-Node + +""" +Functional tests for on-the-fly portfolio history computation. + +Each test seeds persisted Account (current holdings), AccountTrading (trades / +transactions), and exchange price caches, then calls +compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions. + +The compute path reverse-replays trades and transactions from the latest +portfolio to reconstruct past daily holdings, then values each day using daily +prices (or latest tickers when daily prices are missing). +""" + +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + +import octobot_node.protocol.accounts_history as accounts_history_module +from tests.functional_tests.util import accounts_history_test_util as accounts_history_test_util + + +@pytest.mark.asyncio +class TestComputeHistoryFunctionalSimulated: + """End-to-end history compute with seeded sync providers and persisted caches.""" + + async def test_single_buy_trade_produces_two_day_history(self, tmp_path): + """ + Scenario: account holds 1 BTC and 50000 USDT now. + One historical buy of 1 BTC at 30000 USDT on day 1. + Daily price cache has BTC/USDT = 30000 on day 1 and 50000 on day 2. + Expected: day 1 portfolio had 0 BTC + 80000 USDT, day 2 has 1 BTC + 50000 USDT. + """ + # Latest portfolio anchor: what the account holds today. + account = accounts_history_test_util.make_account("a1", {"BTC": 1.0, "USDT": 50000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + # Single BUY on day 1; reverse replay will subtract 1 BTC and credit 30000 USDT. + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "BTC/USDT", + protocol_models.Side.BUY, + quantity=1.0, + price=30000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + + buy_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.BUY_TIME.timestamp()) + ) + buy_day_start = int(buy_day_str) + prior_day_str = str(buy_day_start - 86400) + next_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.DAY_2_TS) + ) + data_root = str(tmp_path) + + # Prior day and day 1 close price 30000; day 2 close price 50000 (current-day valuation). + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": {prior_day_str: 30000.0, buy_day_str: 30000.0, next_day_str: 50000.0}}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": 50000.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + ) + with accounts_history_test_util.with_current_time(accounts_history_test_util.DAY_2_TS): + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.version == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + assert result.history is not None + assert result.history.unit == "USDT" + assert len(result.history.values) == 3 + + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + prior_day_start = buy_day_start - 86400 + next_day_start = int(next_day_str) + assert prior_day_start in day_values + assert buy_day_start in day_values + assert next_day_start in day_values + # Before the buy: 0 BTC + 80000 USDT (50000 + 1 * 30000), valued at 30000/BTC → 80000. + assert day_values[prior_day_start] == pytest.approx(80000.0) + # After the buy on day 1: 1 BTC + 50000 USDT, valued at 30000/BTC → 80000. + assert day_values[buy_day_start] == pytest.approx(80000.0) + # Day 2 forward-filled holdings repriced at 50000/BTC → 100000. + assert day_values[next_day_start] == pytest.approx(100000.0) + + history_by_day = { + int(history_value.timestamp.timestamp()): history_value + for history_value in result.history.values + } + prior_day = history_by_day[prior_day_start] + buy_day = history_by_day[buy_day_start] + assert prior_day.assets is not None and len(prior_day.assets) > 0 + prior_spot_assets = prior_day.assets[0].assets or [] + prior_assets_by_symbol = {asset.symbol: asset for asset in prior_spot_assets} + assert "USDT" in prior_assets_by_symbol + assert prior_assets_by_symbol["USDT"].holdings == pytest.approx(80000.0) + assert prior_assets_by_symbol["USDT"].value == pytest.approx(80000.0) + assert "BTC" not in prior_assets_by_symbol + + buy_spot_assets = buy_day.assets[0].assets or [] + buy_assets_by_symbol = {asset.symbol: asset for asset in buy_spot_assets} + assert buy_assets_by_symbol["BTC"].holdings == pytest.approx(1.0) + assert buy_assets_by_symbol["BTC"].value == pytest.approx(30000.0) + assert buy_assets_by_symbol["USDT"].holdings == pytest.approx(50000.0) + assert buy_assets_by_symbol["USDT"].value == pytest.approx(50000.0) + + async def test_deposit_and_trade_produce_coherent_history(self, tmp_path): + """ + Scenario: account holds 2 BTC and 20000 USDT now. + Day 1: bought 1 BTC at 30000 USDT. + Day 2: deposited 1 BTC. + Daily price for BTC/USDT: day 1 = 30000, day 2 = 35000. + """ + # Latest: 2 BTC + 20000 USDT. + account = accounts_history_test_util.make_account("a1", {"BTC": 2.0, "USDT": 20000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + # Day 1 BUY: reverse removes 1 BTC and adds 30000 USDT → 1 BTC + 20000 USDT. + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "BTC/USDT", + protocol_models.Side.BUY, + quantity=1.0, + price=30000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + # Day 2 deposit: reverse removes 1 BTC → 1 BTC + 20000 USDT before deposit. + deposit = accounts_history_test_util.make_protocol_transaction( + "tx1", + "BTC", + 1.0, + protocol_models.TransactionType.BLOCKCHAIN_DEPOSIT, + accounts_history_test_util.DEPOSIT_TIME, + ) + + buy_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.BUY_TIME.timestamp()) + ) + buy_day_start = int(buy_day_str) + prior_day_str = str(buy_day_start - 86400) + deposit_day_str = str( + accounts_history_test_util.utc_day_start(accounts_history_test_util.DEPOSIT_TIME.timestamp()) + ) + data_root = str(tmp_path) + + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": {prior_day_str: 30000.0, buy_day_str: 30000.0, deposit_day_str: 35000.0}}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"BTC/USDT": 35000.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + transactions=[deposit], + ) + with accounts_history_test_util.with_current_time( + accounts_history_test_util.DEPOSIT_TIME.timestamp(), + ): + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.history is not None + assert len(result.history.values) == 3 + + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + deposit_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.DEPOSIT_TIME.timestamp() + ) + prior_day_start = buy_day_start - 86400 + assert deposit_day_start in day_values + assert buy_day_start in day_values + assert prior_day_start in day_values + # After deposit on day 2: 2 BTC + 20000 USDT @ 35000 → 90000. + assert day_values[deposit_day_start] == pytest.approx(90000.0) + # After buy on day 1 (before deposit): 1 BTC + 20000 USDT @ 30000 → 50000. + assert day_values[buy_day_start] == pytest.approx(50000.0) + # Before buy: 0 BTC + 50000 USDT @ 30000 → 50000. + assert day_values[prior_day_start] == pytest.approx(50000.0) + + async def test_no_account_returns_empty_state(self, tmp_path): + """Missing account id must yield an empty history state, not an error.""" + with accounts_history_test_util.accounts_history_test_environment(tmp_path): + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "missing", + ) + + assert result.history is None + + async def test_no_trading_data_returns_empty_state(self, tmp_path): + """Account exists but AccountTrading was never seeded → no history to compute.""" + account = accounts_history_test_util.make_account("a1", {"BTC": 1.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + _trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + ) + + assert result.history is None + + async def test_ticker_fallback_when_no_daily_prices(self, tmp_path): + """ + When daily price cache is empty, latest ticker is used for all days. + + Scenario: account holds 5 ETH and 1000 USDT now. + One BUY of 5 ETH at 2000 USDT on day 1. + Ticker fallback: ETH/USDT = 2500 for every day. + """ + account = accounts_history_test_util.make_account("a1", {"ETH": 5.0, "USDT": 1000.0}) + exchange_config = accounts_history_test_util.make_exchange_config() + trade = accounts_history_test_util.make_protocol_trade( + "t1", + "ETH/USDT", + protocol_models.Side.BUY, + quantity=5.0, + price=2000.0, + executed_at=accounts_history_test_util.BUY_TIME, + ) + data_root = str(tmp_path) + + # Empty daily prices force valuation to use the latest ticker only. + await accounts_history_test_util.write_daily_prices_cache( + data_root, + "binance", + "spot", + False, + {}, + ) + await accounts_history_test_util.write_latest_tickers_cache( + data_root, + "binance", + "spot", + False, + {"ETH/USDT": 2500.0}, + ) + + with accounts_history_test_util.accounts_history_test_environment(tmp_path) as ( + account_provider, + trading_provider, + ): + accounts_history_test_util.seed_exchange_config( + account_provider, + accounts_history_test_util.TEST_USER_ID, + exchange_config, + ) + accounts_history_test_util.seed_account( + account_provider, + accounts_history_test_util.TEST_USER_ID, + account, + ) + accounts_history_test_util.seed_trading_state( + trading_provider, + accounts_history_test_util.TEST_USER_ID, + "a1", + trades=[trade], + ) + with accounts_history_test_util.with_current_time(accounts_history_test_util.DAY_2_TS): + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + accounts_history_test_util.TEST_USER_ID, + "a1", + data_root=data_root, + ) + + assert result.history is not None + assert len(result.history.values) == 3 + buy_day_start = accounts_history_test_util.utc_day_start( + accounts_history_test_util.BUY_TIME.timestamp() + ) + prior_day_start = buy_day_start - 86400 + day_values = { + int(history_value.timestamp.timestamp()): history_value.total + for history_value in result.history.values + } + # Before buy: 0 ETH + 11000 USDT (1000 + 5 * 2000), all in USDT → 11000. + assert day_values[prior_day_start] == pytest.approx(11000.0) + # After buy: 5 ETH + 1000 USDT @ 2500 ticker → 12500 + 1000 = 13500. + assert day_values[buy_day_start] == pytest.approx(13500.0) diff --git a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py index 17dce2c104..79607d0933 100644 --- a/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py +++ b/packages/node/tests/functional_tests/test_emit_and_copy_grid_automation_signals.py @@ -33,6 +33,7 @@ import starfish_server.storage.filesystem as starfish_filesystem_storage_module import starfish_sharing as starfish_sharing_module +from .util import exchange_account_elements_access as exchange_account_elements_access_module from .util import grid_workflow as grid_sim_util from .util import price_mocks as price_mocks_module from .util import user_action_assertions as user_action_assertions_module @@ -40,11 +41,10 @@ import octobot.community.authentication as community_authentication_module import octobot.community.local_authenticator as local_authenticator_module -import octobot_trading.constants as trading_constants_module import octobot_node.config import octobot_node.constants as node_constants_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module -import octobot_commons.constants as octobot_commons_constants_module import octobot_commons.os_util as commons_os_util_module import octobot_copy.constants as octobot_copy_constants_module import octobot_flow.entities as octobot_flow_entities @@ -68,11 +68,11 @@ _COPY_INIT_USDC = 2000.0 _T_ENQUEUE_SECONDS = 5.0 -_T_GRID_SECONDS = 25.0 -_T_BOOTSTRAP_COPY_SECONDS = 25.0 -_T_POST_SHOCK_SECONDS = 55.0 +_T_GRID_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) +_T_BOOTSTRAP_COPY_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) +_T_POST_SHOCK_SECONDS = workflow_common_module.functional_timeout_seconds(55.0) _T_STOP_SEND_SECONDS = 5.0 -_T_STOP_COMPLETE_SECONDS = 15.0 +_T_STOP_COMPLETE_SECONDS = workflow_common_module.functional_timeout_seconds(15.0) _D_DECIMAL_INCREMENT = decimal.Decimal(str(grid_sim_util.GRID_INCREMENT)) @@ -179,72 +179,6 @@ def _account_for_id( return copy_account raise AssertionError(f"Unexpected account_id lookup: {account_id!r}") -def _d_order_price(raw: typing.Union[int, float, str, decimal.Decimal]) -> decimal.Decimal: - if isinstance(raw, decimal.Decimal): - return raw - return decimal.Decimal(str(raw)) - -def _sorted_limit_prices_from_elements( - exchange_account_elements: typing.Any, - *, - trade_order_side, -) -> list[decimal.Decimal]: - if exchange_account_elements is None: - return [] - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - return [] - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) - open_orders = open_orders or [] - - storage_origin = trading_constants_module.STORAGE_ORIGIN_VALUE - - def _open_order_payload(order_row: typing.Any) -> typing.Any: - """Exchange rows may nest ccxt fields under ``STORAGE_ORIGIN_VALUE``; protocol orders are flat.""" - if isinstance(order_row, dict): - nested = order_row.get(storage_origin) - if isinstance(nested, dict): - return nested - return order_row - nested = getattr(order_row, storage_origin, None) - if nested is not None: - return nested - return order_row - - side_key = trading_enums_module.ExchangeConstantsOrderColumns.SIDE.value - price_col = trading_enums_module.ExchangeConstantsOrderColumns.PRICE.value - type_col = trading_enums_module.ExchangeConstantsOrderColumns.TYPE.value - want_side = trade_order_side.value - limit_type = trading_enums_module.TradeOrderType.LIMIT.value - prices: list[decimal.Decimal] = [] - for order in open_orders: - payload = _open_order_payload(order) - if isinstance(payload, dict): - side = payload.get(side_key) - price_raw = payload.get(price_col) - order_type = payload.get(type_col) - else: - side = getattr(payload, side_key, None) - price_raw = getattr(payload, price_col, None) - order_type = getattr(payload, type_col, None) - if hasattr(side, "value"): - side = side.value - if side != want_side: - continue - if price_raw is None: - continue - if hasattr(order_type, "value"): - order_type = order_type.value - if order_type is not None and order_type != limit_type: - continue - prices.append(_d_order_price(price_raw)) - prices.sort() - return prices - def _assert_open_limit_prices_match_reference( reference_elements: typing.Any, follower_elements: typing.Any, @@ -253,31 +187,16 @@ def _assert_open_limit_prices_match_reference( trading_enums_module.TradeOrderSide.BUY, trading_enums_module.TradeOrderSide.SELL, ): - ref_prices = _sorted_limit_prices_from_elements(reference_elements, trade_order_side=side) - got_prices = _sorted_limit_prices_from_elements(follower_elements, trade_order_side=side) + ref_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + reference_elements, + trade_order_side=side, + ) + got_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + follower_elements, + trade_order_side=side, + ) assert ref_prices == got_prices, f"side={side!r} ref={ref_prices!s} follower={got_prices!s}" -def _portfolio_content_from_exchange_elements(exchange_account_elements: typing.Any) -> dict[str, typing.Any]: - portfolio = getattr(exchange_account_elements, "portfolio", None) - if portfolio is None and isinstance(exchange_account_elements, dict): - portfolio = exchange_account_elements.get("portfolio") - if portfolio is None: - return {} - content = getattr(portfolio, "content", None) - if content is None and isinstance(portfolio, dict): - content = portfolio.get("content") - return content if isinstance(content, dict) else {} - -def _portfolio_row_total(row: typing.Any) -> decimal.Decimal: - total_key = octobot_commons_constants_module.PORTFOLIO_TOTAL - if isinstance(row, dict): - raw = row.get(total_key, row.get("total")) - else: - raw = getattr(row, total_key, None) or getattr(row, "total", None) - if raw is None: - raise AssertionError("portfolio row has no total amount") - return raw if isinstance(raw, decimal.Decimal) else decimal.Decimal(str(raw)) - def _value_weighted_btc_usdc_shares( content: dict[str, typing.Any], *, @@ -286,8 +205,8 @@ def _value_weighted_btc_usdc_shares( """USDC notionals: ``btc_total * btc_usdc_close`` vs USDC total; shares sum to 1.""" for asset in ("BTC", "USDC"): assert asset in content, f"missing portfolio row for {asset}" - btc_total = _portfolio_row_total(content["BTC"]) - usdc_total = _portfolio_row_total(content["USDC"]) + btc_total = exchange_account_elements_access_module.portfolio_row_total(content["BTC"]) + usdc_total = exchange_account_elements_access_module.portfolio_row_total(content["USDC"]) btc_notional_usdc = btc_total * btc_usdc_close usdc_notional = usdc_total total_notional = btc_notional_usdc + usdc_notional @@ -303,8 +222,8 @@ def _assert_btc_usdc_value_shares_match_reference( *, btc_usdc_close: decimal.Decimal, ) -> None: - ref_content = _portfolio_content_from_exchange_elements(reference_elements) - follower_content = _portfolio_content_from_exchange_elements(follower_elements) + ref_content = exchange_account_elements_access_module.portfolio_content_from_elements(reference_elements) + follower_content = exchange_account_elements_access_module.portfolio_content_from_elements(follower_elements) ref_shares = _value_weighted_btc_usdc_shares(ref_content, btc_usdc_close=btc_usdc_close) follower_shares = _value_weighted_btc_usdc_shares(follower_content, btc_usdc_close=btc_usdc_close) # ~3 percentage points slack (master vs copy notionals / float portfolio totals). @@ -318,7 +237,7 @@ def _assert_btc_usdc_value_shares_match_reference( ) def _first_sell_limit_price(exchange_account_elements: typing.Any) -> decimal.Decimal: - sells = _sorted_limit_prices_from_elements( + sells = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( exchange_account_elements, trade_order_side=trading_enums_module.TradeOrderSide.SELL, ) @@ -338,8 +257,10 @@ def _sorted_limit_prices_from_trading_signal_account( *, trade_order_side, ) -> list[decimal.Decimal]: - wrapper = {"orders": {"open_orders": list(trading_signal.account.orders or [])}} - return _sorted_limit_prices_from_elements(wrapper, trade_order_side=trade_order_side) + return exchange_account_elements_access_module.sorted_open_limit_prices_from_protocol_orders( + trading_signal.account.orders, + trade_order_side=trade_order_side, + ) async def _fetch_strategy_signals_from_sync(evm_address: str) -> list[typing.Any]: async with local_authenticator_module.local_user_authenticator() as auth: @@ -360,7 +281,7 @@ def _ladder_limit_prices_match_reference( trading_enums_module.TradeOrderSide.BUY, trading_enums_module.TradeOrderSide.SELL, ): - reference_prices = _sorted_limit_prices_from_elements( + reference_prices = exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( reference_exchange_account_elements, trade_order_side=order_side, ) @@ -413,9 +334,9 @@ async def _poll_state_reader_until( while time.monotonic() < poll_deadline: workflow_rows = await scheduler.INSTANCE.list_workflows_async() for workflow_row in workflow_rows: - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue last_reader = state_reader @@ -629,6 +550,14 @@ def _functional_seed_strategy_for_emit_copy_test(_wallet_address, stored_item_id ): caplog.set_level(logging.INFO) + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state( + user_id, master_account_id + ) + workflow_common_module.seed_empty_account_trading_state( + user_id, copy_account_id + ) + # Step 1 — enqueue master emitting grid signals (pushes to local sync server). try: await asyncio.wait_for( @@ -676,7 +605,7 @@ def _functional_seed_strategy_for_emit_copy_test(_wallet_address, stored_item_id master_workflow_rows_for_user_action_selector = [ workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() - if workflows_util_module.get_automation_id(workflow_row) + if automation_states_loader_module.get_automation_id(workflow_row) == user_action_assertions_module.resolve_create_automation_metadata_id( master_user_action, ) @@ -771,7 +700,7 @@ def _functional_seed_strategy_for_emit_copy_test(_wallet_address, stored_item_id copy_workflow_rows_for_user_action_selector = [ workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() - if workflows_util_module.get_automation_id(workflow_row) + if automation_states_loader_module.get_automation_id(workflow_row) == _COPY_AUTOMATION_ID ] assert copy_workflow_rows_for_user_action_selector @@ -859,7 +788,7 @@ def _functional_seed_strategy_for_emit_copy_test(_wallet_address, stored_item_id key=lambda workflow_status: workflow_status.updated_at or 0, reverse=True, ): - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue matching_parent_id = workflow_row.workflow_id[ : node_constants_module.PARENT_WORKFLOW_ID_LENGTH diff --git a/packages/node/tests/functional_tests/test_global_view_workflow.py b/packages/node/tests/functional_tests/test_global_view_workflow.py new file mode 100644 index 0000000000..8c407e5934 --- /dev/null +++ b/packages/node/tests/functional_tests/test_global_view_workflow.py @@ -0,0 +1,117 @@ +# Drakkar-Software OctoBot-Node + +import asyncio +import pathlib + +import mock +import pytest + +import octobot_node.scheduler.api as scheduler_api_module +import octobot_node.scheduler.tasks as scheduler_tasks_module + +from tests.functional_tests.util import global_view_workflow as global_view_workflow_util +from tests.scheduler import temp_dbos_scheduler + + +@pytest.fixture +def global_view_temp_dbos_scheduler(temp_dbos_scheduler): + return temp_dbos_scheduler + + +@pytest.mark.asyncio +class TestGlobalViewWorkflowFunctional: + async def test_global_view_workflow_updates_all_accounts( + self, + tmp_path: pathlib.Path, + global_view_temp_dbos_scheduler, + ): + async with global_view_workflow_util.global_view_functional_environment(tmp_path) as environment: + workflow_result = await asyncio.wait_for( + global_view_workflow_util.enqueue_and_await_global_view_refresh(), + timeout=global_view_workflow_util.WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + assert workflow_result["refreshed_accounts"] == 3 + + user_id = environment["user_id"] + account_provider = environment["account_provider"] + real_account = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_REAL_ID) + sim_account_1 = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_SIM_1_ID) + sim_account_2 = account_provider.get_item(user_id, global_view_workflow_util.ACCOUNT_SIM_2_ID) + + global_view_workflow_util.assert_real_account_assets(real_account) + global_view_workflow_util.assert_simulated_account_assets(sim_account_1, expected_total=1000.0) + global_view_workflow_util.assert_simulated_account_assets(sim_account_2, expected_total=1000.0) + assert real_account.updated_at is not None + assert sim_account_1.updated_at is not None + assert sim_account_2.updated_at is not None + + async def test_global_view_workflow_triggers_automation_on_filled_order( + self, + tmp_path: pathlib.Path, + global_view_temp_dbos_scheduler, + ): + matching_automation = global_view_workflow_util.build_running_automation_state( + global_view_workflow_util.AUTOMATION_FILL_ID, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + order_ids=[global_view_workflow_util.ORDER_FILL_ID], + ) + non_matching_automation = global_view_workflow_util.build_running_automation_state( + global_view_workflow_util.AUTOMATION_NO_TRIGGER_ID, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + order_ids=[global_view_workflow_util.ORDER_STAYS_OPEN_ID], + ) + trigger_calls: list[str] = [] + + async def record_forced_trigger(automation_id: str, user_id: str) -> None: + trigger_calls.append(automation_id) + + async with global_view_workflow_util.global_view_functional_environment( + tmp_path, + sim_ticker_close_by_symbol={"BTC/USDT": 9000.0}, + ) as environment: + user_id = environment["user_id"] + global_view_workflow_util.seed_account_trading_state( + environment["trading_provider"], + user_id, + account_id=global_view_workflow_util.ACCOUNT_SIM_1_ID, + seeded_orders=[ + { + "exchange_id": global_view_workflow_util.ORDER_FILL_ID, + "price": 10000.0, + "trigger_above": False, + }, + { + "exchange_id": global_view_workflow_util.ORDER_STAYS_OPEN_ID, + "price": 8000.0, + "trigger_above": False, + }, + ], + ) + with ( + mock.patch.object( + scheduler_api_module, + "get_automation_states", + mock.AsyncMock(return_value=[matching_automation, non_matching_automation]), + ), + mock.patch.object( + scheduler_tasks_module, + "send_forced_trigger_to_active_automation", + side_effect=record_forced_trigger, + ), + ): + workflow_result = await asyncio.wait_for( + global_view_workflow_util.enqueue_and_await_global_view_refresh(), + timeout=global_view_workflow_util.WORKFLOW_RESULT_TIMEOUT_SECONDS, + ) + + assert workflow_result["refreshed_accounts"] == 3 + assert trigger_calls == [global_view_workflow_util.AUTOMATION_FILL_ID] + + account_provider = environment["account_provider"] + for account_id in ( + global_view_workflow_util.ACCOUNT_REAL_ID, + global_view_workflow_util.ACCOUNT_SIM_1_ID, + global_view_workflow_util.ACCOUNT_SIM_2_ID, + ): + updated_account = account_provider.get_item(user_id, account_id) + assert updated_account.assets is not None diff --git a/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py new file mode 100644 index 0000000000..b6f316a922 --- /dev/null +++ b/packages/node/tests/functional_tests/test_outdated_reference_account_workflow.py @@ -0,0 +1,199 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +from __future__ import annotations + +import asyncio +import decimal +import logging +import time + +import mock +import pytest + +from .util import authenticator_mocks as authenticator_mocks_module +from .util import grid_workflow as grid_workflow_module +from .util import price_mocks as price_mocks_module +from .util import user_action_assertions as user_action_assertions_module +from .util import workflow_common as workflow_common_module + +import octobot.community.authentication as community_authentication_module +import octobot_flow.entities as octobot_flow_entities +import octobot_node.config +import octobot_node.scheduler +import octobot_node.scheduler.workflows_util as workflows_util_module +import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module + +from tests.scheduler import temp_dbos_scheduler + +_T_ENQUEUE_SECONDS = 5.0 +_T_POLL_SECONDS = workflow_common_module.functional_timeout_seconds(25.0) + + +@pytest.mark.asyncio +class TestOutdatedReferenceAccountWorkflow: + async def test_skips_stale_signal_then_executes_fresh_signal_from_internal_channel( + self, + temp_dbos_scheduler, + caplog, + ): + """ + Copy automation on the simulator exchange: stale TradingSignal is rejected as outdated, + workflow postpones and waits; a fresh signal drives a real mirror limit order. + """ + import octobot_flow.repositories.community.trading_signals_channel as trading_signals_channel_module + import octobot_node.scheduler.internal_trading_signals as internal_trading_signals_module + + from .util import outdated_reference_workflow as outdated_reference_workflow_module + + fresh_order_price_decimal = decimal.Decimal(str(outdated_reference_workflow_module.FRESH_ORDER_PRICE)) + stale_order_price_decimal = decimal.Decimal(str(outdated_reference_workflow_module.STALE_ORDER_PRICE)) + + # Step 0 — Pin BTC/USDC close and mock only exchange/infra providers; DBOS + flow run for real. + patched_fetch_tickers = price_mocks_module.tickers_repository_fetch_tickers_btc_usdc_close_override( + lambda: outdated_reference_workflow_module.FRESH_MARKET_PRICE, + ) + patched_fetch_ohlcv = price_mocks_module.fetch_ohlcv_side_effect_for_close_price( + lambda: outdated_reference_workflow_module.FRESH_MARKET_PRICE, + ) + user_id = workflow_common_module.SIMULATOR_GRID_TEST_COMMUNITY_USER_ID + protocol_account = workflow_common_module.protocol_account_for_functional( + account_id=outdated_reference_workflow_module.COPY_ACCOUNT_ID, + usdc_total=2000.0, + account_name="Outdated reference functional copy account", + ) + copy_user_action = outdated_reference_workflow_module.build_create_copy_follower_user_action() + automation_id = user_action_assertions_module.resolve_create_automation_metadata_id(copy_user_action) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + + def _functional_seed_strategy_for_outdated_reference_test(_wallet_address, stored_item_id): + if stored_item_id == grid_workflow_module.SIMULATOR_COPY_FOLLOWER_STORED_STRATEGY_ID: + return grid_workflow_module.seeded_copy_follower_strategy_for_functional_wallet( + copy_master_strategy_id=outdated_reference_workflow_module.MASTER_STRATEGY_ID, + ) + raise AssertionError(f"unexpected strategy id for functional seed: {stored_item_id!r}") + + await internal_trading_signals_module.subscribe_internal_trading_signal_consumer() + try: + # Step 0 (continued) — Subscribe internal trading-signal consumer and seed account trading state. + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch.object( + octobot_flow_repositories_exchange_module.TickersRepository, + "fetch_tickers", + new=patched_fetch_tickers, + ), + mock.patch.object( + octobot_flow_repositories_exchange_module.OhlcvRepository, + "fetch_ohlcv", + side_effect=patched_fetch_ohlcv, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock(return_value=protocol_account), + get_exchange_config=mock.Mock( + return_value=workflow_common_module.protocol_exchange_config_for_grid_functional(), + ), + ), + ), + mock.patch( + "octobot_sync.sync.collection_providers.StrategyProvider.instance", + return_value=mock.Mock( + get_item=mock.Mock( + side_effect=_functional_seed_strategy_for_outdated_reference_test, + ), + ), + ), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), + mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), + ): + caplog.set_level(logging.INFO) + workflow_common_module.seed_empty_account_trading_state( + user_id, + outdated_reference_workflow_module.COPY_ACCOUNT_ID, + ) + + stale_signal = outdated_reference_workflow_module.build_stale_trading_signal() + fresh_signal = outdated_reference_workflow_module.build_fresh_trading_signal() + + stale_delivery_task = asyncio.create_task( + outdated_reference_workflow_module.deliver_trading_signal_when_pending( + temp_dbos_scheduler, + automation_id, + stale_signal, + _T_POLL_SECONDS, + ) + ) + + # Step 1 — Enqueue AUTOMATION_CREATE for copy follower; expect COMPLETED create result. + try: + await asyncio.wait_for( + workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( + temp_dbos_scheduler, + copy_user_action, + user_id, + ), + timeout=_T_ENQUEUE_SECONDS, + ) + except TimeoutError as exc: + raise AssertionError("execute_user_action timed out enqueueing copy workflow") from exc + + await stale_delivery_task + + await user_action_assertions_module.assert_user_action_selector_completed_automation_create( + user_id=user_id, + user_action_id=copy_user_action.id, + expected_workflow_id=None, + ) + + # Step 2 — Stale signal delivered while copy workflow was pending; expect outdated skip log. + stale_skip_deadline = time.monotonic() + _T_POLL_SECONDS + while time.monotonic() < stale_skip_deadline: + if outdated_reference_workflow_module.caplog_contains_outdated_skip(caplog): + break + await asyncio.sleep(workflow_common_module.DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS) + else: + pytest.fail( + "Timed out waiting for outdated reference account skip log " + f"({outdated_reference_workflow_module.OUTDATED_SKIP_LOG_SUBSTRING!r})" + ) + + await outdated_reference_workflow_module.poll_state_reader_until( + temp_dbos_scheduler, + automation_id, + lambda reader: stale_order_price_decimal + not in outdated_reference_workflow_module.sell_limit_prices_from_reader(reader), + _T_POLL_SECONDS, + "stale sell limit not mirrored", + ) + + # Step 3 — Send fresh TradingSignal after child workflow is pending again; expect mirror at fresh price. + await outdated_reference_workflow_module.deliver_trading_signal_when_pending( + temp_dbos_scheduler, + automation_id, + fresh_signal, + _T_POLL_SECONDS, + ) + fresh_reader = await outdated_reference_workflow_module.poll_state_reader_until( + temp_dbos_scheduler, + automation_id, + lambda reader: fresh_order_price_decimal + in outdated_reference_workflow_module.sell_limit_prices_from_reader(reader), + _T_POLL_SECONDS, + "fresh sell limit mirrored", + ) + + # Step 4 — Assert stale price never mirrored, fresh order present, automation still healthy. + sell_prices = outdated_reference_workflow_module.sell_limit_prices_from_reader(fresh_reader) + assert stale_order_price_decimal not in sell_prices + assert sell_prices.count(fresh_order_price_decimal) == 1 + assert outdated_reference_workflow_module.caplog_contains_outdated_skip(caplog) + finally: + await trading_signals_channel_module.shutdown_internal_trading_signal_channel() diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py index 4a6e978f31..ca748e1a8d 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_2_evaluators_workflow.py @@ -18,12 +18,15 @@ from .util import dag_assertions as dag_assertions_module from .util import dca_workflow as dca_sim_util +from .util import authenticator_mocks as authenticator_mocks_module from .util import price_mocks as price_mocks_module from .util import protocol_assertions as protocol_assertions_module from .util import user_action_assertions as user_action_assertions_module from .util import workflow_common as workflow_common_module +import octobot.community.authentication as community_authentication_module import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_trading.enums as trading_enums_module @@ -109,8 +112,18 @@ def _decline_per_candle(symbol: str) -> float: strategy_id=dca_sim_util.SIMULATOR_DCA_DEFAULT_STRATEGY_ID, ) + authentication_instance = authenticator_mocks_module.build_community_authentication( + workflow_common_module.SIMULATOR_GRID_TEST_PRIVATE_KEY, + workflow_common_module.SIMULATOR_GRID_TEST_WALLET_PASSPHRASE, + ) + # Patch market data and providers, then create the maximum-evaluators DCA automation. with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), mock.patch.object( octobot_flow_repositories_exchange_module.TickersRepository, "fetch_tickers", @@ -141,6 +154,8 @@ def _decline_per_candle(symbol: str) -> float: ), ), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _DCA_ACCOUNT_ID) try: await asyncio.wait_for( workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( @@ -261,7 +276,7 @@ async def _forced_trigger(user_action_id: str) -> None: }, timeout_seconds=_T_STEP_SECONDS, ) - after_strategy_reader = workflows_util_module.get_automation_state_reader( + after_strategy_reader = automation_states_loader_module.get_automation_state_reader( after_strategy_workflow ) assert after_strategy_reader is not None @@ -321,9 +336,9 @@ async def _forced_trigger(user_action_id: str) -> None: last_trade_count: int | None = None while time.monotonic() < btc_only_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_cycle1 = workflows_util_module.get_automation_state_reader(workflow_row) + reader_after_cycle1 = automation_states_loader_module.get_automation_state_reader(workflow_row) if reader_after_cycle1 is None: continue candidate_elements = reader_after_cycle1.state.automation.exchange_account_elements @@ -423,9 +438,9 @@ async def _forced_trigger(user_action_id: str) -> None: async def _poll_cycle2_post_fill_state() -> bool: nonlocal elements_after_cycle2_strategy for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_cycle2 = workflows_util_module.get_automation_state_reader(workflow_row) + reader_after_cycle2 = automation_states_loader_module.get_automation_state_reader(workflow_row) if reader_after_cycle2 is None: continue candidate_elements = reader_after_cycle2.state.automation.exchange_account_elements @@ -448,9 +463,9 @@ async def _poll_cycle2_post_fill_state() -> bool: while time.monotonic() < cycle2_strategy_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_cycle2_strategy = workflows_util_module.get_automation_state_reader( + reader_after_cycle2_strategy = automation_states_loader_module.get_automation_state_reader( workflow_row ) if reader_after_cycle2_strategy is None: @@ -507,9 +522,9 @@ async def _poll_cycle2_post_fill_state() -> bool: elements_after_cycle2_dca: typing.Any = None while time.monotonic() < cycle2_dca_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_cycle2_dca = workflows_util_module.get_automation_state_reader( + reader_after_cycle2_dca = automation_states_loader_module.get_automation_state_reader( workflow_row ) if reader_after_cycle2_dca is None: @@ -587,7 +602,7 @@ async def _poll_cycle2_post_fill_state() -> bool: workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() if workflow_row.status == dbos.WorkflowStatusString.SUCCESS.value - and workflows_util_module.get_automation_id(workflow_row) == metadata_automation_id + and automation_states_loader_module.get_automation_id(workflow_row) == metadata_automation_id ] assert success_rows final_workflow_row = max(success_rows, key=lambda workflow_status: workflow_status.updated_at or 0) diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py index 5dd8e903f7..f6a59055fb 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_dca_no_evaluator_workflow.py @@ -22,6 +22,7 @@ import octobot.community.authentication as community_authentication_module import octobot_flow.entities as octobot_flow_entities import octobot_trading.enums as trading_enums_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module @@ -113,6 +114,8 @@ async def test_trigger_task_dca_simulator_entry_fill_then_stop(self, temp_dbos_s ), ), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _DCA_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE; expect a completed create result and a running workflow. try: await asyncio.wait_for( @@ -143,10 +146,10 @@ async def test_trigger_task_dca_simulator_entry_fill_then_stop(self, temp_dbos_s workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() dca_predicate_met = False for workflow_row in workflow_rows: - workflow_automation_id = workflows_util_module.get_automation_id(workflow_row) + workflow_automation_id = automation_states_loader_module.get_automation_id(workflow_row) if workflow_automation_id != metadata_automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue elements = state_reader.state.automation.exchange_account_elements @@ -247,9 +250,9 @@ async def _enqueue_forced_trigger_and_await(*, user_action_id: str) -> None: open_orders_after_first_fill: list[dict] | None = None while time.monotonic() < partial_fill_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_first_fill = workflows_util_module.get_automation_state_reader( + reader_after_first_fill = automation_states_loader_module.get_automation_state_reader( workflow_row ) if reader_after_first_fill is None: @@ -302,9 +305,9 @@ async def _enqueue_forced_trigger_and_await(*, user_action_id: str) -> None: elements_after_fill: typing.Any = None while time.monotonic() < fill_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_fill = workflows_util_module.get_automation_state_reader(workflow_row) + reader_after_fill = automation_states_loader_module.get_automation_state_reader(workflow_row) if reader_after_fill is None: continue candidate_elements = reader_after_fill.state.automation.exchange_account_elements @@ -417,7 +420,7 @@ async def _enqueue_forced_trigger_and_await(*, user_action_id: str) -> None: workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() if workflow_row.status == dbos.WorkflowStatusString.SUCCESS.value - and workflows_util_module.get_automation_id(workflow_row) == metadata_automation_id + and automation_states_loader_module.get_automation_id(workflow_row) == metadata_automation_id ] assert success_rows, "expected at least one SUCCESS workflow for automation after stop" final_workflow_row = max(success_rows, key=lambda workflow_status: workflow_status.updated_at or 0) diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py index f010003c9d..ba825ef8b7 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_default_config_octobot_process_workflow.py @@ -27,7 +27,8 @@ import octobot_node.config import octobot_node.constants as octobot_node_constants_module import octobot_node.scheduler -import octobot_node.scheduler.generic_process_octobot as generic_process_octobot_module +import octobot_node.scheduler.automations.generic_process_octobot as generic_process_octobot_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_sync.sync.collection_providers as collection_providers_module import tentacles.Meta.DSL_operators.octobot_process_operators.octobot_process_ops as octobot_process_ops @@ -47,6 +48,35 @@ # Spawn real OctoBot child processes on fixed local ports; serialize under pytest-xdist. pytestmark = pytest.mark.xdist_group("octobot_process_functional") +_SHORT_GRACE_RECV_BEFORE_RESCHEDULE_SECONDS = 0.25 + + +@pytest.fixture(autouse=True) +def _short_grace_recv_before_reschedule(monkeypatch, temp_dbos_scheduler): + # Second arg is resume_execution_time (absolute epoch), not a delay. Reschedule path + # passes 0 => zero-timeout recv; rewrite to now+grace so stop can land before next child. + import importlib + automation_workflow_module = importlib.import_module( + "octobot_node.scheduler.workflows.automation_workflow" + ) + real_wait = automation_workflow_module.AutomationWorkflow._wait_and_trigger_on_actions_update + + async def wait_with_short_grace(parsed_inputs, resume_execution_time, *args, **kwargs): + effective_resume_execution_time = resume_execution_time + if resume_execution_time == 0: + effective_resume_execution_time = ( + time.time() + _SHORT_GRACE_RECV_BEFORE_RESCHEDULE_SECONDS + ) + return await real_wait( + parsed_inputs, effective_resume_execution_time, *args, **kwargs + ) + + monkeypatch.setattr( + automation_workflow_module.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + wait_with_short_grace, + ) + class TestStartCheckAndStopDefaultConfigOctobotProcessWorkflow: @pytest.mark.asyncio @@ -118,6 +148,10 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state( + user_id, _GENERIC_PROCESS_ACCOUNT_ID + ) try: await asyncio.wait_for( workflow_common_module.enqueue_user_action_workflow_and_await_terminal_result( @@ -158,9 +192,9 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule workflow_row_matching: typing.Any = None state_reader_matching: typing.Any = None for workflow_row in workflow_rows: - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue workflow_row_matching = workflow_row @@ -263,7 +297,7 @@ async def test_generic_process_default_config_lifecycle(self, temp_dbos_schedule workflow_rows_after_restart = await temp_dbos_scheduler.INSTANCE.list_workflows_async() workflow_row_after_restart: typing.Any = None for workflow_row in workflow_rows_after_restart: - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue if workflow_row.status not in ( dbos.WorkflowStatusString.PENDING.value, @@ -377,9 +411,9 @@ async def test_generic_process_accountless_default_config_lifecycle( workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() workflow_row_matching: typing.Any = None for workflow_row in workflow_rows: - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue workflow_row_matching = workflow_row diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py index 77ad2bab5c..5d2f72f3bb 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow.py @@ -34,6 +34,7 @@ import octobot_flow.entities as octobot_flow_entities import octobot_node.config import octobot_node.scheduler +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module @@ -41,12 +42,12 @@ _T_ENQUEUE_SECONDS = 5.0 -_T_GRID_SECONDS = 20.0 +_T_GRID_SECONDS = workflow_common_module.functional_timeout_seconds(20.0) _T_SIGNAL_SECONDS = 5.0 _T_STOP_SEND_SECONDS = 5.0 -_T_STOP_COMPLETE_SECONDS = 10.0 +_T_STOP_COMPLETE_SECONDS = workflow_common_module.functional_timeout_seconds(10.0) _T_RESTART_ENQUEUE_SECONDS = 10.0 -_T_RESTART_RUNNING_SECONDS = 30.0 +_T_RESTART_RUNNING_SECONDS = workflow_common_module.functional_timeout_seconds(30.0) # Fast poll after stop/signal send; protocol status may flip RUNNING→COMPLETED quickly on CI. _POST_STOP_PROTOCOL_POLL_SECONDS = 0.05 @@ -129,6 +130,8 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _GRID_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE user action; expect a COMPLETED create result and a running workflow. try: await asyncio.wait_for( @@ -159,10 +162,10 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() grid_predicate_met = False for workflow_row in workflow_rows: - workflow_automation_id = workflows_util_module.get_automation_id(workflow_row) + workflow_automation_id = automation_states_loader_module.get_automation_id(workflow_row) if workflow_automation_id != metadata_automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue elements = state_reader.state.automation.exchange_account_elements @@ -258,9 +261,9 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d signal_deadline = time.monotonic() + _T_SIGNAL_SECONDS while time.monotonic() < signal_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_signal = workflows_util_module.get_automation_state_reader( + reader_after_signal = automation_states_loader_module.get_automation_state_reader( workflow_row ) if reader_after_signal is None: @@ -329,9 +332,9 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d key=lambda workflow_status: workflow_status.updated_at or 0, reverse=True, ): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_send = workflows_util_module.get_automation_state_reader(workflow_row) + reader_after_send = automation_states_loader_module.get_automation_state_reader(workflow_row) if reader_after_send is None: continue candidate_elements = reader_after_send.state.automation.exchange_account_elements @@ -413,7 +416,7 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() if workflow_row.status == dbos.WorkflowStatusString.SUCCESS.value - and workflows_util_module.get_automation_id(workflow_row) == metadata_automation_id + and automation_states_loader_module.get_automation_id(workflow_row) == metadata_automation_id ] assert success_rows, "expected at least one SUCCESS workflow for automation after stop" final_workflow_row = max(success_rows, key=lambda workflow_status: workflow_status.updated_at or 0) @@ -458,7 +461,7 @@ async def test_trigger_task_grid_simulator_two_iterations_then_stop(self, temp_d protocol_state_after_restart = None while time.monotonic() < restart_running_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue if workflow_row.status not in ( dbos.WorkflowStatusString.PENDING.value, diff --git a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py index 6e21e04c0e..15aa2199ef 100644 --- a/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py +++ b/packages/node/tests/functional_tests/test_start_check_and_stop_grid_workflow_hollaex_earncurve.py @@ -23,6 +23,7 @@ import octobot_flow.entities as octobot_flow_entities import octobot_node.config import octobot_node.scheduler +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_flow.repositories.exchange as octobot_flow_repositories_exchange_module @@ -129,6 +130,8 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None), mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None), ): + # Seed trading state as CreateAccountActionExecutor would; persist_account_trading requires it. + workflow_common_module.seed_empty_account_trading_state(user_id, _GRID_ACCOUNT_ID) # Step 1 — Enqueue AUTOMATION_CREATE user action; expect a COMPLETED create result and a running workflow. try: await asyncio.wait_for( @@ -160,10 +163,10 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self workflow_rows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() grid_predicate_met = False for workflow_row in workflow_rows: - workflow_automation_id = workflows_util_module.get_automation_id(workflow_row) + workflow_automation_id = automation_states_loader_module.get_automation_id(workflow_row) if workflow_automation_id != metadata_automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue elements = state_reader.state.automation.exchange_account_elements @@ -263,9 +266,9 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self signal_deadline = time.monotonic() + _T_SIGNAL_SECONDS while time.monotonic() < signal_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_signal = workflows_util_module.get_automation_state_reader( + reader_after_signal = automation_states_loader_module.get_automation_state_reader( workflow_row ) if reader_after_signal is None: @@ -335,9 +338,9 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self key=lambda workflow_status: workflow_status.updated_at or 0, reverse=True, ): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue - reader_after_send = workflows_util_module.get_automation_state_reader(workflow_row) + reader_after_send = automation_states_loader_module.get_automation_state_reader(workflow_row) if reader_after_send is None: continue candidate_elements = reader_after_send.state.automation.exchange_account_elements @@ -417,7 +420,7 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self workflow_row for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async() if workflow_row.status == dbos.WorkflowStatusString.SUCCESS.value - and workflows_util_module.get_automation_id(workflow_row) == metadata_automation_id + and automation_states_loader_module.get_automation_id(workflow_row) == metadata_automation_id ] assert success_rows, "expected at least one SUCCESS workflow for automation after stop" final_workflow_row = max(success_rows, key=lambda workflow_status: workflow_status.updated_at or 0) @@ -464,7 +467,7 @@ async def test_trigger_task_grid_hollaex_earncurve_two_iterations_then_stop(self protocol_state_after_restart = None while time.monotonic() < restart_running_deadline: for workflow_row in await temp_dbos_scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != metadata_automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != metadata_automation_id: continue if workflow_row.status not in ( dbos.WorkflowStatusString.PENDING.value, diff --git a/packages/node/tests/functional_tests/util/accounts_history_test_util.py b/packages/node/tests/functional_tests/util/accounts_history_test_util.py new file mode 100644 index 0000000000..f96a734b8b --- /dev/null +++ b/packages/node/tests/functional_tests/util/accounts_history_test_util.py @@ -0,0 +1,222 @@ +# Drakkar-Software OctoBot-Node + +import contextlib +import datetime +import math + +import mock + +import octobot.community.authentication as community_authentication +import octobot_flow.logic.portfolio_history.portfolio_value_history as portfolio_value_history_module +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants +import octobot_sync.sync.collection_providers as collection_providers +import octobot_trading.api as trading_api + + +TEST_USER_ID = "0xaccountshistorytestwallet" +_TEST_PRIVATE_KEY = "accounts-history-test-private-key" +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 1, 12, 0, tzinfo=datetime.UTC) + +DAY_1_TS = 1700000000.0 +DAY_2_TS = DAY_1_TS + 86400 +BUY_TIME = datetime.datetime.fromtimestamp(DAY_1_TS + 3600, tz=datetime.timezone.utc) +DEPOSIT_TIME = datetime.datetime.fromtimestamp(DAY_2_TS + 1800, tz=datetime.timezone.utc) + + +def utc_day_start(timestamp: float) -> int: + return int(math.floor(timestamp / 86400.0) * 86400) + + +def make_account( + account_id: str, + assets: dict[str, float], + exchange: str = "binance", + sandboxed: bool = False, + is_simulated: bool = False, +) -> protocol_models.Account: + detailed_assets = [ + protocol_models.DetailedAsset(symbol=symbol, total=amount, available=amount) + for symbol, amount in assets.items() + ] + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote1", + exchange_config_ids=["cfg1"], + ) + return protocol_models.Account( + id=account_id, + name="Test", + is_simulated=is_simulated, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=detailed_assets, + ) + ], + ) + + +def make_exchange_config( + exchange: str = "binance", + sandboxed: bool = False, +) -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id="cfg1", + name="test", + exchange=exchange, + sandboxed=sandboxed, + historical_trade_symbols=["BTC/USDT"], + ) + + +def make_protocol_trade( + trade_id: str, + symbol: str, + side: protocol_models.Side, + quantity: float, + price: float, + executed_at: datetime.datetime, +) -> protocol_models.Trade: + return protocol_models.Trade( + id=trade_id, + trade_id=trade_id, + type=protocol_models.OrderType.LIMIT, + symbol=symbol, + side=side, + quantity=quantity, + price=price, + status=protocol_models.OrderStatus.FILLED, + executed_at=executed_at, + ) + + +def make_protocol_transaction( + tx_id: str, + asset: str, + amount: float, + tx_type: protocol_models.TransactionType, + timestamp: datetime.datetime, +) -> protocol_models.Transaction: + return protocol_models.Transaction( + id=tx_id, + timestamp=timestamp, + asset=asset, + amount=amount, + type=tx_type, + ) + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def seed_exchange_config( + account_provider: collection_providers.AccountProvider, + user_id: str, + exchange_config: protocol_models.ExchangeConfig, +) -> None: + account_provider.create_exchange_config(user_id, exchange_config) + + +def seed_account( + account_provider: collection_providers.AccountProvider, + user_id: str, + account: protocol_models.Account, +) -> None: + account_provider.create_account(user_id, account) + + +def seed_trading_state( + trading_provider: collection_providers.AccountTradingProvider, + user_id: str, + account_id: str, + *, + trades: list[protocol_models.Trade] | None = None, + transactions: list[protocol_models.Transaction] | None = None, +) -> None: + trading_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_TEST_TIMESTAMP, + trades=trades or None, + transactions=transactions or None, + ), + ) + trading_provider.save_state(user_id, account_id, trading_state) + + +async def write_daily_prices_cache( + data_root: str, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + prices_by_symbol: dict[str, dict[str, float]], +) -> None: + for symbol, closes_by_timestamp in prices_by_symbol.items(): + await trading_api.merge_daily_prices( + exchange_name, + exchange_type, + sandboxed, + symbol, + closes_by_timestamp, + data_root, + ) + + +async def write_latest_tickers_cache( + data_root: str, + exchange_name: str, + exchange_type: str, + sandboxed: bool, + closes_by_symbol: dict[str, float], +) -> None: + await trading_api.update_latest_tickers( + exchange_name, + exchange_type, + sandboxed, + closes_by_symbol, + data_root, + ) + + +@contextlib.contextmanager +def with_current_time(end_timestamp: float): + with mock.patch.object( + portfolio_value_history_module.time, + "time", + return_value=end_timestamp, + ): + yield + + +@contextlib.contextmanager +def accounts_history_test_environment(tmp_path): + account_provider = collection_providers.AccountProvider(base_folder=str(tmp_path)) + trading_provider = collection_providers.AccountTradingProvider(base_folder=str(tmp_path)) + + with ( + _patch_wallet(), + mock.patch.object( + collection_providers.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + ): + yield account_provider, trading_provider diff --git a/packages/node/tests/functional_tests/util/dag_assertions.py b/packages/node/tests/functional_tests/util/dag_assertions.py index 6c591850d4..7730324d38 100644 --- a/packages/node/tests/functional_tests/util/dag_assertions.py +++ b/packages/node/tests/functional_tests/util/dag_assertions.py @@ -11,13 +11,13 @@ import pytest import octobot_flow.enums as octobot_flow_enums_module -import octobot_node.scheduler.workflows_util as workflows_util_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module from . import workflow_common as workflow_common_module def actions_dag_from_workflow_row(workflow_row: typing.Any): - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: return None return state_reader.state.automation.actions_dag @@ -91,9 +91,9 @@ async def wait_for_workflow_matching_automation_id( deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: for workflow_row in await scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue - if workflows_util_module.get_automation_state_reader(workflow_row) is not None: + if automation_states_loader_module.get_automation_state_reader(workflow_row) is not None: return workflow_row await asyncio.sleep(poll_interval_seconds) return None @@ -109,7 +109,7 @@ async def wait_for_executable_action_ids( deadline = time.monotonic() + timeout_seconds while time.monotonic() < deadline: for workflow_row in await scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue actions_dag = actions_dag_from_workflow_row(workflow_row) if actions_dag is None: @@ -137,7 +137,7 @@ async def wait_for_dag_snapshot( last_actions_dag = None while time.monotonic() < deadline: for workflow_row in await scheduler.INSTANCE.list_workflows_async(): - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue actions_dag = actions_dag_from_workflow_row(workflow_row) if actions_dag is None: diff --git a/packages/node/tests/functional_tests/util/dca_workflow.py b/packages/node/tests/functional_tests/util/dca_workflow.py index 5a146f7812..d67f211ea2 100644 --- a/packages/node/tests/functional_tests/util/dca_workflow.py +++ b/packages/node/tests/functional_tests/util/dca_workflow.py @@ -15,6 +15,7 @@ from tests.scheduler.user_actions.user_actions_executor.util import trading_tentacles_test_utils +from . import exchange_account_elements_access as exchange_account_elements_access_module from . import price_mocks as price_mocks_module from . import workflow_common as workflow_common_module @@ -62,18 +63,11 @@ def d_order_price(price: typing.Any) -> decimal.Decimal: def open_order_origins_from_exchange_elements( exchange_account_elements: typing.Any, ) -> list[dict]: - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - return [] - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) + open_orders = exchange_account_elements_access_module.open_orders_from_elements(exchange_account_elements) storage_key = trading_constants_module.STORAGE_ORIGIN_VALUE return [ order_document[storage_key] - for order_document in (open_orders or []) + for order_document in open_orders if isinstance(order_document, dict) and storage_key in order_document ] diff --git a/packages/node/tests/functional_tests/util/exchange_account_elements_access.py b/packages/node/tests/functional_tests/util/exchange_account_elements_access.py new file mode 100644 index 0000000000..ae0ec23807 --- /dev/null +++ b/packages/node/tests/functional_tests/util/exchange_account_elements_access.py @@ -0,0 +1,162 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""Typed accessors for flow exchange account elements in functional tests.""" + +from __future__ import annotations + +import decimal +import enum +import typing + +import octobot_commons.constants as commons_constants +import octobot_flow.entities.accounts.exchange_account_elements as exchange_account_elements_module +import octobot_protocol.models as protocol_models +import octobot_trading.constants as trading_constants +import octobot_trading.enums as trading_enums + +ExchangeAccountElementsInput = ( + exchange_account_elements_module.ExchangeAccountElements + | dict[str, typing.Any] + | None +) + + +def resolve_exchange_account_elements( + elements: ExchangeAccountElementsInput, +) -> exchange_account_elements_module.ExchangeAccountElements | None: + if elements is None: + return None + if isinstance(elements, dict): + return exchange_account_elements_module.ExchangeAccountElements.from_dict(elements) + return elements + + +def open_orders_from_elements(elements: ExchangeAccountElementsInput) -> list[dict]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return [] + return list(resolved.orders.open_orders or []) + + +def trades_from_elements(elements: ExchangeAccountElementsInput) -> list[dict]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return [] + return list(resolved.trades or []) + + +def portfolio_content_from_elements(elements: ExchangeAccountElementsInput) -> dict[str, typing.Any]: + resolved = resolve_exchange_account_elements(elements) + if resolved is None: + return {} + content = resolved.portfolio.content + return content if isinstance(content, dict) else {} + + +def order_storage_payload(order_row: dict) -> dict: + if not isinstance(order_row, dict): + raise TypeError(f"expected order row dict, got {type(order_row).__name__}") + storage_origin = trading_constants.STORAGE_ORIGIN_VALUE + nested = order_row.get(storage_origin) + if isinstance(nested, dict): + return nested + return order_row + + +def order_column_value(payload: dict, column: str) -> typing.Any: + return payload.get(column) + + +def _decimal_order_price(raw: typing.Union[int, float, str, decimal.Decimal]) -> decimal.Decimal: + if isinstance(raw, decimal.Decimal): + return raw + return decimal.Decimal(str(raw)) + + +def _enum_column_value(raw: typing.Any) -> typing.Any: + if isinstance(raw, enum.Enum): + return raw.value + return raw + + +normalize_order_column_value = _enum_column_value + + +def sorted_open_limit_prices_from_elements( + elements: ExchangeAccountElementsInput, + *, + trade_order_side: trading_enums.TradeOrderSide, +) -> list[decimal.Decimal]: + side_key = trading_enums.ExchangeConstantsOrderColumns.SIDE.value + price_col = trading_enums.ExchangeConstantsOrderColumns.PRICE.value + type_col = trading_enums.ExchangeConstantsOrderColumns.TYPE.value + want_side = trade_order_side.value + limit_type = trading_enums.TradeOrderType.LIMIT.value + prices: list[decimal.Decimal] = [] + for order_row in open_orders_from_elements(elements): + if not isinstance(order_row, dict): + raise TypeError(f"expected open order dict, got {type(order_row).__name__}") + payload = order_storage_payload(order_row) + side = _enum_column_value(order_column_value(payload, side_key)) + if side != want_side: + continue + price_raw = order_column_value(payload, price_col) + if price_raw is None: + continue + order_type = _enum_column_value(order_column_value(payload, type_col)) + if order_type is not None and order_type != limit_type: + continue + prices.append(_decimal_order_price(price_raw)) + prices.sort() + return prices + + +def sorted_open_limit_prices_from_protocol_orders( + orders: typing.Iterable[protocol_models.Order] | None, + *, + trade_order_side: trading_enums.TradeOrderSide, +) -> list[decimal.Decimal]: + if trade_order_side == trading_enums.TradeOrderSide.BUY: + want_side = protocol_models.Side.BUY + else: + want_side = protocol_models.Side.SELL + prices: list[decimal.Decimal] = [] + for order in orders or []: + if not isinstance(order, protocol_models.Order): + raise TypeError(f"expected protocol Order, got {type(order).__name__}") + if order.side != want_side: + continue + if order.type != protocol_models.OrderType.LIMIT: + continue + if order.price is None: + continue + prices.append(_decimal_order_price(order.price)) + prices.sort() + return prices + + +def portfolio_row_scalar(row: typing.Any, field_name: str) -> float: + if not isinstance(row, dict): + raise AssertionError(f"portfolio row must be a dict, got {type(row).__name__}: {row!r}") + if field_name == commons_constants.PORTFOLIO_AVAILABLE: + raw_value = row.get(commons_constants.PORTFOLIO_AVAILABLE) + if raw_value is None: + return portfolio_row_scalar(row, commons_constants.PORTFOLIO_TOTAL) + elif field_name == commons_constants.PORTFOLIO_TOTAL: + raw_value = row.get(commons_constants.PORTFOLIO_TOTAL) + else: + raw_value = row.get(field_name) + if raw_value is None: + raise AssertionError(f"portfolio row missing field {field_name!r}: {row!r}") + return float(raw_value) + + +def portfolio_row_total(row: typing.Any) -> decimal.Decimal: + if not isinstance(row, dict): + raise AssertionError(f"portfolio row must be a dict, got {type(row).__name__}: {row!r}") + raw_value = row.get(commons_constants.PORTFOLIO_TOTAL) + if raw_value is None: + raise AssertionError("portfolio row has no total amount") + if isinstance(raw_value, decimal.Decimal): + return raw_value + return decimal.Decimal(str(raw_value)) diff --git a/packages/node/tests/functional_tests/util/global_view_workflow.py b/packages/node/tests/functional_tests/util/global_view_workflow.py new file mode 100644 index 0000000000..feed486b04 --- /dev/null +++ b/packages/node/tests/functional_tests/util/global_view_workflow.py @@ -0,0 +1,382 @@ +# Drakkar-Software OctoBot-Node +"""Shared helpers for global view workflow functional tests.""" + +from __future__ import annotations + +import contextlib +import datetime +import decimal +import typing + +import mock +import pytest + +import octobot.community.authentication as community_authentication_module +import octobot_commons.constants as commons_constants +import octobot_commons.user_root_folder_provider as user_root_folder_provider_module +import octobot_protocol.models as protocol_models +import octobot_sync.server as sync_server_module +import octobot_sync.sync.collection_providers as collection_providers_module +import octobot_trading.exchanges.connectors.ccxt.ccxt_connector as ccxt_connector_module +import octobot_trading.exchanges.types.rest_exchange as rest_exchange_module +import octobot_trading.enums as trading_enums + +import octobot_node.scheduler.workflows_retention as workflows_retention_module + +from tests.functional_tests.util import authenticator_mocks as authenticator_mocks_module +from tests.functional_tests.test_accounts_CRUD_operations import ( + _FUNCTIONAL_BTC_HOLDINGS, + _FUNCTIONAL_ETH_HOLDINGS, + _FUNCTIONAL_SOL_HOLDINGS, + _FUNCTIONAL_USDT_HOLDINGS, + _stub_get_balance_no_network, +) + +_TEST_PRIVATE_KEY = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" +_TEST_WALLET_PASSPHRASE = "globalViewPW1!" +_FUNCTIONAL_TIMESTAMP = datetime.datetime(2026, 4, 1, 12, 0, 0, tzinfo=datetime.UTC) +_EXCHANGE_CONFIG_ID = "global-view-functional-exchange-config" + +ACCOUNT_REAL_ID = "acc-real" +ACCOUNT_SIM_1_ID = "acc-sim-1" +ACCOUNT_SIM_2_ID = "acc-sim-2" + +AUTOMATION_FILL_ID = "automation-gv-fill" +AUTOMATION_NO_TRIGGER_ID = "automation-gv-no-trigger" +ORDER_FILL_ID = "gv-order-fill-1" +ORDER_STAYS_OPEN_ID = "gv-order-stays-open" + +WORKFLOW_RESULT_TIMEOUT_SECONDS = 120.0 + +_FUNCTIONAL_VALUATION_TICKER_CLOSE_BY_SYMBOL = { + "BTC/USDT": 50000.0, + "ETH/USDT": 3000.0, + "SOL/USDT": 100.0, +} + + +async def _stub_get_open_orders_no_network(self, symbol=None, since=None, limit=None, **kwargs): + return [] + + +async def _stub_load_symbol_markets_for_global_view(self, reload=False, market_filter=None): + self.client.markets = { + "BTC/USDT": {"symbol": "BTC/USDT", "active": True, "spot": True, "id": "BTCUSDT"}, + "ETH/USDT": {"symbol": "ETH/USDT", "active": True, "spot": True, "id": "ETHUSDT"}, + "SOL/USDT": {"symbol": "SOL/USDT", "active": True, "spot": True, "id": "SOLUSDT"}, + "SOL/BTC": {"symbol": "SOL/BTC", "active": True, "spot": True, "id": "SOLBTC"}, + } + self.client.symbols = list(self.client.markets.keys()) + + +def derive_user_id() -> str: + return sync_server_module.derive_user_id(_TEST_PRIVATE_KEY) + + +def build_exchange_config() -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id=_EXCHANGE_CONFIG_ID, + name="binance-main", + exchange="binanceus", + sandboxed=False, + ) + + +def build_exchange_account(*, remote_account_id: str) -> protocol_models.ExchangeAccount: + return protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=remote_account_id, + exchange_config_ids=[_EXCHANGE_CONFIG_ID], + ) + + +def build_simulated_account( + *, + account_id: str, + account_name: str, + usdt_total: float = 1000.0, +) -> protocol_models.Account: + return protocol_models.Account( + id=account_id, + name=account_name, + is_simulated=True, + created_at=_FUNCTIONAL_TIMESTAMP, + updated_at=_FUNCTIONAL_TIMESTAMP, + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.DetailedAsset( + symbol="USDT", + total=usdt_total, + available=usdt_total, + ) + ], + ) + ], + specifics=protocol_models.AccountSpecifics( + actual_instance=build_exchange_account(remote_account_id=account_id), + ), + ) + + +def build_real_account() -> protocol_models.Account: + return protocol_models.Account( + id=ACCOUNT_REAL_ID, + name="Global view real account", + is_simulated=False, + created_at=_FUNCTIONAL_TIMESTAMP, + updated_at=_FUNCTIONAL_TIMESTAMP, + authentication_id="global-view-functional-auth", + specifics=protocol_models.AccountSpecifics( + actual_instance=build_exchange_account(remote_account_id=ACCOUNT_REAL_ID), + ), + ) + + +def build_functional_authentication() -> protocol_models.AccountAuthentication: + return protocol_models.AccountAuthentication( + id="global-view-functional-auth", + api_key="functional-test-api-key", + api_secret="functional-test-api-secret", + ) + + +def _protocol_order( + exchange_id: str, + *, + price: float = 10000.0, + trigger_above: bool = False, +) -> protocol_models.Order: + return protocol_models.Order( + id=exchange_id, + symbol="BTC/USDT", + price=price, + quantity=0.01, + filled=0.0, + exchange_id=exchange_id, + side=protocol_models.Side.BUY, + type=protocol_models.OrderType.LIMIT, + status=protocol_models.OrderStatus.OPEN, + created_at=_FUNCTIONAL_TIMESTAMP, + trigger_above=trigger_above, + ) + + +def seed_account_trading_state( + trading_provider: collection_providers_module.AccountTradingProvider, + user_id: str, + *, + account_id: str, + seeded_orders: list[dict[str, typing.Any]], +) -> None: + protocol_orders = [ + _protocol_order( + seeded_order["exchange_id"], + price=float(seeded_order.get("price", 10000.0)), + trigger_above=bool(seeded_order.get("trigger_above", False)), + ) + for seeded_order in seeded_orders + ] + trading_provider.save_state( + user_id, + account_id, + protocol_models.AccountTradingState( + version=collection_providers_module.AccountTradingProvider.STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=_FUNCTIONAL_TIMESTAMP, + orders=protocol_orders, + trades=[], + positions=[], + ), + ), + ) + + +def build_running_automation_state( + automation_id: str, + *, + account_id: str, + order_ids: list[str], +) -> protocol_models.AutomationState: + return protocol_models.AutomationState( + id=automation_id, + status=protocol_models.WorkflowStatus.RUNNING, + metadata=protocol_models.AutomationMetadata( + name=automation_id, + description=automation_id, + ), + exchange_account_ids=[account_id], + orders=[ + protocol_models.OrderSummary(id=order_id, symbol="BTC/USDT") + for order_id in order_ids + ], + ) + + +async def enqueue_and_await_global_view_refresh() -> dict[str, typing.Any]: + import octobot_node.scheduler + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module + + scheduled_time = datetime.datetime.now(datetime.UTC) + workflow_handle = await octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE.enqueue_async( + global_view_workflow_module.GlobalViewRefreshWorkflow.global_view_refresh, + scheduled_time, + None, + ) + return await workflow_handle.get_result() + + +@contextlib.asynccontextmanager +async def global_view_functional_environment( + tmp_path, + *, + sim_ticker_close_by_symbol: dict[str, float] | None = None, +): + user_root_provider = user_root_folder_provider_module.instance() + previous_user_root = user_root_provider.get_root() + test_user_root = tmp_path / "global_view_functional_user_root" + user_root_provider.set_root(str(test_user_root)) + + authentication_instance = authenticator_mocks_module.build_community_authentication( + _TEST_PRIVATE_KEY, + _TEST_WALLET_PASSPHRASE, + ) + user_id = derive_user_id() + + account_provider = collection_providers_module.AccountProvider(base_folder=str(test_user_root)) + authentication_provider = collection_providers_module.AccountAuthenticationProvider( + base_folder=str(test_user_root), + ) + trading_provider = collection_providers_module.AccountTradingProvider(base_folder=str(test_user_root)) + history_provider = collection_providers_module.AccountHistoryProvider(base_folder=str(test_user_root)) + + configured_sim_ticker_close_by_symbol = sim_ticker_close_by_symbol or {} + + import octobot_flow.repositories.exchange.tickers_repository as tickers_repository_module + import octobot_node.scheduler.user_actions.user_actions_executor.util.account_state_updater as account_state_updater_module + + async def patched_fetch_ticker_close_by_symbol(_exchange_manager, symbols): + return { + symbol: configured_sim_ticker_close_by_symbol[symbol] + for symbol in symbols + if symbol in configured_sim_ticker_close_by_symbol + } + + async def patched_fetch_tickers(_self, symbols): + if not symbols: + return {} + close_column = trading_enums.ExchangeConstantsTickersColumns.CLOSE.value + ticker_close_by_symbol = { + **_FUNCTIONAL_VALUATION_TICKER_CLOSE_BY_SYMBOL, + **configured_sim_ticker_close_by_symbol, + } + return { + symbol: {close_column: decimal.Decimal(str(ticker_close_by_symbol[symbol]))} + for symbol in symbols + if symbol in ticker_close_by_symbol + } + + with ( + mock.patch.object( + community_authentication_module.CommunityAuthentication, + "instance", + return_value=authentication_instance, + ), + mock.patch.object( + collection_providers_module.AccountProvider, + "instance", + return_value=account_provider, + ), + mock.patch.object( + collection_providers_module.AccountAuthenticationProvider, + "instance", + return_value=authentication_provider, + ), + mock.patch.object( + collection_providers_module.AccountTradingProvider, + "instance", + return_value=trading_provider, + ), + mock.patch.object( + collection_providers_module.AccountHistoryProvider, + "instance", + return_value=history_provider, + ), + mock.patch.object( + workflows_retention_module, + "should_skip_retention_cleanup_on_this_node", + return_value=False, + ), + mock.patch.object( + account_state_updater_module, + "_fetch_api_key_rights", + mock.AsyncMock(return_value=[]), + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "load_symbol_markets", + _stub_load_symbol_markets_for_global_view, + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "get_balance", + _stub_get_balance_no_network, + ), + mock.patch.object( + ccxt_connector_module.CCXTConnector, + "get_open_orders", + _stub_get_open_orders_no_network, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "fetch_ticker_close_by_symbol", + patched_fetch_ticker_close_by_symbol, + ), + mock.patch.object( + tickers_repository_module.TickersRepository, + "fetch_tickers", + patched_fetch_tickers, + ), + ): + account_provider.create_exchange_config(user_id, build_exchange_config()) + authentication_provider.create_item(user_id, build_functional_authentication()) + account_provider.create_item(user_id, build_real_account()) + account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_1_ID, account_name="Sim 1")) + account_provider.create_item(user_id, build_simulated_account(account_id=ACCOUNT_SIM_2_ID, account_name="Sim 2")) + try: + yield { + "user_id": user_id, + "account_provider": account_provider, + "trading_provider": trading_provider, + "history_provider": history_provider, + } + finally: + user_root_provider.set_root(previous_user_root) + + +def assert_simulated_account_assets( + account: protocol_models.Account, + *, + expected_total: float, +) -> None: + assert account.assets is not None + flattened_assets: list[protocol_models.DetailedAsset] = [] + for assets_for_trading_type in account.assets: + flattened_assets.extend(assets_for_trading_type.assets or []) + assets_by_symbol = {asset.symbol: asset for asset in flattened_assets} + assert "USDT" in assets_by_symbol + assert assets_by_symbol["USDT"].total == pytest.approx(expected_total) + + +def assert_real_account_assets(account: protocol_models.Account) -> None: + assert account.assets is not None + flattened_assets: list[protocol_models.DetailedAsset] = [] + for assets_for_trading_type in account.assets: + flattened_assets.extend(assets_for_trading_type.assets or []) + assets_by_symbol = {asset.symbol: asset for asset in flattened_assets} + assert set(assets_by_symbol) == {"USDT", "BTC", "ETH", "SOL"} + assert assets_by_symbol["USDT"].total == pytest.approx(_FUNCTIONAL_USDT_HOLDINGS) + assert assets_by_symbol["BTC"].total == pytest.approx(_FUNCTIONAL_BTC_HOLDINGS) + assert assets_by_symbol["ETH"].total == pytest.approx(_FUNCTIONAL_ETH_HOLDINGS) + assert assets_by_symbol["SOL"].total == pytest.approx(_FUNCTIONAL_SOL_HOLDINGS) diff --git a/packages/node/tests/functional_tests/util/octobot_process_workflow.py b/packages/node/tests/functional_tests/util/octobot_process_workflow.py index 86aca02d30..abdc1bfe78 100644 --- a/packages/node/tests/functional_tests/util/octobot_process_workflow.py +++ b/packages/node/tests/functional_tests/util/octobot_process_workflow.py @@ -224,16 +224,16 @@ async def wait_for_init_state_ok( while time.monotonic() < deadline: workflow_rows = await scheduler.INSTANCE.list_workflows_async() for workflow_row in workflow_rows: - import octobot_node.scheduler.workflows_util as workflows_util_module + import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module if active_workflows_only and workflow_row.status not in ( dbos.WorkflowStatusString.PENDING.value, dbos.WorkflowStatusString.ENQUEUED.value, ): continue - if workflows_util_module.get_automation_id(workflow_row) != automation_id: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: continue - state_reader = workflows_util_module.get_automation_state_reader(workflow_row) + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) if state_reader is None: continue run_action = get_action_by_id(state_reader.state, GENERIC_PROCESS_ACTION_ID) diff --git a/packages/node/tests/functional_tests/util/outdated_reference_workflow.py b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py new file mode 100644 index 0000000000..0225f8bdf7 --- /dev/null +++ b/packages/node/tests/functional_tests/util/outdated_reference_workflow.py @@ -0,0 +1,170 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +"""Helpers for outdated-reference-account copy automation functional tests.""" + +from __future__ import annotations + +import asyncio +import decimal +import time +import typing + +import dbos +import pytest + +import octobot_commons.timestamp_util as timestamp_util +import octobot_copy.constants as copy_constants +import octobot_flow.entities as octobot_flow_entities +import octobot_node.scheduler.internal_trading_signals as internal_trading_signals_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_protocol.models as protocol_models +import octobot_trading.enums as trading_enums + +from . import exchange_account_elements_access as exchange_account_elements_access_module +from . import grid_workflow as grid_workflow_module +from . import workflow_common as workflow_common_module + +FRESH_MARKET_PRICE = 110_000.0 +STALE_ORDER_PRICE = 67_326.0 +FRESH_ORDER_PRICE = FRESH_MARKET_PRICE * 0.99 +MASTER_STRATEGY_ID = "functional-outdated-reference-master-strategy" +COPY_AUTOMATION_ID = "d4e5f6a7-b8c9-4901-d234-567890abcdef" +COPY_ACCOUNT_ID = "functional_outdated_reference_copy_account" +OUTDATED_SKIP_LOG_SUBSTRING = "Outdated reference account, skipping copy iteration" +_BTC_USDC = "BTC/USDC" + + +def _open_limit_order( + *, + order_id: str, + price: float, + side: protocol_models.Side = protocol_models.Side.SELL, + trigger_above: bool = True, +) -> protocol_models.Order: + return protocol_models.Order( + id=order_id, + symbol=_BTC_USDC, + price=price, + quantity=0.001, + filled=0.0, + exchange_id=f"exchange-{order_id}", + side=side, + type=protocol_models.OrderType.LIMIT, + trigger_above=trigger_above, + reduce_only=False, + is_active=True, + status=protocol_models.OrderStatus.OPEN, + created_at=timestamp_util.utc_datetime_from_timestamp(time.time()), + ) + + +def _copied_account(*, orders: list[protocol_models.Order]) -> protocol_models.CopiedAccount: + return protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=1_710_000_000.0, + copied_assets=[ + protocol_models.CopiedAsset(name="USDC", total=1000.0, available=1000.0, ratio=0.5), + protocol_models.CopiedAsset(name="BTC", total=0.01, available=0.01, ratio=0.5), + ], + orders=orders, + ) + + +def build_stale_trading_signal() -> octobot_flow_entities.TradingSignal: + return octobot_flow_entities.TradingSignal( + strategy_id=MASTER_STRATEGY_ID, + account=_copied_account( + orders=[_open_limit_order(order_id="stale-order", price=STALE_ORDER_PRICE, trigger_above=True)], + ), + ) + + +def build_fresh_trading_signal() -> octobot_flow_entities.TradingSignal: + return octobot_flow_entities.TradingSignal( + strategy_id=MASTER_STRATEGY_ID, + account=_copied_account( + orders=[_open_limit_order(order_id="fresh-order", price=FRESH_ORDER_PRICE, trigger_above=True)], + ), + ) + + +def build_create_copy_follower_user_action() -> protocol_models.UserAction: + return grid_workflow_module.build_create_copy_follower_user_action( + automation_id=COPY_AUTOMATION_ID, + account_id=COPY_ACCOUNT_ID, + name="test_outdated_reference_copy_follower", + strategy_id=MASTER_STRATEGY_ID, + ) + + +def caplog_contains_outdated_skip(caplog) -> bool: + return any( + OUTDATED_SKIP_LOG_SUBSTRING in record.getMessage() + for record in caplog.records + ) + + +async def deliver_trading_signal_when_pending( + scheduler: typing.Any, + automation_id: str, + trading_signal: octobot_flow_entities.TradingSignal, + deadline_seconds: float, +) -> None: + poll_interval = 0.05 + poll_deadline = time.monotonic() + deadline_seconds + while time.monotonic() < poll_deadline: + pending_rows = await scheduler.INSTANCE.list_workflows_async( + status=[ + dbos.WorkflowStatusString.ENQUEUED.value, + dbos.WorkflowStatusString.PENDING.value, + ], + ) + for workflow_row in pending_rows: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: + continue + copied_strategy_ids = automation_states_loader_module.get_automation_copied_strategy_ids(workflow_row) + if trading_signal.strategy_id not in copied_strategy_ids: + continue + await internal_trading_signals_module.send_internal_trading_signal(trading_signal) + return + await asyncio.sleep(poll_interval) + pytest.fail( + f"Timed out delivering trading signal for automation_id={automation_id!r} " + f"strategy_id={trading_signal.strategy_id!r}" + ) + + +def sell_limit_prices_from_reader(state_reader: typing.Any) -> list[decimal.Decimal]: + return exchange_account_elements_access_module.sorted_open_limit_prices_from_elements( + state_reader.state.automation.exchange_account_elements, + trade_order_side=trading_enums.TradeOrderSide.SELL, + ) + + +async def poll_state_reader_until( + scheduler: typing.Any, + automation_id: str, + predicate: typing.Callable[[typing.Any], bool], + deadline_seconds: float, + failure_label: str, +) -> typing.Any: + poll_interval = workflow_common_module.DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS + poll_deadline = time.monotonic() + deadline_seconds + last_reader: typing.Any = None + while time.monotonic() < poll_deadline: + workflow_rows = await scheduler.INSTANCE.list_workflows_async() + for workflow_row in workflow_rows: + if automation_states_loader_module.get_automation_id(workflow_row) != automation_id: + continue + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_row) + if state_reader is None: + continue + last_reader = state_reader + if predicate(state_reader): + return state_reader + await asyncio.sleep(poll_interval) + detail = "no reader" + if last_reader is not None: + sell_prices = sell_limit_prices_from_reader(last_reader) + detail = f"last sell limit prices={sell_prices!s}" + pytest.fail(f"Timed out waiting for {failure_label} ({detail})") diff --git a/packages/node/tests/functional_tests/util/protocol_assertions.py b/packages/node/tests/functional_tests/util/protocol_assertions.py index e5b2fd1dea..ff7d9cbff4 100644 --- a/packages/node/tests/functional_tests/util/protocol_assertions.py +++ b/packages/node/tests/functional_tests/util/protocol_assertions.py @@ -7,8 +7,10 @@ import math import typing +import octobot_commons.constants as commons_constants_module import octobot_protocol.models as protocol_models_module +from . import exchange_account_elements_access as exchange_account_elements_access_module from . import workflow_common as workflow_common_module buy_sell_trade_counts_from_exchange_elements = ( @@ -16,30 +18,6 @@ ) -def portfolio_content_from_exchange_elements(exchange_account_elements: typing.Any) -> dict[str, typing.Any]: - portfolio = getattr(exchange_account_elements, "portfolio", None) - if portfolio is None and isinstance(exchange_account_elements, dict): - portfolio = exchange_account_elements.get("portfolio") - if portfolio is None: - return {} - content = getattr(portfolio, "content", None) - if content is None and isinstance(portfolio, dict): - content = portfolio.get("content") - return content if isinstance(content, dict) else {} - - -def portfolio_row_scalar(row: typing.Any, field_name: str) -> float: - if isinstance(row, dict): - raw_value = row.get(field_name) - else: - raw_value = getattr(row, field_name, None) - if raw_value is None and field_name == "available": - return portfolio_row_scalar(row, "total") - if raw_value is None: - raise AssertionError(f"portfolio row missing field {field_name!r}: {row!r}") - return float(raw_value) - - def assert_protocol_automation_metadata_name( protocol_automation: protocol_models_module.AutomationState, expected_name: str, @@ -85,7 +63,9 @@ def assert_protocol_automation_matches_exchange_account_elements( assert order_summary.symbol == expected_order_symbol, ( f"unexpected OrderSummary.symbol {order_summary.symbol!r}; expected {expected_order_symbol!r}" ) - content = portfolio_content_from_exchange_elements(exchange_account_elements) + content = exchange_account_elements_access_module.portfolio_content_from_elements( + exchange_account_elements, + ) protocol_assets = protocol_automation.assets assert protocol_assets is not None, ( "expected AutomationState.assets to be set" @@ -97,8 +77,12 @@ def assert_protocol_automation_matches_exchange_account_elements( f"missing protocol Asset for portfolio symbol {symbol!r}; " f"protocol asset symbols: {sorted(assets_by_symbol)!r}" ) - expected_total = portfolio_row_scalar(row, "total") - expected_available = portfolio_row_scalar(row, "available") + expected_total = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_TOTAL, + ) + expected_available = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_AVAILABLE, + ) if not math.isclose(float(matching_asset.total), expected_total, rel_tol=1e-9, abs_tol=1e-6): raise AssertionError( f"Asset.total mismatch for {symbol!r}: protocol={matching_asset.total!r} " @@ -153,7 +137,9 @@ def assert_protocol_automation_matches_exchange_account_elements_multi_symbol( f"unexpected OrderSummary.symbol {order_summary.symbol!r}; " f"expected one of {allowed_order_symbols!r}" ) - content = portfolio_content_from_exchange_elements(exchange_account_elements) + content = exchange_account_elements_access_module.portfolio_content_from_elements( + exchange_account_elements, + ) protocol_assets = protocol_automation.assets assert protocol_assets is not None, "expected AutomationState.assets to be set" assets_by_symbol = {asset.symbol: asset for asset in protocol_assets} @@ -163,8 +149,12 @@ def assert_protocol_automation_matches_exchange_account_elements_multi_symbol( f"missing protocol Asset for portfolio symbol {symbol!r}; " f"protocol asset symbols: {sorted(assets_by_symbol)!r}" ) - expected_total = portfolio_row_scalar(row, "total") - expected_available = portfolio_row_scalar(row, "available") + expected_total = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_TOTAL, + ) + expected_available = exchange_account_elements_access_module.portfolio_row_scalar( + row, commons_constants_module.PORTFOLIO_AVAILABLE, + ) if not math.isclose(float(matching_asset.total), expected_total, rel_tol=1e-9, abs_tol=1e-6): raise AssertionError( f"Asset.total mismatch for {symbol!r}: protocol={matching_asset.total!r} " diff --git a/packages/node/tests/functional_tests/util/workflow_common.py b/packages/node/tests/functional_tests/util/workflow_common.py index 9531087b49..b097c68cee 100644 --- a/packages/node/tests/functional_tests/util/workflow_common.py +++ b/packages/node/tests/functional_tests/util/workflow_common.py @@ -7,6 +7,7 @@ import asyncio import datetime import json +import os import time import typing import uuid @@ -15,8 +16,9 @@ import pytest import octobot_sync.chain.evm as sync_evm_module +import octobot_sync.constants as sync_constants_module import octobot_sync.server as sync_server_module -import octobot_trading.constants as trading_constants_module +import octobot_sync.sync.collection_providers as collection_providers_module import octobot_trading.enums as trading_enums_module import octobot_node.constants as node_constants_module @@ -24,10 +26,14 @@ import octobot_node.scheduler import octobot_node.scheduler.workflows import octobot_node.scheduler.api as scheduler_api_module +import octobot_node.scheduler.tasks as scheduler_tasks_module import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module import octobot_node.scheduler.workflows_util as workflows_util_module import octobot_protocol.models as protocol_models_module +from . import exchange_account_elements_access as exchange_account_elements_access_module + # Passphrase for grid functional tests (WalletBackend requires length >= 8). SIMULATOR_GRID_TEST_WALLET_PASSPHRASE = "simgridPW1!" @@ -44,6 +50,34 @@ DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS = 0.5 DEFAULT_GRID_WORKFLOW_POLL_INTERVAL_SECONDS = DEFAULT_WORKFLOW_POLL_INTERVAL_SECONDS +_XDIST_FUNCTIONAL_TIMEOUT_SCALE = 2.25 + + +def functional_timeout_seconds(base_seconds: float) -> float: + """Scale tight poll budgets when pytest-xdist runs concurrent heavy workers.""" + if os.getenv("PYTEST_XDIST_WORKER"): + return base_seconds * _XDIST_FUNCTIONAL_TIMEOUT_SCALE + return base_seconds + + +def seed_empty_account_trading_state(user_id: str, account_id: str) -> None: + """ + Mirror CreateAccountActionExecutor: AccountTradingState must exist before + automation iterations call persist_account_trading. + Call inside auth mock context so encrypted storage can resolve wallet keys. + """ + collection_providers_module.AccountTradingProvider.instance().save_state( + user_id, + account_id, + protocol_models_module.AccountTradingState( + version=sync_constants_module.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models_module.AccountTrading( + updated_at=datetime.datetime.now(datetime.UTC), + ), + ), + ) + + _FUNCTIONAL_PROTOCOL_ACCOUNT_TS = datetime.datetime(2026, 4, 1, 12, 0, 0, tzinfo=datetime.UTC) SIMULATOR_FUNCTIONAL_STRATEGY_VERSION = "1.0.0" @@ -177,41 +211,29 @@ def job_description_dict_from_output( def buy_sell_trade_counts_from_exchange_elements( exchange_account_elements: typing.Any, ) -> tuple[int, int, int]: - if exchange_account_elements is None: + resolved = exchange_account_elements_access_module.resolve_exchange_account_elements( + exchange_account_elements, + ) + if resolved is None: return 0, 0, 0 - orders_container = getattr(exchange_account_elements, "orders", None) - if orders_container is None and isinstance(exchange_account_elements, dict): - orders_container = exchange_account_elements.get("orders") - if orders_container is None: - trades_only = getattr(exchange_account_elements, "trades", None) - if trades_only is None and isinstance(exchange_account_elements, dict): - trades_only = exchange_account_elements.get("trades", []) - return 0, 0, len(trades_only or []) - - open_orders = getattr(orders_container, "open_orders", None) - if open_orders is None and isinstance(orders_container, dict): - open_orders = orders_container.get("open_orders", []) - open_orders = open_orders or [] + open_orders = exchange_account_elements_access_module.open_orders_from_elements(resolved) side_key = trading_enums_module.ExchangeConstantsOrderColumns.SIDE.value - storage_key = trading_constants_module.STORAGE_ORIGIN_VALUE buy_count = 0 sell_count = 0 - for order in open_orders: - if isinstance(order, dict): - inner = order.get(storage_key, {}) - else: - inner = getattr(order, storage_key, {}) - side = inner.get(side_key) if isinstance(inner, dict) else getattr(inner, side_key, None) + for order_row in open_orders: + if not isinstance(order_row, dict): + raise TypeError(f"expected open order dict, got {type(order_row).__name__}") + inner = exchange_account_elements_access_module.order_storage_payload(order_row) + side = exchange_account_elements_access_module.normalize_order_column_value( + exchange_account_elements_access_module.order_column_value(inner, side_key), + ) if side == trading_enums_module.TradeOrderSide.BUY.value: buy_count += 1 elif side == trading_enums_module.TradeOrderSide.SELL.value: sell_count += 1 - trades = getattr(exchange_account_elements, "trades", None) - if trades is None and isinstance(exchange_account_elements, dict): - trades = exchange_account_elements.get("trades", []) - trade_count = len(trades or []) + trade_count = len(exchange_account_elements_access_module.trades_from_elements(resolved)) return buy_count, sell_count, trade_count @@ -285,7 +307,7 @@ async def _list_matching_automation_workflow_rows( return [ workflow_row for workflow_row in workflow_rows - if workflows_util_module.get_automation_id(workflow_row) == automation_id + if automation_states_loader_module.get_automation_id(workflow_row) == automation_id ] @@ -452,8 +474,6 @@ async def enqueue_user_action_workflow_and_await_terminal_result( user_id: str, ): """``execute_user_action`` queues user actions; wait until the USER_ACTION_QUEUE workflow completes.""" - import octobot_node.scheduler.tasks as scheduler_tasks_module - workflow_identifier_encoded = await scheduler_tasks_module.trigger_user_action_workflow( user_action_bundle, user_id, diff --git a/packages/node/tests/protocol/test_accounts_history_compute.py b/packages/node/tests/protocol/test_accounts_history_compute.py new file mode 100644 index 0000000000..666622bd6f --- /dev/null +++ b/packages/node/tests/protocol/test_accounts_history_compute.py @@ -0,0 +1,364 @@ +import datetime +import decimal +import json +import os +import mock +import pytest + +import octobot_commons.constants as commons_constants +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + +import octobot_node.protocol.accounts_history as accounts_history_module + + +def _make_account( + account_id: str, + assets: dict[str, float], + exchange: str = "binance", + sandboxed: bool = False, +) -> protocol_models.Account: + detailed_assets = [ + protocol_models.DetailedAsset(symbol=symbol, total=amount, available=amount) + for symbol, amount in assets.items() + ] + exchange_config = protocol_models.ExchangeConfig( + id="cfg1", name="test", exchange=exchange, sandboxed=sandboxed, + historical_trade_symbols=["BTC/USDT"], + ) + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote1", + exchange_config_ids=["cfg1"], + ) + return protocol_models.Account( + id=account_id, + name="Test", + is_simulated=False, + created_at=datetime.datetime.now(datetime.timezone.utc), + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + assets=[ + protocol_models.DetailedAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=detailed_assets, + ) + ], + ) + + +class TestPortfolioFromAccountAssets: + def test_converts_assets_to_portfolio_dict(self): + account = _make_account("a1", {"BTC": 1.5, "USDT": 10000.0}) + portfolio = accounts_history_module._portfolio_from_account_assets(account) + assert portfolio["BTC"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("1.5") + assert portfolio["USDT"][commons_constants.PORTFOLIO_TOTAL] == decimal.Decimal("10000.0") + + def test_empty_assets(self): + account = mock.MagicMock(spec=protocol_models.Account) + account.assets = None + portfolio = accounts_history_module._portfolio_from_account_assets(account) + assert portfolio == {} + + +class TestComputePortfolioHistoricalValues: + @pytest.mark.asyncio + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + async def test_empty_trading_returns_empty_history(self, mock_trading_provider, mock_account_provider, tmp_path): + account = _make_account("a1", {"BTC": 1.0}) + mock_account_provider.instance.return_value.get_account.return_value = account + mock_account_provider.instance.return_value.get_exchange_config.return_value = protocol_models.ExchangeConfig( + id="cfg1", name="test", exchange="binance", sandboxed=False, + historical_trade_symbols=["BTC/USDT"], + ) + + trading_state = mock.MagicMock() + trading_state.account_trading.trades = [] + trading_state.account_trading.transactions = [] + mock_trading_provider.instance.return_value.load_state.return_value = trading_state + + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + "user1", "a1", + ) + assert result.version == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + # No trades/transactions → no history values. + if result.history is not None: + assert result.history.values is None or len(result.history.values) == 0 + + +class TestBuildPortfolioHistoricalValues: + @pytest.mark.asyncio + @mock.patch( + "octobot_flow.logic.portfolio_history.portfolio_value_history.compute_daily_portfolio_values", + ) + @mock.patch("octobot_trading.api.load_latest_tickers", new_callable=mock.AsyncMock) + @mock.patch("octobot_trading.api.load_daily_prices", new_callable=mock.AsyncMock) + @mock.patch( + "octobot_trading.api.compute_portfolio_historical_holdings_from_latest_portfolio_trades_and_transations", + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountTradingProvider") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_maps_day_assets_to_protocol_models( + self, + mock_account_provider, + mock_trading_provider, + mock_daily_holdings, + mock_load_daily_prices, + mock_load_latest_tickers, + mock_compute_daily_values, + ): + account = _make_account("a1", {"BTC": 1.0, "USDT": 100.0}) + mock_account_provider.instance.return_value.get_account.return_value = account + mock_account_provider.instance.return_value.get_exchange_config.return_value = protocol_models.ExchangeConfig( + id="cfg1", name="test", exchange="binance", sandboxed=False, + historical_trade_symbols=["BTC/USDT"], + ) + trading_state = mock.MagicMock() + trading_state.account_trading.trades = [mock.MagicMock()] + trading_state.account_trading.transactions = [] + mock_trading_provider.instance.return_value.load_state.return_value = trading_state + mock_daily_holdings.return_value = {86400.0: {"BTC": {"total": decimal.Decimal("1")}}} + mock_load_daily_prices.return_value = {} + mock_load_latest_tickers.return_value = {} + mock_compute_daily_values.return_value = [ + protocol_models.PortfolioHistoricalValue( + timestamp=datetime.datetime.fromtimestamp(86400.0, tz=datetime.timezone.utc), + total=41000.0, + assets=[ + protocol_models.HistoricalAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.HistoricalAssetValue( + symbol="BTC", holdings=1.0, value=40000.0, + ), + protocol_models.HistoricalAssetValue( + symbol="USDT", holdings=1000.0, value=1000.0, + ), + ], + ) + ], + ) + ] + + result = await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + "user1", "a1", + ) + + history_value = result.history.values[0] + assert history_value.assets is not None + spot_assets = history_value.assets[0] + assert spot_assets.trading_type == protocol_models.TradingType.SPOT + assets_by_symbol = {asset.symbol: asset for asset in spot_assets.assets} + assert assets_by_symbol["BTC"].holdings == pytest.approx(1.0) + assert assets_by_symbol["BTC"].value == pytest.approx(40000.0) + assert assets_by_symbol["USDT"].holdings == pytest.approx(1000.0) + assert assets_by_symbol["USDT"].value == pytest.approx(1000.0) + + +def _spot_assets( + *assets: tuple[str, float, float], +) -> list[protocol_models.HistoricalAssetsForTradingType]: + return [ + protocol_models.HistoricalAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.HistoricalAssetValue( + symbol=symbol, + holdings=holdings, + value=value, + ) + for symbol, holdings, value in assets + ], + ) + ] + + +def _spot_assets_by_symbol( + history_value: protocol_models.PortfolioHistoricalValue, +) -> dict[str, protocol_models.HistoricalAssetValue]: + assert history_value.assets is not None + spot_assets = history_value.assets[0] + return {asset.symbol: asset for asset in spot_assets.assets} + + +class TestAggregatePortfolioHistoricalValues: + def test_sums_totals_for_overlapping_and_non_overlapping_days(self): + day_one = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + day_two = datetime.datetime(2024, 1, 2, tzinfo=datetime.timezone.utc) + day_three = datetime.datetime(2024, 1, 3, tzinfo=datetime.timezone.utc) + account_a_history = [ + protocol_models.PortfolioHistoricalValue(timestamp=day_one, total=100.0), + protocol_models.PortfolioHistoricalValue(timestamp=day_two, total=200.0), + ] + account_b_history = [ + protocol_models.PortfolioHistoricalValue(timestamp=day_two, total=50.0), + protocol_models.PortfolioHistoricalValue(timestamp=day_three, total=75.0), + ] + + aggregated = accounts_history_module._aggregate_portfolio_historical_values( + [account_a_history, account_b_history], + ) + + assert len(aggregated) == 3 + assert aggregated[0].total == pytest.approx(100.0) + assert aggregated[1].total == pytest.approx(250.0) + assert aggregated[2].total == pytest.approx(75.0) + assert aggregated[0].assets is None + + def test_sums_accounts_on_same_utc_day_with_misaligned_timestamps(self): + day_midnight = datetime.datetime(2026, 2, 26, 0, 0, tzinfo=datetime.timezone.utc) + day_sixteen_hundred = datetime.datetime(2026, 2, 26, 16, 0, tzinfo=datetime.timezone.utc) + account_a_history = [ + protocol_models.PortfolioHistoricalValue(timestamp=day_midnight, total=425.60), + ] + account_b_history = [ + protocol_models.PortfolioHistoricalValue(timestamp=day_sixteen_hundred, total=-237.20), + ] + + aggregated = accounts_history_module._aggregate_portfolio_historical_values( + [account_a_history, account_b_history], + ) + + assert len(aggregated) == 1 + assert aggregated[0].timestamp == day_midnight + assert aggregated[0].total == pytest.approx(188.4) + + def test_sums_assets_by_symbol_across_accounts(self): + day_one = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + account_a_history = [ + protocol_models.PortfolioHistoricalValue( + timestamp=day_one, + total=40100.0, + assets=_spot_assets(("BTC", 1.0, 40000.0), ("USDT", 100.0, 100.0)), + ), + ] + account_b_history = [ + protocol_models.PortfolioHistoricalValue( + timestamp=day_one, + total=23000.0, + assets=_spot_assets(("BTC", 0.5, 20000.0), ("ETH", 2.0, 3000.0)), + ), + ] + + aggregated = accounts_history_module._aggregate_portfolio_historical_values( + [account_a_history, account_b_history], + ) + + assert len(aggregated) == 1 + assert aggregated[0].total == pytest.approx(63100.0) + assets_by_symbol = _spot_assets_by_symbol(aggregated[0]) + assert assets_by_symbol["BTC"].holdings == pytest.approx(1.5) + assert assets_by_symbol["BTC"].value == pytest.approx(60000.0) + assert assets_by_symbol["ETH"].holdings == pytest.approx(2.0) + assert assets_by_symbol["ETH"].value == pytest.approx(3000.0) + assert assets_by_symbol["USDT"].holdings == pytest.approx(100.0) + assert assets_by_symbol["USDT"].value == pytest.approx(100.0) + + def test_buckets_misaligned_timestamps_and_assets_on_same_utc_day(self): + day_midnight = datetime.datetime(2026, 2, 26, 0, 0, tzinfo=datetime.timezone.utc) + day_sixteen_hundred = datetime.datetime(2026, 2, 26, 16, 0, tzinfo=datetime.timezone.utc) + account_a_history = [ + protocol_models.PortfolioHistoricalValue( + timestamp=day_midnight, + total=40000.0, + assets=_spot_assets(("BTC", 1.0, 40000.0)), + ), + ] + account_b_history = [ + protocol_models.PortfolioHistoricalValue( + timestamp=day_sixteen_hundred, + total=20000.0, + assets=_spot_assets(("BTC", 0.5, 20000.0)), + ), + ] + + aggregated = accounts_history_module._aggregate_portfolio_historical_values( + [account_a_history, account_b_history], + ) + + assert len(aggregated) == 1 + assets_by_symbol = _spot_assets_by_symbol(aggregated[0]) + assert assets_by_symbol["BTC"].holdings == pytest.approx(1.5) + assert assets_by_symbol["BTC"].value == pytest.approx(60000.0) + + def test_keeps_assets_from_accounts_that_have_breakdown_when_other_has_none(self): + day_one = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + account_a_history = [ + protocol_models.PortfolioHistoricalValue(timestamp=day_one, total=100.0, assets=None), + ] + account_b_history = [ + protocol_models.PortfolioHistoricalValue( + timestamp=day_one, + total=50.0, + assets=_spot_assets(("USDT", 50.0, 50.0)), + ), + ] + + aggregated = accounts_history_module._aggregate_portfolio_historical_values( + [account_a_history, account_b_history], + ) + + assert aggregated[0].total == pytest.approx(150.0) + assets_by_symbol = _spot_assets_by_symbol(aggregated[0]) + assert assets_by_symbol["USDT"].holdings == pytest.approx(50.0) + assert assets_by_symbol["USDT"].value == pytest.approx(50.0) + + +class TestComputeAggregatedPortfolioHistoricalValues: + @pytest.mark.asyncio + @mock.patch( + "octobot_node.protocol.accounts_history.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions", + new_callable=mock.AsyncMock, + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_filters_accounts_by_is_simulated_and_aggregates( + self, + mock_account_provider, + mock_compute_per_account, + ): + real_account = _make_account("real-1", {"USDT": 100.0}) + simulated_account = _make_account("sim-1", {"USDT": 200.0}) + simulated_account.is_simulated = True + mock_account_provider.instance.return_value.list_accounts.return_value = [ + real_account, + simulated_account, + ] + day_one = datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc) + mock_compute_per_account.side_effect = [ + protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDT", + values=[protocol_models.PortfolioHistoricalValue(timestamp=day_one, total=100.0)], + ), + ), + protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDT", + values=[protocol_models.PortfolioHistoricalValue(timestamp=day_one, total=200.0)], + ), + ), + ] + + result = await accounts_history_module.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + "user1", + is_simulated=False, + ) + + assert mock_compute_per_account.await_count == 1 + mock_compute_per_account.assert_awaited_with("user1", "real-1", data_root=None) + assert result.history is not None + assert result.history.values[0].total == pytest.approx(100.0) + + @pytest.mark.asyncio + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_returns_empty_state_when_no_matching_accounts(self, mock_account_provider): + mock_account_provider.instance.return_value.list_accounts.return_value = [] + + result = await accounts_history_module.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + "user1", + is_simulated=True, + ) + + assert result.history is None diff --git a/packages/node/tests/protocol/test_accounts_trading.py b/packages/node/tests/protocol/test_accounts_trading.py index dac79c4d70..bb38972c08 100644 --- a/packages/node/tests/protocol/test_accounts_trading.py +++ b/packages/node/tests/protocol/test_accounts_trading.py @@ -288,3 +288,75 @@ def test_repeated_update_dedupes_process_bot_trade_without_exchange_trade_id(sel assert saved_state.account_trading.trades is not None assert len(saved_state.account_trading.trades) == 1 assert saved_state.account_trading.trades[0].trade_id == "local-trade-1" + + +class TestResetAccountTradingData: + """Checks :func:`octobot_node.protocol.accounts_trading.reset_account_trading_data`.""" + + def test_creates_empty_state_when_trading_file_is_missing(self): + provider_stub = mock.Mock() + provider_stub.load_state = mock.Mock( + side_effect=collection_errors.CollectionNoDataError("missing trading state"), + ) + with mock.patch.object( + accounts_trading_module.trading_provider.AccountTradingProvider, + "instance", + return_value=provider_stub, + ): + accounts_trading_module.reset_account_trading_data( + _TEST_WALLET_ADDRESS, + _TEST_ACCOUNT_ID, + ) + provider_stub.save_state.assert_called_once() + saved_state = provider_stub.save_state.call_args[0][2] + account_trading = saved_state.account_trading + assert account_trading.orders is None + assert account_trading.trades is None + assert account_trading.transactions is None + assert account_trading.positions is None + + def test_clears_existing_orders_trades_transactions_and_positions(self): + fixture_time = datetime.datetime(2026, 1, 10, tzinfo=datetime.UTC) + existing_state = protocol_models.AccountTradingState( + version=sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION, + account_trading=protocol_models.AccountTrading( + updated_at=fixture_time, + orders=[_sample_order("ord-old")], + trades=[ + accounts_trading_module.trades_protocol.to_protocol_trade( + _exchange_trade_dict("trade-existing"), + ), + ], + positions=[ + accounts_trading_module.positions_protocol.to_protocol_position( + _exchange_position_dict("pos-old"), + ), + ], + transactions=[ + protocol_models.Transaction( + id="tx-1", + timestamp=fixture_time, + asset="SOL", + amount=200.0, + type=protocol_models.TransactionType.BLOCKCHAIN_WITHDRAWAL, + ), + ], + ), + ) + provider_stub = mock.Mock() + provider_stub.load_state = mock.Mock(return_value=existing_state) + with mock.patch.object( + accounts_trading_module.trading_provider.AccountTradingProvider, + "instance", + return_value=provider_stub, + ): + accounts_trading_module.reset_account_trading_data( + _TEST_WALLET_ADDRESS, + _TEST_ACCOUNT_ID, + ) + saved_state = provider_stub.save_state.call_args[0][2] + account_trading = saved_state.account_trading + assert account_trading.orders is None + assert account_trading.trades is None + assert account_trading.transactions is None + assert account_trading.positions is None diff --git a/packages/node/tests/protocol/test_automations.py b/packages/node/tests/protocol/test_automations.py index 610b294fcc..f0a8768c2b 100644 --- a/packages/node/tests/protocol/test_automations.py +++ b/packages/node/tests/protocol/test_automations.py @@ -757,6 +757,76 @@ def test_assets_without_enrichment(self): assert filled.assets[0].available == 1.0 +class TestAutomationTradeSummariesFromExchangeElements: + def test_union_of_archived_and_full_trades(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["old-1", "old-2"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "live-1", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + summaries = automations_protocol._automation_trade_summaries(elements) + summary_ids = {summary.id for summary in summaries} + assert summary_ids == {"old-1", "old-2", "live-1"} + + def test_full_trade_overrides_archived_summary_for_same_id(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["shared-id"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "shared-id", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "shared-id" + assert summaries[0].symbol == "ETH/USDT" + + def test_only_trade_summaries_dict_populated(self): + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["archived-1"]}, + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "archived-1" + assert summaries[0].symbol == "BTC/USDT" + + def test_skips_empty_symbol_or_trade_id_entries(self): + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"": ["id-1"], "BTC/USDT": ["", "valid-id"]}, + ) + summaries = automations_protocol._automation_trade_summaries(elements) + assert len(summaries) == 1 + assert summaries[0].id == "valid-id" + + def test_fill_protocol_state_counts_archived_and_live_trades(self): + order_columns = octobot_trading_enums.ExchangeConstantsOrderColumns + elements = flow_entities.ExchangeAccountElements( + trade_summaries={"BTC/USDT": ["old-1"]}, + trades=[ + { + order_columns.EXCHANGE_TRADE_ID.value: "live-1", + order_columns.SYMBOL.value: "ETH/USDT", + }, + ], + ) + automation = _minimal_automation_details() + automation.exchange_account_elements = elements + flow_state = flow_entities.AutomationState(automation=automation) + filled = automations_protocol._fill_protocol_automation_state(_minimal_protocol_base(), flow_state) + assert filled.trades is not None + assert len(filled.trades) == 2 + trade_ids = {trade.id for trade in filled.trades} + assert trade_ids == {"old-1", "live-1"} + + class TestFillProtocolAutomationStateEmpties: def test_no_exchange_elements_yields_empty_protocol_lists(self): flow_state = flow_entities.AutomationState(automation=_minimal_automation_details()) diff --git a/packages/node/tests/scheduler/__init__.py b/packages/node/tests/scheduler/__init__.py index 9d08455d04..e8830819ab 100644 --- a/packages/node/tests/scheduler/__init__.py +++ b/packages/node/tests/scheduler/__init__.py @@ -91,6 +91,8 @@ def _ensure_scheduler_queues() -> None: octobot_node_enums_module.SchedulerQueues.AUTOMATION_WORKFLOW_QUEUE.value: "AUTOMATION_WORKFLOW_QUEUE", octobot_node_enums_module.SchedulerQueues.USER_ACTION_QUEUE.value: "USER_ACTION_QUEUE", octobot_node_enums_module.SchedulerQueues.DBOS_CLEANUP_QUEUE.value: "DBOS_CLEANUP_QUEUE", + octobot_node_enums_module.SchedulerQueues.GLOBAL_VIEW_QUEUE.value: "GLOBAL_VIEW_QUEUE", + octobot_node_enums_module.SchedulerQueues.PORTFOLIO_HISTORY_QUEUE.value: "PORTFOLIO_HISTORY_QUEUE", } if all(queue_name in registry.queue_info_map for queue_name in queue_bindings): for queue_name, scheduler_attribute in queue_bindings.items(): @@ -115,6 +117,8 @@ def destroy_launched_dbos(*, destroy_registry: bool = False) -> None: octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE = None octobot_node.scheduler.SCHEDULER.USER_ACTION_QUEUE = None octobot_node.scheduler.SCHEDULER.DBOS_CLEANUP_QUEUE = None + octobot_node.scheduler.SCHEDULER.GLOBAL_VIEW_QUEUE = None + octobot_node.scheduler.SCHEDULER.PORTFOLIO_HISTORY_QUEUE = None def init_scheduler(db_file_name: str, application_version: str | None = None): diff --git a/packages/node/tests/scheduler/automations/test_automation_states_loader.py b/packages/node/tests/scheduler/automations/test_automation_states_loader.py new file mode 100644 index 0000000000..a50653d98b --- /dev/null +++ b/packages/node/tests/scheduler/automations/test_automation_states_loader.py @@ -0,0 +1,422 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import json + +import dbos +import mock +import octobot_flow.entities +import pytest + +import octobot_protocol.models as protocol_models +import octobot_node.models as node_models +import octobot_node.protocol.automations as automations_protocol +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.workflows.params as workflow_params + + +_PARENT_WORKFLOW_ID = "741ce171-dac9-40be-83dc-b443c0eaf0e2" +_LOADER_PARENT_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" + + +def _sample_postpone_dag_actions() -> list[dict]: + return [ + {"id": "action_dsl_1", "dsl_script": "1 if True else 2"}, + { + "id": "action_dsl_2", + "dsl_script": "1 if True else 2", + "dependencies": [{"action_id": "action_dsl_1"}], + }, + ] + + +def _actions_dag_from_task_content(task_content: str) -> dict: + return json.loads(task_content)["state"]["automation"]["actions_dag"] + + +def _canonical_actions_dag(actions_dag: dict) -> dict: + automation_state = octobot_flow.entities.AutomationState.from_dict({ + "automation": { + "metadata": {"automation_id": "canonical"}, + "actions_dag": actions_dag, + "execution": {}, + } + }) + return automation_state.to_dict(include_default_values=False)["automation"]["actions_dag"] + + +def _assert_actions_dag_equal(actions_dag_left: dict, actions_dag_right: dict) -> None: + assert _canonical_actions_dag(actions_dag_left) == _canonical_actions_dag(actions_dag_right) + + +def _automation_task_content(*, automation_name: str, automation_id: str = "automation_1") -> str: + return json.dumps( + { + automation_states_loader_module.STATE_KEY: { + "automation": { + "metadata": { + "automation_id": automation_id, + "name": automation_name, + }, + "actions_dag": {"actions": _sample_postpone_dag_actions()}, + "execution": {}, + }, + }, + } + ) + + +def _workflow_status_with_automation_task( + *, + status: str, + input_content: str, + output_content: str | None = None, +) -> mock.Mock: + task = node_models.Task( + name="automation-task", + content=input_content, + type=node_models.TaskType.EXECUTE_ACTIONS.value, + ) + encoded_inputs = workflow_params.AutomationWorkflowInputs(task=task).to_dict( + include_default_values=False + ) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = "parent-workflow-id_1" + workflow_status.status = status + workflow_status.input = {"args": [encoded_inputs], "kwargs": {}} + workflow_status.error = None + if output_content is None: + workflow_status.output = None + else: + workflow_status.output = json.dumps( + workflow_params.AutomationWorkflowOutput(state=output_content).to_dict( + include_default_values=False + ) + ) + return workflow_status + + +def _running_automation_task_content() -> str: + state_dict = { + "automation": { + "metadata": {"automation_id": "automation_1"}, + "actions_dag": { + "actions": [{"id": "a1", "dsl_script": "True"}], + }, + "execution": { + "current_execution": {"scheduled_to": 1, "triggered_at": 2}, + }, + }, + } + return json.dumps({"state": state_dict}) + + +def _build_mock_workflow_status( + task: node_models.Task, + encrypted_state: str, + state_metadata: str, + workflow_id: str = _LOADER_PARENT_ID, +) -> mock.Mock: + output = workflow_params.AutomationWorkflowOutput(state=encrypted_state, state_metadata=state_metadata) + inputs = workflow_params.AutomationWorkflowInputs(task=task, execution_time=0) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.name = "test-task" + workflow_status.status = dbos.WorkflowStatusString.SUCCESS.value + workflow_status.output = json.dumps(output.to_dict()) + workflow_status.input = {"args": [inputs.to_dict()], "kwargs": {}} + workflow_status.created_at = None + workflow_status.updated_at = None + workflow_status.error = None + return workflow_status + + +def _build_mock_workflow_status_error( + task: node_models.Task, + error: Exception, + workflow_id: str = _LOADER_PARENT_ID, +) -> mock.Mock: + inputs = workflow_params.AutomationWorkflowInputs(task=task, execution_time=0) + workflow_status = mock.Mock(spec=dbos.WorkflowStatus) + workflow_status.workflow_id = workflow_id + workflow_status.name = "test-task" + workflow_status.status = dbos.WorkflowStatusString.ERROR.value + workflow_status.output = None + workflow_status.error = error + workflow_status.input = {"args": [inputs.to_dict()], "kwargs": {}} + workflow_status.created_at = None + workflow_status.updated_at = 2 + return workflow_status + + +class TestGetAutomationStateDict: + def test_success_workflow_state_dict_matches_output(self): + input_content = _automation_task_content(automation_name="from-input") + output_content = _automation_task_content(automation_name="from-output") + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.SUCCESS.value, + input_content=input_content, + output_content=output_content, + ) + + state_dict = automation_states_loader_module.get_automation_state_dict(workflow_status) + + assert state_dict is not None + assert state_dict["automation"]["metadata"]["name"] == "from-output" + + +class TestPatchTaskContentDegradedState: + def test_persists_degraded_state_in_task_content(self): + task_content = _automation_task_content(automation_name="copy-grid") + actions_dag_before = _actions_dag_from_task_content(task_content) + + patched_content = automation_states_loader_module.patch_task_content_degraded_state( + task_content, + "not_enough_funds", + "Insufficient funds", + since=1234.5, + ) + + _assert_actions_dag_equal(_actions_dag_from_task_content(patched_content), actions_dag_before) + degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] + assert degraded_state == { + "since": 1234.5, + "error": "not_enough_funds", + "reason": "Insufficient funds", + } + + def test_preserves_existing_degraded_since_on_subsequent_patch(self): + task_content = _automation_task_content(automation_name="copy-grid") + actions_dag_before = _actions_dag_from_task_content(task_content) + task_content = automation_states_loader_module.patch_task_content_degraded_state( + task_content, + "not_enough_funds", + "Insufficient funds", + since=1000.0, + ) + _assert_actions_dag_equal(_actions_dag_from_task_content(task_content), actions_dag_before) + + patched_content = automation_states_loader_module.patch_task_content_degraded_state( + task_content, + "invalid_order", + "Order volume below exchange minimum", + since=2000.0, + ) + + _assert_actions_dag_equal(_actions_dag_from_task_content(patched_content), actions_dag_before) + degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] + assert degraded_state == { + "since": 1000.0, + "error": "invalid_order", + "reason": "Order volume below exchange minimum", + } + + +class TestGetAutomationId: + def test_returns_metadata_automation_id(self): + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.PENDING.value, + input_content=_automation_task_content( + automation_name="dca", + automation_id="automation-dca", + ), + ) + + assert automation_states_loader_module.get_automation_id(workflow_status) == "automation-dca" + + +class TestGetAutomationStateReader: + def test_reader_exposes_automation_name(self): + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.PENDING.value, + input_content=_automation_task_content(automation_name="grid-bot"), + ) + + state_reader = automation_states_loader_module.get_automation_state_reader(workflow_status) + + assert state_reader is not None + state_dict = automation_states_loader_module.get_automation_state_dict(workflow_status) + assert state_dict is not None + assert state_dict["automation"]["metadata"]["name"] == "grid-bot" + + def test_copied_strategy_ids_empty_when_none_configured(self): + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.PENDING.value, + input_content=_automation_task_content(automation_name="grid-bot"), + ) + + assert automation_states_loader_module.get_automation_copied_strategy_ids(workflow_status) == [] + + +class TestParseFlowAutomationState: + def test_parses_task_content_into_flow_automation_state(self): + task = node_models.Task( + name="automation-task", + content=_automation_task_content(automation_name="parsed-bot"), + type=node_models.TaskType.EXECUTE_ACTIONS.value, + ) + + flow_automation_state = automation_states_loader_module.parse_flow_automation_state(task) + + assert flow_automation_state.automation.metadata.automation_id == "automation_1" + + +class TestLoadFlowAutomationStatesById: + @pytest.mark.asyncio + async def test_builds_flow_states_from_sources(self): + task = node_models.Task( + id=_LOADER_PARENT_ID, + name="automation-task", + content=_automation_task_content(automation_name="loaded-bot"), + type=node_models.TaskType.EXECUTE_ACTIONS.value, + ) + source = automations_protocol.AutomationStateSource( + task=task, + workflow_status=dbos.WorkflowStatusString.PENDING.value, + ) + with mock.patch.object( + automation_states_loader_module, + "load_automation_state_sources", + new=mock.AsyncMock(return_value=[source]), + ): + flow_states_by_id = await automation_states_loader_module.load_flow_automation_states_by_id( + "wallet-id", + ) + + assert flow_states_by_id[_LOADER_PARENT_ID].automation.metadata.automation_id == "automation_1" + + +class TestLoadProtocolAutomationStates: + @pytest.mark.asyncio + async def test_error_workflow_reports_failed_not_running(self): + task = node_models.Task( + id=_LOADER_PARENT_ID, + name="failed-automation", + content=_running_automation_task_content(), + type="execute_actions", + ) + error_workflow = _build_mock_workflow_status_error( + task, + RuntimeError("DBOSUnexpectedStepError"), + ) + with mock.patch( + "octobot_node.scheduler.SCHEDULER._get_latest_workflow_for_each_automation", + new=mock.AsyncMock(return_value=[error_workflow]), + ): + automation_states = await automation_states_loader_module.load_protocol_automation_states(None) + + assert len(automation_states) == 1 + assert automation_states[0].id == _LOADER_PARENT_ID + assert automation_states[0].status == protocol_models.WorkflowStatus.FAILED + assert "DBOSUnexpectedStepError" in (automation_states[0].error or "") + assert automation_states[0].error_message is None + + @pytest.mark.asyncio + async def test_success_workflow_with_output_preserves_metadata_name(self): + task = node_models.Task( + id=_LOADER_PARENT_ID, + name="my-automation", + content=_running_automation_task_content(), + type="execute_actions", + ) + success_workflow = _build_mock_workflow_status( + task, + encrypted_state=_running_automation_task_content(), + state_metadata="", + ) + with mock.patch( + "octobot_node.scheduler.SCHEDULER._get_latest_workflow_for_each_automation", + new=mock.AsyncMock(return_value=[success_workflow]), + ): + automation_states = await automation_states_loader_module.load_protocol_automation_states(None) + + assert len(automation_states) == 1 + assert automation_states[0].id == _LOADER_PARENT_ID + assert automation_states[0].metadata.name == "my-automation" + + +class TestLoadWalletAutomationStates: + @pytest.mark.asyncio + async def test_loads_protocol_and_flow_states_from_single_source_fetch(self): + task = node_models.Task( + id=_LOADER_PARENT_ID, + name="wallet-automation", + content=_automation_task_content(automation_name="wallet-bot"), + type=node_models.TaskType.EXECUTE_ACTIONS.value, + ) + source = automations_protocol.AutomationStateSource( + task=task, + workflow_status=dbos.WorkflowStatusString.PENDING.value, + ) + load_sources_mock = mock.AsyncMock(return_value=[source]) + with mock.patch.object( + automation_states_loader_module, + "load_automation_state_sources", + new=load_sources_mock, + ): + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states( + "wallet-id", + ) + + load_sources_mock.assert_awaited_once_with("wallet-id", None, load_output=True) + assert len(wallet_automation_states.protocol_states) == 1 + assert wallet_automation_states.protocol_states[0].id == _LOADER_PARENT_ID + assert wallet_automation_states.flow_states_by_id[_LOADER_PARENT_ID].automation.metadata.automation_id == "automation_1" + + +class TestLoadWalletAutomationStatesForTradeSymbols: + @pytest.mark.asyncio + async def test_uses_enqueued_pending_statuses_and_skips_workflow_outputs(self): + get_latest_mock = mock.AsyncMock(return_value=[]) + scheduler_mock = mock.Mock() + scheduler_mock._get_latest_workflow_for_each_automation = get_latest_mock + with mock.patch( + "octobot_node.scheduler.SCHEDULER", + scheduler_mock, + ): + wallet_automation_states = await automation_states_loader_module.load_wallet_automation_states_for_trade_symbols( + "wallet-id", + ) + + get_latest_mock.assert_awaited_once_with( + "wallet-id", + [ + dbos.WorkflowStatusString.ENQUEUED, + dbos.WorkflowStatusString.PENDING, + ], + load_output=False, + ) + assert wallet_automation_states.protocol_states == [] + assert wallet_automation_states.flow_states_by_id == {} + + +class TestGetAutomationWorkflowStatus: + @pytest.mark.asyncio + async def test_returns_matching_pending_workflow(self): + workflow_status = _workflow_status_with_automation_task( + status=dbos.WorkflowStatusString.PENDING.value, + input_content=_automation_task_content( + automation_name="pending-bot", + automation_id="automation-pending", + ), + ) + with mock.patch.object( + dbos.DBOS, + "list_workflows_async", + new=mock.AsyncMock(return_value=[workflow_status]), + ): + resolved_workflow = await automation_states_loader_module.get_automation_workflow_status( + "automation-pending", + ) + + assert resolved_workflow is workflow_status + + @pytest.mark.asyncio + async def test_raises_when_no_workflow_matches(self): + with mock.patch.object( + dbos.DBOS, + "list_workflows_async", + new=mock.AsyncMock(return_value=[]), + ): + with pytest.raises(ValueError, match="No automation workflow found"): + await automation_states_loader_module.get_automation_workflow_status("missing-automation") diff --git a/packages/node/tests/scheduler/test_generic_process_octobot.py b/packages/node/tests/scheduler/automations/test_generic_process_octobot.py similarity index 99% rename from packages/node/tests/scheduler/test_generic_process_octobot.py rename to packages/node/tests/scheduler/automations/test_generic_process_octobot.py index 0c489efc0b..80423a14ca 100644 --- a/packages/node/tests/scheduler/test_generic_process_octobot.py +++ b/packages/node/tests/scheduler/automations/test_generic_process_octobot.py @@ -18,7 +18,7 @@ import pytest import uuid -from octobot_node.scheduler.generic_process_octobot import create_generic_process_bot +from octobot_node.scheduler.automations.generic_process_octobot import create_generic_process_bot from tests.scheduler import temp_dbos_scheduler diff --git a/packages/node/tests/scheduler/test_octobot_flow_client.py b/packages/node/tests/scheduler/automations/test_octobot_flow_client.py similarity index 99% rename from packages/node/tests/scheduler/test_octobot_flow_client.py rename to packages/node/tests/scheduler/automations/test_octobot_flow_client.py index 4cd1b350c3..60d48077d9 100644 --- a/packages/node/tests/scheduler/test_octobot_flow_client.py +++ b/packages/node/tests/scheduler/automations/test_octobot_flow_client.py @@ -28,7 +28,7 @@ import octobot_trading.enums as trading_enums import octobot_trading.blockchain_wallets.simulator.blockchain_wallet_simulator as blockchain_wallet_simulator import octobot_trading.personal_data.orders.order_factory as order_factory -import octobot_node.scheduler.octobot_flow_client as octobot_flow_client +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client RUN_TESTS = True diff --git a/packages/node/tests/scheduler/automations/test_trade_symbols_resolver.py b/packages/node/tests/scheduler/automations/test_trade_symbols_resolver.py new file mode 100644 index 0000000000..e7f26d5619 --- /dev/null +++ b/packages/node/tests/scheduler/automations/test_trade_symbols_resolver.py @@ -0,0 +1,281 @@ +# Drakkar-Software OctoBot-Node + +import datetime +import mock +import pytest + +import octobot_protocol.models as protocol_models + +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.automations.trade_symbols_resolver as trade_symbols_resolver_module +from tests.scheduler.user_actions.user_actions_executor.util import trading_tentacles_test_utils +import tentacles.Trading.Mode.dca_trading_mode.dca_trading as dca_trading + + +_TEST_WALLET_ID = "0x742d35Cc6634C0532925a3b844Bc9e7595f0bEb" +_TEST_ACCOUNT_ID = "1f2e203f-bdfa-49a6-8427-71c19fdc3327" +_TEST_EXCHANGE_CONFIG_ID = "kraken-spot-config" +_TEST_TIMESTAMP = datetime.datetime(2026, 1, 10, 12, 0, 0, tzinfo=datetime.UTC) + + +def _kraken_exchange_config( + *, + historical_trade_symbols: list[str] | None = None, +) -> protocol_models.ExchangeConfig: + return protocol_models.ExchangeConfig( + id=_TEST_EXCHANGE_CONFIG_ID, + name="kraken-main", + exchange="kraken", + sandboxed=False, + historical_trade_symbols=historical_trade_symbols, + ) + + +def _kraken_account( + account_id: str = _TEST_ACCOUNT_ID, +) -> protocol_models.Account: + exchange_account = protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id=account_id, + exchange_config_ids=[_TEST_EXCHANGE_CONFIG_ID], + ) + return protocol_models.Account( + id=account_id, + name="Kraken real spot", + is_simulated=False, + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + specifics=protocol_models.AccountSpecifics(actual_instance=exchange_account), + ) + + +def _automation_state( + automation_id: str, + *, + account_id: str, + status: protocol_models.WorkflowStatus = protocol_models.WorkflowStatus.RUNNING, + order_symbols: list[str] | None = None, +) -> protocol_models.AutomationState: + return protocol_models.AutomationState( + id=automation_id, + status=status, + metadata=protocol_models.AutomationMetadata( + name=automation_id, + description=automation_id, + ), + exchange_account_ids=[account_id], + orders=[ + protocol_models.OrderSummary(id=f"order-{symbol_index}", symbol=symbol) + for symbol_index, symbol in enumerate(order_symbols or []) + ], + ) + + +def _wallet_automation_states( + *, + protocol_states: list[protocol_models.AutomationState] | None = None, + flow_states_by_id: dict | None = None, +) -> automation_states_loader_module.WalletAutomationStates: + return automation_states_loader_module.WalletAutomationStates( + protocol_states=protocol_states or [], + flow_states_by_id=flow_states_by_id or {}, + ) + + +class TestResolveTradeSymbolsFromConfig: + @pytest.mark.asyncio + async def test_returns_historical_trade_symbols_from_realistic_exchange_config(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config( + historical_trade_symbols=["SOL/USDC"], + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock(return_value=_wallet_automation_states()), + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == ["SOL/USDC"] + + +class TestResolveTradeSymbolsFromAutomations: + @pytest.mark.asyncio + async def test_unions_config_symbols_with_running_automation_order_symbols(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config( + historical_trade_symbols=["SOL/USDC"], + ) + automation = _automation_state( + "automation-dca", + account_id=_TEST_ACCOUNT_ID, + order_symbols=["BTC/USDC"], + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock( + return_value=_wallet_automation_states(protocol_states=[automation]), + ), + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == ["BTC/USDC", "SOL/USDC"] + + @pytest.mark.asyncio + async def test_includes_strategy_traded_symbols_from_running_automation(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config(historical_trade_symbols=[]) + automation = _automation_state( + "automation-dca", + account_id=_TEST_ACCOUNT_ID, + ) + trading_configuration = trading_tentacles_test_utils.trading_tentacles_configuration( + name=dca_trading.DCATradingMode.get_name(), + config=trading_tentacles_test_utils.dca_tentacle_config( + **{dca_trading.DCATradingMode.TRADING_PAIRS: ["BTC/USDC"]}, + ), + ) + stored_strategy = protocol_models.Strategy( + id="strategy-dca-1", + version="1.0.0", + name="DCA strategy", + reference_market="USDC", + created_at=_TEST_TIMESTAMP, + updated_at=_TEST_TIMESTAMP, + configuration=protocol_models.StrategyConfiguration( + actual_instance=trading_configuration, + ), + ) + flow_automation_state = mock.Mock() + flow_automation_state.automation.metadata.strategy_id = "strategy-dca-1" + strategy_provider_mock = mock.Mock() + strategy_provider_mock.get_item.return_value = stored_strategy + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock( + return_value=_wallet_automation_states( + protocol_states=[automation], + flow_states_by_id={"automation-dca": flow_automation_state}, + ), + ), + ), mock.patch.object( + trade_symbols_resolver_module.collection_providers.StrategyProvider, + "instance", + return_value=strategy_provider_mock, + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == ["BTC/USDC"] + strategy_provider_mock.get_item.assert_called_once_with( + _TEST_WALLET_ID, + "strategy-dca-1", + ) + + @pytest.mark.asyncio + async def test_ignores_stopped_automations(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config(historical_trade_symbols=[]) + automation = _automation_state( + "automation-stopped", + account_id=_TEST_ACCOUNT_ID, + status=protocol_models.WorkflowStatus.COMPLETED, + order_symbols=["BTC/USDC"], + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock( + return_value=_wallet_automation_states(protocol_states=[automation]), + ), + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == [] + + @pytest.mark.asyncio + async def test_ignores_automations_bound_to_other_accounts(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config(historical_trade_symbols=[]) + automation = _automation_state( + "automation-foreign", + account_id="other-account-id", + order_symbols=["BTC/USDC"], + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock( + return_value=_wallet_automation_states(protocol_states=[automation]), + ), + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == [] + + +class TestResolveTradeSymbolsWithPreloadedAutomationStates: + @pytest.mark.asyncio + async def test_uses_preloaded_automation_states_without_scheduler_fetches(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config(historical_trade_symbols=["SOL/USDC"]) + automation = _automation_state( + "automation-dca", + account_id=_TEST_ACCOUNT_ID, + order_symbols=["BTC/USDC"], + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=mock.AsyncMock(), + ) as load_wallet_automation_states_mock: + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + automation_states=[automation], + flow_automation_states_by_id={}, + ) + assert resolved_symbols == ["BTC/USDC", "SOL/USDC"] + load_wallet_automation_states_mock.assert_not_awaited() + + @pytest.mark.asyncio + async def test_loads_wallet_automation_states_once_when_both_inputs_missing(self): + account = _kraken_account() + exchange_config = _kraken_exchange_config(historical_trade_symbols=["SOL/USDC"]) + automation = _automation_state( + "automation-dca", + account_id=_TEST_ACCOUNT_ID, + order_symbols=["BTC/USDC"], + ) + load_wallet_automation_states_mock = mock.AsyncMock( + return_value=_wallet_automation_states(protocol_states=[automation]), + ) + with mock.patch.object( + automation_states_loader_module, + "load_wallet_automation_states", + new=load_wallet_automation_states_mock, + ): + resolved_symbols = await trade_symbols_resolver_module.resolve_trade_symbols( + _TEST_WALLET_ID, + account, + exchange_config, + ) + assert resolved_symbols == ["BTC/USDC", "SOL/USDC"] + load_wallet_automation_states_mock.assert_awaited_once_with(_TEST_WALLET_ID) diff --git a/packages/node/tests/scheduler/global_view/test_automation_trigger.py b/packages/node/tests/scheduler/global_view/test_automation_trigger.py new file mode 100644 index 0000000000..35a99fcc47 --- /dev/null +++ b/packages/node/tests/scheduler/global_view/test_automation_trigger.py @@ -0,0 +1,156 @@ +# Drakkar-Software OctoBot-Node + +import mock +import pytest + +import octobot_protocol.models as protocol_models + +import octobot_node.scheduler.global_view.automation_trigger as automation_trigger_module + + +def _automation_state( + automation_id: str, + *, + account_id: str, + order_ids: list[str], +) -> protocol_models.AutomationState: + return protocol_models.AutomationState( + id=automation_id, + status=protocol_models.WorkflowStatus.RUNNING, + metadata=protocol_models.AutomationMetadata( + name=automation_id, + description=automation_id, + ), + exchange_account_ids=[account_id], + orders=[ + protocol_models.OrderSummary(id=order_id, symbol="BTC/USDT") + for order_id in order_ids + ], + ) + + +@pytest.mark.asyncio +class TestAccountHasBoundRunningAutomation: + async def test_true_when_running_automation_lists_account(self): + automation = _automation_state( + "automation-1", + account_id="acc-1", + order_ids=["order-1"], + ) + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock: + scheduler_api_mock.get_automation_states = mock.AsyncMock(return_value=[automation]) + assert await automation_trigger_module.account_has_bound_running_automation( + "wallet-1", + "acc-1", + ) is True + + async def test_false_when_no_running_automation_for_account(self): + automation = _automation_state( + "automation-1", + account_id="other-acc", + order_ids=["order-1"], + ) + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock: + scheduler_api_mock.get_automation_states = mock.AsyncMock(return_value=[automation]) + assert await automation_trigger_module.account_has_bound_running_automation( + "wallet-1", + "acc-1", + ) is False + + +@pytest.mark.asyncio +class TestTriggerAccountAutomations: + async def test_triggers_only_matching_automation_sequentially(self): + matching_automation = _automation_state( + "automation-gv-fill", + account_id="acc-sim-1", + order_ids=["gv-order-fill-1"], + ) + non_matching_automation = _automation_state( + "automation-gv-no-trigger", + account_id="acc-sim-1", + order_ids=["gv-order-stays-open"], + ) + trigger_calls: list[str] = [] + + async def _record_trigger(user_id: str, automation_id: str) -> None: + trigger_calls.append(automation_id) + + with mock.patch.object( + automation_trigger_module, + "scheduler_api", + ) as scheduler_api_mock, mock.patch.object( + automation_trigger_module, + "_trigger_automation_and_wait", + side_effect=_record_trigger, + ): + scheduler_api_mock.get_automation_states = mock.AsyncMock( + return_value=[matching_automation, non_matching_automation], + ) + await automation_trigger_module.trigger_account_automations( + "wallet-1", + "acc-sim-1", + {"gv-order-fill-1"}, + ) + assert trigger_calls == ["automation-gv-fill"] + + +@pytest.mark.asyncio +class TestTriggerAutomationAndWait: + async def test_logs_info_sequence_when_waiting_for_workflow(self): + import octobot_node.scheduler as scheduler_package + + mock_logger = mock.Mock() + scheduler = mock.Mock() + scheduler.resolve_active_automation_workflow_ids_for_parent_id = mock.AsyncMock( + return_value=["workflow-1"], + ) + with ( + mock.patch.object( + scheduler_package, + "SCHEDULER", + scheduler, + ), + mock.patch.object( + automation_trigger_module, + "_logger", + return_value=mock_logger, + ), + mock.patch.object( + automation_trigger_module.scheduler_tasks, + "send_forced_trigger_to_active_automation", + mock.AsyncMock(), + ) as send_trigger_mock, + mock.patch.object( + automation_trigger_module, + "_wait_for_workflow_iteration_success", + mock.AsyncMock(), + ) as wait_mock, + ): + await automation_trigger_module._trigger_automation_and_wait( + "wallet-1", + "automation-1", + ) + + send_trigger_mock.assert_awaited_once_with("automation-1", "wallet-1") + wait_mock.assert_awaited_once_with("workflow-1") + info_messages = [call.args[0] for call in mock_logger.info.call_args_list] + assert info_messages == [ + "Starting forced automation trigger (automation_id=%s, user_id=%s)", + "Resolved active workflow before trigger (automation_id=%s, workflow_id=%s)", + "Forced trigger sent (automation_id=%s, user_id=%s)", + "Waiting for automation workflow iteration (workflow_id=%s)", + "Finished waiting for automation workflow iteration (workflow_id=%s)", + ] + assert mock_logger.info.call_args_list[0].args[1:] == ("automation-1", "wallet-1") + assert mock_logger.info.call_args_list[1].args[1:] == ("automation-1", "workflow-1") + assert mock_logger.info.call_args_list[2].args[1:] == ("automation-1", "wallet-1") + assert mock_logger.info.call_args_list[3].args[1:] == ("workflow-1",) + assert mock_logger.info.call_args_list[4].args[1:] == ("workflow-1",) + mock_logger.warning.assert_not_called() diff --git a/packages/node/tests/scheduler/global_view/test_global_view_workflow.py b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py new file mode 100644 index 0000000000..c22ea60202 --- /dev/null +++ b/packages/node/tests/scheduler/global_view/test_global_view_workflow.py @@ -0,0 +1,189 @@ +# Drakkar-Software OctoBot-Node + +import datetime + +import mock +import pytest + +import octobot_flow.entities +import octobot_protocol.models as protocol_models + +from tests.scheduler import temp_dbos_scheduler + + +class TestGlobalViewScheduleInput: + def test_get_schedule_input_disables_automatic_backfill(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module + + schedule_input = global_view_workflow_module.get_schedule_input() + assert schedule_input["automatic_backfill"] is False + + +@pytest.mark.asyncio +class TestGlobalViewRefreshWorkflowRunGlobalViewRefresh: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_refreshes_all_wallets_and_accounts_in_parallel_per_wallet( + self, + global_view_workflow_module, + ): + wallet_ids = ["wallet-a"] + accounts = [mock.Mock(id="acc-1"), mock.Mock(id="acc-2")] + account_provider = mock.Mock() + account_provider.list_collectable_wallet_ids.return_value = wallet_ids + account_provider.list_items.return_value = accounts + refresh_mock = mock.AsyncMock(return_value=True) + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock, mock.patch.object( + global_view_workflow_module.workflows_retention, + "should_skip_retention_cleanup_on_this_node", + return_value=False, + ), mock.patch.object( + global_view_workflow_module.GlobalViewRefreshWorkflow, + "_refresh_single_account", + refresh_mock, + ): + account_provider_class_mock.instance.return_value = account_provider + result = await global_view_workflow_module.GlobalViewRefreshWorkflow._run_global_view_refresh( + datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC), + ) + assert result["refreshed_accounts"] == 2 + assert refresh_mock.await_count == 2 + + +@pytest.mark.asyncio +class TestRefreshWalletAccounts: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_skips_wallet_when_not_registered(self, global_view_workflow_module): + import octobot.community.wallet_backend.errors as wallet_backend_errors_module + + account_provider = mock.Mock() + account_provider.list_items.side_effect = wallet_backend_errors_module.WalletNotFoundError( + "Wallet not found" + ) + mock_logger = mock.Mock() + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock, mock.patch.object( + global_view_workflow_module.octobot_commons_logging, + "get_logger", + return_value=mock_logger, + ): + account_provider_class_mock.instance.return_value = account_provider + refreshed_count = await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_wallet_accounts( + "wallet-missing", + ) + assert refreshed_count == 0 + mock_logger.warning.assert_called_once() + + async def test_propagates_unexpected_list_items_errors(self, global_view_workflow_module): + account_provider = mock.Mock() + account_provider.list_items.side_effect = RuntimeError("storage failure") + with mock.patch.object( + global_view_workflow_module.collection_providers, + "AccountProvider", + ) as account_provider_class_mock: + account_provider_class_mock.instance.return_value = account_provider + try: + await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_wallet_accounts( + "wallet-1", + ) + raise AssertionError("Expected RuntimeError") + except RuntimeError as error: + assert str(error) == "storage failure" + + +def _exchange_account() -> protocol_models.Account: + account_timestamp = datetime.datetime(2026, 1, 1, tzinfo=datetime.UTC) + return protocol_models.Account( + id="account-1", + name="Kraken Live", + is_simulated=False, + created_at=account_timestamp, + updated_at=account_timestamp, + specifics=protocol_models.AccountSpecifics( + actual_instance=protocol_models.ExchangeAccount( + account_type=protocol_models.AccountType.EXCHANGE, + remote_account_id="remote-1", + exchange_config_ids=["exchange-config-1"], + ) + ), + ) + + +@pytest.mark.asyncio +class TestRefreshSingleAccount: + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + async def test_logs_success_summary_after_refresh(self, global_view_workflow_module): + account = _exchange_account() + refresh_result = octobot_flow.entities.GlobalViewAccountRefreshResult( + updated_account=account, + changed_order_ids={"gone-order-1"}, + open_orders=[{"exchange_id": "stays-order-2"}, {"exchange_id": "stays-order-3"}], + ) + mock_logger = mock.Mock() + with ( + mock.patch.object( + global_view_workflow_module.global_view_executor_module, + "refresh_account_global_view", + mock.AsyncMock(return_value=refresh_result), + ), + mock.patch.object( + global_view_workflow_module.automation_trigger_module, + "trigger_account_automations", + mock.AsyncMock(), + ) as trigger_mock, + mock.patch.object( + global_view_workflow_module.exchange_account_resolver, + "get_exchange_config", + return_value=protocol_models.ExchangeConfig( + id="exchange-config-1", + name="kraken-main", + exchange="kraken", + sandboxed=False, + ), + ), + mock.patch.object( + global_view_workflow_module.octobot_commons_logging, + "get_logger", + return_value=mock_logger, + ), + ): + succeeded = await global_view_workflow_module.GlobalViewRefreshWorkflow._refresh_single_account( + "wallet-1", + account, + ) + assert succeeded is True + trigger_mock.assert_awaited_once_with("wallet-1", "account-1", {"gone-order-1"}) + mock_logger.info.assert_called_once() + info_args = mock_logger.info.call_args.args + assert info_args[0].startswith("Account global view refresh succeeded:") + assert info_args[1:] == ( + "account-1", + "wallet-1", + "Kraken Live", + "kraken", + False, + 2, + 1, + True, + "n/a", + "n/a", + ) diff --git a/packages/node/tests/scheduler/portfolio_history/__init__.py b/packages/node/tests/scheduler/portfolio_history/__init__.py new file mode 100644 index 0000000000..8b13789179 --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/__init__.py @@ -0,0 +1 @@ + diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py new file mode 100644 index 0000000000..2e70d889b7 --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_executor.py @@ -0,0 +1,78 @@ +import mock +import pytest + +import octobot_node.scheduler.automations.automation_states_loader as automation_states_loader_module +import octobot_node.scheduler.portfolio_history.portfolio_history_executor as portfolio_history_executor_module + + +class TestRunPortfolioHistoryCollectionAccountFilter: + @pytest.mark.asyncio + @mock.patch.object( + portfolio_history_executor_module, + "_build_context_for_account", + return_value=None, + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_filters_accounts_when_account_ids_set( + self, mock_account_provider, mock_build_context, + ): + account_one = mock.MagicMock() + account_one.id = "acc-1" + account_two = mock.MagicMock() + account_two.id = "acc-2" + mock_account_provider.instance.return_value.list_items.return_value = [ + account_one, + account_two, + ] + + results = await portfolio_history_executor_module.run_portfolio_history_collection( + "wallet-user-1", + account_ids=["acc-2"], + ) + + assert results == [] + mock_account_provider.instance.return_value.list_items.assert_called_once_with( + "wallet-user-1", + ) + mock_build_context.assert_called_once_with("wallet-user-1", account_two) + + +class TestRunPortfolioHistoryCollectionTradeSymbolLoader: + @pytest.mark.asyncio + @mock.patch( + "octobot_node.scheduler.portfolio_history.portfolio_history_executor.trade_symbols_resolver_module.resolve_trade_symbols", + new_callable=mock.AsyncMock, + return_value=[], + ) + @mock.patch( + "octobot_node.scheduler.portfolio_history.portfolio_history_executor.automation_states_loader_module.load_wallet_automation_states_for_trade_symbols", + new_callable=mock.AsyncMock, + ) + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_uses_trade_symbol_loader_not_full_wallet_loader( + self, + mock_account_provider, + load_for_trade_symbols_mock, + resolve_trade_symbols_mock, + ): + account = mock.MagicMock() + account.id = "acc-1" + mock_account_provider.instance.return_value.list_items.return_value = [account] + wallet_automation_states = automation_states_loader_module.WalletAutomationStates( + protocol_states=[], + flow_states_by_id={}, + ) + load_for_trade_symbols_mock.return_value = wallet_automation_states + + with mock.patch.object( + portfolio_history_executor_module, + "_build_context_for_account", + return_value=mock.MagicMock(), + ), mock.patch( + "octobot_flow.jobs.portfolio_history_job.PortfolioHistoryJob", + ) as portfolio_history_job_class_mock: + portfolio_history_job_class_mock.return_value.run = mock.AsyncMock(return_value=[]) + await portfolio_history_executor_module.run_portfolio_history_collection("wallet-user-1") + + load_for_trade_symbols_mock.assert_awaited_once_with("wallet-user-1") + resolve_trade_symbols_mock.assert_awaited_once() diff --git a/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py new file mode 100644 index 0000000000..692f37a434 --- /dev/null +++ b/packages/node/tests/scheduler/portfolio_history/test_portfolio_history_workflow.py @@ -0,0 +1,117 @@ +import datetime +import mock +import pytest + +import octobot_node.scheduler.workflows.params as workflow_params_module + +from tests.scheduler import temp_dbos_scheduler + + +@pytest.fixture +def portfolio_history_workflow_module(temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow + return portfolio_history_workflow + + +class TestPortfolioHistoryScheduleInput: + def test_get_schedule_input_enables_catch_up_once_on_startup(self, portfolio_history_workflow_module): + schedule_input = portfolio_history_workflow_module.get_schedule_input() + assert schedule_input["automatic_backfill"] is False + assert schedule_input["catch_up_once_on_startup"] is True + assert schedule_input["queue_name"] == "portfolio_history_queue" + + +class TestPortfolioHistoryWorkflow: + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_iterates_wallets_and_delegates( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.return_value = [ + "wallet1", "wallet2", + ] + + mock_result = mock.MagicMock() + mock_result.skipped = False + mock_result.error = None + mock_executor.run_portfolio_history_collection = mock.AsyncMock(return_value=[mock_result]) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + assert mock_executor.run_portfolio_history_collection.call_count == 2 + assert result["succeeded"] == 2 + assert result["failed"] == 0 + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_error_isolation_per_wallet( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.return_value = [ + "wallet1", "wallet2", + ] + mock_executor.run_portfolio_history_collection = mock.AsyncMock( + side_effect=[Exception("fail"), [mock.MagicMock(skipped=False, error=None)]] + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + # wallet1 failed, wallet2 succeeded. + assert result["succeeded"] == 1 + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.logger") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_top_level_exception_is_logged_and_does_not_propagate( + self, mock_account_provider, mock_logger, portfolio_history_workflow_module, + ): + mock_account_provider.instance.return_value.list_collectable_wallet_ids.side_effect = ( + AttributeError("missing method") + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc) + ) + + mock_logger.exception.assert_called_once() + assert result == {"succeeded": 0, "failed": 0, "skipped": 0} + + @pytest.mark.asyncio + @mock.patch("octobot_node.scheduler.workflows.portfolio_history_workflow.portfolio_history_executor_module") + @mock.patch("octobot_sync.sync.collection_providers.AccountProvider") + async def test_uses_wallet_whitelist_from_params_without_listing_all_wallets( + self, mock_account_provider, mock_executor, portfolio_history_workflow_module, + ): + mock_result = mock.MagicMock() + mock_result.skipped = False + mock_result.error = None + mock_executor.run_portfolio_history_collection = mock.AsyncMock(return_value=[mock_result]) + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + + result = await portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection( + datetime.datetime.now(datetime.timezone.utc), + collection_params, + ) + + mock_account_provider.instance.return_value.list_collectable_wallet_ids.assert_not_called() + mock_executor.run_portfolio_history_collection.assert_awaited_once_with( + "wallet-user-1", + account_ids=["acc-1"], + ) + assert result["succeeded"] == 1 + + +class TestPortfolioHistoryCollectionStepRegistration: + def test_run_collection_is_registered_as_dbos_step(self, portfolio_history_workflow_module): + import dbos._registrations as dbos_registrations_module + + run_collection = portfolio_history_workflow_module.PortfolioHistoryWorkflow._run_collection + assert dbos_registrations_module.get_dbos_func_name(run_collection) == "run_portfolio_history_collection" + assert hasattr(run_collection, "dbos_function_name") diff --git a/packages/node/tests/scheduler/test_scheduler.py b/packages/node/tests/scheduler/test_scheduler.py index b81386f497..2ca22325ee 100644 --- a/packages/node/tests/scheduler/test_scheduler.py +++ b/packages/node/tests/scheduler/test_scheduler.py @@ -633,7 +633,7 @@ async def test_get_pending_tasks_uses_task_name_not_workflow_name(self): sched, mock_instance = _make_scheduler_with_mock_instance() mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) - with mock.patch("octobot_node.scheduler.workflows_util.get_automation_state_reader", return_value=None): + with mock.patch("octobot_node.scheduler.automations.automation_states_loader.get_automation_state_reader", return_value=None): executions = await sched.get_pending_tasks() assert len(executions) == 1 @@ -653,76 +653,13 @@ async def test_get_pending_tasks_preserves_none_name_when_task_has_no_name(self) sched, mock_instance = _make_scheduler_with_mock_instance() mock_instance.list_workflows_async = mock.AsyncMock(return_value=[ws]) - with mock.patch("octobot_node.scheduler.workflows_util.get_automation_state_reader", return_value=None): + with mock.patch("octobot_node.scheduler.automations.automation_states_loader.get_automation_state_reader", return_value=None): executions = await sched.get_pending_tasks() assert len(executions) == 1 assert executions[0].name is None -def _running_automation_task_content() -> str: - state_dict = { - "automation": { - "metadata": {"automation_id": "automation_1"}, - "actions_dag": { - "actions": [{"id": "a1", "dsl_script": "True"}], - }, - "execution": { - "current_execution": {"scheduled_to": 1, "triggered_at": 2}, - }, - }, - } - return json.dumps({"state": state_dict}) - - -class TestSchedulerGetAutomationStates: - - @pytest.mark.asyncio - async def test_error_workflow_reports_failed_not_running(self): - """DBOS ERROR with input-only task content must not surface as running in protocol state.""" - task = octobot_node.models.Task( - id=PARENT_ID, - name="failed-automation", - content=_running_automation_task_content(), - type="execute_actions", - ) - error_ws = _build_mock_workflow_status_error(task, RuntimeError("DBOSUnexpectedStepError"), workflow_id=PARENT_ID) - - sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[error_ws]) - - automation_states = await sched.get_automation_states(None) - - assert len(automation_states) == 1 - assert automation_states[0].id == PARENT_ID - assert automation_states[0].status == protocol_models.WorkflowStatus.FAILED - assert "DBOSUnexpectedStepError" in (automation_states[0].error or "") - assert automation_states[0].error_message is None - - @pytest.mark.asyncio - async def test_success_workflow_with_output_preserves_metadata_name(self): - """Completed automation with workflow output must keep task.name in protocol metadata.""" - task = octobot_node.models.Task( - id=PARENT_ID, - name="my-automation", - content=_running_automation_task_content(), - type="execute_actions", - ) - success_ws = _build_mock_workflow_status( - task, - encrypted_state=_running_automation_task_content(), - state_metadata="", - workflow_id=PARENT_ID, - ) - - sched, mock_instance = _make_scheduler_with_mock_instance() - mock_instance.list_workflows_async = mock.AsyncMock(return_value=[success_ws]) - - automation_states = await sched.get_automation_states(None) - - assert len(automation_states) == 1 - assert automation_states[0].id == PARENT_ID - assert automation_states[0].metadata.name == "my-automation" class TestSchedulerListUserActions: diff --git a/packages/node/tests/scheduler/test_schedules.py b/packages/node/tests/scheduler/test_schedules.py index 48834486eb..108c177553 100644 --- a/packages/node/tests/scheduler/test_schedules.py +++ b/packages/node/tests/scheduler/test_schedules.py @@ -42,6 +42,98 @@ def _matching_existing_schedule( } +def _configured_global_view_schedule_input(global_view_workflow_module) -> dict: + return global_view_workflow_module.get_schedule_input() + + +def _matching_existing_global_view_schedule(global_view_workflow_module) -> dict: + schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + return { + "schedule_id": "existing-global-view-schedule-id", + "schedule_name": global_view_workflow_module.SCHEDULE_NAME, + "workflow_name": "ignored-workflow-name", + "workflow_class_name": None, + "schedule": schedule_input["schedule"], + "status": "ACTIVE", + "context": None, + "last_fired_at": "2026-07-13T00:00:00+00:00", + "automatic_backfill": schedule_input.get("automatic_backfill", False), + "cron_timezone": schedule_input.get("cron_timezone"), + "queue_name": schedule_input.get("queue_name"), + } + + +def _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) -> dict: + return portfolio_history_workflow_module.get_schedule_input() + + +def _matching_existing_portfolio_history_schedule(portfolio_history_workflow_module) -> dict: + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + return { + "schedule_id": "existing-portfolio-history-schedule-id", + "schedule_name": portfolio_history_workflow_module.SCHEDULE_NAME, + "workflow_name": "ignored-workflow-name", + "workflow_class_name": None, + "schedule": schedule_input["schedule"], + "status": "ACTIVE", + "context": None, + "last_fired_at": "2026-07-13T00:00:00+00:00", + "automatic_backfill": schedule_input.get("automatic_backfill", False), + "catch_up_once_on_startup": schedule_input.get("catch_up_once_on_startup", False), + "cron_timezone": schedule_input.get("cron_timezone"), + "queue_name": schedule_input.get("queue_name"), + } + + +def _success_workflow_status(): + success_status = mock.Mock(spec=dbos.WorkflowStatus) + success_status.status = dbos.WorkflowStatusString.SUCCESS.value + return success_status + + +def _enqueued_workflow_status(): + enqueued_status = mock.Mock(spec=dbos.WorkflowStatus) + enqueued_status.status = dbos.WorkflowStatusString.ENQUEUED.value + return enqueued_status + + +def _workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=None, +): + portfolio_prefix = f"sched-{portfolio_history_workflow_module.SCHEDULE_NAME}-" + + async def get_workflow_status_async(workflow_id: str): + if workflow_id.startswith(portfolio_prefix): + return _success_workflow_status() + if inner_side_effect is None: + return None + return await inner_side_effect(workflow_id) + + return get_workflow_status_async + + +def _get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + *, + cleanup_existing: dict | None, + global_view_existing: dict | None, + portfolio_history_existing: dict | None, +): + async def get_schedule_async(schedule_name: str): + if schedule_name == dbos_cleanup_workflow_module.SCHEDULE_NAME: + return cleanup_existing + if schedule_name == global_view_workflow_module.SCHEDULE_NAME: + return global_view_existing + if schedule_name == portfolio_history_workflow_module.SCHEDULE_NAME: + return portfolio_history_existing + return None + + return get_schedule_async + + class TestExistingScheduleMatchesConfigured: def test_returns_true_when_all_fields_match(self, temp_dbos_scheduler): import octobot_node.scheduler.schedules as schedules_module @@ -106,6 +198,33 @@ def test_returns_false_when_automatic_backfill_differs(self, temp_dbos_scheduler schedule_input, ) is False + def test_returns_false_when_catch_up_once_on_startup_differs(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + existing_schedule = _matching_existing_portfolio_history_schedule(portfolio_history_workflow_module) + existing_schedule["catch_up_once_on_startup"] = False + + assert schedules_module._existing_schedule_matches_configured( + existing_schedule, + schedule_input, + ) is False + + +class TestGetLatestScheduleTriggerTimeBefore: + def test_daily_cron_returns_latest_slot_before_now(self, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + before = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + trigger_time = schedules_module._get_latest_schedule_trigger_time_before( + schedule_input, + before, + ) + assert trigger_time == datetime.datetime(2026, 7, 15, 3, 0, 0, tzinfo=datetime.timezone.utc) + class TestGetBackfillScheduleDefaultAnchor: def test_returns_utc_now_minus_one_day(self, temp_dbos_scheduler): @@ -185,19 +304,47 @@ def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): yield dbos_cleanup_workflow_module_loaded - async def test_creates_schedule_when_missing(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + @pytest.fixture + def global_view_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.global_view_workflow as global_view_workflow_module_loaded + + yield global_view_workflow_module_loaded + + @pytest.fixture + def portfolio_history_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + yield portfolio_history_workflow_module_loaded + + async def test_creates_schedule_when_missing( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + global_view_schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + portfolio_history_schedule_input = _configured_portfolio_history_schedule_input( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=None, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=None, + global_view_existing=None, + portfolio_history_existing=None, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -214,40 +361,88 @@ async def test_creates_schedule_when_missing(self, dbos_cleanup_workflow_module, ) as backfill_to_thread_mock: await schedules_module.register_schedules(temp_dbos_scheduler) - create_schedule_mock.assert_awaited_once_with( - schedule_name=schedule_input["schedule_name"], - workflow_fn=schedule_input["workflow_fn"], - schedule=schedule_input["schedule"], - context=schedule_input.get("context"), - automatic_backfill=schedule_input.get("automatic_backfill", False), - cron_timezone=schedule_input.get("cron_timezone"), - queue_name=schedule_input.get("queue_name"), + assert create_schedule_mock.await_count == 3 + create_schedule_mock.assert_any_await( + schedule_name=cleanup_schedule_input["schedule_name"], + workflow_fn=cleanup_schedule_input["workflow_fn"], + schedule=cleanup_schedule_input["schedule"], + context=cleanup_schedule_input.get("context"), + automatic_backfill=cleanup_schedule_input.get("automatic_backfill", False), + cron_timezone=cleanup_schedule_input.get("cron_timezone"), + queue_name=cleanup_schedule_input.get("queue_name"), + ) + create_schedule_mock.assert_any_await( + schedule_name=global_view_schedule_input["schedule_name"], + workflow_fn=global_view_schedule_input["workflow_fn"], + schedule=global_view_schedule_input["schedule"], + context=global_view_schedule_input.get("context"), + automatic_backfill=global_view_schedule_input.get("automatic_backfill", False), + cron_timezone=global_view_schedule_input.get("cron_timezone"), + queue_name=global_view_schedule_input.get("queue_name"), + ) + create_schedule_mock.assert_any_await( + schedule_name=portfolio_history_schedule_input["schedule_name"], + workflow_fn=portfolio_history_schedule_input["workflow_fn"], + schedule=portfolio_history_schedule_input["schedule"], + context=portfolio_history_schedule_input.get("context"), + automatic_backfill=portfolio_history_schedule_input.get("automatic_backfill", False), + cron_timezone=portfolio_history_schedule_input.get("cron_timezone"), + queue_name=portfolio_history_schedule_input.get("queue_name"), ) apply_schedules_mock.assert_not_awaited() backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Creating schedule %s (%s)", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Creating schedule %s (%s)", + global_view_schedule_input["schedule_name"], + global_view_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Creating schedule %s (%s)", + portfolio_history_schedule_input["schedule_name"], + portfolio_history_schedule_input["schedule"], ) - async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + async def test_keeps_schedule_when_config_matches( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) - existing_schedule = _matching_existing_schedule( + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + global_view_schedule_input = _configured_global_view_schedule_input(global_view_workflow_module) + existing_cleanup_schedule = _matching_existing_schedule( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_cleanup_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -259,6 +454,11 @@ async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_mo "apply_schedules_async", new_callable=mock.AsyncMock, ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), + ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, ) as backfill_to_thread_mock: @@ -267,30 +467,53 @@ async def test_keeps_schedule_when_config_matches(self, dbos_cleanup_workflow_mo create_schedule_mock.assert_not_awaited() apply_schedules_mock.assert_not_awaited() backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Keeping existing schedule %s (%s)", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Keeping existing schedule %s (%s)", + global_view_schedule_input["schedule_name"], + global_view_schedule_input["schedule"], ) - async def test_recreates_schedule_when_config_differs(self, dbos_cleanup_workflow_module, temp_dbos_scheduler): + async def test_recreates_schedule_when_config_differs( + self, + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + temp_dbos_scheduler, + ): import octobot_node.scheduler.schedules as schedules_module - schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) - existing_schedule = _matching_existing_schedule( + cleanup_schedule_input = _configured_cleanup_schedule_input(dbos_cleanup_workflow_module) + existing_cleanup_schedule = _matching_existing_schedule( dbos_cleanup_workflow_module, temp_dbos_scheduler, automatic_backfill=False, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) mock_logger = mock.Mock() with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", - return_value=schedule_input, + return_value=cleanup_schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_cleanup_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -302,23 +525,35 @@ async def test_recreates_schedule_when_config_differs(self, dbos_cleanup_workflo "apply_schedules_async", new_callable=mock.AsyncMock, ) as apply_schedules_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), + ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, ) as backfill_to_thread_mock: await schedules_module.register_schedules(temp_dbos_scheduler) create_schedule_mock.assert_not_awaited() - apply_schedules_mock.assert_awaited_once_with([schedule_input]) + apply_schedules_mock.assert_awaited_once_with([cleanup_schedule_input]) backfill_to_thread_mock.assert_not_awaited() - mock_logger.info.assert_called_once_with( + mock_logger.info.assert_any_call( "Updating schedule %s (%s): configuration changed", - schedule_input["schedule_name"], - schedule_input["schedule"], + cleanup_schedule_input["schedule_name"], + cleanup_schedule_input["schedule"], + ) + mock_logger.info.assert_any_call( + "Keeping existing schedule %s (%s)", + global_view_workflow_module.SCHEDULE_NAME, + _configured_global_view_schedule_input(global_view_workflow_module)["schedule"], ) async def test_backfills_when_last_fired_at_is_null( self, dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -328,6 +563,12 @@ async def test_backfills_when_last_fired_at_is_null( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) workflow_id = ( @@ -348,8 +589,14 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -372,8 +619,9 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - new_callable=mock.AsyncMock, - return_value=None, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", side_effect=backfill_to_thread_side_effect, @@ -409,6 +657,8 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): async def test_skips_backfill_when_all_window_slots_terminal( self, dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -418,6 +668,12 @@ async def test_skips_backfill_when_all_window_slots_terminal( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None anchor = datetime.datetime(2026, 7, 14, 6, 30, 0, tzinfo=datetime.timezone.utc) backfill_end = datetime.datetime(2026, 7, 15, 8, 30, 0, tzinfo=datetime.timezone.utc) @@ -440,8 +696,14 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -464,7 +726,10 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - side_effect=get_workflow_status_side_effect, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=get_workflow_status_side_effect, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, @@ -489,6 +754,8 @@ async def get_workflow_status_side_effect(requested_workflow_id: str): async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_window( self, dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -499,6 +766,12 @@ async def test_backfill_logs_unchanged_when_slot_already_terminal_in_mixed_windo dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None existing_schedule["schedule"] = "0 * * * *" anchor = datetime.datetime(2026, 7, 15, 10, 0, 0, tzinfo=datetime.timezone.utc) @@ -533,8 +806,14 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -557,7 +836,10 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", - side_effect=get_workflow_status_side_effect, + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + inner_side_effect=get_workflow_status_side_effect, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", side_effect=backfill_to_thread_side_effect, @@ -589,6 +871,8 @@ async def backfill_to_thread_side_effect(func, *args, **kwargs): async def test_skips_backfill_when_last_fired_at_set( self, dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -598,14 +882,26 @@ async def test_skips_backfill_when_last_fired_at_set( dbos_cleanup_workflow_module, temp_dbos_scheduler, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) with mock.patch( "octobot_node.scheduler.schedules.dbos_cleanup_workflow.get_schedule_input", return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, @@ -616,6 +912,11 @@ async def test_skips_backfill_when_last_fired_at_set( temp_dbos_scheduler.INSTANCE, "apply_schedules_async", new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + side_effect=_workflow_status_async_with_portfolio_terminal( + portfolio_history_workflow_module, + ), ), mock.patch( "octobot_node.scheduler.schedules.asyncio.to_thread", new_callable=mock.AsyncMock, @@ -627,6 +928,8 @@ async def test_skips_backfill_when_last_fired_at_set( async def test_skips_backfill_when_automatic_backfill_false( self, dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, temp_dbos_scheduler, ): import octobot_node.scheduler.schedules as schedules_module @@ -638,6 +941,12 @@ async def test_skips_backfill_when_automatic_backfill_false( temp_dbos_scheduler, automatic_backfill=False, ) + existing_global_view_schedule = _matching_existing_global_view_schedule( + global_view_workflow_module, + ) + existing_portfolio_history_schedule = _matching_existing_portfolio_history_schedule( + portfolio_history_workflow_module, + ) existing_schedule["last_fired_at"] = None with mock.patch( @@ -645,11 +954,21 @@ async def test_skips_backfill_when_automatic_backfill_false( return_value=schedule_input, ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", - new_callable=mock.AsyncMock, - return_value=existing_schedule, + side_effect=_get_schedule_async_side_effect( + dbos_cleanup_workflow_module, + global_view_workflow_module, + portfolio_history_workflow_module, + cleanup_existing=existing_schedule, + global_view_existing=existing_global_view_schedule, + portfolio_history_existing=existing_portfolio_history_schedule, + ), ), mock.patch( "octobot_node.scheduler.schedules.dbos.DBOS.create_schedule_async", new_callable=mock.AsyncMock, + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_success_workflow_status(), ), mock.patch( "octobot_node.scheduler.schedules._get_logger", return_value=mock.Mock(), @@ -664,3 +983,217 @@ async def test_skips_backfill_when_automatic_backfill_false( await schedules_module.register_schedules(temp_dbos_scheduler) backfill_to_thread_mock.assert_not_awaited() + + +class TestMaybeCatchUpScheduleOnceOnStartup: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def portfolio_history_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + yield portfolio_history_workflow_module_loaded + + async def test_backfills_latest_slot_when_missing(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + trigger_time = datetime.datetime(2026, 7, 15, 3, 0, 0, tzinfo=datetime.timezone.utc) + workflow_id = schedules_module.build_scheduled_workflow_id( + portfolio_history_workflow_module.SCHEDULE_NAME, + trigger_time, + ) + mock_handle = mock.Mock() + mock_handle.get_workflow_id.return_value = workflow_id + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + async def backfill_to_thread_side_effect(func, *args, **kwargs): + return func(*args, **kwargs) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=None, + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + side_effect=backfill_to_thread_side_effect, + ) as backfill_to_thread_mock, mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.backfill_schedule", + return_value=[mock_handle], + ) as backfill_schedule_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_awaited_once() + backfill_schedule_mock.assert_called_once() + schedule_name, backfill_start, backfill_end = backfill_schedule_mock.call_args[0] + assert schedule_name == portfolio_history_workflow_module.SCHEDULE_NAME + assert backfill_start == trigger_time - datetime.timedelta(seconds=1) + assert backfill_end == trigger_time + datetime.timedelta(seconds=1) + + async def test_skips_when_latest_slot_terminal(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_success_workflow_status(), + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_not_awaited() + + async def test_skips_when_latest_slot_in_progress(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + now = datetime.datetime(2026, 7, 15, 13, 0, 0, tzinfo=datetime.timezone.utc) + datetime_class_mock = mock.Mock(wraps=datetime.datetime) + datetime_class_mock.now.return_value = now + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value=_matching_existing_portfolio_history_schedule(portfolio_history_workflow_module), + ), mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_workflow_status_async", + new_callable=mock.AsyncMock, + return_value=_enqueued_workflow_status(), + ), mock.patch( + "octobot_node.scheduler.schedules.datetime", + mock.Mock( + datetime=datetime_class_mock, + timezone=datetime.timezone, + timedelta=datetime.timedelta, + ), + ), mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + backfill_to_thread_mock.assert_not_awaited() + + async def test_skips_when_flag_disabled(self, portfolio_history_workflow_module, temp_dbos_scheduler): + import octobot_node.scheduler.schedules as schedules_module + + schedule_input = _configured_portfolio_history_schedule_input(portfolio_history_workflow_module) + schedule_input["catch_up_once_on_startup"] = False + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + ) as get_schedule_mock, mock.patch( + "octobot_node.scheduler.schedules.asyncio.to_thread", + new_callable=mock.AsyncMock, + ) as backfill_to_thread_mock: + await schedules_module._maybe_catch_up_schedule_once_on_startup( + portfolio_history_workflow_module.SCHEDULE_NAME, + schedule_input, + ) + + get_schedule_mock.assert_not_awaited() + backfill_to_thread_mock.assert_not_awaited() + + +class TestUpdateCleanupScheduleCron: + pytestmark = pytest.mark.asyncio + + @pytest.fixture + def dbos_cleanup_workflow_module(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module_loaded + + yield dbos_cleanup_workflow_module_loaded + + async def test_applies_schedule_when_cron_changes( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + desired_cron = "0 */6 * * *" + schedule_input = dbos_cleanup_workflow_module.get_schedule_input(cron=desired_cron) + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value={"schedule": "0 0 * * *"}, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + result = await schedules_module.update_cleanup_schedule_cron( + temp_dbos_scheduler, + desired_cron, + ) + + assert result == {"changed": True, "cron": desired_cron} + apply_schedules_mock.assert_awaited_once_with([schedule_input]) + + async def test_skips_apply_when_cron_unchanged( + self, + dbos_cleanup_workflow_module, + temp_dbos_scheduler, + ): + import octobot_node.scheduler.schedules as schedules_module + + desired_cron = dbos_cleanup_workflow_module.SCHEDULE_CRON + + with mock.patch( + "octobot_node.scheduler.schedules.dbos.DBOS.get_schedule_async", + new_callable=mock.AsyncMock, + return_value={"schedule": desired_cron}, + ), mock.patch.object( + temp_dbos_scheduler.INSTANCE, + "apply_schedules_async", + new_callable=mock.AsyncMock, + ) as apply_schedules_mock: + result = await schedules_module.update_cleanup_schedule_cron( + temp_dbos_scheduler, + desired_cron, + ) + + assert result == {"changed": False, "cron": desired_cron} + apply_schedules_mock.assert_not_awaited() diff --git a/packages/node/tests/scheduler/test_task_context.py b/packages/node/tests/scheduler/test_task_context.py index cb08e5f9ef..e82efb393a 100644 --- a/packages/node/tests/scheduler/test_task_context.py +++ b/packages/node/tests/scheduler/test_task_context.py @@ -17,7 +17,11 @@ import pytest import mock -import octobot_node.scheduler.octobot_flow_client as octobot_flow_client +import octobot_flow.entities as flow_entities_module +import octobot_flow.logic.accounts.account_state_persistence as account_state_persistence_module +import octobot_trading.enums as trading_enums_module + +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client from octobot_node.scheduler.task_context import encrypted_task from octobot_node.models import Task @@ -211,3 +215,67 @@ def test_encrypted_task_per_task_ecdsa_key_takes_precedence(self) -> None: mock_decrypt.assert_called_once_with( "encrypted_content", "metadata", user_ecdsa_public_key=b"per_task_key" ) + + +class TestEncryptedTaskTrimmedState: + def test_encrypts_trimmed_next_actions_description_state(self) -> None: + mock_settings = mock.Mock() + mock_settings.TASKS_SERVER_RSA_PRIVATE_KEY = None + mock_settings.TASKS_USER_ECDSA_PUBLIC_KEY = None + + order_columns = trading_enums_module.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(120) + ] + elements = flow_entities_module.ExchangeAccountElements(trades=trades) + automation_state = flow_entities_module.AutomationState( + automation=flow_entities_module.AutomationDetails( + metadata=flow_entities_module.AutomationMetadata(automation_id="automation-1"), + exchange_account_elements=elements, + ), + ) + job_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + auth_details={}, + params={}, + ) + to_update_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[mock.Mock()], + next_actions_description=job_description, + actions_dag=mock.Mock(), + ) + captured_state_dicts: list[dict] = [] + + def capture_encrypt(state_dict: dict) -> tuple[str, str]: + captured_state_dicts.append(state_dict) + return "encrypted_payload", "encryption_meta" + + mock_encrypt = mock.Mock(side_effect=capture_encrypt) + + with mock.patch("octobot_node.config.settings", mock_settings), \ + mock.patch( + "octobot_node.scheduler.task_context.encryption.get_next_encrypted_if_needed_content_and_metadata", + mock_encrypt, + ): + task = Task(name="test_task", content="plain") + with encrypted_task(task, to_update_result=to_update_result): + job_description.state = automation_state.to_dict(include_default_values=False) + account_state_persistence_module.trim_live_trades_in_iteration_state( + job_description.state, + 100, + ) + + assert len(captured_state_dicts) == 1 + restored_state = flow_entities_module.AutomationState.from_dict( + captured_state_dicts[0]["state"] + ) + trimmed_elements = restored_state.automation.exchange_account_elements + assert trimmed_elements is not None + assert len(trimmed_elements.trades) == 100 + assert len(trimmed_elements.trade_summaries.get("BTC/USDT", [])) == 20 + assert to_update_result.maybe_encrypted_next_actions_description == "encrypted_payload" diff --git a/packages/node/tests/scheduler/test_tasks.py b/packages/node/tests/scheduler/test_tasks.py index 9f6bdc7a02..42cbc4c0c8 100644 --- a/packages/node/tests/scheduler/test_tasks.py +++ b/packages/node/tests/scheduler/test_tasks.py @@ -189,6 +189,55 @@ async def test_enqueues_execute_user_action_workflow_with_encoded_inputs(self, t assert enqueue_keyword_arguments["inputs"] == expected_inputs_encoded +class TestTriggerPortfolioHistoryCollection: + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await octobot_node.scheduler.tasks.trigger_portfolio_history_collection() + + @pytest.mark.asyncio + async def test_enqueues_portfolio_history_collection_on_portfolio_history_queue(self, temp_dbos_scheduler): + import datetime + import octobot_node.scheduler.workflows.portfolio_history_workflow as portfolio_history_workflow_module_loaded + + expected_workflow_id = "portfolio-history-workflow-test-id" + scheduled_time = datetime.datetime(2026, 3, 15, 12, 0, 0, tzinfo=datetime.UTC) + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + with ( + mock.patch( + "octobot_commons.timestamp_util.utc_now_datetime", + return_value=scheduled_time, + ), + mock.patch.object( + temp_dbos_scheduler.PORTFOLIO_HISTORY_QUEUE, + "enqueue_async", + mock.AsyncMock(), + ) as mock_enqueue_async_operation, + ): + mock_workflow_enqueue_handle = mock.Mock() + mock_workflow_enqueue_handle.workflow_id = expected_workflow_id + mock_enqueue_async_operation.return_value = mock_workflow_enqueue_handle + + enqueue_function_result = await octobot_node.scheduler.tasks.trigger_portfolio_history_collection( + collection_params, + ) + + assert enqueue_function_result == expected_workflow_id + mock_enqueue_async_operation.assert_awaited_once() + positional_workflow_targets, enqueue_keyword_arguments = mock_enqueue_async_operation.call_args + assert ( + positional_workflow_targets[0] + is portfolio_history_workflow_module_loaded.PortfolioHistoryWorkflow.portfolio_history_collection + ) + assert positional_workflow_targets[1] == scheduled_time + assert positional_workflow_targets[2] == collection_params.to_dict(include_default_values=False) + assert enqueue_keyword_arguments == {} + + class TestSendToActiveAutomationWorkflow: _TEST_WALLET_ADDRESS = "0xaaabbbcccddd" _TEST_PARENT_AUTOMATION_ID = "00000000-0000-4000-8000-000000000099" diff --git a/packages/node/tests/scheduler/test_user_action_executor_factory.py b/packages/node/tests/scheduler/test_user_action_executor_factory.py index 302c762ab0..20216b55cd 100644 --- a/packages/node/tests/scheduler/test_user_action_executor_factory.py +++ b/packages/node/tests/scheduler/test_user_action_executor_factory.py @@ -153,6 +153,29 @@ def test_returns_refresh_accounts_executor_class(self): resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) assert resolved_executor_cls is user_actions_executor_package.RefreshAccountsActionExecutor + def test_returns_update_historical_exchanges_data_executor_class(self): + configuration_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action_model = self._user_action( + action_identifier="ua-update-historical", + configuration_inner=configuration_inner, + ) + resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) + assert resolved_executor_cls is user_actions_executor_package.UpdateHistoricalExchangesDataActionExecutor + + def test_returns_reset_account_trading_data_executor_class(self): + configuration_inner = protocol_models.ResetAccountTradingDataConfiguration( + action_type=protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA, + account_ids=["acc-1"], + ) + user_action_model = self._user_action( + action_identifier="ua-reset-account-trading", + configuration_inner=configuration_inner, + ) + resolved_executor_cls = executor_factory_module.user_action_executor_factory(user_action_model) + assert resolved_executor_cls is user_actions_executor_package.ResetAccountTradingDataActionExecutor + def test_returns_create_exchange_config_executor_class(self): configuration_inner = protocol_models.CreateExchangeConfigConfiguration( action_type=protocol_models.UserActionType.EXCHANGE_CONFIG_CREATE, diff --git a/packages/node/tests/scheduler/test_workflows_retention.py b/packages/node/tests/scheduler/test_workflows_retention.py index e992ad0d06..1d9350c90b 100644 --- a/packages/node/tests/scheduler/test_workflows_retention.py +++ b/packages/node/tests/scheduler/test_workflows_retention.py @@ -1,4 +1,4 @@ -# Drakkar-Software OctoBot-Node +# Drakkar-Software OctoBot-Node # Copyright (c) 2025 Drakkar-Software, All rights reserved. import datetime @@ -18,6 +18,8 @@ _AUTOMATION_WORKFLOW_NAME = "execute_automation" _DBOS_CLEANUP_WORKFLOW_NAME = "dbos_cleanup" +_GLOBAL_VIEW_WORKFLOW_NAME = "global_view_refresh" +_PORTFOLIO_HISTORY_WORKFLOW_NAME = "portfolio_history_collection" _PARENT_WORKFLOW_ID_A = "741ce171-dac9-40be-83dc-b443c0eaf0e2" _PARENT_WORKFLOW_ID_B = "852df282-edb0-51cf-94ed-c554d1fbf1f3" @@ -407,7 +409,7 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s sched = scheduler_module.Scheduler() mock_instance = mock.Mock() mock_instance.list_workflows_async = mock.AsyncMock( - side_effect=[automation_workflows, cleanup_workflows] + side_effect=[automation_workflows, cleanup_workflows, [], []] ) mock_instance.delete_workflows_async = mock.AsyncMock() mock_engine = mock.Mock() @@ -418,10 +420,13 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s sched.INSTANCE = mock_instance mock_logger = mock.Mock() - with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch.object( + with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=None, + ), mock.patch.object( workflows_retention, - "AUTOMATION_EXECUTION_RETENTION_SECONDS", - retention_seconds, + "select_retention_seconds_for_database_size", + return_value=retention_seconds, ), mock.patch( "octobot_node.scheduler.workflows_retention._get_logger", return_value=mock_logger, @@ -431,7 +436,11 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s assert summary == { "deleted_by_automation": {_PARENT_WORKFLOW_ID_A: 1}, "deleted_cleanup_executions": 1, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 2, + "database_size_bytes": None, + "retention_seconds": retention_seconds, } mock_instance.delete_workflows_async.assert_awaited_once_with( [ @@ -440,12 +449,14 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s ], delete_children=False, ) - mock_connection.execute.assert_called_once() mock_logger.info.assert_any_call( - "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs", + "Deleting %s outdated workflow executions: %s automation groups, %s cleanup runs, " + "%s global view runs, %s portfolio history runs", 2, 1, 1, + 0, + 0, ) mock_logger.info.assert_any_call("DBOS cleanup summary: %s", summary) @@ -453,13 +464,16 @@ async def test_returns_per_automation_summary_and_deletes_once(self, temp_dbos_s async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_scheduler): sched = scheduler_module.Scheduler() mock_instance = mock.Mock() - mock_instance.list_workflows_async = mock.AsyncMock(side_effect=[[], []]) + mock_instance.list_workflows_async = mock.AsyncMock(side_effect=[[], [], [], []]) mock_instance.delete_workflows_async = mock.AsyncMock() mock_instance._sys_db.engine = mock.Mock() sched.INSTANCE = mock_instance mock_logger = mock.Mock() with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=None, + ), mock.patch( "octobot_node.scheduler.workflows_retention._get_logger", return_value=mock_logger, ): @@ -468,13 +482,18 @@ async def test_skips_delete_and_vacuum_when_nothing_to_delete(self, temp_dbos_sc assert summary == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, + "database_size_bytes": None, + "retention_seconds": workflows_retention.RETENTION_SECONDS_2_DAYS, } mock_instance.delete_workflows_async.assert_not_called() mock_instance._sys_db.engine.begin.assert_not_called() mock_logger.info.assert_called_once_with("DBOS cleanup summary: %s", summary) + class TestShouldSkipRetentionCleanupForScheduledTime: @pytest.mark.asyncio async def test_returns_true_when_scheduler_not_initialized(self, temp_dbos_scheduler): @@ -593,3 +612,309 @@ async def test_returns_false_when_latest_terminal_cleanup_is_older_than_backfill ) assert result is False + + +class TestGetOutdatedTerminalWorkflowIds: + def test_deletes_old_terminal_workflows(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id="global-view-old", + updated_at=old_updated_at, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + _workflow_status_row( + workflow_id="global-view-recent", + updated_at=now_ms - 1, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_terminal_workflow_ids( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == ["global-view-old"] + + def test_skips_non_terminal_workflows(self): + now_ms = 10_000_000 + retention_seconds = 100.0 + old_updated_at = now_ms - int(retention_seconds * 1000) - 1 + workflows = [ + _workflow_status_row( + workflow_id="portfolio-history-pending", + updated_at=old_updated_at, + status=dbos.WorkflowStatusString.PENDING.value, + name=_PORTFOLIO_HISTORY_WORKFLOW_NAME, + ), + ] + + deleted_ids = workflows_retention.get_outdated_terminal_workflow_ids( + workflows, + retention_seconds=retention_seconds, + now_ms=now_ms, + ) + + assert deleted_ids == [] + + +class TestSelectRetentionSecondsForDatabaseSize: + def test_returns_2_day_retention_when_size_unknown(self): + assert workflows_retention.select_retention_seconds_for_database_size(None) == workflows_retention.RETENTION_SECONDS_2_DAYS + + def test_returns_2_day_retention_below_first_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES - 1 + assert workflows_retention.select_retention_seconds_for_database_size(size_bytes) == workflows_retention.RETENTION_SECONDS_2_DAYS + + def test_returns_1_day_retention_between_tiers(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES + assert workflows_retention.select_retention_seconds_for_database_size(size_bytes) == workflows_retention.RETENTION_SECONDS_1_DAY + + def test_returns_6_hour_retention_at_second_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES + assert workflows_retention.select_retention_seconds_for_database_size(size_bytes) == workflows_retention.RETENTION_SECONDS_6_HOURS + + +class TestSelectCleanupCronForDatabaseSize: + def test_returns_daily_cron_when_size_unknown(self): + assert workflows_retention.select_cleanup_cron_for_database_size(None) == workflows_retention.DBOS_CLEANUP_CRON_DAILY + + def test_returns_daily_cron_below_first_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES - 1 + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_DAILY + + def test_returns_12h_cron_between_tiers(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_12H + + def test_returns_6h_cron_at_second_tier(self): + size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES + assert workflows_retention.select_cleanup_cron_for_database_size(size_bytes) == workflows_retention.DBOS_CLEANUP_CRON_6H + + +class TestGetSchedulerDatabaseSizeBytes: + def test_sums_sqlite_file_and_sidecars(self, tmp_path): + sqlite_path = tmp_path / "tasks.db" + sqlite_path.write_bytes(b"x" * 10) + wal_path = tmp_path / "tasks.db-wal" + wal_path.write_bytes(b"y" * 5) + mock_dbos_instance = mock.Mock() + + with mock.patch( + "octobot_node.config.settings.SCHEDULER_SQLITE_FILE", + str(sqlite_path), + ), mock.patch( + "octobot_node.config.settings.SCHEDULER_POSTGRES_URL", + None, + ): + size_bytes = workflows_retention.get_scheduler_database_size_bytes(mock_dbos_instance) + + assert size_bytes == 15 + + def test_returns_none_when_sqlite_file_missing(self): + mock_dbos_instance = mock.Mock() + mock_logger = mock.Mock() + with mock.patch( + "octobot_node.config.settings.SCHEDULER_SQLITE_FILE", + "missing-tasks.db", + ), mock.patch( + "octobot_node.config.settings.SCHEDULER_POSTGRES_URL", + None, + ), mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + size_bytes = workflows_retention.get_scheduler_database_size_bytes(mock_dbos_instance) + + assert size_bytes is None + mock_logger.exception.assert_called_once() + + +class TestCleanupOutdatedAutomationExecutionsScheduledWorkflows: + @pytest.mark.asyncio + async def test_deletes_outdated_global_view_and_portfolio_history_workflows(self, temp_dbos_scheduler): + now_ms = 10_000_000 + retention_seconds = 100.0 + cutoff_ms = now_ms - int(retention_seconds * 1000) + global_view_workflows = [ + _workflow_status_row( + workflow_id="global-view-old", + updated_at=cutoff_ms - 1, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + ] + portfolio_history_workflows = [ + _workflow_status_row( + workflow_id="portfolio-history-old", + updated_at=cutoff_ms - 1, + name=_PORTFOLIO_HISTORY_WORKFLOW_NAME, + ), + ] + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock( + side_effect=[[], [], global_view_workflows, portfolio_history_workflows] + ) + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=None, + ), mock.patch.object( + workflows_retention, + "select_retention_seconds_for_database_size", + return_value=retention_seconds, + ): + summary = await workflows_retention.cleanup_outdated_automation_executions(sched) + + assert summary["deleted_global_view_executions"] == 1 + assert summary["deleted_portfolio_history_executions"] == 1 + assert summary["total_deleted"] == 2 + assert summary["retention_seconds"] == retention_seconds + mock_instance.delete_workflows_async.assert_awaited_once_with( + ["global-view-old", "portfolio-history-old"], + delete_children=False, + ) + + @pytest.mark.asyncio + async def test_uses_shorter_retention_when_database_is_large(self, temp_dbos_scheduler): + now_ms = 10_000_000 + six_hour_retention_seconds = workflows_retention.RETENTION_SECONDS_6_HOURS + twelve_hours_ms = int(12 * 60 * 60 * 1000) + updated_at_ms = now_ms - twelve_hours_ms + global_view_workflows = [ + _workflow_status_row( + workflow_id="global-view-12h-old", + updated_at=updated_at_ms, + name=_GLOBAL_VIEW_WORKFLOW_NAME, + ), + ] + + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_instance.list_workflows_async = mock.AsyncMock( + side_effect=[[], [], global_view_workflows, []] + ) + mock_instance.delete_workflows_async = mock.AsyncMock() + mock_instance._sys_db.engine = mock.Mock() + sched.INSTANCE = mock_instance + large_database_size_bytes = workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES + mock_logger = mock.Mock() + + with mock.patch("octobot_node.scheduler.workflows_retention.time.time", return_value=now_ms / 1000), mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=large_database_size_bytes, + ), mock.patch( + "octobot_node.scheduler.workflows_retention._get_logger", + return_value=mock_logger, + ): + summary = await workflows_retention.cleanup_outdated_automation_executions(sched) + + assert summary["retention_seconds"] == six_hour_retention_seconds + assert summary["database_size_bytes"] == large_database_size_bytes + assert summary["deleted_global_view_executions"] == 1 + assert summary["total_deleted"] == 1 + mock_instance.delete_workflows_async.assert_awaited_once_with( + ["global-view-12h-old"], + delete_children=False, + ) + mock_logger.info.assert_any_call( + "Using retention %s s for database size %s bytes", + six_hour_retention_seconds, + large_database_size_bytes, + ) + + +class TestFinalizeDbosCleanupRun: + @pytest.mark.asyncio + async def test_adds_size_and_updates_schedule(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + sched.INSTANCE = mock_instance + cleanup_summary = { + "deleted_by_automation": {}, + "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, + "total_deleted": 0, + } + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": True, "cron": workflows_retention.DBOS_CLEANUP_CRON_12H}), + ) as update_schedule_mock: + summary = await workflows_retention.finalize_dbos_cleanup_run(sched, cleanup_summary) + + assert summary["database_size_bytes"] == workflows_retention.DBOS_CLEANUP_SIZE_TIER_1_BYTES + assert summary["cleanup_schedule_cron"] == workflows_retention.DBOS_CLEANUP_CRON_12H + assert summary["cleanup_schedule_updated"] is True + update_schedule_mock.assert_awaited_once_with( + sched, + workflows_retention.DBOS_CLEANUP_CRON_12H, + ) + + @pytest.mark.asyncio + async def test_vacuums_when_workflows_were_deleted(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=0, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": False, "cron": workflows_retention.DBOS_CLEANUP_CRON_DAILY}), + ): + await workflows_retention.finalize_dbos_cleanup_run( + sched, + {"total_deleted": 3}, + ) + + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" + + @pytest.mark.asyncio + async def test_vacuums_when_large_database_and_nothing_deleted(self, temp_dbos_scheduler): + sched = scheduler_module.Scheduler() + mock_instance = mock.Mock() + mock_connection = mock.Mock() + mock_engine = mock.Mock() + mock_engine.begin.return_value.__enter__ = mock.Mock(return_value=mock_connection) + mock_engine.begin.return_value.__exit__ = mock.Mock(return_value=False) + mock_instance._sys_db.engine = mock_engine + sched.INSTANCE = mock_instance + + with mock.patch( + "octobot_node.scheduler.workflows_retention.get_scheduler_database_size_bytes", + return_value=workflows_retention.DBOS_CLEANUP_SIZE_TIER_2_BYTES, + ), mock.patch( + "octobot_node.scheduler.schedules.update_cleanup_schedule_cron", + mock.AsyncMock(return_value={"changed": False, "cron": workflows_retention.DBOS_CLEANUP_CRON_6H}), + ): + await workflows_retention.finalize_dbos_cleanup_run( + sched, + {"total_deleted": 0}, + ) + + mock_connection.execute.assert_called_once() + assert mock_connection.execute.call_args[0][0].text == "VACUUM" diff --git a/packages/node/tests/scheduler/test_workflows_util_automation_state.py b/packages/node/tests/scheduler/test_workflows_util_automation_state.py index 716471d521..c3aabccc71 100644 --- a/packages/node/tests/scheduler/test_workflows_util_automation_state.py +++ b/packages/node/tests/scheduler/test_workflows_util_automation_state.py @@ -10,6 +10,8 @@ import octobot_node.scheduler.workflows.params as workflow_params import octobot_node.scheduler.workflows_util as workflows_util +_AUTOMATION_STATE_KEY = "state" + _PARENT_WORKFLOW_ID = "741ce171-dac9-40be-83dc-b443c0eaf0e2" @@ -35,7 +37,7 @@ def _workflow_status_row( def _automation_task_content(*, automation_name: str) -> str: return json.dumps( { - workflows_util.STATE_KEY: { + _AUTOMATION_STATE_KEY: { "automation": { "metadata": { "automation_id": "automation_1", @@ -227,60 +229,3 @@ def test_error_workflow_uses_output_state(self): assert resolved_task is not None assert resolved_task.content == output_content - -class TestGetAutomationStateDict: - def test_success_workflow_state_dict_matches_output(self): - input_content = _automation_task_content(automation_name="from-input") - output_content = _automation_task_content(automation_name="from-output") - workflow_status = _workflow_status_with_automation_task( - status=dbos.WorkflowStatusString.SUCCESS.value, - input_content=input_content, - output_content=output_content, - ) - - state_dict = workflows_util.get_automation_state_dict(workflow_status) - - assert state_dict is not None - assert state_dict["automation"]["metadata"]["name"] == "from-output" - - -class TestPatchTaskContentDegradedState: - def test_persists_degraded_state_in_task_content(self): - task_content = _automation_task_content(automation_name="copy-grid") - - patched_content = workflows_util.patch_task_content_degraded_state( - task_content, - "not_enough_funds", - "Insufficient funds", - since=1234.5, - ) - - degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] - assert degraded_state == { - "since": 1234.5, - "error": "not_enough_funds", - "reason": "Insufficient funds", - } - - def test_preserves_existing_degraded_since_on_subsequent_patch(self): - task_content = _automation_task_content(automation_name="copy-grid") - task_content = workflows_util.patch_task_content_degraded_state( - task_content, - "not_enough_funds", - "Insufficient funds", - since=1000.0, - ) - - patched_content = workflows_util.patch_task_content_degraded_state( - task_content, - "invalid_order", - "Order volume below exchange minimum", - since=2000.0, - ) - - degraded_state = json.loads(patched_content)["state"]["automation"]["execution"]["degraded_state"] - assert degraded_state == { - "since": 1000.0, - "error": "invalid_order", - "reason": "Order volume below exchange minimum", - } diff --git a/packages/node/tests/scheduler/user_actions/test_user_action_util.py b/packages/node/tests/scheduler/user_actions/test_user_action_util.py index 9e2fe98159..86597bb3d9 100644 --- a/packages/node/tests/scheduler/user_actions/test_user_action_util.py +++ b/packages/node/tests/scheduler/user_actions/test_user_action_util.py @@ -234,6 +234,27 @@ def test_account_auth_actions_return_account_auth(self, action_type): == protocol_models.UserActionResultType.ACCOUNT_AUTH ) + def test_update_historical_exchanges_data_returns_account(self): + configuration_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = _user_action_with_configuration(configuration_inner) + assert ( + user_action_util.resolve_user_action_result_type(user_action) + == protocol_models.UserActionResultType.ACCOUNT + ) + + def test_reset_account_trading_data_returns_account(self): + configuration_inner = protocol_models.ResetAccountTradingDataConfiguration( + action_type=protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA, + account_ids=["acc-1"], + ) + user_action = _user_action_with_configuration(configuration_inner) + assert ( + user_action_util.resolve_user_action_result_type(user_action) + == protocol_models.UserActionResultType.ACCOUNT + ) + class TestBuildSynthesizedFailureUserActionResult: def test_automation_result(self): diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py index 57d6ae3f4f..5a1926e327 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/account/test_create_account.py @@ -17,6 +17,7 @@ import mock import pytest +import octobot_sync.constants as sync_constants import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_protocol.models as protocol_models @@ -38,11 +39,16 @@ async def test_calls_provider_create_with_wallet_and_account(self): ) user_action = protocol_models.UserAction(id="ua-create", configuration=account_executor_test_utils.wrap_configuration(inner)) provider_mock = mock.Mock() + trading_provider_mock = mock.Mock() with ( mock.patch( "octobot_sync.sync.collection_providers.AccountProvider.instance", return_value=provider_mock, ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountTradingProvider.instance", + return_value=trading_provider_mock, + ), mock.patch.object( account_state_updater_module, "update_account_state", @@ -52,6 +58,14 @@ async def test_calls_provider_create_with_wallet_and_account(self): executor = create_account_executor.CreateAccountActionExecutor(account_executor_test_utils.WALLET_ADDRESS) await executor.execute(user_action) provider_mock.create_item.assert_called_once_with(account_executor_test_utils.WALLET_ADDRESS, account_model) + trading_provider_mock.save_state.assert_called_once() + saved_user_id, saved_account_id, saved_trading_state = trading_provider_mock.save_state.call_args.args + assert saved_user_id == account_executor_test_utils.WALLET_ADDRESS + assert saved_account_id == "new-acc" + assert saved_trading_state.version == sync_constants.USER_ACCOUNTS_TRADING_STATE_VERSION + assert saved_trading_state.account_trading.orders is None + assert saved_trading_state.account_trading.trades is None + assert saved_trading_state.account_trading.positions is None provider_assertions.assert_user_action_terminal_state( user_action=user_action, expected_status=protocol_models.UserActionStatus.COMPLETED, diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_reset_account_trading_data.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_reset_account_trading_data.py new file mode 100644 index 0000000000..b0188f216a --- /dev/null +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_reset_account_trading_data.py @@ -0,0 +1,154 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors + +from .account import account_executor_test_utils +from . import provider_assertions + +import octobot_node.errors as node_errors + +import octobot_node.scheduler.user_actions.user_actions_executor.account.reset_account_trading_data as reset_account_trading_data_executor + + +class TestResetAccountTradingDataActionExecutorExecute: + @pytest.mark.asyncio + async def test_clears_trading_data_and_completes(self): + reset_inner = protocol_models.ResetAccountTradingDataConfiguration( + action_type=protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA, + account_ids=["acc-owned"], + ) + user_action = protocol_models.UserAction( + id="ua-reset-account-trading", + configuration=account_executor_test_utils.wrap_configuration(reset_inner), + ) + owned_account = account_executor_test_utils.minimal_exchange_account(account_id="acc-owned") + account_provider_mock = mock.Mock() + account_provider_mock.get_item.return_value = owned_account + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=account_provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.reset_account_trading_data.accounts_trading_module.reset_account_trading_data", + ) as reset_mock, + ): + executor = reset_account_trading_data_executor.ResetAccountTradingDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + account_provider_mock.get_item.assert_called_once_with( + account_executor_test_utils.WALLET_ADDRESS, + "acc-owned", + ) + reset_mock.assert_called_once_with( + account_executor_test_utils.WALLET_ADDRESS, + "acc-owned", + ) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="account", + expect_error_details=False, + ) + + @pytest.mark.asyncio + async def test_raises_when_account_id_not_owned(self): + reset_inner = protocol_models.ResetAccountTradingDataConfiguration( + action_type=protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA, + account_ids=["foreign-acc"], + ) + user_action = protocol_models.UserAction( + id="ua-reset-account-trading-foreign", + configuration=account_executor_test_utils.wrap_configuration(reset_inner), + ) + account_provider_mock = mock.Mock() + account_provider_mock.get_item.side_effect = collection_errors.ItemNotFoundError("missing") + with ( + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=account_provider_mock, + ), + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.account.reset_account_trading_data.accounts_trading_module.reset_account_trading_data", + ) as reset_mock, + ): + executor = reset_account_trading_data_executor.ResetAccountTradingDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(collection_errors.ItemNotFoundError): + await executor.execute(user_action) + reset_mock.assert_not_called() + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.ACCOUNT_NOT_FOUND, + ) + + @pytest.mark.asyncio + async def test_raises_when_account_ids_empty(self): + reset_inner = protocol_models.ResetAccountTradingDataConfiguration.model_construct( + action_type=protocol_models.UserActionType.RESET_ACCOUNT_TRADING_DATA, + account_ids=[], + ) + user_action = protocol_models.UserAction( + id="ua-reset-account-trading-empty", + configuration=protocol_models.UserActionConfiguration.model_construct( + actual_instance=reset_inner, + ), + ) + executor = reset_account_trading_data_executor.ResetAccountTradingDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(node_errors.InvalidUserActionPayloadError, match="at least one account id"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + @pytest.mark.asyncio + async def test_raises_when_payload_type_is_wrong(self): + inner = protocol_models.DeleteAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_DELETE, + id="acc-1", + ) + user_action = protocol_models.UserAction( + id="ua-reset-account-trading-wrong", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + executor = reset_account_trading_data_executor.ResetAccountTradingDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(node_errors.InvalidUserActionPayloadError, match="ResetAccountTradingDataConfiguration"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INVALID_CONFIGURATION, + ) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py new file mode 100644 index 0000000000..42a41cfb0d --- /dev/null +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/test_update_historical_exchanges_data.py @@ -0,0 +1,207 @@ +# Drakkar-Software OctoBot-Node +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock +import pytest + +import octobot_protocol.models as protocol_models +import octobot_sync.sync.collection_backend.errors as collection_errors + +from .account import account_executor_test_utils +from . import provider_assertions + +import octobot_node.errors as node_errors + +import octobot_node.scheduler.tasks as scheduler_tasks_module +import octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data as update_historical_exchanges_data_executor +import octobot_node.scheduler.workflows.params as workflow_params_module +import octobot_node.scheduler.user_actions.user_action_post_actions as user_action_post_actions_module + +from tests.scheduler import temp_dbos_scheduler + + +class TestUpdateHistoricalExchangesDataActionExecutorExecute: + @pytest.mark.asyncio + async def test_sets_portfolio_history_collection_params_and_completes(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = protocol_models.UserAction( + id="ua-update-historical", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + collection_params = executor.post_actions.portfolio_history_collection_params + assert collection_params is not None + assert collection_params.wallet_ids == [account_executor_test_utils.WALLET_ADDRESS] + assert collection_params.account_ids is None + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.COMPLETED, + result_channel="account", + expect_error_details=False, + ) + + @pytest.mark.asyncio + async def test_validates_account_ids_belong_to_user(self): + owned_account = account_executor_test_utils.minimal_exchange_account(account_id="acc-owned") + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + account_ids=["acc-owned"], + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-scoped", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + provider_mock = mock.Mock() + provider_mock.get_item.return_value = owned_account + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + await executor.execute(user_action) + provider_mock.get_item.assert_called_once_with( + account_executor_test_utils.WALLET_ADDRESS, + "acc-owned", + ) + collection_params = executor.post_actions.portfolio_history_collection_params + assert collection_params is not None + assert collection_params.account_ids == ["acc-owned"] + + @pytest.mark.asyncio + async def test_raises_when_account_id_not_owned(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + account_ids=["foreign-acc"], + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-foreign", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + provider_mock = mock.Mock() + provider_mock.get_item.side_effect = collection_errors.ItemNotFoundError("missing") + with ( + mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=True, + ), + mock.patch( + "octobot_sync.sync.collection_providers.AccountProvider.instance", + return_value=provider_mock, + ), + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(collection_errors.ItemNotFoundError): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.ACCOUNT_NOT_FOUND, + ) + + @pytest.mark.asyncio + async def test_raises_when_scheduler_not_initialized(self): + update_inner = protocol_models.UpdateHistoricalExchangesDataConfiguration( + action_type=protocol_models.UserActionType.UPDATE_HISTORICAL_EXCHANGES_DATA, + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-no-scheduler", + configuration=account_executor_test_utils.wrap_configuration(update_inner), + ) + with mock.patch( + "octobot_node.scheduler.user_actions.user_actions_executor.historical_data.update_historical_exchanges_data.scheduler_module.is_initialized", + return_value=False, + ): + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(RuntimeError, match="Scheduler is not initialized"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INTERNAL_ERROR, + ) + + @pytest.mark.asyncio + async def test_raises_when_payload_type_is_wrong(self): + inner = protocol_models.DeleteAccountConfiguration( + action_type=protocol_models.UserActionType.ACCOUNT_DELETE, + id="acc-1", + ) + user_action = protocol_models.UserAction( + id="ua-update-historical-wrong", + configuration=account_executor_test_utils.wrap_configuration(inner), + ) + executor = update_historical_exchanges_data_executor.UpdateHistoricalExchangesDataActionExecutor( + account_executor_test_utils.WALLET_ADDRESS, + ) + with pytest.raises(node_errors.InvalidUserActionPayloadError, match="UpdateHistoricalExchangesDataConfiguration"): + await executor.execute(user_action) + provider_assertions.assert_user_action_terminal_state( + user_action=user_action, + expected_status=protocol_models.UserActionStatus.FAILED, + result_channel="account", + expect_error_details=True, + expected_error_message=protocol_models.AccountActionResultErrorMessage.INVALID_CONFIGURATION, + ) + + +class TestUpdateHistoricalExchangesDataAfterUserActionExecution: + @pytest.mark.asyncio + async def test_triggers_portfolio_history_collection_when_post_action_set(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.user_action_workflow as user_action_workflow_module + + trigger_mock = mock.AsyncMock(return_value="portfolio-history-workflow-id") + collection_params = workflow_params_module.PortfolioHistoryCollectionParams( + wallet_ids=["wallet-user-1"], + account_ids=["acc-1"], + ) + execution_result = workflow_params_module.UserActionExecutionResult( + updated_user_action=protocol_models.UserAction(id="ua-update-historical"), + post_actions=user_action_post_actions_module.UserActionPostActions( + portfolio_history_collection_params=collection_params, + ), + ) + with mock.patch.object( + scheduler_tasks_module, + "trigger_portfolio_history_collection", + trigger_mock, + ): + await user_action_workflow_module.UserActionWorkflow.after_user_action_execution(execution_result) + trigger_mock.assert_awaited_once_with(collection_params) diff --git a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py index 8b67437d9c..b2a280ba2c 100644 --- a/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py +++ b/packages/node/tests/scheduler/user_actions/user_actions_executor/util/test_account_state_updater.py @@ -550,23 +550,6 @@ async def test_returns_optimistic_rights_when_permissions_fetch_is_not_supported ] -class TestAccountStateUpdaterAccountPermissionsFromApiKeyRights: - def test_maps_known_api_key_rights_to_account_permissions(self): - account_permissions = account_state_updater_module._account_permissions_from_api_key_rights([ - trading_enums.APIKeyRights.READING, - trading_enums.APIKeyRights.SPOT_TRADING, - trading_enums.APIKeyRights.FUTURES_TRADING, - trading_enums.APIKeyRights.WITHDRAWALS, - trading_enums.APIKeyRights.MARGIN_TRADING, - ]) - assert account_permissions == [ - protocol_models.AccountPermission.READ, - protocol_models.AccountPermission.SPOT_TRADING, - protocol_models.AccountPermission.FUTURES_TRADING, - protocol_models.AccountPermission.WITHDRAW, - ] - - class TestAccountStateUpdaterAssetsFromBalance: def test_maps_non_zero_holdings_to_detailed_assets(self): balance = { diff --git a/packages/node/tests/scheduler/workflows/test_automation_workflow.py b/packages/node/tests/scheduler/workflows/test_automation_workflow.py index f16b61146b..f2aab64a5f 100644 --- a/packages/node/tests/scheduler/workflows/test_automation_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_automation_workflow.py @@ -16,6 +16,7 @@ import asyncio import contextlib +import copy import importlib import json import os @@ -29,9 +30,13 @@ import octobot_trading.constants import octobot_trading.errors as octobot_trading_errors +import octobot_trading.enums as trading_enums import octobot_commons.cryptography +import octobot.community.wallet_backend.errors as wallet_backend_errors_module + import octobot_copy.constants as copy_constants +import octobot_copy.errors as copy_errors import octobot_protocol.models as protocol_models import octobot_node.config import octobot_node.constants @@ -41,7 +46,7 @@ import octobot_node.errors as errors import octobot_node.models import octobot_node.scheduler.workflows.params as params -import octobot_node.scheduler.octobot_flow_client as octobot_flow_client +import octobot_node.scheduler.automations.octobot_flow_client as octobot_flow_client import octobot_node.scheduler.encryption.task_inputs as task_inputs_encryption import octobot_node.scheduler.task_context as task_context @@ -98,10 +103,92 @@ def _automation_state_dict_with_scheduled_to( return state +def _sample_postpone_dag_actions() -> list[dict[str, typing.Any]]: + return [ + {"id": "action_dsl_1", "dsl_script": "1 if True else 2"}, + { + "id": "action_dsl_2", + "dsl_script": "1 if True else 2", + "dependencies": [{"action_id": "action_dsl_1"}], + }, + ] + + +def _actions_dag_from_task_content(task_content: str) -> dict: + return json.loads(task_content)["state"]["automation"]["actions_dag"] + + +def _actions_dag_from_execute_iteration_result(result: dict) -> dict: + next_description = json.loads(result["next_iteration_description"]) + return next_description["state"]["automation"]["actions_dag"] + + +def _actions_dag_from_enqueued_inputs(enqueued_inputs: dict) -> dict: + return json.loads(enqueued_inputs["task"]["content"])["state"]["automation"]["actions_dag"] + + +def _canonical_actions_dag(actions_dag: dict) -> dict: + automation_state = octobot_flow.entities.AutomationState.from_dict({ + "automation": { + "metadata": {"automation_id": "canonical"}, + "actions_dag": actions_dag, + "execution": {}, + } + }) + return automation_state.to_dict(include_default_values=False)["automation"]["actions_dag"] + + +def _assert_actions_dag_equal(actions_dag_left: dict, actions_dag_right: dict) -> None: + assert _canonical_actions_dag(actions_dag_left) == _canonical_actions_dag(actions_dag_right) + + +def _assert_actions_dag_unchanged(task_content: str, result: dict) -> None: + _assert_actions_dag_equal( + _actions_dag_from_task_content(task_content), + _actions_dag_from_execute_iteration_result(result), + ) + + +def _assert_skip_postpone_preserves_state(task_content: str, result: dict) -> None: + assert result["next_iteration_description"] == task_content + _assert_actions_dag_unchanged(task_content, result) + + +def _assert_trading_postpone_preserves_dag(task_content: str, result: dict) -> None: + _assert_actions_dag_unchanged(task_content, result) + + +def _automation_state_with_trade_count(trade_count: int) -> octobot_flow.entities.AutomationState: + order_columns = trading_enums.ExchangeConstantsOrderColumns + trades = [ + { + order_columns.EXCHANGE_TRADE_ID.value: f"trade-{trade_index}", + order_columns.SYMBOL.value: "BTC/USDT", + order_columns.TIMESTAMP.value: float(trade_index), + } + for trade_index in range(trade_count) + ] + elements = octobot_flow.entities.ExchangeAccountElements() + elements.trades = trades + exchange_details = octobot_flow.entities.ExchangeAccountDetails() + exchange_details.exchange_details.exchange_account_id = "acc-sync-1" + automation_state = octobot_flow.entities.AutomationState( + automation=octobot_flow.entities.AutomationDetails( + metadata=octobot_flow.entities.AutomationMetadata(automation_id="automation_1"), + ), + exchange_account_details=exchange_details, + ) + automation_state.automation.exchange_account_elements = elements + return automation_state + + def _octobot_actions_job_mock_class_pending_priority_skipped( *, automation_inner_state: dict[str, typing.Any], - skip_error: "octobot_flow.errors.PendingPriorityActionsSkippedError", + skip_error: typing.Union[ + "octobot_flow.errors.PendingPriorityActionsSkippedError", + copy_errors.OutdatedReferenceAccountError, + ], ) -> mock.Mock: async def run_raises(*args, **kwargs): raise skip_error @@ -242,6 +329,13 @@ def _trading_signal_update_envelope(signal_dicts: list[dict]) -> dict[str, typin ).to_dict(include_default_values=False) +def _forced_trigger_update_envelope() -> dict[str, typing.Any]: + return params.AutomationWorkflowActionUpdate( + actions_type=octobot_node.enums.AutomationWorkflowActionTypes.FORCED_TRIGGER.value, + actions_details=[], + ).to_dict(include_default_values=False) + + @pytest.fixture def parsed_inputs(): task = octobot_node.models.Task( @@ -756,7 +850,7 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( expected_error_status, expected_retry_delay_seconds, ): - task_content = json.dumps({"state": _automation_state_dict([])}) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( @@ -772,14 +866,14 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( "octobot_node.scheduler.workflows.automation_workflow.time.time", return_value=fixed_now, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_not_called() + persist_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) assert parsed_progress_status.error == expected_error_status assert parsed_progress_status.error_message == str(run_side_effect) @@ -793,6 +887,42 @@ async def test_execute_iteration_postponed_error_sets_postponed_iteration( assert degraded_state["error"] == expected_error_status assert degraded_state["reason"] == str(run_side_effect) assert degraded_state["since"] == fixed_now + _assert_trading_postpone_preserves_dag(task_content, result) + + @pytest.mark.asyncio + @required_imports + async def test_execute_iteration_postponed_error_round_trips_serialization( + self, + import_automation_workflow, + task, + ): + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ): + raw_result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + iteration_result = params.AutomationWorkflowIterationResult.from_dict(raw_result) + assert iteration_result.progress_status.postponed_iteration is True + assert iteration_result.has_next_actions is True + assert iteration_result.progress_status.error == ( + octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value + ) + _assert_trading_postpone_preserves_dag(task_content, raw_result) @pytest.mark.asyncio @required_imports @@ -845,8 +975,6 @@ async def test_execute_iteration_passes_trading_signals_to_octobot_actions_job( async def test_execute_iteration_persists_open_orders_to_account_trading( self, import_automation_workflow, task ): - import octobot_trading.enums as trading_enums - task.user_id = "0xwallet-trading-sync" task.content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", @@ -893,19 +1021,16 @@ async def test_execute_iteration_persists_open_orders_to_account_trading( "OctoBotActionsJob", mock_octobot_actions_job_class, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_called_once_with( + persist_account_trading_mock.assert_called_once_with( task.user_id, - "acc-sync-1", - [open_order], - [], - [], + automation_state.to_dict(include_default_values=False), ) @pytest.mark.asyncio @@ -913,9 +1038,6 @@ async def test_execute_iteration_persists_open_orders_to_account_trading( async def test_execute_iteration_continues_when_trading_persistence_wallet_missing( self, import_automation_workflow, task ): - import octobot.community.wallet_backend.errors as wallet_backend_errors_module - import octobot_trading.enums as trading_enums - task.user_id = "0xwallet-trading-sync" task.content = json.dumps({"params": {"ACTIONS": "trade", "EXCHANGE_FROM": "binance", "ORDER_SYMBOL": "ETH/BTC", "ORDER_AMOUNT": 1, "ORDER_TYPE": "market", @@ -957,8 +1079,8 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi "OctoBotActionsJob", mock_octobot_actions_job_class, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading", side_effect=wallet_backend_errors_module.WalletNotFoundError("Wallet not found"), ): result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( @@ -969,6 +1091,145 @@ async def test_execute_iteration_continues_when_trading_persistence_wallet_missi assert parsed_progress_status.error is None +class TestPersistBeforeTrimTrades: + @pytest.mark.asyncio + @required_imports + async def test_persist_receives_full_trades_then_state_trimmed_to_live_window( + self, import_automation_workflow, task + ): + task.user_id = "0xwallet-trading-sync" + task.content = json.dumps({"params": {"ACTIONS": "trade"}}) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + automation_state = _automation_state_with_trade_count(120) + next_actions_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + ) + action = octobot_flow.entities.ConfiguredActionDetails(id="action_1", action="trade") + mock_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[action], + next_actions_description=next_actions_description, + has_next_actions=True, + actions_dag=None, + should_stop=False, + ) + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_on_result=lambda result_ref: _apply_octobot_actions_job_result_template(result_ref, mock_result), + ) + persisted_state_snapshots: list[dict] = [] + + def capture_persisted_state(user_id, state_dict): + if state_dict is not None: + persisted_state_snapshots.append(copy.deepcopy(state_dict)) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + task_context, + "encrypted_task", + mock.MagicMock(), + ) as mock_encrypted, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + side_effect=capture_persisted_state, + ): + mock_encrypted.return_value.__enter__ = mock.Mock(return_value=None) + mock_encrypted.return_value.__exit__ = mock.Mock(return_value=None) + await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + assert len(persisted_state_snapshots) == 1 + persisted_state = octobot_flow.entities.AutomationState.from_dict(persisted_state_snapshots[0]) + assert len(persisted_state.automation.exchange_account_elements.trades) == 120 + trimmed_state = octobot_flow.entities.AutomationState.from_dict(mock_result.next_actions_description.state) + trimmed_elements = trimmed_state.automation.exchange_account_elements + assert len(trimmed_elements.trades) == octobot_node.constants.AUTOMATION_LIVE_STATE_MAX_TRADES + assert len(trimmed_elements.trade_summaries.get("BTC/USDT", [])) == 20 + + @pytest.mark.asyncio + @required_imports + async def test_trim_runs_when_wallet_persistence_skipped(self, import_automation_workflow, task): + task.user_id = "0xwallet-trading-sync" + task.content = json.dumps({"params": {"ACTIONS": "trade"}}) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + automation_state = _automation_state_with_trade_count(120) + next_actions_description = octobot_flow_client.OctoBotActionsJobDescription( + state=automation_state.to_dict(include_default_values=False), + ) + mock_result = octobot_flow_client.OctoBotActionsJobResult( + processed_actions=[], + next_actions_description=next_actions_description, + has_next_actions=True, + actions_dag=None, + should_stop=False, + ) + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_on_result=lambda result_ref: _apply_octobot_actions_job_result_template(result_ref, mock_result), + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + task_context, + "encrypted_task", + mock.MagicMock(), + ) as mock_encrypted, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "trim_live_trades_in_iteration_state", + ) as trim_mock: + mock_encrypted.return_value.__enter__ = mock.Mock(return_value=None) + mock_encrypted.return_value.__exit__ = mock.Mock(return_value=None) + await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + trim_mock.assert_called_once_with( + mock_result.next_actions_description.state, + octobot_node.constants.AUTOMATION_LIVE_STATE_MAX_TRADES, + ) + + @pytest.mark.asyncio + @required_imports + async def test_postponed_iteration_skips_persist_and_trim( + self, import_automation_workflow, task + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + skip_error = octobot_flow.errors.PendingPriorityActionsSkippedError("skipped") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_mock, mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "trim_live_trades_in_iteration_state", + ) as trim_mock: + result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + persist_mock.assert_not_called() + trim_mock.assert_not_called() + _assert_skip_postpone_preserves_state(task_content, result) + + class TestExecuteIterationPendingPriorityActionsSkippedError: @pytest.mark.asyncio @required_imports @@ -976,7 +1237,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( self, import_automation_workflow, task ): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -997,21 +1260,21 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( "octobot_node.scheduler.workflows.automation_workflow.time.time", return_value=fixed_now, ), mock.patch.object( - octobot_node.scheduler.workflows.automation_workflow.accounts_trading_protocol, - "update_account_trading", - ) as update_account_trading_mock: + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( inputs, None ) - update_account_trading_mock.assert_not_called() + persist_account_trading_mock.assert_not_called() parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) assert parsed_progress_status.postponed_iteration is True assert parsed_progress_status.next_step_at == scheduled_to assert parsed_progress_status.error is None assert parsed_progress_status.error_message is None assert result["has_next_actions"] is True - assert result["next_iteration_description"] == task_content + _assert_skip_postpone_preserves_state(task_content, result) next_iteration_description = json.loads(result["next_iteration_description"]) execution = next_iteration_description["state"]["automation"].get("execution", {}) assert "degraded_state" not in execution @@ -1020,7 +1283,9 @@ async def test_postpones_iteration_at_scheduled_to_without_degraded_state( @required_imports async def test_logs_pending_priority_skipped_error(self, import_automation_workflow, task): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1051,7 +1316,9 @@ async def test_logs_pending_priority_skipped_error(self, import_automation_workf @required_imports async def test_postponed_log_uses_none_error_fields(self, import_automation_workflow, task): scheduled_to = 5000.0 - automation_inner_state = _automation_state_dict_with_scheduled_to(scheduled_to) + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) task_content = json.dumps({"state": automation_inner_state}) task.content = task_content inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) @@ -1083,6 +1350,158 @@ async def test_postponed_log_uses_none_error_fields(self, import_automation_work ) +class TestExecuteIterationOutdatedReferenceAccountError: + @pytest.mark.asyncio + @required_imports + async def test_postpones_iteration_at_scheduled_to_without_degraded_state( + self, import_automation_workflow, task + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=outdated_error, + ) + fixed_now = 1000.0 + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ) as persist_account_trading_mock: + result = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow.execute_iteration( + inputs, None + ) + + persist_account_trading_mock.assert_not_called() + parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) + assert parsed_progress_status.postponed_iteration is True + assert parsed_progress_status.next_step_at == scheduled_to + assert parsed_progress_status.error is None + assert parsed_progress_status.error_message is None + assert result["has_next_actions"] is True + _assert_skip_postpone_preserves_state(task_content, result) + next_iteration_description = json.loads(result["next_iteration_description"]) + execution = next_iteration_description["state"]["automation"].get("execution", {}) + assert "degraded_state" not in execution + + @pytest.mark.asyncio + @required_imports + async def test_logs_outdated_reference_account_info(self, import_automation_workflow, task): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + outdated_error = copy_errors.OutdatedReferenceAccountError("reference account is outdated") + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=outdated_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + await automation_workflow.execute_iteration(inputs, None) + + mock_logger.info.assert_any_call( + f"Outdated reference account, skipping copy iteration: {outdated_error}" + ) + mock_logger.error.assert_not_called() + + @pytest.mark.parametrize( + "skip_error,expected_log_method,expected_log_message", + [ + pytest.param( + octobot_flow.errors.PendingPriorityActionsSkippedError("stale priority skipped"), + "error", + "Pending priority actions were skipped: stale priority skipped", + id="pending_priority_skipped", + ), + pytest.param( + copy_errors.OutdatedReferenceAccountError("reference account is outdated"), + "info", + "Outdated reference account, skipping copy iteration: reference account is outdated", + id="outdated_reference_account", + ), + ], + ) + @pytest.mark.asyncio + @required_imports + async def test_shared_skip_handler_postpones_without_degraded_state( + self, + import_automation_workflow, + task, + skip_error, + expected_log_method, + expected_log_message, + ): + scheduled_to = 5000.0 + automation_inner_state = _automation_state_dict_with_scheduled_to( + scheduled_to, _sample_postpone_dag_actions() + ) + task_content = json.dumps({"state": automation_inner_state}) + task.content = task_content + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict(include_default_values=False) + mock_octobot_actions_job_class = _octobot_actions_job_mock_class_pending_priority_skipped( + automation_inner_state=automation_inner_state, + skip_error=skip_error, + ) + mock_logger = mock.Mock() + automation_workflow = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + + with mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + automation_workflow, + "get_logger", + return_value=mock_logger, + ): + result = await automation_workflow.execute_iteration(inputs, None) + + if expected_log_method == "error": + getattr(mock_logger, expected_log_method).assert_called_once_with(expected_log_message) + else: + getattr(mock_logger, expected_log_method).assert_any_call(expected_log_message) + parsed_progress_status = params.ProgressStatus.model_validate(result["progress_status"]) + assert parsed_progress_status.postponed_iteration is True + assert parsed_progress_status.error is None + _assert_skip_postpone_preserves_state(task_content, result) + + +class TestShouldRetryOutdatedReferenceAccountError: + def test_should_retry_returns_false_for_outdated_reference_account_error( + self, import_automation_workflow + ): + assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_retry( + copy_errors.OutdatedReferenceAccountError("reference account is outdated") + ) is False + + class TestExecuteAutomationPostponedIteration: @pytest.mark.asyncio @required_imports @@ -1276,6 +1695,93 @@ async def test_process_pending_schedules_next_when_no_priority_actions( mock_wait.assert_awaited_once_with(parsed_inputs, 0) mock_schedule.assert_called_once() + @pytest.mark.asyncio + async def test_process_pending_ignores_forced_trigger_after_iteration( + self, import_automation_workflow, parsed_inputs, iteration_result + ): + mock_wait = mock.AsyncMock( + side_effect=[ + _forced_trigger_update_envelope(), + None, + ] + ) + mock_schedule = mock.AsyncMock() + mock_iteration = mock.AsyncMock() + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_schedule_next_iteration", + mock_schedule, + ): + should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( + parsed_inputs, iteration_result + ) + assert should_continue is True + assert mock_wait.await_count == 2 + mock_iteration.assert_not_awaited() + mock_schedule.assert_called_once() + + @pytest.mark.asyncio + async def test_process_pending_still_processes_user_actions_after_ignoring_forced_trigger( + self, import_automation_workflow, parsed_inputs, iteration_result + ): + result_with_next = params.AutomationWorkflowIterationResult( + progress_status=params.ProgressStatus( + latest_step="step_1", + next_step="step_2", + next_step_at=0.0, + remaining_steps=1, + error=None, + should_stop=False, + ), + next_iteration_description='{"state": {"automation": {}}}', + has_next_actions=True, + ) + mock_wait = mock.AsyncMock( + side_effect=[ + _forced_trigger_update_envelope(), + _user_actions_update_envelope([{"action": "stop"}]), + None, + ] + ) + mock_iteration = mock.AsyncMock( + return_value=result_with_next.to_dict(include_default_values=False) + ) + mock_schedule = mock.AsyncMock() + + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_wait_and_trigger_on_actions_update", + mock_wait, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "execute_iteration", + mock_iteration, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_should_continue_workflow", + mock.Mock(return_value=True), + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "_schedule_next_iteration", + mock_schedule, + ): + should_continue, _ = await octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._process_pending_priority_actions_and_reschedule( + parsed_inputs, iteration_result + ) + assert should_continue is True + assert mock_wait.await_count == 3 + mock_iteration.assert_awaited_once() + mock_schedule.assert_called_once() + @pytest.mark.asyncio async def test_process_pending_returns_false_when_should_stop(self, import_automation_workflow, parsed_inputs, iteration_result): iteration_result.progress_status.should_stop = True @@ -1536,6 +2042,25 @@ def test_should_continue_returns_stop_on_error_when_error(self, import_automatio parsed_inputs, progress, False ) is False + def test_should_continue_returns_true_when_postponed_iteration_despite_error( + self, import_automation_workflow, parsed_inputs + ): + progress = params.ProgressStatus( + error=octobot_flow.enums.ActionErrorStatus.INTERNAL_ERROR.value, + postponed_iteration=True, + should_stop=False, + ) + with mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_continue_workflow( + parsed_inputs, progress, False + ) is True + workflow_logger.error.assert_not_called() + def test_should_continue_returns_false_when_should_stop(self, import_automation_workflow, parsed_inputs): progress = params.ProgressStatus(error=None, should_stop=True) assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._should_continue_workflow( @@ -1555,6 +2080,170 @@ def test_should_continue_returns_true_by_no_reason_to_stop(self, import_automati ) is True +class TestExecuteAutomationPostponedFailedRequestIntegration: + def setup_method(self): + octobot_trading.constants.ALLOW_FUNDS_TRANSFER = True + self._no_encrypt_rsa = mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_RSA_PRIVATE_KEY", None) + self._no_encrypt_ecdsa = mock.patch.object(octobot_node.config.settings, "TASKS_SERVER_ECDSA_PRIVATE_KEY", None) + self._no_encrypt_rsa.start() + self._no_encrypt_ecdsa.start() + + def teardown_method(self): + octobot_trading.constants.ALLOW_FUNDS_TRANSFER = False + self._no_encrypt_rsa.stop() + self._no_encrypt_ecdsa.stop() + + @pytest.mark.asyncio + @required_imports + async def test_execute_automation_reschedules_after_failed_request_via_dbos_step( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, _ = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task = octobot_node.models.Task( + name="postponed_failed_request", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + fixed_now = 1000.0 + recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" + real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) + automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + with mock.patch(recv_path, mock.AsyncMock(return_value=None)), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, + "enqueue_async", + enqueue_mock, + ), mock.patch.object( + automation_wf, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + automation_wf.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + assert workflow_result is None + enqueue_mock.assert_called_once() + enqueued_inputs = enqueue_mock.call_args.kwargs["inputs"] + assert enqueued_inputs["execution_time"] == ( + fixed_now + octobot_node.constants.DEFAULT_WORKFLOW_RESCHEDULE_IN_SECONDS + ) + workflows = await temp_dbos_scheduler.INSTANCE.list_workflows_async() + assert len(workflows) >= 2 + error_messages = [str(call) for call in workflow_logger.error.call_args_list] + assert not any("unrecoverable iteration error" in message for message in error_messages) + assert not any("Automation stopped (remaining steps:" in message for message in error_messages) + info_messages = [str(call) for call in workflow_logger.info.call_args_list] + assert any("Enqueuing next iteration" in message for message in info_messages) + _assert_actions_dag_equal( + _actions_dag_from_enqueued_inputs(enqueued_inputs), + _actions_dag_from_task_content(task_content), + ) + + @pytest.mark.asyncio + @required_imports + async def test_execute_automation_reschedules_after_postponed_iteration_with_priority_trading_signal( + self, + import_automation_workflow, + temp_dbos_scheduler, + ): + failed_request_error = octobot_trading_errors.FailedRequest("Exchange API request failed") + mock_octobot_actions_job_class, run_mock = _octobot_actions_job_mock_class( + run_side_effect=failed_request_error, + ) + trading_signal = octobot_flow.entities.TradingSignal( + account=protocol_models.CopiedAccount( + version=copy_constants.COPIED_ACCOUNT_VERSION, + updated_at=time.time(), + copied_assets=[], + ), + strategy_id="test-strategy-id", + ) + trading_signal_envelope = _trading_signal_update_envelope( + [trading_signal.to_dict(include_default_values=False)] + ) + task_content = json.dumps({"state": _automation_state_dict(_sample_postpone_dag_actions())}) + task = octobot_node.models.Task( + name="postponed_failed_request_priority_signal", + content=task_content, + type=octobot_node.models.TaskType.EXECUTE_ACTIONS.value, + ) + inputs = params.AutomationWorkflowInputs(task=task, execution_time=0).to_dict( + include_default_values=False + ) + inputs["task"] = task.model_dump(exclude_defaults=True) + fixed_now = 1000.0 + recv_path = "octobot_node.scheduler.workflows.automation_workflow.SCHEDULER.INSTANCE.recv_async" + real_enqueue_async = temp_dbos_scheduler.AUTOMATION_WORKFLOW_QUEUE.enqueue_async + enqueue_mock = mock.AsyncMock(wraps=real_enqueue_async) + automation_wf = octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow + with mock.patch( + recv_path, + mock.AsyncMock(side_effect=[None, trading_signal_envelope, None]), + ), mock.patch( + "octobot_node.scheduler.workflows.automation_workflow.time.time", + return_value=fixed_now, + ), mock.patch.object( + octobot_flow_client, + "OctoBotActionsJob", + mock_octobot_actions_job_class, + ), mock.patch.object( + octobot_node.scheduler.workflows.automation_workflow.account_state_persistence_module, + "persist_account_trading_from_iteration_state", + ), mock.patch.object( + octobot_node.scheduler.SCHEDULER.AUTOMATION_WORKFLOW_QUEUE, + "enqueue_async", + enqueue_mock, + ), mock.patch.object( + automation_wf, + "get_logger", + return_value=mock.Mock(), + ) as get_logger_mock: + workflow_logger = get_logger_mock.return_value + handle = await temp_dbos_scheduler.INSTANCE.start_workflow_async( + automation_wf.execute_automation, + inputs=inputs, + ) + workflow_result = await handle.get_result() + + assert workflow_result is None + assert run_mock.await_count == 2 + enqueue_mock.assert_called_once() + error_messages = [str(call) for call in workflow_logger.error.call_args_list] + assert not any("unrecoverable iteration error" in message for message in error_messages) + assert not any("Automation stopped (remaining steps:" in message for message in error_messages) + info_messages = [str(call) for call in workflow_logger.info.call_args_list] + assert any("Enqueuing next iteration" in message for message in info_messages) + enqueued_inputs = enqueue_mock.call_args.kwargs["inputs"] + _assert_actions_dag_equal( + _actions_dag_from_enqueued_inputs(enqueued_inputs), + _actions_dag_from_task_content(task_content), + ) + + class TestGetActionsSummary: def test_get_actions_summary_empty_returns_empty_string(self, import_automation_workflow): assert octobot_node.scheduler.workflows.automation_workflow.AutomationWorkflow._get_actions_summary([]) == "" diff --git a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py index 61eb5832e2..c0802bbf7f 100644 --- a/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py +++ b/packages/node/tests/scheduler/workflows/test_dbos_cleanup_workflow.py @@ -35,14 +35,17 @@ async def test_delegates_to_cleanup_outdated_automation_executions(self, dbos_cl ), mock.patch( "octobot_node.scheduler.workflows_retention.cleanup_outdated_automation_executions", mock.AsyncMock(return_value=expected_summary), - ) as cleanup_mock: + ) as cleanup_mock, mock.patch( + "octobot_node.scheduler.workflows_retention.finalize_dbos_cleanup_run", + mock.AsyncMock(return_value={**expected_summary, "database_size_bytes": 1024, "cleanup_schedule_cron": "0 0 * * *", "cleanup_schedule_updated": False}), + ) as finalize_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( datetime.datetime.now(datetime.timezone.utc), - None, ) cleanup_mock.assert_awaited_once() - assert result == expected_summary + finalize_mock.assert_awaited_once() + assert result["total_deleted"] == 3 @pytest.mark.asyncio async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module): @@ -62,7 +65,6 @@ async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module ) as skip_for_scheduled_time_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( datetime.datetime.now(datetime.timezone.utc), - None, ) cleanup_mock.assert_not_called() @@ -71,6 +73,8 @@ async def test_skips_cleanup_on_consumer_only(self, dbos_cleanup_workflow_module assert result == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -93,7 +97,6 @@ async def test_skips_cleanup_when_newer_execution_already_ran(self, dbos_cleanup ) as cleanup_mock: result = await dbos_cleanup_workflow_module.DbosCleanupWorkflow._cleanup_outdated_automation_executions( scheduled_time, - None, ) cleanup_mock.assert_not_called() @@ -104,6 +107,8 @@ async def test_skips_cleanup_when_newer_execution_already_ran(self, dbos_cleanup assert result == { "deleted_by_automation": {}, "deleted_cleanup_executions": 0, + "deleted_global_view_executions": 0, + "deleted_portfolio_history_executions": 0, "total_deleted": 0, } @@ -123,3 +128,10 @@ def test_returns_daily_schedule_input(self, temp_dbos_scheduler): "queue_name": octobot_node.enums.SchedulerQueues.DBOS_CLEANUP_QUEUE.value, } + def test_accepts_custom_cron(self, temp_dbos_scheduler): + import octobot_node.scheduler.workflows.dbos_cleanup_workflow as dbos_cleanup_workflow_module + + schedule_input = dbos_cleanup_workflow_module.get_schedule_input(cron="0 */6 * * *") + + assert schedule_input["schedule"] == "0 */6 * * *" + diff --git a/packages/protocol/.openapi-generator/FILES b/packages/protocol/.openapi-generator/FILES index de6cd26a7a..e9604e0d7b 100644 --- a/packages/protocol/.openapi-generator/FILES +++ b/packages/protocol/.openapi-generator/FILES @@ -67,6 +67,7 @@ docs/ExchangeAccount.md docs/ExchangeConfig.md docs/ExchangeConfigActionResult.md docs/ExchangeConfigActionResultErrorMessage.md +docs/Fee.md docs/GenericAccount.md docs/GenericProcessConfiguration.md docs/GenericWorkflowConfiguration.md @@ -96,6 +97,7 @@ docs/PositionStatus.md docs/PositionSummary.md docs/ProposedActionEntry.md docs/RefreshAccountsConfiguration.md +docs/ResetAccountTradingDataConfiguration.md docs/RestartAutomationConfiguration.md docs/Side.md docs/SignalAutomationConfiguration.md @@ -118,6 +120,9 @@ docs/TradingTentaclesConfiguration.md docs/TradingType.md docs/TrailingProfile.md docs/TrailingProfileType.md +docs/Transaction.md +docs/TransactionType.md +docs/UpdateHistoricalExchangesDataConfiguration.md docs/UserAction.md docs/UserActionConfiguration.md docs/UserActionResult.md @@ -199,6 +204,7 @@ octobot_protocol/models/exchange_account.py octobot_protocol/models/exchange_config.py octobot_protocol/models/exchange_config_action_result.py octobot_protocol/models/exchange_config_action_result_error_message.py +octobot_protocol/models/fee.py octobot_protocol/models/generic_account.py octobot_protocol/models/generic_process_configuration.py octobot_protocol/models/generic_workflow_configuration.py @@ -228,6 +234,7 @@ octobot_protocol/models/position_status.py octobot_protocol/models/position_summary.py octobot_protocol/models/proposed_action_entry.py octobot_protocol/models/refresh_accounts_configuration.py +octobot_protocol/models/reset_account_trading_data_configuration.py octobot_protocol/models/restart_automation_configuration.py octobot_protocol/models/side.py octobot_protocol/models/signal_automation_configuration.py @@ -250,6 +257,9 @@ octobot_protocol/models/trading_tentacles_configuration.py octobot_protocol/models/trading_type.py octobot_protocol/models/trailing_profile.py octobot_protocol/models/trailing_profile_type.py +octobot_protocol/models/transaction.py +octobot_protocol/models/transaction_type.py +octobot_protocol/models/update_historical_exchanges_data_configuration.py octobot_protocol/models/user_action.py octobot_protocol/models/user_action_configuration.py octobot_protocol/models/user_action_result.py @@ -329,6 +339,7 @@ test/test_exchange_account.py test/test_exchange_config.py test/test_exchange_config_action_result.py test/test_exchange_config_action_result_error_message.py +test/test_fee.py test/test_generic_account.py test/test_generic_process_configuration.py test/test_generic_workflow_configuration.py @@ -358,6 +369,7 @@ test/test_position_status.py test/test_position_summary.py test/test_proposed_action_entry.py test/test_refresh_accounts_configuration.py +test/test_reset_account_trading_data_configuration.py test/test_restart_automation_configuration.py test/test_side.py test/test_signal_automation_configuration.py @@ -380,6 +392,9 @@ test/test_trading_tentacles_configuration.py test/test_trading_type.py test/test_trailing_profile.py test/test_trailing_profile_type.py +test/test_transaction.py +test/test_transaction_type.py +test/test_update_historical_exchanges_data_configuration.py test/test_user_action.py test/test_user_action_configuration.py test/test_user_action_result.py diff --git a/packages/protocol/docs/AccountTrading.md b/packages/protocol/docs/AccountTrading.md index 44050729ca..45cc3fcddd 100644 --- a/packages/protocol/docs/AccountTrading.md +++ b/packages/protocol/docs/AccountTrading.md @@ -10,6 +10,7 @@ Name | Type | Description | Notes **orders** | [**List[Order]**](Order.md) | | [optional] **trades** | [**List[Trade]**](Trade.md) | | [optional] **positions** | [**List[Position]**](Position.md) | | [optional] +**transactions** | [**List[Transaction]**](Transaction.md) | | [optional] ## Example diff --git a/packages/protocol/docs/ExchangeConfig.md b/packages/protocol/docs/ExchangeConfig.md index cd6ce7a34f..ed57508def 100644 --- a/packages/protocol/docs/ExchangeConfig.md +++ b/packages/protocol/docs/ExchangeConfig.md @@ -11,6 +11,7 @@ Name | Type | Description | Notes **exchange** | **str** | | **sandboxed** | **bool** | | [default to False] **url** | **str** | | [optional] +**historical_trade_symbols** | **List[str]** | | [optional] ## Example diff --git a/packages/protocol/docs/Fee.md b/packages/protocol/docs/Fee.md new file mode 100644 index 0000000000..c1e39e0c61 --- /dev/null +++ b/packages/protocol/docs/Fee.md @@ -0,0 +1,31 @@ +# Fee + +TradeFee + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**amount** | **float** | | +**currency** | **str** | | + +## Example + +```python +from octobot_protocol.models.fee import Fee + +# TODO update the JSON string below +json = "{}" +# create an instance of Fee from a JSON string +fee_instance = Fee.from_json(json) +# print the JSON string representation of the object +print(Fee.to_json()) + +# convert the object into a dict +fee_dict = fee_instance.to_dict() +# create an instance of Fee from a dict +fee_from_dict = Fee.from_dict(fee_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/ResetAccountTradingDataConfiguration.md b/packages/protocol/docs/ResetAccountTradingDataConfiguration.md new file mode 100644 index 0000000000..e34a00a161 --- /dev/null +++ b/packages/protocol/docs/ResetAccountTradingDataConfiguration.md @@ -0,0 +1,31 @@ +# ResetAccountTradingDataConfiguration + +ResetAccountTradingDataConfiguration + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**action_type** | [**UserActionType**](UserActionType.md) | reset_account_trading_data | +**account_ids** | **List[str]** | | + +## Example + +```python +from octobot_protocol.models.reset_account_trading_data_configuration import ResetAccountTradingDataConfiguration + +# TODO update the JSON string below +json = "{}" +# create an instance of ResetAccountTradingDataConfiguration from a JSON string +reset_account_trading_data_configuration_instance = ResetAccountTradingDataConfiguration.from_json(json) +# print the JSON string representation of the object +print(ResetAccountTradingDataConfiguration.to_json()) + +# convert the object into a dict +reset_account_trading_data_configuration_dict = reset_account_trading_data_configuration_instance.to_dict() +# create an instance of ResetAccountTradingDataConfiguration from a dict +reset_account_trading_data_configuration_from_dict = ResetAccountTradingDataConfiguration.from_dict(reset_account_trading_data_configuration_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/Trade.md b/packages/protocol/docs/Trade.md index 3e60e90eec..9c4d371846 100644 --- a/packages/protocol/docs/Trade.md +++ b/packages/protocol/docs/Trade.md @@ -13,6 +13,7 @@ Name | Type | Description | Notes **side** | [**Side**](Side.md) | | **quantity** | **float** | | **price** | **float** | | +**fee** | [**Fee**](Fee.md) | | [optional] **status** | [**OrderStatus**](OrderStatus.md) | | **executed_at** | **datetime** | | diff --git a/packages/protocol/docs/Transaction.md b/packages/protocol/docs/Transaction.md new file mode 100644 index 0000000000..e968030b81 --- /dev/null +++ b/packages/protocol/docs/Transaction.md @@ -0,0 +1,34 @@ +# Transaction + +Transaction + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**id** | **str** | | +**timestamp** | **datetime** | | +**asset** | **str** | | +**amount** | **float** | | +**type** | [**TransactionType**](TransactionType.md) | | + +## Example + +```python +from octobot_protocol.models.transaction import Transaction + +# TODO update the JSON string below +json = "{}" +# create an instance of Transaction from a JSON string +transaction_instance = Transaction.from_json(json) +# print the JSON string representation of the object +print(Transaction.to_json()) + +# convert the object into a dict +transaction_dict = transaction_instance.to_dict() +# create an instance of Transaction from a dict +transaction_from_dict = Transaction.from_dict(transaction_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/TransactionType.md b/packages/protocol/docs/TransactionType.md new file mode 100644 index 0000000000..a8f42a33eb --- /dev/null +++ b/packages/protocol/docs/TransactionType.md @@ -0,0 +1,17 @@ +# TransactionType + +TransactionType + +## Enum + +* `BLOCKCHAIN_DEPOSIT` (value: `'blockchain_deposit'`) + +* `BLOCKCHAIN_WITHDRAWAL` (value: `'blockchain_withdrawal'`) + +* `FUNDING_FEE` (value: `'funding_fee'`) + +* `TRADING_FEE` (value: `'trading_fee'`) + +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md b/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md new file mode 100644 index 0000000000..50cd8fd6bd --- /dev/null +++ b/packages/protocol/docs/UpdateHistoricalExchangesDataConfiguration.md @@ -0,0 +1,31 @@ +# UpdateHistoricalExchangesDataConfiguration + +UpdateHistoricalExchangesDataConfiguration + +## Properties + +Name | Type | Description | Notes +------------ | ------------- | ------------- | ------------- +**action_type** | [**UserActionType**](UserActionType.md) | update_historical_exchanges_data | +**account_ids** | **List[str]** | | [optional] + +## Example + +```python +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration + +# TODO update the JSON string below +json = "{}" +# create an instance of UpdateHistoricalExchangesDataConfiguration from a JSON string +update_historical_exchanges_data_configuration_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json) +# print the JSON string representation of the object +print(UpdateHistoricalExchangesDataConfiguration.to_json()) + +# convert the object into a dict +update_historical_exchanges_data_configuration_dict = update_historical_exchanges_data_configuration_instance.to_dict() +# create an instance of UpdateHistoricalExchangesDataConfiguration from a dict +update_historical_exchanges_data_configuration_from_dict = UpdateHistoricalExchangesDataConfiguration.from_dict(update_historical_exchanges_data_configuration_dict) +``` +[[Back to Model list]](../README.md#documentation-for-models) [[Back to API list]](../README.md#documentation-for-api-endpoints) [[Back to README]](../README.md) + + diff --git a/packages/protocol/docs/UserActionConfiguration.md b/packages/protocol/docs/UserActionConfiguration.md index 895dfee816..acaf0c7b78 100644 --- a/packages/protocol/docs/UserActionConfiguration.md +++ b/packages/protocol/docs/UserActionConfiguration.md @@ -11,7 +11,7 @@ Name | Type | Description | Notes **automation_id** | **str** | | **signal_type** | [**AutomationSignalType**](AutomationSignalType.md) | | **signal_payload** | [**SignalAutomationConfigurationSignalPayload**](SignalAutomationConfigurationSignalPayload.md) | | [optional] -**account_ids** | **List[str]** | | [optional] +**account_ids** | **List[str]** | | ## Example diff --git a/packages/protocol/docs/UserActionType.md b/packages/protocol/docs/UserActionType.md index 8d10bdec95..c942855d30 100644 --- a/packages/protocol/docs/UserActionType.md +++ b/packages/protocol/docs/UserActionType.md @@ -28,6 +28,10 @@ UserActionType * `EXCHANGE_CONFIG_DELETE` (value: `'exchange_config_delete'`) +* `UPDATE_HISTORICAL_EXCHANGES_DATA` (value: `'update_historical_exchanges_data'`) + +* `RESET_ACCOUNT_TRADING_DATA` (value: `'reset_account_trading_data'`) + * `STRATEGY_CREATE` (value: `'strategy_create'`) * `STRATEGY_EDIT` (value: `'strategy_edit'`) diff --git a/packages/protocol/octobot_protocol/models/__init__.py b/packages/protocol/octobot_protocol/models/__init__.py index ab3b57bf78..46292fbe8d 100644 --- a/packages/protocol/octobot_protocol/models/__init__.py +++ b/packages/protocol/octobot_protocol/models/__init__.py @@ -82,6 +82,7 @@ from octobot_protocol.models.exchange_config import ExchangeConfig from octobot_protocol.models.exchange_config_action_result import ExchangeConfigActionResult from octobot_protocol.models.exchange_config_action_result_error_message import ExchangeConfigActionResultErrorMessage +from octobot_protocol.models.fee import Fee from octobot_protocol.models.generic_account import GenericAccount from octobot_protocol.models.generic_process_configuration import GenericProcessConfiguration from octobot_protocol.models.generic_workflow_configuration import GenericWorkflowConfiguration @@ -111,6 +112,7 @@ from octobot_protocol.models.position_summary import PositionSummary from octobot_protocol.models.proposed_action_entry import ProposedActionEntry from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.reset_account_trading_data_configuration import ResetAccountTradingDataConfiguration from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.side import Side from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration @@ -133,6 +135,9 @@ from octobot_protocol.models.trading_type import TradingType from octobot_protocol.models.trailing_profile import TrailingProfile from octobot_protocol.models.trailing_profile_type import TrailingProfileType +from octobot_protocol.models.transaction import Transaction +from octobot_protocol.models.transaction_type import TransactionType +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration from octobot_protocol.models.user_action import UserAction from octobot_protocol.models.user_action_configuration import UserActionConfiguration from octobot_protocol.models.user_action_result import UserActionResult diff --git a/packages/protocol/octobot_protocol/models/account_trading.py b/packages/protocol/octobot_protocol/models/account_trading.py index ddc5ab2253..74f2e3b854 100644 --- a/packages/protocol/octobot_protocol/models/account_trading.py +++ b/packages/protocol/octobot_protocol/models/account_trading.py @@ -23,6 +23,7 @@ from octobot_protocol.models.order import Order from octobot_protocol.models.position import Position from octobot_protocol.models.trade import Trade +from octobot_protocol.models.transaction import Transaction from typing import Optional, Set from typing_extensions import Self from pydantic_core import to_jsonable_python @@ -35,7 +36,8 @@ class AccountTrading(BaseModel): orders: Optional[List[Order]] = None trades: Optional[List[Trade]] = None positions: Optional[List[Position]] = None - __properties: ClassVar[List[str]] = ["updated_at", "orders", "trades", "positions"] + transactions: Optional[List[Transaction]] = None + __properties: ClassVar[List[str]] = ["updated_at", "orders", "trades", "positions", "transactions"] model_config = ConfigDict( validate_by_name=True, @@ -97,6 +99,13 @@ def to_dict(self) -> Dict[str, Any]: if _item_positions: _items.append(_item_positions.to_dict()) _dict['positions'] = _items + # override the default output from pydantic by calling `to_dict()` of each item in transactions (list) + _items = [] + if self.transactions: + for _item_transactions in self.transactions: + if _item_transactions: + _items.append(_item_transactions.to_dict()) + _dict['transactions'] = _items return _dict @classmethod @@ -112,7 +121,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "updated_at": obj.get("updated_at"), "orders": [Order.from_dict(_item) for _item in obj["orders"]] if obj.get("orders") is not None else None, "trades": [Trade.from_dict(_item) for _item in obj["trades"]] if obj.get("trades") is not None else None, - "positions": [Position.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None + "positions": [Position.from_dict(_item) for _item in obj["positions"]] if obj.get("positions") is not None else None, + "transactions": [Transaction.from_dict(_item) for _item in obj["transactions"]] if obj.get("transactions") is not None else None }) return _obj diff --git a/packages/protocol/octobot_protocol/models/exchange_config.py b/packages/protocol/octobot_protocol/models/exchange_config.py index 6c585bec66..8d260a5004 100644 --- a/packages/protocol/octobot_protocol/models/exchange_config.py +++ b/packages/protocol/octobot_protocol/models/exchange_config.py @@ -32,7 +32,8 @@ class ExchangeConfig(BaseModel): exchange: StrictStr sandboxed: StrictBool url: Optional[StrictStr] = None - __properties: ClassVar[List[str]] = ["id", "name", "exchange", "sandboxed", "url"] + historical_trade_symbols: Optional[List[StrictStr]] = None + __properties: ClassVar[List[str]] = ["id", "name", "exchange", "sandboxed", "url", "historical_trade_symbols"] model_config = ConfigDict( validate_by_name=True, @@ -89,7 +90,8 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "name": obj.get("name"), "exchange": obj.get("exchange"), "sandboxed": obj.get("sandboxed") if obj.get("sandboxed") is not None else False, - "url": obj.get("url") + "url": obj.get("url"), + "historical_trade_symbols": obj.get("historical_trade_symbols") }) return _obj diff --git a/packages/protocol/octobot_protocol/models/fee.py b/packages/protocol/octobot_protocol/models/fee.py new file mode 100644 index 0000000000..eddb354b42 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/fee.py @@ -0,0 +1,90 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Union +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class Fee(BaseModel): + """ + TradeFee + """ # noqa: E501 + amount: Union[StrictFloat, StrictInt] + currency: StrictStr + __properties: ClassVar[List[str]] = ["amount", "currency"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of Fee from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of Fee from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "amount": obj.get("amount"), + "currency": obj.get("currency") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/reset_account_trading_data_configuration.py b/packages/protocol/octobot_protocol/models/reset_account_trading_data_configuration.py new file mode 100644 index 0000000000..83086225d4 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/reset_account_trading_data_configuration.py @@ -0,0 +1,92 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List +from typing_extensions import Annotated +from octobot_protocol.models.user_action_type import UserActionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class ResetAccountTradingDataConfiguration(BaseModel): + """ + ResetAccountTradingDataConfiguration + """ # noqa: E501 + action_type: UserActionType = Field(description="reset_account_trading_data") + account_ids: Annotated[List[StrictStr], Field(min_length=1)] + __properties: ClassVar[List[str]] = ["action_type", "account_ids"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of ResetAccountTradingDataConfiguration from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of ResetAccountTradingDataConfiguration from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "action_type": obj.get("action_type"), + "account_ids": obj.get("account_ids") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/trade.py b/packages/protocol/octobot_protocol/models/trade.py index e85839b871..ae9fa6df55 100644 --- a/packages/protocol/octobot_protocol/models/trade.py +++ b/packages/protocol/octobot_protocol/models/trade.py @@ -19,7 +19,8 @@ from datetime import datetime from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr -from typing import Any, ClassVar, Dict, List, Union +from typing import Any, ClassVar, Dict, List, Optional, Union +from octobot_protocol.models.fee import Fee from octobot_protocol.models.order_status import OrderStatus from octobot_protocol.models.order_type import OrderType from octobot_protocol.models.side import Side @@ -38,9 +39,10 @@ class Trade(BaseModel): side: Side quantity: Union[StrictFloat, StrictInt] price: Union[StrictFloat, StrictInt] + fee: Optional[Fee] = None status: OrderStatus executed_at: datetime - __properties: ClassVar[List[str]] = ["id", "trade_id", "type", "symbol", "side", "quantity", "price", "status", "executed_at"] + __properties: ClassVar[List[str]] = ["id", "trade_id", "type", "symbol", "side", "quantity", "price", "fee", "status", "executed_at"] model_config = ConfigDict( validate_by_name=True, @@ -81,6 +83,9 @@ def to_dict(self) -> Dict[str, Any]: exclude=excluded_fields, exclude_none=True, ) + # override the default output from pydantic by calling `to_dict()` of fee + if self.fee: + _dict['fee'] = self.fee.to_dict() return _dict @classmethod @@ -100,6 +105,7 @@ def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: "side": obj.get("side"), "quantity": obj.get("quantity"), "price": obj.get("price"), + "fee": Fee.from_dict(obj["fee"]) if obj.get("fee") is not None else None, "status": obj.get("status"), "executed_at": obj.get("executed_at") }) diff --git a/packages/protocol/octobot_protocol/models/transaction.py b/packages/protocol/octobot_protocol/models/transaction.py new file mode 100644 index 0000000000..8cf4e95cda --- /dev/null +++ b/packages/protocol/octobot_protocol/models/transaction.py @@ -0,0 +1,98 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from datetime import datetime +from pydantic import BaseModel, ConfigDict, StrictFloat, StrictInt, StrictStr +from typing import Any, ClassVar, Dict, List, Union +from octobot_protocol.models.transaction_type import TransactionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class Transaction(BaseModel): + """ + Transaction + """ # noqa: E501 + id: StrictStr + timestamp: datetime + asset: StrictStr + amount: Union[StrictFloat, StrictInt] + type: TransactionType + __properties: ClassVar[List[str]] = ["id", "timestamp", "asset", "amount", "type"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of Transaction from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of Transaction from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "id": obj.get("id"), + "timestamp": obj.get("timestamp"), + "asset": obj.get("asset"), + "amount": obj.get("amount"), + "type": obj.get("type") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/transaction_type.py b/packages/protocol/octobot_protocol/models/transaction_type.py new file mode 100644 index 0000000000..c8dfd6849c --- /dev/null +++ b/packages/protocol/octobot_protocol/models/transaction_type.py @@ -0,0 +1,39 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import json +from enum import Enum +from typing_extensions import Self + + +class TransactionType(str, Enum): + """ + TransactionType + """ + + """ + allowed enum values + """ + BLOCKCHAIN_DEPOSIT = 'blockchain_deposit' + BLOCKCHAIN_WITHDRAWAL = 'blockchain_withdrawal' + FUNDING_FEE = 'funding_fee' + TRADING_FEE = 'trading_fee' + + @classmethod + def from_json(cls, json_str: str) -> Self: + """Create an instance of TransactionType from a JSON string""" + return cls(json.loads(json_str)) + + diff --git a/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py b/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py new file mode 100644 index 0000000000..d5b901bc13 --- /dev/null +++ b/packages/protocol/octobot_protocol/models/update_historical_exchanges_data_configuration.py @@ -0,0 +1,91 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +from __future__ import annotations +import pprint +import re # noqa: F401 +import json + +from pydantic import BaseModel, ConfigDict, Field, StrictStr +from typing import Any, ClassVar, Dict, List, Optional +from octobot_protocol.models.user_action_type import UserActionType +from typing import Optional, Set +from typing_extensions import Self +from pydantic_core import to_jsonable_python + +class UpdateHistoricalExchangesDataConfiguration(BaseModel): + """ + UpdateHistoricalExchangesDataConfiguration + """ # noqa: E501 + action_type: UserActionType = Field(description="update_historical_exchanges_data") + account_ids: Optional[List[StrictStr]] = None + __properties: ClassVar[List[str]] = ["action_type", "account_ids"] + + model_config = ConfigDict( + validate_by_name=True, + validate_by_alias=True, + validate_assignment=True, + protected_namespaces=(), + ) + + + def to_str(self) -> str: + """Returns the string representation of the model using alias""" + return pprint.pformat(self.model_dump(by_alias=True)) + + def to_json(self) -> str: + """Returns the JSON representation of the model using alias""" + return json.dumps(to_jsonable_python(self.to_dict())) + + @classmethod + def from_json(cls, json_str: str) -> Optional[Self]: + """Create an instance of UpdateHistoricalExchangesDataConfiguration from a JSON string""" + return cls.from_dict(json.loads(json_str)) + + def to_dict(self) -> Dict[str, Any]: + """Return the dictionary representation of the model using alias. + + This has the following differences from calling pydantic's + `self.model_dump(by_alias=True)`: + + * `None` is only added to the output dict for nullable fields that + were set at model initialization. Other fields with value `None` + are ignored. + """ + excluded_fields: Set[str] = set([ + ]) + + _dict = self.model_dump( + by_alias=True, + exclude=excluded_fields, + exclude_none=True, + ) + return _dict + + @classmethod + def from_dict(cls, obj: Optional[Dict[str, Any]]) -> Optional[Self]: + """Create an instance of UpdateHistoricalExchangesDataConfiguration from a dict""" + if obj is None: + return None + + if not isinstance(obj, dict): + return cls.model_validate(obj) + + _obj = cls.model_validate({ + "action_type": obj.get("action_type"), + "account_ids": obj.get("account_ids") + }) + return _obj + + diff --git a/packages/protocol/octobot_protocol/models/user_action_configuration.py b/packages/protocol/octobot_protocol/models/user_action_configuration.py index f4f6866acc..b7464acbd3 100644 --- a/packages/protocol/octobot_protocol/models/user_action_configuration.py +++ b/packages/protocol/octobot_protocol/models/user_action_configuration.py @@ -32,14 +32,16 @@ from octobot_protocol.models.edit_exchange_config_configuration import EditExchangeConfigConfiguration from octobot_protocol.models.edit_strategy_configuration import EditStrategyConfiguration from octobot_protocol.models.refresh_accounts_configuration import RefreshAccountsConfiguration +from octobot_protocol.models.reset_account_trading_data_configuration import ResetAccountTradingDataConfiguration from octobot_protocol.models.restart_automation_configuration import RestartAutomationConfiguration from octobot_protocol.models.signal_automation_configuration import SignalAutomationConfiguration from octobot_protocol.models.stop_automation_configuration import StopAutomationConfiguration +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration from pydantic import StrictStr, Field from typing import Union, List, Set, Optional, Dict from typing_extensions import Literal, Self -USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration"] +USERACTIONCONFIGURATION_ONE_OF_SCHEMAS = ["CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "ResetAccountTradingDataConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration", "UpdateHistoricalExchangesDataConfiguration"] class UserActionConfiguration(BaseModel): """ @@ -61,28 +63,32 @@ class UserActionConfiguration(BaseModel): oneof_schema_7_validator: Optional[EditAccountConfiguration] = None # data type: DeleteAccountConfiguration oneof_schema_8_validator: Optional[DeleteAccountConfiguration] = None + # data type: UpdateHistoricalExchangesDataConfiguration + oneof_schema_9_validator: Optional[UpdateHistoricalExchangesDataConfiguration] = None + # data type: ResetAccountTradingDataConfiguration + oneof_schema_10_validator: Optional[ResetAccountTradingDataConfiguration] = None # data type: CreateExchangeConfigConfiguration - oneof_schema_9_validator: Optional[CreateExchangeConfigConfiguration] = None + oneof_schema_11_validator: Optional[CreateExchangeConfigConfiguration] = None # data type: EditExchangeConfigConfiguration - oneof_schema_10_validator: Optional[EditExchangeConfigConfiguration] = None + oneof_schema_12_validator: Optional[EditExchangeConfigConfiguration] = None # data type: DeleteExchangeConfigConfiguration - oneof_schema_11_validator: Optional[DeleteExchangeConfigConfiguration] = None + oneof_schema_13_validator: Optional[DeleteExchangeConfigConfiguration] = None # data type: RefreshAccountsConfiguration - oneof_schema_12_validator: Optional[RefreshAccountsConfiguration] = None + oneof_schema_14_validator: Optional[RefreshAccountsConfiguration] = None # data type: CreateStrategyConfiguration - oneof_schema_13_validator: Optional[CreateStrategyConfiguration] = None + oneof_schema_15_validator: Optional[CreateStrategyConfiguration] = None # data type: EditStrategyConfiguration - oneof_schema_14_validator: Optional[EditStrategyConfiguration] = None + oneof_schema_16_validator: Optional[EditStrategyConfiguration] = None # data type: DeleteStrategyConfiguration - oneof_schema_15_validator: Optional[DeleteStrategyConfiguration] = None + oneof_schema_17_validator: Optional[DeleteStrategyConfiguration] = None # data type: CreateAccountAuthConfiguration - oneof_schema_16_validator: Optional[CreateAccountAuthConfiguration] = None + oneof_schema_18_validator: Optional[CreateAccountAuthConfiguration] = None # data type: EditAccountAuthConfiguration - oneof_schema_17_validator: Optional[EditAccountAuthConfiguration] = None + oneof_schema_19_validator: Optional[EditAccountAuthConfiguration] = None # data type: DeleteAccountAuthConfiguration - oneof_schema_18_validator: Optional[DeleteAccountAuthConfiguration] = None - actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]] = None - one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration" } + oneof_schema_20_validator: Optional[DeleteAccountAuthConfiguration] = None + actual_instance: Optional[Union[CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration]] = None + one_of_schemas: Set[str] = { "CreateAccountAuthConfiguration", "CreateAccountConfiguration", "CreateAutomationConfiguration", "CreateExchangeConfigConfiguration", "CreateStrategyConfiguration", "DeleteAccountAuthConfiguration", "DeleteAccountConfiguration", "DeleteExchangeConfigConfiguration", "DeleteStrategyConfiguration", "EditAccountAuthConfiguration", "EditAccountConfiguration", "EditAutomationConfiguration", "EditExchangeConfigConfiguration", "EditStrategyConfiguration", "RefreshAccountsConfiguration", "ResetAccountTradingDataConfiguration", "RestartAutomationConfiguration", "SignalAutomationConfiguration", "StopAutomationConfiguration", "UpdateHistoricalExchangesDataConfiguration" } model_config = ConfigDict( validate_assignment=True, @@ -148,6 +154,16 @@ def actual_instance_must_validate_oneof(cls, v): error_messages.append(f"Error! Input type `{type(v)}` is not `DeleteAccountConfiguration`") else: match += 1 + # validate data type: UpdateHistoricalExchangesDataConfiguration + if not isinstance(v, UpdateHistoricalExchangesDataConfiguration): + error_messages.append(f"Error! Input type `{type(v)}` is not `UpdateHistoricalExchangesDataConfiguration`") + else: + match += 1 + # validate data type: ResetAccountTradingDataConfiguration + if not isinstance(v, ResetAccountTradingDataConfiguration): + error_messages.append(f"Error! Input type `{type(v)}` is not `ResetAccountTradingDataConfiguration`") + else: + match += 1 # validate data type: CreateExchangeConfigConfiguration if not isinstance(v, CreateExchangeConfigConfiguration): error_messages.append(f"Error! Input type `{type(v)}` is not `CreateExchangeConfigConfiguration`") @@ -200,10 +216,10 @@ def actual_instance_must_validate_oneof(cls, v): match += 1 if match > 1: # more than 1 match - raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when setting `actual_instance` in UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) else: return v @@ -298,6 +314,11 @@ def from_json(cls, json_str: str) -> Self: instance.actual_instance = EditExchangeConfigConfiguration.from_json(json_str) return instance + # check if data type is `ResetAccountTradingDataConfiguration` + if _data_type == "reset_account_trading_data": + instance.actual_instance = ResetAccountTradingDataConfiguration.from_json(json_str) + return instance + # check if data type is `CreateStrategyConfiguration` if _data_type == "strategy_create": instance.actual_instance = CreateStrategyConfiguration.from_json(json_str) @@ -313,6 +334,11 @@ def from_json(cls, json_str: str) -> Self: instance.actual_instance = EditStrategyConfiguration.from_json(json_str) return instance + # check if data type is `UpdateHistoricalExchangesDataConfiguration` + if _data_type == "update_historical_exchanges_data": + instance.actual_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json_str) + return instance + # deserialize data into CreateAutomationConfiguration try: instance.actual_instance = CreateAutomationConfiguration.from_json(json_str) @@ -361,6 +387,18 @@ def from_json(cls, json_str: str) -> Self: match += 1 except (ValidationError, ValueError) as e: error_messages.append(str(e)) + # deserialize data into UpdateHistoricalExchangesDataConfiguration + try: + instance.actual_instance = UpdateHistoricalExchangesDataConfiguration.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) + # deserialize data into ResetAccountTradingDataConfiguration + try: + instance.actual_instance = ResetAccountTradingDataConfiguration.from_json(json_str) + match += 1 + except (ValidationError, ValueError) as e: + error_messages.append(str(e)) # deserialize data into CreateExchangeConfigConfiguration try: instance.actual_instance = CreateExchangeConfigConfiguration.from_json(json_str) @@ -424,10 +462,10 @@ def from_json(cls, json_str: str) -> Self: if match > 1: # more than 1 match - raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("Multiple matches found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) elif match == 0: # no match - raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration. Details: " + ", ".join(error_messages)) + raise ValueError("No match found when deserializing the JSON string into UserActionConfiguration with oneOf schemas: CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration. Details: " + ", ".join(error_messages)) else: return instance @@ -441,7 +479,7 @@ def to_json(self) -> str: else: return json.dumps(self.actual_instance) - def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration]]: + def to_dict(self) -> Optional[Union[Dict[str, Any], CreateAccountAuthConfiguration, CreateAccountConfiguration, CreateAutomationConfiguration, CreateExchangeConfigConfiguration, CreateStrategyConfiguration, DeleteAccountAuthConfiguration, DeleteAccountConfiguration, DeleteExchangeConfigConfiguration, DeleteStrategyConfiguration, EditAccountAuthConfiguration, EditAccountConfiguration, EditAutomationConfiguration, EditExchangeConfigConfiguration, EditStrategyConfiguration, RefreshAccountsConfiguration, ResetAccountTradingDataConfiguration, RestartAutomationConfiguration, SignalAutomationConfiguration, StopAutomationConfiguration, UpdateHistoricalExchangesDataConfiguration]]: """Returns the dict representation of the actual instance""" if self.actual_instance is None: return None diff --git a/packages/protocol/octobot_protocol/models/user_action_type.py b/packages/protocol/octobot_protocol/models/user_action_type.py index 8836a794d0..4ced73b50a 100644 --- a/packages/protocol/octobot_protocol/models/user_action_type.py +++ b/packages/protocol/octobot_protocol/models/user_action_type.py @@ -38,6 +38,8 @@ class UserActionType(str, Enum): EXCHANGE_CONFIG_CREATE = 'exchange_config_create' EXCHANGE_CONFIG_EDIT = 'exchange_config_edit' EXCHANGE_CONFIG_DELETE = 'exchange_config_delete' + UPDATE_HISTORICAL_EXCHANGES_DATA = 'update_historical_exchanges_data' + RESET_ACCOUNT_TRADING_DATA = 'reset_account_trading_data' STRATEGY_CREATE = 'strategy_create' STRATEGY_EDIT = 'strategy_edit' STRATEGY_DELETE = 'strategy_delete' diff --git a/packages/protocol/octobot_protocol_ts/models/AccountTrading.ts b/packages/protocol/octobot_protocol_ts/models/AccountTrading.ts index ca6592c759..14f4a4792a 100644 --- a/packages/protocol/octobot_protocol_ts/models/AccountTrading.ts +++ b/packages/protocol/octobot_protocol_ts/models/AccountTrading.ts @@ -13,6 +13,7 @@ import { Order } from '../models/Order'; import { Position } from '../models/Position'; import { Trade } from '../models/Trade'; +import { Transaction } from '../models/Transaction'; /** * AccountTrading @@ -22,6 +23,7 @@ export class AccountTrading { 'orders'?: Array; 'trades'?: Array; 'positions'?: Array; + 'transactions'?: Array; static readonly discriminator: string | undefined = undefined; @@ -51,6 +53,12 @@ export class AccountTrading { "baseName": "positions", "type": "Array", "format": "" + }, + { + "name": "transactions", + "baseName": "transactions", + "type": "Array", + "format": "" } ]; static getAttributeTypeMap() { diff --git a/packages/protocol/octobot_protocol_ts/models/ExchangeConfig.ts b/packages/protocol/octobot_protocol_ts/models/ExchangeConfig.ts index 83d50f31e5..8ab78f2998 100644 --- a/packages/protocol/octobot_protocol_ts/models/ExchangeConfig.ts +++ b/packages/protocol/octobot_protocol_ts/models/ExchangeConfig.ts @@ -20,6 +20,7 @@ export class ExchangeConfig { 'exchange': string; 'sandboxed': boolean; 'url'?: string; + 'historical_trade_symbols'?: Array; static readonly discriminator: string | undefined = undefined; @@ -55,6 +56,12 @@ export class ExchangeConfig { "baseName": "url", "type": "string", "format": "" + }, + { + "name": "historical_trade_symbols", + "baseName": "historical_trade_symbols", + "type": "Array", + "format": "" } ]; static getAttributeTypeMap() { diff --git a/packages/protocol/octobot_protocol_ts/models/Fee.ts b/packages/protocol/octobot_protocol_ts/models/Fee.ts new file mode 100644 index 0000000000..a84086032b --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/Fee.ts @@ -0,0 +1,45 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* TradeFee +*/ +export class Fee { + 'amount': number; + 'currency': string; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "amount", + "baseName": "amount", + "type": "number", + "format": "" + }, + { + "name": "currency", + "baseName": "currency", + "type": "string", + "format": "" + } ]; + + static getAttributeTypeMap() { + return Fee.attributeTypeMap; + } + + public constructor() { + } +} diff --git a/packages/protocol/octobot_protocol_ts/models/ResetAccountTradingDataConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/ResetAccountTradingDataConfiguration.ts new file mode 100644 index 0000000000..03d5269516 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/ResetAccountTradingDataConfiguration.ts @@ -0,0 +1,51 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { UserActionType } from '../models/UserActionType'; + +/** +* ResetAccountTradingDataConfiguration +*/ +export class ResetAccountTradingDataConfiguration { + /** + * reset_account_trading_data + */ + 'action_type': 'reset_account_trading_data'; + 'account_ids': Array; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "action_type", + "baseName": "action_type", + "type": "UserActionType", + "format": "" + }, + { + "name": "account_ids", + "baseName": "account_ids", + "type": "Array", + "format": "" + } ]; + + static getAttributeTypeMap() { + return ResetAccountTradingDataConfiguration.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/Trade.ts b/packages/protocol/octobot_protocol_ts/models/Trade.ts index 27bb93d3f7..1a2831d83c 100644 --- a/packages/protocol/octobot_protocol_ts/models/Trade.ts +++ b/packages/protocol/octobot_protocol_ts/models/Trade.ts @@ -10,6 +10,7 @@ * Do not edit the class manually. */ +import { Fee } from '../models/Fee'; import { OrderStatus } from '../models/OrderStatus'; import { OrderType } from '../models/OrderType'; import { Side } from '../models/Side'; @@ -25,6 +26,7 @@ export class Trade { 'side': Side; 'quantity': number; 'price': number; + 'fee'?: Fee; 'status': OrderStatus; 'executed_at': string; @@ -75,6 +77,12 @@ export class Trade { "type": "number", "format": "" }, + { + "name": "fee", + "baseName": "fee", + "type": "Fee", + "format": "" + }, { "name": "status", "baseName": "status", diff --git a/packages/protocol/octobot_protocol_ts/models/Transaction.ts b/packages/protocol/octobot_protocol_ts/models/Transaction.ts new file mode 100644 index 0000000000..57790bacf6 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/Transaction.ts @@ -0,0 +1,69 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { TransactionType } from '../models/TransactionType'; + +/** +* Transaction +*/ +export class Transaction { + 'id': string; + 'timestamp': string; + 'asset': string; + 'amount': number; + 'type': TransactionType; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "id", + "baseName": "id", + "type": "string", + "format": "" + }, + { + "name": "timestamp", + "baseName": "timestamp", + "type": "Date", + "format": "date-time" + }, + { + "name": "asset", + "baseName": "asset", + "type": "string", + "format": "" + }, + { + "name": "amount", + "baseName": "amount", + "type": "number", + "format": "" + }, + { + "name": "type", + "baseName": "type", + "type": "TransactionType", + "format": "" + } ]; + + static getAttributeTypeMap() { + return Transaction.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/TransactionType.ts b/packages/protocol/octobot_protocol_ts/models/TransactionType.ts new file mode 100644 index 0000000000..94768f74d5 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/TransactionType.ts @@ -0,0 +1,17 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + + +/** +* TransactionType +*/ +export type TransactionType = 'blockchain_deposit' | 'blockchain_withdrawal' | 'funding_fee' | 'trading_fee' diff --git a/packages/protocol/octobot_protocol_ts/models/UpdateHistoricalExchangesDataConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/UpdateHistoricalExchangesDataConfiguration.ts new file mode 100644 index 0000000000..a176660907 --- /dev/null +++ b/packages/protocol/octobot_protocol_ts/models/UpdateHistoricalExchangesDataConfiguration.ts @@ -0,0 +1,51 @@ +/** + * OctoBot protocol types + * No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + * + * OpenAPI spec version: 1.0.0 + * + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { UserActionType } from '../models/UserActionType'; + +/** +* UpdateHistoricalExchangesDataConfiguration +*/ +export class UpdateHistoricalExchangesDataConfiguration { + /** + * update_historical_exchanges_data + */ + 'action_type': 'update_historical_exchanges_data'; + 'account_ids'?: Array; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: {[index: string]: string} | undefined = undefined; + + static readonly attributeTypeMap: Array<{name: string, baseName: string, type: string, format: string}> = [ + { + "name": "action_type", + "baseName": "action_type", + "type": "UserActionType", + "format": "" + }, + { + "name": "account_ids", + "baseName": "account_ids", + "type": "Array", + "format": "" + } ]; + + static getAttributeTypeMap() { + return UpdateHistoricalExchangesDataConfiguration.attributeTypeMap; + } + + public constructor() { + } +} + + diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts index 2bc1cdbf1e..d59d9798f4 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionConfiguration.ts @@ -25,16 +25,18 @@ import { EditAutomationConfiguration } from '../models/EditAutomationConfigurati import { EditExchangeConfigConfiguration } from '../models/EditExchangeConfigConfiguration'; import { EditStrategyConfiguration } from '../models/EditStrategyConfiguration'; import { RefreshAccountsConfiguration } from '../models/RefreshAccountsConfiguration'; +import { ResetAccountTradingDataConfiguration } from '../models/ResetAccountTradingDataConfiguration'; import { RestartAutomationConfiguration } from '../models/RestartAutomationConfiguration'; import { SignalAutomationConfiguration } from '../models/SignalAutomationConfiguration'; import { StopAutomationConfiguration } from '../models/StopAutomationConfiguration'; +import { UpdateHistoricalExchangesDataConfiguration } from '../models/UpdateHistoricalExchangesDataConfiguration'; /** * @type UserActionConfiguration * Type * @export */ -export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | RestartAutomationConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration; +export type UserActionConfiguration = CreateAccountAuthConfiguration | CreateAccountConfiguration | CreateAutomationConfiguration | CreateExchangeConfigConfiguration | CreateStrategyConfiguration | DeleteAccountAuthConfiguration | DeleteAccountConfiguration | DeleteExchangeConfigConfiguration | DeleteStrategyConfiguration | EditAccountAuthConfiguration | EditAccountConfiguration | EditAutomationConfiguration | EditExchangeConfigConfiguration | EditStrategyConfiguration | RefreshAccountsConfiguration | ResetAccountTradingDataConfiguration | RestartAutomationConfiguration | SignalAutomationConfiguration | StopAutomationConfiguration | UpdateHistoricalExchangesDataConfiguration; /** * @type UserActionConfigurationClass @@ -59,9 +61,11 @@ export class UserActionConfigurationClass { "exchange_config_create": "CreateExchangeConfigConfiguration", "exchange_config_delete": "DeleteExchangeConfigConfiguration", "exchange_config_edit": "EditExchangeConfigConfiguration", + "reset_account_trading_data": "ResetAccountTradingDataConfiguration", "strategy_create": "CreateStrategyConfiguration", "strategy_delete": "DeleteStrategyConfiguration", "strategy_edit": "EditStrategyConfiguration", + "update_historical_exchanges_data": "UpdateHistoricalExchangesDataConfiguration", }; } @@ -81,3 +85,5 @@ export class UserActionConfigurationClass { + + diff --git a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts index d87710ca2f..fd0f4cd0b6 100644 --- a/packages/protocol/octobot_protocol_ts/models/UserActionType.ts +++ b/packages/protocol/octobot_protocol_ts/models/UserActionType.ts @@ -14,4 +14,4 @@ /** * UserActionType */ -export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_restart' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' +export type UserActionType = 'automation_create' | 'automation_edit' | 'automation_stop' | 'automation_restart' | 'automation_signal' | 'account_create' | 'account_edit' | 'account_delete' | 'accounts_refresh' | 'exchange_config_create' | 'exchange_config_edit' | 'exchange_config_delete' | 'update_historical_exchanges_data' | 'reset_account_trading_data' | 'strategy_create' | 'strategy_edit' | 'strategy_delete' | 'account_auth_create' | 'account_auth_edit' | 'account_auth_delete' diff --git a/packages/protocol/octobot_protocol_ts/models/index.ts b/packages/protocol/octobot_protocol_ts/models/index.ts index 7928e134ce..287a9c277f 100644 --- a/packages/protocol/octobot_protocol_ts/models/index.ts +++ b/packages/protocol/octobot_protocol_ts/models/index.ts @@ -67,6 +67,7 @@ export * from "./ExchangeAccount"; export * from "./ExchangeConfig"; export * from "./ExchangeConfigActionResult"; export * from "./ExchangeConfigActionResultErrorMessage"; +export * from "./Fee"; export * from "./GenericAccount"; export * from "./GenericProcessConfiguration"; export * from "./GenericWorkflowConfiguration"; @@ -96,6 +97,7 @@ export * from "./PositionStatus"; export * from "./PositionSummary"; export * from "./ProposedActionEntry"; export * from "./RefreshAccountsConfiguration"; +export * from "./ResetAccountTradingDataConfiguration"; export * from "./RestartAutomationConfiguration"; export * from "./Side"; export * from "./SignalAutomationConfiguration"; @@ -116,6 +118,9 @@ export * from "./TradingTentaclesConfiguration"; export * from "./TradingType"; export * from "./TrailingProfile"; export * from "./TrailingProfileType"; +export * from "./Transaction"; +export * from "./TransactionType"; +export * from "./UpdateHistoricalExchangesDataConfiguration"; export * from "./UserAction"; export * from "./UserActionConfiguration"; export * from "./UserActionResult"; diff --git a/packages/protocol/openapi.json b/packages/protocol/openapi.json index d61609cce9..5459ae669e 100644 --- a/packages/protocol/openapi.json +++ b/packages/protocol/openapi.json @@ -279,6 +279,22 @@ } } }, + "Fee": { + "description": "TradeFee", + "type": "object", + "required": [ + "amount", + "currency" + ], + "properties": { + "amount": { + "type": "number" + }, + "currency": { + "type": "string" + } + } + }, "Trade": { "description": "Trade", "type": "object", @@ -315,6 +331,9 @@ "price": { "type": "number" }, + "fee": { + "$ref": "#/components/schemas/Fee" + }, "status": { "$ref": "#/components/schemas/OrderStatus" }, @@ -818,6 +837,45 @@ } } }, + "TransactionType": { + "description": "TransactionType", + "type": "string", + "enum": [ + "blockchain_deposit", + "blockchain_withdrawal", + "funding_fee", + "trading_fee" + ] + }, + "Transaction": { + "description": "Transaction", + "type": "object", + "required": [ + "id", + "timestamp", + "asset", + "amount", + "type" + ], + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time" + }, + "asset": { + "type": "string" + }, + "amount": { + "type": "number" + }, + "type": { + "$ref": "#/components/schemas/TransactionType" + } + } + }, "AccountTrading": { "description": "AccountTrading", "type": "object", @@ -846,6 +904,12 @@ "items": { "$ref": "#/components/schemas/Position" } + }, + "transactions": { + "type": "array", + "items": { + "$ref": "#/components/schemas/Transaction" + } } } }, @@ -874,6 +938,12 @@ }, "url": { "type": "string" + }, + "historical_trade_symbols": { + "type": "array", + "items": { + "type": "string" + } } } }, @@ -1926,6 +1996,46 @@ } } }, + "UpdateHistoricalExchangesDataConfiguration": { + "description": "UpdateHistoricalExchangesDataConfiguration", + "type": "object", + "required": [ + "action_type" + ], + "properties": { + "action_type": { + "$ref": "#/components/schemas/UserActionType", + "description": "update_historical_exchanges_data" + }, + "account_ids": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ResetAccountTradingDataConfiguration": { + "description": "ResetAccountTradingDataConfiguration", + "type": "object", + "required": [ + "action_type", + "account_ids" + ], + "properties": { + "action_type": { + "$ref": "#/components/schemas/UserActionType", + "description": "reset_account_trading_data" + }, + "account_ids": { + "type": "array", + "minItems": 1, + "items": { + "type": "string" + } + } + } + }, "CreateExchangeConfigConfiguration": { "description": "CreateExchangeConfigConfiguration", "type": "object", @@ -2159,6 +2269,8 @@ "exchange_config_create", "exchange_config_edit", "exchange_config_delete", + "update_historical_exchanges_data", + "reset_account_trading_data", "strategy_create", "strategy_edit", "strategy_delete", @@ -2358,6 +2470,12 @@ { "$ref": "#/components/schemas/DeleteAccountConfiguration" }, + { + "$ref": "#/components/schemas/UpdateHistoricalExchangesDataConfiguration" + }, + { + "$ref": "#/components/schemas/ResetAccountTradingDataConfiguration" + }, { "$ref": "#/components/schemas/CreateExchangeConfigConfiguration" }, @@ -2400,6 +2518,8 @@ "account_create": "#/components/schemas/CreateAccountConfiguration", "account_edit": "#/components/schemas/EditAccountConfiguration", "account_delete": "#/components/schemas/DeleteAccountConfiguration", + "update_historical_exchanges_data": "#/components/schemas/UpdateHistoricalExchangesDataConfiguration", + "reset_account_trading_data": "#/components/schemas/ResetAccountTradingDataConfiguration", "exchange_config_create": "#/components/schemas/CreateExchangeConfigConfiguration", "exchange_config_edit": "#/components/schemas/EditExchangeConfigConfiguration", "exchange_config_delete": "#/components/schemas/DeleteExchangeConfigConfiguration", diff --git a/packages/protocol/test/test_account_trading.py b/packages/protocol/test/test_account_trading.py index d1e022fd11..d8cac32991 100644 --- a/packages/protocol/test/test_account_trading.py +++ b/packages/protocol/test/test_account_trading.py @@ -94,6 +94,9 @@ def make_instance(self, include_optional) -> AccountTrading: side = 'buy', quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = 'pending_creation', executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -107,6 +110,14 @@ def make_instance(self, include_optional) -> AccountTrading: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ] ) else: diff --git a/packages/protocol/test/test_account_trading_state.py b/packages/protocol/test/test_account_trading_state.py index 02360cac9c..70222bc0ed 100644 --- a/packages/protocol/test/test_account_trading_state.py +++ b/packages/protocol/test/test_account_trading_state.py @@ -96,6 +96,9 @@ def make_instance(self, include_optional) -> AccountTradingState: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -109,6 +112,14 @@ def make_instance(self, include_optional) -> AccountTradingState: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ) ) else: @@ -174,6 +185,9 @@ def make_instance(self, include_optional) -> AccountTradingState: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -187,6 +201,14 @@ def make_instance(self, include_optional) -> AccountTradingState: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ), ) """ diff --git a/packages/protocol/test/test_account_trading_with_account_id.py b/packages/protocol/test/test_account_trading_with_account_id.py index 14adcdd2c6..48c970e9f2 100644 --- a/packages/protocol/test/test_account_trading_with_account_id.py +++ b/packages/protocol/test/test_account_trading_with_account_id.py @@ -96,6 +96,9 @@ def make_instance(self, include_optional) -> AccountTradingWithAccountId: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -109,6 +112,14 @@ def make_instance(self, include_optional) -> AccountTradingWithAccountId: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ) ) else: diff --git a/packages/protocol/test/test_accounts_state.py b/packages/protocol/test/test_accounts_state.py index d347396eb5..03fdfb35c7 100644 --- a/packages/protocol/test/test_accounts_state.py +++ b/packages/protocol/test/test_accounts_state.py @@ -69,7 +69,10 @@ def make_instance(self, include_optional) -> AccountsState: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ] ) else: diff --git a/packages/protocol/test/test_create_exchange_config_configuration.py b/packages/protocol/test/test_create_exchange_config_configuration.py index 18af585bee..cfa806fc48 100644 --- a/packages/protocol/test/test_create_exchange_config_configuration.py +++ b/packages/protocol/test/test_create_exchange_config_configuration.py @@ -41,7 +41,10 @@ def make_instance(self, include_optional) -> CreateExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ) else: return CreateExchangeConfigConfiguration( @@ -51,7 +54,10 @@ def make_instance(self, include_optional) -> CreateExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ), + url = '', + historical_trade_symbols = [ + '' + ], ), ) """ diff --git a/packages/protocol/test/test_debug.py b/packages/protocol/test/test_debug.py index 714cd0887c..9daad058d8 100644 --- a/packages/protocol/test/test_debug.py +++ b/packages/protocol/test/test_debug.py @@ -145,7 +145,10 @@ def make_instance(self, include_optional) -> Debug: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ], account_tradings = [ octobot_protocol.models.account_trading_with_account_id.AccountTradingWithAccountId( @@ -210,6 +213,9 @@ def make_instance(self, include_optional) -> Debug: side = , quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = , executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ) ], @@ -223,6 +229,14 @@ def make_instance(self, include_optional) -> Debug: mark_price = 1.337, liquidation_price = 1.337, status = 'open', ) + ], + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) ], ), ) ], local_strategies = [ diff --git a/packages/protocol/test/test_debug_state.py b/packages/protocol/test/test_debug_state.py index 7e9a2381b2..1bb4ab87e8 100644 --- a/packages/protocol/test/test_debug_state.py +++ b/packages/protocol/test/test_debug_state.py @@ -135,13 +135,24 @@ def make_instance(self, include_optional) -> DebugState: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ], account_tradings = [ octobot_protocol.models.account_trading_with_account_id.AccountTradingWithAccountId( account_id = '', account_trading = octobot_protocol.models.account_trading.AccountTrading( - updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), ), ) + updated_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + transactions = [ + octobot_protocol.models.transaction.Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', ) + ], ), ) ], local_strategies = [ octobot_protocol.models.strategy.Strategy( diff --git a/packages/protocol/test/test_edit_exchange_config_configuration.py b/packages/protocol/test/test_edit_exchange_config_configuration.py index e241890163..ea17e3d34b 100644 --- a/packages/protocol/test/test_edit_exchange_config_configuration.py +++ b/packages/protocol/test/test_edit_exchange_config_configuration.py @@ -42,7 +42,10 @@ def make_instance(self, include_optional) -> EditExchangeConfigConfiguration: name = '', exchange = '', sandboxed = True, - url = '', ) + url = '', + historical_trade_symbols = [ + '' + ], ) ) else: return EditExchangeConfigConfiguration( diff --git a/packages/protocol/test/test_exchange_config.py b/packages/protocol/test/test_exchange_config.py index e8d1ee6bbd..7fde94d3cc 100644 --- a/packages/protocol/test/test_exchange_config.py +++ b/packages/protocol/test/test_exchange_config.py @@ -39,7 +39,10 @@ def make_instance(self, include_optional) -> ExchangeConfig: name = '', exchange = '', sandboxed = True, - url = '' + url = '', + historical_trade_symbols = [ + '' + ] ) else: return ExchangeConfig( diff --git a/packages/protocol/test/test_fee.py b/packages/protocol/test/test_fee.py new file mode 100644 index 0000000000..c728329c75 --- /dev/null +++ b/packages/protocol/test/test_fee.py @@ -0,0 +1,54 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.fee import Fee + +class TestFee(unittest.TestCase): + """Fee unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> Fee: + """Test Fee + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `Fee` + """ + model = Fee() + if include_optional: + return Fee( + amount = 1.337, + currency = '' + ) + else: + return Fee( + amount = 1.337, + currency = '', + ) + """ + + def testFee(self): + """Test Fee""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_reset_account_trading_data_configuration.py b/packages/protocol/test/test_reset_account_trading_data_configuration.py new file mode 100644 index 0000000000..f8d16f61a1 --- /dev/null +++ b/packages/protocol/test/test_reset_account_trading_data_configuration.py @@ -0,0 +1,58 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.reset_account_trading_data_configuration import ResetAccountTradingDataConfiguration + +class TestResetAccountTradingDataConfiguration(unittest.TestCase): + """ResetAccountTradingDataConfiguration unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> ResetAccountTradingDataConfiguration: + """Test ResetAccountTradingDataConfiguration + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `ResetAccountTradingDataConfiguration` + """ + model = ResetAccountTradingDataConfiguration() + if include_optional: + return ResetAccountTradingDataConfiguration( + action_type = 'automation_create', + account_ids = [ + '' + ] + ) + else: + return ResetAccountTradingDataConfiguration( + action_type = 'automation_create', + account_ids = [ + '' + ], + ) + """ + + def testResetAccountTradingDataConfiguration(self): + """Test ResetAccountTradingDataConfiguration""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_trade.py b/packages/protocol/test/test_trade.py index 232ffcec76..3960b720ff 100644 --- a/packages/protocol/test/test_trade.py +++ b/packages/protocol/test/test_trade.py @@ -42,6 +42,9 @@ def make_instance(self, include_optional) -> Trade: side = 'buy', quantity = 1.337, price = 1.337, + fee = octobot_protocol.models.fee.Fee( + amount = 1.337, + currency = '', ), status = 'pending_creation', executed_at = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f') ) diff --git a/packages/protocol/test/test_transaction.py b/packages/protocol/test/test_transaction.py new file mode 100644 index 0000000000..760003ef3c --- /dev/null +++ b/packages/protocol/test/test_transaction.py @@ -0,0 +1,60 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.transaction import Transaction + +class TestTransaction(unittest.TestCase): + """Transaction unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> Transaction: + """Test Transaction + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `Transaction` + """ + model = Transaction() + if include_optional: + return Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit' + ) + else: + return Transaction( + id = '', + timestamp = datetime.datetime.strptime('2013-10-20 19:20:30.00', '%Y-%m-%d %H:%M:%S.%f'), + asset = '', + amount = 1.337, + type = 'blockchain_deposit', + ) + """ + + def testTransaction(self): + """Test Transaction""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_transaction_type.py b/packages/protocol/test/test_transaction_type.py new file mode 100644 index 0000000000..07fe0560ae --- /dev/null +++ b/packages/protocol/test/test_transaction_type.py @@ -0,0 +1,33 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.transaction_type import TransactionType + +class TestTransactionType(unittest.TestCase): + """TransactionType unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def testTransactionType(self): + """Test TransactionType""" + # inst = TransactionType() + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_update_historical_exchanges_data_configuration.py b/packages/protocol/test/test_update_historical_exchanges_data_configuration.py new file mode 100644 index 0000000000..faaaf9d2a4 --- /dev/null +++ b/packages/protocol/test/test_update_historical_exchanges_data_configuration.py @@ -0,0 +1,55 @@ +# coding: utf-8 + +""" + OctoBot protocol types + + No description provided (generated by Openapi Generator https://github.com/openapitools/openapi-generator) + + The version of the OpenAPI document: 1.0.0 + Generated by OpenAPI Generator (https://openapi-generator.tech) + + Do not edit the class manually. +""" # noqa: E501 + + +import unittest + +from octobot_protocol.models.update_historical_exchanges_data_configuration import UpdateHistoricalExchangesDataConfiguration + +class TestUpdateHistoricalExchangesDataConfiguration(unittest.TestCase): + """UpdateHistoricalExchangesDataConfiguration unit test stubs""" + + def setUp(self): + pass + + def tearDown(self): + pass + + def make_instance(self, include_optional) -> UpdateHistoricalExchangesDataConfiguration: + """Test UpdateHistoricalExchangesDataConfiguration + include_optional is a boolean, when False only required + params are included, when True both required and + optional params are included """ + # uncomment below to create an instance of `UpdateHistoricalExchangesDataConfiguration` + """ + model = UpdateHistoricalExchangesDataConfiguration() + if include_optional: + return UpdateHistoricalExchangesDataConfiguration( + action_type = 'automation_create', + account_ids = [ + '' + ] + ) + else: + return UpdateHistoricalExchangesDataConfiguration( + action_type = 'automation_create', + ) + """ + + def testUpdateHistoricalExchangesDataConfiguration(self): + """Test UpdateHistoricalExchangesDataConfiguration""" + # inst_req_only = self.make_instance(include_optional=False) + # inst_req_and_optional = self.make_instance(include_optional=True) + +if __name__ == '__main__': + unittest.main() diff --git a/packages/protocol/test/test_user_action_configuration.py b/packages/protocol/test/test_user_action_configuration.py index f2afd8dd10..a2df617c6b 100644 --- a/packages/protocol/test/test_user_action_configuration.py +++ b/packages/protocol/test/test_user_action_configuration.py @@ -68,6 +68,9 @@ def make_instance(self, include_optional) -> UserActionConfiguration: id = '', automation_id = '', signal_type = 'actions', + account_ids = [ + '' + ], ) """ diff --git a/packages/sync/octobot_sync/constants.py b/packages/sync/octobot_sync/constants.py index ea4509d05a..b3bc4b9c81 100644 --- a/packages/sync/octobot_sync/constants.py +++ b/packages/sync/octobot_sync/constants.py @@ -42,6 +42,7 @@ EXCHANGE_ACCOUNTS_STATE_VERSION = "1.0.0" USER_ACCOUNTS_AUTH_STATE_VERSION = "1.0.0" USER_ACCOUNTS_TRADING_STATE_VERSION = "1.0.0" +USER_ACCOUNTS_HISTORY_STATE_VERSION = "1.0.0" USER_STRATEGIES_STATE_VERSION = "1.0.0" USER_DATA_STATE_VERSION = "1.0.0" USER_ACTIONS_STATE_VERSION = "1.0.0" diff --git a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py index 05370b1a2f..1ddd09d511 100644 --- a/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py +++ b/packages/sync/octobot_sync/sync/collection_backend/base_local_collection_storage.py @@ -61,6 +61,16 @@ def _file_path(self, storage_key: str) -> pathlib.Path: filename = f"{self._sanitize_storage_key(storage_key)}.json" return self._root / filename + def list_wallet_storage_keys(self) -> list[str]: + """Return wallet user ids that have a persisted state file at the collection root.""" + if not self._root.exists(): + return [] + return sorted( + storage_path.stem + for storage_path in self._root.glob("*.json") + if storage_path.is_file() + ) + def _missing_data_error(self, storage_key: str) -> collection_errors.CollectionNoDataError: return collection_errors.CollectionNoDataError( f"{self.collection} file does not exist for user_id {storage_key}" diff --git a/packages/sync/octobot_sync/sync/collection_providers/__init__.py b/packages/sync/octobot_sync/sync/collection_providers/__init__.py index 3a70e50ea0..66f51a8f49 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/__init__.py +++ b/packages/sync/octobot_sync/sync/collection_providers/__init__.py @@ -17,6 +17,7 @@ import octobot_sync.sync.collection_providers.user_account_authentication_provider as user_account_authentication_provider_module import octobot_sync.sync.collection_providers.user_account_provider as user_account_provider_module +import octobot_sync.sync.collection_providers.user_account_history_provider as user_account_history_provider_module import octobot_sync.sync.collection_providers.user_account_trading_provider as user_account_trading_provider_module import octobot_sync.sync.collection_providers.user_strategy_provider as user_strategy_provider_module @@ -25,11 +26,13 @@ user_account_authentication_provider_module.AccountAuthenticationProvider ) AccountTradingProvider = user_account_trading_provider_module.AccountTradingProvider +AccountHistoryProvider = user_account_history_provider_module.AccountHistoryProvider StrategyProvider = user_strategy_provider_module.StrategyProvider __all__ = [ "AccountProvider", "AccountAuthenticationProvider", "AccountTradingProvider", + "AccountHistoryProvider", "StrategyProvider", ] diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py new file mode 100644 index 0000000000..a7ecc5ca56 --- /dev/null +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_history_provider.py @@ -0,0 +1,38 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + + +import octobot_commons.singleton.singleton_class as singleton_class +import octobot_sync.constants as sync_constants +import octobot_protocol.models as protocol_models +import octobot_sync.enums as sync_enums + +import octobot_sync.sync.collection_backend.single_item_local_collection_provider as single_item_provider + + +class AccountHistoryProvider( + single_item_provider.SingleItemLocalCollectionProvider[protocol_models.PortfolioHistoricalValuesState], + singleton_class.Singleton, +): + """ + Singleton provider for per-account portfolio historical values. + + Each account is stored in its own encrypted file under + ``//.json``. + """ + COLLECTION = sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + STATE_VERSION = sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + STATE_CLASS = protocol_models.PortfolioHistoricalValuesState diff --git a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py index c7d731ae10..5cb37641eb 100644 --- a/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py +++ b/packages/sync/octobot_sync/sync/collection_providers/user_account_provider.py @@ -17,6 +17,9 @@ import typing +import octobot.community.authentication as community_authentication +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_commons.logging as commons_logging import octobot_commons.singleton.singleton_class as singleton_class import octobot_sync.constants as sync_constants import octobot_protocol.models as protocol_models @@ -127,3 +130,22 @@ def update_exchange_config( def delete_exchange_config(self, address: str, config_id: str) -> None: self._delete_item_for_key(address, self.EXCHANGE_CONFIGS_KEY, config_id) + + def list_registered_wallet_ids(self) -> list[str]: + return self._storage.list_wallet_storage_keys() + + def list_collectable_wallet_ids(self) -> list[str]: + logger = commons_logging.get_logger(self.__class__.__name__) + community_auth = community_authentication.CommunityAuthentication.instance() + collectable_wallet_ids = [] + for wallet_id in self.list_registered_wallet_ids(): + try: + community_auth.get_wallet_by_user_id(wallet_id) + except wallet_backend_errors.WalletNotFoundError: + logger.debug( + "Skipping wallet %s: not registered locally", + wallet_id, + ) + continue + collectable_wallet_ids.append(wallet_id) + return collectable_wallet_ids diff --git a/packages/sync/tests/sync/collection_providers/test_account_history_provider.py b/packages/sync/tests/sync/collection_providers/test_account_history_provider.py new file mode 100644 index 0000000000..9dbf1178a1 --- /dev/null +++ b/packages/sync/tests/sync/collection_providers/test_account_history_provider.py @@ -0,0 +1,147 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime +import json + +import mock +import octobot.community.authentication as community_authentication +import octobot_sync.sync.collection_backend.single_item_local_collection_storage as single_item_storage_module +import octobot_sync.sync.collection_providers.user_account_history_provider as history_provider_module +import octobot_sync.constants as sync_constants +import octobot_protocol.models as protocol_models +import octobot_sync.enums as sync_enums +import octobot_sync.sync.collection_backend.errors as collection_errors + +_TEST_ADDRESS = "0xaaabbbcccddd" +_TEST_ACCOUNT_ID = "acc-42" +_TEST_PRIVATE_KEY = "private-key" + + +def _patch_wallet(private_key: str = _TEST_PRIVATE_KEY): + wallet = mock.Mock() + wallet.private_key = private_key + auth = mock.Mock() + auth.get_wallet_by_user_id.return_value = wallet + return mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ) + + +def _sample_history_state(updated_at: datetime.datetime) -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDT", + values=[ + protocol_models.PortfolioHistoricalValue( + timestamp=updated_at, + total=1000.0, + ) + ], + ), + ) + + +class TestAccountHistoryProviderCollection: + def test_collection_is_USER_ACCOUNTS_HISTORY(self): + assert ( + history_provider_module.AccountHistoryProvider.COLLECTION + == sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + ) + + def test_storage_collection_matches(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + assert provider._storage.collection == sync_enums.Collections.USER_ACCOUNTS_HISTORY.value + + def test_storage_is_single_item_local_collection_storage(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + assert isinstance(provider._storage, single_item_storage_module.SingleItemLocalCollectionStorage) + + +class TestAccountHistoryProviderStateFormat: + def test_state_version_matches_constant(self): + assert ( + history_provider_module.AccountHistoryProvider.STATE_VERSION + == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + ) + + def test_state_class_is_portfolio_historical_values_state(self): + assert ( + history_provider_module.AccountHistoryProvider.STATE_CLASS + is protocol_models.PortfolioHistoricalValuesState + ) + + +class TestAccountHistoryProviderLoadSaveState: + def test_save_and_load_state_per_account_id(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + updated_time = datetime.datetime(2026, 1, 16, tzinfo=datetime.UTC) + updated_state = _sample_history_state(updated_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + loaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, updated_state) + reloaded_state = provider.load_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + assert loaded_state.history.values[0].timestamp == fixture_time + assert reloaded_state.history.values[0].timestamp == updated_time + + def test_load_state_encrypted_reads_persisted_blob(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + encrypted_blob = provider.load_state_encrypted(_TEST_ADDRESS, _TEST_ACCOUNT_ID) + assert "iv" in encrypted_blob + assert "data" in encrypted_blob + + +class TestAccountHistoryProviderSeparateFiles: + def test_accounts_use_separate_files(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + first_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + second_time = datetime.datetime(2026, 1, 16, tzinfo=datetime.UTC) + first_state = _sample_history_state(first_time) + second_state = _sample_history_state(second_time) + with _patch_wallet(): + provider.save_state(_TEST_ADDRESS, "acc-1", first_state) + provider.save_state(_TEST_ADDRESS, "acc-2", second_state) + loaded_first = provider.load_state(_TEST_ADDRESS, "acc-1") + loaded_second = provider.load_state(_TEST_ADDRESS, "acc-2") + assert loaded_first.history.values[0].timestamp == first_time + assert loaded_second.history.values[0].timestamp == second_time + + +class TestAccountHistoryProviderMissingState: + def test_load_state_raises_when_file_missing(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + with _patch_wallet(): + try: + provider.load_state(_TEST_ADDRESS, "missing-account") + raise AssertionError("Expected CollectionNoDataError") + except collection_errors.CollectionNoDataError: + pass + + +class TestAccountHistoryProviderWalletKey: + def test_save_requires_wallet_private_key(self, tmp_path): + provider = history_provider_module.AccountHistoryProvider(base_folder=str(tmp_path)) + fixture_time = datetime.datetime(2026, 1, 15, tzinfo=datetime.UTC) + history_state = _sample_history_state(fixture_time) + auth = mock.Mock() + auth.get_wallet_by_user_id.side_effect = KeyError("wallet not found") + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=auth, + ): + try: + provider.save_state(_TEST_ADDRESS, _TEST_ACCOUNT_ID, history_state) + raise AssertionError("Expected KeyError from missing wallet") + except KeyError: + pass diff --git a/packages/sync/tests/sync/collection_providers/test_user_account_provider.py b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py new file mode 100644 index 0000000000..beaa56c092 --- /dev/null +++ b/packages/sync/tests/sync/collection_providers/test_user_account_provider.py @@ -0,0 +1,54 @@ +# Drakkar-Software OctoBot-Sync +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# This library is free software; you can redistribute it and/or +# modify it under the terms of the GNU Lesser General Public +# License as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# Lesser General Public License for more details. +# +# You should have received a copy of the GNU Lesser General Public +# License along with this library. + +import mock + +import octobot.community.authentication as community_authentication +import octobot.community.wallet_backend.errors as wallet_backend_errors +import octobot_sync.sync.collection_providers.user_account_provider as account_provider_module + + +class TestListCollectableWalletIds: + def test_returns_only_wallets_registered_locally(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [ + "wallet-known", + "wallet-missing", + ] + community_auth = mock.Mock() + community_auth.get_wallet_by_user_id.side_effect = lambda wallet_id: ( + mock.Mock() + if wallet_id == "wallet-known" + else (_ for _ in ()).throw(wallet_backend_errors.WalletNotFoundError("missing")) + ) + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=community_auth, + ): + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == ["wallet-known"] + + def test_returns_empty_when_no_registered_wallets(self): + provider = mock.Mock(spec=account_provider_module.AccountProvider) + provider.list_registered_wallet_ids.return_value = [] + with mock.patch.object( + community_authentication.CommunityAuthentication, + "instance", + return_value=mock.Mock(), + ): + result = account_provider_module.AccountProvider.list_collectable_wallet_ids(provider) + assert result == [] diff --git a/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py b/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py index 00835bc156..a5e8224c2c 100644 --- a/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py +++ b/packages/tentacles/Backtesting/collectors/exchanges/exchange_bot_snapshot_data_collector/bot_snapshot_with_history_collector.py @@ -30,7 +30,7 @@ import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.symbols.symbol_util as symbol_util -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_backtesting.data as data import octobot_trading.api as trading_api import octobot_trading.errors as trading_errors @@ -396,7 +396,7 @@ async def get_ohlcv_history(self, exchange, symbol, time_frame): async def _import_candles_from_datafile(self, exchange, symbol, time_frame): return importers.import_ohlcvs( await self.database.select(backtesting_enums.ExchangeDataTables.OHLCV, - size=databases.SQLiteDatabase.DEFAULT_SIZE, + size=backtesting_databases.BacktestingDataSQLiteDatabase.DEFAULT_SIZE, exchange_name=exchange, symbol=symbol.symbol_str, time_frame=time_frame.value) ) diff --git a/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py b/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py index bd8c96bcae..e46b7c1266 100644 --- a/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py +++ b/packages/tentacles/Backtesting/collectors/exchanges/exchange_history_collector/tests/test_history_collector.py @@ -19,7 +19,7 @@ import json import asyncio -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_commons.symbols as commons_symbols import octobot_commons.enums as commons_enums import octobot_commons.constants as commons_constants @@ -59,7 +59,7 @@ async def data_collector(exchange_name, tentacles_setup_config, symbols, time_fr @contextlib.asynccontextmanager async def collector_database(collector): - database = databases.SQLiteDatabase(collector.file_path) + database = backtesting_databases.BacktestingDataSQLiteDatabase(collector.file_path) try: await database.initialize() yield database diff --git a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py index 716f8bec7d..4f1f9b1a04 100644 --- a/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py +++ b/packages/tentacles/Backtesting/converters/exchanges/legacy_data_converter/legacy_converter.py @@ -24,7 +24,7 @@ import octobot_backtesting.converters as converters import octobot_backtesting.data as backtesting_data import octobot_backtesting.enums as backtesting_enums -import octobot_commons.databases as databases +import octobot_backtesting.databases as backtesting_databases import octobot_commons.constants as commons_constants import octobot_commons.enums as commons_enums import octobot_commons.symbols.symbol_util as symbol_util @@ -78,8 +78,10 @@ async def can_convert(self, ) -> bool: async def convert(self) -> bool: try: - self.database = databases.SQLiteDatabase( - path.join(backtesting_constants.BACKTESTING_FILE_PATH, self.converted_file)) + self.database = backtesting_databases.BacktestingDataSQLiteDatabase( + path.join(backtesting_constants.BACKTESTING_FILE_PATH, self.converted_file), + read_only=False, + ) await self.database.initialize() await self._create_description() for time_frame in self.time_frames: diff --git a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py index ff0dfd4496..19fe5c56b4 100644 --- a/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py +++ b/packages/tentacles/Meta/DSL_operators/exchange_operators/exchange_personal_data_operators/copy_exchange_account_operators.py @@ -23,6 +23,7 @@ import octobot_commons.enums as commons_enums import octobot_commons.dsl_interpreter as dsl_interpreter import octobot_commons.errors as commons_errors +import octobot_commons.logging as commons_logging import octobot_commons.symbols.symbol_util as symbol_util @@ -33,6 +34,7 @@ import octobot_protocol.models as protocol_models import octobot_copy.copiers +import octobot_copy.copiers.formatter as copy_formatter import octobot_copy.entities import octobot_copy.constants @@ -198,6 +200,10 @@ async def pre_compute(self) -> None: copier_exchange_manager, copier_trading_mode, ) + commons_logging.get_logger("copy_exchange_account_operator").info( + f"Copying account for strategy {strategy_id!r}, account: " + f"{copy_formatter.format_reference_account_summary(reference_account)}" + ) copy_result = await account_copier.copy_account() self.value = self.create_re_callable_result_dict( keyword=self.get_name(), diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py index 46ff619d97..7f29b0493f 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/octobot_process_ops.py @@ -51,7 +51,9 @@ import octobot_sync.sync.collection_backend.errors as collection_errors import octobot_sync.sync.collection_providers as collection_providers -DEFAULT_PING_WAITING_TIME = 2.0 +DEFAULT_PING_WAITING_TIME = 30.0 +FAST_PING_WAITING_TIME = 5.0 +STEADY_PING_WAITING_TIME = 60.0 DEFAULT_ENSURE_TIMEOUT = 120.0 DEFAULT_FORCE_KILL_EXIT_WAIT_SECONDS = 5.0 DEFAULT_DSL_PROFILE_ID = "non-trading" @@ -209,6 +211,32 @@ def _in_restart_grace_period( return (now - last_updated_at) < ping_timeout +def _resolve_recall_waiting_time( + recall_state: octobot_process_state_import.OctobotProcessState, + loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], + *, + dsl_waiting_time: typing.Optional[float], + now: float, + ping_timeout: float, + stored_pid_running: bool, +) -> float: + child_confirmed_alive = _is_child_confirmed_alive(loaded_state) + is_running = stored_pid_running or child_confirmed_alive + if _in_restart_grace_period( + recall_state, + loaded_state, + now=now, + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ): + return FAST_PING_WAITING_TIME + if recall_state.init_state_ok and is_running: + if dsl_waiting_time is not None: + return float(dsl_waiting_time) + return STEADY_PING_WAITING_TIME + return FAST_PING_WAITING_TIME + + def _should_use_recall_path( recall_state: octobot_process_state_import.OctobotProcessState, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState], @@ -1130,7 +1158,7 @@ def _emit_ensure_recall( octobot_flow_entities.PostIterationActionsDetails( updated_exchange_account_elements=( parsed_process_bot_state.exchange_account_elements.to_dict( - include_default_values=True + include_default_values=False ) ), ).to_dict(include_default_values=False) @@ -1148,7 +1176,7 @@ async def _pre_compute_recall_path( last_result: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ping_timeout: float, loaded_state: typing.Optional[process_bot_state_import.ProcessBotState] = None, ) -> None: @@ -1205,6 +1233,14 @@ async def _pre_compute_recall_path( self.pid = resolved_pid recall_state = _apply_resolved_pid_to_state(recall_state, resolved_pid) _get_logger().info("process state path (re-call path): %s", state_path) + resolved_recall_interval = _resolve_recall_waiting_time( + recall_state, + loaded_state, + dsl_waiting_time=dsl_waiting_time, + now=now, + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ) # Running: stored recall pid or child-confirmed-alive → init_state_ok, optional EAE. is_running = stored_pid_running or child_confirmed_alive if is_running: @@ -1224,7 +1260,7 @@ async def _pre_compute_recall_path( state=updated, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded_state, ) return @@ -1242,7 +1278,7 @@ async def _pre_compute_recall_path( state=recall_state.model_copy(update={"state_file_path": state_path}), last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded_state, ) @@ -1254,7 +1290,7 @@ async def _pre_compute_first_spawn( last_result: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ) -> None: # One-time (or re-) materialization, free ports, env, and `Popen` at project root. exchange_auth_overrides = params.get("exchange_auth_data") @@ -1334,6 +1370,16 @@ async def _pre_compute_first_spawn( ) # First process state check after spawn (init cap still uses `state.started_waiting_at`). loaded = await _load_process_bot_state(state_file_path) + ping_timeout = float(params.get("ping_timeout") or DEFAULT_ENSURE_TIMEOUT) + stored_pid_running = state.pid > 0 and process_util.pid_is_running(state.pid) + resolved_recall_interval = _resolve_recall_waiting_time( + state, + loaded, + dsl_waiting_time=dsl_waiting_time, + now=time.time(), + ping_timeout=ping_timeout, + stored_pid_running=stored_pid_running, + ) is_live = loaded is not None and _is_process_state_alive(loaded) if is_live: state_pid = loaded.metadata.pid @@ -1355,11 +1401,19 @@ async def _pre_compute_first_spawn( ) ready = state.model_copy(update={"init_state_ok": True}) _release_child_listen_ports(web_port, node_port, user_folder) + resolved_recall_interval = _resolve_recall_waiting_time( + ready, + loaded, + dsl_waiting_time=dsl_waiting_time, + now=time.time(), + ping_timeout=ping_timeout, + stored_pid_running=True, + ) self._emit_ensure_recall( state=ready, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded, ) return @@ -1377,7 +1431,7 @@ async def _pre_compute_first_spawn( state=state, last_result=last_result, start_time=start_time, - recall_interval=recall_interval, + recall_interval=resolved_recall_interval, parsed_process_bot_state=loaded, ) @@ -1426,7 +1480,7 @@ async def _pre_compute_update_config_refresh( params: dict, *, start_time: float, - recall_interval: float, + dsl_waiting_time: typing.Optional[float], ping_timeout: float, ) -> None: # Resolve prior child layout from re-call payload; required for stop, wait, and paths to remove. @@ -1469,7 +1523,7 @@ async def _pre_compute_update_config_refresh( params, {}, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) async def pre_compute(self) -> None: @@ -1520,7 +1574,12 @@ async def pre_compute(self) -> None: last_result = self.get_last_execution_result(params) or {} start_time = time.time() ping_timeout = float(params.get("ping_timeout") or DEFAULT_ENSURE_TIMEOUT) - recall_interval = float(params.get("waiting_time") or DEFAULT_PING_WAITING_TIME) + waiting_time_param = params.get("waiting_time") + dsl_waiting_time = None + if waiting_time_param is not None: + waiting_time_value = float(waiting_time_param) + if waiting_time_value != DEFAULT_PING_WAITING_TIME: + dsl_waiting_time = waiting_time_value if self.matches_operator_signal(dsl_interpreter.OperatorSignal.UPDATE_CONFIG.value): await self._pre_compute_update_config_refresh( last_result, @@ -1528,7 +1587,7 @@ async def pre_compute(self) -> None: working_directory, params, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ping_timeout=ping_timeout, ) return @@ -1558,7 +1617,7 @@ async def pre_compute(self) -> None: params, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) return # 3. Recall vs respawn from liveness/grace rules. @@ -1573,7 +1632,7 @@ async def pre_compute(self) -> None: recall_state, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ping_timeout=ping_timeout, loaded_state=loaded_state, ) @@ -1600,7 +1659,7 @@ async def pre_compute(self) -> None: params, last_result, start_time=start_time, - recall_interval=recall_interval, + dsl_waiting_time=dsl_waiting_time, ) diff --git a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py index 9f3a8473cd..b099a6a779 100644 --- a/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py +++ b/packages/tentacles/Meta/DSL_operators/octobot_process_operators/tests/test_octobot_process_ops.py @@ -115,7 +115,7 @@ def _healthy_recall_inner( user_root = user_root or "/x/ub" state_fn = os.path.join(user_root, octobot_constants.PROCESS_BOT_STATE_FILE_NAME) return { - "waiting_time": octobot_process_ops.DEFAULT_PING_WAITING_TIME, + "waiting_time": octobot_process_ops.STEADY_PING_WAITING_TIME, "last_execution_time": 0.0, "http_base_url": "http://127.0.0.1:20050", "web_port": 20050, @@ -1957,7 +1957,7 @@ async def test_returns_recallable_with_init_state_ok_after_first_spawn(self, tmp assert le.get("init_state_ok") is True assert le.get("http_base_url", "").startswith("http://") assert le.get("pid") == 10001 - assert le.get("waiting_time") == octobot_process_ops.DEFAULT_PING_WAITING_TIME + assert le.get("waiting_time") == octobot_process_ops.STEADY_PING_WAITING_TIME assert octobot_flow_entities.PostIterationActionsDetails.__name__ in le post = octobot_flow_entities.PostIterationActionsDetails.from_dict( le[octobot_flow_entities.PostIterationActionsDetails.__name__] @@ -2318,7 +2318,7 @@ async def test_does_not_apply_init_timeout_after_init_state_ok(self, tmp_path): class TestEnsureOctobotProcessWaitingTimeConstantInPayload: - async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): + async def test_waiting_time_uses_fast_tier_during_init_even_with_dsl_override(self, tmp_path): start_script = tmp_path / "start.py" start_script.write_text("#", encoding="utf-8") op = EnsureOctobotProcessOperator( @@ -2361,6 +2361,32 @@ async def test_waiting_time_uses_parameter_for_recall_emissions(self, tmp_path): assert isinstance(op.value, dict) le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] assert isinstance(le, dict) + assert le.get("waiting_time") == octobot_process_ops.FAST_PING_WAITING_TIME + + async def test_dsl_waiting_time_overrides_steady_tier_when_child_is_healthy(self, tmp_path): + inner = _healthy_recall_inner(tmp_path=tmp_path, init_state_ok=True) + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + waiting_time=7.0, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == inner["pid"], + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ): + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] assert le.get("waiting_time") == 7.0 @@ -3927,3 +3953,187 @@ async def live_state_with_same_pid(*_unused): spawn_mock.assert_not_called() rebind_mock.assert_not_called() assert op.pid == 25044 + + +def _minimal_octobot_process_state(**updates) -> octobot_process_state_import.OctobotProcessState: + base_state = { + "http_base_url": "http://127.0.0.1:20050", + "web_port": 20050, + "node_port": 30050, + "user_root": "/x/ub", + "user_folder": "ub", + "log_folder": "/x/l", + "profile_id": "p", + "pid": 10001, + "state_file_path": "/x/ub/process_bot_state.json", + "executor_id": TEST_EXECUTOR_ID, + } + base_state.update(updates) + return octobot_process_state_import.OctobotProcessState.model_validate(base_state) + + +class TestTwoTierPingWaitingTime: + def test_fast_when_init_state_not_ok(self): + recall_state = _minimal_octobot_process_state(init_state_ok=False) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + def test_steady_when_init_ok_and_child_running(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=True, + ) + assert resolved_interval == octobot_process_ops.STEADY_PING_WAITING_TIME + + def test_restart_grace_window_uses_fast_tier(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True, pid=0) + loaded_state = process_bot_state_import.ProcessBotState( + metadata=process_bot_state_import.Metadata(updated_at=995.0), + ) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + loaded_state, + dsl_waiting_time=None, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + def test_dsl_waiting_time_overrides_steady_only(self): + recall_state = _minimal_octobot_process_state(init_state_ok=True) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=7.0, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=True, + ) + assert resolved_interval == 7.0 + + def test_dsl_waiting_time_does_not_override_fast_tier(self): + recall_state = _minimal_octobot_process_state(init_state_ok=False) + resolved_interval = octobot_process_ops._resolve_recall_waiting_time( + recall_state, + None, + dsl_waiting_time=7.0, + now=1000.0, + ping_timeout=30.0, + stored_pid_running=False, + ) + assert resolved_interval == octobot_process_ops.FAST_PING_WAITING_TIME + + async def test_executor_restart_first_spawn_uses_fast_waiting_time(self, tmp_path): + (tmp_path / "start.py").write_text("#", encoding="utf-8") + inner = _healthy_recall_inner( + pid=10002, + tmp_path=tmp_path, + executor_id="old-executor-id", + ) + op = octobot_process_ops.create_octobot_process_operators( + None, TEST_EXECUTOR_ID + )[0]( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=_re_calling_ensure_value(inner), + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + process_util, + "pid_is_running", + return_value=False, + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": inner["user_root"], + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_return_none_mock), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock: + spawn_mock.return_value = mock.Mock(spec=["pid"], pid=30005) + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + assert le.get("waiting_time") == octobot_process_ops.FAST_PING_WAITING_TIME + assert le.get("init_state_ok") is False + + +class TestEmitEnsureRecallEaeSerialization: + async def test_emitted_post_iteration_eae_omits_default_only_fields(self, tmp_path): + start_script = tmp_path / "start.py" + start_script.write_text("#", encoding="utf-8") + op = EnsureOctobotProcessOperator( + user_folder="ub", + user_id=_PROCESS_TEST_USER_ID, + profile_data=_MINIMAL_PROFILE_DATA, + last_execution_result=None, + ) + with mock.patch.object( + octobot_process_ops.os, + "getcwd", + return_value=str(tmp_path), + ), mock.patch.object( + octobot_process_ops, + "ensure_user_profile_and_layout", + new=mock.AsyncMock( + return_value={ + "user_root": str( + tmp_path / commons_constants.USER_FOLDER / commons_constants.AUTOMATIONS_FOLDER / "ub" + ), + "profile_id": "x", + "already_prepared": True, + } + ), + ), mock.patch.object( + octobot_process_ops, + "_listen_port_pair_with_shared_scan_offset", + return_value=(20050, 30050), + ), mock.patch.object( + process_util, + "spawn_managed_subprocess", + ) as spawn_mock, mock.patch.object( + process_util, + "pid_is_running", + side_effect=lambda process_id: process_id == 10001, + ), mock.patch.object( + octobot_process_ops, + "_load_process_bot_state", + new=mock.AsyncMock(side_effect=_async_live_process_bot_state_with_pid_10001), + ): + spawn_mock.return_value.pid = 10001 + await op.pre_compute() + le = op.value[dsl_interpreter.ReCallingOperatorResult.__name__]["last_execution_result"] + post_iteration_payload = le[octobot_flow_entities.PostIterationActionsDetails.__name__] + emitted_eae_dict = post_iteration_payload["updated_exchange_account_elements"] + assert "trade_summaries" not in emitted_eae_dict + diff --git a/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py index 22d9cb4563..fbe030d715 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/configuration/exchanges_configuration.py @@ -13,25 +13,11 @@ # # You should have received a copy of the GNU Lesser General Public # License along with this library. -import octobot_commons.constants as constants - -# TODO later: find a way to store this in exchange tentacles instead and use exchange.get_default_reference_market -# Issue: hollaex based exchanages require an exchange configuration to be identified as such -_SPECIFIC_REFERENCE_MARKET_PER_EXCHANGE: dict[str, str] = { - "coinbase": "USDC", - "binance": "USDC", -} _EXCHANGES_WITH_DIFFERENT_PUBLIC_DATA_AFTER_AUTH = set[str]([ "mexc", "lbank", ]) -def get_default_reference_market_per_exchange(exchanges: list[str]) -> dict[str, str]: - return {exchange: get_default_exchange_reference_market(exchange) for exchange in exchanges} - -def get_default_exchange_reference_market(exchange: str) -> str: - return _SPECIFIC_REFERENCE_MARKET_PER_EXCHANGE.get(exchange, constants.DEFAULT_REFERENCE_MARKET) - def is_exchange_with_different_public_data_after_auth(exchange: str) -> bool: return exchange in _EXCHANGES_WITH_DIFFERENT_PUBLIC_DATA_AFTER_AUTH diff --git a/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py b/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py index c63b6507b6..16c2903f56 100644 --- a/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py +++ b/packages/tentacles/Meta/Keywords/scripting_library/configuration/indexes_configuration.py @@ -24,10 +24,10 @@ import octobot_evaluators.constants as evaluators_constants import octobot_trading.constants as trading_constants +import octobot_trading.api.exchange as exchange_api import tentacles.Trading.Mode.index_trading_mode.index_trading as index_trading import octobot_copy.enums as rebalancer_enums -import tentacles.Meta.Keywords.scripting_library.configuration.exchanges_configuration as exchanges_configuration def create_index_config_from_tentacles_config( @@ -36,7 +36,7 @@ def create_index_config_from_tentacles_config( ) -> commons_profiles.ProfileData: trading_mode_config = tentacles_config[0].config distribution = trading_mode_config[index_trading.IndexTradingModeProducer.INDEX_CONTENT] - reference_market = exchanges_configuration.get_default_exchange_reference_market(exchange) + reference_market = exchange_api.get_default_exchange_reference_market(exchange) # replace USD by reference market for element in distribution: if element[rebalancer_enums.DistributionKeys.NAME] == "USD": diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py index 7f34fad531..292bc2239b 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/main.py @@ -30,6 +30,7 @@ setup, exchanges, wallets, + accounts, debug, config, logs, @@ -38,7 +39,7 @@ ) except ImportError: from api.route_provider import register_all_provider_routes # type: ignore[no-redef] - from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, debug, config, logs, octobots, dsl # type: ignore[no-redef] + from api.routes import login, nodes, users, tasks, setup, exchanges, wallets, accounts, debug, config, logs, octobots, dsl # type: ignore[no-redef] def build_api_router() -> APIRouter: @@ -52,6 +53,7 @@ def build_api_router() -> APIRouter: api_router.include_router(tasks.router, prefix="/tasks") api_router.include_router(octobots.router, prefix="/octobots") api_router.include_router(nodes.router, prefix="/nodes") + api_router.include_router(accounts.router, prefix="/accounts") api_router.include_router(debug.router, prefix="/debug") api_router.include_router(config.router, prefix="/config") api_router.include_router(logs.router, prefix="/logs") diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/accounts.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/accounts.py new file mode 100644 index 0000000000..ee6dbd282b --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/accounts.py @@ -0,0 +1,78 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import json +import typing + +from fastapi import APIRouter, Query +from fastapi.responses import JSONResponse + +import octobot_node.protocol.accounts_history as accounts_history_module + +try: + from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + ) +except ImportError: + from api.deps import CurrentUser # type: ignore[no-redef] + from api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + ) + +router = APIRouter(tags=["accounts"]) + + +@router.get("/aggregated/historical-values") +async def get_aggregated_account_historical_values( + current_user: CurrentUser, + is_simulated: bool = Query(...), + wallet_address: typing.Optional[str] = Query(default=None), +) -> JSONResponse: + """Compute aggregated portfolio historical values for all matching accounts.""" + ensure_debug_routes_enabled() + ensure_scheduler_initialized() + resolved_user_id = resolve_user_id(current_user, wallet_address) + history_state = ( + await accounts_history_module.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + resolved_user_id, + is_simulated=is_simulated, + ) + ) + return JSONResponse(content=json.loads(history_state.to_json())) + + +@router.get("/{account_id}/historical-values") +async def get_account_historical_values( + account_id: str, + current_user: CurrentUser, + wallet_address: typing.Optional[str] = Query(default=None), +) -> JSONResponse: + """Compute portfolio historical values on-the-fly for an account.""" + ensure_debug_routes_enabled() + ensure_scheduler_initialized() + resolved_user_id = resolve_user_id(current_user, wallet_address) + history_state = ( + await accounts_history_module.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions( + resolved_user_id, + account_id, + ) + ) + return JSONResponse(content=json.loads(history_state.to_json())) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py index d75e9ac785..06e65e2f9e 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/debug.py @@ -20,7 +20,6 @@ from fastapi import APIRouter, Body, HTTPException, Query, Response, status from fastapi.responses import JSONResponse -import octobot_node.config import octobot_node.models import octobot_node.protocol.debug as debug_protocol import octobot_node.protocol.user_actions as user_actions_protocol @@ -29,10 +28,18 @@ try: from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] - from tentacles.Services.Interfaces.node_api_interface.api.user_id import evm_to_user_id # type: ignore[no-redef] + from tentacles.Services.Interfaces.node_api_interface.api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + ) except ImportError: from api.deps import CurrentUser # type: ignore[no-redef] - from api.user_id import evm_to_user_id # type: ignore[no-redef] + from api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + ) router = APIRouter(tags=["debug"]) @@ -66,37 +73,6 @@ def _parse_user_action_payload(payload: typing.Any) -> protocol_models.UserActio return user_action -def _resolve_wallet_address( - current_user: octobot_node.models.User, - wallet_address: typing.Optional[str], -) -> str: - """Return the resolved EVM wallet address (normalized to lowercase).""" - if wallet_address is None: - return current_user.email - normalized_wallet_address = wallet_address.lower() - if normalized_wallet_address == current_user.email.lower(): - return normalized_wallet_address - if current_user.is_superuser: - return normalized_wallet_address - raise HTTPException( - status_code=status.HTTP_403_FORBIDDEN, - detail="Wallet address does not belong to the authenticated user", - ) - - -def _resolve_user_id( - current_user: octobot_node.models.User, - wallet_address: typing.Optional[str], -) -> str: - """Resolve EVM wallet address to the Starfish user_id used by the sync-core. - - The HTTP debug API accepts the EVM address for user-facing consistency, but all - internal protocol and scheduler calls use the Starfish user_id. - """ - evm_address = _resolve_wallet_address(current_user, wallet_address) - return evm_to_user_id(evm_address) - - def _extract_automation_parent_id( user_action: protocol_models.UserAction, ) -> typing.Optional[str]: @@ -133,11 +109,11 @@ async def _resolve_execution_user_id( wallet-scoped lookup. Admins may act on another wallet's automation without passing ``wallet_address``, but only after API-side authorization and owner resolution here. """ - # Explicit wallet override: admin-gated in _resolve_wallet_address. + # Explicit wallet override: admin-gated in resolve_wallet_address. if wallet_address is not None: - return _resolve_user_id(current_user, wallet_address) + return resolve_user_id(current_user, wallet_address) - caller_user_id = _resolve_user_id(current_user, None) + caller_user_id = resolve_user_id(current_user, None) parent_automation_id = _extract_automation_parent_id(user_action) if parent_automation_id is None: return caller_user_id @@ -181,22 +157,6 @@ async def _resolve_execution_user_id( ) -def _ensure_debug_routes_enabled() -> None: - if octobot_node.config.settings.is_node_side_encryption_enabled: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail="Debug routes are disabled when node-side encryption is enabled", - ) - - -def _ensure_scheduler_initialized() -> None: - if not octobot_node.scheduler.is_initialized(): - raise HTTPException( - status_code=status.HTTP_503_SERVICE_UNAVAILABLE, - detail="Scheduler not initialized", - ) - - @router.get("/", response_model=protocol_models.DebugState) async def get_debug( current_user: CurrentUser, @@ -207,9 +167,9 @@ async def get_debug( Requires authenticated user (``CurrentUser`` / HTTP Basic wallet + passphrase). Missing or invalid credentials return 401. """ - _ensure_debug_routes_enabled() - _ensure_scheduler_initialized() - resolved_user_id = _resolve_user_id(current_user, wallet_address) + ensure_debug_routes_enabled() + ensure_scheduler_initialized() + resolved_user_id = resolve_user_id(current_user, wallet_address) debug_state = await debug_protocol.get_debug_state(resolved_user_id) return JSONResponse(content=json.loads(debug_state.to_json())) @@ -225,8 +185,8 @@ async def execute_user_action( Requires authenticated user (``CurrentUser`` / HTTP Basic wallet + passphrase). Missing or invalid credentials return 401. """ - _ensure_debug_routes_enabled() - _ensure_scheduler_initialized() + ensure_debug_routes_enabled() + ensure_scheduler_initialized() user_action = _parse_user_action_payload(payload) resolved_user_id = await _resolve_execution_user_id(current_user, wallet_address, user_action) try: diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py index a2a0fe1ae7..a1512e53dc 100644 --- a/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/routes/octobots.py @@ -19,7 +19,7 @@ import octobot_node.errors as node_errors import octobot_node.scheduler -import octobot_node.scheduler.generic_process_octobot as generic_process_octobot_module +import octobot_node.scheduler.automations.generic_process_octobot as generic_process_octobot_module try: from tentacles.Services.Interfaces.node_api_interface.api.deps import CurrentUser # type: ignore[no-redef] diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/api/wallet_route_helpers.py b/packages/tentacles/Services/Interfaces/node_api_interface/api/wallet_route_helpers.py new file mode 100644 index 0000000000..5c5983539a --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/api/wallet_route_helpers.py @@ -0,0 +1,75 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. +# +# OctoBot is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either +# version 3.0 of the License, or (at your option) any later version. +# +# OctoBot is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +# General Public License for more details. +# +# You should have received a copy of the GNU General Public +# License along with OctoBot. If not, see . + +import typing + +from fastapi import HTTPException, status + +import octobot_node.config +import octobot_node.models +import octobot_node.scheduler + +try: + from tentacles.Services.Interfaces.node_api_interface.api.user_id import evm_to_user_id # type: ignore[no-redef] +except ImportError: + from api.user_id import evm_to_user_id # type: ignore[no-redef] + + +def resolve_wallet_address( + current_user: octobot_node.models.User, + wallet_address: typing.Optional[str], +) -> str: + """Return the resolved EVM wallet address (normalized to lowercase).""" + if wallet_address is None: + return current_user.email + normalized_wallet_address = wallet_address.lower() + if normalized_wallet_address == current_user.email.lower(): + return normalized_wallet_address + if current_user.is_superuser: + return normalized_wallet_address + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="Wallet address does not belong to the authenticated user", + ) + + +def resolve_user_id( + current_user: octobot_node.models.User, + wallet_address: typing.Optional[str], +) -> str: + """Resolve EVM wallet address to the Starfish user_id used by the sync-core. + + The HTTP debug API accepts the EVM address for user-facing consistency, but all + internal protocol and scheduler calls use the Starfish user_id. + """ + evm_address = resolve_wallet_address(current_user, wallet_address) + return evm_to_user_id(evm_address) + + +def ensure_debug_routes_enabled() -> None: + if octobot_node.config.settings.is_node_side_encryption_enabled: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail="Debug routes are disabled when node-side encryption is enabled", + ) + + +def ensure_scheduler_initialized() -> None: + if not octobot_node.scheduler.is_initialized(): + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Scheduler not initialized", + ) diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_accounts.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_accounts.py new file mode 100644 index 0000000000..157ab3b6d2 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_routes_accounts.py @@ -0,0 +1,124 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import datetime +import mock +import pytest + +import octobot_node.config +import octobot_protocol.models as protocol_models +import octobot_sync.constants as sync_constants + +from .conftest import TENANT_USER_ID + + +def _sample_history_state() -> protocol_models.PortfolioHistoricalValuesState: + return protocol_models.PortfolioHistoricalValuesState( + version=sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION, + history=protocol_models.PortfolioHistoricalValues( + unit="USDT", + values=[ + protocol_models.PortfolioHistoricalValue( + timestamp=datetime.datetime(2024, 1, 1, tzinfo=datetime.timezone.utc), + total=1000.0, + assets=[ + protocol_models.HistoricalAssetsForTradingType( + trading_type=protocol_models.TradingType.SPOT, + assets=[ + protocol_models.HistoricalAssetValue( + symbol="USDT", + holdings=1000.0, + value=1000.0, + ) + ], + ) + ], + ) + ], + ), + ) + + +class TestGetAccountHistoricalValues: + def test_returns_history_state_for_account(self, tenant_client, mock_auth): + history_state = _sample_history_state() + mock_compute = mock.AsyncMock(return_value=history_state) + with mock.patch( + "octobot_node.protocol.accounts_history.compute_portfolio_historical_values_from_latest_portfolio_trades_and_transactions", + new=mock_compute, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + response = tenant_client.get("/api/v1/accounts/acc-1/historical-values") + assert response.status_code == 200 + body = response.json() + assert body["version"] == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + assert body["history"]["unit"] == "USDT" + assert body["history"]["values"][0]["total"] == 1000.0 + mock_compute.assert_awaited_once_with(TENANT_USER_ID, "acc-1") + + def test_returns_404_when_debug_routes_disabled(self, tenant_client, mock_auth): + with mock.patch.object( + octobot_node.config.settings, + "is_node_side_encryption_enabled", + True, + ): + response = tenant_client.get("/api/v1/accounts/acc-1/historical-values") + assert response.status_code == 404 + + def test_returns_503_when_scheduler_not_initialized(self, tenant_client, mock_auth): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): + response = tenant_client.get("/api/v1/accounts/acc-1/historical-values") + assert response.status_code == 503 + + def test_returns_401_without_auth(self, client, mock_auth): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + response = client.get("/api/v1/accounts/acc-1/historical-values") + assert response.status_code == 401 + + +class TestGetAggregatedAccountHistoricalValues: + def test_returns_aggregated_history_state(self, tenant_client, mock_auth): + history_state = _sample_history_state() + mock_compute = mock.AsyncMock(return_value=history_state) + with mock.patch( + "octobot_node.protocol.accounts_history.compute_aggregated_portfolio_historical_values_from_latest_portfolio_trades_and_transactions", + new=mock_compute, + ): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + response = tenant_client.get( + "/api/v1/accounts/aggregated/historical-values", + params={"is_simulated": False}, + ) + assert response.status_code == 200 + body = response.json() + assert body["version"] == sync_constants.USER_ACCOUNTS_HISTORY_STATE_VERSION + assert body["history"]["unit"] == "USDT" + mock_compute.assert_awaited_once_with(TENANT_USER_ID, is_simulated=False) + + def test_returns_404_when_debug_routes_disabled(self, tenant_client, mock_auth): + with mock.patch.object( + octobot_node.config.settings, + "is_node_side_encryption_enabled", + True, + ): + response = tenant_client.get( + "/api/v1/accounts/aggregated/historical-values", + params={"is_simulated": True}, + ) + assert response.status_code == 404 + + def test_returns_503_when_scheduler_not_initialized(self, tenant_client, mock_auth): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): + response = tenant_client.get( + "/api/v1/accounts/aggregated/historical-values", + params={"is_simulated": True}, + ) + assert response.status_code == 503 + + def test_returns_401_without_auth(self, client, mock_auth): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=True): + response = client.get( + "/api/v1/accounts/aggregated/historical-values", + params={"is_simulated": False}, + ) + assert response.status_code == 401 diff --git a/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_wallet_route_helpers.py b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_wallet_route_helpers.py new file mode 100644 index 0000000000..d733c7c082 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_api_interface/tests/test_wallet_route_helpers.py @@ -0,0 +1,88 @@ +# This file is part of OctoBot Node (https://github.com/Drakkar-Software/OctoBot-Node) +# Copyright (c) 2025 Drakkar-Software, All rights reserved. + +import uuid + +import mock +import pytest +from fastapi import HTTPException + +import octobot_node.config +import octobot_node.models +import octobot_node.scheduler + +try: + from tentacles.Services.Interfaces.node_api_interface.api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + resolve_wallet_address, + ) + import tentacles.Services.Interfaces.node_api_interface.api.wallet_route_helpers as wallet_route_helpers_module # type: ignore[no-redef] +except ImportError: + from api.wallet_route_helpers import ( # type: ignore[no-redef] + ensure_debug_routes_enabled, + ensure_scheduler_initialized, + resolve_user_id, + resolve_wallet_address, + ) + import api.wallet_route_helpers as wallet_route_helpers_module # type: ignore[no-redef] + + +def _user(email: str, is_superuser: bool = False) -> octobot_node.models.User: + return octobot_node.models.User( + id=uuid.uuid5(uuid.NAMESPACE_URL, email), + email=email, + is_active=True, + is_superuser=is_superuser, + full_name="Test User", + ) + + +class TestResolveWalletAddress: + def test_returns_authenticated_wallet_when_query_param_missing(self): + current_user = _user("0xabc") + assert resolve_wallet_address(current_user, None) == "0xabc" + + def test_returns_matching_wallet_for_same_user(self): + current_user = _user("0xAbC") + assert resolve_wallet_address(current_user, "0xabc") == "0xabc" + + def test_allows_superuser_to_target_other_wallet(self): + current_user = _user("0xowner", is_superuser=True) + assert resolve_wallet_address(current_user, "0xother") == "0xother" + + def test_forbids_non_superuser_other_wallet(self): + current_user = _user("0xowner", is_superuser=False) + with pytest.raises(HTTPException) as error: + resolve_wallet_address(current_user, "0xother") + assert error.value.status_code == 403 + + +class TestResolveUserId: + @mock.patch.object(wallet_route_helpers_module, "evm_to_user_id", return_value="starfish-user") + def test_maps_wallet_to_starfish_user_id(self, mock_evm_to_user_id): + current_user = _user("0xabc") + assert resolve_user_id(current_user, None) == "starfish-user" + mock_evm_to_user_id.assert_called_once_with("0xabc") + + +class TestEnsureDebugRoutesEnabled: + def test_raises_404_when_node_side_encryption_enabled(self): + with mock.patch.object( + type(octobot_node.config.settings), + "is_node_side_encryption_enabled", + new_callable=mock.PropertyMock, + return_value=True, + ): + with pytest.raises(HTTPException) as error: + ensure_debug_routes_enabled() + assert error.value.status_code == 404 + + +class TestEnsureSchedulerInitialized: + def test_raises_503_when_scheduler_not_initialized(self): + with mock.patch("octobot_node.scheduler.is_initialized", return_value=False): + with pytest.raises(HTTPException) as error: + ensure_scheduler_initialized() + assert error.value.status_code == 503 diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/charts/SvgLineChart.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/charts/SvgLineChart.tsx new file mode 100644 index 0000000000..df388a3428 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Common/charts/SvgLineChart.tsx @@ -0,0 +1,215 @@ +import { useLayoutEffect, useMemo, useRef, useState, type ReactNode } from "react" +import { createPortal } from "react-dom" + +import { + buildLinearTicks, + buildLinePath, + domainFromPoints, + findNearestPointIndex, + formatDefaultYTick, + getChartTooltipViewportPosition, + scaleLinear, + type ChartTooltipViewportPosition, + type SvgLineChartPoint, +} from "@/lib/charts/svg-line-chart" + +const CHART_PADDING = { + top: 12, + right: 12, + bottom: 24, + left: 48, +} + +export type SvgLineChartProps = { + points: SvgLineChartPoint[] + width?: number + height?: number + ariaLabel?: string + renderTooltip?: (point: SvgLineChartPoint, index: number) => ReactNode + emptyMessage?: string + formatYTick?: (value: number) => string +} + +type TooltipState = { + index: number +} + +export function SvgLineChart({ + points, + width = 640, + height = 220, + ariaLabel = "Line chart", + renderTooltip, + emptyMessage = "No data", + formatYTick = formatDefaultYTick, +}: SvgLineChartProps) { + const svgRef = useRef(null) + const [tooltipState, setTooltipState] = useState(null) + const [tooltipPosition, setTooltipPosition] = + useState(null) + + const plotWidth = width - CHART_PADDING.left - CHART_PADDING.right + const plotHeight = height - CHART_PADDING.top - CHART_PADDING.bottom + + const { path, scaledPoints, xScale, yScale, yTicks } = useMemo(() => { + const xDomain = domainFromPoints(points, "x") + const yDomain = domainFromPoints(points, "y") + const computedXScale = scaleLinear( + xDomain.min, + xDomain.max, + CHART_PADDING.left, + CHART_PADDING.left + plotWidth, + ) + const computedYScale = scaleLinear( + yDomain.min, + yDomain.max, + CHART_PADDING.top + plotHeight, + CHART_PADDING.top, + ) + return { + path: buildLinePath(points, computedXScale, computedYScale), + scaledPoints: points.map((point) => ({ + x: computedXScale.toSvg(point.x), + y: computedYScale.toSvg(point.y), + })), + xScale: computedXScale, + yScale: computedYScale, + yTicks: buildLinearTicks(yDomain.min, yDomain.max), + } + }, [points, plotHeight, plotWidth]) + + const activePoint = + tooltipState === null ? null : points[tooltipState.index] + const activeScaledPoint = + tooltipState === null ? null : scaledPoints[tooltipState.index] + + useLayoutEffect(() => { + if ( + tooltipState === null || + activeScaledPoint === null || + svgRef.current === null + ) { + setTooltipPosition(null) + return + } + const svgBounds = svgRef.current.getBoundingClientRect() + setTooltipPosition( + getChartTooltipViewportPosition( + svgBounds, + activeScaledPoint.x, + activeScaledPoint.y, + width, + height, + ), + ) + }, [activeScaledPoint, height, tooltipState, width]) + + if (points.length === 0) { + return ( +

+ {emptyMessage} +

+ ) + } + + const updateTooltipFromClientPosition = (clientX: number) => { + const svgElement = svgRef.current + if (!svgElement) { + return + } + const bounds = svgElement.getBoundingClientRect() + const relativeX = + ((clientX - bounds.left) / bounds.width) * width + const dataX = xScale.min + ((relativeX - CHART_PADDING.left) / plotWidth) * (xScale.max - xScale.min) + const nearestIndex = findNearestPointIndex(points, dataX) + if (nearestIndex === null) { + return + } + setTooltipState({ index: nearestIndex }) + } + + const tooltipContent = + tooltipState !== null && + activePoint !== null && + renderTooltip?.(activePoint, tooltipState.index) + + return ( +
+ setTooltipState(null)} + onMouseMove={(event) => + updateTooltipFromClientPosition(event.clientX) + } + > + {yTicks.map((tickValue) => { + const tickY = yScale.toSvg(tickValue) + return ( + + + + {formatYTick(tickValue)} + + + ) + })} + + + {scaledPoints.map((scaledPoint, pointIndex) => ( + + ))} + + {tooltipContent && + tooltipPosition !== null && + createPortal( +
+ {tooltipContent} +
, + document.body, + )} +
+ ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx index 6ecfc2e5a8..b14241baaa 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/DebugTabsPanel.tsx @@ -9,11 +9,13 @@ import type { UserAction, } from "@/client" import { DebugTabDeleteControls } from "@/components/Debug/DebugTabDeleteControls" +import { AccountsHistoryDialog } from "@/components/Debug/dialogs/AccountsHistoryDialog" import { AccountsTable } from "@/components/Debug/tables/AccountsTable" import { AutomationsTable } from "@/components/Debug/tables/AutomationsTable" import { ExchangeConfigsTable } from "@/components/Debug/tables/ExchangeConfigsTable" import { StrategiesTable } from "@/components/Debug/tables/StrategiesTable" import { UserActionsTable } from "@/components/Debug/tables/UserActionsTable" +import { Button } from "@/components/ui/button" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { DEBUG_DELETABLE_TAB_VALUES } from "@/lib/debug/constants" import type { ExecuteActionDraft } from "@/lib/debug/types" @@ -26,6 +28,7 @@ import { buildAutomationStopUserActionJson, buildExchangeConfigEditUserActionJson, buildStrategyEditUserActionJson, + buildUpdateHistoricalExchangesDataUserActionJson, } from "@/lib/debug/user-action-templates" const DELETABLE_TABS = new Set(DEBUG_DELETABLE_TAB_VALUES) @@ -58,6 +61,7 @@ function DebugTabsPanelComponent({ const [activeTab, setActiveTab] = useState("automations") const [deleteMode, setDeleteMode] = useState(false) const [selectedIds, setSelectedIds] = useState>(new Set()) + const [accountsHistoryOpen, setAccountsHistoryOpen] = useState(false) const isDeletableTab = DELETABLE_TABS.has(activeTab) const canDelete = isDeletableTab && !isImportedMode @@ -189,6 +193,8 @@ function DebugTabsPanelComponent({ rows={accounts} exchangeConfigs={exchangeConfigs} accountTradings={accountTradings} + walletQueryParam={walletQueryParam} + isImportedMode={isImportedMode} onEdit={(account) => onOpenExecuteAction({ actionType: "account_edit", @@ -201,6 +207,30 @@ function DebugTabsPanelComponent({ jsonText: buildAutomationCreateUserActionJsonForAccount(account), }) } + onUpdateHistory={(account) => + onOpenExecuteAction({ + actionType: "update_historical_exchanges_data", + jsonText: buildUpdateHistoricalExchangesDataUserActionJson(account.id), + }) + } + /> + {!isImportedMode ? ( +
+ +
+ ) : null} + diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/PortfolioHistoryChart.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/PortfolioHistoryChart.tsx new file mode 100644 index 0000000000..9f44d3ccdd --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/PortfolioHistoryChart.tsx @@ -0,0 +1,101 @@ +import { Loader2 } from "lucide-react" +import { useMemo } from "react" + +import { SvgLineChart } from "@/components/Common/charts/SvgLineChart" +import { + formatPortfolioHistoryTooltip, + getPortfolioHistoryChartPoints, +} from "@/lib/debug/portfolio-history-chart" +import type { PortfolioHistoricalValuesState } from "@/lib/debug/portfolio-historical-values-types" + +type PortfolioHistoryChartProps = { + state: PortfolioHistoricalValuesState | undefined + isLoading: boolean + error?: unknown + isImportedMode?: boolean +} + +export function PortfolioHistoryChart({ + state, + isLoading, + error, + isImportedMode = false, +}: PortfolioHistoryChartProps) { + const chartData = useMemo( + () => getPortfolioHistoryChartPoints(state), + [state], + ) + const chartPoints = useMemo( + () => chartData.map((point) => ({ x: point.x, y: point.y })), + [chartData], + ) + const unit = state?.history?.unit ?? "USDT" + + if (isImportedMode) { + return ( +

+ History unavailable in imported snapshot. +

+ ) + } + + if (isLoading) { + return ( +
+ + Loading history… +
+ ) + } + + if (error) { + return ( +

+ Failed to load portfolio history. +

+ ) + } + + return ( + { + const historyValue = chartData[index]?.historyValue + if (!historyValue) { + return null + } + const tooltip = formatPortfolioHistoryTooltip(historyValue, unit) + return ( +
+
{tooltip.timestampLabel}
+
Total: {tooltip.totalLabel}
+ {tooltip.assetRows.length > 0 ? ( + + + + + + + + + + {tooltip.assetRows.map((assetRow) => ( + + + + + + ))} + +
AssetHoldings{tooltip.valueColumnLabel}
{assetRow.symbol}{assetRow.holdings}{assetRow.value}
+ ) : ( +

No holdings breakdown.

+ )} +
+ ) + }} + /> + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx new file mode 100644 index 0000000000..658bd09393 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountDetailDialog.tsx @@ -0,0 +1,95 @@ +import { useQuery } from "@tanstack/react-query" +import { useMemo } from "react" + +import type { Account, ExchangeConfig } from "@/client" +import { PortfolioHistoryChart } from "@/components/Debug/PortfolioHistoryChart" +import { CollapsibleJsonView } from "@/components/ui/collapsible-json-view" +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { getAccountHistoricalValuesQueryOptions } from "@/lib/debug/queries" +import { getNegativeHoldingsWarnings } from "@/lib/debug/portfolio-history-warnings" + +type AccountDetailDialogProps = { + account: Account | null + open: boolean + onOpenChange: (open: boolean) => void + walletQueryParam?: string + isImportedMode?: boolean + exchangeConfigs?: ExchangeConfig[] +} + +export function AccountDetailDialog({ + account, + open, + onOpenChange, + walletQueryParam, + isImportedMode = false, + exchangeConfigs = [], +}: AccountDetailDialogProps) { + const historyQuery = useQuery({ + ...getAccountHistoricalValuesQueryOptions( + account?.id, + walletQueryParam, + open && !isImportedMode, + ), + }) + + const negativeHoldingsWarnings = useMemo( + () => + getNegativeHoldingsWarnings( + historyQuery.data, + historyQuery.data?.history?.unit ?? "USDT", + account, + exchangeConfigs, + ), + [account, exchangeConfigs, historyQuery.data], + ) + + return ( + + + + {account?.name ?? "Account"} + + Portfolio history and full JSON payload + + +
+ + {negativeHoldingsWarnings.length > 0 ? ( +
+

Negative holdings detected in history

+
    + {negativeHoldingsWarnings.map((warning) => ( +
  • + {warning.symbol} has negative holdings. Add{" "} + {warning.suggestedTradeSymbol} to{" "} + historical_trade_symbols + {warning.exchangeConfigLabel + ? ` on exchange config "${warning.exchangeConfigLabel}".` + : " on the linked exchange config."} +
    • + ))} +
+
+ ) : null} + {account != null ? ( + + ) : ( +

—

+ )} +
+ + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx new file mode 100644 index 0000000000..a93aeb1a67 --- /dev/null +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/dialogs/AccountsHistoryDialog.tsx @@ -0,0 +1,139 @@ +import { useQuery } from "@tanstack/react-query" +import { useMemo, useState } from "react" + +import type { Account } from "@/client" +import { PortfolioHistoryChart } from "@/components/Debug/PortfolioHistoryChart" +import { CopyableIdCell } from "@/components/Common/Tables/CopyableIdCell" +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog" +import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" +import { getAggregatedAccountHistoricalValuesQueryOptions } from "@/lib/debug/queries" + +type AccountsHistoryDialogProps = { + accounts: Account[] + open: boolean + onOpenChange: (open: boolean) => void + walletQueryParam?: string + isImportedMode?: boolean +} + +type AccountsHistoryTabProps = { + accounts: Account[] + isSimulated: boolean + isActive: boolean + dialogOpen: boolean + walletQueryParam?: string + isImportedMode: boolean +} + +function AccountsHistoryTab({ + accounts, + isSimulated, + isActive, + dialogOpen, + walletQueryParam, + isImportedMode, +}: AccountsHistoryTabProps) { + const tabAccounts = useMemo( + () => accounts.filter((account) => account.is_simulated === isSimulated), + [accounts, isSimulated], + ) + const historyQuery = useQuery({ + ...getAggregatedAccountHistoricalValuesQueryOptions( + isSimulated, + walletQueryParam, + dialogOpen && isActive && !isImportedMode, + ), + }) + + return ( +
+ +
+

+ Accounts ({tabAccounts.length}) +

+ {tabAccounts.length === 0 ? ( +

+ No {isSimulated ? "simulated" : "real"} accounts. +

+ ) : ( +
    + {tabAccounts.map((account) => ( +
  • + {account.name} + +
    • + ))} +
+ )} +
+
+ ) +} + +export function AccountsHistoryDialog({ + accounts, + open, + onOpenChange, + walletQueryParam, + isImportedMode = false, +}: AccountsHistoryDialogProps) { + const [activeTab, setActiveTab] = useState<"real" | "simulated">("real") + + return ( + + + + Accounts history + + Aggregated portfolio value across all accounts of the selected type. + + + setActiveTab(value as "real" | "simulated")} + className="flex min-h-0 flex-1 flex-col" + > + + Real + Simulated + + + + + + + + + + + ) +} diff --git a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx index faeaf035b8..93eece2f24 100644 --- a/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx +++ b/packages/tentacles/Services/Interfaces/node_web_interface/src/components/Debug/tables/AccountsTable.tsx @@ -1,4 +1,4 @@ -import { Eye, Pencil, Play } from "lucide-react" +import { Eye, History, Pencil, Play } from "lucide-react" import { useMemo, useState } from "react" import type { @@ -12,9 +12,9 @@ import { ColumnFilterInput } from "@/components/Common/Tables/ColumnFilterInput" import { CopyableIdCell } from "@/components/Common/Tables/CopyableIdCell" import { SortableTableHead } from "@/components/Common/Tables/SortableTableHead" import { AssetsPortfolioCell } from "@/components/Debug/cells/AssetsPortfolioCell" +import { AccountDetailDialog } from "@/components/Debug/dialogs/AccountDetailDialog" import { AutomationTradingCountCell } from "@/components/Debug/cells/AutomationTradingCountCell" import { DebugStatusCell } from "@/components/Debug/cells/DebugStatusCell" -import { JsonDetailDialog } from "@/components/Debug/dialogs/JsonDetailDialog" import { Table, TableBody, @@ -47,16 +47,22 @@ type AccountsTableProps = { rows: Account[] exchangeConfigs: ExchangeConfig[] accountTradings: AccountTradingWithAccountId[] + walletQueryParam?: string + isImportedMode?: boolean onEdit?: (account: Account) => void onStartAutomation?: (account: Account) => void + onUpdateHistory?: (account: Account) => void } export function AccountsTable({ rows, exchangeConfigs, accountTradings, + walletQueryParam, + isImportedMode = false, onEdit, onStartAutomation, + onUpdateHistory, }: AccountsTableProps) { const [detail, setDetail] = useState(null) const [sort, setSort] = useState>({ @@ -91,7 +97,7 @@ export function AccountsTable({ ] const accountColumnCount = accountColumns.length + 1 - const actionsHeadClass = "w-24" + const actionsHeadClass = "w-32" if (rows.length === 0) { return ( @@ -295,6 +301,14 @@ export function AccountsTable({ > +