WIP [Activity] add onboarding state

additional_tests/exchanges_tests/abstract_authenticated_exchange_tester.py

@@ -142,6 +142,7 @@ class AbstractAuthenticatedExchangeTester:
     DEFAULT_MAX_DEFAULT_ORDERS_COUNT = trading_constants.DEFAULT_MAX_DEFAULT_ORDERS_COUNT
     DEFAULT_MAX_STOP_ORDERS_COUNT = trading_constants.DEFAULT_MAX_STOP_ORDERS_COUNT
     SLEEP_SECONDS_BEFORE_CHECKING_PORTFOLIO = 0  # used to wait before fetching portfolio after creating/cancelling an order
+    ENABLE_MARKET_ORDER_CONVERSION_CHECKS = False  # set True to run after_market_order_created during market order tests
 
     # Implement all "test_[name]" methods, call super() to run the test, pass to ignore it.
     # Override the "inner_test_[name]" method to override a test content.
@@ -714,6 +715,8 @@ async def inner_test_create_and_fill_market_orders(self):
         post_buy_portfolio = {}
         try:
             self.check_created_market_order(first_market_order, size, side)
+            if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS:
+                await self.after_market_order_created(first_market_order)
             filled_order = await self.wait_for_fill(first_market_order)
             parsed_filled_order = personal_data.create_order_instance_from_raw(
                 self.exchange_manager.trader,
@@ -735,6 +738,8 @@ async def inner_test_create_and_fill_market_orders(self):
             other_side = trading_enums.TradeOrderSide.SELL if side == trading_enums.TradeOrderSide.BUY else trading_enums.TradeOrderSide.BUY
             second_market_order = await self.create_market_order(current_price, mirror_size, other_side)
             self.check_created_market_order(second_market_order, mirror_size, other_side)
+            if self.ENABLE_MARKET_ORDER_CONVERSION_CHECKS:
+                await self.after_market_order_created(second_market_order)
             await self.wait_for_fill(second_market_order)
             await self.sleep_before_checking_portfolio()
             post_sell_portfolio = await self.get_portfolio()
@@ -1702,6 +1707,9 @@ async def _get_recent_trades_until(self, validation_func, timeout):
             f"message: {message}"
         )
 
+    async def after_market_order_created(self, market_order) -> None:
+        pass
+
     async def wait_for_fill(self, order):
         def parse_is_filled(raw_order):
             return personal_data.parse_order_status(raw_order) in {trading_enums.OrderStatus.FILLED,

additional_tests/exchanges_tests/test_coinrabbit.py

@@ -56,7 +56,39 @@ class TestCoinRabbitAuthenticatedExchange(
     TOP_UP_CODE = "usdt"
     TOP_UP_NETWORK = "eth"
     TOP_UP_AMOUNT = "0"
-
+    ENABLE_MARKET_ORDER_CONVERSION_CHECKS = True
+    MARKET_FILL_TIMEOUT = 600  # 10 min provisional for first live run; tune after observing fill duration
+
+    def check_raw_closed_orders(self, closed_orders):
+        print(f"{len(closed_orders)} closed orders: {closed_orders}")
+        for raw_order in closed_orders:
+            order_info = raw_order.get("info", {})
+            assert raw_order.get("status") == "closed"
+            assert order_info.get("status", "").lower() == "closed"
+            if order_info.get("updated_at"):
+                last_update_timestamp = raw_order.get("lastUpdateTimestamp")
+                created_at_timestamp = raw_order.get("timestamp")
+                assert last_update_timestamp, "closed order with updated_at must expose lastUpdateTimestamp"
+                assert created_at_timestamp and last_update_timestamp >= created_at_timestamp
+        super().check_raw_closed_orders(closed_orders)
+
+    async def after_market_order_created(self, market_order) -> None:
+        client = self.exchange_manager.exchange.connector.client
+        open_orders = await client.fetch_open_orders(self.SYMBOL)
+        assert isinstance(open_orders, list)
+        for open_order in open_orders:
+            assert open_order["status"] == "open"
+        active_orders = await client.fetch_orders(self.SYMBOL, params={"status": "active"})
+        assert isinstance(active_orders, list)
+        for active_order in active_orders:
+            assert active_order["status"] == "open"
+            assert active_order["info"]["status"].lower() == "active"
+        created_order_id = str(market_order.exchange_order_id)
+        open_order_ids = {str(open_order.get("id")) for open_order in open_orders}
+        assert created_order_id in open_order_ids, (
+            f"market order {created_order_id} must appear in fetch_open_orders while converting"
+        )
+        print(f"{created_order_id} in {len(open_order_ids)} open_order_ids: {open_order_ids}")
 
     async def test_get_portfolio(self):
         async with self.local_exchange_manager():
@@ -115,8 +147,7 @@ async def test_cancel_uncancellable_order(self):
     async def test_create_and_cancel_limit_orders(self):
         await super().test_create_and_cancel_limit_orders()
 
-    # TODO: wait for portfolio settlement (not only order fill) before check_portfolio_changed; fetchOrder closed
-    # status from the ccxt fee heuristic does not mean balances have settled (used→free can lag).
+    # Portfolio settlement (used→free) can lag behind order closed status; sleep_before_checking_portfolio handles this.
     async def test_create_and_fill_market_orders(self):
         await super().test_create_and_fill_market_orders()
 

octobot/cli.py

@@ -57,6 +57,7 @@
     import octobot.logger as octobot_logger
     import octobot.community as octobot_community
     import octobot.community.errors
+    import octobot.community.node_journal.lifecycle as journal_startup
     import octobot.limits as limits
 except ImportError as err:
     traceback.print_exc()
@@ -478,8 +479,33 @@ def _log_startup_distribution_mode(logger, args) -> None:
     logger.info(startup_message)
 
 
+def _record_cli_startup_failure(
+    error: Exception,
+    startup_phase: str,
+    config: configuration.Configuration | None = None,
+) -> None:
+    try:
+        if config is not None:
+            distribution = configuration_manager.get_distribution(config.config)
+        elif constants.FORCED_DISTRIBUTION:
+            distribution = enums.OctoBotDistribution(constants.FORCED_DISTRIBUTION)
+        else:
+            return
+        if distribution is not enums.OctoBotDistribution.NODE:
+            return
+        journal_startup.record_node_startup_failed(
+            error,
+            startup_phase=startup_phase,
+            force_exit=True,
+            config=config,
+        )
+    except Exception:
+        pass
+
+
 def start_octobot(args, default_config_file=None):
     logger = None
+    config = None
     try:
         if args.version:
             print(constants.LONG_VERSION)
@@ -518,8 +544,6 @@ def start_octobot(args, default_config_file=None):
         if not config.is_loaded():
             raise errors.ConfigError
 
-        octobot_community.ActivityMetrics.initialize_tracker(config)
-
         # Handle utility methods before bot initializing if possible
         if args.encrypter:
             commands.exchange_keys_encrypter()
@@ -531,6 +555,7 @@ def start_octobot(args, default_config_file=None):
         community_auth = None if args.backtesting else asyncio.run(
             _get_authenticated_community_if_possible(config, logger)
         )
+        journal_startup.initialize_journal(config)
 
         # Startup order matters: sync user and tentacles/community config must run before
         # profile activation. First boot (empty user/) has no profiles until tentacles
@@ -604,20 +629,23 @@ def start_octobot(args, default_config_file=None):
         force_error_exit = False
     except errors.RemoteConfigError as err:
         logger.exception(err)
+        _record_cli_startup_failure(err, "remote_config", config)
         force_error_exit = True
 
     except errors.ConfigError as err:
         logger.error("OctoBot can't start without a valid " + common_constants.CONFIG_FILE
                      + " configuration file.\nError: " + str(err) + "\nYou can use " +
                      constants.DEFAULT_CONFIG_FILE + " as an example to fix it.")
+        _record_cli_startup_failure(err, "config", config)
         force_error_exit = True
 
-    except errors.NoProfileError:
+    except errors.NoProfileError as err:
         logger.error("Missing default profiles. OctoBot can't start without a valid default profile configuration. "
                      "Please make sure that the {config.profiles_path} "
                      f"folder is accessible. To reinstall default profiles, delete the "
                      f"'{tentacles_manager_constants.TENTACLES_PATH}' "
                      f"folder or start OctoBot with the following arguments: tentacles --install --all")
+        _record_cli_startup_failure(err, "no_profile", config)
         force_error_exit = True
 
     except ModuleNotFoundError as err:
@@ -626,18 +654,21 @@ def start_octobot(args, default_config_file=None):
                          "please use the following command:\nstart.py tentacles --install --all")
         else:
             logger.exception(err)
+        _record_cli_startup_failure(err, "module_not_found", config)
         force_error_exit = True
 
-    except errors.ConfigEvaluatorError:
+    except errors.ConfigEvaluatorError as err:
         logger.error("OctoBot can't start without a valid  configuration file.\n"
                      "This file is generated on tentacle "
                      "installation using the following command:\nstart.py tentacles --install --all")
+        _record_cli_startup_failure(err, "config_evaluator", config)
         force_error_exit = True
 
-    except errors.ConfigTradingError:
+    except errors.ConfigTradingError as err:
         logger.error("OctoBot can't start without a valid configuration file.\n"
                      "This file is generated on tentacle "
                      "installation using the following command:\nstart.py tentacles --install --all")
+        _record_cli_startup_failure(err, "config_trading", config)
         force_error_exit = True
     if force_error_exit:
         octobot_community.flush_tracker()

octobot/commands.py

@@ -38,6 +38,8 @@
 import octobot.constants as constants
 import octobot.community.tentacles_packages as community_tentacles_packages
 import octobot.configuration_manager as configuration_manager
+import octobot.enums as enums
+import octobot.community.node_journal.lifecycle as journal_startup
 
 COMMANDS_LOGGER_NAME = "Commands"
 IGNORED_COMMAND_WHEN_RESTART = ["-u", "--update"]
@@ -340,6 +342,18 @@ def run_bot(bot, logger):
     bot.task_manager.run_forever(start_bot(bot, logger))
 
 
+def _record_bot_initialize_failure(bot, error: Exception, *, catch: bool) -> None:
+    if configuration_manager.get_distribution(bot.config) is not enums.OctoBotDistribution.NODE:
+        return
+    edited_config = bot.get_edited_config(constants.CONFIG_KEY, dict_only=False)
+    journal_startup.record_node_startup_failed(
+        error,
+        startup_phase="bot_initialize",
+        force_exit=not catch,
+        config=edited_config,
+    )
+
+
 async def start_bot(bot, logger, catch=False):
     try:
         # load tentacles details
@@ -354,6 +368,7 @@ async def start_bot(bot, logger, catch=False):
 
     except Exception as e:
         logger.exception(e)
+        _record_bot_initialize_failure(bot, e, catch=catch)
         if not catch:
             raise
         stop_bot(bot)

octobot/community/__init__.py

@@ -64,9 +64,6 @@
     get_current_octobots_stats,
     can_read_metrics,
 )
-from octobot.community.activity_analysis.activity_metrics import (
-    ActivityMetrics,
-)
 from octobot.community.authentication import (
     CommunityAuthentication,
 )
@@ -123,7 +120,6 @@
     "get_community_metrics",
     "get_current_octobots_stats",
     "can_read_metrics",
-    "ActivityMetrics",
     "CommunityAuthentication",
     "CommunityTentaclesPackage",
     "CommunitySupports",