fix(launch): stop dropping the initial message when provider init outlives the client

src/cli_agent_orchestrator/cli/commands/launch.py

@@ -50,6 +50,96 @@
 )
 _FORWARDED_ENV_MAX_VALUE_BYTES = 2048
 
+# How long the CLI waits for a freshly created terminal to report ready before
+# attaching / sending MESSAGE. Also the FLOOR on the ``POST /sessions`` read
+# budget (see ``_create_session_timeout``): the two must not disagree about how
+# long provider init is allowed to take, or the create call gives up on work
+# the very next step is still willing to wait for.
+_READINESS_WAIT_TIMEOUT = 120
+
+# Slack added to the create budget on top of the two init-timeout windows.
+# Server-side, ``POST /sessions`` also spends time that NEITHER init timeout
+# bounds: pane/window creation runs BEFORE ``initialize()``
+# (``terminal_service.create_session``/``create_window``), and inside
+# ``initialize()`` there are fixed unbudgeted gaps — e.g. codex's
+# ``await asyncio.sleep(2.0)`` shell warm-up, plus send_keys latency and the
+# tail of a 1s poll interval on either side of each bounded wait. Without this,
+# the sum of those steps could exceed the budget even when the client and
+# server agree exactly on the init timeouts (a 60/20 config leaves the client
+# at 140s against a worst-case ~141s server path), reopening the same
+# silent-drop hole this constant's neighbours exist to close.
+_CREATE_OVERHEAD_MARGIN = 30
+
+
+def _effective_init_timeout(agent_profile, settings):
+    """Largest ``provider_init_timeout`` the server might apply for this launch.
+
+    Providers do NOT agree on where this value comes from, so the client cannot
+    assume either source: ``claude_code``/``antigravity_cli``/``kimi_cli`` route
+    through ``BaseProvider.get_init_timeout(profile)``, which prefers the
+    profile's own ``provider_init_timeout`` override (that override exists so a
+    containerized profile whose wrapped CLI is slow can raise its init cap
+    without touching global config), while ``codex``/``copilot_cli`` read the
+    global setting directly. Take the MAX of the two: it is the only bound
+    guaranteed to cover whichever the target provider actually uses, and being
+    generous here only ever costs an unused ceiling on a request that succeeds
+    or fails on its own long before it.
+
+    A profile that cannot be loaded (missing / malformed) falls back to the
+    global value, matching ``get_init_timeout``'s own no-profile behaviour.
+    """
+    global_timeout = int(settings["provider_init_timeout"])
+    try:
+        from cli_agent_orchestrator.utils.agent_profiles import load_agent_profile
+
+        override = load_agent_profile(agent_profile).provider_init_timeout
+    except Exception:
+        # Deliberately broad: this is a timeout hint, never a reason to fail a
+        # launch the server would have accepted. The real profile load happens
+        # server-side and reports its own errors.
+        return global_timeout
+    if override is None:
+        return global_timeout
+    return max(global_timeout, int(override))
+
+
+def _create_session_timeout(settings, agent_profile=None):
+    """Read budget for ``POST /sessions``, which initializes the provider inline.
+
+    ``POST /sessions`` runs the provider's FULL ``initialize()`` synchronously
+    server-side (``session_service.create_session`` -> ``create_terminal`` ->
+    ``await provider_instance.initialize()``), so this request's read budget has
+    to cover ``provider_init_timeout`` — NOT the generic ``mcp_request_timeout``
+    (30s) meant for ordinary tool calls.
+
+    With the 30s budget, a slow provider cold start (codex with MCP servers, on
+    a container, under a concurrent fan-out) outlived the client: ``requests``
+    raised ``ReadTimeout``, ``launch`` turned that into "Failed to connect to
+    cao-server", and the create-then-send flow below never ran — so MESSAGE was
+    silently never delivered even though the session, the terminal, and a
+    healthy idle TUI all existed server-side. Nothing retried, because from the
+    server's point of view the launch had succeeded.
+
+    Server-side, the init timeout applies TWICE per init — once to
+    ``wait_for_shell``, again to the final ``wait_until_status`` — with the
+    startup-prompt/trust-dialog handler in between, so cover the sum rather
+    than a single init timeout, plus ``_CREATE_OVERHEAD_MARGIN`` for the steps
+    neither window bounds. The init timeout is resolved per-launch via
+    ``_effective_init_timeout``, so a profile that raises its own cap widens
+    this budget with it instead of leaving the client to give up first.
+
+    ``settings.get`` for the handler timeout: a hand-edited partial
+    settings.json that omits the key must not turn a widened budget into a
+    ``KeyError`` traceback at launch.
+    """
+    init_timeout = _effective_init_timeout(agent_profile, settings)
+    return max(
+        2 * init_timeout
+        + int(settings.get("startup_prompt_handler_timeout", 20))
+        + _CREATE_OVERHEAD_MARGIN,
+        _READINESS_WAIT_TIMEOUT,
+    )
+
 
 def _parse_env_pairs(pairs):
     """Parse repeated ``KEY=VALUE`` entries into a dict, validating each.
@@ -297,8 +387,11 @@ def launch(
         # Forwarded env vars travel in the JSON body so values (which may
         # contain secrets) don't end up in cao-server's HTTP access log.
         # See issue #248.
-        request_timeout = get_server_settings()["mcp_request_timeout"]
-        post_kwargs: dict = {"params": params, "timeout": request_timeout}
+        settings = get_server_settings()
+        post_kwargs: dict = {
+            "params": params,
+            "timeout": _create_session_timeout(settings, agents),
+        }
         if forwarded_env:
             post_kwargs["json"] = {"env_vars": forwarded_env}
 
@@ -324,13 +417,14 @@ def launch(
             ready = wait_until_terminal_status(
                 terminal["id"],
                 {TerminalStatus.IDLE, TerminalStatus.COMPLETED},
-                timeout=120,
+                timeout=_READINESS_WAIT_TIMEOUT,
             )
             if not ready:
                 click.echo(
                     click.style(
-                        f"  Warning: {terminal['id']} did not reach idle within 120s — "
-                        "attaching anyway; input may be unreliable until init completes.",
+                        f"  Warning: {terminal['id']} did not reach idle within "
+                        f"{_READINESS_WAIT_TIMEOUT}s — attaching anyway; input may be "
+                        "unreliable until init completes.",
                         fg="yellow",
                     )
                 )
@@ -339,29 +433,28 @@ def launch(
             ready = wait_until_terminal_status(
                 terminal["id"],
                 {TerminalStatus.IDLE, TerminalStatus.COMPLETED},
-                timeout=120,
+                timeout=_READINESS_WAIT_TIMEOUT,
             )
             if not ready:
                 raise click.ClickException(
-                    f"Conductor {terminal['id']} did not become ready within 120s"
+                    f"Conductor {terminal['id']} did not become ready within "
+                    f"{_READINESS_WAIT_TIMEOUT}s"
                 )
-            request_timeout = get_server_settings()["mcp_request_timeout"]
             response = requests.post(
                 f"{API_BASE_URL}/terminals/{terminal['id']}/input",
                 params={"message": message},
-                timeout=request_timeout,
+                timeout=settings["mcp_request_timeout"],
             )
             response.raise_for_status()
             time.sleep(3)
             if is_async:
                 click.echo(f"Message sent to {terminal['name']}. Running in background.")
                 return
             poll_until_done(terminal["id"], timeout=300)
-            request_timeout = get_server_settings()["mcp_request_timeout"]
             output_resp = requests.get(
                 f"{API_BASE_URL}/terminals/{terminal['id']}/output",
                 params={"mode": "last"},
-                timeout=request_timeout,
+                timeout=settings["mcp_request_timeout"],
             )
             output_resp.raise_for_status()
             output = output_resp.json().get("output", "")

src/cli_agent_orchestrator/providers/codex.py

@@ -455,6 +455,16 @@ def _build_codex_command(self) -> str:
     async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
         """Dismiss startup prompts that block readiness.
 
+        Every backend call here (get_history/send_keys/send_special_key) is a
+        blocking subprocess exec, and this loop makes one per second for up to
+        ``timeout`` seconds. cao-server runs a SINGLE event loop, so leaving
+        them loop-side froze every other concurrent request — including every
+        other terminal's own init — for the duration. They are offloaded to
+        threads for the same reason claude_code's startup handler was in #451;
+        codex was the slowest remaining provider still doing this, which is why
+        a concurrent fan-out of codex launches blew the CLI's own read budget on
+        POST /sessions (see _create_session_timeout in cli/commands/launch.py).
+
         Handles two classes of blocking dialog in a single poll loop:
 
         1. Workspace trust prompt (two variants):
@@ -472,7 +482,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
         trust_dismissed = False
         update_dismissed = False
         while time.time() - start_time < timeout:
-            output = get_backend().get_history(self.session_name, self.window_name)
+            output = await asyncio.to_thread(
+                get_backend().get_history, self.session_name, self.window_name
+            )
             if not output:
                 await asyncio.sleep(1.0)
                 continue
@@ -484,7 +496,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
 
                 logger.info("Codex workspace trust prompt (v1) detected, auto-accepting")
                 status_monitor.notify_input_sent(self.terminal_id)
-                get_backend().send_special_key(self.session_name, self.window_name, "Enter")
+                await asyncio.to_thread(
+                    get_backend().send_special_key, self.session_name, self.window_name, "Enter"
+                )
                 trust_dismissed = True
                 await asyncio.sleep(1.0)
                 continue
@@ -500,7 +514,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
 
                 logger.info("Codex workspace trust prompt (v2) detected, auto-accepting")
                 status_monitor.notify_input_sent(self.terminal_id)
-                get_backend().send_special_key(self.session_name, self.window_name, "Enter")
+                await asyncio.to_thread(
+                    get_backend().send_special_key, self.session_name, self.window_name, "Enter"
+                )
                 trust_dismissed = True
                 await asyncio.sleep(1.0)
                 continue
@@ -512,10 +528,18 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
                     "Codex update-available dialog detected, selecting " "'Skip until next version'"
                 )
                 status_monitor.notify_input_sent(self.terminal_id)
-                get_backend().send_keys(self.session_name, self.window_name, "3", enter_count=0)
+                await asyncio.to_thread(
+                    get_backend().send_keys,
+                    self.session_name,
+                    self.window_name,
+                    "3",
+                    enter_count=0,
+                )
                 # TUI rendering latency: '3' highlights the menu item, Enter confirms.
                 await asyncio.sleep(0.3)
-                get_backend().send_special_key(self.session_name, self.window_name, "Enter")
+                await asyncio.to_thread(
+                    get_backend().send_special_key, self.session_name, self.window_name, "Enter"
+                )
                 update_dismissed = True
                 await asyncio.sleep(1.0)
                 continue
@@ -540,7 +564,9 @@ async def _handle_trust_prompt(self, timeout: float = 20.0) -> None:
 
         pane_tail = ""
         try:
-            output = get_backend().get_history(self.session_name, self.window_name)
+            output = await asyncio.to_thread(
+                get_backend().get_history, self.session_name, self.window_name
+            )
             if output:
                 pane_tail = "\n".join(output.splitlines()[-10:])
         except Exception:
@@ -561,8 +587,10 @@ async def initialize(self) -> bool:
 
         # Capture the shell process name before launching codex — used later to
         # detect when codex has exited and the pane is back to a bare shell.
-        self.shell_baseline = get_backend().get_pane_current_command(
-            self.session_name, self.window_name
+        # Offloaded like the rest of this method's backend calls (#451): each is
+        # a blocking subprocess exec on cao-server's single shared event loop.
+        self.shell_baseline = await asyncio.to_thread(
+            get_backend().get_pane_current_command, self.session_name, self.window_name
         )
 
         # Send a warm-up command before launching codex.
@@ -572,7 +600,9 @@ async def initialize(self) -> bool:
         # external input that must be allowed to drive PROCESSING transitions
         # past any previously-latched ready state.
         status_monitor.notify_input_sent(self.terminal_id)
-        get_backend().send_keys(self.session_name, self.window_name, "echo ready")
+        await asyncio.to_thread(
+            get_backend().send_keys, self.session_name, self.window_name, "echo ready"
+        )
         await asyncio.sleep(2.0)
 
         # Build command with flags and agent profile (developer_instructions).
@@ -582,10 +612,19 @@ async def initialize(self) -> bool:
         #   caused by the shell_snapshot subprocess inheriting stdin.
         command = self._build_codex_command()
         status_monitor.notify_input_sent(self.terminal_id)
-        get_backend().send_keys(self.session_name, self.window_name, command)
+        await asyncio.to_thread(
+            get_backend().send_keys, self.session_name, self.window_name, command
+        )
 
-        # Handle workspace trust prompt if it appears (new/untrusted directories)
-        await self._handle_trust_prompt(timeout=20.0)
+        # Handle workspace trust prompt if it appears (new/untrusted directories).
+        # Timeout comes from settings so an operator on a slow/containerized host
+        # can widen it; the hard-coded 20.0 could not be raised without a code
+        # change, and a cold codex start that renders its first frame later than
+        # that fell through to "startup prompt handler timed out" and then had to
+        # be rescued by the wait_until_status below.
+        await self._handle_trust_prompt(
+            timeout=float(get_server_settings()["startup_prompt_handler_timeout"])
+        )
 
         if not await wait_until_status(
             self.terminal_id,