From d40f78af971b928bc2be4460d4c931e017f55e8f Mon Sep 17 00:00:00 2001 From: Bingran You Date: Mon, 14 Sep 2026 09:49:40 -0700 Subject: [PATCH 1/3] fix(sandbox): upload denylist overlay in Daytona DinD --- src/benchflow/sandbox/daytona_dind.py | 2 + tests/test_daytona_dind_compose_up.py | 90 +++++++++++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/src/benchflow/sandbox/daytona_dind.py b/src/benchflow/sandbox/daytona_dind.py index fb840a33c..00ffb5009 100644 --- a/src/benchflow/sandbox/daytona_dind.py +++ b/src/benchflow/sandbox/daytona_dind.py @@ -25,6 +25,7 @@ from benchflow.sandbox._compose import ( COMPOSE_BASE_PATH, COMPOSE_BUILD_PATH, + COMPOSE_NET_ADMIN_PATH, COMPOSE_NO_NETWORK_PATH, COMPOSE_PREBUILT_PATH, COMPOSE_UP_RETRY_DELAYS_SEC, @@ -473,6 +474,7 @@ async def start(self, force_build: bool) -> None: COMPOSE_BUILD_PATH, COMPOSE_PREBUILT_PATH, COMPOSE_NO_NETWORK_PATH, + COMPOSE_NET_ADMIN_PATH, ): await env._sdk_upload_file(path, f"{self._COMPOSE_DIR}/{path.name}") diff --git a/tests/test_daytona_dind_compose_up.py b/tests/test_daytona_dind_compose_up.py index 96c92e57b..49a810378 100644 --- a/tests/test_daytona_dind_compose_up.py +++ b/tests/test_daytona_dind_compose_up.py @@ -12,6 +12,7 @@ import logging import subprocess from types import SimpleNamespace +from unittest.mock import AsyncMock import pytest @@ -225,6 +226,95 @@ async def vm_exec(command, cwd=None, env=None, timeout_sec=None): await strategy._run_pre_compose_hook() +@pytest.mark.asyncio +async def test_denylist_build_uploads_every_referenced_compose_file( + monkeypatch, tmp_path +): + """Guards the DinD overlay fix against the regression introduced by f396c355.""" + from benchflow.sandbox import daytona as daytona_module + + class FakeClientManager: + @classmethod + async def get_instance(cls): + return object() + + monkeypatch.setattr(daytona_module, "DaytonaClientManager", FakeClientManager) + monkeypatch.setattr( + daytona_module, + "Resources", + lambda **kwargs: SimpleNamespace(**kwargs), + ) + monkeypatch.setattr( + daytona_module, + "Image", + SimpleNamespace(base=lambda image: SimpleNamespace(image=image)), + ) + monkeypatch.setattr( + daytona_module, + "CreateSandboxFromImageParams", + lambda **kwargs: SimpleNamespace(**kwargs), + ) + + available_paths: set[str] = set() + + async def upload_file(_source, destination): + available_paths.add(destination) + + async def upload_dir(_source, destination): + available_paths.add(f"{destination}/docker-compose.yaml") + + task_config = SimpleNamespace( + allow_internet=True, + build_timeout_sec=1200, + cpus=4, + docker_image=None, + memory_mb=8192, + network_mode="denylist", + storage_mb=10240, + ) + env = SimpleNamespace( + _auto_delete_interval=0, + _auto_stop_interval=0, + _create_sandbox=AsyncMock(), + _kwargs={}, + _sdk_upload_dir=upload_dir, + _sdk_upload_file=upload_file, + environment_dir=tmp_path, + environment_name="task", + logger=logging.getLogger("test.daytona.dind.overlays"), + task_env_config=task_config, + ) + strategy = _DaytonaDinD.__new__(_DaytonaDinD) + strategy._env = env + strategy._vm_exec = AsyncMock( + return_value=ExecResult(stdout="", stderr="", return_code=0) + ) + strategy._wait_for_docker_daemon = AsyncMock() + strategy._run_pre_compose_hook = AsyncMock() + strategy._compose_up_with_retry = AsyncMock() + strategy._wait_for_main_container = AsyncMock() + + referenced_at_build: list[str] = [] + + async def compose_exec(subcommand, timeout_sec=None): + del timeout_sec + if subcommand == ["build"]: + flags = strategy._compose_file_flags() + referenced_at_build.extend( + flags[index + 1] for index, item in enumerate(flags) if item == "-f" + ) + assert set(referenced_at_build) <= available_paths + return ExecResult(stdout="", stderr="", return_code=0) + + strategy._compose_exec = compose_exec + + await strategy.start(force_build=False) + + assert f"{strategy._COMPOSE_DIR}/docker-compose-net-admin.yaml" in ( + referenced_at_build + ) + + class AsyncNoop: """A no-op async callable to replace ``asyncio.sleep`` in retry tests.""" From 7bd54896dfd1a61ae4b85e5ba279f33d35d3d882 Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 16 Oct 2026 15:31:56 -0700 Subject: [PATCH 2/3] test(sandbox): stub the DinD daemon guard in the overlay upload test main added _guard_daemon_from_containers to the DinD start path after this test was written; its fake VM exec does not answer the guard, so startup was refused before the uploads were checked. The guard has its own tests in test_sdk_lockdown.py. --- tests/test_daytona_dind_compose_up.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_daytona_dind_compose_up.py b/tests/test_daytona_dind_compose_up.py index 49a810378..8675b9468 100644 --- a/tests/test_daytona_dind_compose_up.py +++ b/tests/test_daytona_dind_compose_up.py @@ -290,6 +290,8 @@ async def upload_dir(_source, destination): return_value=ExecResult(stdout="", stderr="", return_code=0) ) strategy._wait_for_docker_daemon = AsyncMock() + # The daemon guard has its own tests in test_sdk_lockdown.py. + strategy._guard_daemon_from_containers = AsyncMock() strategy._run_pre_compose_hook = AsyncMock() strategy._compose_up_with_retry = AsyncMock() strategy._wait_for_main_container = AsyncMock() From c5e673a342431a330bbe42ddb018a17ef260f43c Mon Sep 17 00:00:00 2001 From: Bingran You Date: Fri, 16 Oct 2026 16:30:47 -0700 Subject: [PATCH 3/3] fix(sandbox): retry Daytona gateway errors and delete sandboxes a failed create left The Daytona SDK creates a sandbox and then waits for it to start inside one create() call. A 502 while refreshing its state raised out of that wait, so benchflow got no handle: the sandbox kept running with nothing able to delete it, and the attempt was not retried because 502/503/504 were not transient (DaytonaBadGatewayError / DaytonaServiceUnavailableError in newer SDKs, a status-coded base DaytonaError in older ones). Treat gateway statuses (and DaytonaConnectionTimeoutError) as transient; the status-code fallback applies only to the base class. Every sandbox one DaytonaSandbox creates carries a benchflow.create_attempt label; the sandboxes carrying it, other than the one in use, are deleted after a failed or timed-out attempt (shielded and bounded when cancelled), after a retry succeeds (a sandbox Daytona committed after its error answer), and in stop(). The lookup and deletes retry gateway errors; only that exact label is matched, client-side as well as in the query, so other sandboxes on a shared API key are never touched. Adds CHANGELOG entries for this and the DinD overlay fix. --- CHANGELOG.md | 19 ++ src/benchflow/sandbox/daytona.py | 97 ++++++++- src/benchflow/sandbox/daytona_pty.py | 13 ++ tests/test_daytona_create_cleanup.py | 296 +++++++++++++++++++++++++++ 4 files changed, 424 insertions(+), 1 deletion(-) create mode 100644 tests/test_daytona_create_cleanup.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 32ef41144..50dc5475e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,25 @@ ## [Unreleased] +### Fixed + +- **Compose tasks with a URL denylist start on Daytona.** Daytona DinD added + `docker-compose-net-admin.yaml` to every denylist compose command but never + uploaded it, so such tasks failed before their image build with `open + /benchflow/compose/docker-compose-net-admin.yaml: no such file or + directory`; FrontierPhysics `ion-trap-3d-vs-2d` and + `twisted-slinky-wave-coupling` could not run their oracles on Daytona. +- **Daytona gateway errors are retried, and a failed sandbox creation leaves no + sandbox running.** The SDK creates a sandbox and then waits for it to start + inside one call; a 502 during that wait failed the attempt with no retry + (502/503/504 were not transient) and left the new sandbox running with + nothing able to delete it. 502/503/504 (and `DaytonaConnectionTimeoutError`) + now retry, and every sandbox a `DaytonaSandbox` creates carries a + `benchflow.create_attempt` label: after a failed attempt, after a retry + succeeds, and in `stop()`, sandboxes carrying it other than the one in use + are deleted. Only that label is matched, so other sandboxes on a shared API + key are never touched. + ## 0.7.11 — 2026-10-16 ### Added diff --git a/src/benchflow/sandbox/daytona.py b/src/benchflow/sandbox/daytona.py index b91aa4076..2d0dfc1a8 100644 --- a/src/benchflow/sandbox/daytona.py +++ b/src/benchflow/sandbox/daytona.py @@ -140,6 +140,7 @@ async def __aexit__(self, *args: object) -> None: FileDownloadRequest: Any = None FileUpload: Any = None Image: Any = None +ListSandboxesQuery: Any = None Resources: Any = None SessionExecuteRequest: Any = None SnapshotState: Any = None @@ -157,7 +158,7 @@ def _load_daytona_sdk() -> None: global AsyncDaytona, AsyncSandbox global CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams global DaytonaNotFoundError, FileDownloadRequest, FileUpload - global Image, Resources, SessionExecuteRequest, SnapshotState + global Image, ListSandboxesQuery, Resources, SessionExecuteRequest, SnapshotState if _DAYTONA_SDK_LOADED: return @@ -180,6 +181,7 @@ def _load_daytona_sdk() -> None: FileDownloadRequest = _daytona.FileDownloadRequest FileUpload = _daytona.FileUpload Image = _daytona.Image + ListSandboxesQuery = _daytona.ListSandboxesQuery Resources = _daytona.Resources SessionExecuteRequest = _daytona.SessionExecuteRequest SnapshotState = _snapshot.SnapshotState @@ -243,6 +245,17 @@ def build_sync_client(api_key: str | None = None) -> Any: # sandbox creation (#532). ``_stop_sandbox`` deliberately uses a smaller budget. _DAYTONA_RETRY_ATTEMPTS = 3 _DAYTONA_STOP_RETRY_ATTEMPTS = 2 +# Label tying every sandbox one DaytonaSandbox creates to it, so a sandbox whose +# creation failed after Daytona made it can still be found and deleted; see +# ``DaytonaSandbox._create_sandbox``. +_CREATE_ATTEMPT_LABEL = "benchflow.create_attempt" +# Budget for that lookup and delete, which meet the same gateway errors. +_LEFTOVER_RETRY = retry( + stop=stop_after_attempt(4), + wait=wait_exponential(multiplier=1, min=1, max=8), + retry=_DAYTONA_TRANSIENT_RETRY, + reraise=True, +) # Shared tenacity policy for the idempotent Daytona SDK calls — session-command # polling and filesystem up/download. Three attempts with exponential backoff, @@ -468,6 +481,15 @@ async def _create_sandbox( daytona = await self._client_manager.get_client() build_timeout = round(self.task_env_config.build_timeout_sec) hard_timeout = build_timeout + _STARTUP_HARD_TIMEOUT_BUFFER_SEC + # The SDK creates the sandbox, then waits for it to start inside the + # same call. When that wait fails (a 502 while refreshing its state, + # say) the sandbox exists but no handle comes back, so neither the + # cleanup above nor stop() can reach it. Every attempt carries this + # instance's label, so whatever a failed one left can be found. + params.labels = { + **(params.labels or {}), + _CREATE_ATTEMPT_LABEL: self._create_label(), + } create_task = asyncio.ensure_future( daytona.create( @@ -489,6 +511,8 @@ async def _create_sandbox( f"(build_timeout={build_timeout}s + buffer={_STARTUP_HARD_TIMEOUT_BUFFER_SEC}s)" ) create_task.cancel() + self._create_failures = getattr(self, "_create_failures", 0) + 1 + await self._delete_create_leftovers(daytona) raise except asyncio.CancelledError: try: @@ -498,7 +522,72 @@ async def _create_sandbox( self._on_sandbox_created() except (TimeoutError, asyncio.CancelledError, Exception): create_task.cancel() + self._create_failures = getattr(self, "_create_failures", 0) + 1 + with contextlib.suppress(Exception, asyncio.CancelledError): + await asyncio.wait_for( + asyncio.shield(self._delete_create_leftovers(daytona)), + timeout=30, + ) + raise + except Exception: + if self._sandbox is None: + self._create_failures = getattr(self, "_create_failures", 0) + 1 + await self._delete_create_leftovers(daytona) raise + if getattr(self, "_create_failures", 0): + # A sandbox Daytona made after an earlier attempt's error answer. + await self._delete_create_leftovers(daytona, keep=self._sandbox) + + def _create_label(self) -> str: + """This instance's creation label, the same value for every attempt.""" + label = getattr(self, "_create_label_value", None) + if label is None: + label = self._create_label_value = uuid4().hex + return label + + async def _delete_create_leftovers(self, daytona: Any, keep: Any = None) -> None: + """Delete the sandboxes this instance's failed creations left, best effort. + + Only sandboxes carrying this instance's own label are touched, so a + shared API key's other sandboxes are never at risk; ``keep`` is the + sandbox in use. The lookup and each delete retry gateway errors. + """ + label = self._create_label() + keep_id = getattr(keep, "id", None) + try: + found = await self._list_create_leftovers(daytona, label) + except Exception as err: + self.logger.warning( + f"Could not look for a sandbox left by a failed creation: {err}" + ) + return + for sandbox in found: + if sandbox.id == keep_id: + continue + try: + await self._delete_leftover(sandbox) + self.logger.warning( + f"Deleted sandbox {sandbox.id} left by a failed creation" + ) + except Exception as err: + self.logger.warning( + f"Could not delete sandbox {sandbox.id} left by a failed " + f"creation: {err}" + ) + + @_LEFTOVER_RETRY + async def _list_create_leftovers(self, daytona: Any, label: str) -> list[Any]: + query = ListSandboxesQuery(labels={_CREATE_ATTEMPT_LABEL: label}) + return [ + sandbox + async for sandbox in daytona.list(query) + if (getattr(sandbox, "labels", None) or {}).get(_CREATE_ATTEMPT_LABEL) + == label + ] + + @_LEFTOVER_RETRY + async def _delete_leftover(self, sandbox: Any) -> None: + await sandbox.delete() @retry( stop=stop_after_attempt(_DAYTONA_STOP_RETRY_ATTEMPTS), @@ -828,6 +917,12 @@ async def start(self, force_build: bool) -> None: return await self._strategy.start(force_build) async def stop(self, delete: bool) -> None: + if getattr(self, "_create_failures", 0) and self._client_manager: + # A sandbox Daytona made after its creation call failed can surface + # late; sweep once more before the client goes away. + with contextlib.suppress(Exception): + daytona = await self._client_manager.get_client() + await self._delete_create_leftovers(daytona, keep=self._sandbox) return await self._strategy.stop(delete) async def exec( diff --git a/src/benchflow/sandbox/daytona_pty.py b/src/benchflow/sandbox/daytona_pty.py index eaf0c5245..255a067cc 100644 --- a/src/benchflow/sandbox/daytona_pty.py +++ b/src/benchflow/sandbox/daytona_pty.py @@ -81,11 +81,18 @@ def _daytona_preflight() -> None: _DAYTONA_TRANSIENT_RETRY_CLASS_NAMES = frozenset( { + "DaytonaBadGatewayError", "DaytonaConnectionError", + "DaytonaConnectionTimeoutError", "DaytonaRateLimitError", + "DaytonaServiceUnavailableError", "DaytonaTimeoutError", } ) +# Gateway statuses Daytona's API front end returns while its backend is +# briefly unreachable. SDKs older than the typed gateway classes above raise +# the base ``DaytonaError`` with the status code; typed classes go by name. +_DAYTONA_TRANSIENT_STATUS_CODES = frozenset({502, 503, 504}) _DAYTONA_EMPTY_EXIT_CODE_MARKERS = ( "failed to convert exit code to int", 'strconv.Atoi: parsing "": invalid syntax', @@ -100,6 +107,12 @@ def _is_daytona_transient_retry_error(exc: BaseException) -> bool: marker in str(exc) for marker in _DAYTONA_EMPTY_EXIT_CODE_MARKERS ): return True + if ( + exc_type.__module__.startswith("daytona.") + and exc_type.__name__ == "DaytonaError" + and getattr(exc, "status_code", None) in _DAYTONA_TRANSIENT_STATUS_CODES + ): + return True return ( exc_type.__module__.startswith("daytona.") and exc_type.__name__ in _DAYTONA_TRANSIENT_RETRY_CLASS_NAMES diff --git a/tests/test_daytona_create_cleanup.py b/tests/test_daytona_create_cleanup.py new file mode 100644 index 000000000..5f131ae12 --- /dev/null +++ b/tests/test_daytona_create_cleanup.py @@ -0,0 +1,296 @@ +"""Daytona sandbox creation: gateway errors retry, and failed attempts leave no sandbox behind. + +Found in a FrontierPhysics oracle sweep: a 502 while the SDK waited for a new +sandbox to start failed the attempt without a retry, and the sandbox it had +already created kept running with nothing able to delete it. +""" + +from __future__ import annotations + +import asyncio +import logging +from types import SimpleNamespace + +import pytest +from tenacity import wait_none + +import benchflow.sandbox.daytona as daytona_mod +from benchflow.sandbox.daytona import DaytonaSandbox +from benchflow.sandbox.daytona_pty import _is_daytona_transient_retry_error + +LABEL = daytona_mod._CREATE_ATTEMPT_LABEL + + +def _daytona_error(name: str, **attrs: object) -> Exception: + cls = type(name, (Exception,), {"__module__": "daytona.common.errors"}) + error = cls(f"{name} from the Daytona SDK") + for key, value in attrs.items(): + setattr(error, key, value) + return error + + +@pytest.mark.parametrize( + "error", + [ + _daytona_error("DaytonaBadGatewayError", status_code=502), + _daytona_error("DaytonaServiceUnavailableError", status_code=503), + _daytona_error("DaytonaConnectionTimeoutError"), + # SDKs before the typed gateway classes raise the base class with the status. + _daytona_error("DaytonaError", status_code=502), + _daytona_error("DaytonaError", status_code=503), + _daytona_error("DaytonaError", status_code=504), + ], +) +def test_gateway_errors_are_transient(error: Exception) -> None: + assert _is_daytona_transient_retry_error(error) + + +@pytest.mark.parametrize( + "error", + [ + _daytona_error("DaytonaError", status_code=400), + _daytona_error("DaytonaError", status_code=500), + _daytona_error("DaytonaError"), + # A typed class goes by its name, whatever status it carries. + _daytona_error("DaytonaAuthenticationError", status_code=502), + RuntimeError("502 in a message from elsewhere"), + type("Other", (Exception,), {"status_code": 502})("not from the SDK"), + ], +) +def test_other_errors_stay_permanent(error: Exception) -> None: + assert not _is_daytona_transient_retry_error(error) + + +@pytest.mark.parametrize( + ("status", "transient"), + [(502, True), (503, True), (504, True), (401, False), (500, False)], +) +def test_installed_sdk_status_mapping(status: int, transient: bool) -> None: + """The installed SDK's own error for each status, not a forged class.""" + errors = pytest.importorskip("daytona.common.errors") + error = errors.create_daytona_error("gateway answer", status_code=status) + assert _is_daytona_transient_retry_error(error) is transient + + +class _FakeSandbox: + def __init__(self, sandbox_id: str, labels: dict[str, str], registry: dict) -> None: + self.id = sandbox_id + self.labels = dict(labels) + self._registry = registry + + async def delete(self) -> None: + self._registry.pop(self.id, None) + + +class _FakeDaytona: + """Daytona as seen through the SDK: create() may make the sandbox and still raise. + + ``plan`` holds one entry per create() call: "ok", an exception raised after + the sandbox exists, ("late", exc) to raise first and let the sandbox appear + after the next list(), or "hang" to make it and never return. + """ + + def __init__(self, plan: list, registry: dict) -> None: + self.plan = list(plan) + self.registry = registry + self.creates: list[dict[str, str]] = [] + self.list_queries: list[dict[str, str]] = [] + self.list_failures: list[Exception] = [] + self.honour_filter = True + self._late: list[_FakeSandbox] = [] + + async def create(self, params, timeout): + del timeout + step = self.plan.pop(0) + self.creates.append(dict(params.labels)) + sandbox = _FakeSandbox(f"sb-{len(self.creates)}", params.labels, self.registry) + if isinstance(step, tuple) and step[0] == "late": + self._late.append(sandbox) + raise step[1] + self.registry[sandbox.id] = sandbox + if step == "hang": + await asyncio.sleep(3600) + if isinstance(step, Exception): + raise step + return sandbox + + async def list(self, query): + self.list_queries.append(dict(query.labels)) + if self.list_failures: + raise self.list_failures.pop(0) + for sandbox in list(self.registry.values()): + # The server honours the label filter (unless a test says otherwise). + if not self.honour_filter or all( + sandbox.labels.get(k) == v for k, v in query.labels.items() + ): + yield sandbox + for sandbox in self._late: + self.registry[sandbox.id] = sandbox + self._late.clear() + + +@pytest.fixture +def no_backoff(monkeypatch): + monkeypatch.setattr( + daytona_mod, "ListSandboxesQuery", lambda labels: SimpleNamespace(labels=labels) + ) + for fn in ( + DaytonaSandbox._create_sandbox, + DaytonaSandbox._list_create_leftovers, + DaytonaSandbox._delete_leftover, + ): + monkeypatch.setattr(fn.retry, "wait", wait_none()) + + +def _sandbox_under_test(daytona: _FakeDaytona) -> DaytonaSandbox: + env = DaytonaSandbox.__new__(DaytonaSandbox) + + class _Manager: + async def get_client(self): + return daytona + + env._client_manager = _Manager() + env._sandbox = None + env.task_env_config = SimpleNamespace(build_timeout_sec=60) + env.logger = logging.getLogger("test.daytona.create") + env._on_sandbox_created = lambda: None + return env + + +def _params() -> SimpleNamespace: + return SimpleNamespace(labels={"benchflow.managed": "1"}) + + +@pytest.mark.asyncio +async def test_failed_creation_deletes_only_its_own_leftover(no_backoff) -> None: + registry: dict = {} + registry["other"] = _FakeSandbox("other", {LABEL: "another-run"}, registry) + daytona = _FakeDaytona( + [RuntimeError("Failure during waiting for sandbox to start")], registry + ) + env = _sandbox_under_test(daytona) + + with pytest.raises(RuntimeError, match="Failure during waiting"): + await env._create_sandbox(_params()) + + assert list(registry) == ["other"] + assert daytona.list_queries == [{LABEL: daytona.creates[0][LABEL]}] + + +@pytest.mark.asyncio +async def test_a_server_ignoring_the_filter_still_spares_other_sandboxes( + no_backoff, +) -> None: + registry: dict = {} + registry["unlabelled"] = _FakeSandbox( + "unlabelled", {"benchflow.managed": "1"}, registry + ) + registry["other"] = _FakeSandbox("other", {LABEL: "another-run"}, registry) + daytona = _FakeDaytona( + [RuntimeError("Failure during waiting for sandbox to start")], registry + ) + daytona.honour_filter = False + env = _sandbox_under_test(daytona) + + with pytest.raises(RuntimeError, match="Failure during waiting"): + await env._create_sandbox(_params()) + + assert sorted(registry) == ["other", "unlabelled"] + + +@pytest.mark.asyncio +async def test_gateway_error_is_retried_after_the_leftover_is_deleted( + no_backoff, +) -> None: + registry: dict = {} + daytona = _FakeDaytona( + [_daytona_error("DaytonaError", status_code=502), "ok"], registry + ) + env = _sandbox_under_test(daytona) + + await env._create_sandbox(_params()) + + assert len(daytona.creates) == 2 + assert daytona.creates[0][LABEL] == daytona.creates[1][LABEL] + assert list(registry) == ["sb-2"] + assert env._sandbox is registry["sb-2"] + + +@pytest.mark.asyncio +async def test_the_lookup_retries_gateway_errors_too(no_backoff) -> None: + registry: dict = {} + daytona = _FakeDaytona([RuntimeError("start-wait failed")], registry) + daytona.list_failures = [_daytona_error("DaytonaBadGatewayError", status_code=502)] + env = _sandbox_under_test(daytona) + + with pytest.raises(RuntimeError, match="start-wait failed"): + await env._create_sandbox(_params()) + + assert registry == {} + assert len(daytona.list_queries) == 2 + + +@pytest.mark.asyncio +async def test_a_sandbox_that_appears_late_is_deleted_after_the_retry( + no_backoff, +) -> None: + registry: dict = {} + late = ("late", _daytona_error("DaytonaError", status_code=504)) + daytona = _FakeDaytona([late, "ok"], registry) + env = _sandbox_under_test(daytona) + + await env._create_sandbox(_params()) + + assert list(registry) == ["sb-2"] + assert env._sandbox is registry["sb-2"] + + +@pytest.mark.asyncio +async def test_a_timed_out_creation_leaves_nothing(no_backoff, monkeypatch) -> None: + monkeypatch.setattr(daytona_mod, "_STARTUP_HARD_TIMEOUT_BUFFER_SEC", 0.05) + registry: dict = {} + daytona = _FakeDaytona(["hang", "hang", "hang"], registry) + env = _sandbox_under_test(daytona) + env.task_env_config = SimpleNamespace(build_timeout_sec=0) + + with pytest.raises(TimeoutError): + await env._create_sandbox(_params()) + + assert registry == {} + + +@pytest.mark.asyncio +async def test_stop_sweeps_what_a_failed_lookup_missed(no_backoff) -> None: + registry: dict = {} + daytona = _FakeDaytona([RuntimeError("start-wait failed")], registry) + daytona.list_failures = [RuntimeError("lookup refused")] + env = _sandbox_under_test(daytona) + stopped = [] + + async def strategy_stop(delete): + stopped.append(delete) + + env._strategy = SimpleNamespace(stop=strategy_stop) + + with pytest.raises(RuntimeError, match="start-wait failed"): + await env._create_sandbox(_params()) + assert list(registry) == ["sb-1"] + + await env.stop(delete=True) + + assert registry == {} + assert stopped == [True] + + +@pytest.mark.asyncio +async def test_successful_creation_looks_nothing_up(no_backoff) -> None: + registry: dict = {} + daytona = _FakeDaytona(["ok"], registry) + env = _sandbox_under_test(daytona) + + await env._create_sandbox(_params()) + + assert env._sandbox is registry["sb-1"] + assert env._sandbox.labels["benchflow.managed"] == "1" + assert env._sandbox.labels[LABEL] + assert daytona.list_queries == []