fix(sandbox): Daytona DinD denylist overlay, gateway retries and leaked sandboxes

CHANGELOG.md

@@ -2,6 +2,25 @@
 
 ## [Unreleased]
 
+### Fixed
+
+- **Compose tasks with a URL denylist start on Daytona.** Daytona DinD added
+  `docker-compose-net-admin.yaml` to every denylist compose command but never
+  uploaded it, so such tasks failed before their image build with `open
+  /benchflow/compose/docker-compose-net-admin.yaml: no such file or
+  directory`; FrontierPhysics `ion-trap-3d-vs-2d` and
+  `twisted-slinky-wave-coupling` could not run their oracles on Daytona.
+- **Daytona gateway errors are retried, and a failed sandbox creation leaves no
+  sandbox running.** The SDK creates a sandbox and then waits for it to start
+  inside one call; a 502 during that wait failed the attempt with no retry
+  (502/503/504 were not transient) and left the new sandbox running with
+  nothing able to delete it. 502/503/504 (and `DaytonaConnectionTimeoutError`)
+  now retry, and every sandbox a `DaytonaSandbox` creates carries a
+  `benchflow.create_attempt` label: after a failed attempt, after a retry
+  succeeds, and in `stop()`, sandboxes carrying it other than the one in use
+  are deleted. Only that label is matched, so other sandboxes on a shared API
+  key are never touched.
+
 ## 0.7.11 — 2026-10-16
 
 ### Added

src/benchflow/sandbox/daytona.py

@@ -140,6 +140,7 @@ async def __aexit__(self, *args: object) -> None:
 FileDownloadRequest: Any = None
 FileUpload: Any = None
 Image: Any = None
+ListSandboxesQuery: Any = None
 Resources: Any = None
 SessionExecuteRequest: Any = None
 SnapshotState: Any = None
@@ -157,7 +158,7 @@ def _load_daytona_sdk() -> None:
     global AsyncDaytona, AsyncSandbox
     global CreateSandboxFromImageParams, CreateSandboxFromSnapshotParams
     global DaytonaNotFoundError, FileDownloadRequest, FileUpload
-    global Image, Resources, SessionExecuteRequest, SnapshotState
+    global Image, ListSandboxesQuery, Resources, SessionExecuteRequest, SnapshotState
 
     if _DAYTONA_SDK_LOADED:
         return
@@ -180,6 +181,7 @@ def _load_daytona_sdk() -> None:
     FileDownloadRequest = _daytona.FileDownloadRequest
     FileUpload = _daytona.FileUpload
     Image = _daytona.Image
+    ListSandboxesQuery = _daytona.ListSandboxesQuery
     Resources = _daytona.Resources
     SessionExecuteRequest = _daytona.SessionExecuteRequest
     SnapshotState = _snapshot.SnapshotState
@@ -243,6 +245,17 @@ def build_sync_client(api_key: str | None = None) -> Any:
 # sandbox creation (#532). ``_stop_sandbox`` deliberately uses a smaller budget.
 _DAYTONA_RETRY_ATTEMPTS = 3
 _DAYTONA_STOP_RETRY_ATTEMPTS = 2
+# Label tying every sandbox one DaytonaSandbox creates to it, so a sandbox whose
+# creation failed after Daytona made it can still be found and deleted; see
+# ``DaytonaSandbox._create_sandbox``.
+_CREATE_ATTEMPT_LABEL = "benchflow.create_attempt"
+# Budget for that lookup and delete, which meet the same gateway errors.
+_LEFTOVER_RETRY = retry(
+    stop=stop_after_attempt(4),
+    wait=wait_exponential(multiplier=1, min=1, max=8),
+    retry=_DAYTONA_TRANSIENT_RETRY,
+    reraise=True,
+)
 
 # Shared tenacity policy for the idempotent Daytona SDK calls — session-command
 # polling and filesystem up/download. Three attempts with exponential backoff,
@@ -468,6 +481,15 @@ async def _create_sandbox(
         daytona = await self._client_manager.get_client()
         build_timeout = round(self.task_env_config.build_timeout_sec)
         hard_timeout = build_timeout + _STARTUP_HARD_TIMEOUT_BUFFER_SEC
+        # The SDK creates the sandbox, then waits for it to start inside the
+        # same call. When that wait fails (a 502 while refreshing its state,
+        # say) the sandbox exists but no handle comes back, so neither the
+        # cleanup above nor stop() can reach it. Every attempt carries this
+        # instance's label, so whatever a failed one left can be found.
+        params.labels = {
+            **(params.labels or {}),
+            _CREATE_ATTEMPT_LABEL: self._create_label(),
+        }
 
         create_task = asyncio.ensure_future(
             daytona.create(
@@ -489,6 +511,8 @@ async def _create_sandbox(
                 f"(build_timeout={build_timeout}s + buffer={_STARTUP_HARD_TIMEOUT_BUFFER_SEC}s)"
             )
             create_task.cancel()
+            self._create_failures = getattr(self, "_create_failures", 0) + 1
+            await self._delete_create_leftovers(daytona)
             raise
         except asyncio.CancelledError:
             try:
@@ -498,7 +522,72 @@ async def _create_sandbox(
                 self._on_sandbox_created()
             except (TimeoutError, asyncio.CancelledError, Exception):
                 create_task.cancel()
+                self._create_failures = getattr(self, "_create_failures", 0) + 1
+                with contextlib.suppress(Exception, asyncio.CancelledError):
+                    await asyncio.wait_for(
+                        asyncio.shield(self._delete_create_leftovers(daytona)),
+                        timeout=30,
+                    )
+            raise
+        except Exception:
+            if self._sandbox is None:
+                self._create_failures = getattr(self, "_create_failures", 0) + 1
+                await self._delete_create_leftovers(daytona)
             raise
+        if getattr(self, "_create_failures", 0):
+            # A sandbox Daytona made after an earlier attempt's error answer.
+            await self._delete_create_leftovers(daytona, keep=self._sandbox)
+
+    def _create_label(self) -> str:
+        """This instance's creation label, the same value for every attempt."""
+        label = getattr(self, "_create_label_value", None)
+        if label is None:
+            label = self._create_label_value = uuid4().hex
+        return label
+
+    async def _delete_create_leftovers(self, daytona: Any, keep: Any = None) -> None:
+        """Delete the sandboxes this instance's failed creations left, best effort.
+
+        Only sandboxes carrying this instance's own label are touched, so a
+        shared API key's other sandboxes are never at risk; ``keep`` is the
+        sandbox in use. The lookup and each delete retry gateway errors.
+        """
+        label = self._create_label()
+        keep_id = getattr(keep, "id", None)
+        try:
+            found = await self._list_create_leftovers(daytona, label)
+        except Exception as err:
+            self.logger.warning(
+                f"Could not look for a sandbox left by a failed creation: {err}"
+            )
+            return
+        for sandbox in found:
+            if sandbox.id == keep_id:
+                continue
+            try:
+                await self._delete_leftover(sandbox)
+                self.logger.warning(
+                    f"Deleted sandbox {sandbox.id} left by a failed creation"
+                )
+            except Exception as err:
+                self.logger.warning(
+                    f"Could not delete sandbox {sandbox.id} left by a failed "
+                    f"creation: {err}"
+                )
+
+    @_LEFTOVER_RETRY
+    async def _list_create_leftovers(self, daytona: Any, label: str) -> list[Any]:
+        query = ListSandboxesQuery(labels={_CREATE_ATTEMPT_LABEL: label})
+        return [
+            sandbox
+            async for sandbox in daytona.list(query)
+            if (getattr(sandbox, "labels", None) or {}).get(_CREATE_ATTEMPT_LABEL)
+            == label
+        ]
+
+    @_LEFTOVER_RETRY
+    async def _delete_leftover(self, sandbox: Any) -> None:
+        await sandbox.delete()
 
     @retry(
         stop=stop_after_attempt(_DAYTONA_STOP_RETRY_ATTEMPTS),
@@ -828,6 +917,12 @@ async def start(self, force_build: bool) -> None:
         return await self._strategy.start(force_build)
 
     async def stop(self, delete: bool) -> None:
+        if getattr(self, "_create_failures", 0) and self._client_manager:
+            # A sandbox Daytona made after its creation call failed can surface
+            # late; sweep once more before the client goes away.
+            with contextlib.suppress(Exception):
+                daytona = await self._client_manager.get_client()
+                await self._delete_create_leftovers(daytona, keep=self._sandbox)
         return await self._strategy.stop(delete)
 
     async def exec(

src/benchflow/sandbox/daytona_dind.py

@@ -25,6 +25,7 @@
 from benchflow.sandbox._compose import (
     COMPOSE_BASE_PATH,
     COMPOSE_BUILD_PATH,
+    COMPOSE_NET_ADMIN_PATH,
     COMPOSE_NO_NETWORK_PATH,
     COMPOSE_PREBUILT_PATH,
     COMPOSE_UP_RETRY_DELAYS_SEC,
@@ -473,6 +474,7 @@ async def start(self, force_build: bool) -> None:
                 COMPOSE_BUILD_PATH,
                 COMPOSE_PREBUILT_PATH,
                 COMPOSE_NO_NETWORK_PATH,
+                COMPOSE_NET_ADMIN_PATH,
             ):
                 await env._sdk_upload_file(path, f"{self._COMPOSE_DIR}/{path.name}")
 

src/benchflow/sandbox/daytona_pty.py

@@ -81,11 +81,18 @@ def _daytona_preflight() -> None:
 
 _DAYTONA_TRANSIENT_RETRY_CLASS_NAMES = frozenset(
     {
+        "DaytonaBadGatewayError",
         "DaytonaConnectionError",
+        "DaytonaConnectionTimeoutError",
         "DaytonaRateLimitError",
+        "DaytonaServiceUnavailableError",
         "DaytonaTimeoutError",
     }
 )
+# Gateway statuses Daytona's API front end returns while its backend is
+# briefly unreachable. SDKs older than the typed gateway classes above raise
+# the base ``DaytonaError`` with the status code; typed classes go by name.
+_DAYTONA_TRANSIENT_STATUS_CODES = frozenset({502, 503, 504})
 _DAYTONA_EMPTY_EXIT_CODE_MARKERS = (
     "failed to convert exit code to int",
     'strconv.Atoi: parsing "": invalid syntax',
@@ -100,6 +107,12 @@ def _is_daytona_transient_retry_error(exc: BaseException) -> bool:
         marker in str(exc) for marker in _DAYTONA_EMPTY_EXIT_CODE_MARKERS
     ):
         return True
+    if (
+        exc_type.__module__.startswith("daytona.")
+        and exc_type.__name__ == "DaytonaError"
+        and getattr(exc, "status_code", None) in _DAYTONA_TRANSIENT_STATUS_CODES
+    ):
+        return True
     return (
         exc_type.__module__.startswith("daytona.")
         and exc_type.__name__ in _DAYTONA_TRANSIENT_RETRY_CLASS_NAMES