Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/benchflow/rollout/_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,15 @@ def _computed_deadline_sec(cfg: RolloutConfig) -> float:
``timeout_sec`` overrides, user-loop rounds (each round runs one prompt
plus a soft verify), and :func:`effective_install_timeout` — the single
source of truth for the install budget — per distinct agent.

Task budgets are read after the run's ``config_override`` is merged in, as
``Rollout.setup()`` does, so a budget the run raises raises the backstop.
"""
from benchflow._utils.config_override import apply_config_override
from benchflow.agents.install import effective_install_timeout
from benchflow.task import Task

tcfg = Task(cfg.task_path).config
tcfg = apply_config_override(Task(cfg.task_path).config, cfg.config_override)
agent_default = float(cfg.timeout or tcfg.agent.timeout_sec or 900.0)
verifier_sec = float(tcfg.verifier.timeout_sec or 900.0)
build_sec = float(tcfg.sandbox.build_timeout_sec or 600.0)
Expand Down
29 changes: 29 additions & 0 deletions tests/test_rollout_hard_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,35 @@ def test_caller_timeout_override_dominates_task_budget(self, tmp_path, monkeypat
assert base is not None and raised is not None
assert raised >= base + 7200 - 60

@pytest.mark.parametrize(
("overlay", "added_sec"),
[
({"agent": {"timeout_sec": 7200}}, 7200 - 60),
({"sandbox": {"build_timeout_sec": 1800}}, 1800 - 600),
],
ids=["agent-timeout", "build-timeout"],
)
def test_config_override_budgets_dominate_task_budget(
self, tmp_path, monkeypatch, overlay, added_sec
):
"""A run's ``config_override`` (``--config-override``) is merged into
the task config in ``Rollout.setup()``, so the raised budget is the one
the phase enforces; the backstop must be derived from it as well.

Guards the fix for #1134: the deadline from #949 re-read the task
budgets without the override introduced in 0a645223, and killed runs
that were still inside their raised budget.
"""
monkeypatch.delenv("BENCHFLOW_ROLLOUT_HARD_DEADLINE", raising=False)
_write_task(tmp_path / "t")
base = hard_deadline_sec(RolloutConfig(task_path=tmp_path / "t"))
raised = hard_deadline_sec(
RolloutConfig(task_path=tmp_path / "t", config_override=overlay)
)
assert base is not None
# Exact, so the unreadable-task fallback cannot pass for the result.
assert raised == base + added_sec

def test_user_loop_rounds_dominate(self, tmp_path, monkeypatch):
"""A user-loop run legitimately spends max_user_rounds x (agent +
soft-verify); the backstop must cover it, not fire mid-loop."""
Expand Down
Loading