Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion src/benchflow/rollout/_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,15 @@ def _computed_deadline_sec(cfg: RolloutConfig) -> float:
``timeout_sec`` overrides, user-loop rounds (each round runs one prompt
plus a soft verify), and :func:`effective_install_timeout` — the single
source of truth for the install budget — per distinct agent.

Task budgets are read after the run's ``config_override`` is merged in, as
``Rollout.setup()`` does, so a budget the run raises raises the backstop.
"""
from benchflow._utils.config_override import apply_config_override
from benchflow.agents.install import effective_install_timeout
from benchflow.task import Task

tcfg = Task(cfg.task_path).config
tcfg = apply_config_override(Task(cfg.task_path).config, cfg.config_override)
agent_default = float(cfg.timeout or tcfg.agent.timeout_sec or 900.0)
verifier_sec = float(tcfg.verifier.timeout_sec or 900.0)
build_sec = float(tcfg.sandbox.build_timeout_sec or 600.0)
Expand Down
29 changes: 29 additions & 0 deletions tests/test_rollout_hard_deadline.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,35 @@ def test_caller_timeout_override_dominates_task_budget(self, tmp_path, monkeypat
assert base is not None and raised is not None
assert raised >= base + 7200 - 60

@pytest.mark.parametrize(
("overlay", "added_sec"),
[
({"agent": {"timeout_sec": 7200}}, 7200 - 60),
({"sandbox": {"build_timeout_sec": 1800}}, 1800 - 600),
],
ids=["agent-timeout", "build-timeout"],
)
def test_config_override_budgets_dominate_task_budget(
self, tmp_path, monkeypatch, overlay, added_sec
):
"""A run's ``config_override`` (``--config-override``) is merged into
the task config in ``Rollout.setup()``, so the raised budget is the one
the phase enforces; the backstop must be derived from it as well.

Guards the fix from PR #1161 for #1134: the deadline from #949 re-read the task
budgets without the override introduced in 0a645223, and killed runs
that were still inside their raised budget.
"""
monkeypatch.delenv("BENCHFLOW_ROLLOUT_HARD_DEADLINE", raising=False)
_write_task(tmp_path / "t")
base = hard_deadline_sec(RolloutConfig(task_path=tmp_path / "t"))
raised = hard_deadline_sec(
RolloutConfig(task_path=tmp_path / "t", config_override=overlay)
)
assert base is not None
# Exact, so the unreadable-task fallback cannot pass for the result.
assert raised == base + added_sec

def test_user_loop_rounds_dominate(self, tmp_path, monkeypatch):
"""A user-loop run legitimately spends max_user_rounds x (agent +
soft-verify); the backstop must cover it, not fire mid-loop."""
Expand Down
Loading