diff --git a/src/benchflow/rollout/_deadline.py b/src/benchflow/rollout/_deadline.py index 397646e08..d6a8d58af 100644 --- a/src/benchflow/rollout/_deadline.py +++ b/src/benchflow/rollout/_deadline.py @@ -80,11 +80,15 @@ def _computed_deadline_sec(cfg: RolloutConfig) -> float: ``timeout_sec`` overrides, user-loop rounds (each round runs one prompt plus a soft verify), and :func:`effective_install_timeout` — the single source of truth for the install budget — per distinct agent. + + Task budgets are read after the run's ``config_override`` is merged in, as + ``Rollout.setup()`` does, so a budget the run raises raises the backstop. """ + from benchflow._utils.config_override import apply_config_override from benchflow.agents.install import effective_install_timeout from benchflow.task import Task - tcfg = Task(cfg.task_path).config + tcfg = apply_config_override(Task(cfg.task_path).config, cfg.config_override) agent_default = float(cfg.timeout or tcfg.agent.timeout_sec or 900.0) verifier_sec = float(tcfg.verifier.timeout_sec or 900.0) build_sec = float(tcfg.sandbox.build_timeout_sec or 600.0) diff --git a/tests/test_rollout_hard_deadline.py b/tests/test_rollout_hard_deadline.py index 9ef1c3358..894688256 100644 --- a/tests/test_rollout_hard_deadline.py +++ b/tests/test_rollout_hard_deadline.py @@ -82,6 +82,35 @@ def test_caller_timeout_override_dominates_task_budget(self, tmp_path, monkeypat assert base is not None and raised is not None assert raised >= base + 7200 - 60 + @pytest.mark.parametrize( + ("overlay", "added_sec"), + [ + ({"agent": {"timeout_sec": 7200}}, 7200 - 60), + ({"sandbox": {"build_timeout_sec": 1800}}, 1800 - 600), + ], + ids=["agent-timeout", "build-timeout"], + ) + def test_config_override_budgets_dominate_task_budget( + self, tmp_path, monkeypatch, overlay, added_sec + ): + """A run's ``config_override`` (``--config-override``) is merged into + the task config in ``Rollout.setup()``, so the raised budget is the one + the phase enforces; the backstop must be derived from it as well. + + Guards the fix from PR #1161 for #1134: the deadline from #949 re-read the task + budgets without the override introduced in 0a645223, and killed runs + that were still inside their raised budget. + """ + monkeypatch.delenv("BENCHFLOW_ROLLOUT_HARD_DEADLINE", raising=False) + _write_task(tmp_path / "t") + base = hard_deadline_sec(RolloutConfig(task_path=tmp_path / "t")) + raised = hard_deadline_sec( + RolloutConfig(task_path=tmp_path / "t", config_override=overlay) + ) + assert base is not None + # Exact, so the unreadable-task fallback cannot pass for the result. + assert raised == base + added_sec + def test_user_loop_rounds_dominate(self, tmp_path, monkeypatch): """A user-loop run legitimately spends max_user_rounds x (agent + soft-verify); the backstop must cover it, not fire mid-loop."""