From eb2b3bb765abd5e84d38c519b6f41ecb0660d7f8 Mon Sep 17 00:00:00 2001 From: Onkar Chougule Date: Wed, 9 Sep 2026 16:40:56 +0530 Subject: [PATCH 1/3] Make LLM CI selection authoritative by default Signed-off-by: Onkar Chougule --- scripts/ci_impact/cli.py | 5 +++-- scripts/ci_impact/core.py | 13 +++++++------ scripts/ci_impact/llm.py | 19 +++---------------- 3 files changed, 13 insertions(+), 24 deletions(-) diff --git a/scripts/ci_impact/cli.py b/scripts/ci_impact/cli.py index ebee87142a..d279272c6d 100644 --- a/scripts/ci_impact/cli.py +++ b/scripts/ci_impact/cli.py @@ -15,7 +15,7 @@ import sys from pathlib import Path -from .core import STAGES, ImpactPlan, build_plan, write_plan +from .core import STAGES, ImpactPlan, build_plan, is_hard_full_path, write_plan from .llm import ( LLMSelection, LLMStageError, @@ -78,7 +78,8 @@ def _plan(args: argparse.Namespace) -> int: catalog = load_catalog(args.catalog, plan.head) plan = expand_plan_with_catalog(plan, catalog) write_plan(plan, args.deterministic_output) - if plan.mode == "full": + hard_full_change = any(is_hard_full_path(path) for path in plan.changed_files) + if plan.mode == "full" and (args.force_full or hard_full_change): selection = _skipped_llm_selection() plan.llm = selection.to_dict() write_llm_artifact(selection, args.llm_output) diff --git a/scripts/ci_impact/core.py b/scripts/ci_impact/core.py index fb2b2a454c..c01c927aef 100644 --- a/scripts/ci_impact/core.py +++ b/scripts/ci_impact/core.py @@ -37,11 +37,7 @@ "weight_free_qaic", ) -HARD_FULL_FILES = { - "pyproject.toml", - "scripts/Jenkinsfile", - "scripts/JenkinsFileFullCi", -} +HARD_FULL_FILES = {"pyproject.toml", "scripts/Jenkinsfile", "scripts/JenkinsFileFullCi"} IGNORED_FILES = { ".gitignore", ".pre-commit-config.yaml", @@ -626,6 +622,11 @@ def _empty_plan( ) +def is_hard_full_path(path: str) -> bool: + """Return whether a changed path must bypass LLM test selection.""" + return path in HARD_FULL_FILES or path.startswith("scripts/ci_impact/") + + def build_plan(repo: Path, base: str, head: str = "HEAD", force_full: bool = False) -> ImpactPlan: repo = repo.resolve() merge_base, head_sha, changes = resolve_changes(repo, base, head) @@ -635,7 +636,7 @@ def build_plan(repo: Path, base: str, head: str = "HEAD", force_full: bool = Fal return _empty_plan("no_tests", merge_base, head_sha, changes, ["no changed files"], []) paths = {change.path for change in changes} - hard = sorted(path for path in paths if path in HARD_FULL_FILES or path.startswith("scripts/ci_impact/")) + hard = sorted(path for path in paths if is_hard_full_path(path)) if hard: reasons = [f"unconditional full-CI path: {path}" for path in hard] return _empty_plan("full", merge_base, head_sha, changes, reasons, []) diff --git a/scripts/ci_impact/llm.py b/scripts/ci_impact/llm.py index ad7f92e265..8830f67549 100644 --- a/scripts/ci_impact/llm.py +++ b/scripts/ci_impact/llm.py @@ -22,9 +22,9 @@ from dataclasses import asdict, dataclass from pathlib import Path -from .core import SCHEMA_VERSION, STAGES, ImpactPlan, TestCase +from .core import SCHEMA_VERSION, STAGES, ImpactPlan, TestCase, is_hard_full_path -DEFAULT_MODEL = "azure::gpt-5.5" +DEFAULT_MODEL = "gpt5.6-terra" DEFAULT_REASONING_EFFORT = "high" MAX_PROMPT_BYTES = 400_000 MAX_RESPONSE_BYTES = 1_000_000 @@ -36,15 +36,6 @@ QUERY_TOOL_PATH = Path(__file__).with_name("query.py") TOOL_POLICY_PATH = Path(__file__).with_name("tool_policy.py") HOOK_AUDIT_NAME = ".ci-impact-qgenie-audit.jsonl" -LLM_REFINABLE_FULL_REASONS = ( - "global pytest behavior changed:", - "source snapshot generation failed", - "unsafe static analysis for ", - "unclassified production/configuration changes", - "unparsable model inventory:", -) - - class LLMStageError(RuntimeError): """Raised when mandatory LLM selection cannot be completed safely.""" @@ -700,11 +691,7 @@ def merge_selection( plan.stages[stage]["enabled"] = True return plan - if ( - plan.mode == "full" - and plan.reasons - and all(reason.startswith(LLM_REFINABLE_FULL_REASONS) for reason in plan.reasons) - ): + if plan.mode == "full" and not any(is_hard_full_path(path) for path in plan.changed_files): for stage in STAGES: plan.stages[stage]["enabled"] = False plan.reasons.append("LLM bounded a deterministic static-analysis full-CI fallback") From 7bf14ca9a470b9d130e1e8bf43431a35e409320d Mon Sep 17 00:00:00 2001 From: Onkar Chougule Date: Wed, 9 Sep 2026 16:42:09 +0530 Subject: [PATCH 2/3] Test LLM refinement of static full fallbacks Signed-off-by: Onkar Chougule --- tests/ci_impact/test_core.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/ci_impact/test_core.py b/tests/ci_impact/test_core.py index 5d13b34649..5dddf85316 100644 --- a/tests/ci_impact/test_core.py +++ b/tests/ci_impact/test_core.py @@ -564,13 +564,23 @@ def test_llm_can_escalate_to_full_ci() -> None: assert all(stage["enabled"] for stage in merged.stages.values()) -def test_llm_can_refine_static_analysis_full_plan() -> None: +@pytest.mark.parametrize( + "reason", + [ + "unsafe static analysis for QEfficient/base/modeling_qeff.py", + "unclassified production/configuration changes", + "unparsable model inventory: tests/configs/models.json", + "production changes produced no confident deterministic test matches", + "dependency closure reaches every Jenkins stage", + ], +) +def test_llm_can_refine_deterministic_full_plan(reason: str) -> None: deterministic = ImpactPlan( mode="full", base="base", head="head", changed_files=["QEfficient/base/modeling_qeff.py"], - reasons=["unsafe static analysis for QEfficient/base/modeling_qeff.py"], + reasons=[reason], unresolved=["QEFFBaseModel: dynamic reflection"], stages=_stage_plans(enabled=True), ) From 08175328c7129f55aa9e33913e5ab6dd5a3f24e8 Mon Sep 17 00:00:00 2001 From: Onkar Chougule Date: Wed, 9 Sep 2026 16:47:11 +0530 Subject: [PATCH 3/3] Fix CI impact module formatting Signed-off-by: Onkar Chougule --- scripts/ci_impact/llm.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/ci_impact/llm.py b/scripts/ci_impact/llm.py index 8830f67549..9e33e6848a 100644 --- a/scripts/ci_impact/llm.py +++ b/scripts/ci_impact/llm.py @@ -36,6 +36,8 @@ QUERY_TOOL_PATH = Path(__file__).with_name("query.py") TOOL_POLICY_PATH = Path(__file__).with_name("tool_policy.py") HOOK_AUDIT_NAME = ".ci-impact-qgenie-audit.jsonl" + + class LLMStageError(RuntimeError): """Raised when mandatory LLM selection cannot be completed safely."""