Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
68 commits
Select commit Hold shift + click to select a range
2d746ac
[jwbron/live-eval-corpus] review: live-enabled corpus format and ten …
jwbron Jul 9, 2026
6727eb6
[jwbron/live-eval-producer-staging] review: live-producer prompt extr…
jwbron Jul 9, 2026
5c00cf3
[jwbron/live-eval-producer-staging] review: the live producer and SDK…
jwbron Jul 9, 2026
52073ef
[jwbron/live-eval-ab-runner] review: the live A/B runner (phase 3)
jwbron Jul 9, 2026
93770e6
[jwbron/live-eval-ab-ci] review: per-PR live A/B workflow (phase 4)
jwbron Jul 9, 2026
30e7527
[jwbron/live-eval-corpus] review: exclude eval-corpus trees from lint…
jwbron Jul 9, 2026
19f06cb
[jwbron/live-eval-producer-staging] Merge branch 'jwbron/live-eval-co…
jwbron Jul 9, 2026
d1b5288
[jwbron/live-eval-ab-runner] Merge branch 'jwbron/live-eval-producer-…
jwbron Jul 9, 2026
254dfc4
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 9, 2026
917cc11
[jwbron/live-eval-producer-staging] review: namespace live finding id…
jwbron Jul 9, 2026
0ee295a
[jwbron/live-eval-ab-runner] Merge branch 'jwbron/live-eval-producer-…
jwbron Jul 9, 2026
58cd4ed
[jwbron/live-eval-ab-runner] review: judge failures degrade the A/B r…
jwbron Jul 9, 2026
b19b08d
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 9, 2026
bced291
[jwbron/review-trial-skill] review: add the review-trial skill (live …
jwbron Jul 9, 2026
da1115c
[tmp-refresh] Merge remote-tracking branch 'origin/main' into tmp-ref…
jwbron Jul 9, 2026
a6883be
[tmp-refresh] Merge remote-tracking branch 'origin/jwbron/live-eval-c…
jwbron Jul 9, 2026
0d02672
[tmp-refresh] Merge remote-tracking branch 'origin/jwbron/live-eval-p…
jwbron Jul 9, 2026
93d8dec
[tmp-refresh] Merge remote-tracking branch 'origin/jwbron/live-eval-a…
jwbron Jul 9, 2026
e3b34eb
[tmp-refresh] Merge remote-tracking branch 'origin/jwbron/live-eval-a…
jwbron Jul 9, 2026
491a983
[jwbron/review-rereview-accountability] review: re-review accountabil…
jwbron Jul 9, 2026
7b5318c
[jwbron/review-out-of-lane] review: hand off out-of-lane observations…
jwbron Jul 9, 2026
5dd182b
[jwbron/review-out-artifact-upload] review: fix the out/ artifact upl…
jwbron Jul 9, 2026
92bffa2
[jwbron/review-rereview-accountability] review: prettier-format the r…
jwbron Jul 9, 2026
2be8ede
[jwbron/review-out-of-lane] Merge branch 'jwbron/review-rereview-acco…
jwbron Jul 9, 2026
3a8fc5c
[jwbron/review-out-of-lane] review: prettier-format finding-schema
jwbron Jul 9, 2026
813767c
[jwbron/review-dispatch-tax] review: dedupe lens discipline snippets …
jwbron Jul 9, 2026
2812679
[jwbron/live-eval-corpus] review: route the specialist lens on each l…
jwbron Jul 9, 2026
2ce35a0
[jwbron/live-eval-producer-staging] Merge branch 'jwbron/live-eval-co…
jwbron Jul 9, 2026
c0fece2
[jwbron/live-eval-ab-runner] Merge branch 'jwbron/live-eval-producer-…
jwbron Jul 9, 2026
e02ac40
[jwbron/live-eval-ab-runner] review: carry agent-failure reasons into…
jwbron Jul 9, 2026
d225bd4
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 9, 2026
cbc838d
[jwbron/live-eval-ab-runner] review: close every A/B report with a pe…
jwbron Jul 9, 2026
63097f1
[jwbron/live-eval-corpus] Merge remote-tracking branch 'origin/jwbron…
jwbron Jul 9, 2026
9012508
[jwbron/live-eval-producer-staging] Merge remote-tracking branch 'ori…
jwbron Jul 9, 2026
25133b4
[jwbron/live-eval-producer-staging] Merge branch 'jwbron/live-eval-co…
jwbron Jul 9, 2026
0a3d212
[jwbron/live-eval-ab-runner] Merge remote-tracking branch 'origin/jwb…
jwbron Jul 9, 2026
a547972
[jwbron/live-eval-ab-runner] Merge branch 'jwbron/live-eval-producer-…
jwbron Jul 9, 2026
391151b
[jwbron/live-eval-ab-ci] Merge remote-tracking branch 'origin/jwbron/…
jwbron Jul 9, 2026
d2c4c70
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 9, 2026
996766f
[jwbron/live-eval-ab-runner] review: identity short-circuit, gate-fli…
jwbron Jul 10, 2026
fb81be8
[jwbron/live-eval-ab-runner] review: judge economics (Haiku pin, retr…
jwbron Jul 10, 2026
a659be8
[jwbron/live-eval-ab-ci] review: document why the baseline is the bas…
jwbron Jul 10, 2026
b7c3786
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 10, 2026
541e413
[jwbron/review-trial-skill] Merge branch 'jwbron/live-eval-ab-ci' int…
jwbron Jul 10, 2026
fd42efd
[jwbron/review-out-artifact-upload] Merge branch 'jwbron/review-trial…
jwbron Jul 10, 2026
6d3459a
[jwbron/review-rereview-accountability] Merge branch 'jwbron/review-o…
jwbron Jul 10, 2026
1438a67
[jwbron/review-out-of-lane] Merge branch 'jwbron/review-rereview-acco…
jwbron Jul 10, 2026
98157c2
[jwbron/review-dispatch-tax] Merge branch 'jwbron/review-out-of-lane'…
jwbron Jul 10, 2026
193ae69
[jwbron/live-eval-ab-runner] review: type the judge response via the …
jwbron Jul 10, 2026
7a2065c
[jwbron/live-eval-ab-ci] Merge branch 'jwbron/live-eval-ab-runner' in…
jwbron Jul 10, 2026
00ce9d4
[jwbron/review-trial-skill] Merge branch 'jwbron/live-eval-ab-ci' int…
jwbron Jul 10, 2026
4bd445c
[jwbron/review-out-artifact-upload] Merge branch 'jwbron/review-trial…
jwbron Jul 10, 2026
da1e1db
[jwbron/review-rereview-accountability] Merge branch 'jwbron/review-o…
jwbron Jul 10, 2026
1bc9e08
[jwbron/review-out-of-lane] Merge branch 'jwbron/review-rereview-acco…
jwbron Jul 10, 2026
0123a92
[jwbron/review-dispatch-tax] Merge branch 'jwbron/review-out-of-lane'…
jwbron Jul 10, 2026
80da666
Merge branch 'main' into jwbron/live-eval-corpus
jwbron Jul 10, 2026
82bc5ed
Merge branch 'jwbron/live-eval-corpus' into jwbron/live-eval-producer…
jwbron Jul 10, 2026
7f2fc3a
Merge branch 'jwbron/live-eval-producer-staging' into jwbron/live-eva…
jwbron Jul 10, 2026
9cc1a89
Merge branch 'jwbron/live-eval-ab-runner' into jwbron/live-eval-ab-ci
jwbron Jul 10, 2026
da93344
Merge branch 'jwbron/live-eval-ab-ci' into jwbron/review-trial-skill
jwbron Jul 10, 2026
aa5a246
Merge branch 'jwbron/review-trial-skill' into jwbron/review-out-artif…
jwbron Jul 10, 2026
248cdf8
Merge branch 'jwbron/review-out-artifact-upload' into jwbron/review-r…
jwbron Jul 10, 2026
51691de
Merge branch 'jwbron/review-rereview-accountability' into jwbron/revi…
jwbron Jul 10, 2026
a752c5c
Merge branch 'jwbron/review-out-of-lane' into jwbron/review-dispatch-tax
jwbron Jul 10, 2026
6c496fe
[landtmp] Merge remote-tracking branch 'origin/main' into landtmp
jwbron Jul 10, 2026
eeed413
[jwbron/review-dispatch-tax] review: verify the disciplines extractio…
jwbron Jul 13, 2026
0345c27
[jwbron/review-dispatch-tax] Merge remote-tracking branch 'origin/mai…
jwbron Jul 13, 2026
c965cd3
[jwbron/review-dispatch-tax] review: drop the per-lens out-of-lane co…
jwbron Jul 13, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/review-dispatch-tax.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"review": patch
---

Dispatch-tax trim: dedupe the specialist-lens discipline snippets. The v1.4.0 re-run's cost premium vs the v1.3.1 baseline sat at dispatch time (uncached input +51-65%, cache writes +18-34% per run), partly because the shared discipline blocks (bounded investigation, untrusted input, lens-owned skills with quote-the-rule, the schema-rules trailer, the tri-state hunt contract) were stamped verbatim into every one of the eleven specialist-lens definitions and paid on every dispatch. The shared text now lives once, in a marker-delimited "REVIEW DISCIPLINES" section of review.md's main body; Step 1 stages it to `/tmp/gh-aw/review/disciplines.md` with one whole-line-anchored `sed` extraction of the engine-provided rendered prompt (`$GH_AW_PROMPT`), verified by grep with a byte-for-byte heredoc fallback, and each lens carries only a pointer plus its own domain notes (its investigation examples, review rules, hunts, and output JSON, all unchanged). Behavior-neutral by construction: the instruction content is unchanged, the three variants that had already drifted apart are unified on the fullest wording, and the label-shape reviewers (whose variants differ materially) keep their own copies. Pinned by `lib/disciplines.test.ts`: the section extracts cleanly under the exact sed range semantics, every lens points at the staged file and carries no residual copy, and the label-shape reviewers still carry theirs.
192 changes: 192 additions & 0 deletions workflows/review/lib/disciplines.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,192 @@
import {execFileSync} from "node:child_process";
import {readFileSync} from "node:fs";
import {join} from "node:path";

import {describe, it, expect} from "vitest";

import {SPECIALIST_LENSES} from "./router.ts";

/**
* Dispatch-tax dedupe smoke tests.
*
* The measured cost premium of the v1.4.0 re-run sat at dispatch time
* (uncached input +51-65%, cache writes +18-34% per run), partly because
* discipline snippets (bounded investigation, quote-the-rule via lens-owned
* skills, the schema-rules trailer, ...) were stamped verbatim into every one
* of the eleven specialist-lens definitions and paid on every dispatch. The
* shared text now lives once, in the marker-delimited REVIEW DISCIPLINES
* section of review.md's main body, which Step 1 stages to
* `/tmp/gh-aw/review/disciplines.md` with a mechanical `sed` range extraction.
*
* These tests pin the contract: the section exists and extracts cleanly (the
* same line-range semantics the sed command uses), it carries every section
* the lens pointer names, each lens definition points at the staged file and
* no longer carries its own copy, and the label-shape reviewers (whose
* variants differ materially) still carry theirs.
*/

const reviewMdPath = join(__dirname, "..", "review.md");
const reviewMd = readFileSync(reviewMdPath, "utf8");

const BEGIN = "<!-- BEGIN REVIEW DISCIPLINES -->";
const END = "<!-- END REVIEW DISCIPLINES -->";

/**
* The exact sed range expression review.md Step 1 runs (whole-line anchored).
* The anchors are load-bearing: without `^...$`, the range would open at the
* first backticked *mention* of the marker in the prose, ~1075 lines early.
*/
const SED_RANGE = `/^${BEGIN}$/,/^${END}$/p`;

/**
* The sed range extraction from review.md Step 1, replicated line-for-line.
* The Step 1 command anchors both patterns to whole lines (`^...$`), so only
* the marker lines themselves open/close the range — never prose that mentions
* a marker, and never the sed command's own text.
*/
const sedRangeExtract = (text: string): string => {
const lines = text.split("\n");
const out: string[] = [];
let inRange = false;
for (const line of lines) {
if (!inRange && line === BEGIN) {
inRange = true;
}
if (inRange) {
out.push(line);
if (line === END) {
break;
}
}
}
return out.join("\n");
};

/** The main body (the orchestrator prompt): everything before the first agent. */
const mainBody = reviewMd.slice(0, reviewMd.indexOf("\n## agent: `"));

/** One lens's definition section, as the sub-agent extractor would cut it. */
const lensSection = (lens: string): string => {
const start = reviewMd.indexOf(`## agent: \`${lens}\``);
expect(start).toBeGreaterThan(-1);
const rest = reviewMd.slice(start);
const next = rest.indexOf("\n## agent: `");
return next === -1 ? rest : rest.slice(0, next);
};

const DISCIPLINE_HEADINGS = [
"## Staged inputs",
"## Untrusted input",
"## Read every line",
"## Bounded investigation",
"## Lens-owned skills",
"## Out-of-lane handoff",
"## Structured finding schema and hunts",
];

describe("the shared disciplines section", () => {
it("lives in the main body (the staged prompt), not in an agent section", () => {
expect(mainBody).toContain(BEGIN);
expect(mainBody).toContain(END);
});

it("extracts cleanly with the Step 1 sed range semantics", () => {
const extracted = sedRangeExtract(reviewMd);
expect(extracted.startsWith(BEGIN)).toBe(true);
expect(extracted.endsWith(END)).toBe(true);
for (const heading of DISCIPLINE_HEADINGS) {
expect(extracted).toContain(`\n${heading}\n`);
}
});

it("keeps the quote-the-rule discipline verbatim", () => {
const extracted = sedRangeExtract(reviewMd);
expect(extracted).toContain(
"Flag a skill violation only when you can quote **both** the exact rule",
);
expect(extracted).toContain("no spirit-of-the-doc inference");
});

it("keeps the tri-state hunt contract", () => {
const extracted = sedRangeExtract(reviewMd);
for (const state of ["`found`", "`ran`", "`not-applicable`"]) {
expect(extracted).toContain(state);
}
});

it("is staged by Step 1 with the sed extraction and a fallback guard", () => {
expect(mainBody).toContain("$GH_AW_PROMPT");
expect(mainBody).toContain("/tmp/gh-aw/review/disciplines.md");
expect(mainBody).toContain(
"grep -q '## Structured finding schema and hunts'",
);
});

it("extracts identically under real sed (not only the modeled replica)", () => {
// The replica above models the sed semantics; this runs the real
// thing, so a drift in the Step 1 command (say, dropped `^...$`
// anchors) fails here even if the model still passes. Pin the range
// expression to the one review.md actually documents, then run it
// against the real file.
expect(mainBody).toContain(`sed -n '${SED_RANGE}'`);
const staged = execFileSync("sed", ["-n", SED_RANGE, reviewMdPath], {
encoding: "utf8",
});
// sed prints each selected line with its newline; the replica joins
// without a trailing one.
expect(staged).toBe(`${sedRangeExtract(reviewMd)}\n`);
expect(staged.startsWith(BEGIN)).toBe(true);
for (const heading of DISCIPLINE_HEADINGS) {
expect(staged).toContain(`\n${heading}\n`);
}
});
});

describe("each specialist lens definition", () => {
for (const lens of SPECIALIST_LENSES) {
it(`${lens}: points at the staged disciplines and carries no copy`, () => {
const section = lensSection(lens);
expect(section).toContain("**Shared disciplines first.**");
expect(section).toContain("/tmp/gh-aw/review/disciplines.md");
// The deduped blocks must be gone from the lens definition.
expect(section).not.toContain("**Bounded investigation.**");
expect(section).not.toContain("**Untrusted input.**");
expect(section).not.toContain("Read from disk:");
expect(section).not.toContain("Schema rules");
expect(section).not.toContain(
"**Hand off, never drop, an out-of-lane observation.**",
);
// The domain-specific content stays.
expect(section).toContain("### Review rules");
expect(section).toContain("### Incident-derived hunts (tri-state)");
expect(section).toContain("### Output");
expect(section).toContain(
"{{#runtime-import .github/aw/review/skills.md}}",
);
expect(section).toContain(`\`lens\` is exactly \`${lens}\``);
expect(section).toContain(
"Domain notes for §Bounded investigation",
);
});
}
});

describe("the label-shape reviewers still carry their own disciplines", () => {
// Their variants differ materially (CLI cap invocation with per-agent id
// semantics, `discussion` instead of `evidence_trace`), so they were
// deliberately left out of the dedupe.
for (const agent of [
"correctness-reviewer",
"skill-auditor",
"claim-validator",
"holistic",
"completeness",
"test-adequacy",
"first-principles",
"conventions",
]) {
it(`${agent}: keeps its own bounded-investigation block`, () => {
expect(lensSection(agent)).toContain("**Bounded investigation.**");
});
}
});
Loading
Loading