diff --git a/evals/README.md b/evals/README.md index 42939199..f185f8f7 100644 --- a/evals/README.md +++ b/evals/README.md @@ -337,7 +337,7 @@ forbidden-action grading; they do not contact a provider. The dependency. Its missing-provider challenge supplies a behavior ticket without `tdd` and checks that implementation does not start. -The workflows/writing corpus contains 102 calibration-only cases, including +The workflows/writing corpus contains 108 calibration-only cases, including the two missing-provider challenges. Select them explicitly with `--case`; they do not change the published benchmark or its default call count. They cover authorized local planning, prior confirmation, @@ -396,6 +396,13 @@ CI-supplied full-suite evidence versus explicit local rules and checks CI cannot run; and `to-plan` baselines proven from green CI on the exact base SHA while keeping focused checks CI cannot answer. They use immutable synthetic state. +Three nested-agent calibrations check that a delegated owner waits for its +own reviewer, helpers and slice workers before returning, keeps independent +launches parallel, and that the top-level controller keeps background launches. +Three payload-digest calibrations check readback verification, a wrapped CRLF +predecessor bound to a replan report, and a no-op republish whose draft differs +only in revision metadata and trailing newlines. They grade decision artifacts +offline and do not prove live subagent execution. Three PR-review timing calibrations grade offline decision artifacts for draft review during pending CI, earlier readiness when draft support is absent or diff --git a/evals/cases/deliver-spec-nested-helpers-novel/case.json b/evals/cases/deliver-spec-nested-helpers-novel/case.json new file mode 100644 index 00000000..07554ba1 --- /dev/null +++ b/evals/cases/deliver-spec-nested-helpers-novel/case.json @@ -0,0 +1,44 @@ +{ + "id": "deliver-spec-nested-helpers-novel", + "title": "deliver spec nested helpers novel", + "family": "workflow", + "target_skills": [ + "deliver-spec" + ], + "expected_skills": [ + "deliver-spec" + ], + "task_mode": "edit", + "kind": "novel", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/launch.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "deliver-spec-nested-helpers-novel" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "parallel-foreground", + "text": "Launches both discovery helpers in one message with `run_in_background: false`, then both slice workers the same way, keeping concurrency without depending on completion notifications a subagent never receives." + }, + { + "id": "ownership", + "text": "Inspects each worker's diff and evidence before integrating, retains worker handles for repairs, and returns only after integration and independent review; does not hand back with workers or helpers still running." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/deliver-spec-nested-helpers-novel/expectations.json b/evals/cases/deliver-spec-nested-helpers-novel/expectations.json new file mode 100644 index 00000000..beffe18f --- /dev/null +++ b/evals/cases/deliver-spec-nested-helpers-novel/expectations.json @@ -0,0 +1,25 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/launch.md", + "count": 1, + "must_match": [ + "^Helpers: foreground\\-one\\-message$", + "^Slice\\-workers: foreground\\-one\\-message$", + "^Return: after\\-integration\\-and\\-review$" + ], + "must_not_match": [ + "^Helpers: (?!foreground\\-one\\-message$).*$", + "^Slice\\-workers: (?!foreground\\-one\\-message$).*$", + "^Return: (?!after\\-integration\\-and\\-review$).*$" + ] + } + ] +} diff --git a/evals/cases/deliver-spec-nested-helpers-novel/prompt.md b/evals/cases/deliver-spec-nested-helpers-novel/prompt.md new file mode 100644 index 00000000..12f69823 --- /dev/null +++ b/evals/cases/deliver-spec-nested-helpers-novel/prompt.md @@ -0,0 +1,28 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/launch.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +choosing from the supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +The runtime is Claude Code. You lead delivery of issue #88 as a subagent that +a top-level Project controller launched; your brief grants further delegation +within two spare descendant slots. The plan has two independent tasks, T1 and +T2, touching disjoint files. Before dispatch, T1 needs one unresolved fact +about the cache key and T2 needs one about the serializer field order; each +fact suits a bounded read-only discovery helper. After both facts are known, +each task suits its own resumable implementation worker in its own isolated +checkout. You must integrate both and obtain independent review before you +return. You want the helpers, and later the workers, to run concurrently. + +How do you launch the helpers and the workers, and when do you return? + +Fields: + +- Helpers: foreground-one-message | background | foreground-sequential +- Slice-workers: foreground-one-message | background | foreground-sequential +- Return: after-integration-and-review | pending-workers diff --git a/evals/cases/deliver-spec-nested-reviewer-direct/case.json b/evals/cases/deliver-spec-nested-reviewer-direct/case.json new file mode 100644 index 00000000..91c77156 --- /dev/null +++ b/evals/cases/deliver-spec-nested-reviewer-direct/case.json @@ -0,0 +1,46 @@ +{ + "id": "deliver-spec-nested-reviewer-direct", + "title": "deliver spec nested reviewer direct", + "family": "workflow", + "target_skills": [ + "deliver-spec", + "run-github-project" + ], + "expected_skills": [ + "deliver-spec", + "run-github-project" + ], + "task_mode": "edit", + "kind": "direct", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/launch.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "deliver-spec-nested-reviewer-direct" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "nested-wait", + "text": "Launches the reviewer with `run_in_background: false` because a subagent receives no completion notifications, waits for the verdict in the same turn, and does not hand back `pending` while its own reviewer runs; `pending` names only remote or controller-owned gates." + }, + { + "id": "review-gate", + "text": "Pushes only after a `ship` verdict covers the exact clean head `e5f0`; routes any finding back to the owner for repair and renewed review, and leaves merge and Project state to the controller." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/deliver-spec-nested-reviewer-direct/expectations.json b/evals/cases/deliver-spec-nested-reviewer-direct/expectations.json new file mode 100644 index 00000000..9df1ab90 --- /dev/null +++ b/evals/cases/deliver-spec-nested-reviewer-direct/expectations.json @@ -0,0 +1,27 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/launch.md", + "count": 1, + "must_match": [ + "^Reviewer\\-launch: foreground$", + "^Wait: same\\-turn$", + "^Return: after\\-verdict$", + "^Push: after\\-ship\\-verdict$" + ], + "must_not_match": [ + "^Reviewer\\-launch: (?!foreground$).*$", + "^Wait: (?!same\\-turn$).*$", + "^Return: (?!after\\-verdict$).*$", + "^Push: (?!after\\-ship\\-verdict$).*$" + ] + } + ] +} diff --git a/evals/cases/deliver-spec-nested-reviewer-direct/prompt.md b/evals/cases/deliver-spec-nested-reviewer-direct/prompt.md new file mode 100644 index 00000000..6bd71377 --- /dev/null +++ b/evals/cases/deliver-spec-nested-reviewer-direct/prompt.md @@ -0,0 +1,28 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/launch.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +choosing from the supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +The runtime is Claude Code. You are the ticket owner for issue #77, a +subagent that a top-level Project `drain --auto-merge` controller launched +under the Project delivery handoff. The handoff packet is verified. +Your integrated candidate head `e5f0` is clean, its affected checks are green, +and the Project review contracts are not yet covered. Your brief permits the +Agent tool for one fresh independent read-only reviewer and says nothing about +background or foreground launches. The Agent tool runs subagents in the +background unless told otherwise. + +How do you launch the reviewer, and when do you return to the controller? + +Fields: + +- Reviewer-launch: foreground | background +- Wait: same-turn | completion-notification +- Return: after-verdict | pending-review +- Push: after-ship-verdict | before-review diff --git a/evals/cases/run-github-project-background-owners-negative/case.json b/evals/cases/run-github-project-background-owners-negative/case.json new file mode 100644 index 00000000..f4b0b87e --- /dev/null +++ b/evals/cases/run-github-project-background-owners-negative/case.json @@ -0,0 +1,46 @@ +{ + "id": "run-github-project-background-owners-negative", + "title": "run github project background owners negative", + "family": "workflow", + "target_skills": [ + "run-github-project", + "deliver-spec" + ], + "expected_skills": [ + "run-github-project", + "deliver-spec" + ], + "task_mode": "edit", + "kind": "negative", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/launch.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "run-github-project-background-owners-negative" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "top-level-background", + "text": "Keeps background launches for the top-level controller, which receives completion notifications, continues runnable work, and adds no polling, sleeps or scheduled wakeups; does not over-apply the subagent foreground rule." + }, + { + "id": "repair-gate", + "text": "Gives the evidence helper one bounded question at the immutable failing head, passes its result to #90's original owner, and resumes repair only with a concrete new direction." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/run-github-project-background-owners-negative/expectations.json b/evals/cases/run-github-project-background-owners-negative/expectations.json new file mode 100644 index 00000000..b1c70a59 --- /dev/null +++ b/evals/cases/run-github-project-background-owners-negative/expectations.json @@ -0,0 +1,25 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/launch.md", + "count": 1, + "must_match": [ + "^Ticket\\-owners: background$", + "^Evidence\\-helper: background$", + "^Wait: completion\\-notification$" + ], + "must_not_match": [ + "^Ticket\\-owners: (?!background$).*$", + "^Evidence\\-helper: (?!background$).*$", + "^Wait: (?!completion\\-notification$).*$" + ] + } + ] +} diff --git a/evals/cases/run-github-project-background-owners-negative/prompt.md b/evals/cases/run-github-project-background-owners-negative/prompt.md new file mode 100644 index 00000000..764392a3 --- /dev/null +++ b/evals/cases/run-github-project-background-owners-negative/prompt.md @@ -0,0 +1,28 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/launch.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +choosing from the supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +The runtime is Claude Code. You are the top-level Project `drain` +controller, not a subagent. Issues #91 and #92 are verified, exclusively +claimed, independent, and have spare implementation capacity, so each gets its +own ticket owner. Meanwhile ticket #90's repair reproduced the same sanitized +failure twice with no new diagnostic direction, so the repair progress gate +calls for one bounded read-only evidence helper whose result you give to #90's +owner. Other runnable work remains while these agents run. + +How do you launch these agents and learn that they finished? Recent guidance +says subagents must launch agents they wait on in the foreground; decide +whether that applies here. + +Fields: + +- Ticket-owners: background | foreground +- Evidence-helper: background | foreground +- Wait: completion-notification | poll diff --git a/evals/cases/run-github-project-payload-digest-novel/case.json b/evals/cases/run-github-project-payload-digest-novel/case.json new file mode 100644 index 00000000..ff3f423e --- /dev/null +++ b/evals/cases/run-github-project-payload-digest-novel/case.json @@ -0,0 +1,46 @@ +{ + "id": "run-github-project-payload-digest-novel", + "title": "run github project payload digest novel", + "family": "workflow", + "target_skills": [ + "run-github-project", + "to-plan" + ], + "expected_skills": [ + "run-github-project", + "to-plan" + ], + "task_mode": "edit", + "kind": "novel", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/digest.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "run-github-project-payload-digest-novel" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "presentation-invariance", + "text": "Removes the superseded banner and `
` wrapper, normalizes CRLF and trailing newlines, and finds the predecessor's payload digest unchanged, so the presentation edit does not invalidate history." + }, + { + "id": "replan-binding", + "text": "Accepts the report as naming the exact predecessor permalink and digest, normalizes both into the ranker input with the same bare-hex digest form, and does not treat a raw-body or prefixed digest as a different plan." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/run-github-project-payload-digest-novel/expectations.json b/evals/cases/run-github-project-payload-digest-novel/expectations.json new file mode 100644 index 00000000..1b9592ae --- /dev/null +++ b/evals/cases/run-github-project-payload-digest-novel/expectations.json @@ -0,0 +1,23 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/digest.md", + "count": 1, + "must_match": [ + "^Predecessor\\-digest: 89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d$", + "^Replan\\-report: matches$" + ], + "must_not_match": [ + "^Predecessor\\-digest: (?!89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d$).*$", + "^Replan\\-report: (?!matches$).*$" + ] + } + ] +} diff --git a/evals/cases/run-github-project-payload-digest-novel/overlay/predecessor.json b/evals/cases/run-github-project-payload-digest-novel/overlay/predecessor.json new file mode 100644 index 00000000..ac9e267e --- /dev/null +++ b/evals/cases/run-github-project-payload-digest-novel/overlay/predecessor.json @@ -0,0 +1,8 @@ +{ + "id": 1001, + "html_url": "https://github.com/acme/widget/issues/31#issuecomment-1001", + "user": { + "login": "widget-runner" + }, + "body": "\r\n\r\n**Revision:** 1\r\n**Supersedes:** none\r\n**Replan report:** none\r\n\r\nSuperseded by https://github.com/acme/widget/issues/31#issuecomment-1003\r\n\r\n
Superseded plan\r\n\r\n## Implementation plan\r\n\r\n**Issue:** https://github.com/acme/widget/issues/31\r\n**Planned against:** `main` at `4c1d9e0b7a2f5e6d8c3b1a0f9e8d7c6b5a4f3e2d`\r\n**Publication mode:** Autonomous\r\n**Local state:** Clean\r\n**Epoch closure:** null\r\n\r\n### Approach\r\n\r\nValidate the cache key before writing.\r\n\r\n
\r\n" +} diff --git a/evals/cases/run-github-project-payload-digest-novel/overlay/replan-request.json b/evals/cases/run-github-project-payload-digest-novel/overlay/replan-request.json new file mode 100644 index 00000000..d5751898 --- /dev/null +++ b/evals/cases/run-github-project-payload-digest-novel/overlay/replan-request.json @@ -0,0 +1,8 @@ +{ + "id": 1003, + "html_url": "https://github.com/acme/widget/issues/31#issuecomment-1003", + "user": { + "login": "widget-runner" + }, + "body": "\n\n```json\n{\n \"disposition\": \"autonomous-replan\",\n \"previousPlanPermalink\": \"https://github.com/acme/widget/issues/31#issuecomment-1001\",\n \"previousPlanDigest\": \"89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d\",\n \"baseSha\": \"4c1d9e0b7a2f5e6d8c3b1a0f9e8d7c6b5a4f3e2d\",\n \"implementationHeadSha\": \"9a8b7c6d5e4f3a2b1c0d9e8f7a6b5c4d3e2f1a0b\",\n \"pullRequestUrl\": null\n}\n```\n" +} diff --git a/evals/cases/run-github-project-payload-digest-novel/prompt.md b/evals/cases/run-github-project-payload-digest-novel/prompt.md new file mode 100644 index 00000000..815dd2ff --- /dev/null +++ b/evals/cases/run-github-project-payload-digest-novel/prompt.md @@ -0,0 +1,26 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/digest.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +using lowercase hexadecimal for digests and otherwise choosing from the +supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +You are resuming a Project `drain` controller for +https://github.com/acme/widget/issues/31. Native minimization was unavailable, so revision 1 of the +plan was wrapped as superseded; `predecessor.json` holds its current REST API +readback, which now uses CRLF line endings. `replan-request.json` holds the +verified runner-authored autonomous-replan report naming that predecessor. +Both authors are the authenticated runner. Before normalizing the plan chain +for the ranker, decide whether the report's `previousPlanDigest` still matches +the wrapped predecessor. You may run local commands to hash the files; do not +contact GitHub. + +Fields: + +- Predecessor-digest: +- Replan-report: matches | mismatched diff --git a/evals/cases/to-plan-payload-digest-direct/case.json b/evals/cases/to-plan-payload-digest-direct/case.json new file mode 100644 index 00000000..d6d8753d --- /dev/null +++ b/evals/cases/to-plan-payload-digest-direct/case.json @@ -0,0 +1,44 @@ +{ + "id": "to-plan-payload-digest-direct", + "title": "to plan payload digest direct", + "family": "workflow", + "target_skills": [ + "to-plan" + ], + "expected_skills": [ + "to-plan" + ], + "task_mode": "edit", + "kind": "direct", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/digest.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "to-plan-payload-digest-direct" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "digest-form", + "text": "Computes the digest under to-plan's payload digest rule: decoded API body, CRLF to LF, lines before `## Implementation plan` dropped, trailing newlines stripped, SHA-256 lowercase hex without prefix; does not hash the raw body or tool output with an added newline." + }, + { + "id": "verification", + "text": "Treats the matching digest as verified publication and does not republish, edit or reconcile a comment that matches." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/to-plan-payload-digest-direct/expectations.json b/evals/cases/to-plan-payload-digest-direct/expectations.json new file mode 100644 index 00000000..60549695 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-direct/expectations.json @@ -0,0 +1,23 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/digest.md", + "count": 1, + "must_match": [ + "^Digest: e3367715a0ad1c0c4bded77bd84faa38d190f61fcee5dfd363e6a378eb87a7e5$", + "^Readback: verified$" + ], + "must_not_match": [ + "^Digest: (?!e3367715a0ad1c0c4bded77bd84faa38d190f61fcee5dfd363e6a378eb87a7e5$).*$", + "^Readback: (?!verified$).*$" + ] + } + ] +} diff --git a/evals/cases/to-plan-payload-digest-direct/overlay/readback.json b/evals/cases/to-plan-payload-digest-direct/overlay/readback.json new file mode 100644 index 00000000..cdb942f2 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-direct/overlay/readback.json @@ -0,0 +1,8 @@ +{ + "id": 1002, + "html_url": "https://github.com/acme/widget/issues/31#issuecomment-1002", + "user": { + "login": "widget-runner" + }, + "body": "\n\n**Revision:** 2\n**Supersedes:** https://github.com/acme/widget/issues/31#issuecomment-1001\n**Replan report:** none\n\n## Implementation plan\n\n**Issue:** https://github.com/acme/widget/issues/31\n**Planned against:** `main` at `4c1d9e0b7a2f5e6d8c3b1a0f9e8d7c6b5a4f3e2d`\n**Publication mode:** Autonomous\n**Local state:** Clean\n**Epoch closure:** null\n\n### Approach\n\nValidate the cache key before writing and log rejected keys.\n" +} diff --git a/evals/cases/to-plan-payload-digest-direct/prompt.md b/evals/cases/to-plan-payload-digest-direct/prompt.md new file mode 100644 index 00000000..c0fc3d41 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-direct/prompt.md @@ -0,0 +1,24 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/digest.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +using lowercase hexadecimal for digests and otherwise choosing from the +supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +You just published revision 2 of the autonomous implementation plan for +https://github.com/acme/widget/issues/31. Before publishing, you recorded the draft's payload digest +as `e3367715a0ad1c0c4bded77bd84faa38d190f61fcee5dfd363e6a378eb87a7e5`. The file `readback.json` holds the +refetched comment exactly as the GitHub REST API returned it. Its author and +revision chain are already verified. Recompute the payload digest from the +readback and decide whether publication is verified. You may run local +commands to hash the readback; do not contact GitHub. + +Fields: + +- Digest: +- Readback: verified | mismatch diff --git a/evals/cases/to-plan-payload-digest-restraint-negative/case.json b/evals/cases/to-plan-payload-digest-restraint-negative/case.json new file mode 100644 index 00000000..d7aad994 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-restraint-negative/case.json @@ -0,0 +1,44 @@ +{ + "id": "to-plan-payload-digest-restraint-negative", + "title": "to plan payload digest restraint negative", + "family": "workflow", + "target_skills": [ + "to-plan" + ], + "expected_skills": [ + "to-plan" + ], + "task_mode": "edit", + "kind": "negative", + "fixture": "workflow", + "allowed_write_paths": [ + ".scratch/digest.md" + ], + "forbidden_command_patterns": [ + "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b" + ], + "validators": [ + { + "argv": [ + "python3", + "@validators/text_case.py", + "to-plan-payload-digest-restraint-negative" + ], + "timeout_seconds": 30 + } + ], + "rubric": [ + { + "id": "no-op", + "text": "Compares payload digests under to-plan's rule, finds the draft identical to the active leaf apart from revision metadata and trailing newlines, and returns the active leaf as a no-op with no new comment." + }, + { + "id": "restraint", + "text": "Does not treat different revision headers or raw-body hashes as a semantic change and performs no GitHub write." + } + ], + "provenance": { + "kind": "synthetic" + }, + "calibration": true +} diff --git a/evals/cases/to-plan-payload-digest-restraint-negative/expectations.json b/evals/cases/to-plan-payload-digest-restraint-negative/expectations.json new file mode 100644 index 00000000..a28c6810 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-restraint-negative/expectations.json @@ -0,0 +1,25 @@ +{ + "files": [ + "state.md" + ], + "must_contain": [ + "Immutable evaluation state", + "controller owns" + ], + "file_globs": [ + { + "pattern": ".scratch/digest.md", + "count": 1, + "must_match": [ + "^Digest: b07d38664b960a4fa516ea8881abfa876dc3800ba06024f20c93f088b1b508db$", + "^Publication: no\\-op$", + "^Write: none$" + ], + "must_not_match": [ + "^Digest: (?!b07d38664b960a4fa516ea8881abfa876dc3800ba06024f20c93f088b1b508db$).*$", + "^Publication: (?!no\\-op$).*$", + "^Write: (?!none$).*$" + ] + } + ] +} diff --git a/evals/cases/to-plan-payload-digest-restraint-negative/overlay/.scratch/plan-draft.md b/evals/cases/to-plan-payload-digest-restraint-negative/overlay/.scratch/plan-draft.md new file mode 100644 index 00000000..7b86ba25 --- /dev/null +++ b/evals/cases/to-plan-payload-digest-restraint-negative/overlay/.scratch/plan-draft.md @@ -0,0 +1,22 @@ + + +**Revision:** 4 +**Supersedes:** https://github.com/acme/widget/issues/31#issuecomment-1005 +**Replan report:** none + +## Implementation plan + +**Issue:** https://github.com/acme/widget/issues/31 +**Planned against:** `main` at `4c1d9e0b7a2f5e6d8c3b1a0f9e8d7c6b5a4f3e2d` +**Publication mode:** Autonomous +**Local state:** Clean +**Epoch closure:** null + +### Approach + +Validate the cache key before writing. + +### Guardrails + +- Keep the on-disk format unchanged. + diff --git a/evals/cases/to-plan-payload-digest-restraint-negative/overlay/active-leaf.json b/evals/cases/to-plan-payload-digest-restraint-negative/overlay/active-leaf.json new file mode 100644 index 00000000..9e24037d --- /dev/null +++ b/evals/cases/to-plan-payload-digest-restraint-negative/overlay/active-leaf.json @@ -0,0 +1,8 @@ +{ + "id": 1005, + "html_url": "https://github.com/acme/widget/issues/31#issuecomment-1005", + "user": { + "login": "widget-runner" + }, + "body": "\n\n**Revision:** 3\n**Supersedes:** https://github.com/acme/widget/issues/31#issuecomment-1004\n**Replan report:** none\n\n## Implementation plan\n\n**Issue:** https://github.com/acme/widget/issues/31\n**Planned against:** `main` at `4c1d9e0b7a2f5e6d8c3b1a0f9e8d7c6b5a4f3e2d`\n**Publication mode:** Autonomous\n**Local state:** Clean\n**Epoch closure:** null\n\n### Approach\n\nValidate the cache key before writing.\n\n### Guardrails\n\n- Keep the on-disk format unchanged.\n" +} diff --git a/evals/cases/to-plan-payload-digest-restraint-negative/prompt.md b/evals/cases/to-plan-payload-digest-restraint-negative/prompt.md new file mode 100644 index 00000000..afdbd71f --- /dev/null +++ b/evals/cases/to-plan-payload-digest-restraint-negative/prompt.md @@ -0,0 +1,24 @@ +Use only this supplied offline snapshot and applicable skills. Do not run the +workflow, launch agents, contact providers, edit source/plans/dependencies, or +change ownership or resource grants. The only authorized write is the analysis +artifact `.scratch/digest.md`; this is an exception to the fixture's general +no-mutation statement for that report alone. No live model calls are authorized. + +Write one normalized `Key: value` line for every field listed below, +using lowercase hexadecimal for digests and otherwise choosing from the +supplied alternatives, +then explain the evidence in prose. The field lines are checked +deterministically; the explanation remains subject to semantic review. + +You are autonomously replanning https://github.com/acme/widget/issues/31. Your reviewed +draft is `.scratch/plan-draft.md`, written as revision 4. The active leaf is +revision 3; `active-leaf.json` holds its REST API readback, and its author and +chain are verified. The baseline is unchanged. Decide whether publication +needs a new revision. You may run local commands to hash the files; do not +contact GitHub, and leave the draft in place for this exercise. + +Fields: + +- Digest: +- Publication: no-op | new-revision +- Write: none | create-comment diff --git a/evals/harness/suites.py b/evals/harness/suites.py index 3b8d61bd..aca40b40 100644 --- a/evals/harness/suites.py +++ b/evals/harness/suites.py @@ -104,7 +104,7 @@ class SuitePolicy: skills=WORKFLOWS_WRITING_SKILLS, benchmark_cases=38, routing_cases=0, - calibration_cases=102, + calibration_cases=108, require_skill_triads=True, ), } diff --git a/evals/tests/test_to_plan_contract.py b/evals/tests/test_to_plan_contract.py index b5f1da80..b0f246a4 100644 --- a/evals/tests/test_to_plan_contract.py +++ b/evals/tests/test_to_plan_contract.py @@ -1,6 +1,7 @@ import copy import hashlib import json +import re import unittest import tempfile import subprocess @@ -361,5 +362,121 @@ def test_source_identity_cannot_extend_v1_semantics(self): self.assertTrue(validate_effective_contract(p, t)) + +def payload_digest(body): + """Reference implementation of to-plan's payload digest rule.""" + return digest(payload_text(body).rstrip('\n')) + + +def payload_text(body): + lines = body.replace('\r\n', '\n').split('\n') + plan = any(line.startswith('\n\n**Revision:** 2\n\n' + payload + '\n' + wrapped = ('\n\n**Revision:** 2\n\nSuperseded by x\n\n' + '
Superseded plan\n\n' + payload + '\n\n
\n') + variants = [body, body + '\n', body.rstrip('\n'), body.replace('\n', '\r\n'), + wrapped, wrapped.replace('\n', '\r\n')] + self.assertEqual({digest(payload)}, {payload_digest(item) for item in variants}) + self.assertNotEqual(digest(body), payload_digest(body)) + self.assertNotEqual(payload_digest(body), payload_digest(body.replace('null', '{}'))) + + def test_legacy_digests_of_the_same_comment_are_recognized(self): + body = '\n\n**Revision:** 1\n\n## Implementation plan\n\nStep one.\n' + for stored in (digest(body), digest(body.rstrip('\n')), 'sha256:' + digest(body)): + with self.subTest(stored=stored): + self.assertTrue(legacy_identity(stored, body)) + # The pre-rule extracted payload, trailing newline kept. + self.assertTrue(legacy_identity(digest('## Implementation plan\n\nStep one.\n'), body)) + self.assertFalse(legacy_identity(digest(body + '\n'), body)) + # Wrapping makes a raw-body legacy digest unverifiable, so the fallback skips it. + wrapped = body.replace('## Implementation plan', 'Superseded by x\n\n
Old\n\n## Implementation plan') + '\n
\n' + self.assertEqual(payload_digest(body), payload_digest(wrapped)) + self.assertFalse(legacy_identity(digest(body), wrapped)) + self.assertFalse(legacy_identity(digest(body.replace('one', 'two')), body)) + + def test_runner_markers_keep_their_whole_body(self): + body = '\n\nBlocked: merge.\n' + self.assertEqual(digest(body.rstrip('\n')), payload_digest(body + '\n')) + wrapped = 'Superseded by x\n\n
Old\n\n' + body + '\n
\n' + self.assertEqual(payload_digest(body), payload_digest(wrapped)) + self.assertTrue(legacy_identity(digest(payload_text(wrapped)), wrapped)) + + def test_fast_path_revision_starts_at_its_heading(self): + payload = '## Fast path\n\nContract digest: abc\n\nEdit one file.' + body = ('\n\n**Revision:** 1\n' + '**Supersedes:** none\n**Replan report:** none\n\n' + payload + '\n') + self.assertEqual(digest(payload), payload_digest(body)) + with self.assertRaises(StopIteration): + payload_digest('\n\nNo heading.\n') + + def test_wrapper_without_its_closer_is_a_blocker(self): + body = ('\n\nSuperseded by x\n\n' + '
Old\n\n## Implementation plan\n\nStep one.\n' + '\nAppended instruction.\n') + with self.assertRaises(ValueError): + payload_digest(body) + + def test_payload_details_block_is_not_a_wrapper(self): + payload = '## Implementation plan\n\n
Notes\n\nKeep.\n\n
' + body = '\n\n**Revision:** 1\n\n' + payload + '\n' + self.assertEqual(digest(payload), payload_digest(body)) + + def test_eval_case_overlays_match_expected_digests(self): + cases = { + 'to-plan-payload-digest-direct': ('readback.json', 'Digest'), + 'run-github-project-payload-digest-novel': ('predecessor.json', 'Predecessor-digest'), + 'to-plan-payload-digest-restraint-negative': ('active-leaf.json', 'Digest'), + } + for case_id, (name, field) in cases.items(): + with self.subTest(case=case_id): + directory = self.root / 'evals/cases' / case_id + body = json.loads((directory / 'overlay' / name).read_text(encoding='utf-8'))['body'] + rules = json.loads((directory / 'expectations.json').read_text(encoding='utf-8')) + patterns = rules['file_globs'][0]['must_match'] + self.assertIn(f'^{re.escape(field)}: {payload_digest(body)}$', patterns) + novel = self.root / 'evals/cases/run-github-project-payload-digest-novel/overlay' + report = json.loads((novel / 'replan-request.json').read_text(encoding='utf-8'))['body'] + recorded = json.loads(report.split('```json\n')[1].split('\n```')[0])['previousPlanDigest'] + predecessor = json.loads((novel / 'predecessor.json').read_text(encoding='utf-8'))['body'] + self.assertEqual(recorded, payload_digest(predecessor)) + negative = self.root / 'evals/cases/to-plan-payload-digest-restraint-negative/overlay' + draft = (negative / '.scratch/plan-draft.md').read_text(encoding='utf-8') + leaf = json.loads((negative / 'active-leaf.json').read_text(encoding='utf-8'))['body'] + self.assertEqual(payload_digest(leaf), payload_digest(draft)) + self.assertNotEqual(digest(leaf), digest(draft)) + + if __name__ == '__main__': unittest.main() diff --git a/evals/tests/test_workflows_writing_matrix.py b/evals/tests/test_workflows_writing_matrix.py index eb7926b9..91fa4480 100644 --- a/evals/tests/test_workflows_writing_matrix.py +++ b/evals/tests/test_workflows_writing_matrix.py @@ -276,7 +276,7 @@ def test_has_skill_triads_and_workflow_calibration_coverage_without_routing(self benchmark = [case for case in report.cases if not case.calibration] calibration = [case for case in report.cases if case.calibration] self.assertEqual(38, len(benchmark)) - self.assertEqual(102, len(calibration)) + self.assertEqual(108, len(calibration)) self.assertIn("grounded-writing", PUBLIC_SKILLS) self.assertNotIn("implement", PUBLIC_SKILLS) self.assertEqual(38, len(filter_cases(report.cases, case_ids=None, skills=None))) @@ -385,6 +385,12 @@ def test_has_skill_triads_and_workflow_calibration_coverage_without_routing(self "run-github-project-preparation-direct", "deliver-spec-behavioral-dependency-novel", "run-github-project-shared-seam-negative", + "deliver-spec-nested-reviewer-direct", + "deliver-spec-nested-helpers-novel", + "run-github-project-background-owners-negative", + "to-plan-payload-digest-direct", + "run-github-project-payload-digest-novel", + "to-plan-payload-digest-restraint-negative", }, {case.id for case in calibration}, ) @@ -666,6 +672,9 @@ def test_automatic_conditions_exclude_explicit_only_workflow_skills(self): "run-github-project-preparation-direct", "deliver-spec-behavioral-dependency-novel", "run-github-project-shared-seam-negative", + "to-plan-payload-digest-direct", + "run-github-project-payload-digest-novel", + "to-plan-payload-digest-restraint-negative", }, {case.id for case, _ in conditions}, ) @@ -1582,6 +1591,98 @@ def check(values): result = check({**fields, key: f"{fields[key]}\n{key}: {unsafe[key]}"}) self.assertEqual(1, result.returncode, result.stderr) + def test_nested_agent_and_payload_digest_decisions_reject_unsafe_variants(self): + reports = { + "deliver-spec-nested-reviewer-direct": { + "Reviewer-launch": "foreground", "Wait": "same-turn", + "Return": "after-verdict", "Push": "after-ship-verdict", + }, + "deliver-spec-nested-helpers-novel": { + "Helpers": "foreground-one-message", + "Slice-workers": "foreground-one-message", + "Return": "after-integration-and-review", + }, + "run-github-project-background-owners-negative": { + "Ticket-owners": "background", "Evidence-helper": "background", + "Wait": "completion-notification", + }, + "to-plan-payload-digest-direct": { + "Digest": "e3367715a0ad1c0c4bded77bd84faa38d190f61fcee5dfd363e6a378eb87a7e5", + "Readback": "verified", + }, + "run-github-project-payload-digest-novel": { + "Predecessor-digest": "89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d", + "Replan-report": "matches", + }, + "to-plan-payload-digest-restraint-negative": { + "Digest": "b07d38664b960a4fa516ea8881abfa876dc3800ba06024f20c93f088b1b508db", + "Publication": "no-op", "Write": "none", + }, + } + # Each unsafe value is the decision or digest the observed failure produces. + unsafe = { + "deliver-spec-nested-reviewer-direct": { + "Reviewer-launch": "background", "Wait": "completion-notification", + "Return": "pending-review", "Push": "before-review", + }, + "deliver-spec-nested-helpers-novel": { + "Helpers": "background", "Slice-workers": "background", + "Return": "pending-workers", + }, + "run-github-project-background-owners-negative": { + "Ticket-owners": "foreground", "Evidence-helper": "foreground", "Wait": "poll", + }, + "to-plan-payload-digest-direct": { + "Digest": "42c657ca3e56807cf8972dfdf006dec7777155a0564f132be776c6d3a138b8c7", + "Readback": "mismatch", + }, + "run-github-project-payload-digest-novel": { + "Predecessor-digest": "103dae42f31ef781fbad9a7d90e4cfa61df40570015331723b84a0c89e425c3f", + "Replan-report": "mismatched", + }, + "to-plan-payload-digest-restraint-negative": { + "Digest": "3588a82491c459569f43cffa357293d49632572393c1f8f7b0ee828495e2e725", + "Publication": "new-revision", "Write": "create-comment", + }, + } + cases = {case.id: case for case in validate_corpus(REPO_ROOT, suite="workflows-writing").cases} + validator = REPO_ROOT / "evals/validators/text_case.py" + for case_id, fields in reports.items(): + case = cases[case_id] + self.assertTrue(case.calibration) + (artifact_path,) = case.allowed_write_paths + with tempfile.TemporaryDirectory() as directory: + workspace = Path(directory) + (workspace / "state.md").write_bytes( + (REPO_ROOT / "evals/fixtures/workflow/state.md").read_bytes() + ) + artifact = workspace / artifact_path + artifact.parent.mkdir() + + def check(values): + artifact.write_text( + "\n".join(f"{key}: {value}" for key, value in values.items()) + "\n", + encoding="utf-8", + ) + return subprocess.run( + [sys.executable, "-B", str(validator), case_id], + cwd=workspace, text=True, capture_output=True, check=False, + ) + + with self.subTest(case=case_id, variant="accepted"): + result = check(fields) + self.assertEqual(0, result.returncode, result.stderr) + for key in fields: + with self.subTest(case=case_id, missing=key): + result = check({name: value for name, value in fields.items() if name != key}) + self.assertEqual(1, result.returncode, result.stderr) + with self.subTest(case=case_id, changed=key): + result = check({**fields, key: unsafe[case_id][key]}) + self.assertEqual(1, result.returncode, result.stderr) + with self.subTest(case=case_id, contradictory=key): + result = check({**fields, key: f"{fields[key]}\n{key}: {unsafe[case_id][key]}"}) + self.assertEqual(1, result.returncode, result.stderr) + def test_plan_artifact_validator_rejects_cyclic_task_dependencies(self): validator = REPO_ROOT / "evals/validators/text_case.py" with tempfile.TemporaryDirectory() as temp_dir: diff --git a/evals/validators/plan_contract.py b/evals/validators/plan_contract.py index 5d9ee331..a4d08bcc 100644 --- a/evals/validators/plan_contract.py +++ b/evals/validators/plan_contract.py @@ -1,7 +1,7 @@ """Offline effective-contract checks; trusted inputs must be evaluator-owned. Records contain provider URL, author, exact body, and semantic digest. Full-plan -bodies are already extracted semantic payloads under their approved convention; +bodies are already extracted payloads under to-plan's payload digest rule; amendments contain one JSON fence. The caller authenticates that normalization. This does not establish live permissions, semantic compatibility or review quality. """ diff --git a/references/subagent-selection.md b/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/references/subagent-selection.md +++ b/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/deliver-spec/references/project-handoff.md b/skills/deliver-spec/references/project-handoff.md index 89d678ec..c96251dd 100644 --- a/skills/deliver-spec/references/project-handoff.md +++ b/skills/deliver-spec/references/project-handoff.md @@ -71,7 +71,9 @@ handoff or take over a Project claim. passes; resume the same context when actionable. Remote waiting consumes no active-agent capacity. 7. Return `ready`, `pending`, `blocked`, or `no-change` with exact evidence. - The controller alone claims, assigns, changes labels/Project Status, records + `pending` names only remote or controller-owned gates; never return while a + reviewer or helper you launched is still running. The controller alone + claims, assigns, changes labels/Project Status, records [ticket pauses](../../run-github-project/references/authority-and-pauses.md), manages capacity, merges, closes issues, and reconciles terminal state. Return missing grants and material decisions to that controller even when diff --git a/skills/deliver-spec/references/subagent-selection.md b/skills/deliver-spec/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/skills/deliver-spec/references/subagent-selection.md +++ b/skills/deliver-spec/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/gradle-run/references/subagent-selection.md b/skills/gradle-run/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/skills/gradle-run/references/subagent-selection.md +++ b/skills/gradle-run/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/implement-with-subagents/references/subagent-selection.md b/skills/implement-with-subagents/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/skills/implement-with-subagents/references/subagent-selection.md +++ b/skills/implement-with-subagents/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/run-github-project/references/authority-and-pauses.md b/skills/run-github-project/references/authority-and-pauses.md index 700d919d..b0761ad3 100644 --- a/skills/run-github-project/references/authority-and-pauses.md +++ b/skills/run-github-project/references/authority-and-pauses.md @@ -62,7 +62,7 @@ integrity blocker. completed work is preserved. Use nulls for absent artifacts. Keep secrets, local paths, and unsanitized logs out of the comment. 3. Refetch the comment and live ticket/artifact state. Verify the authenticated - runner author, identifiers, payload digest, and existing authority lease + runner author, identifiers, [payload digest](../../to-plan/references/github-mode.md#payload-digest), and existing authority lease when claimed. Reconcile ambiguous publication before retrying. Update the ticket only when that record is verified; report a failed write precisely and continue other safe work rather than claiming the update succeeded. diff --git a/skills/run-github-project/references/drain-scheduler.md b/skills/run-github-project/references/drain-scheduler.md index bee154bd..00519f9c 100644 --- a/skills/run-github-project/references/drain-scheduler.md +++ b/skills/run-github-project/references/drain-scheduler.md @@ -453,8 +453,8 @@ releasing the slot: preserved assignment and `In progress` Status. Never include local paths, secrets, or unsanitized logs. 2. Refetch the comment, assignment, Project Status, PR head, and checks. Require - the marker author to be the authenticated runner and compute a digest from - its immutable payload. Reconcile an ambiguous create before retrying. Keep + the marker author to be the authenticated runner and compute its + [payload digest](../../to-plan/references/github-mode.md#payload-digest). Reconcile an ambiguous create before retrying. Keep the slot occupied when the record cannot be verified. 3. Preserve the verified record permalink and digest with the branch, worktree, PR, failed-check evidence, and repair history. Release every named resource, diff --git a/skills/run-github-project/references/normalized-ticket.md b/skills/run-github-project/references/normalized-ticket.md index 3902f83f..1fbddc0f 100644 --- a/skills/run-github-project/references/normalized-ticket.md +++ b/skills/run-github-project/references/normalized-ticket.md @@ -79,7 +79,7 @@ or pause field for this ranker CLI. Use this shape for execution contenders: "commentId": "IC_plan", "permalink": "https://github.com/owner/repository/issues/42#issuecomment-1", "author": "octocat", - "digest": "sha256:plan-payload", + "digest": "", "createdAt": "2026-07-28T09:00:00Z", "publishedAt": "2026-07-28T09:00:00Z", "updatedAt": "2026-07-28T09:00:00Z", @@ -180,14 +180,14 @@ include these optional fields inside `wayfinderReconciliation`: ```json { - "payloadDigest": "sha256:original-marker-payload", + "payloadDigest": "", "configurationRenewal": { "commentId": "IC_renewal", "permalink": "https://github.com/owner/repository/issues/52#issuecomment-3", "author": "octocat", "markerCommentId": "IC_reconciliation", "markerPermalink": "https://github.com/owner/repository/issues/52#issuecomment-2", - "markerPayloadDigest": "sha256:original-marker-payload", + "markerPayloadDigest": "", "originalConfigurationDigest": "sha256:configuration", "configurationDigest": "sha256:renewed-configuration" } @@ -195,9 +195,9 @@ include these optional fields inside `wayfinderReconciliation`: ``` Keep the marker's original `configurationDigest` unchanged. Compute -`payloadDigest` from its freshly read semantic payload, excluding only a -presentation wrapper; retain every identity, configuration value and planned -mutation. The normalized renewal binds that payload, comment ID and permalink +`payloadDigest` from its freshly read marker body under the +[payload digest](../../to-plan/references/github-mode.md#payload-digest) rule; retain every identity, configuration value and +planned mutation. The normalized renewal binds that payload, comment ID and permalink to its original digest and the current CLI configuration digest. Require all fields nonempty and its author to be the authenticated runner. A supplied malformed, foreign, stale or mismatched renewal blocks recovery even when the @@ -274,9 +274,12 @@ normalize it into `implementationPlans`. Require runner authorship. A v1 comment is a revision-one root. A v2 comment records its positive `revision`, predecessor permalink in `supersedes`, triggering report permalink in `replanRequest` when applicable, -and payload publication time in `publishedAt`. Compute `digest` from the -semantic plan payload, excluding any superseded banner or `
` wrapper, -so presentation-only edits do not invalidate history. +and payload publication time in `publishedAt`. Compute `digest` under the +[payload digest](../../to-plan/references/github-mode.md#payload-digest) rule, which +excludes the marker, revision metadata and any superseded wrapper, so +presentation-only edits do not invalidate history. Normalize a verified legacy +`previousPlanDigest` in a replan report to that rule's digest under the same +rule's legacy path, so an older report still matches its plan. Require one root, contiguous revisions, one child per revision, and one unminimized leaf. The ranker returns that leaf as the synthesized @@ -309,7 +312,7 @@ For an automatic requeue, normalize the verified report comment as: "createdAt": "2026-07-28T11:00:00Z", "disposition": "autonomous-replan", "previousPlanPermalink": "https://github.com/owner/repository/issues/42#issuecomment-1", - "previousPlanDigest": "sha256:plan-payload", + "previousPlanDigest": "", "baseSha": "0123456789abcdef", "implementationHeadSha": "fedcba9876543210", "pullRequestUrl": null diff --git a/skills/run-github-project/references/project-config.md b/skills/run-github-project/references/project-config.md index dbf847e2..49b39105 100644 --- a/skills/run-github-project/references/project-config.md +++ b/skills/run-github-project/references/project-config.md @@ -196,8 +196,8 @@ and for active, planning, paused, parked and Wayfinder recovery leases. old/new commits and configuration digests, stable IDs and old/new names, complete comparison and policy evidence, lease/artifact identities and predecessor renewal permalink/digest (or original lease for the root). - Bind every retained historical marker by comment ID, permalink and semantic - payload digest. Never edit that marker's original configuration digest. + Bind every retained historical marker by comment ID, permalink and + [payload digest](../../to-plan/references/github-mode.md#payload-digest). Never edit that marker's original configuration digest. Reconcile an ambiguous create before retrying; refetch and verify exact author, body, payload digest and a single unforked, gap-free renewal lineage. 5. Renew the controller's effective leases only after every affected record is diff --git a/skills/run-github-project/references/subagent-selection.md b/skills/run-github-project/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/skills/run-github-project/references/subagent-selection.md +++ b/skills/run-github-project/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/run-github-project/references/todo-lane.md b/skills/run-github-project/references/todo-lane.md index 0315ae75..af660670 100644 --- a/skills/run-github-project/references/todo-lane.md +++ b/skills/run-github-project/references/todo-lane.md @@ -36,7 +36,7 @@ deliver-spec's [fast path](../../deliver-spec/references/fast-path.md). When in doubt, plan it normally. 3. In the controller lane, publish one runner-authored `` comment with revision one, - `Supersedes: none`, `Replan report: none`, a `Fast path` heading, the + `Supersedes: none`, `Replan report: none`, a `## Fast path` heading, the planned branch and base SHA, the classification evidence, the issue contract digest and the brief plan. Verify it as in step 7 below, reconciling an ambiguous create by exact revision and digest. @@ -88,13 +88,16 @@ deliver-spec's [fast path](../../deliver-spec/references/fast-path.md). a new v2 revision when the substantive plan changed, or return the identical active leaf as a no-op. Never edit a semantic plan payload in place. 7. Refetch all marker comments and verify the unique active leaf, revision - chain, authenticated-runner author, payload digest, permalink, planned + chain, authenticated-runner author, [payload digest](../../to-plan/references/github-mode.md#payload-digest), permalink, planned branch, planned SHA, replan-report link, and timestamps. After a new leaf is verified, minimize its predecessor as `OUTDATED`. When native minimization - is unavailable, prepend a superseded banner and wrap the unchanged payload - in `
`, then refetch and verify its payload digest. If both - presentation operations fail after bounded reconciliation, report the - hygiene failure but continue because the chain is authoritative. Treat + is unavailable, add a superseded banner and `
` opening before its + first payload line (`## Implementation plan` or `## Fast path`) and a final + `
` line, leaving the payload unchanged, then refetch and verify + its payload digest. Skip that edit when a record names the predecessor by + a legacy digest, which the edit would make unverifiable. If both + presentation operations fail or are skipped after bounded reconciliation, + report the hygiene failure but continue because the chain is authoritative. Treat missing `to-plan` as an issue-local planning blocker; it must not block implementation items with current plans. 8. Move the item to `Ready to implement` and refetch its current Status, diff --git a/skills/run-github-project/references/wayfinder-lane.md b/skills/run-github-project/references/wayfinder-lane.md index 497bfb11..1f6030a0 100644 --- a/skills/run-github-project/references/wayfinder-lane.md +++ b/skills/run-github-project/references/wayfinder-lane.md @@ -84,7 +84,7 @@ is the canonical Wayfinder session boundary. The drain controller may continue with another AFK child only through another fresh context. For a research child, require the provider to invoke the installed `research` -skill in a background subagent. Give it the ticket question and map Notes, +skill in a separate subagent. Give it the ticket question and map Notes, require primary-source citations in one repository Markdown artifact on a throwaway `research/` branch in an isolated research worktree, and return its linked context pointer to the Wayfinder owner. A drain invocation may diff --git a/skills/run-github-project/references/workflow-providers.md b/skills/run-github-project/references/workflow-providers.md index f3049e32..5ed347b6 100644 --- a/skills/run-github-project/references/workflow-providers.md +++ b/skills/run-github-project/references/workflow-providers.md @@ -43,7 +43,7 @@ continue ordinary planning and execution. Never install it implicitly or copy its decision-map procedure into this workflow. `research` is required only for a Wayfinder research child. The Wayfinder owner -must invoke it in a background subagent; a generic helper is not an equivalent +must invoke it in a separate subagent; a generic helper is not an equivalent provider. If it is unavailable, block only research children and continue other authorized lanes. Never install it implicitly. diff --git a/skills/shepherd/references/subagent-selection.md b/skills/shepherd/references/subagent-selection.md index b9cb103f..cda84ab5 100644 --- a/skills/shepherd/references/subagent-selection.md +++ b/skills/shepherd/references/subagent-selection.md @@ -89,8 +89,13 @@ log. caller grants further delegation. - With no user model-selection policy, leave `model` unset so it is inherited; set effort only when the user asks. -- Run workers in the background and wait for the completion notification; do - not poll or sleep. +- A top-level session runs workers in the background and waits for the + completion notification; do not poll or sleep. +- A subagent receives no completion notifications; they go to the top-level + session. When it launches an agent whose result it needs before returning, + such as a reviewer, evidence or discovery helper, or slice worker, it passes + `run_in_background: false` and puts independent launches in one message to + keep them parallel. - Before dispatching background writers, list the commit and validation commands they will run and have the user approve them. A worker denied permission anyway reports a blocker and does not seek a workaround. diff --git a/skills/to-plan/references/github-mode.md b/skills/to-plan/references/github-mode.md index 5b04ab9a..df76125d 100644 --- a/skills/to-plan/references/github-mode.md +++ b/skills/to-plan/references/github-mode.md @@ -109,10 +109,45 @@ Plan comments are the only GitHub state this skill may mutate. Never change the issue body, labels, assignee, relationships, Project fields, status, or any non-plan comment. -Compute a semantic payload digest without the marker, revision metadata, or -superseded wrapper. The complete `Epoch closure` line, including retained-work -and evidence dispositions, is semantic payload: always include it in the digest, -never discard it as revision metadata. If the active leaf already has the same payload and +### Payload digest + +Every full-plan and runner-marker payload digest uses this one form, so any +agent and any tool reproduces it: + +1. Start from the comment's `body` string as decoded from the API JSON. +2. Replace each CRLF with LF. +3. Find the first payload line: in a `to-plan:implementation-plan` comment, + the first `## Implementation plan` or `## Fast path` line; otherwise the + marker line. A plan comment with neither line is a blocker. Drop every line + before it (marker, revision metadata, superseded banner or `
` + opening). If a dropped line opened `
`, the last nonempty line must + be `
`: drop it, and treat a missing closer as a blocker. +4. Strip all trailing LF characters. This absorbs the newline that tools such + as `jq -r` add. +5. Hash the UTF-8 bytes with SHA-256 and record lowercase hexadecimal with no + prefix. + +For example, the body +`\r\n\r\n**Revision:** 1\r\n\r\n## Implementation plan\r\n\r\nStep one.\r\n` +has the payload `## Implementation plan\n\nStep one.` and the digest +`bcce4e2c20ff2a6fa053e9560880d1491c6458c51ed25e4ad6aa0b847d6c4da7`. + +A record published before this rule may carry another digest of the same +comment. After verifying that comment's author and chain, accept the stored +value as a legacy identity only when it equals the SHA-256 hexadecimal, with +or without a `sha256:` prefix, of the decoded body, raw or after steps 2 and 3, +either as is or with trailing LF characters stripped. When normalizing, replace a verified legacy value with +this rule's digest. A published amendment chain keeps the plan digest it +pinned. Any other mismatch stays a blocker, and new records carry only this +rule's digest. + +The complete `Epoch closure` line, including retained-work and evidence +dispositions, is semantic payload: always include it in the digest, never +discard it as revision metadata. + +### Publication + +If the active leaf already has the same payload and baseline, perform no write, delete only an exact matching temporary draft after verification, and return the leaf as a no-op. Otherwise: @@ -126,9 +161,12 @@ verification, and return the leaf as a no-op. Otherwise: 3. Require one root, no fork or gap, and the new comment as the unique unminimized leaf. 4. Minimize the predecessor as `OUTDATED`. If unavailable, edit only that - runner-owned predecessor to add a superseded-by link and wrap its unchanged - payload in `
`, then verify its digest. Report failure of both - presentation methods without invalidating the verified new leaf. + runner-owned predecessor to add a superseded-by link and `
` opening + before its first payload line (`## Implementation plan` or `## Fast path`) + and a final `
` line, leaving the payload unchanged, then verify + its digest. Skip that edit when a record names the predecessor by a legacy + digest, which the edit would make unverifiable. Report failure or a skip of + both presentation methods without invalidating the verified new leaf. 5. Delete only the exact draft after the active leaf is verified. An amendment is a separate append-only component of one effective contract, diff --git a/skills/to-plan/references/integration-amendments.md b/skills/to-plan/references/integration-amendments.md index 043fa17f..f47ec017 100644 --- a/skills/to-plan/references/integration-amendments.md +++ b/skills/to-plan/references/integration-amendments.md @@ -126,8 +126,9 @@ reconciliation, not an automatic retry or silently refreshed hash. 1. Enumerate all full-plan and amendment marker comments, including minimized history. Apply GitHub mode's existing v1/v2 plan-chain checks. Pin the current - full-plan revision and semantic digest using its existing approved digest - convention; do not retroactively rehash or rewrite a legacy plan. + full-plan revision and its [payload digest](github-mode.md#payload-digest); + an existing chain keeps the verified legacy plan digest it pinned. Do not + retroactively rehash or rewrite a legacy plan. 2. Define the new amendment's canonical semantic payload as a JSON object with all fields in the template other than publication metadata, review record, and the two self-referential digests. Include source manifest, full-plan