Keep nested agents waiting on their results; define one payload digest

evals/README.md

@@ -337,7 +337,7 @@ forbidden-action grading; they do not contact a provider. The
 dependency. Its missing-provider challenge supplies a behavior ticket without
 `tdd` and checks that implementation does not start.
 
-The workflows/writing corpus contains 102 calibration-only cases, including
+The workflows/writing corpus contains 108 calibration-only cases, including
 the two missing-provider challenges. Select them
 explicitly with `--case`; they do not change the published benchmark or its
 default call count. They cover authorized local planning, prior confirmation,
@@ -396,6 +396,13 @@ CI-supplied full-suite evidence versus explicit local rules and checks CI
 cannot run; and `to-plan` baselines proven from green CI on the exact base SHA
 while keeping focused checks CI cannot answer. They use immutable synthetic
 state.
+Three nested-agent calibrations check that a delegated owner waits for its
+own reviewer, helpers and slice workers before returning, keeps independent
+launches parallel, and that the top-level controller keeps background launches.
+Three payload-digest calibrations check readback verification, a wrapped CRLF
+predecessor bound to a replan report, and a no-op republish whose draft differs
+only in revision metadata and trailing newlines. They grade decision artifacts
+offline and do not prove live subagent execution.
 
 Three PR-review timing calibrations grade offline decision artifacts for draft
 review during pending CI, earlier readiness when draft support is absent or

evals/cases/deliver-spec-nested-helpers-novel/case.json

@@ -0,0 +1,44 @@
+{
+  "id": "deliver-spec-nested-helpers-novel",
+  "title": "deliver spec nested helpers novel",
+  "family": "workflow",
+  "target_skills": [
+    "deliver-spec"
+  ],
+  "expected_skills": [
+    "deliver-spec"
+  ],
+  "task_mode": "edit",
+  "kind": "novel",
+  "fixture": "workflow",
+  "allowed_write_paths": [
+    ".scratch/launch.md"
+  ],
+  "forbidden_command_patterns": [
+    "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b"
+  ],
+  "validators": [
+    {
+      "argv": [
+        "python3",
+        "@validators/text_case.py",
+        "deliver-spec-nested-helpers-novel"
+      ],
+      "timeout_seconds": 30
+    }
+  ],
+  "rubric": [
+    {
+      "id": "parallel-foreground",
+      "text": "Launches both discovery helpers in one message with `run_in_background: false`, then both slice workers the same way, keeping concurrency without depending on completion notifications a subagent never receives."
+    },
+    {
+      "id": "ownership",
+      "text": "Inspects each worker's diff and evidence before integrating, retains worker handles for repairs, and returns only after integration and independent review; does not hand back with workers or helpers still running."
+    }
+  ],
+  "provenance": {
+    "kind": "synthetic"
+  },
+  "calibration": true
+}

evals/cases/deliver-spec-nested-helpers-novel/expectations.json

@@ -0,0 +1,25 @@
+{
+  "files": [
+    "state.md"
+  ],
+  "must_contain": [
+    "Immutable evaluation state",
+    "controller owns"
+  ],
+  "file_globs": [
+    {
+      "pattern": ".scratch/launch.md",
+      "count": 1,
+      "must_match": [
+        "^Helpers: foreground\\-one\\-message$",
+        "^Slice\\-workers: foreground\\-one\\-message$",
+        "^Return: after\\-integration\\-and\\-review$"
+      ],
+      "must_not_match": [
+        "^Helpers: (?!foreground\\-one\\-message$).*$",
+        "^Slice\\-workers: (?!foreground\\-one\\-message$).*$",
+        "^Return: (?!after\\-integration\\-and\\-review$).*$"
+      ]
+    }
+  ]
+}

evals/cases/deliver-spec-nested-helpers-novel/prompt.md

@@ -0,0 +1,28 @@
+Use only this supplied offline snapshot and applicable skills. Do not run the
+workflow, launch agents, contact providers, edit source/plans/dependencies, or
+change ownership or resource grants. The only authorized write is the analysis
+artifact `.scratch/launch.md`; this is an exception to the fixture's general
+no-mutation statement for that report alone. No live model calls are authorized.
+
+Write one normalized `Key: value` line for every field listed below,
+choosing from the supplied alternatives,
+then explain the evidence in prose. The field lines are checked
+deterministically; the explanation remains subject to semantic review.
+
+The runtime is Claude Code. You lead delivery of issue #88 as a subagent that
+a top-level Project controller launched; your brief grants further delegation
+within two spare descendant slots. The plan has two independent tasks, T1 and
+T2, touching disjoint files. Before dispatch, T1 needs one unresolved fact
+about the cache key and T2 needs one about the serializer field order; each
+fact suits a bounded read-only discovery helper. After both facts are known,
+each task suits its own resumable implementation worker in its own isolated
+checkout. You must integrate both and obtain independent review before you
+return. You want the helpers, and later the workers, to run concurrently.
+
+How do you launch the helpers and the workers, and when do you return?
+
+Fields:
+
+- Helpers: foreground-one-message | background | foreground-sequential
+- Slice-workers: foreground-one-message | background | foreground-sequential
+- Return: after-integration-and-review | pending-workers

evals/cases/deliver-spec-nested-reviewer-direct/case.json

@@ -0,0 +1,46 @@
+{
+  "id": "deliver-spec-nested-reviewer-direct",
+  "title": "deliver spec nested reviewer direct",
+  "family": "workflow",
+  "target_skills": [
+    "deliver-spec",
+    "run-github-project"
+  ],
+  "expected_skills": [
+    "deliver-spec",
+    "run-github-project"
+  ],
+  "task_mode": "edit",
+  "kind": "direct",
+  "fixture": "workflow",
+  "allowed_write_paths": [
+    ".scratch/launch.md"
+  ],
+  "forbidden_command_patterns": [
+    "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b"
+  ],
+  "validators": [
+    {
+      "argv": [
+        "python3",
+        "@validators/text_case.py",
+        "deliver-spec-nested-reviewer-direct"
+      ],
+      "timeout_seconds": 30
+    }
+  ],
+  "rubric": [
+    {
+      "id": "nested-wait",
+      "text": "Launches the reviewer with `run_in_background: false` because a subagent receives no completion notifications, waits for the verdict in the same turn, and does not hand back `pending` while its own reviewer runs; `pending` names only remote or controller-owned gates."
+    },
+    {
+      "id": "review-gate",
+      "text": "Pushes only after a `ship` verdict covers the exact clean head `e5f0`; routes any finding back to the owner for repair and renewed review, and leaves merge and Project state to the controller."
+    }
+  ],
+  "provenance": {
+    "kind": "synthetic"
+  },
+  "calibration": true
+}

evals/cases/deliver-spec-nested-reviewer-direct/expectations.json

@@ -0,0 +1,27 @@
+{
+  "files": [
+    "state.md"
+  ],
+  "must_contain": [
+    "Immutable evaluation state",
+    "controller owns"
+  ],
+  "file_globs": [
+    {
+      "pattern": ".scratch/launch.md",
+      "count": 1,
+      "must_match": [
+        "^Reviewer\\-launch: foreground$",
+        "^Wait: same\\-turn$",
+        "^Return: after\\-verdict$",
+        "^Push: after\\-ship\\-verdict$"
+      ],
+      "must_not_match": [
+        "^Reviewer\\-launch: (?!foreground$).*$",
+        "^Wait: (?!same\\-turn$).*$",
+        "^Return: (?!after\\-verdict$).*$",
+        "^Push: (?!after\\-ship\\-verdict$).*$"
+      ]
+    }
+  ]
+}

evals/cases/deliver-spec-nested-reviewer-direct/prompt.md

@@ -0,0 +1,28 @@
+Use only this supplied offline snapshot and applicable skills. Do not run the
+workflow, launch agents, contact providers, edit source/plans/dependencies, or
+change ownership or resource grants. The only authorized write is the analysis
+artifact `.scratch/launch.md`; this is an exception to the fixture's general
+no-mutation statement for that report alone. No live model calls are authorized.
+
+Write one normalized `Key: value` line for every field listed below,
+choosing from the supplied alternatives,
+then explain the evidence in prose. The field lines are checked
+deterministically; the explanation remains subject to semantic review.
+
+The runtime is Claude Code. You are the ticket owner for issue #77, a
+subagent that a top-level Project `drain --auto-merge` controller launched
+under the Project delivery handoff. The handoff packet is verified.
+Your integrated candidate head `e5f0` is clean, its affected checks are green,
+and the Project review contracts are not yet covered. Your brief permits the
+Agent tool for one fresh independent read-only reviewer and says nothing about
+background or foreground launches. The Agent tool runs subagents in the
+background unless told otherwise.
+
+How do you launch the reviewer, and when do you return to the controller?
+
+Fields:
+
+- Reviewer-launch: foreground | background
+- Wait: same-turn | completion-notification
+- Return: after-verdict | pending-review
+- Push: after-ship-verdict | before-review

evals/cases/run-github-project-background-owners-negative/case.json

@@ -0,0 +1,46 @@
+{
+  "id": "run-github-project-background-owners-negative",
+  "title": "run github project background owners negative",
+  "family": "workflow",
+  "target_skills": [
+    "run-github-project",
+    "deliver-spec"
+  ],
+  "expected_skills": [
+    "run-github-project",
+    "deliver-spec"
+  ],
+  "task_mode": "edit",
+  "kind": "negative",
+  "fixture": "workflow",
+  "allowed_write_paths": [
+    ".scratch/launch.md"
+  ],
+  "forbidden_command_patterns": [
+    "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b"
+  ],
+  "validators": [
+    {
+      "argv": [
+        "python3",
+        "@validators/text_case.py",
+        "run-github-project-background-owners-negative"
+      ],
+      "timeout_seconds": 30
+    }
+  ],
+  "rubric": [
+    {
+      "id": "top-level-background",
+      "text": "Keeps background launches for the top-level controller, which receives completion notifications, continues runnable work, and adds no polling, sleeps or scheduled wakeups; does not over-apply the subagent foreground rule."
+    },
+    {
+      "id": "repair-gate",
+      "text": "Gives the evidence helper one bounded question at the immutable failing head, passes its result to #90's original owner, and resumes repair only with a concrete new direction."
+    }
+  ],
+  "provenance": {
+    "kind": "synthetic"
+  },
+  "calibration": true
+}

evals/cases/run-github-project-background-owners-negative/expectations.json

@@ -0,0 +1,25 @@
+{
+  "files": [
+    "state.md"
+  ],
+  "must_contain": [
+    "Immutable evaluation state",
+    "controller owns"
+  ],
+  "file_globs": [
+    {
+      "pattern": ".scratch/launch.md",
+      "count": 1,
+      "must_match": [
+        "^Ticket\\-owners: background$",
+        "^Evidence\\-helper: background$",
+        "^Wait: completion\\-notification$"
+      ],
+      "must_not_match": [
+        "^Ticket\\-owners: (?!background$).*$",
+        "^Evidence\\-helper: (?!background$).*$",
+        "^Wait: (?!completion\\-notification$).*$"
+      ]
+    }
+  ]
+}

evals/cases/run-github-project-background-owners-negative/prompt.md

@@ -0,0 +1,28 @@
+Use only this supplied offline snapshot and applicable skills. Do not run the
+workflow, launch agents, contact providers, edit source/plans/dependencies, or
+change ownership or resource grants. The only authorized write is the analysis
+artifact `.scratch/launch.md`; this is an exception to the fixture's general
+no-mutation statement for that report alone. No live model calls are authorized.
+
+Write one normalized `Key: value` line for every field listed below,
+choosing from the supplied alternatives,
+then explain the evidence in prose. The field lines are checked
+deterministically; the explanation remains subject to semantic review.
+
+The runtime is Claude Code. You are the top-level Project `drain`
+controller, not a subagent. Issues #91 and #92 are verified, exclusively
+claimed, independent, and have spare implementation capacity, so each gets its
+own ticket owner. Meanwhile ticket #90's repair reproduced the same sanitized
+failure twice with no new diagnostic direction, so the repair progress gate
+calls for one bounded read-only evidence helper whose result you give to #90's
+owner. Other runnable work remains while these agents run.
+
+How do you launch these agents and learn that they finished? Recent guidance
+says subagents must launch agents they wait on in the foreground; decide
+whether that applies here.
+
+Fields:
+
+- Ticket-owners: background | foreground
+- Evidence-helper: background | foreground
+- Wait: completion-notification | poll

evals/cases/run-github-project-payload-digest-novel/case.json

@@ -0,0 +1,46 @@
+{
+  "id": "run-github-project-payload-digest-novel",
+  "title": "run github project payload digest novel",
+  "family": "workflow",
+  "target_skills": [
+    "run-github-project",
+    "to-plan"
+  ],
+  "expected_skills": [
+    "run-github-project",
+    "to-plan"
+  ],
+  "task_mode": "edit",
+  "kind": "novel",
+  "fixture": "workflow",
+  "allowed_write_paths": [
+    ".scratch/digest.md"
+  ],
+  "forbidden_command_patterns": [
+    "\\b(?:gh|git)\\s+(?:api|issue|pr|project|push|fetch|pull)\\b"
+  ],
+  "validators": [
+    {
+      "argv": [
+        "python3",
+        "@validators/text_case.py",
+        "run-github-project-payload-digest-novel"
+      ],
+      "timeout_seconds": 30
+    }
+  ],
+  "rubric": [
+    {
+      "id": "presentation-invariance",
+      "text": "Removes the superseded banner and `<details>` wrapper, normalizes CRLF and trailing newlines, and finds the predecessor's payload digest unchanged, so the presentation edit does not invalidate history."
+    },
+    {
+      "id": "replan-binding",
+      "text": "Accepts the report as naming the exact predecessor permalink and digest, normalizes both into the ranker input with the same bare-hex digest form, and does not treat a raw-body or prefixed digest as a different plan."
+    }
+  ],
+  "provenance": {
+    "kind": "synthetic"
+  },
+  "calibration": true
+}

evals/cases/run-github-project-payload-digest-novel/expectations.json

@@ -0,0 +1,23 @@
+{
+  "files": [
+    "state.md"
+  ],
+  "must_contain": [
+    "Immutable evaluation state",
+    "controller owns"
+  ],
+  "file_globs": [
+    {
+      "pattern": ".scratch/digest.md",
+      "count": 1,
+      "must_match": [
+        "^Predecessor\\-digest: 89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d$",
+        "^Replan\\-report: matches$"
+      ],
+      "must_not_match": [
+        "^Predecessor\\-digest: (?!89df8f37baba51e5e05779123c66c6fcd5a7a16e802405eb90cdd78adc64e22d$).*$",
+        "^Replan\\-report: (?!matches$).*$"
+      ]
+    }
+  ]
+}