|
| 1 | +#!/usr/bin/env bash |
| 2 | +# SPDX-License-Identifier: MPL-2.0 |
| 3 | +# Acceptance checks for the documented contract, not runtime enforcement. |
| 4 | +# Offline: reads documentation only; never calls GitHub or enables the fleet. |
| 5 | +# Run with: bash tests/review-thread-stewardship-test.sh |
| 6 | +set -euo pipefail |
| 7 | +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" |
| 8 | + |
| 9 | +# Limit assertions to the relevant section so another bot or metarule cannot |
| 10 | +# accidentally satisfy them. Collapse wrapping without snapshotting whole docs. |
| 11 | +section() { |
| 12 | + local file="$1" heading="$2" level="$3" |
| 13 | + awk -v heading="$heading" -v level="$level" ' |
| 14 | + $0 == heading { found = 1; next } |
| 15 | + found && $0 ~ "^" level " " { exit } |
| 16 | + found { print } |
| 17 | + ' "$file" |
| 18 | +} |
| 19 | +normalize() { awk '{$1=$1; printf "%s ", $0}'; } |
| 20 | +require() { |
| 21 | + local label="$1" pattern="$2" content="$3" |
| 22 | + if ! grep -Eq -- "$pattern" <<< "$content"; then |
| 23 | + printf 'FAIL: %s\n' "$label" >&2 |
| 24 | + exit 1 |
| 25 | + fi |
| 26 | + printf 'PASS: %s\n' "$label" |
| 27 | +} |
| 28 | + |
| 29 | +operations="$(section "$root/docs/BOT-OPERATIONS.adoc" \ |
| 30 | + '==== Review-Thread Stewardship (metarule)' '====')" |
| 31 | +tokens="$(section "$root/docs/TOKEN-ARCHITECTURE.adoc" \ |
| 32 | + '=== 3. echidnabot' '===')" |
| 33 | + |
| 34 | +# Keep each guard in its own numbered step; a disclaimer elsewhere must not |
| 35 | +# conceal a weakened matching, resolution, or invalid-finding instruction. |
| 36 | +step() { |
| 37 | + local number="$1" |
| 38 | + awk -v number="$number" ' |
| 39 | + /^[0-9]+\. / { |
| 40 | + if (found) exit |
| 41 | + if ($1 == number ".") found = 1 |
| 42 | + } |
| 43 | + found && /^\.\.\.\.$/ { exit } |
| 44 | + found { print } |
| 45 | + ' <<< "$operations" | normalize |
| 46 | +} |
| 47 | +match_step="$(step 1)" |
| 48 | +fix_step="$(step 2)" |
| 49 | +reply_step="$(step 3)" |
| 50 | +resolve_step="$(step 4)" |
| 51 | +invalid_step="$(step 5)" |
| 52 | + |
| 53 | +require 'candidate filtering uses the same PR, file/line class and unresolved threads' \ |
| 54 | + 'same PR and file/line class only to filter candidate unresolved' "$match_step" |
| 55 | +require 'candidate filtering requires a verified finding-to-thread match' \ |
| 56 | + 'Verify the finding-to-thread match' "$match_step" |
| 57 | +require 'reply and resolution require exactly one match' \ |
| 58 | + 'exactly one match before replying or resolving' "$match_step" |
| 59 | +require 'zero or ambiguous matches leave every thread unchanged' \ |
| 60 | + 'Leave all threads unchanged if zero or multiple matches' "$match_step" |
| 61 | +require 'the executor lands the fix on the PR branch' \ |
| 62 | + 'fix lands on the PR branch \(robot-repo-automaton' "$fix_step" |
| 63 | +require 'identity findings cannot be automatically fixed' \ |
| 64 | + 'HYP-S010 are auto_fixable=false' "$fix_step" |
| 65 | +require 'identity direction belongs to the owner' \ |
| 66 | + 'bot proposes and replies, the owner rules on direction' "$fix_step" |
| 67 | +require 'echidnabot replies with both the fix commit and reasoning' \ |
| 68 | + 'echidnabot replies to the thread with the fix commit and reasoning' "$reply_step" |
| 69 | + |
| 70 | +# REST takes a top-level comment ID; GraphQL takes a review-thread node ID. |
| 71 | +reply_endpoint='POST /repos/{owner}/{repo}/pulls/{pr}/comments/{id}/replies' |
| 72 | +require 'operations document the REST reply endpoint' \ |
| 73 | + "$(sed 's/[][\\.^$*+?(){}|]/\\&/g' <<< "$reply_endpoint")" "$reply_step" |
| 74 | +require 'REST replies use the top-level review-comment ID' \ |
| 75 | + 'where \{id\} is the top-level review-comment ID' "$reply_step" |
| 76 | +require 'resolution follows both a landed fix and a posted reply' \ |
| 77 | + "only after the required fix lands and Step 3's reply is posted" "$resolve_step" |
| 78 | +require 'resolution uses a GraphQL thread ID from the PR' \ |
| 79 | + "GraphQL mutation resolveReviewThread \(thread id from the PR's reviewThreads\)" "$resolve_step" |
| 80 | +require 'identity threads wait for owner direction, approval and a landed fix' \ |
| 81 | + 'HYP-S010 stay open until the owner directs and approves a fix and that fix lands' "$resolve_step" |
| 82 | +require 'invalid findings take the skip path instead of normal resolution' \ |
| 83 | + 'For verified-invalid findings, follow Step 5 instead' "$resolve_step" |
| 84 | +require 'unaddressed findings must never be resolved' \ |
| 85 | + 'Never resolve a thread whose finding is not addressed' "$invalid_step" |
| 86 | +require 'invalid findings receive a skip explanation and remain unresolved' \ |
| 87 | + 'verified as no-longer-valid, post a "skipped with a brief reason" reply explaining why and leave the thread unresolved' "$invalid_step" |
| 88 | +require 'alignment findings require checking for template residue first' \ |
| 89 | + 'check which side is the residue before automating' "$(normalize <<< "$operations")" |
| 90 | + |
| 91 | +reply_row="$(awk '/^\| Reply to a review thread \|/ {print}' <<< "$tokens")" |
| 92 | +resolve_row="$(awk '/^\| Resolve a review thread \|/ {print}' <<< "$tokens")" |
| 93 | +require 'echidnabot owns exactly one reply action and one resolve action' \ |
| 94 | + '^1 1$' "$(grep -c '^| Reply to a review thread |' <<< "$tokens") $(grep -c '^| Resolve a review thread |' <<< "$tokens")" |
| 95 | +require 'token docs use the same REST endpoint as operations' \ |
| 96 | + "$(sed 's/[][\\.^$*+?(){}|]/\\&/g' <<< "${reply_endpoint//\{id\}/\{comment_id\}}")" "$reply_row" |
| 97 | +require 'token docs distinguish REST comment IDs from GraphQL thread IDs' \ |
| 98 | + 'top-level review-comment ID, distinct from the GraphQL review-thread ID' "$reply_row" |
| 99 | +require 'review replies require no additional token scope' \ |
| 100 | + 'no new scope, `repo` covers it' "$reply_row" |
| 101 | +require 'token docs name the GraphQL mutation and threadId argument' \ |
| 102 | + 'mutation resolveReviewThread\(input: \{threadId\}\)' "$resolve_row" |
| 103 | +require 'token docs restrict resolution to addressed findings after the reply' \ |
| 104 | + 'resolves only threads whose finding the fleet addressed, with the fix-commit reply posted first' "$resolve_row" |
| 105 | +require 'operations link to the token permission document' \ |
| 106 | + 'link:TOKEN-ARCHITECTURE\.adoc\[' "$(normalize <<< "$operations")" |
| 107 | +require 'token reply action refers back to the operations contract' \ |
| 108 | + 'BOT-OPERATIONS\.adoc' "$reply_row" |
![coderabbitai[bot]](https://avatars.githubusercontent.com/in/347564?v=4&size=40){kind=avatar}
0 commit comments