diff --git a/.github/workflows/actions.lock b/.github/workflows/actions.lock index 98a4ad877..0698ea837 100644 --- a/.github/workflows/actions.lock +++ b/.github/workflows/actions.lock @@ -37,6 +37,10 @@ workflows: - 'haskell-actions/setup@0f8e8c99d88aeb3fbfd523f1ef2c6f762d10d64d' - 'hyperpolymath/standards@d55304a01b5260b0511aae0d14ed8b8770b00293' - 'oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6' + '.github/workflows/code-coverage.yml': + - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' + - 'actions/upload-code-coverage@2b21a77928be8d5168c2b9581a67f2adbebacc52' + - 'dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8' '.github/workflows/codeql-reusable.yml': - 'actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1' - 'github/codeql-action@b96794f015dfd88f77b49b1c93e0fa7110f94c63' @@ -235,6 +239,11 @@ dependencies: repo_id: 496012378 uses: - 'actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f' + 'actions/upload-code-coverage@2b21a77928be8d5168c2b9581a67f2adbebacc52': + ref: 'v1.4.4' + commit: 'sha1-2b21a77928be8d5168c2b9581a67f2adbebacc52' + owner_id: 44036562 + repo_id: 1220171870 'denoland/setup-deno@22d081ff2d3a40755e97629de92e3bcbfa7cf2ed': ref: 'v2.0.5' commit: 'sha1-22d081ff2d3a40755e97629de92e3bcbfa7cf2ed' diff --git a/.github/workflows/code-coverage.yml b/.github/workflows/code-coverage.yml new file mode 100644 index 000000000..6266b8d15 --- /dev/null +++ b/.github/workflows/code-coverage.yml @@ -0,0 +1,210 @@ +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell (hyperpolymath) +# +# code-coverage.yml — publish real line-coverage data to GitHub Code Quality. +# +# WHY THIS EXISTS +# --------------- +# The main-gate ruleset carries a `code_coverage` rule (minimum line coverage). +# GitHub evaluates that rule against **uploaded** Cobertura reports; it runs +# nothing itself and it does not wait for an upload to arrive. A coverage rule +# with no uploader therefore blocks every pull request forever while looking, +# in the check list, like a rule rather than a fault. That is the same defect as +# a required status context no pull request can publish; see +# docs/audits/audit-ci-context-producibility-2026-09-20.adoc, which records five +# instances of it in a single day. +# +# The alternative — delete the rule — was offered and declined. So the number +# has to be real: this workflow measures what is measurable here and uploads it. +# +# WHAT IS MEASURED, AND WHAT IS NOT +# --------------------------------- +# Measured: line coverage of the six independent Rust crates in this tree +# (`cargo llvm-cov`, Cobertura output). +# +# NOT measured: the 245 shell scripts under scripts/ and the 756 AsciiDoc files +# that are the bulk of this repository. Shell coverage needs kcov, is materially +# more fragile, and belongs in its own change with its own review. Until that +# lands, the floor set in the ruleset describes the Rust crates only. Saying so +# is the point: a coverage number that quietly excludes most of the tree is +# worse than no number, because it reads as a guarantee. +# +# WHY push AND pull_request +# ------------------------- +# Code Quality compares pull-request coverage against the **default branch** +# baseline. Without a `push` trigger on main there is no baseline, and the rule +# has nothing to compare against. Both triggers are load-bearing. +# +# WHY NO `paths:` FILTER +# ---------------------- +# A path-filtered workflow never reports on pull requests that miss the filter. +# This job must be a required status check (see below), so a filter would leave +# those pull requests waiting for a check that never starts. That is deadlock, +# not optimisation — the same reasoning as lock-sync-gate.yml. +# +# WHY THE CHECK MUST BE REQUIRED +# ------------------------------ +# The coverage rule evaluates "data that has already been uploaded". Merge +# before this job finishes and the rule can evaluate an empty set and pass. +# Requiring the job is what makes the rule mean anything, and it is also what +# makes a fork pull request — whose upload the action deliberately skips — wait +# for the same check as everything else instead of slipping through. +# +# ONE JOB PER CRATE, ONE LABEL PER UPLOAD +# --------------------------------------- +# GitHub aggregates the uploaded reports for a branch, so six labels are six +# contributions to one number rather than six competing numbers. A matrix keeps +# each crate's build isolated: these are independent crates with no workspace +# root, and one broken build must not mask another crate's coverage. +# +# SCOPE OF THE MATRIX +# ------------------- +# Six crates live outside `9-archive/`. Eight more Cargo manifests live *inside* +# it. Archive contents are excluded on purpose: measuring code that is retained +# for the historical record puts a coverage floor on work nobody is meant to +# change. The exclusion is stated rather than silent, because the aggregate +# describes these six crates and nothing else. +# +# FIRST RUN, RECORDED (2026-10-25) +# -------------------------------- +# All six legs measured successfully. All six uploads then failed with +# `Coverage upload failed (HTTP 404): Not Found` — the Code Quality ingestion +# endpoint does not exist until Code Quality is enabled for the repository, and +# that is a settings change, not a file. The job stayed red rather than +# reporting a passing upload that never happened, which is the behaviour this +# file intends. `fail-on-error` must therefore stay at its default until the +# setting is on; turning it off here would only hide the 404. +# +# The measured per-crate figures are recorded in +# docs/audits/audit-ci-context-producibility-2026-09-20.adoc under "Correction, +# 2026-10-25", including the aggregate the ruleset's floor has to sit at or +# below. They are kept there rather than duplicated here so the two cannot drift +# apart — this file states the mechanism, that document states the numbers. + +name: Code coverage + +on: + push: + branches: [main, master] + pull_request: + # Deliberately no workflow_dispatch: the upload action needs a pull request + # number for any non-default branch and skips uploads it cannot attribute, so + # a manual run would appear to work and publish nothing. + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + coverage: + # The matrix suffix is generated by GitHub from the raw `include` values, so + # without this the required check would be named + # `Rust coverage (, )` — the path baked into a context name + # that must stay stable for as long as the ruleset requires it. Naming the + # slug explicitly keeps the published context readable and short. + name: Rust coverage (${{ matrix.name }}) + runs-on: ubuntu-24.04 + timeout-minutes: 30 + permissions: + contents: read + # Required to upload coverage data. Granted at job scope rather than + # workflow scope; nothing else here needs it. + code-quality: write + strategy: + fail-fast: false + matrix: + # `name` is a stable slug used in the upload label. Deriving the label + # from `strategy.job-index` would renumber every report whenever an + # entry is inserted, which silently invalidates history comparisons. + include: + - crate: .verisimdb/ecosystem-ingest + name: ecosystem-ingest + - crate: 1-formats/sub-specs/inline-annotations/extractor + name: inline-annotations-extractor + - crate: 2-protocols/k9-coordination/tools/k9-init + name: k9-init + - crate: hooks/playbook-to-recipe + name: playbook-to-recipe + - crate: tools/js-runtimes + name: js-runtimes + - crate: tools/uuid-v8 + name: uuid-v8 + steps: + - name: Check out + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # Measure the tree the upload is attributed to. On a `pull_request` + # event the default checkout is GitHub's synthetic merge commit, but + # the upload action attributes the report to + # `github.event.pull_request.head.sha`. When the head is behind the + # base, the two trees differ and base-branch changes land in coverage + # totals recorded against a commit that does not contain them. On + # `push` there is no pull request object, so this falls back to + # `github.sha` — the commit being pushed. + ref: ${{ github.event.pull_request.head.sha || github.sha }} + + - name: Install Rust toolchain + # `toolchain:` is mandatory under a SHA pin. + uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # stable + with: + toolchain: stable + components: llvm-tools-preview + + - name: Install cargo-llvm-cov + # Version-pinned: `cargo install` without a version silently picks up + # whatever crates.io serves that day, and a coverage tool that changes + # under you changes the number the ruleset is measuring. + run: cargo install cargo-llvm-cov --version 0.9.1 --locked + + - name: Measure line coverage + working-directory: ${{ matrix.crate }} + run: | + set -euo pipefail + # Deliberately not `--locked`: four of these six crates do not commit a + # Cargo.lock, so `--locked` fails there rather than measuring them. + cargo llvm-cov --cobertura --output-path cobertura.xml + # Read the totals Cobertura puts on the root element. The rule compares + # a weighted aggregate across every uploaded report, so the raw counts + # matter as much as the ratio: a floor can only be chosen from data + # that shows how much of the aggregate each crate actually carries. + read -r PCT COVERED VALID <<< "$(awk -F'"' '/> "$GITHUB_STEP_SUMMARY" + # Also emit the number as an annotation. The step summary is only in the + # run's own page; an annotation shows up on the pull request and is + # readable through the API, which is what makes the figure auditable + # after the fact rather than "trust the green tick". + echo "::notice title=Line coverage: ${{ matrix.name }}::${PCT}% of instrumented lines are covered by tests (${COVERED}/${VALID})." + # A crate that instruments nothing would otherwise report a cheerful + # 0.00% or, worse, divide by zero. Both mean "this measured nothing", + # which is a different statement from "this is untested". + if [ "$VALID" -eq 0 ]; then + echo "::warning title=No instrumented lines: ${{ matrix.name }}::cargo llvm-cov reported no instrumented lines; this crate contributes nothing to the aggregate." + fi + + - name: Upload coverage report + # Fork pull requests cannot receive the write permission this needs; the + # action detects that and skips rather than failing. `fail-on-error` + # stays at its default (true), so a genuine upload failure fails the + # job. A coverage job that goes green without uploading is the exact + # defect this file exists to prevent. + uses: actions/upload-code-coverage@2b21a77928be8d5168c2b9581a67f2adbebacc52 # v1.4.4 + with: + file: ${{ matrix.crate }}/cobertura.xml + language: Rust + label: code-coverage/rust-${{ matrix.name }} diff --git a/.github/workflows/required-contexts.yml b/.github/workflows/required-contexts.yml index b8dcd7bf4..69905f579 100644 --- a/.github/workflows/required-contexts.yml +++ b/.github/workflows/required-contexts.yml @@ -108,7 +108,11 @@ jobs: fi # Findings = every required context that cannot be satisfied on a pull # request. `app-owned` and `producible` are not findings. - jq -r '[.results[] | select(.verdict == "PR-UNPRODUCIBLE" or .verdict == "UNSATISFIABLE") | .context] | .[]' \ + # Three verdicts are findings, and the third is easy to forget: a + # data-dependent RULE (`code_coverage`, `code_quality`) never appears + # as a status context, so it cannot be caught by looking at contexts, + # yet it blocks every pull request exactly like an unproducible one. + jq -r '[.results[] | select(.verdict == "PR-UNPRODUCIBLE" or .verdict == "UNSATISFIABLE" or .verdict == "UNSATISFIED-RULE") | .context] | .[]' \ "$JSON" > "$RUNNER_TEMP/findings.txt" awk '{ sub(/#.*/, ""); gsub(/^[ \t]+|[ \t]+$/, ""); if ($0 != "") print }' \ "$ALLOW" > "$RUNNER_TEMP/allow.txt" @@ -146,9 +150,9 @@ jobs: echo "::warning::$n_stale declared exemption(s) no longer match anything — remove them from $ALLOW" fi if [ "$n_new" -gt 0 ]; then - echo "::error::$n_new required context(s) cannot be published on a pull request and are not declared in $ALLOW" + echo "::error::$n_new requirement(s) cannot be satisfied on a pull request and are not declared in $ALLOW — see the findings above; a \`rule:\` entry is a rule whose data nothing uploads, a bare name is a status context nothing publishes" sed 's/^/ /' "$RUNNER_TEMP/undeclared.txt" - echo "Every pull request waits for a check run that cannot happen. Either publish it on pull_request, require the name the PR-time workflow actually emits, or declare it in $ALLOW with a reason." + echo "For a status context: publish it on pull_request, or require the name the PR-time workflow actually emits. For a rule: upload the data it needs, or remove the rule — a coverage threshold on a repository that publishes no coverage blocks every pull request forever, and an undecidable gate is one people bypass. Declaring it in $ALLOW is the last resort, not the repair." exit 1 fi - echo "All unsatisfiable required contexts are declared in $ALLOW ($n_find declared)." + echo "All unsatisfiable requirements are declared in $ALLOW ($n_find declared)." diff --git a/.machine_readable/required-contexts-allow.txt b/.machine_readable/required-contexts-allow.txt index 750812e93..b542df171 100644 --- a/.machine_readable/required-contexts-allow.txt +++ b/.machine_readable/required-contexts-allow.txt @@ -6,6 +6,14 @@ # # Every line is a debt with a name and a reason: `context` # +# THE LIST IS EMPTY, AND THAT IS THE POINT +# ---------------------------------------- +# This file's preamble has always said that removing the last entry is the +# goal and not an edge case. The one entry has now been removed, because the +# defect it recorded was repaired at the decision layer rather than declared +# here. Empty is the correct resting state; if an entry comes back, it is a +# regression with a review trail, not a configuration. +# # WHY THIS FILE EXISTS INSTEAD OF `continue-on-error: true` # --------------------------------------------------------- # The alternative was an always-green advisory job, and this repository has @@ -15,33 +23,47 @@ # was never wired, so it never disagreed with anything.) # # So the job FAILS on any unsatisfiable required context that is not listed -# here, and passes only for the ones below, which are reported on every run. -# A new entry cannot be added quietly: this file is the review surface. -# -# Removing the last entry is the point, not an edge case. -# -# The known entry is a branch RULE, not a missing workflow. The scan it names -# does run on every pull request — as `Hypatia verdict grid / Hypatia -# Neurosymbolic Analysis`, the fork-governance lane. The required context names -# the OTHER call site of the same reusable workflow: hypatia-scan.yml, whose -# `scan` job is deliberately push/schedule-only so the same findings are not -# uploaded to code scanning twice per pull request. So the defect is a context -# name pointing at the post-merge call site of a scan that is gated pre-merge -# under a different name. -# -# Consequence, measured on #1336: 65 checks green, reviewDecision clear, -# mergeable, and still BLOCKED — because the required check can never appear. -# Every pull request in this repository is in that state, and the only way work -# lands is the ruleset's `always` bypass, which discards the other nineteen -# required checks in the same action. That is the whole failure mode this -# repository keeps re-learning: the bypass is not an escape hatch used -# occasionally, it is the actual merge path, and it makes the nineteen -# decorative. -# -# Repair (decision layer, needs repository admin — the ruleset, not this file): -# replace the context below with `Hypatia verdict grid / Hypatia Neurosymbolic -# Analysis` in ruleset 23787415, leaving the other nineteen untouched. +# here. A new entry cannot be added quietly: this file is the review surface, +# and the gate also warns when a listed context stops appearing as a finding, so +# this file cannot quietly rot into a museum of old exemptions either. +# +# RECORD OF THE RETIRED ENTRY +# --------------------------- +# Context: `scan / Hypatia Neurosymbolic Analysis` +# Retired: 2026-10-24 +# +# It named the second call site of hypatia-scan-reusable.yml. `hypatia-scan.yml` +# declares that job as push/schedule-only on purpose, so the same findings are +# not uploaded to code scanning twice per pull request; the pull-request call +# site is the fork-governance lane, which publishes +# `Hypatia verdict grid / Hypatia Neurosymbolic Analysis`. A required check is +# matched by name, and the names differ, so the required one could never be +# produced while a pull request was open. +# +# Measured consequence: #1336 reached 65 passing checks, a cleared +# reviewDecision and mergeable=MERGEABLE while mergeStateStatus stayed BLOCKED. +# #1337, which carried the repair, merged 79 minutes BEFORE the ruleset was +# corrected — so the fix provably did not enable its own merge. #1337's merge +# used the ruleset's `always` bypass, which discards every required check in the +# same action. This record does not establish that #1336 merged or used the bypass. +# +# Retired by the decision layer, as this file asked: +# ruleset 23787415, required_status_checks: +# scan / Hypatia Neurosymbolic Analysis +# -> Hypatia verdict grid / Hypatia Neurosymbolic Analysis +# bypass_actors: always on RepositoryRole:5 -> removed +# strict_required_status_checks_policy: false -> true +# + `Repo self-tests` required (21 contexts, all producible on a pull request) +# +# Doing it in this file instead would have required continuing to bypass, and a +# list of declared exceptions is only safer than a bypass while it stays short. # # Format: one context per line; `#` starts a comment; blank lines ignored. - -scan / Hypatia Neurosymbolic Analysis +# +# An empty file (or one holding only comments) asserts that no repository-owned +# `PR-UNPRODUCIBLE` or `UNSATISFIABLE` finding is undeclared. It does +# NOT assert that every required context is satisfiable: the reconciliation step +# excludes `app-owned` contexts (those produced by a GitHub App other than +# Actions), and it only judges contexts that are findings. Both exclusions are +# deliberate — but they mean empty is a narrower claim than "all good", and +# this file should not be read as one. diff --git a/0-AI-MANIFEST.deed b/0-AI-MANIFEST.deed index 3b1c3fd2e..096f82b86 100644 --- a/0-AI-MANIFEST.deed +++ b/0-AI-MANIFEST.deed @@ -29,7 +29,8 @@ (rule :priority 10 :text "No A2ML/DEED duplication — root must NOT contain STATE.deed, META.deed, etc.") (rule :priority 20 :text "The repo deed routes to state evidence; generated metadata does not confer normative authority.") (rule :priority 30 :text "Initial licence assignment is permitted: agents may add the correct SPDX header to new files under the repository policy and set a brand-new repository's initial LICENSE. Changes to existing licences, LICENSE files or SPDX declarations are Manual-Only and restricted to registered codeowner-admins. Agents must flag licence drift for review; never sweep or relicense existing content. See LICENSE, .machine_readable/licensing-policy.toml, and .claude/CLAUDE.md.") - (rule :priority 40 :text "Author attribution — Jonathan D.A. Jewell ")) + (rule :priority 40 :text "Author attribution — Jonathan D.A. Jewell ") + (rule :priority 50 :text "Never merge through a branch-ruleset bypass to get past a required status check. A bypass discards every required check in that ruleset in a single action, so using one converts the rest of the decision layer into advice. If a pull request is green everywhere and still BLOCKED, diagnose it with scripts/check-required-contexts.sh and consult the audit verdict before choosing a repair. UNSATISFIED-RULE for data-dependent code_coverage or code_quality rules calls for inspecting the required data and repository settings. When the verdict identifies an unproducible required context, repair the context with scripts/fix-required-context-name.sh (report-only by default) and flag it for a human to apply. Downgrading the gate — continue-on-error, || true, or a new line in .machine_readable/required-contexts-allow.txt — is not a repair. See docs/audits/audit-ci-context-producibility-2026-09-20.adoc.")) (startup-checklist :steps ("Read THIS file, then README.adoc." diff --git a/CICD-WORKFLOW-CATALOG.md b/CICD-WORKFLOW-CATALOG.md index c5d88db0e..c215f7425 100644 --- a/CICD-WORKFLOW-CATALOG.md +++ b/CICD-WORKFLOW-CATALOG.md @@ -43,6 +43,8 @@ These workflows **block** merges if they fail. They enforce critical estate-wide | `governance-reusable.yml` | Reusable governance | standards | Yes | | `hypatia-scan.yml` | Security scanning with Hypatia | rsr-template-repo | Yes | | `hypatia-scan-reusable.yml` | Reusable Hypatia scan | standards | Yes | +| `fork-governance.yml` | The one PR comment (DEED · K9-DEED conformance · Hypatia verdict grid) plus the PR-time scanner. This is the lane that publishes `Hypatia verdict grid / Hypatia Neurosymbolic Analysis`, so a ruleset requiring that context depends on this workflow running on every PR | standards | `fork-governance-reusable.yml` | +| `required-contexts.yml` | Asserts every context the branch ruleset requires is actually publishable on a pull request. A required check nothing can publish makes every PR undecidable; this gate catches that before the ruleset has to be bypassed | standards | No | | `main-estate-audit.yml` | Estate audit checks | rsr-template-repo | Yes | | `scorecard.yml` | OSSF Scorecard | rsr-template-repo | Yes | | `scorecard-reusable.yml` | Reusable Scorecard | standards | Yes | @@ -51,7 +53,7 @@ These workflows **block** merges if they fail. They enforce critical estate-wide | `guix-policy.yml` | Guix/Nix package policy | knot-rider (canonical) | No | | `secret-scanner.yml` | Secrets detection | rsr-template-repo | Yes | | `secret-scanner-reusable.yml` | Reusable secrets scanner | standards | Yes | -| `runtime-policy.yml` | Runtime security policies | rsr-template-repo | No | +| ~~`runtime-policy.yml`~~ | **RETIRED:** removed from `rsr-template-repo`; `tests/workflows/governance_policy_test.sh` asserts the file and its lock record must not exist. Listed here only so the retirement is discoverable rather than looking like an omission | — | No | | `static-analysis-gate.yml` | Static analysis gate | rsr-template-repo | No | | `wellknown-enforcement.yml` | .well-known file enforcement | rsr-template-repo | No | diff --git a/EXPLAINME.adoc b/EXPLAINME.adoc index 7825a7a81..37e95d149 100644 --- a/EXPLAINME.adoc +++ b/EXPLAINME.adoc @@ -46,6 +46,12 @@ stick. Former satellite repos were absorbed as subdirectories on 2026-02-08. | SHA-pinned GitHub Actions | Prevents supply-chain attacks from mutable tags. +| A required check must be producible on a pull request +| A gate only means something if a pull request can satisfy it. A required context that only a post-merge run publishes makes every pull request undecidable, and forces the bypass that discards every other check. Producibility is therefore checked, not assumed (`scripts/check-required-contexts.sh`). + +| No standing bypass on the default branch +| An `always` bypass actor turns 21 required checks into advice, because using it once discards all of them at once. The emergency path is disabling the ruleset, which is an auditable act, not a silent merge. + | Guix-first package management | Reproducible builds via Guix. | Elegance is the default arm (proposed) diff --git a/README.adoc b/README.adoc index 7324c5483..f75bfb92d 100644 --- a/README.adoc +++ b/README.adoc @@ -290,6 +290,47 @@ The link:.github/workflows/k9-contractile.yml[DEED–K9 workflow] publishes one compact PR report, with expandable findings and links to dated evidence. See link:1-formats/k9/tools/CONFORMANCE-REPORTING.adoc[its scope, colours and reproduction guide]. +=== What actually blocks a merge + +`main` is gated by the ruleset *main gate: append-only + required checks + +signatures + scanning*: 21 required status checks, branches required to be up to +date, and **no bypass actors**. Nothing merges without producing all 21. + +That last clause is the one that is easy to lose, so it is worth stating what a +gate is *not*. A required check that no pull request can publish does not make +the gate stricter — it makes every pull request undecidable. On 2026-10-24 this +repository merged red repeatedly for exactly that reason, and not because a check +failed: pull request #1336 stood at 65 passing checks, a cleared review and +`mergeable=MERGEABLE` while `mergeStateStatus` stayed `BLOCKED`, because the +required context named the post-merge call site of a scan that runs pre-merge +under a different name. The only way through was the ruleset bypass, which +discards all 21 required checks in the same action. + +Three things now keep that from recurring: + +* *Producibility is a gate.* + link:.github/workflows/required-contexts.yml[`required-contexts.yml`] runs + link:scripts/check-required-contexts.sh[`scripts/check-required-contexts.sh`] + on every pull request and weekly. It rejects undeclared findings: + `PR-UNPRODUCIBLE`, `UNSATISFIABLE`, and `UNSATISFIED-RULE` for data-dependent + rules such as `code_coverage` and `code_quality`. App-owned contexts are + excluded from findings, and declared findings are permitted. +* *Exemptions are declared, dated and shrink-only.* + link:.machine_readable/required-contexts-allow.txt[`required-contexts-allow.txt`] + lists known-unsatisfiable contexts with a reason. It is currently **empty**, + which is its intended resting state; the gate warns when a listed entry stops + appearing as a finding, so it cannot rot into a museum of stale exemptions. + Silence is never achieved with `continue-on-error` or `|| true` — both were tried here and + both produced an always-green job that reported a gate it never ran. +* *The repair is scripted.* + link:scripts/fix-required-context-name.sh[`scripts/fix-required-context-name.sh`] + repoints a single named context at a single named ruleset, refuses every state + it cannot justify, and verifies the read-back. Branch protection is edited by + a narrow instrument or not at all. + +The full measurement, including the 3h20m of red `main` that motivated it, is in +link:docs/audits/audit-ci-context-producibility-2026-09-20.adoc[the context-producibility audit]. + === CI/CD Workflows Copy these workflows to `.github/workflows/` in your repository: diff --git a/coordination.k9 b/coordination.k9 index 55ac7e649..ea1ac7184 100644 --- a/coordination.k9 +++ b/coordination.k9 @@ -20,7 +20,7 @@ metadata: schema: k9-coordination schema_version: 1.2.0 generated_by: manual - last_updated: 2026-10-14 + last_updated: 2026-10-24 project: name: "standards" @@ -88,6 +88,14 @@ invariants: rule: "All GitHub Actions must be SHA-pinned, not version-tagged" reason: "Version tags are mutable. SHA pins are immutable. This is a security invariant." severity: high + - id: required-checks-are-producible + rule: "Every context required by a branch ruleset MUST be publishable on a pull request. A context only a post-merge run produces is a defect, never a stricter gate." + reason: "A required check nothing can publish makes every pull request undecidable, and the only way through is a bypass that discards every other required check at once. Measured 2026-10-24: #1336 had 65 passing checks, a cleared review and mergeable=MERGEABLE while BLOCKED. Every required context must be checked; scripts/check-required-contexts.sh checks contexts within its scope on every pull request and weekly, excluding app-owned contexts from findings." + severity: critical + - id: no-standing-bypass + rule: "No ruleset guarding the default branch may carry an `always` bypass actor. The emergency path is disabling the ruleset deliberately, which is auditable." + reason: "Bypass is all-or-nothing: one use discards every required check in the ruleset. A standing bypass therefore makes the whole decision layer advisory, and leaves no trace at merge time." + severity: critical protected: - path: component-readiness-grades/ @@ -182,6 +190,12 @@ innervation: - trigger: "file-create" guard: "No banned language files (*.ts, *.res, *.py, *.go, *.java, *.kt)" action: "Reject with language policy link" + - trigger: "pull_request" + guard: "Every required status context is publishable on a pull request" + action: "Fail with the workflow and event list that could not publish it; never silence with continue-on-error" + - trigger: "ruleset-edit" + guard: "No `always` bypass actor on a ruleset guarding the default branch" + action: "Refuse the edit; the emergency path is disabling the ruleset deliberately" do_not_create: - pattern: **/*.ts diff --git a/docs/audits/audit-ci-context-producibility-2026-09-20.adoc b/docs/audits/audit-ci-context-producibility-2026-09-20.adoc index 64315ff7f..c71885ad3 100644 --- a/docs/audits/audit-ci-context-producibility-2026-09-20.adoc +++ b/docs/audits/audit-ci-context-producibility-2026-09-20.adoc @@ -535,3 +535,243 @@ does not do this: it derives contexts from checks that run and then has to pick a ruleset, and with six additive branch rulesets it cannot justify a choice, so it reports `AMBIGUOUS` and declines. A scripted single-field edit is the safe instrument; a general-purpose rewriter is not. + +=== Applied: the decision layer, verified + +The repair ran on 2026-10-24 and was verified by reading the ruleset back, not +by trusting the script's success message. + +*Scope of this table, stated because the numbers differ elsewhere:* it covers +**one** ruleset, `23787415` (*main gate*), which is the one that carried both +the unproducible context and the bypass that made the red merges possible. The +repository has six active branch rulesets. See the caveat after the table. + +[cols="2,1,1",options="header"] +|=== +| Field | Before | After + +| `required_status_checks` (Hypatia) +| `scan / Hypatia Neurosymbolic Analysis` +| `Hypatia verdict grid / Hypatia Neurosymbolic Analysis` + +| `bypass_actors` +| `RepositoryRole:5`, `always` +| none in this Hypatia ruleset + +| `strict_required_status_checks_policy` +| `false` +| `true` + +| Required contexts in this Hypatia ruleset +| 20 (1 unproducible) +| 21, all producible on a pull request +|=== + +`Repo self-tests` was added as the twenty-first. It is the check that caught both +regressions in this audit — the `|| true` that swallowed the gate's exit, and +inline Python that was banned repository-wide — and under the old ruleset it +could not block either. A check that detects a defect and cannot stop it is a +reporter, not a gate; it is now load-bearing. + +The earlier gate re-run on 2026-10-24, after the context repair and before +data-dependent rules were included in the audit, reported `inspected: true`, +`producible=23`, `app-owned=1`, and **zero findings**. This is a historical +measurement, not the current PR gate result. Those counts cover the *whole +inspected ruleset set*, not the +21 contexts of the one ruleset above: 23 contexts are producible and 1 is owned +by a GitHub App, across all six active rulesets, and 21 + 3 = 24 required +context entries include duplicates across rulesets. The two totals are different +views and neither is wrong. + +`.machine_readable/required-contexts-allow.txt` is therefore header-only: the +entry it carried is retired and recorded there, not silently deleted, because a +declaration that disappears without a trace is indistinguishable from one that +was never true. + +==== The timestamp that settles the argument + +#1337 carried the repair and merged at **21:38:20Z**. The ruleset was corrected +at **22:57** — seventy-nine minutes later. The fix therefore provably did not +enable its own merge, and that merge again went through the `always` bypass. + +The unproducible context and bypass are now gone from the repaired Hypatia +ruleset. Legacy branch protection remains unverified; the other active +rulesets' remaining bypass actors are recorded below. This does not establish +that every path to `main` enforces all checks. + +==== The bypass removal was not complete, and the residue is recorded here + +Removing `bypass_actors` from `23787415` did not remove bypass from the +repository. Three of the six active branch rulesets still carry +`RepositoryRole:5` with `bypass_mode: always`, measured 2026-10-24 after the +repair. What each one actually protects, and whether the bypass matters: + +[cols="1,2,2",options="header"] +|=== +| Ruleset | Its only rule | Is that rule enforced anywhere else? + +| `24255701` Require-Linear-History +| `required_linear_history` +| **No.** This is the one real gap. `23787415` has `non_fast_forward`, which + stops history being *rewritten*, but not `required_linear_history`, which + stops merge commits being *created*. Linear history is therefore bypassable + by any admin today. + +| `24587723` Lock-Floor +| 2 status checks +| Yes — both are also required by `23787415`, which has no bypass. + +| `24309995` Secret-Scan-Floor +| `scan / gitleaks` +| Yes — also required by `23787415`, which has no bypass. +|=== + +So the accurate statement is narrower than "the bypass is gone" and different +from "three gates are open": + +* All **21 distinct required contexts** are unbypassable, because `23787415` + enforces them all and carries no bypass actor. Lock-Floor and Secret-Scan-Floor + duplicate two and one of them respectively, so their bypasses discard nothing + that is not still required next door. +* **`required_linear_history` is genuinely bypassable.** It is the only rule in + the repository that appears in exactly one ruleset, and that ruleset still + grants `RepositoryRole:5` bypass mode `always`. Append-only merging is + therefore still advice for an admin. +* Those two duplicate rulesets should still be cleaned up, not because they + currently weaken anything but because a redundant bypass is a bypass waiting + to become load-bearing the day someone removes the check from the main gate. + +This is the same failure mode the audit describes, one layer down: the headline +gate was fixed where the symptom was loudest, and three quieter rulesets were +not examined. It is recorded rather than fixed here because it is a +branch-protection write, and because a second unverified ruleset edit on the same +day is how protection gets dropped by accident. + +==== The two data-dependent rules cannot be satisfied by writing code + +`code_coverage` and `code_quality` are the fourth and fifth instances of the +same defect, and unlike the first three they cannot be repaired in this +repository by any change to this repository's files. + +What they require, per GitHub's documentation: GitHub **Code Quality enabled** +for the repository, and then either a Cobertura XML upload through +`actions/upload-code-coverage@v1` (for coverage) or the `CodeQL - Code Quality` +check running on pull requests (for quality). Neither prerequisite is a file in +this tree. Code Quality is a repository setting, and the upload permission is +`code-quality: write`, which no workflow here currently requests. + +Measured state of the tree those rules are judging: + +[cols="1,1",options="header"] +|=== +| Fact | Value + +| AsciiDoc files | 756 +| Shell scripts | 245 +| Shell test scripts | 90 +| Rust files | 8, across five separate crates, none with a test file +| Coverage instrumentation available in CI | none (no `kcov`, no `bashcov`) +| `code_coverage` rule | `minimum_coverage: 80` +|=== + +An 80% line-coverage floor on a repository that is 90% prose and shell, with no +coverage instrumentation for either, is not a threshold that a code change can +reach. Standing up shell coverage is a project, not a tidy, and even then the +number would describe the `scripts/` directory rather than the artifact this +repository exists to produce. + +So the honest options are these two, and they are both settings: + +. **Remove the two rules** from `23787415`. The repository keeps 21 required + status contexts, all producible, and the other five rule types. +. **Keep them, but make them true.** Enable Code Quality, add the Cobertura + upload with `code-quality: write`, and lower `minimum_coverage` to a figure + the tree actually meets — a floor nobody can reach is not a quality bar, it is + a permanent block. Note that the coverage rule evaluates only data already + uploaded and *does not wait* for uploads, so the job that uploads coverage + must itself be a required status context or the rule can pass on no data. + +The failure mechanism is the *required-contexts workflow*: the audit reports +these data-dependent rules as `UNSATISFIED-RULE`, and the workflow rejects +undeclared findings. Its failing status context then blocks the PR through +`required_status_checks`. This does not establish that GitHub’s coverage rule +itself blocks every pull request: as noted above, it can pass when no coverage +data has been uploaded. Choose either settings repair above based on the data +and enforcement the repository intends to require; a context-name repair does +not address these rule findings. + +==== Correction, 2026-10-25 — coverage data *can* be produced in this repository + +The subsection above is wrong on its central claim, and the error is the kind +this audit exists to catch, so it is recorded rather than quietly edited. + +It said `code_coverage` "cannot be repaired in this repository by any change to +this repository's files". Workflow changes can produce coverage data, but +completing the rule repair also requires GitHub Code Quality to be enabled in +the repository settings before uploads can succeed. The separate `code_quality` +rule still requires that setting and GitHub's `CodeQL - Code Quality` check; +no repository file can enable it. The table it printed was also wrong on the +size and testability of the Rust surface: + +[cols="2,1,2",options="header"] +|=== +| Claim in the table | Measured | How + +| Rust files: 8, across five separate crates +| 45 files; 14 `Cargo.toml`. Outside `9-archive/`: 8 files in 6 crates +| `git ls-files '*.rs'`, `git ls-files '*Cargo.toml'` + +| none with a test file +| all six non-archive crates contain tests +| `#[test]` / `#[cfg(test)]` / assertion macros per crate +|=== + +The count of 8 was right for the non-archive tree and was then stated as the +whole tree; "five crates" and "no test files" were wrong in both cardinality and +fact. The conclusion drawn from them — that no code change could produce +coverage data — followed from the error, not from the tree. + +What the tree can actually be measured for, measured rather than estimated +(`.github/workflows/code-coverage.yml`, run +https://github.com/hyperpolymath/standards/actions/runs/40146453852[40146453852]): + +[cols="2,1,1,1",options="header"] +|=== +| Crate | Covered | Instrumented | Line coverage + +| `js-runtimes` | 282 | 601 | 46.92% +| `k9-init` | 222 | 326 | 68.10% +| `ecosystem-ingest` | 340 | 501 | 67.86% +| `inline-annotations-extractor` | 211 | 291 | 72.51% +| `playbook-to-recipe` | 179 | 266 | 67.29% +| `uuid-v8` | 102 | 126 | 80.95% +| *Aggregate* | *1336* | *2111* | *63.29%* +|=== + +So the 80% floor is **16.71 points** above what the tree currently meets. It is +not a quality bar; no pull request could hold it, and it would block +`23787415`'s twenty-one satisfiable contexts from ever being discharged. The +correct floor is a number at or below the aggregate the tree actually produces, +after which `max_drop: 5` becomes the real mechanism: a ratchet against +regression rather than an aspiration nobody reaches. + +Three limits on that number, stated because a coverage figure that hides its +scope reads as a guarantee it cannot keep: + +. It describes these six crates and nothing else. The 245 shell scripts and 756 + AsciiDoc files, which are the bulk of this repository, are not instrumented + and contribute nothing to the aggregate. The floor must not be read as a + statement about them. +. Eight further `Cargo.toml` manifests sit under `9-archive/`. They are excluded + deliberately: a coverage floor over retained historical code would demand work + on files nobody is meant to change again. +. The aggregate is only defined once a baseline exists. Code Quality compares a + pull request against the default branch, so `main` must upload at least once + (the workflow's `push` trigger) before the rule has anything to compare. + Until then the rule's behaviour on an absent baseline is GitHub's to decide, + not this repository's. + +What remains true from the subsection above: **`code_quality` still cannot be +satisfied by a file**. It requires Code Quality to be enabled for the +repository, which is a settings change, and the `CodeQL - Code Quality` check +then appears on its own. diff --git a/docs/wikis/developers/index.adoc b/docs/wikis/developers/index.adoc index 9f24b0720..f5b553e55 100644 --- a/docs/wikis/developers/index.adoc +++ b/docs/wikis/developers/index.adoc @@ -49,7 +49,46 @@ NOTE: Scaffold. The link spine is complete; the connective prose is `TODO`. * The workflow-pin staleness gate: `scripts/check-workflow-staleness.sh` (recency window — link:../../decisions/ADR-003-workflow-pin-staleness-window.adoc[ADR-003]). * Language policy, no-JS, secret scanning, SPARK gate. -* TODO: "decode a red gate and the fastest green path". + +==== Green everywhere, still BLOCKED + +If every check is green, the review is clear, GitHub says `mergeable=MERGEABLE`, +and `mergeStateStatus` still reads `BLOCKED`, inspect the audit verdict before +choosing a repair. Green checks alone do not establish that a required status +context is missing. Unsatisfied data-dependent rules such as `rule:code_coverage` +or `rule:code_quality` appear as `UNSATISFIED-RULE`; inspect their required data +and repository settings before considering a status-context repair. + +[source,bash] +---- +# which required contexts can actually be published on a pull request? +bash scripts/check-required-contexts.sh . --json /tmp/rc.json +jq -r '.results[] | select(.verdict == "PR-UNPRODUCIBLE" or .verdict == "UNSATISFIABLE" or .verdict == "UNSATISFIED-RULE") | "\(.verdict)\t\(.context)"' /tmp/rc.json +---- + +The usual cause is a context name that points at a *call site* rather than a +check. A job that calls a reusable workflow is reported as +` / `, so two workflows running the same +reusable publish two different names — and the ruleset can require the one that +only runs after merge. `PR-UNPRODUCIBLE` in the output is exactly that, and the +verdict comes with the workflow and event list that published it. + +When the verdict identifies a context-name issue, +`scripts/fix-required-context-name.sh` reports the +one-line ruleset diff and `--apply` writes it (needs repository admin). Do not +add the missing check to `.machine_readable/required-contexts-allow.txt` unless +it genuinely cannot be produced — that file is a shrink-only list of accepted +debts, and it is empty today. + +==== Decoding a red gate fast + +* Read the *step* that failed, not the job title; a job named for a check that + ends in `|| true` or `continue-on-error` is not a gate at all. +* `gh pr checks ` for the rollup, `gh api …/check-runs//annotations` for + the actual message. `gh run view --log-failed` often returns nothing useful. +* A check that exists in CI but not in the ruleset cannot block anything. If a + lane catches a real regression and does not gate, say so — `Repo self-tests` + caught two in one session and now gates. === 7. Reference diff --git a/docs/wikis/maintainers/index.adoc b/docs/wikis/maintainers/index.adoc index b5e98e9f6..3b35ced37 100644 --- a/docs/wikis/maintainers/index.adoc +++ b/docs/wikis/maintainers/index.adoc @@ -27,6 +27,29 @@ NOTE: Scaffold. The link spine is complete; the connective prose is `TODO`. Their generators and the `registry-verify` gate were retired on 2026-10-09; see `REGISTRY.adoc`. +=== The merge decision layer is part of the standard + +Branch protection is not infrastructure that someone remembers to configure; it +is the thing that decides whether any of the other guardrails can bite. + +* *Check producibility before trusting a required check.* A required context + that no pull request can publish blocks every pull request forever, and the + only way through is a bypass that discards the rest. Run + `scripts/check-required-contexts.sh` locally; it is also a PR gate and a + weekly cron. (`docs/audits/audit-ci-context-producibility-2026-09-20.adoc`.) +* *Never add a standing bypass.* Bypass is all-or-nothing: using it once + discards every required check in the ruleset. If the ruleset is blocking + something that should land, fix the ruleset — the escape hatch is to disable + it deliberately, which leaves an audit trail. +* *Edit protection with the narrow instrument.* + `scripts/fix-required-context-name.sh` (report-only by default) repoints one + named context in one named ruleset and refuses when the before-state is not + what it expects. Do not hand-edit a ruleset payload with ad-hoc `jq`. +* *Exemptions are reviewed, dated and shrink-only.* + `.machine_readable/required-contexts-allow.txt` is empty by design. Adding a + line is a public promise that the context cannot be produced; the gate warns + when an entry goes stale, so it cannot quietly become permanent. + == Learning path === 1. Drift / staleness automation diff --git a/scripts/check-required-contexts.sh b/scripts/check-required-contexts.sh index 6c3ed64fa..3644f8f45 100755 --- a/scripts/check-required-contexts.sh +++ b/scripts/check-required-contexts.sh @@ -108,6 +108,7 @@ APP_NAMED_DEFAULT="github-advanced-security Dependabot Dependabot Updates Code s APP_NAMED="${APP_NAMED_CONTEXTS:-$APP_NAMED_DEFAULT}" GH_ACTIONS_APP=15368 RAW="https://raw.githubusercontent.com" +MERGE_BRANCH="" say() { printf '%s\n' "$*"; } info() { printf ' %s\n' "$*"; } @@ -357,7 +358,23 @@ required_json() { # emit {"context":..,"app_id":..,"source":..} objects note "could not read branch rulesets for $repo (API error, rate limit, or missing permission) — nothing inspected" >&2 return 1 fi - br="$(gh api "repos/$repo" 2>/dev/null | jq -r 'if type=="object" then (.default_branch // "main") else "main" end' 2>/dev/null || echo main)" + # MERGE_BRANCH decides which rulesets are considered to apply, so it must be + # the real default branch or the audit must stop. Defaulting to "main" when + # the lookup fails is not a harmless fallback: on a repository whose default + # branch is `master`, `main` matches no ref filter, every rule is filtered + # out, and the audit reports a clean gate that was never examined. That is the + # same false-clean this check exists to catch, so an unreadable or malformed + # repository response is an inspection failure instead. + local details + details="$(gh api "repos/$repo" 2>/dev/null || true)" + br="$(printf '%s' "$details" | jq -r ' + if type == "object" and (.default_branch | type) == "string" and (.default_branch | length) > 0 + then .default_branch else empty end' 2>/dev/null)" + if [ -z "$br" ]; then + note "could not resolve the default branch of $repo — cannot tell which rules apply, nothing inspected" >&2 + return 1 + fi + MERGE_BRANCH="$br" for id in $(printf '%s' "$list" | jq -r '.[]? | select(.target=="branch") | .id'); do body="$(gh api "repos/$repo/rulesets/$id" 2>/dev/null || true)" printf '%s' "$body" | jq -c ' @@ -377,12 +394,217 @@ required_json() { # emit {"context":..,"app_id":..,"source":..} objects return 0 } +# Rule types whose satisfaction depends on DATA being uploaded by something, +# rather than on a workflow existing. GitHub evaluates these itself, so they +# never appear as a status context and the producibility check above cannot see +# them — which is exactly how they go unnoticed while blocking every pull +# request. Measured 2026-10-24: `code_coverage` (minimum_coverage 80) and +# `code_quality` (severity all) appeared on the main gate of a repository whose +# only coverage job reports SKIPPED and which uploads no code-quality results. +# Neither rule is in the repository's own config/rulesets/*.json; they arrived +# through a UI edit. +# +# The repair differs per type and is not a rename: either upload the data, or +# remove the rule. Keeping the rule and hoping is what makes every PR +# undecidable, and an undecidable gate is a gate people bypass. +DATA_RULES='code_coverage code_quality' +# Locate a workflow in the tree that uploads data for a data-dependent rule, and +# echo its path; echo nothing when there is none. This changes the WORDING of the +# finding only, never whether it is a finding: a rule with an uploader in the +# tree is still unsatisfiable until the repository setting it needs is on, so it +# must keep failing rather than becoming a warning. What it stops is the audit +# asserting that a repository "never uploads" while a workflow in that repository +# does the uploading — an error message that is confidently wrong sends the +# reader looking for data that is already being produced. +# `code_quality` deliberately has no producer here: GitHub runs that analysis +# itself once the feature is enabled, so no file can supply it. +data_rule_producer() { + local match + case "${1:-}" in + code_coverage) + match="$(grep -rl --include='*.yml' --include='*.yaml' \ + 'actions/upload-code-coverage' "$ROOT/.github/workflows" 2>/dev/null | head -1)" + [ -n "$match" ] || return 0 + # Strip the scan root by expansion, not by regex: ROOT is usually a temp + # path, and its dots and slashes would be live regex metacharacters. + printf '%s\n' "${match#"$ROOT"/}" + ;; + *) return 0 ;; + esac +} + +# Print evidence that a data-dependent rule is demonstrably fed, or nothing when +# there is none. Empty output means the finding stands. +# +# This is the difference between *unverified* and *unsatisfiable*, and it is the +# whole point of the check. A rule whose data pipeline has actually run is not a +# rule that blocks every pull request, and reporting it as one is its own defect: +# a permanent alarm is how people learn to ignore the alarm. +# +# The evidence is deliberately an OUTCOME, never an intention. For coverage that +# means a completed run that SUCCEEDED. A workflow merely existing proves +# nothing — every run on this branch failed with HTTP 404 while Code Quality was +# off, and a check that accepted file presence would have called that satisfied +# and let the block through. Returns 1 when a lookup fails, so an API problem +# becomes an inspection failure instead of a silent pass. +data_rule_evidence() { + local type="${1:-}" repo="${GITHUB_REPOSITORY:-}" producer body + [ -n "$repo" ] || return 1 + case "$type" in + code_coverage) + producer="$(data_rule_producer code_coverage)" + [ -n "$producer" ] || return 0 + # A successful run is not yet evidence: the upload action SKIPS rather + # than fails when it cannot attribute the report (fork pull requests, and + # after this repository was renamed, every PR whose event payload still + # carries the old head-repo name). Such a run concludes green having + # uploaded nothing. The ingestion endpoint belongs to Code Quality, so no + # upload can be accepted before that product exists — which makes its + # presence the precondition for any coverage run to have meant anything. + local q + q="$(data_rule_evidence code_quality)" || return 1 + [ -n "$q" ] || return 0 + # Address the workflow by file name; inspect several successful runs so a + # recent fork run cannot hide an earlier run from this repository. + body="$(gh api "repos/$repo/actions/workflows/${producer##*/}/runs?status=success&per_page=30" 2>/dev/null)" || return 1 + [ -n "$body" ] || return 1 + printf '%s' "$body" | jq -es ' + length == 1 and (.[0] | type == "object" and + (.workflow_runs | type == "array")) + ' >/dev/null 2>&1 || return 1 + printf '%s' "$body" | jq -r --arg repo "$repo" ' + [.workflow_runs[] | select((.head_repository.full_name // "") == $repo)][0] // empty + | "a successful \(.path // "coverage") run on \(.head_branch // "?") (\((.head_sha // "")[0:12]))"' \ + 2>/dev/null + ;; + code_quality) + # GitHub runs this analysis itself once the feature is enabled, so there is + # no uploader in the tree to inspect; the workflow it registers is the + # signal that the feature exists for this repository. + # `--slurp` is not available in every gh release (2.23.0, shipped by + # Debian, rejects it outright), and depending on it made this check fail + # closed everywhere that version is installed. `--paginate` alone emits + # one JSON document per page, which jq's own slurp mode folds into the + # array of pages. + body="$(gh api "repos/$repo/actions/workflows?per_page=100" --paginate 2>/dev/null)" || return 1 + [ -n "$body" ] || return 1 + # Reject malformed or incomplete catalogues, including a successful + # response whose pagination stopped too early: every page must agree on + # total_count, and the pages must actually contain that many workflows. + printf '%s' "$body" | jq -es ' + length > 0 and + all(.[]; type == "object" and (.total_count | type == "number") and + (.workflows | type == "array")) and + (.[0].total_count as $total | + all(.[]; .total_count == $total) and + ([.[].workflows[]] | length) == $total) + ' >/dev/null 2>&1 || return 1 + printf '%s' "$body" | jq -sr ' + [.[].workflows[]? | select(.path == "dynamic/github-code-quality/codeql")] + | .[0] // empty + | "the GitHub-generated workflow \(.path) (\(.name))"' 2>/dev/null + ;; + *) return 0 ;; + esac +} + +# Emit tab-separated finding name, app id 0 and UNSATISFIED-RULE for each +# DATA_RULES match in an active branch ruleset whose ref filters match MERGE_BRANCH. +# Presence alone is flagged; uploaded coverage or quality data is not checked. +# Uses GITHUB_REPOSITORY, resolving and setting it with gh when unset. +# Reuses MERGE_BRANCH resolved by required_json. Returns 0 with no findings for +# file fixtures; unavailable inspection or a ruleset read/parse failure returns 1. +# Output may be partial on failure; run_audit discards those partial findings. +data_rules_json() { + [ -f "${REQUIRED_CONTEXTS_JSON:-}" ] && return 0 # offline fixtures: none + command -v gh >/dev/null 2>&1 && command -v jq >/dev/null 2>&1 || return 1 + # Resolve the repo the same way required_json does. Returning early on an + # unset GITHUB_REPOSITORY made this check silently report nothing in exactly + # the local, hand-run case where someone is trying to find out why a green PR + # will not merge. + [ -n "${GITHUB_REPOSITORY:-}" ] || GITHUB_REPOSITORY="$(gh repo view --json nameWithOwner -q .nameWithOwner 2>/dev/null)" + [ -n "${GITHUB_REPOSITORY:-}" ] || return 1 + local repo="$GITHUB_REPOSITORY" id body + local list + [ -n "$MERGE_BRANCH" ] || return 1 + # Evidence is fetched lazily, per rule type, the first time an applicable rule + # of that type is seen — never up front. Fetching it unconditionally made an + # unrelated Actions API failure abort the whole audit on repositories with no + # data-dependent rules at all: protection could be read perfectly well, but + # the run reported `inspected: false` because of an endpoint it did not need. + local ev_cov="" ev_qual="" got_cov=0 got_qual=0 applicable + # Defined once, used twice: the applicability probe and the emission below must + # never disagree about which rulesets apply to the merge branch. + local ref_match_jq=' # GitHub ref filters use pathname globs: * and ? do not cross /. + def matches_ref: + if . == "~ALL" or . == "~DEFAULT_BRANCH" then true + else + ([scan("\\*\\*/|\\*\\*|\\*|\\?|\\[[^]]+\\]|.") | + if . == "**/" then "(.*/)?" + elif . == "**" then ".*" + elif . == "*" then "[^/]*" + elif . == "?" then "[^/]" + elif startswith("[") then sub("^\\[!"; "[^") + else gsub("(?[.()+^$|{}\\\\])"; "\\\(.c)") end] | join("")) as $pattern + | ("refs/heads/" + $branch) | test("^" + $pattern + "$") + end; +' + list="$(gh api "repos/$repo/rulesets" 2>/dev/null)" || return 1 + printf '%s' "$list" | jq -es ' + length == 1 and (.[0] | type == "array" and all(.[]; type == "object" and + (.target | type == "string") and (.id | type == "number"))) + ' >/dev/null 2>&1 || return 1 + for id in $(printf '%s' "$list" | jq -r '.[]? | select(.target=="branch") | .id'); do + body="$(gh api "repos/$repo/rulesets/$id" 2>/dev/null)" || return 1 + printf '%s' "$body" | jq -es ' + length == 1 and (.[0] | type == "object" and + (.enforcement | IN("active", "evaluate", "disabled")) and + (.rules | type == "array" and all(.[]; type == "object" and (.type | type == "string"))) and + (.conditions.ref_name | type == "object" and + (.include | type == "array" and all(.[]; type == "string")) and + (.exclude | type == "array" and all(.[]; type == "string")))) + ' >/dev/null 2>&1 || return 1 + applicable="$(printf '%s' "$body" | jq -r --arg types "$DATA_RULES" --arg branch "$MERGE_BRANCH" \ + "$ref_match_jq"' + if .enforcement == "active" then + select(any(.conditions.ref_name.include[]; matches_ref)) + | select(all(.conditions.ref_name.exclude[]; matches_ref | not)) + | .rules[]? | select(.type as $t | ($types | split(" ") | index($t))) | .type + else empty end')" || return 1 + case "$applicable" in + *code_coverage*) [ "$got_cov" = 1 ] || { ev_cov="$(data_rule_evidence code_coverage)" || return 1; got_cov=1; } ;; + esac + case "$applicable" in + *code_quality*) [ "$got_qual" = 1 ] || { ev_qual="$(data_rule_evidence code_quality)" || return 1; got_qual=1; } ;; + esac + printf '%s' "$body" | jq -r --arg types "$DATA_RULES" --arg id "$id" --arg branch "$MERGE_BRANCH" \ + --arg ev_cov "$ev_cov" --arg ev_qual "$ev_qual" "$ref_match_jq"' + if .enforcement == "active" then + select(any(.conditions.ref_name.include[]; matches_ref)) + | select(all(.conditions.ref_name.exclude[]; matches_ref | not)) + | .rules[]? | select(.type as $t | ($types | split(" ") | index($t))) + # A rule with evidence of real data keeps its row, because a reader of + # the report deserves to see what the audit found and accepted. It is + # just no longer a finding: the verdict carries the claim, not the + # absence of a row. + | (if .type == "code_coverage" then $ev_cov + elif .type == "code_quality" then $ev_qual + else "" end) as $ev + | "rule:\(.type) in ruleset \($id)\t0\t" + + (if $ev != "" then "VERIFIED-RULE\t\($ev)" else "UNSATISFIED-RULE" end) + else empty end' 2>/dev/null || return 1 + done + return 0 +} + # ------------------------------------------------------------------ audit ----- # Print the required-context audit using workflows beneath ROOT; if JSON_OUT is -# set, attempt to write its JSON summary. Returns 1 for UNSATISFIABLE or -# PR-UNPRODUCIBLE findings when STRICT=1, otherwise 0. -# Rule-reading failures return 1 and emit inspected:false in the JSON report. -# JSON write failures do not affect the return status. +# set, replace any existing report and attempt to write its JSON summary. +# Returns 1 for UNSATISFIABLE, PR-UNPRODUCIBLE or UNSATISFIED-RULE findings +# when STRICT=1, otherwise 0. A required_json or data_rules_json failure returns +# 1 regardless of STRICT and marks the JSON report inspected:false. +# JSON write failures do not affect the return status. Data-rule findings appear +# in both results and the JSON unsatisfiable count. run_audit() { local pubb pubb_pr required ROWS unsatisfiable=0 line ctx app verdict inspected=1 pubb="$(mktemp)"; pubb_pr="$(mktemp)"; required="$(mktemp)"; ROWS="$(mktemp)" @@ -390,10 +612,19 @@ run_audit() { publishable_names pr | sort -u >"$pubb_pr" # Never let a failed inspection leave a stale or empty successful report. [ -z "$JSON_OUT" ] || rm -f "$JSON_OUT" + # Data-dependent rules are findings in their own right; they are appended to + # ROWS before the context loop so they appear in the same report, the same + # JSON and the same allow-list reconciliation. A context name that cannot be + # produced and a rule whose data is never uploaded fail the same way: every + # pull request becomes undecidable. if ! required_json >"$required"; then err "required contexts not inspected — could not read branch rules" inspected=0 : >"$required" # discard any partial results from the failed inspection + elif ! data_rules_json >>"$ROWS"; then + err "data-dependent rules not inspected — could not read branch rules" + inspected=0 + : >"$ROWS" # discard any partial findings from the failed inspection fi say "" @@ -489,6 +720,16 @@ EOS esac done <"$required" + # Count rule findings before serializing the total into the JSON report. + local n_rule n_verified + n_rule="$(awk -F'\t' '$3=="UNSATISFIED-RULE"' "$ROWS" 2>/dev/null | wc -l | tr -d ' ')" + case "$n_rule" in ''|*[!0-9]*) n_rule=0 ;; esac + # A rule that is demonstrably fed is reported, not hidden. Silence would make + # the audit unable to answer "was this rule considered at all?" — the same + # question that goes unanswered when a check quietly never runs. + n_verified="$(awk -F'\t' '$3=="VERIFIED-RULE"' "$ROWS" 2>/dev/null | wc -l | tr -d ' ')" + case "$n_verified" in ''|*[!0-9]*) n_verified=0 ;; esac + unsatisfiable=$((unsatisfiable + n_rule)) if [ -n "$JSON_OUT" ]; then jq -Rs --arg repo "${GITHUB_REPOSITORY:-$ROOT}" --argjson n "$unsatisfiable" \ --argjson inspected "$inspected" --rawfile _ /dev/null ' @@ -497,6 +738,44 @@ EOS | {context: .[0], app_id: (.[1]|tonumber), verdict: .[2]}))}' "$ROWS" >"$JSON_OUT" 2>/dev/null || true info "json report written to $JSON_OUT" fi + # Data-rule findings are appended to ROWS directly, so they never pass through + # the per-context case above. Without counting them here, a strict run whose + # ONLY finding was a data-dependent rule returned 0 — the gate would print the + # reason a pull request cannot merge and then declare success, which is the + # defect this script exists to catch, committed by the script itself. + if [ "$n_verified" -gt 0 ]; then + while IFS=$'\t' read -r r _ _ ev; do + [ -n "$r" ] || continue + info "satisfied $r — $ev" + done </dev/null) +EOF + fi + if [ "$n_rule" -gt 0 ]; then + while IFS= read -r r; do + [ -n "$r" ] || continue + # `$r` is "rule: in ruleset "; the repair depends on the type, + # and so does the sentence. All three branches are findings. + local rtype producer msg + rtype="$(printf '%s' "$r" | sed 's/^rule:\([^ ]*\) .*/\1/')" + producer="$(data_rule_producer "$rtype")" + if [ "$rtype" = code_quality ]; then + msg="required $r needs Code Quality enabled for this repository — a repository setting, and no file in the tree can supply it; every pull request waits for a result that cannot exist" + elif [ -n "$producer" ]; then + msg="required $r has an uploader in the tree ($producer), but that upload cannot succeed until Code Quality is enabled for this repository; every pull request waits for data that is uploaded and rejected" + else + msg="required $r needs data this repository never uploads — every pull request waits for a result that cannot exist; upload the data, or remove the rule" + fi + if [ "$STRICT" = 1 ]; then + err "$msg" + else + warn "$msg" + fi + done </dev/null) +EOF + fi + rm -f "$pubb" "$pubb_pr" "$required" "$ROWS" [ "$inspected" = 1 ] || return 1 [ "$unsatisfiable" -gt 0 ] && [ "$STRICT" = 1 ] && return 1 diff --git a/scripts/check-workflow-staleness.sh b/scripts/check-workflow-staleness.sh index fcff1f39c..646354aea 100755 --- a/scripts/check-workflow-staleness.sh +++ b/scripts/check-workflow-staleness.sh @@ -130,19 +130,30 @@ STANDARDS_REUSABLES=( "scorecard-reusable.yml" ) -# Determine if we are in standards repo +# Determine if we are in the canonical standards repo. The owner changed from +# hyperpolymath to metadatastician; both slugs resolve to GitHub repo 1116521501. +# These are exact identities, not an exemption for arbitrary standards forks. IS_STANDARDS=false -if [ "$GITHUB_REPOSITORY" = "hyperpolymath/standards" ]; then - IS_STANDARDS=true -else - # Fallback: check Git remote origin URL - if [ -d "$REPO_ROOT/.git" ]; then - REMOTE_URL=$(git -C "$REPO_ROOT" remote get-url origin 2>/dev/null || echo "") - if [[ "$REMOTE_URL" =~ "hyperpolymath/standards" ]]; then - IS_STANDARDS=true - fi - fi +REMOTE_URL="" +if [ -d "$REPO_ROOT/.git" ]; then + REMOTE_URL=$(git -C "$REPO_ROOT" remote get-url origin 2>/dev/null || echo "") fi +for standards_repo in hyperpolymath/standards metadatastician/standards; do + if [ "${GITHUB_REPOSITORY:-}" = "$standards_repo" ]; then + IS_STANDARDS=true + break + fi + # Local runs have no Actions context. Match complete HTTPS/SSH origins so a + # consumer named standards-something cannot inherit the canonical exemption. + case "$REMOTE_URL" in + "https://github.com/$standards_repo" | "https://github.com/$standards_repo.git" | \ + "git@github.com:$standards_repo" | "git@github.com:$standards_repo.git" | \ + "ssh://git@github.com/$standards_repo" | "ssh://git@github.com/$standards_repo.git") + IS_STANDARDS=true + break + ;; + esac +done # ── Locate a git checkout that carries standards history ──────────────────── # We need the commit graph (not just HEAD) to answer "is this pin an ancestor diff --git a/scripts/fix-required-context-name.sh b/scripts/fix-required-context-name.sh index 21c537e19..628cf0b56 100755 --- a/scripts/fix-required-context-name.sh +++ b/scripts/fix-required-context-name.sh @@ -159,10 +159,29 @@ if ! "$GH" api -X PUT "repos/$REPO/rulesets/$RULESET" --input "$tmp/after.json" fi # Verify against the server, not against the payload we sent. -"$GH" api "repos/$REPO/rulesets/$RULESET" >"$tmp/verify.json" +# Verify against the server, and treat an unverifiable result as a failure: a +# failed GET leaves an empty file, jq yields empty strings, and `[ "" -ne 1 ]` +# is a syntax error that evaluates false — which would skip the mismatch branch +# and print success after verifying nothing. +if ! "$GH" api "repos/$REPO/rulesets/$RULESET" >"$tmp/verify.json" 2>"$tmp/err"; then + echo "FATAL: could not read back the ruleset after writing it — UNVERIFIED" >&2 + sed 's/^/ /' "$tmp/err" >&2 + exit 1 +fi +if ! jq -e '.rules | type == "array"' "$tmp/verify.json" >/dev/null 2>&1; then + echo "FATAL: read-back is not a ruleset payload — UNVERIFIED" >&2 + exit 1 +fi v_new="$(jq -r --arg c "$NEW_CONTEXT" '[.rules[] | select(.type=="required_status_checks") | .parameters.required_status_checks[]? | select(.context==$c)] | length' "$tmp/verify.json")" v_old="$(jq -r --arg c "$OLD_CONTEXT" '[.rules[] | select(.type=="required_status_checks") | .parameters.required_status_checks[]? | select(.context==$c)] | length' "$tmp/verify.json")" v_cnt="$(jq -r '[.rules[] | select(.type=="required_status_checks") | .parameters.required_status_checks[]?] | length' "$tmp/verify.json")" +for count in "$v_new" "$v_old" "$v_cnt" "$n_ctx"; do + case "$count" in + ''|*[!0-9]*) + echo "FATAL: read-back gave non-numeric counts — UNVERIFIED" >&2 + exit 1 ;; + esac +done if [ "$v_new" -ne 1 ] || [ "$v_old" -ne 0 ] || [ "$v_cnt" -ne "$n_ctx" ]; then echo "FATAL: read-back does not match what was intended." >&2 diff --git a/scripts/remove-ruleset-bypass.sh b/scripts/remove-ruleset-bypass.sh new file mode 100755 index 000000000..1b4930bd5 --- /dev/null +++ b/scripts/remove-ruleset-bypass.sh @@ -0,0 +1,200 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# remove-ruleset-bypass.sh — remove `always` bypass actors from branch rulesets. +# +# WHY THIS EXISTS +# --------------- +# On 2026-10-24 the bypass was removed from `23787415` (main gate), which is the +# ruleset that had actually been used to merge red. Three other active branch +# rulesets were left alone, and two of them still matter less than they look: +# +# 24255701 Require-Linear-History required_linear_history ← the real gap +# 24587723 Lock-Floor 2 status checks ← duplicated +# 24309995 Secret-Scan-Floor 1 status check ← duplicated +# +# Lock-Floor and Secret-Scan-Floor require contexts that `23787415` also +# requires, and `23787415` now has no bypass, so their bypasses discard nothing +# that is not still required next door. `required_linear_history` is the only +# rule in the repository that lives in exactly one ruleset — so its bypass is +# the difference between "append-only merging is enforced" and "append-only +# merging is advice for an admin". +# +# The duplicated ones are still worth clearing: a redundant bypass is a bypass +# waiting to become load-bearing the day somebody removes that check from the +# main gate. +# +# WHAT IT REFUSES +# --------------- +# * a ruleset that is not `active` (editing it changes nothing while looking +# like it did) — reported and skipped rather than written; +# * a payload that would change anything other than `bypass_actors` — the rules, +# conditions, name, target and enforcement are asserted identical before send; +# * a read-back that disagrees with the intent. +# +# Removing every bypass means the emergency path is disabling the ruleset +# deliberately, which is an auditable act rather than a silent merge. That is +# the trade this makes, and it is stated in the output rather than assumed. +# +# Requires `administration: write`. Installation tokens report repo admin=true +# and still get 403 on ruleset writes; use an admin user token. +# +# Usage: +# scripts/remove-ruleset-bypass.sh # report; writes nothing +# scripts/remove-ruleset-bypass.sh --apply # remove from all targeted +# ONLY=24255701 scripts/remove-ruleset-bypass.sh --apply +# REPO=owner/name scripts/remove-ruleset-bypass.sh + +set -uo pipefail + +REPO="${REPO:-hyperpolymath/standards}" +ONLY="${ONLY:-}" # space-separated ruleset ids; empty = every active branch ruleset +GH="${GH:-gh}" # injectable for scripts/tests/remove-ruleset-bypass-test.sh + +APPLY=0 +case "${1:-}" in + --apply) APPLY=1 ;; + -h|--help) sed -n '2,52p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;; + "") ;; + *) echo "unknown argument: $1 (try --help)" >&2; exit 2 ;; +esac + +command -v "$GH" >/dev/null || { echo "'$GH' is required" >&2; exit 2; } +command -v jq >/dev/null || { echo "jq is required" >&2; exit 2; } + +tmp="$(mktemp -d)"; trap 'rm -rf "$tmp"' EXIT + +echo "== $REPO" + +if ! "$GH" api "repos/$REPO/rulesets" >"$tmp/list.json" 2>"$tmp/err"; then + echo "FATAL: cannot list rulesets: $(head -1 "$tmp/err")" >&2 + echo " (403 here means the token cannot see protection at all, not that" >&2 + echo " none exists)" >&2 + exit 1 +fi + +# Filter the list locally rather than with `gh --jq`: the fetch above already +# returned it, and a local jq keeps this working against any gh version and any +# wrapper that does not pass --jq through. +ids="$(jq -r '.[] | select(.target=="branch" and .source_type=="Repository") | .id' "$tmp/list.json")" +if [ -n "$ONLY" ]; then + for id in $ONLY; do + if ! grep -qxF "$id" <<<"$ids"; then + echo "FATAL: targeted ruleset $id is not a repository-owned branch ruleset" >&2 + exit 1 + fi + done + ids="$ONLY" +fi +[ -z "$ids" ] && { echo "no branch rulesets to inspect"; exit 0; } + +# Never rewrite a bypass away by accident on a ruleset that is currently doing +# something we do not understand: this refuses on any surprise. +changed=0; skipped=0; target=0 + +for id in $ids; do + if ! "$GH" api "repos/$REPO/rulesets/$id" >"$tmp/rs.json" 2>"$tmp/err"; then + if [ -n "$ONLY" ]; then + echo "FATAL: cannot read targeted ruleset $id: $(head -1 "$tmp/err")" >&2 + exit 1 + fi + echo "SKIP $id — cannot read it: $(head -1 "$tmp/err")" >&2 + skipped=$((skipped+1)); continue + fi + + name="$(jq -r '.name' "$tmp/rs.json")" + enforcement="$(jq -r '.enforcement' "$tmp/rs.json")" + if ! jq -e '.bypass_actors | type == "array"' "$tmp/rs.json" >/dev/null 2>&1; then + echo "FATAL: bypass actors for $id are unknown — expected an array" >&2 + exit 1 + fi + n_bypass="$(jq -r '.bypass_actors | length' "$tmp/rs.json")" + + if [ "$n_bypass" -eq 0 ]; then + printf ' ok %-9s %-58s no bypass\n' "$id" "$name" + continue + fi + + actors="$(jq -r '[.bypass_actors[] | "\(.actor_type):\(.actor_id)=\(.bypass_mode)"] | join(", ")' "$tmp/rs.json")" + + if [ "$enforcement" != "active" ]; then + printf ' SKIP %-9s %-58s bypass=[%s] but enforcement=%s\n' "$id" "$name" "$actors" "$enforcement" + skipped=$((skipped+1)); continue + fi + + target=$((target+1)) + printf ' FIX %-9s %-58s bypass=[%s]\n' "$id" "$name" "$actors" + + # Build the writable payload ONCE, then derive the before/after pair from it. + # Comparing the payload against the raw API response is wrong: the response + # also carries read-only fields (`id`, `node_id`, `_links`, `created_at`, + # `source`, `current_user_can_bypass`) that are not part of a PUT body, so that + # comparison fails on a payload that is in fact correct. The assertion that + # matters is that MY transformation changed nothing but bypass_actors, so the + # two sides must both be payloads built the same way from the same source. + jq '{name, target, enforcement, conditions, rules, bypass_actors}' \ + "$tmp/rs.json" >"$tmp/before-payload.json" + jq '.bypass_actors = []' "$tmp/before-payload.json" >"$tmp/payload.json" + + if ! diff -q \ + <(jq -S 'del(.bypass_actors)' "$tmp/before-payload.json") \ + <(jq -S 'del(.bypass_actors)' "$tmp/payload.json") >/dev/null; then + echo "FATAL: payload would change more than bypass_actors for $id. Refusing." >&2 + exit 1 + fi + if [ "$(jq -r '.bypass_actors|length' "$tmp/payload.json")" -ne 0 ]; then + echo "FATAL: payload still carries bypass actors for $id. Refusing." >&2 + exit 1 + fi + + if [ "$APPLY" -eq 0 ]; then + changed=$((changed+1)); continue + fi + + if ! "$GH" api -X PUT "repos/$REPO/rulesets/$id" --input "$tmp/payload.json" >/dev/null 2>"$tmp/err"; then + echo "FATAL: write refused for $id:" >&2 + sed 's/^/ /' "$tmp/err" >&2 + echo " 'Resource not accessible by integration' means an installation" >&2 + echo " token: admin=true is not administration:write. Use a user token." >&2 + exit 1 + fi + + # Verify against the server, and treat an unverifiable result as a failure. + # A GET that fails leaves an empty file; jq then yields empty strings, and + # `[ "" -ne 0 ]` is a *syntax* error that evaluates false — so the mismatch + # branch is skipped and the script would print "verified" after performing no + # verification at all. The request must succeed, the response must parse, + # and the complete writable ruleset must match the intended payload. + if ! "$GH" api "repos/$REPO/rulesets/$id" >"$tmp/verify.json" 2>"$tmp/err"; then + echo "FATAL: could not read back $id after writing it — protection is UNVERIFIED" >&2 + sed 's/^/ /' "$tmp/err" >&2 + exit 1 + fi + if ! jq -e '.bypass_actors | type == "array"' "$tmp/verify.json" >/dev/null 2>&1; then + echo "FATAL: read-back for $id is not a ruleset payload — protection is UNVERIFIED" >&2 + exit 1 + fi + v_rules="$(jq -r '.rules|length' "$tmp/verify.json" 2>/dev/null)" + if ! jq -e --slurpfile intended "$tmp/payload.json" ' + {name, target, enforcement, conditions, rules, bypass_actors} == $intended[0] + ' "$tmp/verify.json" >/dev/null 2>&1; then + echo "FATAL: read-back mismatch for $id — writable ruleset differs from intended payload" >&2 + exit 1 + fi + echo " verified: 0 bypass actors, $v_rules rules unchanged" + changed=$((changed+1)) +done + +echo +if [ "$APPLY" -eq 0 ]; then + if [ "$target" -eq 0 ]; then + echo "REPORT-ONLY: no active branch ruleset carries a bypass actor." + else + echo "REPORT-ONLY: $target ruleset(s) would lose their bypass. Re-run with --apply." + echo "The emergency path becomes: disable the ruleset deliberately (audited)," + echo "not merge through it silently." + fi +else + echo "done: $changed ruleset(s) updated, $skipped skipped." +fi diff --git a/scripts/tests/check-required-contexts-test.sh b/scripts/tests/check-required-contexts-test.sh index cbd70d70c..dce70c062 100755 --- a/scripts/tests/check-required-contexts-test.sh +++ b/scripts/tests/check-required-contexts-test.sh @@ -32,9 +32,291 @@ fi if output="$(bash "$GATE" --self-test 2>&1)"; then printf '%s\n' "$output" echo "PASS $0" - exit 0 +else + printf '%s\n' "$output" + echo "::error file=$GATE::fixture suite failed" + exit 1 +fi + +# Exercise live-inspection paths with a fake gh, including failures on the +# second read (data rules) after the context inspection succeeded. +work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT +mkdir -p "$work/bin" "$work/repo/.github/workflows" +cat >"$work/bin/gh" <<'STUB' +#!/usr/bin/env bash +path="${2:-}" +case "$path" in + repos/example/repo) + case "${REPO_MODE:-ok}" in + ok) printf '{"default_branch":"release/current"}' ;; + master) printf '{"default_branch":"master"}' ;; + fail) echo '{"message":"Not Found"}'; exit 1 ;; + no-branch) printf '{"name":"repo"}' ;; + esac ;; + */rulesets) + n="$(cat "$FIXTURES/list-count" 2>/dev/null || echo 0)" + n=$((n+1)); echo "$n" >"$FIXTURES/list-count" + if [ "$n" -gt 1 ]; then + case "$MODE" in + list-fail) cat "$FIXTURES/list.json"; exit 1 ;; + list-invalid) echo '{"message":"error"}'; exit 0 ;; + list-empty) exit 0 ;; + esac + fi + cat "$FIXTURES/list.json" ;; + */rulesets/*) + id="${path##*/}" + n="$(cat "$FIXTURES/detail-$id-count" 2>/dev/null || echo 0)" + n=$((n+1)); echo "$n" >"$FIXTURES/detail-$id-count" + if [ "$n" -gt 1 ]; then + case "$MODE:$id" in + detail-fail:2) cat "$FIXTURES/2.json"; exit 1 ;; + detail-invalid:2) echo '{"message":"error"}'; exit 0 ;; + detail-empty:2) exit 0 ;; + detail-malformed:2) echo '{'; exit 0 ;; + detail-shape:2) echo '{"enforcement":"active","rules":{}}'; exit 0 ;; + esac + fi + cat "$FIXTURES/$id.json" ;; + # Evidence lookups. Defaults are deliberately empty: "no evidence" must be the + # resting state, or every test would pass by accident. + */actions/workflows/*/runs\?status=success\&per_page=30) + case "${RUNS_MODE:-none}" in + ok|foreign|fork-first|missing-head) + jq -n --arg mode "$RUNS_MODE" ' + {id:9, path:".github/workflows/coverage.yml", head_sha:"abcdef1234567890", + head_branch:"main", conclusion:"success", + head_repository:{full_name:"example/repo"}} as $run + | ($run | .id = 10 | .head_repository.full_name = "example/fork") as $fork + | {workflow_runs: (if $mode == "foreign" then [$fork] + elif $mode == "fork-first" then [$fork, $run] + elif $mode == "missing-head" then [$run | del(.head_repository)] + else [$run] end)}' ;; + invalid) echo '{"workflow_runs":{}}'; exit 0 ;; + fail) echo '{"message":"Not Found"}'; exit 1 ;; + *) echo '{"workflow_runs":[]}' ;; + esac ;; + */actions/workflows\?per_page=100) + # gh --paginate emits one JSON document per page; jq folds them, so the stub + # must too. --slurp is deliberately NOT required here: it does not exist in + # every gh release, and the script must not depend on it. + [[ " $* " == *" --paginate "* ]] || exit 1 + [[ " $* " != *" --slurp "* ]] || exit 1 + case "${WF_MODE:-none}" in + quality) echo '{"total_count":1,"workflows":[{"name":"Code Quality","path":"dynamic/github-code-quality/codeql"}]}' ;; + local-quality) echo '{"total_count":1,"workflows":[{"name":"Code Quality","path":".github/workflows/code-quality.yml"}]}' ;; + quality-later) + jq -n '[range(100) | {name:"CI", path:".github/workflows/ci-\(.).yml"}] as $first + | {total_count:101, workflows:$first}, + {total_count:101, workflows:[{name:"Renamed analysis", path:"dynamic/github-code-quality/codeql"}]}' ;; + incomplete) echo '{"total_count":2,"workflows":[{"name":"Code Quality","path":"dynamic/github-code-quality/codeql"}]}' ;; + inconsistent-total) printf '%s\n' '{"total_count":1,"workflows":[]}' '{"total_count":2,"workflows":[{"path":"dynamic/github-code-quality/codeql"}]}' ;; + page-fail) echo '{"total_count":0,"workflows":[]}'; exit 1 ;; + empty-pages) : ;; + invalid) echo '{"total_count":0,"workflows":{}}' ;; + *) echo '{"total_count":0,"workflows":[]}' ;; + esac ;; + */protection/required_status_checks) echo '{"message":"not found"}'; exit 1 ;; + *) exit 1 ;; +esac +STUB +chmod +x "$work/bin/gh" +rule() { # id enforcement includes excludes + jq -n --argjson id "$1" --arg e "$2" --argjson inc "$3" --argjson exc "$4" '{ + id: $id, target: "branch", enforcement: $e, + conditions: {ref_name: {include: $inc, exclude: $exc}}, + rules: [{type:"code_coverage"}, {type:"code_quality"}] + }' >"$work/$1.json" +} +rule 1 active '["~DEFAULT_BRANCH"]' '[]' +rule 2 active '["refs/heads/release/*"]' '[]' +rule 3 evaluate '["~ALL"]' '[]' +rule 4 disabled '["~ALL"]' '[]' +rule 5 active '["refs/heads/main"]' '[]' +rule 6 active '["~ALL"]' '["refs/heads/release/**"]' +rule 7 active '["refs/heads/*"]' '[]' # * must not cross a slash +rule 8 active '["refs/heads/**/curren?"]' '[]' +rule 9 active '["refs/heads/release/[c]urrent"]' '[]' +rule 10 active '["refs/heads/release/current+"]' '[]' # literal regex metachar +rule 11 active '["~ALL"]' '["~DEFAULT_BRANCH"]' +rule 12 active '["refs/heads/release/current"]' '[]' +rule 13 active '["~ALL"]' '["refs/heads/main"]' +rule 14 active '["~ALL"]' '["refs/heads/release/current"]' +jq -n '[range(1;15) | {id:., target:"branch"}]' >"$work/list.json" +pass=0; fail=0 +run() { + rm -f "$work"/*-count + (unset REQUIRED_CONTEXTS_JSON + PATH="$work/bin:$PATH" FIXTURES="${FIXDIR:-$work}" MODE="$1" GITHUB_REPOSITORY=example/repo \ + RUNS_MODE="${RUNS_MODE:-none}" WF_MODE="${WF_MODE:-none}" REPO_MODE="${REPO_MODE:-ok}" \ + REQUIRED_CONTEXTS_STRICT="$2" bash "$GATE" "$work/repo" --no-network --json "$work/report.json" + ) >"$work/out" 2>&1 + rc=$? +} +check() { # label expected-exit jq-assertion + if [ "$rc" = "$2" ] && jq -e "$3" "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS $1" + else + fail=$((fail+1)); echo "FAIL $1 (exit $rc)"; cat "$work/out" "$work/report.json" + fi +} +run ok 0 +check 'only active rules matching the merge branch are counted in JSON' 0 \ + '.inspected == true and .unsatisfiable == 12 and (.results | length) == 12 and + ([.results[].context | capture("ruleset (?[0-9]+)").id] | unique) == ["1","12","13","2","8","9"]' +run ok 1 +check 'strict mode fails when findings consist only of data rules' 1 '.inspected == true and .unsatisfiable == 12' +# An uploader in the tree must change the EXPLANATION and not the verdict. The +# rule stays unsatisfiable until Code Quality is enabled for the repository, so a +# producer must never turn the finding into a pass; what it must do is stop the +# audit claiming the repository "never uploads" while a workflow does. +cat >"$work/repo/.github/workflows/coverage.yml" <<'YML' +name: Coverage +on: [pull_request] +jobs: + c: + runs-on: ubuntu-latest + steps: + - uses: actions/upload-code-coverage@0000000000000000000000000000000000000000 +YML +run ok 1 +if [ "$rc" = 1 ] && grep -q 'has an uploader in the tree (.github/workflows/coverage.yml)' "$work/out" && \ + ! grep -q 'never uploads' "$work/out"; then + pass=$((pass+1)); echo "PASS an uploader in the tree changes the wording and not the verdict" +else + fail=$((fail+1)); echo "FAIL an uploader in the tree changes the wording and not the verdict (exit $rc)"; cat "$work/out" +fi +# Negative control: with no uploader the old sentence must come back, or the test +# above would pass for a script that never looked at the tree at all. +rm -f "$work/repo/.github/workflows/coverage.yml" +run ok 1 +if [ "$rc" = 1 ] && grep -q 'never uploads' "$work/out"; then + pass=$((pass+1)); echo "PASS with no uploader the finding still says nothing uploads" +else + fail=$((fail+1)); echo "FAIL with no uploader the finding still says nothing uploads (exit $rc)"; cat "$work/out" +fi + +# Evidence: a successful run is what makes a data rule satisfied, and file +# presence alone must never be enough. Each fixture below holds one variable. +cat >"$work/repo/.github/workflows/coverage.yml" <<'YML' +name: Coverage +on: [pull_request] +jobs: + c: + runs-on: ubuntu-latest + steps: + - uses: actions/upload-code-coverage@0000000000000000000000000000000000000000 +YML +# A green coverage run is not evidence on its own. The upload action skips +# rather than fails when it cannot attribute the report — fork PRs, and every PR +# whose head-repo name went stale when this repository was renamed. Those runs +# conclude success having uploaded nothing, so while Code Quality is off (and the +# ingestion endpoint therefore absent) a green run must verify nothing. +RUNS_MODE=ok run ok 1 +if [ "$rc" = 1 ] && jq -e '.inspected == true and .unsatisfiable == 12' "$work/report.json" >/dev/null && + ! grep -q 'satisfied rule:code_coverage' "$work/out"; then + pass=$((pass+1)); echo "PASS a green coverage run cannot verify the rule while Code Quality is off" +else + fail=$((fail+1)); echo "FAIL a successful coverage run satisfies only the coverage rule (exit $rc)"; cat "$work/out" fi +# Code Quality must be present for a coverage upload to have any chance of being +# accepted, so these evidence tests enable it and then vary only the run data. +RUNS_MODE=fork-first WF_MODE=quality run ok 0 +check 'coverage evidence selects a repository run after a newer fork run' 0 \ + '.inspected == true and .unsatisfiable == 0 and + ([.results[] | select(.verdict == "VERIFIED-RULE" and (.context | startswith("rule:code_coverage")))] | length) == 6' +for mode in foreign missing-head; do + RUNS_MODE="$mode" run ok 1 + check "$mode coverage runs cannot verify a rule" 1 \ + '.inspected == true and .unsatisfiable == 12 and + ([.results[] | select(.verdict == "VERIFIED-RULE")] | length) == 0' +done +# Same tree, no successful run: the uploader exists and proves nothing. This is +# the branch's first six runs, which all failed 404 while the feature was off. +run ok 1 +if [ "$rc" = 1 ] && jq -e '.inspected == true and .unsatisfiable == 12' "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS an uploader with no successful run is not evidence" +else + fail=$((fail+1)); echo "FAIL an uploader with no successful run is not evidence (exit $rc)"; cat "$work/out" +fi +WF_MODE=quality run ok 1 +if [ "$rc" = 1 ] && jq -e '.inspected == true and .unsatisfiable == 6' "$work/report.json" >/dev/null && + grep -q 'satisfied rule:code_quality' "$work/out"; then + pass=$((pass+1)); echo "PASS a Code Quality workflow satisfies the quality rule independently" +else + fail=$((fail+1)); echo "FAIL a Code Quality workflow satisfies the quality rule independently (exit $rc)"; cat "$work/out" +fi +WF_MODE=local-quality run ok 1 +check 'a repository workflow named Code Quality cannot verify the quality rule' 1 \ + '.inspected == true and .unsatisfiable == 12 and + ([.results[] | select(.verdict == "VERIFIED-RULE")] | length) == 0' +WF_MODE=quality-later run ok 1 +check 'a generated quality workflow beyond 100 entries is found by path despite its renamed display name' 1 \ + '.inspected == true and .unsatisfiable == 6 and + ([.results[] | select(.verdict == "VERIFIED-RULE" and (.context | startswith("rule:code_quality")))] | length) == 6' +RUNS_MODE=ok WF_MODE=quality run ok 1 +if [ "$rc" = 0 ] && jq -e '.inspected == true and .unsatisfiable == 0' "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS strict mode passes when both data rules are demonstrably fed" +else + fail=$((fail+1)); echo "FAIL strict mode passes when both data rules are demonstrably fed (exit $rc)"; cat "$work/out" +fi +# A lookup that fails must not be read as "no data" (a silent finding) nor as +# "data present" (a silent pass). It is an inspection failure. +RUNS_MODE=invalid WF_MODE=quality run ok 1 +if [ "$rc" = 1 ] && jq -e '.inspected == false and .results == []' "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS a malformed runs response is an inspection failure, not a verdict" +else + fail=$((fail+1)); echo "FAIL a malformed runs response is an inspection failure, not a verdict (exit $rc)"; cat "$work/out" +fi +WF_MODE=invalid run ok 1 +if [ "$rc" = 1 ] && jq -e '.inspected == false and .results == []' "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS a malformed workflows response is an inspection failure, not a verdict" +else + fail=$((fail+1)); echo "FAIL a malformed workflows response is an inspection failure, not a verdict (exit $rc)"; cat "$work/out" +fi +for mode in incomplete inconsistent-total page-fail empty-pages; do + WF_MODE="$mode" run ok 1 + check "$mode workflow catalogue is an inspection failure" 1 '.inspected == false and .results == []' +done +# A ruleset set with no data-dependent rules. Evidence for those rules must not +# be fetched at all, because the endpoint it lives on is unrelated to reading +# protection — and a failure there must not condemn a perfectly readable audit. +mkdir -p "$work/nodata" +echo '[{"id":1,"target":"branch"}]' >"$work/nodata/list.json" +jq -n '{id:1, target:"branch", enforcement:"active", + conditions:{ref_name:{include:["~ALL"], exclude:[]}}, + rules:[{type:"deletion"}]}' >"$work/nodata/1.json" +FIXDIR="$work/nodata" RUNS_MODE=fail run ok 1 +if [ "$rc" = 0 ] && jq -e '.inspected == true and .unsatisfiable == 0' "$work/report.json" >/dev/null; then + pass=$((pass+1)); echo "PASS a failing evidence endpoint cannot condemn an audit that needs no evidence" +else + fail=$((fail+1)); echo "FAIL a failing evidence endpoint cannot condemn an audit that needs no evidence (exit $rc)"; cat "$work/out" "$work/report.json" +fi + +# An unresolvable default branch must not be silently replaced by "main": on a +# repository whose default branch is `master`, `main` matches no ref filter, the +# rule below is filtered out, and the audit reports a clean gate it never +# examined. +mkdir -p "$work/master" +echo '[{"id":1,"target":"branch"}]' >"$work/master/list.json" +jq -n '{id:1, target:"branch", enforcement:"active", + conditions:{ref_name:{include:["refs/heads/master"], exclude:[]}}, + rules:[{type:"code_quality"}]}' >"$work/master/1.json" +for mode in fail no-branch; do + FIXDIR="$work/master" REPO_MODE="$mode" run ok 0 + check "an unreadable default branch ($mode) is an inspection failure, not an empty success" 1 \ + '.inspected == false and .results == []' +done +# The same fixtures with a readable branch must still find the rule: the guard +# must not be so broad that it hides real findings. +FIXDIR="$work/master" REPO_MODE=master run ok 1 +check "a readable default branch still reports the rule it applies to" 1 \ + '.inspected == true and .unsatisfiable == 1' -printf '%s\n' "$output" -echo "::error file=$GATE::fixture suite failed" -exit 1 +rm -f "$work/repo/.github/workflows/coverage.yml" +for mode in list-fail list-invalid list-empty detail-fail detail-invalid detail-empty detail-malformed detail-shape; do + run "$mode" 0 + check "$mode does not become an empty success or retain partial findings" 1 '.inspected == false and .results == []' +done +printf '\nAPI fixture tests: %s passed, %s failed\n' "$pass" "$fail" +[ "$fail" -eq 0 ] diff --git a/scripts/tests/check-workflow-staleness-test.sh b/scripts/tests/check-workflow-staleness-test.sh index 46c75d14b..d33171a28 100755 --- a/scripts/tests/check-workflow-staleness-test.sh +++ b/scripts/tests/check-workflow-staleness-test.sh @@ -201,6 +201,45 @@ run_case "consumer direct Scorecard SARIF upload fails" 1 "$R" run_case "canonical standards Scorecard SARIF uploader passes" 0 "$R" \ GITHUB_REPOSITORY=hyperpolymath/standards +# The canonical repository moved owners; both names resolve to GitHub repo +# 1116521501. Its own workflows must not be classified as consumer copies. +R="$TEST_DIR/standards-self"; mk_repo "$R" +cp "$TEST_DIR/sarif/.github/workflows/scorecard.yml" "$R/.github/workflows/scorecard-reusable.yml" +touch "$R/.github/workflows/scorecard-enforcer.yml" +run_case "canonical standards retains both structural exemptions" 0 "$R" \ + GITHUB_REPOSITORY=hyperpolymath/standards +run_case "renamed standards retains both structural exemptions" 0 "$R" \ + GITHUB_REPOSITORY=metadatastician/standards + +# Local runs have no Actions context; identify the exact GitHub origin, never +# a consumer whose name merely contains the standards slug. +git -C "$R" init -q +git -C "$R" remote add origin https://github.com/metadatastician/standards.git +for origin in \ + https://github.com/hyperpolymath/standards.git \ + https://github.com/metadatastician/standards \ + https://github.com/metadatastician/standards.git \ + git@github.com:metadatastician/standards.git \ + ssh://git@github.com/metadatastician/standards.git; do + git -C "$R" remote set-url origin "$origin" + run_case "canonical origin $origin is recognised without Actions context" 0 "$R" \ + GITHUB_REPOSITORY= +done +for origin in \ + https://github.com/hyperpolymath/standards-consumer.git \ + https://github.com/metadatastician/standards-consumer.git \ + https://example.com/metadatastician/standards.git; do + git -C "$R" remote set-url origin "$origin" + run_case "consumer origin $origin still fails structural checks" 1 "$R" \ + GITHUB_REPOSITORY= +done +git -C "$R" remote remove origin +run_case "a consumer named standards does not gain exemptions" 1 "$R" \ + GITHUB_REPOSITORY=example/standards +write_pin "$R/.github/workflows/governance.yml" "governance-reusable.yml" "$FORGED" +run_case "renamed standards still rejects forged reusable pins" 1 "$R" \ + GITHUB_REPOSITORY=metadatastician/standards + # ── 10. No workflows dir -> pass ──────────────────────────────────────────── R="$TEST_DIR/empty"; mkdir -p "$R" run_case "no .github/workflows passes" 0 "$R" diff --git a/scripts/tests/fix-required-context-name-test.sh b/scripts/tests/fix-required-context-name-test.sh index f9e156598..f658d70de 100755 --- a/scripts/tests/fix-required-context-name-test.sh +++ b/scripts/tests/fix-required-context-name-test.sh @@ -87,7 +87,8 @@ out=""; rc=0 run() { # fixture-file [arg] : >"$work/written" FIXTURE="$1" WRITTEN="$work/written" PUT_CAPTURE="$work/payload.json" \ - REFUSE="${REFUSE:-0}" GH="$work/gh" \ + REFUSE="${REFUSE:-0}" GH="$work/gh" PATH="$work/bin:$PATH" \ + COUNT_TARGET="${COUNT_TARGET:-}" COUNT_VALUE="${COUNT_VALUE:-}" \ bash "$script" "${2:-}" >"$work/out" 2>&1 rc=$? out="$(cat "$work/out")" @@ -150,6 +151,43 @@ REFUSE=1; run "$work/old-only.json" --apply; check "403 exits 1" 1 "$rc" REFUSE=0 check_says "403 explains the installation-token trap" "installation token" "$out" +# Each query result must be validated independently. Simulate a jq query that +# produces no count or a non-numeric count while the other queries still work. +mkdir -p "$work/bin" +real_jq="$(command -v jq)" +cat >"$work/bin/jq" <<'SH' +#!/usr/bin/env bash +query="$*" +field="" +case "$query" in + *verify.json*) + case "$query" in + *'Hypatia verdict grid / Hypatia Neurosymbolic Analysis'*) field=v_new ;; + *'scan / Hypatia Neurosymbolic Analysis'*) field=v_old ;; + *'required_status_checks[]?] | length'*) field=v_cnt ;; + esac ;; + *before.json*) + case "$query" in *'required_status_checks[]?] | length'*) field=n_ctx ;; esac ;; +esac +if [ -n "$field" ] && [ "$field" = "$COUNT_TARGET" ]; then + printf '%s' "$COUNT_VALUE"; exit 1 +fi +exec "$REAL_JQ" "$@" +SH +chmod +x "$work/bin/jq" +export REAL_JQ="$real_jq" +for field in v_new v_old v_cnt n_ctx; do + for value in '' invalid; do + COUNT_TARGET="$field" COUNT_VALUE="$value" run "$work/old-only.json" --apply + check "$field value '$value' exits 1" 1 "$rc" + if [ "$field" = n_ctx ]; then + check_says "invalid original count refused before write" "Refusing to send" "$out" + else + check_says "$field value '$value' is unverified" "UNVERIFIED" "$out" + fi + done +done + # 8. unknown argument is not silently ignored run "$work/old-only.json" --force; check "unknown arg exits 2" 2 "$rc" diff --git a/scripts/tests/remove-ruleset-bypass-test.sh b/scripts/tests/remove-ruleset-bypass-test.sh new file mode 100755 index 000000000..f1a61db88 --- /dev/null +++ b/scripts/tests/remove-ruleset-bypass-test.sh @@ -0,0 +1,212 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# SPDX-FileCopyrightText: 2026 Jonathan D.A. Jewell +# +# Offline tests for scripts/remove-ruleset-bypass.sh. +# +# This script edits branch protection for six rulesets at once, so the tests +# concentrate on the two ways that goes wrong: writing when it should have only +# reported, and writing a payload that changes something other than the bypass. +# A fake `gh` (via $GH) serves fixtures and records every PUT. +# +# Requires jq. No network. + +set -uo pipefail + +here="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +script="$here/../remove-ruleset-bypass.sh" +pass=0; fail=0 +work="$(mktemp -d)"; trap 'rm -rf "$work"' EXIT + +# Fixtures: a bypassed ruleset, one with no bypass, and an inactive one. +mk() { # id name enforcement bypass + jq -n --argjson id "$1" --arg n "$2" --arg e "$3" --argjson b "$4" '{ + id: $id, node_id: "RRS_x", name: $n, target: "branch", source_type: "Repository", + source: "hyperpolymath/standards", enforcement: $e, + created_at: "2026-10-06T14:41:36.643Z", current_user_can_bypass: "always", + _links: {self: {href: "x"}, html: {href: "y"}}, + conditions: {ref_name: {exclude: [], include: ["~DEFAULT_BRANCH"]}}, + bypass_actors: (if $b == 1 then [{actor_id: 5, actor_type: "RepositoryRole", bypass_mode: "always"}] else [] end), + rules: [{type: "required_status_checks", parameters: { + strict_required_status_checks_policy: true, do_not_enforce_on_create: false, + required_status_checks: [{context: "scan / gitleaks", integration_id: 15368}]}}] + }' +} + +mk 1 "Require-Linear-History" active 1 >"$work/rs1.json" +mk 2 "Lock-Floor" active 1 >"$work/rs2.json" +mk 3 "main gate" active 0 >"$work/rs3.json" +mk 4 "Dormant" evaluate 1 >"$work/rs4.json" + +cat >"$work/gh" <<'SH' +#!/usr/bin/env bash +set -uo pipefail +method=GET; args=(); payload="" +while [ $# -gt 0 ]; do + case "$1" in + -X) method="$2"; shift 2 ;; + --input) payload="$2"; shift 2 ;; + --jq) jqexpr="$2"; shift 2 ;; + *) args+=("$1"); shift ;; + esac +done +path="${args[*]:-}" +if [ "$method" = PUT ]; then + id="${path##*/}" + cp "${payload:?}" "$PUT_DIR/$id.json" + if [ "${REFUSE:-0}" = 1 ]; then + echo '{"message":"Resource not accessible by integration","status":"403"}' >&2 + exit 1 + fi + # Serve the written payload back on subsequent reads. + jq "${MUTATION:-.}" "$PUT_DIR/$id.json" | jq '. + {id: 1, _links: {}}' >"$SERVE_DIR/$id.json" + exit 0 +fi +case "$path" in + */rulesets) cat "$SERVE_DIR/list.json" ;; + */rulesets/*) + id="${path##*/}" + echo "$id" >>"$SERVE_DIR/reads" + if [ "${READ_FAIL:-}" = "$id" ]; then + echo "fixture read failure" >&2; exit 1 + fi + if [ -n "${BYPASS_SHAPE:-}" ] && [ ! -f "$PUT_DIR/$id.json" ]; then + jq "$BYPASS_SHAPE" "$SRC_DIR/rs$id.json"; exit 0 + fi + if [ "${VERIFY_FAIL:-0}" = 1 ] && [ -f "$PUT_DIR/$id.json" ]; then + echo "curl: (52) Empty reply from server" >&2; exit 1 # GET after PUT + fi + if [ "${CORRUPT:-0}" = 1 ] && [ -f "$PUT_DIR/$id.json" ]; then + echo '{"message":"Not Found"}'; exit 0 # parses, wrong shape + fi + if [ -f "$SERVE_DIR/$id.json" ]; then cat "$SERVE_DIR/$id.json"; else cat "$SRC_DIR/rs$id.json"; fi ;; + *) echo "unexpected: $path" >&2; exit 1 ;; +esac +SH +chmod +x "$work/gh" + +setup() { + rm -rf "$work/src" "$work/serve" "$work/put" + mkdir -p "$work/src" "$work/serve" "$work/put" + cp "$work"/rs*.json "$work/src/" + cp "$work"/rs*.json "$work/serve/" + jq -n '([range(1;5) | {id:.,target:"branch",source_type:"Repository"}] + [{id:5,target:"branch",source_type:"Organization"},{id:6,target:"tag",source_type:"Repository"}])' \ + >"$work/serve/list.json" +} + +out=""; rc=0 +run() { + SRC_DIR="$work/src" SERVE_DIR="$work/serve" PUT_DIR="$work/put" \ + REFUSE="${REFUSE:-0}" VERIFY_FAIL="${VERIFY_FAIL:-0}" CORRUPT="${CORRUPT:-0}" \ + READ_FAIL="${READ_FAIL:-}" BYPASS_SHAPE="${BYPASS_SHAPE:-}" MUTATION="${MUTATION:-.}" \ + GH="$work/gh" REPO="example/repo" ONLY="${ONLY:-}" \ + bash "$script" "$@" >"$work/out" 2>&1 + rc=$?; out="$(cat "$work/out")" +} +check() { local l="$1" w="$2" g="$3" + if [ "$g" = "$w" ]; then pass=$((pass+1)); echo "PASS $l" + else fail=$((fail+1)); echo "FAIL $l — want $w got $g"; fi; } +check_says() { local l="$1" w="$2" + if printf '%s' "$out" | grep -qF "$w"; then pass=$((pass+1)); echo "PASS $l" + else fail=$((fail+1)); echo "FAIL $l — output lacks: $w"; printf '%s\n' "$out" | tail -4; fi; } + +# 1. report-only: identifies the three bypasses, writes nothing +setup; run +check "report-only exits 0" 0 "$rc" +check_says "report-only counts them" "2 ruleset(s) would lose their bypass" +if [ -z "$(ls -A "$work/put")" ]; then pass=$((pass+1)); echo "PASS report-only wrote nothing" +else fail=$((fail+1)); echo "FAIL report-only wrote: $(ls "$work/put")"; fi + +# 2. --apply writes each, and only bypass_actors changes +setup; run --apply +check "apply exits 0" 0 "$rc" +# rs1 and rs2 are active WITH a bypass; rs3 has none; rs4 is inactive and must +# be skipped. Two is the correct number, not four. +check "apply wrote the 2 active bypassed rulesets" 2 "$(ls "$work/put" | wc -l | tr -d ' ')" +check "all payloads have zero bypass" "0 0" "$(for f in "$work/put"/*.json; do jq -r '.bypass_actors|length' "$f"; done | tr '\n' ' ' | sed 's/ $//')" +check "rules preserved in every payload" "1 1" "$(for f in "$work/put"/*.json; do jq -r '.rules|length' "$f"; done | tr '\n' ' ' | sed 's/ $//')" +check "conditions preserved" "1 1" "$(for f in "$work/put"/*.json; do jq -r 'if .conditions.ref_name.include==["~DEFAULT_BRANCH"] then 1 else 0 end' "$f"; done | tr '\n' ' ' | sed 's/ $//')" +check "read-only fields not sent" "0 0" "$(for f in "$work/put"/*.json; do jq -r 'if has("id") or has("_links") or has("created_at") then 1 else 0 end' "$f"; done | tr '\n' ' ' | sed 's/ $//')" + +# 3. an inactive ruleset is skipped, not written +setup; ONLY=4 run --apply +check "inactive skipped, exit 0" 0 "$rc" +check_says "inactive reported as skipped" "enforcement=evaluate" +check "inactive not written" 0 "$(ls "$work/put" | wc -l | tr -d ' ')" + +# 4. a ruleset with no bypass is left alone +setup; ONLY=3 run --apply +check "no-bypass exits 0" 0 "$rc" +check "no-bypass not written" 0 "$(ls "$work/put" | wc -l | tr -d ' ')" + +# 5. targeted rewrite only touches the named ruleset +setup; ONLY=2 run --apply +check "targeted wrote exactly 1" 1 "$(ls "$work/put" | wc -l | tr -d ' ')" +check "targeted wrote the right one" "2.json" "$(ls "$work/put")" + +# 6. the API refuses (installation token): explain the trap, exit 1 +setup; REFUSE=1; run --apply +check "403 exits 1" 1 "$rc" +check_says "403 explains the installation-token trap" "installation" +REFUSE=0 + +# 7. the read-back GET fails after a successful PUT. Reported by Codex on #1338 +# and a genuine bug in the first version: an empty verify.json made +# `[ "" -ne 0 ]` a syntax error, which evaluated false, skipped the mismatch +# branch, and printed "verified" after verifying nothing. +setup; VERIFY_FAIL=1; run --apply +check "read-back failure exits 1" 1 "$rc" +check_says "read-back failure says UNVERIFIED" "UNVERIFIED" +if printf '%s' "$out" | grep -q "verified: 0 bypass"; then + fail=$((fail+1)); echo "FAIL read-back failure still claimed verification" +else pass=$((pass+1)); echo "PASS read-back failure did not claim verification"; fi +VERIFY_FAIL=0 + +# 8. the read-back returns valid JSON but not a ruleset (no bypass_actors array) +setup; CORRUPT=1; run --apply +check "corrupt read-back exits 1" 1 "$rc" +check_says "corrupt read-back says UNVERIFIED" "UNVERIFIED" +CORRUPT=0 + +# Inherited and non-branch rulesets must never reach the repository endpoint. +setup; run --apply +check "inherited and tag rulesets not read" 0 "$(grep -Ec '^(5|6)$' "$work/serve/reads")" +setup; ONLY=5 run --apply +check "explicit inherited target refused" 1 "$rc" +check "explicit inherited target not written" 0 "$(ls "$work/put" | wc -l | tr -d ' ')" + +# A failed explicit target is a failure; broad scans retain their skip policy. +setup; ONLY='1 2' READ_FAIL=2 run --apply +check "unreadable explicit target fails" 1 "$rc" +check_says "unreadable explicit target named" "cannot read targeted ruleset 2" +setup; READ_FAIL=2 run --apply +check "unreadable non-targeted ruleset skipped" 0 "$rc" +check_says "unreadable non-targeted ruleset named" "SKIP 2" + +for shape in 'del(.bypass_actors)' '.bypass_actors=null' '.bypass_actors={}' '.bypass_actors=""'; do + setup; ONLY=1 BYPASS_SHAPE="$shape" run --apply + check "unknown bypass shape $shape fails" 1 "$rc" + check_says "unknown bypass shape $shape reported" "expected an array" + check "unknown bypass shape $shape not written" 0 "$(ls "$work/put" | wc -l | tr -d ' ')" +done + +# Equal rule counts cannot hide changed rules or other writable fields. +for mutation in \ + '.rules[0].parameters.required_status_checks[0].context="different check"' \ + '.conditions.ref_name.include=["~ALL"]' \ + '.enforcement="disabled"' \ + '.name="different name"' \ + '.target="tag"' \ + '.bypass_actors=[{actor_type:"RepositoryRole",actor_id:5,bypass_mode:"always"}]'; do + setup; ONLY=1 MUTATION="$mutation" run --apply + check "read-back mutation $mutation fails" 1 "$rc" + check_says "read-back mutation $mutation diagnosed" "read-back mismatch" +done + +# 9. unknown argument is not silently ignored +run --force +check "unknown arg exits 2" 2 "$rc" + +echo +echo "self-test: $pass passed, $fail failed" +[ "$fail" -eq 0 ]