coverage-hourly.yml

name: Coverage Hourly

on:
  schedule:
    # Off the top-of-the-hour slot: GitHub delays or drops scheduled runs in
    # its most congested window, and this cron was observed firing every
    # 2-5h instead of hourly (#6627). :23 is not used by any other workflow.
    - cron: '23 * * * *'
  # Non-schedule backstop (#10877): schedule: delivery is starved (~one tick
  # per 6-7h), so also measure v5 after each successful post-merge image build,
  # mirroring tagged-release.yml's workflow_run trigger (#5318). The `gate`
  # job de-duplicates against cron and repeat events.
  workflow_run:
    workflows: ["Build and Push Docker Image"]
    types: [completed]
  workflow_dispatch:
  # No pull_request trigger anymore (#4112). The pre-merge half of this gate —
  # added in #2350's aftermath and moved to v4 in #3903 — now lives in
  # v2-tests.yml's `coverage` job, which parses the coverprofile emitted by the
  # SAME sharded `go test -short -race -count=1` run that gates the PR, so PRs
  # run the test suite exactly once instead of twice. This workflow remains the
  # scheduled hourly monitor for the v5 branch (the default branch, #9632),
  # including the create-issue-on-coverage-drop machinery below.
  #
  # v5 is covered by the pre-merge half too: v2-tests.yml's `pull_request`
  # filter is `branches: [v4, v5, v6]`, so the coverage floor already gates
  # every v5 PR. This workflow is the hourly monitor for whatever lands on
  # v5 between PRs (merge commits, direct pushes) and the issue-filer of
  # record for a sustained drop.
  #
  # KEEP THE FLOORS IN SYNC: PKG_THRESHOLD below is duplicated in
  # v2-tests.yml's coverage job; a floor change must land in both.

permissions:
  actions: read
  contents: read
  issues: write

jobs:
  gate:
    # Non-schedule backstop (#10877): repository-wide schedule: delivery is
    # starved (~one tick per 6-7h), so this workflow also runs when its
    # upstream workflow completes. Bounded de-dup: skip when another run of
    # this workflow for the same commit started within the last 30
    # minutes (cron + event must not double-run).
    if: >-
      github.event_name != 'workflow_run' || (
      github.event.workflow_run.conclusion == 'success' &&
      github.event.workflow_run.head_branch == 'v5' )
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      run: ${{ steps.dedup.outputs.run }}
    steps:
      - name: Skip when a recent run already covers this commit
        id: dedup
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          EVENT: ${{ github.event_name }}
          SHA: ${{ github.event.workflow_run.head_sha || github.sha }}
          SELF_RUN_ID: ${{ github.run_id }}
        run: |
          set -euo pipefail
          since=$(date -u -d '30 minutes ago' +%Y-%m-%dT%H:%M:%SZ)
          # runs[].head_sha is the v5 HEAD at trigger time, not the upstream
          # build's SHA, and a run whose `coverage` job this gate skipped still
          # concludes "success". Counting those as covering the commit made every
          # gate-only run block the next real measurement. Only a run whose
          # coverage job ran, or is still running, counts.
          recent=0
          for id in $(gh api "repos/${REPO}/actions/workflows/coverage-hourly.yml/runs?per_page=30&created=>=${since}" \
              --jq "[.workflow_runs[] | select(.id != (env.SELF_RUN_ID | tonumber)) | select(.status != \"completed\" or .conclusion == \"success\") | select(.event == \"workflow_dispatch\" | not) | select(.head_sha == env.SHA or (env.EVENT == \"schedule\"))] | .[].id" || true); do
            measured=$(gh api "repos/${REPO}/actions/runs/${id}/jobs" \
              --jq '[.jobs[] | select(.name == "coverage") | select(.status != "completed" or .conclusion == "success")] | length' || echo 0)
            if [ "${measured:-0}" != "0" ]; then
              recent=$((recent + 1))
            fi
          done
          if [ "$EVENT" = "workflow_dispatch" ] || [ "$recent" = "0" ]; then
            echo "run=true" >> "$GITHUB_OUTPUT"
          else
            echo "::notice::${recent} recent coverage-hourly.yml run(s) already measured ${SHA}; skipping"
            echo "run=false" >> "$GITHUB_OUTPUT"
          fi

  coverage:
    needs: gate
    if: needs.gate.outputs.run == 'true'
    runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }}
    # Hard cap so a hung runner / stuck -race build auto-fails instead of
    # blocking PR merges indefinitely (the go test -timeout only bounds the test
    # step, not checkout/setup/build/post-processing or a wedged runner). #3615.
    # Two sequential full go test passes now run ~18–20 min on shared runners, so
    # 20 minutes was cancelling the second pass on green runs; 30 still auto-fails
    # hung runners. #15266.
    timeout-minutes: 30
    # Surfaced so create-issue-on-coverage-drop can report WHAT failed and on
    # WHICH commit (#3903). Without these the filed issue could only say
    # "packages below 90%" and quote github.sha — the SHA of the branch that
    # TRIGGERED the run, not the one the checkout above measured.
    outputs:
      below_threshold: ${{ steps.coverage.outputs.below_threshold }}
      failed_packages: ${{ steps.coverage.outputs.failed_packages }}
      total: ${{ steps.coverage.outputs.total }}
      measured_sha: ${{ steps.coverage.outputs.measured_sha }}
      measured_ref: ${{ steps.coverage.outputs.measured_ref }}
    defaults:
      run:
        working-directory: src
    steps:
      # On `schedule`/`workflow_dispatch` there is no PR ref, so pin explicitly
      # to the branch whose floor this gate reports on hourly.
      #
      # That branch is v5 (#9632) — the default branch and the one Tide
      # actually merges to. It was v4 (#3903) before that, and v2 before that;
      # each move happened because the previous branch stopped being where the
      # fleet lands work, so a gate measuring it was reporting on a branch
      # nobody could fix from. Pinning to a frozen branch has repeatedly cost
      # more than "stale numbers": moving v2->v4 surfaced 14 packages that
      # never existed on v2 and so had never been subject to any floor.
      #
      # (This workflow no longer runs on pull_request — the pre-merge floor
      # check lives in v2-tests.yml now, see #4112 — so the ref can be pinned
      # unconditionally.)
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: 'v5'

      # Populate the shared node-local Go tool cache under the runner pods'
      # node-wide lock before setup-go, so parallel jobs on a cold cache (a
      # src/go.mod bump) stop clobbering each other's rm -rf + copy (#11175,
      # #10234). Never fails the job.
      - name: Warm shared Go tool cache (serialize the cold-cache population)
        # The checkout above is pinned to the measured ref; skip when that ref
        # does not carry the script yet (same reasoning as the cache guard below).
        run: |
          guard="$GITHUB_WORKSPACE/.github/scripts/go-toolcache-guard.sh"
          if [ -f "$guard" ]; then
            bash "$guard"
          else
            echo "::notice::go-toolcache-guard.sh not present on the measured ref; setup-go populates the tool cache itself"
          fi
      - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
        with:
          go-version-file: src/go.mod
          cache: false
      - name: Guard shared Go cache (fall back when the runner volume is full)
        # The checkout above is pinned to the MEASURED ref, not the branch this
        # workflow file lives on, so the helper script only exists when that ref
        # carries it. v4 predates #9338, and calling the missing script exited
        # 127 before any test ran - every hourly run went red and filed a
        # "packages below 90% threshold" issue that no test could fix (#9632).
        # The guard never fails a job by design, so skipping it when absent is
        # the same "keep original cache settings" outcome.
        run: |
          guard="$GITHUB_WORKSPACE/.github/scripts/go-cache-guard.sh"
          if [ -f "$guard" ]; then
            bash "$guard"
          else
            echo "::notice::go-cache-guard.sh not present on the measured ref; keeping default Go cache settings"
          fi

      - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
        with:
          node-version: '22'
      - name: Guard shared node tool cache (repair a half-written entry)
        # setup-node reports "Found in cache" for a node/<ver>/x64 entry whose
        # .complete marker exists but bin/npm does not (#10505, #10518). The
        # canary repairs this too, but its cron is starved to a ~5h cadence, so
        # heal the node from the hot path: drop the marker (never the directory,
        # a sibling job may be mid-run) so the next job re-extracts.
        if: ${{ always() }}
        run: |
          set -u
          command -v npm >/dev/null 2>&1 && exit 0
          for marker in "${RUNNER_TOOL_CACHE:-/nonexistent}"/node/*/x64.complete; do
            [ -e "$marker" ] || continue
            dir="${marker%.complete}"
            [ -x "$dir/bin/npm" ] && continue
            echo "::warning::node tool cache $dir has a .complete marker but no executable bin/npm; removing the marker so the next job on $RUNNER_NAME rebuilds it"
            rm -f "$marker"
          done
          echo "::error::npm is not on PATH after actions/setup-node on $RUNNER_NAME (half-written shared tool cache, see #10518); marker removed, re-run this job"
          exit 1

      # The coverage run must NOT abort this job on a test failure: the
      # per-package check below is what turns a failure into a readable report.
      # Previously a failing package exited non-zero here and the check step was
      # `skipped` entirely, so the gate never actually ran (see the comment on
      # the parsing loop).
      - name: Test with coverage
        id: cover
        continue-on-error: true
        run: |
          go test ./pkg/... ./cmd/... ./internal/... ./crux/... -short -race -coverprofile=coverage.out -count=1 -timeout 600s \
            2>&1 | tee race-run.log
          exit "${PIPESTATUS[0]}"

      - name: Check per-package coverage
        id: coverage
        run: |
          THRESHOLD=90
          FAILED=""
          BROKEN=""
          REPORT=""

          # Per-package threshold overrides. A handful of packages are heavily
          # coupled to the live runtime (fixed /data,/proc,/var/run paths, real
          # github.com/platform.claude.com device-flow, a live tmux agent buffer,
          # an in-cluster k8s serviceaccount) and cannot be exercised hermetically
          # in CI without invasive production seams. They get a documented lower
          # floor so the gate still catches regressions below their real ceiling,
          # rather than blocking forever on untestable code. Raise these toward 90
          # as seams are added; do not add new entries without justification here.
          #
          # KEEP IN SYNC with the copy of this map in v2-tests.yml's coverage
          # job (the pre-merge half of this gate, #4112).
          declare -A PKG_THRESHOLD=(
            [dashboard]=84   # 86.2% on 2026-09-01; the residue is the inception tmux watcher + copilot/claude device-flow + /data file I/O + k8s
            [hub]=87         # kubectl-shelling + live-GitHub paths; see #2355. Was 89 against v2's 89.9%; v4/v5 measure 87.6%.
            [crux/agent]=89  # pre-existing shortfall inherited from #2350, NOT a new regression. See #2355. Keyed crux/<pkg> since the module cut (#13536)

            # ── Floors recorded when the gate moved from v2 to v4 (#3903) ─────
            #
            # These are a RATCHET, NOT A TARGET. Every entry below is a package
            # the gate had never measured: either it does not exist on v2 at all
            # (the gate's old ref), or it grew on v4 while the gate watched v2.
            # None of this is a regression that slipped past the gate — the gate
            # was pointed somewhere else.
            #
            # Each floor is the value measured on v4 at d89121f5, rounded DOWN to
            # the integer the check compares, so the gate starts catching further
            # slippage immediately instead of blocking every v4 PR on day one.
            # The alternative — switching to v4 with the bare 90% default — turns
            # CI red for 14 packages at once and blocks work that has nothing to
            # do with coverage, which is how a gate gets disabled rather than
            # satisfied.
            #
            # Raise these toward 90 as tests land; never raise one to paper over
            # a drop. A package that improves should have its floor moved up in
            # the same PR, so the ratchet only turns one way.
            #
            # ── Ratchet turned for #4570 ─────────────────────────────────────
            # sandbox/ioscan/pushbroker: tests added in the same PR, so the
            #   floors move with the gain (67.6→98.5, 71.0→97.1, 72.8→100.0).
            # intent/tracing/dashboard: NO code change — these had drifted far
            #   above floors recorded at d89121f5 and were no longer catching
            #   anything. Re-measured on v4 at cd8aee5d and moved up to what
            #   they actually hold, with ~1 point of headroom.
            # scheduler (90.4) and escalation (93.2) now clear the 90% default
            #   and no longer need an entry at all.
            [intent]=87
            [tracing]=85
            [review]=80
            [retro]=82
            [github]=88      # 88.9% on 2026-10-05
            [hivectl]=88
            [knowledge]=89
            [crux/proxy]=89
            [ioscan]=96
            [crux/sandbox]=98
            [crux/pushbroker]=99
            [crux/kubejob]=90    # new in #6311; sandbox.Launcher over the in-cluster API, tested against httptest

            # ── Stale module-path parser repaired on v5 (#6066) ─────────────
            # These packages were pre-existing below-default floors that became
            # visible again when the v2-tests coverage classifier was corrected
            # from the old github.com/kubestellar/hive module path to
            # github.com/hivecommons/hive. Record their current ceilings so the
            # parser fix can gate new packages (pkg/effects, pkg/hub/spoke)
            # without turning unrelated backlog into a blocker.
            [convergence/mutation]=86   # 86.7% on PR #6076
            [convergence/audit]=92      # measured 95.6% on v5 after #9930 (in-package Run/reconcile/publish tests, #9928); was 6 (seed tests only, #9688)
            [dashboard/webstatic]=88  # 88.9% on PR #6076
            [github/automerge]=88   # 88.6% on PR #6076
            [mint]=89       # 89.1% on PR #6076
            [turn]=87       # 87.1% on PR #6076

            # ── Ratchet turned for #7146 ─────────────────────────────────────
            # spoke had NO entry, so it sat on the 90 default and flapped across
            # it (run 35026215085 failed at 89.6% with green runs either side and
            # no spoke change between them) — the classic signature of timing
            # dependent paths drifting in and out of the profile. #7146 proposed
            # dropping spoke to 89 to quiet the noise; that would turn the
            # ratchet the wrong way. #7130 covered those paths instead, and spoke
            # now measures a flat 96.3% over five consecutive runs with the gate's
            # own command (`-short -count=1`, no -race) — zero variance, so the
            # flap is gone at the source. Floored at what it holds with headroom.
            [spoke]=95       # 96.3% on 2026-09-16 after #7130; was unfloored/flapping

            # ── ./cmd/... floors (#4701) ─────────────────────────────────────
            # CI never ran ./cmd/... at all before #4701 — four cmd/bd tests
            # were merged already-failing and stayed red undetected. cmd keys
            # are prefixed ("cmd/<name>") so cmd/hivectl can never collide with
            # pkg/hivectl's floor. Recorded just below what v4 measures after
            # #4700/#4719 so the ratchet can turn as CLI tests land.
            # ── Ratchet turned for #7407 ────────────────────────────────────
            # NO code change — the cmd floors had drifted 27-82 points below
            # what the packages hold, so deleting most of the bd or apiproxy
            # CLI tests would still have passed the gate. Re-measured on
            # 2026-09-17 with the gate's own command (`-short -count=1`), three
            # consecutive runs, zero variance. Floors are set ~2-3 points under
            # the LOWER of the two independent measurements (linux, as the gate
            # runs, and darwin, where a contributor reproduces locally) so
            # normal variance cannot fail CI on either platform:
            #   cmd/bd          95.7 linux / 94.4 darwin
            #   cmd/apiproxy    95.6 linux / 94.4 darwin
            #   cmd/hive-backup 90.4 linux / 87.8 darwin
            #   cmd/hive        44.8 linux / 51.6 darwin
            #   cmd/hivectl    100.0 both
            [cmd/bd]=92       # 95.7% on 2026-09-17; was 58
            [cmd/hive]=42     # 44.8% on 2026-09-17; bulk is main()/run-loop wiring; was 17
            [cmd/apiproxy]=92 # 95.6% on 2026-09-17; was 13
            [cmd/hive-backup]=85 # 90.4% on 2026-09-17; was 68
            # cmd/hivectl gained tests in a3f9842c and holds 100.0%, so it is no
            # longer exempt below — floored here instead of falling back to the
            # 90 default (#7407).
            [cmd/hivectl]=97  # 100.0% on 2026-09-17; was exempt as an untested main
            [cmd/flue-fixture]=75 # 80.0% on 2026-09-22 (#8361): main() is the single uncovered statement; run() and signalContext() are tested in-process and the binary is smoke-tested as a child process, which yields no profile
            [cmd/crux]=75     # 80.0% on 2026-10-23 (#13536): same shape as flue-fixture; main() is the one uncovered statement, run() is tested in-process. floor_key maps BOTH ./cmd/crux (root host) and ./crux/cmd/crux (the module's own binary, #14138) here; the module one measured 80.0% on 2026-10-25
            [crux/cli]=92     # 97.8% on 2026-10-23 (#13536): the crux CLI against a fake crux.Runtime plus a real unix control socket; the residue is Main() and write-error branches
          )

          # Thin flag-parsing mains with no test files, wrapping packages the
          # gate already floors (pkg/hivectl at 88, pkg/hubbackup at the 90
          # default). Exempt from the NO TEST FILES hard failure by name so any
          # NEW untested cmd package still fails the gate. Remove an entry the
          # moment its package gains a test file.
          #
          # Empty since #7407: cmd/hivectl was the only entry and it gained
          # src/cmd/hivectl/main_test.go in a3f9842c, so `go test` no longer
          # emits a "[no test files]" line for it and the exemption could never
          # match. The mechanism below is intact and still hard-fails any new
          # untested cmd main; it simply has nothing to exempt today.
          UNTESTED_CMD_MAINS=''

          # Capture the full `go test -cover` output ONCE, then classify every
          # line. The previous version piped through `grep '^ok'`, which silently
          # dropped two categories that matter most:
          #
          #   FAIL  <pkg>                    -> a broken package scored nothing
          #   ?     <pkg> [no test files]    -> an untested package scored nothing
          #
          # Both were therefore incapable of failing the gate, so adding a
          # package with zero tests was the one guaranteed way to stay green.
          # They are now hard failures in their own right.
          go test -cover ./pkg/... ./cmd/... ./internal/... ./crux/... -short -count=1 -timeout 600s 2>&1 | tee cover-run.log || true

          # Floor-map key for a package import path: cmd packages are keyed
          # with a "cmd/" prefix so cmd/hivectl can never resolve to
          # pkg/hivectl's floor (#4701).
          #
          # pkg packages are keyed by their full path below pkg/ for the same
          # reason: a bare basename makes every nested package an alias for the
          # top-level one that shares its name, so they silently share a floor.
          # v5 is the live example -- it carries both pkg/spoke (96.3%) and
          # pkg/hub/spoke (90.7%), which both keyed to "spoke", so the [spoke]=95
          # floor #7146 set for pkg/spoke would have been enforced against
          # pkg/hub/spoke too and failed it. Keys are now "spoke" vs "hub/spoke".
          #
          # The nested Crux module (src/crux, #13536) is keyed "crux/<path>"
          # with the module root package as "crux", so crux/config can never
          # alias pkg/config and crux/agent keeps the floor pkg/agent carried
          # before the module cut.
          floor_key() {
            case "$1" in
              */cmd/*)  echo "cmd/${1##*/}" ;;
              */pkg/*)  echo "${1##*/pkg/}" ;;
              */crux/*) echo "crux/${1##*/crux/}" ;;
              */crux)   echo "crux" ;;
              *)        echo "${1##*/}" ;;
            esac
          }

          while IFS= read -r line; do
            case "$line" in
              ok*)
                pkgpath=$(echo "$line" | awk '{print $2}')
                pkg=$(floor_key "$pkgpath")
                pct=$(echo "$line" | grep -o '[0-9]*\.[0-9]*%' | tr -d '%')
                if [ -z "$pct" ]; then continue; fi
                int_pct=$(echo "$pct" | cut -d. -f1)
                pkg_threshold=${PKG_THRESHOLD[$pkg]:-$THRESHOLD}
                if [ "$int_pct" -lt "$pkg_threshold" ]; then
                  FAILED="${FAILED}${pkg}: ${pct}% (floor ${pkg_threshold}%)\n"
                fi
                REPORT="${REPORT}${pkg}: ${pct}%\n"
                ;;
              FAIL*)
                pkgpath=$(echo "$line" | awk '{print $2}')
                # Ignore the bare trailing "FAIL" summary line, which has no pkg.
                if [ -n "$pkgpath" ] && [ "$pkgpath" != "FAIL" ]; then
                  pkg=$(floor_key "$pkgpath")
                  BROKEN="${BROKEN}${pkg}: TESTS FAILING\n"
                  REPORT="${REPORT}${pkg}: FAIL\n"
                fi
                ;;
              \?*)
                pkgpath=$(echo "$line" | awk '{print $2}')
                pkg=$(floor_key "$pkgpath")
                case " $UNTESTED_CMD_MAINS " in
                  *" $pkg "*)
                    REPORT="${REPORT}${pkg}: no tests (exempt thin main)\n"
                    ;;
                  *)
                    BROKEN="${BROKEN}${pkg}: NO TEST FILES\n"
                    REPORT="${REPORT}${pkg}: no tests\n"
                    ;;
                esac
                ;;
              *coverage:*)
                # Go >=1.20 with -cover prints no-test packages as a
                # tab-prefixed "<pkg>\t\tcoverage: 0.0% of statements" line
                # instead of the old "?   <pkg> [no test files]" line.
                pkgpath=$(echo "$line" | awk '{print $1}')
                case "$pkgpath" in
                  */pkg/*|*/cmd/*|*/internal/*)
                    pkg=$(floor_key "$pkgpath")
                    case " $UNTESTED_CMD_MAINS " in
                      *" $pkg "*)
                        REPORT="${REPORT}${pkg}: no tests (exempt thin main)\n"
                        ;;
                      *)
                        BROKEN="${BROKEN}${pkg}: NO TEST FILES\n"
                        REPORT="${REPORT}${pkg}: no tests\n"
                        ;;
                    esac
                    ;;
                esac
                ;;
            esac
          done < cover-run.log

          TOTAL=$(go tool cover -func=coverage.out 2>/dev/null | grep '^total:' | awk '{print $3}')
          echo "Total coverage: ${TOTAL:-unavailable}"
          echo "total=${TOTAL:-unavailable}" >> "$GITHUB_OUTPUT"

          # Report the commit that was actually MEASURED, read from the working
          # tree rather than from github.sha (#3903). On a schedule these differ:
          # github.sha is the default branch's head, while the checkout above
          # pinned an explicit ref. The filed issue used to quote github.sha, so
          # it named a commit the run had never tested — anyone bisecting from it
          # was looking at the wrong tree.
          MEASURED_SHA=$(git rev-parse HEAD 2>/dev/null || echo unknown)
          MEASURED_REF=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown)
          echo "Measured ${MEASURED_REF} @ ${MEASURED_SHA}"
          echo "measured_sha=${MEASURED_SHA}" >> "$GITHUB_OUTPUT"
          echo "measured_ref=${MEASURED_REF}" >> "$GITHUB_OUTPUT"

          {
            echo "## Per-package coverage"
            echo ""
            echo -e "${REPORT}"
          } >> "$GITHUB_STEP_SUMMARY"

          # Written once the verdict is known and before either outcome (the
          # `exit 1` shortfall below or success), so it means "measurement
          # reached a verdict", not "gate passed". The profile artifact cannot
          # serve as this signal: a hung run leaves a partial cover-run.log
          # that the always() upload still ships during cancellation cleanup
          # (#12951).
          echo "$GITHUB_SHA" > coverage-measured.marker

          if [ -n "$BROKEN" ] || [ -n "$FAILED" ]; then
            echo "below_threshold=true" >> "$GITHUB_OUTPUT"
            {
              [ -n "$BROKEN" ] && echo -e "### Packages that cannot be scored\n${BROKEN}"
              [ -n "$FAILED" ] && echo -e "### Packages below their floor\n${FAILED}"
            } >> "$GITHUB_STEP_SUMMARY"
            echo -e "failed_packages<<EOF\n${BROKEN}${FAILED}EOF" >> "$GITHUB_OUTPUT"
            echo -e "full_report<<EOF\n${REPORT}EOF" >> "$GITHUB_OUTPUT"
            echo ""
            echo "COVERAGE GATE FAILED:"
            [ -n "$BROKEN" ] && echo -e "${BROKEN}"
            [ -n "$FAILED" ] && echo -e "${FAILED}"
            # Exit non-zero so the gate actually BLOCKS. The previous version
            # computed this report and then exited 0 regardless, so even a
            # correctly-detected shortfall never failed anything.
            exit 1
          fi

          echo "below_threshold=false" >> "$GITHUB_OUTPUT"
          echo "All packages meet their coverage floor" >> "$GITHUB_STEP_SUMMARY"

      # `if: always()` keeps the profile when the gate above fails; `uses:`
      # steps ignore defaults.run.working-directory, hence the src/ prefix.
      - name: Upload coverage profile
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: coverage-profile-${{ github.run_id }}-${{ github.run_attempt }}
          path: |
            src/coverage.out
            src/cover-run.log
            src/race-run.log
          if-no-files-found: ignore
          retention-days: 14

      - name: Upload measurement-complete marker
        if: always()
        uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        with:
          name: coverage-measured-${{ github.run_id }}-${{ github.run_attempt }}
          path: src/coverage-measured.marker
          if-no-files-found: ignore
          retention-days: 14

  create-issue-on-coverage-drop:
    # Scheduled runs only. The bare `failure()` here previously fired on ANY
    # job failure regardless of cause, which is why issue #2191 accumulated 41
    # comments titled "packages below 90% threshold" when the real cause was a
    # test failure aborting the run. Now that the gate also runs on
    # pull_request, an unscoped condition would additionally file an issue for
    # every contributor's red PR.
    # A cancelled coverage job carries no outputs. If it was cancelled AFTER the
    # measurement finished (timeout during post-step cleanup, #12944) the
    # measurement-complete marker artifact exists and filing would only produce a false "measured unknown"
    # alarm, so the script below exits early. If it was cancelled BEFORE the
    # marker was written (a hung suite hitting the job timeout, #12951) the
    # issue is still filed so a persistent hang does not go unreported.
    if: always() && github.event_name == 'schedule' && needs.coverage.result != 'success' && needs.coverage.result != 'skipped'
    runs-on: ${{ fromJSON(vars.HIVE_RUNNER_LABELS || '["ubuntu-latest"]') }}
    timeout-minutes: 10
    needs: coverage
    permissions:
      contents: read
      issues: write
      actions: read
    steps:
      - name: Create or update coverage issue
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        with:
          script: |
            const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;

            // Report what the coverage job MEASURED, not what triggered the run
            // (#3903). On a schedule, context.sha is the default branch's head
            // while the job checks out an explicit ref — the old body quoted
            // context.sha and hardcoded "Branch: v2", so #3903 was filed naming
            // a v4 commit for a run that had measured v2. Anyone starting from
            // that SHA was reading the wrong tree.
            const measuredSha = '${{ needs.coverage.outputs.measured_sha }}' || context.sha;
            const measuredRef = '${{ needs.coverage.outputs.measured_ref }}' || 'unknown';
            const failing = `${{ needs.coverage.outputs.failed_packages }}`.trim();
            const total = '${{ needs.coverage.outputs.total }}' || 'unavailable';
            const shortSha = measuredSha.substring(0, 7);

            // Cancelled jobs propagate no outputs, so the marker artifact is the
            // only evidence that the measurement completed (#12944, #12951). The
            // profile artifact is unreliable here: partial logs from a hung run
            // are still uploaded during cancellation cleanup.
            const cancelled = '${{ needs.coverage.result }}' === 'cancelled';
            if (cancelled) {
              const markerName = 'coverage-measured-${{ github.run_id }}-${{ github.run_attempt }}';
              const arts = await github.paginate(github.rest.actions.listWorkflowRunArtifacts, {
                owner: context.repo.owner,
                repo: context.repo.repo,
                run_id: context.runId,
                per_page: 100,
              });
              if (arts.some(a => a.name === markerName && !a.expired)) {
                core.info('Coverage job was cancelled after the measurement completed; no issue (#12944).');
                return;
              }
            }

            // The gate fails for two DIFFERENT reasons and they need different
            // fixes: a package under its floor, or a package that could not be
            // scored at all (tests failing / no test files). Conflating them is
            // how #2191 collected 41 comments about coverage when the cause was
            // a broken test, and how #3903 was titled "below 90%" when nothing
            // was below its floor. State the cause up front.
            const cannotScore = /TESTS FAILING|NO TEST FILES/.test(failing);
            // No measured ref means the coverage step never wrote its outputs:
            // the job died earlier (checkout, setup, a workflow step, runner).
            // Calling that "below their coverage floor" sent #9632 to agents
            // that added tests for a workflow failure (#9645).
            const neverMeasured = !'${{ needs.coverage.outputs.measured_ref }}';
            const cause = cancelled
              ? 'The coverage job was **cancelled before any coverage verdict reached this job** (for example a hung test suite hitting the job timeout, or a cancellation during setup). No verdict was recorded, so this is not evidence of a coverage shortfall - open the run log to see where it stopped.'
              : neverMeasured
              ? 'The coverage job **failed before any package was measured** (workflow, setup, or runner failure). This is not a coverage shortfall and adding tests will not fix it - open the run log for the failing step.'
              : cannotScore
              ? 'One or more packages **could not be scored** (tests failing, or no test files). That is a broken-test problem, not a coverage shortfall — fix the tests first; the floor cannot be evaluated until they run.'
              : 'One or more packages are **below their coverage floor**.';
            const detail = failing
              ? `\n\n### What the gate reported\n\n\`\`\`\n${failing}\n\`\`\``
              : '';

            // The title must not claim a shortfall the body contradicts:
            // triage reads titles first (#9664 audit). Both keep the
            // [Coverage] prefix the dedupe below matches on.
            const title = cancelled
              ? `[Coverage] gate job cancelled before a coverage verdict`
              : neverMeasured
              ? `[Coverage] gate job failed before measuring coverage`
              : `[Coverage] packages below 90% threshold`;
            const body = `## Coverage gate failed\n\n`
              + `- **Measured:** \`${measuredRef}\` @ \`${measuredSha}\`\n`
              + `- **Run:** ${runUrl}\n`
              + `- **Total coverage:** ${total}\n`
              + `- **Default threshold:** 90% per package (see PKG_THRESHOLD in the workflow for documented per-package floors)\n\n`
              + `${cause}${detail}\n\n`
              + `Close this issue when the gate is green again.`;

            const existing = await github.rest.issues.listForRepo({
              owner: context.repo.owner,
              repo: context.repo.repo,
              state: 'open',
              labels: 'ci-coverage',
              per_page: 10,
            });
            const dup = existing.data.find(i => i.title.startsWith('[Coverage]'));
            if (dup) {
              await github.rest.issues.createComment({
                owner: context.repo.owner,
                repo: context.repo.repo,
                issue_number: dup.number,
                body: `Gate still failing on \`${measuredRef}\` @ \`${shortSha}\`: ${runUrl}${detail}`,
              });
              return;
            }

            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title,
              body,
              labels: ['ci-coverage', 'kind/bug'],
            });