diff --git a/changelog.d/fixed-hub-forge-unreachable-badge.md b/changelog.d/fixed-hub-forge-unreachable-badge.md new file mode 100644 index 0000000000..e67ea35bbf --- /dev/null +++ b/changelog.d/fixed-hub-forge-unreachable-badge.md @@ -0,0 +1 @@ +- hub: hives whose spoke reports `forge_unreachable` are labelled "forge unreachable (github_host)" instead of generic stale badges, and the condition no longer counts against their health or advisory status diff --git a/src/docs/advisory-staleness.md b/src/docs/advisory-staleness.md index 15e37f3a75..e9010d1411 100644 --- a/src/docs/advisory-staleness.md +++ b/src/docs/advisory-staleness.md @@ -15,6 +15,11 @@ never re-derives it. ## The gates +0. **Forge unreachable** — a beat carrying `ForgeUnreachable` is never stale, + whatever the digest's age or error. The spoke cannot reach its GitHub host + from the pod, so an ageing digest is a symptom of the egress fault rather + than a wedge of its own; the row shows a `forge unreachable (github_host)` + pill instead (#17433). 1. **Participation** — no post time *and* no error means "not an advisory hive (or a spoke too old to report)". Unknown, never stale. 2. **Reported error** — flagged with its cause, unless the App is still @@ -49,6 +54,7 @@ never re-derives it. | Pure PR/merge hive | Suppressed | Suppressed | unchanged (expected suppression) | | Every agent paused or off-schedule, digest ageing | Flagged stale — the pause itself lit the pill | Suppressed — no agents, no findings, no digest | Gate 3 (`allAgentsQuietByDesign`, #4528); counted as `hidden_stale` in diagnostics | | Spoke heartbeat collect timed out, beat carries cached/minimal stats (`statsStale`) | `advisory-stale` alert raised from the last-good `advisoryLastPostedAt` while fresh digests were posting | No fleet alert — the post time is not a fresh observation | The hub alert rule skips `StatsStale` beats (#12842); health verdict already reads "heartbeat stats stale" | +| Spoke cannot reach its GitHub host from the pod (`ForgeUnreachable`) | Generic stale pills lit the row while the real fault was cluster egress | No stale-advisory pill — the row shows `forge unreachable (github_host)` instead | `advisoryStale` returns early on `ForgeUnreachable` beats (#17433) | ## Diagnostics diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 40f12e74f3..91265f4c73 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -33,39 +33,39 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:531` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1774` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1775` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1780` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1781` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1709` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1710` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1711` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1712` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1713` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1721` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1722` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1723` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1724` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1715` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1716` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1717` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1718` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1719` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1727` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1728` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1729` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1730` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | | `GET` | `/api/hub/activity` | Signed-in user | Read-only hub-wide activity view ([#15951](https://github.com/hivecommons/hive/issues/15951), RFC [#15861](https://github.com/hivecommons/hive/issues/15861)) merged from each spoke's heartbeat `activity` digest, held in hub memory only. Authorization is per user and per hive before aggregation: hub admins see every registered hive; everyone else only hives they own or hold an unexpired grant for; anonymous → `401`. Returns `hives[]` (`hive_id`, `name`, `status` `ok`/`stale`/`not_reporting`/`missing`, `reporting`, `stale` once the block is older than two heartbeats, `last_heartbeat`, `received_at`, `generated_at`, `totals` — `items`, `by_stage`, `by_modifier`, counting private and truncated items too — `truncated` and `events_dropped`), `items[]` (`hive_id`, `repo`, `n`, `kind`, `title`, `author_kind`, `stage`, `stage_n`, `since`, `mods`, `private`) ordered needs-human/needs-decision, stalled, furthest stage, oldest, a response-level `truncated` (cap 2000), and `events[]` ([#15964](https://github.com/hivecommons/hive/issues/15964): `hive_id`, `received_at`, `seq`, `type` `opened`/`stage`/`merged`, `repo`, `n`, `kind`, `title`, `from`, `stage`, `at`, `private`), oldest first, from a fleet-wide in-memory ring of at most 1,000 events, none held longer than 24 hours (ageing out is not counted in `events_dropped`; when full, the oldest event of the hive holding the most events is dropped, counted in that hive's `events_dropped`, so one hive cannot evict the others' events). Events are filtered per user and per hive exactly like items, before aggregation, so an event for a hive the viewer cannot see is never delivered; `mod`/`author_kind` filters keep only events whose item is still on the hive's board. A spoke block carries at most 200 events; drops are disclosed per hive as `events_dropped`. New events change the `ETag`. Each digest carries at most 100 items by default and never more than the 250-item hub ceiling; private-repository items appear only when the spoke opts in. Filters: `hive`, `repo`, `stage`, `mod`, `author_kind`, `q` (title substring or `#`), each ≤ 200 characters (else `400`). `ETag` + `If-None-Match` → `304`; `Cache-Control: private, no-cache`, `Vary: Cookie, Authorization`. No actions | `pkg/hub/activity.go` | | `GET` | `/api/hub/activity/public` | Public | Public-safe activity view ([#15966](https://github.com/hivecommons/hive/issues/15966)): the `/api/hub/activity` shape for anyone, never reading a cookie, token or `Authorization` header (credentials do not change the response). Only hives the registry lists as public on github.com, and only items and events not marked `private`; `totals`, per-hive `truncated` and `events_dropped`, hive owners and per-user grants are omitted. Same filters (`400` on an invalid one). The body is built at most once per 2 s per parsed filter and shared by every anonymous caller, so rows, `generated_at` and the `ETag` change at most every 2 s; query strings that parse to the default view share one entry. Toggling a hive's visibility drops the cache at once. `ETag` + `If-None-Match` → `304`; `Cache-Control: no-cache` | `pkg/hub/activity_public.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1732` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1742` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1743` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1744` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1745` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1746` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1747` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1748` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1749` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1752` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1755` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1756` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1763` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1764` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1765` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1787` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1789` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1738` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1748` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1749` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1750` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1751` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1752` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1753` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1754` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1755` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1758` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1761` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1762` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1769` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1770` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1771` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1793` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1795` | diff --git a/src/pkg/hub/advisory_staleness.go b/src/pkg/hub/advisory_staleness.go index fa71d7e717..245ef415ba 100644 --- a/src/pkg/hub/advisory_staleness.go +++ b/src/pkg/hub/advisory_staleness.go @@ -193,6 +193,9 @@ func carryAdvisoryPostTime(entry *RegistryEntry, prev RegistryEntry) { // returns false — never a false alarm — matching the rule the codebase already // applies to StartedAt and GitHubAPIURL. func advisoryStale(e RegistryEntry, now time.Time) (stale bool, reason string) { + if e.ForgeUnreachable { + return false, "" + } return advisoryStaleFromFreshness(e, advisoryFreshnessFor(e, now)) } diff --git a/src/pkg/hub/alerts.go b/src/pkg/hub/alerts.go index b759598fb4..dd0f65f465 100644 --- a/src/pkg/hub/alerts.go +++ b/src/pkg/hub/alerts.go @@ -637,6 +637,9 @@ type alertHive struct { // key material. InferenceAuthError string + // ForgeUnreachable exempts the github_auth check from the failing-check rule. + ForgeUnreachable bool + // GitHubAppRequired / GitHubAppState are the spoke's own report of whether // its config demands App auth and, when App auth is failing, WHY (the // github.AppAuthState wire token). Carried verbatim: the spoke owns the @@ -692,6 +695,7 @@ func alertHiveFromEntry(h MyHiveEntry) alertHive { InactiveAgentsReason: h.InactiveAgentsReason, InferenceAuthError: h.InferenceAuthError, + ForgeUnreachable: h.ForgeUnreachable, GitHubAppRequired: h.GitHubAppRequired, GitHubAppState: h.GitHubAppState, @@ -1141,6 +1145,9 @@ func evaluateAlertsInFleet(state *alertState, hives, fleet []alertHive, driftAle // seeded with a placeholder org/repo, which the check flags by design // until the slot is claimed (#12843). failing := failingHealthChecks(h.Health) + if h.ForgeUnreachable { + failing = withoutForgeChecks(failing) + } if h.IsPlaceholder { failing = withoutAuthClassChecks(failing, h.ProvStatus == statusAvailable) } diff --git a/src/pkg/hub/assets/dashboard.html b/src/pkg/hub/assets/dashboard.html index b4aa0af25f..220d0efbad 100644 --- a/src/pkg/hub/assets/dashboard.html +++ b/src/pkg/hub/assets/dashboard.html @@ -3847,7 +3847,15 @@

'; } + /* forgeUnreachableSummary renders the hub-computed "forge unreachable (host)" + label (h.forgeUnreachableLabel). Amber: the pod is healthy, the cluster's + egress to the forge is not. */ + function forgeUnreachableSummary(h) { + if (!h || !h.forgeUnreachableLabel) return ''; + return amberNudgePill('The spoke cannot reach its GitHub host from the pod; the hive itself is healthy', h.forgeUnreachableLabel); + } + + /* amberNudgePill is the shared "worth noticing, not a row failure" pill used + by the stale-advisory and forge-unreachable summaries. */ + function amberNudgePill(title, label) { + var col = '#d29922'; + return '' + + esc(label) + ''; + } + /* advisoryStaleSummary renders a small "stale advisory" pill next to the failing-check pill when the hub has flagged this hive's advisory digest as stale. The flag (h.advisoryStale) and its reason are computed server-side @@ -3891,12 +3917,7 @@

' + - 'stale advisory'; + return amberNudgePill(h.advisoryStaleReason || 'Advisory digest has gone stale', 'stale advisory'); } /* advisoryFindingsSummary renders the finding count from this hive's last @@ -9530,7 +9551,7 @@

' + bulkCheckboxCell(h, section || 'all') + '' + ('⋮' + pendingBadge + '') + '' + - '' + (function() { var isHostedRow = h.hiveType === 'hosted' || (h.id && (h.id.startsWith('hosted-') || h.id.startsWith('saas-'))); var dh = isHostedRow && h.id ? ('/api/saas/hives/' + encodeURIComponent(h.id) + '/open') : (rb ? esc(rb) : ''); /* Label derived from the PROJECT (org + primary repo), not by splitting h.name — see hiveLabel. */ var label = hiveLabel(h); var orgName = label.line1; var repoName = label.line2; /* rp is the repo path shown in the GitHub-icon tooltip; use the same doubling-safe path the label shows (repoDisplayLine / hiveLabel), never org + '/' + primaryRepo which doubles the owner on a github.io/GHE hive whose primaryRepo is already 'owner/repo'. */ var rp = repoName || ''; /* The icon href must resolve to a real GitHub path. When primaryRepo already carries 'owner/repo', that pair IS the owner/repo the URL needs (not org, which may be a mis-parsed host); otherwise fall back to org + primaryRepo. */ var hasRepoPath = h.primaryRepo && h.primaryRepo.indexOf('/') !== -1; var rpOwner = hasRepoPath ? h.primaryRepo.split('/')[0] : h.org; var rpName = hasRepoPath ? h.primaryRepo.split('/').slice(1).join('/') : h.primaryRepo; var rpHref = ghRepoURL(hiveForgeHost(h), rpOwner, rpName); var ghIcon = (rp && rpHref) ? '' : (rp ? '' : ''); /* vanityDisplay is the friendly host shown in the status bar on hover. rb is resolvedBase(h), which the hub has already overlaid with the claimed vanity_url; it is only a DISPLAY url here, never the click target — see ssoDisplayLink. Empty for a row with no vanity host, which falls back to today's /open href. Only meaningful on hosted rows: a non-hosted row's dh IS rb already. */ var vanityDisplay = isHostedRow && rb ? rb : ''; var link = function(text, bold) { if (dh) { return ssoDisplayLink(dh, vanityDisplay, text, bold ? 'hive-name-link' : 'hive-sub-link'); } var s = bold ? 'font-weight:700;color:inherit' : 'color:#6b7280;font-weight:400'; return '' + esc(text) + ''; }; /* repoLink wraps the second line (the org/repo) in a link to the ACTUAL FORGE REPO (github.com / GHE host), NOT the spoke dashboard: the top line already opens the dashboard, so the repo path should take an operator to the code. rpHref is the same doubling-safe forge URL the GitHub icon uses (ghRepoURL(hiveForgeHost(h), rpOwner, rpName)); when it is empty (BYO/mis-parsed host) fall back to plain non-link text, never to the dashboard. */ var repoLink = function(text) { if (rpHref) { return '' + esc(text) + ''; } return '' + esc(text) + ''; }; var line1 = dot + ' ' + link(orgName, true) + heartbeatHeart(h) + nameEditAffordance(h); var fcPill = h.online ? failingCheckSummary(h) : ''; /* Advisory-staleness pill sits right beside the failing-check pill: both are "something is quietly wrong with this working hive" signals, and advisoryStaleSummary already self-suppresses (empty string) unless the hub flagged the digest stale, so unaffected rows are pixel-identical. */ var advPill = h.online ? advisoryStaleSummary(h) : ''; /* Advisory finding-count pill rides beside the staleness pill and carries the "(top N)" marker when the spoke capped its digest. */ var advCountPill = h.online ? advisoryFindingsSummary(h) : ''; /* Dead-link pill is deliberately NOT gated on h.online: the entire point is a hive that IS online and heartbeating while its public URL is broken, so gating it the way the other pills are gated would hide exactly the case it exists to surface. */ var dlPill = deadLinkSummary(h); var privatePill = privateURLSummary(h); /* Inference-auth pill: like the dead-link pill it is NOT gated on h.online, because the hive being online while every inference call 401s is exactly the case it surfaces. */ var iaPill = inferenceAuthSummary(h); var rsPill = restartStormSummary(h); /* Inline access faces sit on the name cell's second line, immediately after this row's own role badge: the badge already answers "what am I on this hive", so the co-members read as the natural continuation of the same thought, in the one cell that is left-aligned and has room to grow. It also keeps them out of the 16 dense metric columns, none of which is about people. Empty string when the viewer is the only member, so those rows are pixel-identical to today. */ var accessFaces = hiveAccessAvatars(h); /* Keyed off repoName, not orgName: line 1 now always carries SOME identity, so the presence of a second line is decided purely by whether there is a repo to put on it. Without a repo the row still shows the GitHub icon, role badge, faces and failing-check pill on the compact variant. */ var line2 = repoName ? '
' + repoLink(repoName) + ' ' + ghIcon + ' ' + roleBadge(h.role) + accessFaces + fcPill + advPill + advCountPill + dlPill + privatePill + iaPill + rsPill + '
' : '
' + ghIcon + ' ' + roleBadge(h.role) + accessFaces + fcPill + advPill + advCountPill + dlPill + privatePill + iaPill + rsPill + '
'; var line3 = pendingPill ? '
' + pendingPill + '
' : ''; return line1 + line2 + line3; })() + '' + + '' + (function() { var isHostedRow = h.hiveType === 'hosted' || (h.id && (h.id.startsWith('hosted-') || h.id.startsWith('saas-'))); var dh = isHostedRow && h.id ? ('/api/saas/hives/' + encodeURIComponent(h.id) + '/open') : (rb ? esc(rb) : ''); /* Label derived from the PROJECT (org + primary repo), not by splitting h.name — see hiveLabel. */ var label = hiveLabel(h); var orgName = label.line1; var repoName = label.line2; /* rp is the repo path shown in the GitHub-icon tooltip; use the same doubling-safe path the label shows (repoDisplayLine / hiveLabel), never org + '/' + primaryRepo which doubles the owner on a github.io/GHE hive whose primaryRepo is already 'owner/repo'. */ var rp = repoName || ''; /* The icon href must resolve to a real GitHub path. When primaryRepo already carries 'owner/repo', that pair IS the owner/repo the URL needs (not org, which may be a mis-parsed host); otherwise fall back to org + primaryRepo. */ var hasRepoPath = h.primaryRepo && h.primaryRepo.indexOf('/') !== -1; var rpOwner = hasRepoPath ? h.primaryRepo.split('/')[0] : h.org; var rpName = hasRepoPath ? h.primaryRepo.split('/').slice(1).join('/') : h.primaryRepo; var rpHref = ghRepoURL(hiveForgeHost(h), rpOwner, rpName); var ghIcon = (rp && rpHref) ? '' : (rp ? '' : ''); /* vanityDisplay is the friendly host shown in the status bar on hover. rb is resolvedBase(h), which the hub has already overlaid with the claimed vanity_url; it is only a DISPLAY url here, never the click target — see ssoDisplayLink. Empty for a row with no vanity host, which falls back to today's /open href. Only meaningful on hosted rows: a non-hosted row's dh IS rb already. */ var vanityDisplay = isHostedRow && rb ? rb : ''; var link = function(text, bold) { if (dh) { return ssoDisplayLink(dh, vanityDisplay, text, bold ? 'hive-name-link' : 'hive-sub-link'); } var s = bold ? 'font-weight:700;color:inherit' : 'color:#6b7280;font-weight:400'; return '' + esc(text) + ''; }; /* repoLink wraps the second line (the org/repo) in a link to the ACTUAL FORGE REPO (github.com / GHE host), NOT the spoke dashboard: the top line already opens the dashboard, so the repo path should take an operator to the code. rpHref is the same doubling-safe forge URL the GitHub icon uses (ghRepoURL(hiveForgeHost(h), rpOwner, rpName)); when it is empty (BYO/mis-parsed host) fall back to plain non-link text, never to the dashboard. */ var repoLink = function(text) { if (rpHref) { return '' + esc(text) + ''; } return '' + esc(text) + ''; }; var line1 = dot + ' ' + link(orgName, true) + heartbeatHeart(h) + nameEditAffordance(h); var fcPill = h.online ? failingCheckSummary(h) : ''; /* Forge egress broken, pod healthy: the labelled pill replaces the stale-advisory pill and the github_auth entry (failingChecks already drops it), but any other failing check still renders beside it. Not re-gated on h.online here because the hub already blanks forgeUnreachableLabel for offline hives (forgeUnreachableLabel in Go), so the rule cannot drift between the API and this row. */ var fuPill = forgeUnreachableSummary(h); /* Advisory-staleness pill sits right beside the failing-check pill: both are "something is quietly wrong with this working hive" signals, and advisoryStaleSummary already self-suppresses (empty string) unless the hub flagged the digest stale, so unaffected rows are pixel-identical. */ var advPill = h.online && !fuPill ? advisoryStaleSummary(h) : ''; /* Advisory finding-count pill rides beside the staleness pill and carries the "(top N)" marker when the spoke capped its digest. */ var advCountPill = h.online ? advisoryFindingsSummary(h) : ''; /* Dead-link pill is deliberately NOT gated on h.online: the entire point is a hive that IS online and heartbeating while its public URL is broken, so gating it the way the other pills are gated would hide exactly the case it exists to surface. */ var dlPill = deadLinkSummary(h); var privatePill = privateURLSummary(h); /* Inference-auth pill: like the dead-link pill it is NOT gated on h.online, because the hive being online while every inference call 401s is exactly the case it surfaces. */ var iaPill = inferenceAuthSummary(h); var rsPill = restartStormSummary(h); /* Inline access faces sit on the name cell's second line, immediately after this row's own role badge: the badge already answers "what am I on this hive", so the co-members read as the natural continuation of the same thought, in the one cell that is left-aligned and has room to grow. It also keeps them out of the 16 dense metric columns, none of which is about people. Empty string when the viewer is the only member, so those rows are pixel-identical to today. */ var accessFaces = hiveAccessAvatars(h); /* Keyed off repoName, not orgName: line 1 now always carries SOME identity, so the presence of a second line is decided purely by whether there is a repo to put on it. Without a repo the row still shows the GitHub icon, role badge, faces and failing-check pill on the compact variant. */ var line2 = repoName ? '
' + repoLink(repoName) + ' ' + ghIcon + ' ' + roleBadge(h.role) + accessFaces + fuPill + fcPill + advPill + advCountPill + dlPill + privatePill + iaPill + rsPill + '
' : '
' + ghIcon + ' ' + roleBadge(h.role) + accessFaces + fuPill + fcPill + advPill + advCountPill + dlPill + privatePill + iaPill + rsPill + '
'; var line3 = pendingPill ? '
' + pendingPill + '
' : ''; return line1 + line2 + line3; })() + '' + '' + locationCell + '' + '' + uptimeCell(h) + '' + /* No white-space:nowrap on the cell itself: the stacked lines each diff --git a/src/pkg/hub/forge_unreachable.go b/src/pkg/hub/forge_unreachable.go new file mode 100644 index 0000000000..17016f6102 --- /dev/null +++ b/src/pkg/hub/forge_unreachable.go @@ -0,0 +1,31 @@ +package hub + +import "strings" + +// forgeUnreachableLabel is the row label for a hive whose latest beat said the +// forge host cannot be reached from the pod; "" when it did not. Also "" once +// markStaleHives flips the hive offline: the flag is carried over from the last +// beat, and a hive that has stopped beating is not "healthy apart from forge +// egress" — the offline row must not claim it is. +func forgeUnreachableLabel(e RegistryEntry) string { + if !e.ForgeUnreachable || !e.Online { + return "" + } + if host := strings.TrimSpace(e.GitHubHost); host != "" { + return "forge unreachable (" + host + ")" + } + return "forge unreachable" +} + +// withoutForgeChecks drops the github_auth check, whose failure is a symptom +// of forge egress being down rather than of the hive being unhealthy. +func withoutForgeChecks(names []string) []string { + kept := names[:0:0] + for _, n := range names { + if n == healthCheckGitHubAuth { + continue + } + kept = append(kept, n) + } + return kept +} diff --git a/src/pkg/hub/forge_unreachable_test.go b/src/pkg/hub/forge_unreachable_test.go new file mode 100644 index 0000000000..242eed037b --- /dev/null +++ b/src/pkg/hub/forge_unreachable_test.go @@ -0,0 +1,81 @@ +package hub + +import ( + "testing" + "time" +) + +func TestForgeUnreachableLabel(t *testing.T) { + cases := []struct { + name string + e RegistryEntry + want string + }{ + {"host", RegistryEntry{Online: true, ForgeUnreachable: true, GitHubHost: "github.ibm.com"}, "forge unreachable (github.ibm.com)"}, + {"no host", RegistryEntry{Online: true, ForgeUnreachable: true}, "forge unreachable"}, + {"old spoke", RegistryEntry{Online: true, GitHubHost: "github.ibm.com"}, ""}, + {"offline", RegistryEntry{ForgeUnreachable: true, GitHubHost: "github.ibm.com"}, ""}, + } + for _, tc := range cases { + if got := forgeUnreachableLabel(tc.e); got != tc.want { + t.Errorf("%s: label = %q, want %q", tc.name, got, tc.want) + } + } +} + +func TestForgeUnreachable_NotCountedAgainstHealth(t *testing.T) { + e := RegistryEntry{ + ID: "h1", Online: true, ACMMLevel: 3, + GitHubAppState: "unreachable", GitHubAppRequired: true, + } + rollup := agentFleetRollup{Known: 1, Running: 1, Expected: 1} + app := GitHubAppHealth{Bucket: ghAppBucketBroken} + now := time.Now() + + before := hiveHealthFor(e, rollup, app, 0, now) + if before.State != HealthStateRed { + t.Fatalf("baseline broken-app beat should be red, got %q", before.State) + } + + e.ForgeUnreachable = true + after := hiveHealthFor(e, rollup, app, 0, now) + if after.State == HealthStateUnknown || after.State == HealthStateRed { + t.Fatalf("forge-unreachable hive must not be penalised, got %q (%s)", after.State, after.Reason) + } +} + +// A beat carrying both flags still has cached stats: the hub must not assert +// health from them, but it names the forge as the cause instead of a generic +// stale verdict. +func TestForgeUnreachable_StaleStatsStayUnknown(t *testing.T) { + e := RegistryEntry{ + ID: "h1", Online: true, StatsStale: true, ForgeUnreachable: true, ACMMLevel: 3, + } + rollup := agentFleetRollup{Known: 1, Running: 1, Expected: 1} + v := hiveHealthFor(e, rollup, GitHubAppHealth{Bucket: ghAppBucketOK}, 0, time.Now()) + if v.State != HealthStateUnknown { + t.Fatalf("stale stats must stay unknown even when forge unreachable, got %q", v.State) + } + if v.Reason != "heartbeat stats stale (forge unreachable)" { + t.Fatalf("reason = %q", v.Reason) + } +} + +func TestForgeUnreachable_NotStaleAdvisory(t *testing.T) { + e := advisoryModeEntry() + e.AdvisoryLastPostedAt = rfc3339Ago(30 * 24 * time.Hour) + if stale, _ := advisoryStale(e, advNow); !stale { + t.Fatal("baseline: aged digest should be stale") + } + e.ForgeUnreachable = true + if stale, reason := advisoryStale(e, advNow); stale { + t.Fatalf("forge-unreachable hive must not be stale-advisory, got %q", reason) + } +} + +func TestForgeUnreachable_WithoutForgeChecks(t *testing.T) { + got := withoutForgeChecks([]string{"github_auth", "disk"}) + if len(got) != 1 || got[0] != "disk" { + t.Fatalf("got %v", got) + } +} diff --git a/src/pkg/hub/health_verdict.go b/src/pkg/hub/health_verdict.go index 6968226353..d4259cf237 100644 --- a/src/pkg/hub/health_verdict.go +++ b/src/pkg/hub/health_verdict.go @@ -79,6 +79,14 @@ func hiveHealthFor(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealth func hiveHealthBase(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealth, queuedWork int, now time.Time) HealthVerdict { v := HealthVerdict{QueuedWork: queuedWork} + // Forge egress is broken, not the pod: the github_auth failure it causes + // must not read as a hive fault. StatsStale is deliberately NOT cleared + // here — cached stats are still cached, and the stale gate below never + // asserts health from them; the gate only names the forge as the cause. + if e.ForgeUnreachable { + app = GitHubAppHealth{Bucket: ghAppBucketUnknown} + } + // --- Unknown gates first: never claim health for a hive we can't see. --- if !e.Online { v.State = HealthStateUnknown @@ -89,6 +97,9 @@ func hiveHealthBase(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealt if e.StatsStale { v.State = HealthStateUnknown v.Reason = "heartbeat stats stale" + if e.ForgeUnreachable { + v.Reason = "heartbeat stats stale (forge unreachable)" + } v.OutputKind = outputKindForLevel(e.ACMMLevel) return v } diff --git a/src/pkg/hub/heartbeat.go b/src/pkg/hub/heartbeat.go index 1545db6c2d..242a9a3383 100644 --- a/src/pkg/hub/heartbeat.go +++ b/src/pkg/hub/heartbeat.go @@ -296,6 +296,7 @@ type HeartbeatPayload struct { GitBranch string `json:"git_branch,omitempty"` ImageRef string `json:"image_ref,omitempty"` GitHubHost string `json:"github_host,omitempty"` + ForgeUnreachable bool `json:"forge_unreachable,omitempty"` Timestamp string `json:"timestamp"` GitHubAppRequired bool `json:"github_app_required,omitempty"` GitHubAppPermIssue string `json:"github_app_perm_issue,omitempty"` diff --git a/src/pkg/hub/saas_hives_api.go b/src/pkg/hub/saas_hives_api.go index d7a205513a..71e215330d 100644 --- a/src/pkg/hub/saas_hives_api.go +++ b/src/pkg/hub/saas_hives_api.go @@ -174,6 +174,12 @@ type MyHiveEntry struct { AdvisoryStale bool `json:"advisoryStale,omitempty"` AdvisoryStaleReason string `json:"advisoryStaleReason,omitempty"` + // ForgeUnreachableLabel is "forge unreachable (host)" when the latest beat + // reported ForgeUnreachable and the hive is still online; the UI renders it + // in place of the generic failing-check / stale-advisory pills. Empty once + // the hive goes offline (forgeUnreachableLabel), like AdvisoryStale. + ForgeUnreachableLabel string `json:"forgeUnreachableLabel,omitempty"` + // AutoUpgradeBlocked is true when this hive has auto-upgrade ON but the hub // will REFUSE to arm it: upgradeCollectible() is false, so the spoke cannot // pull the instruction off its own heartbeat and triggerAutoUpgrades() @@ -1031,6 +1037,8 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values result[i].BudgetHealth = budgetHealthFor(result[i].RegistryEntry) result[i].GitHubAppHealth = githubAppHealthFor(result[i].RegistryEntry, journeyNow) + result[i].ForgeUnreachableLabel = forgeUnreachableLabel(result[i].RegistryEntry) + // Advisory-staleness pill, computed on read (same as Journey) so the // gating — advisory-mode participation, app-can-write, past-threshold — // lives ONLY in Go and the browser just renders the flag. diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index aa651b7e70..d107147380 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -338,7 +338,13 @@ type RegistryEntry struct { // from cache. The hive is online, but health/agent-derived truth must be // rendered as stale/unknown unless the beat explicitly carried a fresh // in-memory agent overlay. - StatsStale bool `json:"statsStale,omitempty"` + StatsStale bool `json:"statsStale,omitempty"` + // ForgeUnreachable is the spoke-reported flag that the GitHub host in + // GitHubHost cannot be reached from the pod (forge probe timed out or + // failed at the transport layer). The pod is healthy; the break is egress, + // so the hub labels the row by host and exempts it from health/advisory + // penalties. Old spokes omit it (false = today's behaviour). + ForgeUnreachable bool `json:"forgeUnreachable,omitempty"` Upgrading bool `json:"upgrading,omitempty"` UpgradeTarget string `json:"upgradeTarget,omitempty"` UpgradeStartedAt time.Time `json:"upgradeStartedAt,omitempty"` @@ -2123,6 +2129,7 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) { }(), Online: true, StatsStale: payload.StatsStale, + ForgeUnreachable: payload.ForgeUnreachable, GitHubAppRequired: payload.GitHubAppRequired, // GitHubAppPermIssue is a full sentence the drift hover renders // verbatim ("The GitHub App is configured but has no installation.