🐛 fix(hub): label forge-unreachable hives by host instead of generic stale badges

changelog.d/fixed-hub-forge-unreachable-badge.md

@@ -0,0 +1 @@
+- hub: hives whose spoke reports `forge_unreachable` are labelled "forge unreachable (github_host)" instead of generic stale badges, and the condition no longer counts against their health or advisory status

src/docs/advisory-staleness.md

@@ -15,6 +15,11 @@ never re-derives it.
 
 ## The gates
 
+0. **Forge unreachable** — a beat carrying `ForgeUnreachable` is never stale,
+   whatever the digest's age or error. The spoke cannot reach its GitHub host
+   from the pod, so an ageing digest is a symptom of the egress fault rather
+   than a wedge of its own; the row shows a `forge unreachable (github_host)`
+   pill instead (#17433).
 1. **Participation** — no post time *and* no error means "not an advisory hive
    (or a spoke too old to report)". Unknown, never stale.
 2. **Reported error** — flagged with its cause, unless the App is still
@@ -49,6 +54,7 @@ never re-derives it.
 | Pure PR/merge hive | Suppressed | Suppressed | unchanged (expected suppression) |
 | Every agent paused or off-schedule, digest ageing | Flagged stale — the pause itself lit the pill | Suppressed — no agents, no findings, no digest | Gate 3 (`allAgentsQuietByDesign`, #4528); counted as `hidden_stale` in diagnostics |
 | Spoke heartbeat collect timed out, beat carries cached/minimal stats (`statsStale`) | `advisory-stale` alert raised from the last-good `advisoryLastPostedAt` while fresh digests were posting | No fleet alert — the post time is not a fresh observation | The hub alert rule skips `StatsStale` beats (#12842); health verdict already reads "heartbeat stats stale" |
+| Spoke cannot reach its GitHub host from the pod (`ForgeUnreachable`) | Generic stale pills lit the row while the real fault was cluster egress | No stale-advisory pill — the row shows `forge unreachable (github_host)` instead | `advisoryStale` returns early on `ForgeUnreachable` beats (#17433) |
 
 ## Diagnostics
 

src/pkg/hub/advisory_staleness.go

@@ -193,6 +193,9 @@ func carryAdvisoryPostTime(entry *RegistryEntry, prev RegistryEntry) {
 // returns false — never a false alarm — matching the rule the codebase already
 // applies to StartedAt and GitHubAPIURL.
 func advisoryStale(e RegistryEntry, now time.Time) (stale bool, reason string) {
+	if e.ForgeUnreachable {
+		return false, ""
+	}
 	return advisoryStaleFromFreshness(e, advisoryFreshnessFor(e, now))
 }
 

src/pkg/hub/alerts.go

@@ -637,6 +637,9 @@ type alertHive struct {
 	// key material.
 	InferenceAuthError string
 
+	// ForgeUnreachable exempts the github_auth check from the failing-check rule.
+	ForgeUnreachable bool
+
 	// GitHubAppRequired / GitHubAppState are the spoke's own report of whether
 	// its config demands App auth and, when App auth is failing, WHY (the
 	// github.AppAuthState wire token). Carried verbatim: the spoke owns the
@@ -692,6 +695,7 @@ func alertHiveFromEntry(h MyHiveEntry) alertHive {
 		InactiveAgentsReason: h.InactiveAgentsReason,
 
 		InferenceAuthError: h.InferenceAuthError,
+		ForgeUnreachable:   h.ForgeUnreachable,
 
 		GitHubAppRequired: h.GitHubAppRequired,
 		GitHubAppState:    h.GitHubAppState,
@@ -1141,6 +1145,9 @@ func evaluateAlertsInFleet(state *alertState, hives, fleet []alertHive, driftAle
 		// seeded with a placeholder org/repo, which the check flags by design
 		// until the slot is claimed (#12843).
 		failing := failingHealthChecks(h.Health)
+		if h.ForgeUnreachable {
+			failing = withoutForgeChecks(failing)
+		}
 		if h.IsPlaceholder {
 			failing = withoutAuthClassChecks(failing, h.ProvStatus == statusAvailable)
 		}

src/pkg/hub/forge_unreachable.go

@@ -0,0 +1,31 @@
+package hub
+
+import "strings"
+
+// forgeUnreachableLabel is the row label for a hive whose latest beat said the
+// forge host cannot be reached from the pod; "" when it did not. Also "" once
+// markStaleHives flips the hive offline: the flag is carried over from the last
+// beat, and a hive that has stopped beating is not "healthy apart from forge
+// egress" — the offline row must not claim it is.
+func forgeUnreachableLabel(e RegistryEntry) string {
+	if !e.ForgeUnreachable || !e.Online {
+		return ""
+	}
+	if host := strings.TrimSpace(e.GitHubHost); host != "" {
+		return "forge unreachable (" + host + ")"
+	}
+	return "forge unreachable"
+}
+
+// withoutForgeChecks drops the github_auth check, whose failure is a symptom
+// of forge egress being down rather than of the hive being unhealthy.
+func withoutForgeChecks(names []string) []string {
+	kept := names[:0:0]
+	for _, n := range names {
+		if n == healthCheckGitHubAuth {
+			continue
+		}
+		kept = append(kept, n)
+	}
+	return kept
+}

src/pkg/hub/forge_unreachable_test.go

@@ -0,0 +1,81 @@
+package hub
+
+import (
+	"testing"
+	"time"
+)
+
+func TestForgeUnreachableLabel(t *testing.T) {
+	cases := []struct {
+		name string
+		e    RegistryEntry
+		want string
+	}{
+		{"host", RegistryEntry{Online: true, ForgeUnreachable: true, GitHubHost: "github.ibm.com"}, "forge unreachable (github.ibm.com)"},
+		{"no host", RegistryEntry{Online: true, ForgeUnreachable: true}, "forge unreachable"},
+		{"old spoke", RegistryEntry{Online: true, GitHubHost: "github.ibm.com"}, ""},
+		{"offline", RegistryEntry{ForgeUnreachable: true, GitHubHost: "github.ibm.com"}, ""},
+	}
+	for _, tc := range cases {
+		if got := forgeUnreachableLabel(tc.e); got != tc.want {
+			t.Errorf("%s: label = %q, want %q", tc.name, got, tc.want)
+		}
+	}
+}
+
+func TestForgeUnreachable_NotCountedAgainstHealth(t *testing.T) {
+	e := RegistryEntry{
+		ID: "h1", Online: true, ACMMLevel: 3,
+		GitHubAppState: "unreachable", GitHubAppRequired: true,
+	}
+	rollup := agentFleetRollup{Known: 1, Running: 1, Expected: 1}
+	app := GitHubAppHealth{Bucket: ghAppBucketBroken}
+	now := time.Now()
+
+	before := hiveHealthFor(e, rollup, app, 0, now)
+	if before.State != HealthStateRed {
+		t.Fatalf("baseline broken-app beat should be red, got %q", before.State)
+	}
+
+	e.ForgeUnreachable = true
+	after := hiveHealthFor(e, rollup, app, 0, now)
+	if after.State == HealthStateUnknown || after.State == HealthStateRed {
+		t.Fatalf("forge-unreachable hive must not be penalised, got %q (%s)", after.State, after.Reason)
+	}
+}
+
+// A beat carrying both flags still has cached stats: the hub must not assert
+// health from them, but it names the forge as the cause instead of a generic
+// stale verdict.
+func TestForgeUnreachable_StaleStatsStayUnknown(t *testing.T) {
+	e := RegistryEntry{
+		ID: "h1", Online: true, StatsStale: true, ForgeUnreachable: true, ACMMLevel: 3,
+	}
+	rollup := agentFleetRollup{Known: 1, Running: 1, Expected: 1}
+	v := hiveHealthFor(e, rollup, GitHubAppHealth{Bucket: ghAppBucketOK}, 0, time.Now())
+	if v.State != HealthStateUnknown {
+		t.Fatalf("stale stats must stay unknown even when forge unreachable, got %q", v.State)
+	}
+	if v.Reason != "heartbeat stats stale (forge unreachable)" {
+		t.Fatalf("reason = %q", v.Reason)
+	}
+}
+
+func TestForgeUnreachable_NotStaleAdvisory(t *testing.T) {
+	e := advisoryModeEntry()
+	e.AdvisoryLastPostedAt = rfc3339Ago(30 * 24 * time.Hour)
+	if stale, _ := advisoryStale(e, advNow); !stale {
+		t.Fatal("baseline: aged digest should be stale")
+	}
+	e.ForgeUnreachable = true
+	if stale, reason := advisoryStale(e, advNow); stale {
+		t.Fatalf("forge-unreachable hive must not be stale-advisory, got %q", reason)
+	}
+}
+
+func TestForgeUnreachable_WithoutForgeChecks(t *testing.T) {
+	got := withoutForgeChecks([]string{"github_auth", "disk"})
+	if len(got) != 1 || got[0] != "disk" {
+		t.Fatalf("got %v", got)
+	}
+}

src/pkg/hub/health_verdict.go

@@ -79,6 +79,14 @@ func hiveHealthFor(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealth
 func hiveHealthBase(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealth, queuedWork int, now time.Time) HealthVerdict {
 	v := HealthVerdict{QueuedWork: queuedWork}
 
+	// Forge egress is broken, not the pod: the github_auth failure it causes
+	// must not read as a hive fault. StatsStale is deliberately NOT cleared
+	// here — cached stats are still cached, and the stale gate below never
+	// asserts health from them; the gate only names the forge as the cause.
+	if e.ForgeUnreachable {
+		app = GitHubAppHealth{Bucket: ghAppBucketUnknown}
+	}
+
 	// --- Unknown gates first: never claim health for a hive we can't see. ---
 	if !e.Online {
 		v.State = HealthStateUnknown
@@ -89,6 +97,9 @@ func hiveHealthBase(e RegistryEntry, rollup agentFleetRollup, app GitHubAppHealt
 	if e.StatsStale {
 		v.State = HealthStateUnknown
 		v.Reason = "heartbeat stats stale"
+		if e.ForgeUnreachable {
+			v.Reason = "heartbeat stats stale (forge unreachable)"
+		}
 		v.OutputKind = outputKindForLevel(e.ACMMLevel)
 		return v
 	}

src/pkg/hub/heartbeat.go

@@ -296,6 +296,7 @@ type HeartbeatPayload struct {
 	GitBranch                    string                          `json:"git_branch,omitempty"`
 	ImageRef                     string                          `json:"image_ref,omitempty"`
 	GitHubHost                   string                          `json:"github_host,omitempty"`
+	ForgeUnreachable             bool                            `json:"forge_unreachable,omitempty"`
 	Timestamp                    string                          `json:"timestamp"`
 	GitHubAppRequired            bool                            `json:"github_app_required,omitempty"`
 	GitHubAppPermIssue           string                          `json:"github_app_perm_issue,omitempty"`

src/pkg/hub/saas_hives_api.go

@@ -174,6 +174,12 @@ type MyHiveEntry struct {
 	AdvisoryStale       bool   `json:"advisoryStale,omitempty"`
 	AdvisoryStaleReason string `json:"advisoryStaleReason,omitempty"`
 
+	// ForgeUnreachableLabel is "forge unreachable (host)" when the latest beat
+	// reported ForgeUnreachable and the hive is still online; the UI renders it
+	// in place of the generic failing-check / stale-advisory pills. Empty once
+	// the hive goes offline (forgeUnreachableLabel), like AdvisoryStale.
+	ForgeUnreachableLabel string `json:"forgeUnreachableLabel,omitempty"`
+
 	// AutoUpgradeBlocked is true when this hive has auto-upgrade ON but the hub
 	// will REFUSE to arm it: upgradeCollectible() is false, so the spoke cannot
 	// pull the instruction off its own heartbeat and triggerAutoUpgrades()
@@ -1031,6 +1037,8 @@ func (s *HubServer) buildMyHivesResponse(username string, queryValues url.Values
 		result[i].BudgetHealth = budgetHealthFor(result[i].RegistryEntry)
 		result[i].GitHubAppHealth = githubAppHealthFor(result[i].RegistryEntry, journeyNow)
 
+		result[i].ForgeUnreachableLabel = forgeUnreachableLabel(result[i].RegistryEntry)
+
 		// Advisory-staleness pill, computed on read (same as Journey) so the
 		// gating — advisory-mode participation, app-can-write, past-threshold —
 		// lives ONLY in Go and the browser just renders the flag.

src/pkg/hub/server.go

@@ -338,7 +338,13 @@ type RegistryEntry struct {
 	// from cache. The hive is online, but health/agent-derived truth must be
 	// rendered as stale/unknown unless the beat explicitly carried a fresh
 	// in-memory agent overlay.
-	StatsStale       bool      `json:"statsStale,omitempty"`
+	StatsStale bool `json:"statsStale,omitempty"`
+	// ForgeUnreachable is the spoke-reported flag that the GitHub host in
+	// GitHubHost cannot be reached from the pod (forge probe timed out or
+	// failed at the transport layer). The pod is healthy; the break is egress,
+	// so the hub labels the row by host and exempts it from health/advisory
+	// penalties. Old spokes omit it (false = today's behaviour).
+	ForgeUnreachable bool      `json:"forgeUnreachable,omitempty"`
 	Upgrading        bool      `json:"upgrading,omitempty"`
 	UpgradeTarget    string    `json:"upgradeTarget,omitempty"`
 	UpgradeStartedAt time.Time `json:"upgradeStartedAt,omitempty"`
@@ -2123,6 +2129,7 @@ func (s *HubServer) handleHeartbeat(w http.ResponseWriter, r *http.Request) {
 		}(),
 		Online:            true,
 		StatsStale:        payload.StatsStale,
+		ForgeUnreachable:  payload.ForgeUnreachable,
 		GitHubAppRequired: payload.GitHubAppRequired,
 		// GitHubAppPermIssue is a full sentence the drift hover renders
 		// verbatim ("The GitHub App is configured but has no installation.