Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
44 changes: 44 additions & 0 deletions bin/contributor-relay.js
Original file line number Diff line number Diff line change
Expand Up @@ -1776,6 +1776,10 @@ function makeHub(url, token, disabled = false, contributorId = '', repoScope = n
// (quota_hold, capability quota_hold_v1). Reset on every auth_ok: a fresh
// connection on the hub starts out not held.
quotaHoldSent: false,
// What this hub's CURRENT connection was last told about this relay
// working ANOTHER hive's task (busy_elsewhere, capability
// busy_elsewhere_v1). Reset on every auth_ok, like quotaHoldSent.
busyElsewhereSent: false,
// Set by handleTLSHostMismatch when the dialed host is not in the served
// certificate; parks the hub (no reconnect loop) until contributor.env
// changes. relocationAttempted bounds the hub-registry re-resolution to
Expand Down Expand Up @@ -1916,6 +1920,9 @@ function writeRelayStatus() {
// Every quota hold transition already ends here (it changes relayReadiness),
// so this is the one place the hub is told — see reportQuotaHold.
reportQuotaHoldToHubs();
// Every currentTask transition (assign, complete, fail, revoke) ends here
// too, so the same holds for busy_elsewhere — see reportBusyElsewhere.
reportBusyElsewhereToHubs();
}

// ── Quota hold report (quota_hold_v1) ────────────────────────────────────────
Expand Down Expand Up @@ -1950,6 +1957,39 @@ function reportQuotaHoldToHubs() {
for (const hub of hubs) reportQuotaHold(hub);
}

// ── Busy on another hive (busy_elsewhere_v1) ─────────────────────────────────
// A relay subscribed to several hives works one primary task at a time. While
// that task belongs to hive B, hive A still sees a connected relay that
// heartbeats, holds its "Acting as" role and has been refused nothing, so A's
// contributor-first gate (RoleCoverage) counted it as covering A's lane and
// withheld A's resident agent while nobody worked that lane. Tell every OTHER
// hub that advertises busy_elsewhere_v1 when such a task starts and when it
// ends; a hub without the token never sees the frame, and keeps today's
// reading. The frame carries only when the other task started — never which
// hive, repository or issue — so one hive learns nothing about another's work.
// The local review cycle belongs to no hub and is not reported.
function busyElsewhereFor(hub) {
if (!currentTask || isLocalOnlyTask(currentTask)) return false;
const owner = currentTask._hub;
return !!owner && owner !== hub;
}

function reportBusyElsewhere(hub) {
if (!hub || !hub.authenticated || hub.disabled) return;
if (!Array.isArray(hub.serverCapabilities) || !hub.serverCapabilities.includes('busy_elsewhere_v1')) return;
const busy = busyElsewhereFor(hub);
if (busy === hub.busyElsewhereSent) return;
if (!hub.ws || hub.ws.readyState !== WebSocket.OPEN) return;
hub.busyElsewhereSent = busy;
sendTo(hub, busy
? { type: 'busy_elsewhere', seq: nextSeq(), state: 'busy', since: currentTaskStartedAt || new Date().toISOString() }
: { type: 'busy_elsewhere', seq: nextSeq(), state: 'clear' });
}

function reportBusyElsewhereToHubs() {
for (const hub of hubs) reportBusyElsewhere(hub);
}

let progressInterval = null;
let verdictWatchInterval = null;
let verdictFastPathLine = null;
Expand Down Expand Up @@ -9810,6 +9850,10 @@ function handleMessage(data, hub) {
// before anything else so contributor-first cover is right from the start.
hub.quotaHoldSent = false;
reportQuotaHold(hub);
// Same for busy_elsewhere: a new connection starts out not busy on the
// hub, so restate another hive's running task to it.
hub.busyElsewhereSent = false;
reportBusyElsewhere(hub);
declareStandby(hub);
hub.scopeSent = false;
negotiateRepoScope(hub);
Expand Down
79 changes: 79 additions & 0 deletions bin/contributor-relay.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -17338,6 +17338,85 @@ test('#16463 standby acknowledgement does not outlive the connection it was gran
}
});

// ---------------------------------------------------------------------------
// busy_elsewhere_v1: a relay working one hive's task tells its OTHER hives, so
// they stop counting it as contributor-first cover. Observed on 2026-11-09: a
// relay set "Acting as: scanner" on projectbluefin worked hivecommons tasks
// from 14:25Z to 14:32Z and sent projectbluefin no ready.
// ---------------------------------------------------------------------------

function busyElsewhereFrames(sent) {
return sent.filter(m => m.type === 'busy_elsewhere');
}

function busyAuth(relay, hub, capabilities = ['busy_elsewhere_v1']) {
relay.handleMessage(JSON.stringify({ type: 'auth_ok', contributor_id: 'c1', trust_tier: 'contributor', server_capabilities: capabilities }), hub);
}

// Rotate the polling slot to hive B (as a task_unavailable from A would) and
// take B's task through the real assignment handler.
function takeTaskFromHubB(relay, hubs, taskId) {
relay.sendReadyForNextTask('unavailable');
relay.handleMessage(JSON.stringify({ type: 'task_assign', task_id: taskId, task_gen: 1, kind: 'issue', repo: 'org/b', number: 7, title: 'work on hive B' }), hubs[1]);
assert.strictEqual(relay.getCurrentTask() && relay.getCurrentTask().task_id, taskId, 'precondition: the relay took hive B\'s task');
}

test('busy_elsewhere_v1: another hive\'s task is reported to the other hive, restated on reconnect, and cleared when it finishes', () => {
const relay = loadRelay({ env: MULTI_HUB_ENV });
const log = console.log; const warn = console.warn;
console.log = () => {}; console.warn = () => {};
try {
const { hubs, sentA, sentB } = attachHubSinks(relay);
busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]);
assert.deepStrictEqual(busyElsewhereFrames([...sentA, ...sentB]), [], 'an idle relay is busy nowhere');

takeTaskFromHubB(relay, hubs, 'b-1');
const frames = busyElsewhereFrames(sentA);
assert.deepStrictEqual(frames.map(m => m.state), ['busy'], 'hive A must hear about hive B\'s task once');
assert.deepStrictEqual(Object.keys(frames[0]).sort(), ['seq', 'since', 'state', 'type'],
'the frame names no hive, repository or task: one hive learns nothing about another\'s work');
assert.ok(!Number.isNaN(Date.parse(frames[0].since)), `since must be a timestamp: ${frames[0].since}`);
assert.deepStrictEqual(busyElsewhereFrames(sentB), [], 'the hive that owns the task is never told it is busy elsewhere');

// A reconnect to hive A is a new connection that knows nothing: restate.
busyAuth(relay, hubs[0]);
assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'busy']);

relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null });
assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'busy', 'clear'],
'finishing hive B\'s task must clear the report on hive A');
assert.deepStrictEqual(busyElsewhereFrames(sentB), []);
} finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); }
});

test('busy_elsewhere_v1: a revoked task clears the report', () => {
const relay = loadRelay({ env: MULTI_HUB_ENV });
const log = console.log; const warn = console.warn;
console.log = () => {}; console.warn = () => {};
try {
const { hubs, sentA } = attachHubSinks(relay);
busyAuth(relay, hubs[0]); busyAuth(relay, hubs[1]);
takeTaskFromHubB(relay, hubs, 'b-2');
relay.handleMessage(JSON.stringify({ type: 'task_revoke', task_id: 'b-2', reason: 'operator stop' }), hubs[1]);
assert.strictEqual(relay.getCurrentTask(), null, 'precondition: the revoke ended the task');
assert.deepStrictEqual(busyElsewhereFrames(sentA).map(m => m.state), ['busy', 'clear']);
} finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); }
});

test('busy_elsewhere_v1: a hive that does not advertise the token never sees the frame', () => {
const relay = loadRelay({ env: MULTI_HUB_ENV });
const log = console.log; const warn = console.warn;
console.log = () => {}; console.warn = () => {};
try {
const { hubs, sentA, sentB } = attachHubSinks(relay);
busyAuth(relay, hubs[0], ['quota_hold_v1']); busyAuth(relay, hubs[1]);
takeTaskFromHubB(relay, hubs, 'b-3');
busyAuth(relay, hubs[0], ['quota_hold_v1']);
relay.finishCurrentTask({ completionSignal: 'verdict', summary: 'done', tmuxLines: [], prURL: null, noWork: null });
assert.deepStrictEqual(busyElsewhereFrames([...sentA, ...sentB]), [], 'an old hub must see exactly today\'s traffic');
} finally { console.log = log; console.warn = warn; relay.setCurrentTask(null); teardown(relay); }
});

// ---------------------------------------------------------------------------
// quota_hold_v1: a held relay tells the hub, so it stops counting as
// contributor-first cover. Observed on 2026-11-08: a relay acting as scanner
Expand Down
1 change: 1 addition & 0 deletions changelog.d/fixed-busy-elsewhere-relay-not-cover.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- Contributor-first lanes are no longer left unworked by a relay that is busy on another hive. A relay subscribed to several hives works one task at a time, and while that task belonged to another hive it still looked to this one like a connected relay holding its role, so the governor kept the resident agent's kicks withheld while nobody worked the lane (seen on 2026-11-09: a relay set "Acting as: scanner" on one hive worked another hive's tasks for seven minutes and sent the first hive no request for work). Relays now tell every other hive that advertises the new `busy_elsewhere_v1` capability when they start and finish another hive's task, and the hub leaves such a relay out of coverage until the task ends or the relay asks it for work again. The frame names no hive, repository or issue. Older relays and older hubs behave as before.
8 changes: 8 additions & 0 deletions src/docs/agent-configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -1394,6 +1394,14 @@ A relay counts as covering the role only when all of these hold:
tells the hub (`quota_hold_v1`, see
[the contributor relay guide](contributor-relay.md#a-held-relay-tells-the-hub-quota_hold_v1));
an older relay is still counted while held;
- it is not busy on another hive. A relay subscribed to several hives works
one task at a time; while that task belongs to another hive it stays
connected here but asks this hive for nothing. A relay new enough to know
about this tells every other hive it is subscribed to (`busy_elsewhere_v1`, see
[the contributor relay guide](contributor-relay.md#a-relay-busy-on-another-hive-tells-the-hub-busy_elsewhere_v1))
and clears it when that task ends; its next request for work here clears it
too. A relay that this hive has also handed a task or a fix-pr task still
counts, and an older relay is still counted while busy elsewhere;
- it is not in a failure-streak pause;
- it has not used up its hourly or daily task allowance
(`hub.tier_limits.<tier>.max_per_hour` / `max_per_day` for the
Expand Down
40 changes: 40 additions & 0 deletions src/docs/contributor-relay.md
Original file line number Diff line number Diff line change
Expand Up @@ -2021,6 +2021,46 @@ about it:

The expired-CLI-login hold is not reported this way yet.

### A relay busy on another hive tells the hub (`busy_elsewhere_v1`)

A relay subscribed to several hives works one task at a time. While that task
belongs to hive B, hive A still sees a connected relay that answers heartbeats,
holds its "Acting as" role and has been refused nothing. Before this, a relay
acting as a role on A kept counting as
[contributor-first](agent-configuration.md#contributor-first-lanes-contributor_first)
cover there for as long as it worked B's tasks, so A's resident agent was held
back while nobody worked A's lane.

Every other hub that advertises `busy_elsewhere_v1` in
`auth_ok.server_capabilities` now hears about it:

```jsonc
{ "type": "busy_elsewhere", "seq": 61, "state": "busy", "since": "2026-11-09T14:25:03Z" }
{ "type": "busy_elsewhere", "seq": 62, "state": "clear" }
```

- The relay sends `busy` when it starts a task from another hive and `clear`
when that task completes, fails or is revoked. It sends each transition once
per connection, and restates a running task right after `auth_ok`, because a
new connection starts out not busy on the hub. The hive that owns the task
is never sent the frame.
- The frame says only when the other task started. It never names the other
hive, repository or issue.
- The hub records it on the connection and uses it only for contributor-first
coverage. It changes no lease, assignment or standby state. A relay this hub
has also handed a task or a fix-pr task still counts.
- There is no timer on the hub. The flag goes away with the connection (a
socket close or a relay restart), and the relay's next `ready` to this hub
clears it, since a relay asks for work only when it holds none. A timer
would start counting a relay that is still busy on a long task again; a flag
left set by mistake only means the resident agent works the lane, which is
what happens without `contributor_first`.
- A hub that does not advertise the token never receives the frame. A relay
that never sends it is read as not busy elsewhere. Either way the behaviour
is exactly what it was before.

The local PR-review cycle belongs to no hive and is not reported.

### Provider adapter sources

The `src/pkg/rotation` probers normalize each provider's usage into the reading
Expand Down
2 changes: 1 addition & 1 deletion src/docs/design/agent-state-inventory.md
Original file line number Diff line number Diff line change
Expand Up @@ -180,7 +180,7 @@ key generations (`/data/saas/hub-generations.json`).

| # | State | Where (v5) | On a spoke (dashboard) pod roll | Verdict |
|---|-------|------------|--------------------------------|---------|
| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:975`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** |
| 27 | Completed-task ledger + per-task cooldowns + PR URLs | `completedTasks` / `completedTaskCooldown` / `completedTaskPRURL` (`contribute_ws.go:985`), persisted to `/data/contributors/completed-tasks.json` | Survives; was moved to the PVC ledger through the #3792 → #3989 lineage after re-offer loops | **fixed** |
| 28 | Failure cooldowns + quarantine counters | `failedTasks` / `consecutiveFailures` (`:373`, `:381`), persisted to `/data/contributors/failed-tasks.json` | Survives (#2435 livelock fix, made durable with the ledger) | **fixed** |
| 29 | No-PR completion streaks | `noPRStreaks` (`:394`), `/data/contributors/no-pr-streaks.json` | Survives (#3980; geometric backoff must outlive the completion entry it escalates) | **fixed** |
| 30 | `no_work_needed` verdicts | `noWorkVerdicts` (`:416`), `/data/contributors/no-work-verdicts.json` | Survives (#3987/#3997) — "persisted in the same PVC-backed ledger dir as the cooldowns so a pod restart does not forget the verdict" | **fixed** |
Expand Down
30 changes: 28 additions & 2 deletions src/pkg/dashboard/contribute_coverage.go
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,15 @@ import (
// Observed on 2026-11-08: a relay held at 15% of its weekly window kept
// the scanner lane "covered" while nobody worked it. A relay too old to
// send quota_hold is still counted, as before;
// - it has not reported that it is busy on another hive (busy_elsewhere,
// capability busy_elsewhere_v1) while holding no task from this one. A
// relay subscribed to several hives works one task at a time; while that
// task belongs to another hive it heartbeats and keeps its role here but
// asks this hub for nothing. Observed on 2026-11-09: a relay set "Acting
// as: scanner" on one hive spent the next seven minutes on another hive's
// tasks and sent the first hive no ready. Its next ready here clears the
// report (see busyElsewhereLocked). A relay too old to send busy_elsewhere
// is still counted, as before;
// - it is not in a failure-streak pause: a relay whose runtime keeps dying
// is connected but is not doing the lane's work;
// - its identity has not used up its hourly or daily task allowance
Expand All @@ -60,7 +69,9 @@ import (
// since. Standby, quota hold, a used-up allowance and a closed tier do not
// wait for the task to end: each means the relay will not take the lane's NEXT
// item, which is what the governor is deciding whether to withhold. (Holding
// max_concurrent tasks is different: that relay is busy, and counts.) Whether
// max_concurrent tasks is different: that relay is busy, and counts. So is a
// relay busy on another hive that this hub has also handed a task or a fix-pr
// task: it is working this lane too.) Whether
// the relay is ALLOWED the role (trust tier, grants, the delegatable list) was
// settled when the hub set c.role; nothing is re-derived here.
func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.ContributorCoverage {
Expand All @@ -84,7 +95,7 @@ func (h *ContributeWSHub) RoleCoverage(now time.Time) map[string]governor.Contri
login = strings.TrimSpace(c.profile.GitHubUsername)
tier = c.profile.TrustTier
}
notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld
notTakingWork := c.standbyDeclared || len(c.standby) > 0 || c.quotaHeld || c.busyElsewhereLocked()
queueEmpty := c.noWorkForWindowLocked(now)
c.mu.Unlock()
if role == "" || !fresh || login == "" || notTakingWork || queueEmpty {
Expand Down Expand Up @@ -190,6 +201,21 @@ func (c *ContributorConnection) noWorkForWindowLocked(now time.Time) bool {
return now.Sub(c.lastIdleReasonSince) >= coverageNoWorkWindow
}

// busyElsewhereLocked reports whether c has said it is working another hive's
// task (busy_elsewhere) and this hub has handed it no task of its own, primary
// or fix-pr. The caller holds c.mu.
//
// There is no wall-clock expiry. The flag is per connection, so a socket close
// or a relay restart drops it with the connection; the relay clears it when the
// other task ends, and the relay's next ready here clears it too (handleReady).
// A timeout would only re-count a relay that is still busy on a long task —
// the bug this check exists to stop — while a flag left set by mistake errs
// the safe way: the resident agent works the lane, as it would without
// contributor_first.
func (c *ContributorConnection) busyElsewhereLocked() bool {
return c.busyElsewhere && c.currentTask == nil && len(c.fixTasks) == 0
}

// contributionsSuspended mirrors the task-selection gate in
// contribute_select.go: with hub.contribute_suspended set, no relay is handed
// work, whatever role it holds.
Expand Down
Loading
Loading