diff --git a/agent_sys/cli/environment.py b/agent_sys/cli/environment.py index c70fff411..bf3cdd328 100644 --- a/agent_sys/cli/environment.py +++ b/agent_sys/cli/environment.py @@ -37,6 +37,7 @@ from env_mgr.fs.domain import DomainRegistry from env_mgr.isolation.policy import Granted, Mode, interpreter_grants from env_mgr.isolation.probe import Availability, probe, select +from env_mgr.prefix import CLAUDE_CONFIG_ENV_VAR, Prefix from env_mgr.protocols import Context, DomainKind, NoConfinement, Tier __all__ = [ @@ -282,6 +283,47 @@ def confinement(availability: Availability | None = None) -> str: # Credentials +def _probe_environment() -> dict[str, str]: + """The ambient environment **plus** the o11y prefix's `CLAUDE_CONFIG_DIR`. + + Gate 1 covers *agent* children; this subprocess is not one, so it dropped a + JSONL into `~/.claude/projects` every run — measured. Copied, not replaced: + a bare `env={...}` strips `PATH`, and a probe that cannot run refuses the + whole run. Never into our own `os.environ`. + """ + env = dict(os.environ) + env[CLAUDE_CONFIG_ENV_VAR] = str(Prefix.resolve(os.environ).claude_home) + return env + + +#: Where the probe runs. Its own directory, because AgentsView names a project +#: after the session's cwd — resolving the git *main repository* when there is +#: one — so inheriting the caller's put ten identical probe transcripts into the +#: real `infera` project. A plain directory falls back to its basename, and +#: `probe` is what these sessions are. +PROBE_DIR = "probe" + + +def probe_cwd(prefix: Prefix) -> Path: + return prefix.state / PROBE_DIR + + +def _probe_cwd_or_none(prefix: Prefix) -> str | None: + """The probe's own directory, or `None` if we could not make one. + + **A cwd is not worth failing the run for.** `preflight_credentials` aborts + everything when it fails, and a child refuses a cwd that does not exist — + so an unwritable prefix must fall back to the old behaviour, not turn a + misfiled transcript into a dead deployment. + """ + try: + cwd = probe_cwd(prefix) + cwd.mkdir(parents=True, exist_ok=True) + except OSError: + return None + return str(cwd) + + def preflight_credentials(*, cli: str = BACKEND, timeout: float = 90.0) -> str: """Ask the backend whether it can run at all, **before any zone is built**. @@ -293,7 +335,12 @@ def preflight_credentials(*, cli: str = BACKEND, timeout: float = 90.0) -> str: `CredentialsMissing` carrying **stdout and stderr both** on failure. **It does not test what the run does, and saying so is the point.** This - runs `claude -p` *unconfined*, against the operator's own config directory. + runs `claude -p` *unconfined*, against the operator's own credentials — but + not their own config directory: `CLAUDE_CONFIG_DIR` points into + `~/.infera_agent_sys` like every other `claude` child we spawn, so the + transcript lands there. Measured to keep authentication working; see + `_probe_environment`. Not the relocation the table below is about. + A confined task gets a different arm: `material.deploy` points `CLAUDE_CONFIG_DIR` into the zone — correctly, it is what removed the `$HOME` grant — which also moves away the `env` block in `~/.claude/settings.json` @@ -330,6 +377,8 @@ def preflight_credentials(*, cli: str = BACKEND, timeout: float = 90.0) -> str: try: done = subprocess.run( # noqa: S603 — `binary` came from `shutil.which` [binary, "-p", "Reply with exactly one word: ready"], + env=_probe_environment(), + cwd=_probe_cwd_or_none(Prefix.resolve(os.environ)), capture_output=True, text=True, timeout=timeout, diff --git a/agent_sys/cli/events.py b/agent_sys/cli/events.py index 582447d9e..4c1465132 100644 --- a/agent_sys/cli/events.py +++ b/agent_sys/cli/events.py @@ -22,7 +22,7 @@ __all__ = ["SCHEMA_VERSION", "Event", "EventKind"] -SCHEMA_VERSION = "1.3" +SCHEMA_VERSION = "1.4" """The schema of the machine-readable stream. Criterion 14 makes this an interface: **bump it on any change to `EventKind`, @@ -68,6 +68,16 @@ one that was never declared, and the difference is the whole claim the run is making. +**1.4** — `O11Y_PANEL`. The AgentsView panel's URL, and the notice that its +binary was fetched for the first time, were `log.info` calls. **Nothing in this +repository configures `logging`**, so the root logger sits at `WARNING` with no +handler and both lines were discarded — while the o11y failure paths, being +`log.warning`, reached stderr through `logging.lastResort`. Failures were +visible and successes were not, and the tests did not notice because +`caplog.at_level("INFO")` forces the level from pytest's side. A fact the user +is meant to read belongs in the stream, which is the thing in this package +whose job is being read; `logging` here is for the operator's diary. + `docs/interfaces.md` §5.7: once the whole-system CLI wants the same stream, two artefacts share this constant with no bump policy. That is open. """ @@ -96,6 +106,7 @@ class EventKind(str, Enum): PERMISSIONS_DISABLED = "permissions_disabled" ZONE_PREPARED = "zone_prepared" ACCESS_DENIED = "access_denied" + O11Y_PANEL = "o11y_panel" # what this run did NOT check, and why. Absent is not the same as dropped. VALIDATION_DROPPED = "validation_dropped" diff --git a/agent_sys/cli/main.py b/agent_sys/cli/main.py index 0acd84a24..96b965914 100644 --- a/agent_sys/cli/main.py +++ b/agent_sys/cli/main.py @@ -21,9 +21,10 @@ import argparse import logging +import os import shutil import sys -from collections.abc import Sequence +from collections.abc import Callable, Sequence from contextlib import ExitStack from pathlib import Path from typing import Any, TextIO @@ -46,6 +47,16 @@ from cli.render.machine import JsonLinesRenderer from cli.stream import Stream from env_mgr import meta +from env_mgr.o11y.agentsview import ( + RECIPE_PATH, + freshly_installed, + ensure_installed, + ensure_run_project, + ensure_running, + pinned_version, + resolve_port, +) +from env_mgr.prefix import Prefix from env_mgr.prepare import EnvManager, permissions_enforced from env_mgr.protocols import NoConfinement, PrepareRefused, UnresolvedGrant from env_mgr.remote.connection import sync_transport @@ -173,6 +184,21 @@ def parser() -> argparse.ArgumentParser: "ends in seconds regardless; this only bounds one that never stops" ), ) + run.add_argument( + "--agentsview-port", + type=int, + default=None, + metavar="N", + help=( + "port for the AgentsView o11y panel (default 18888; " + "a port already in use is a warning and a skip)" + ), + ) + run.add_argument( + "--no-agentsview", + action="store_true", + help="do not start the AgentsView o11y panel", + ) return top @@ -209,7 +235,16 @@ def main(argv: Sequence[str] | None = None) -> int: try: if args.verb == "show": return _show(args, stream) - return _run(args, stream) + # The one call site: the daemon outlives the run, so it starts + # once per invocation and its result never reaches the exit code. + # Not for `--dry-run` (whose contract is *resolve everything, do + # nothing*) or `--clean` (which deletes every run and exits). + panel_url = _start_o11y( + args.agentsview_port, + disabled=args.no_agentsview or args.dry_run or args.clean, + stream=stream, + ) + return _run(args, stream, panel_url) except package.PackageNotFound as exc: return _fail(stream, PRECONDITION, str(exc)) except SpecInvalid as exc: @@ -226,6 +261,110 @@ def main(argv: Sequence[str] | None = None) -> int: return UNEXPECTED_FAILURE # pragma: no cover — ExitStack always returns above +def _install_item(prefix: Prefix) -> Callable[[], Sequence[Any]]: + """The recipe call `ensure_installed` injects rather than performs. + + **This is o11y-shaped code living in `cli/`, and review asked why. It is + here because it cannot be under `env_mgr/`.** Spec §9 walls `recipe`, + `runner` and `installers` off from every module there, and + `tests/env_mgr/test_imports.py` enforces it structurally — it derives the + "above the wall" set from the filesystem and walks it with `rglob`, so a new + subpackage is covered the moment it exists, with `env_mgr/cli.py` the single + exemption. This function's whole body is `load_recipe` + `runner.run`, so + any home under `env_mgr/o11y/` fails that test. A first draft of + `ensure_installed` did exactly that and failed exactly that test, which is + why it takes an injected callable rather than looking the recipe up itself. + + Moving it would mean either putting it in `env_mgr/cli.py` — legal, but that + is env_mgr's command-line entry point and it would be there for the + exemption rather than because it belongs — or widening the exemption, which + weakens a guard whose own docstring records a module going unchecked when + the list was maintained by hand. Neither is this function's call to make. + + **Zero-argument, not a precomputed list**: a list evaluated at the call site + would run the installer before `--dry-run` could stop it. `target.path` is + overridden because the checked-in recipe's value is a placeholder — nothing + in `env_mgr` expands `${VAR}` in a YAML value. + """ + + def call() -> Sequence[Any]: + from env_mgr.recipe import load_recipe + from env_mgr.runner import Filters, run + + target, items = load_recipe(RECIPE_PATH) + target.path = str(prefix.root) + outs, _status = run(target, items, "install", Filters(item="agentsview")) + return outs + + return call + + +def _start_o11y( + port_flag: int | None, disabled: bool, stream: Stream | None = None +) -> str | None: + """The one call site. Returns the panel URL, or None, and never raises. + + **Also o11y-shaped code in `cli/`, and also deliberate.** Review asked for + it to live in `env_mgr/o11y/`, and the destination is right; the move is a + refactor rather than a relocation, because this function is tied to `cli/` + at two points. It calls `_install_item`, which cannot leave (see there). And + it emits on the `Stream`, so moving it as written would have `env_mgr` + importing `cli` — a library importing its own consumer, which is a worse + inversion than the one being fixed. The honest shape is + `start_panel(prefix, port_flag, install_item, announce)` in the o11y package + with a four-line adapter here, and it re-points the ten or so tests that + patch `cli_main.ensure_installed` / `cli_main.ensure_running` at module + level. Worth doing on its own, not folded into a review-fix. + + **The bare `except Exception` is the point**: everything inside + `ensure_running` already degrades to a warning, and this catches what that + module has not thought of. A side-car that can abort a run is a worse bug + than a missing panel. + + **Success goes to the `stream`, failure to `logging`.** Both were + `log.info`, and this package never configures `logging` — so the root + logger sits at `WARNING` with no handler and they reached nobody, while the + warnings still reached stderr through `lastResort`. `stream` is optional + because the failure-mode tests are not about it; `main` always passes one. + + `os.environ` is read here and never written. + """ + if disabled: + return None + + def say(message: str, **fields: Any) -> None: + if stream is not None: + stream.emit(EventKind.O11Y_PANEL, message, **fields) + + try: + prefix = Prefix.resolve(os.environ) + installed = ensure_installed(prefix, _install_item(prefix)) + if not installed.running: + # `ensure_installed` has already logged the one warning. Starting a + # daemon whose binary is absent would only add a second. + return None + if freshly_installed(installed.reason): + # Only on the run that downloaded: a line on every run is how a + # real warning gets scrolled past. Says what arrived and where, + # because a 45 MB download nobody asked for should be inspectable. + version = pinned_version() + path = str(prefix.bin / "agentsview") + message = ( + f"fetched the o11y panel binary (agentsview v{version}, " + f"from github.com/kenn-io/agentsview) into {path}" + ) + log.info("agentsview: %s", message) + say(message, version=version, path=path, installed=True) + status = ensure_running(prefix, port=resolve_port(port_flag, os.environ)) + if status.running: + log.info("agentsview: o11y panel at %s", status.url) + say(f"panel at {status.url}", url=status.url) + return status.url + except Exception as e: # noqa: BLE001 + log.warning("agentsview: o11y start-up failed (%s); continuing without a panel.", e) + return None + + def _fail(stream: Stream, code: int, message: str, *, kind: EventKind | None = None) -> int: stream.emit(kind or EventKind.RUN_COMPLETE, message, exit_code=code, ok=False) return code @@ -259,12 +398,12 @@ def _show(args: argparse.Namespace, stream: Stream) -> int: # run -def _run(args: argparse.Namespace, stream: Stream) -> int: +def _run(args: argparse.Namespace, stream: Stream, panel_url: str | None = None) -> int: if args.clean: return _clean(args, stream) if args.dry_run: return _dry_run(args, stream) - return _real_run(args, stream) + return _real_run(args, stream, panel_url) def _clean(args: argparse.Namespace, stream: Stream) -> int: @@ -326,7 +465,7 @@ def _layout(args: argparse.Namespace) -> Layout: return layout_for(root).create() -def _real_run(args: argparse.Namespace, stream: Stream) -> int: +def _real_run(args: argparse.Namespace, stream: Stream, panel_url: str | None = None) -> int: """Everything. Needs credentials, a sandbox, and a model. The order of the two preconditions is measured rather than aesthetic: the @@ -351,6 +490,18 @@ def _real_run(args: argparse.Namespace, stream: Stream) -> int: promises = expectations.for_package(package.locate(args.package)) layout = _layout(args) + # **Here, and not in `_start_o11y`, because the run id does not exist yet + # when the panel starts.** Before any task runs, so the mapping is in place + # before the first transcript is ingested -- measured: a mapping that + # exists at ingest labels the session at sync time, with no second call. + mapped = ensure_run_project(panel_url, layout.run) + if mapped.running: + stream.emit( + EventKind.O11Y_PANEL, + f"this run is project {mapped.reason!r} on the panel", + project=mapped.reason, + run=str(layout.run), + ) root = package.locate(args.package) # **Read once, at start-up, and it is the run's fact rather than a task's.** # `env_mgr.prepare.permissions_enforced()` is the single reader of the diff --git a/agent_sys/cli/render/human.py b/agent_sys/cli/render/human.py index dffe19264..a199fcc05 100644 --- a/agent_sys/cli/render/human.py +++ b/agent_sys/cli/render/human.py @@ -35,6 +35,7 @@ EventKind.PERMISSIONS_DISABLED: "NO SANDBOX", EventKind.VALIDATION_DROPPED: "DROPPED", EventKind.ZONE_PREPARED: "zone", + EventKind.O11Y_PANEL: "o11y", EventKind.ACCESS_DENIED: " denied", EventKind.TASK_DISPATCHED: "dispatch", EventKind.PHASE_START: " phase", diff --git a/agent_sys/docs/TODO.md b/agent_sys/docs/TODO.md index 8e7329d28..61b18ee4d 100644 --- a/agent_sys/docs/TODO.md +++ b/agent_sys/docs/TODO.md @@ -37,11 +37,78 @@ unclaimed. **Not blocked on anyone — nobody has them.** | # | Item | Why it is not already fixed | |---|---|---| | 4b | **A typo'd `kind` in `Task.kinds` is caught by nothing at runtime** | `_participates` turns it into a no-op, and §4.16's narrowing removed the last place it would have raised. Probably `closure` check 6, at load time. **Reported twice by `env_mgr`, still unowned** | -| 4c | **`examples/demo/logic/store.py`'s F-D5 fallback cannot work under confinement** | It reads `AGENT_SYS_DEMO_STORE` and walks to a manifest; `env_mgr`'s `p11` measured **EACCES on the store root from a confined body**. `demo`'s file, with a second defect in it that is `handoff`'s. **The declared route is `materials.json`** — reaching a non-target artefact means *declaring* it (`inputs: ['summary', 'facts']`, permitted by spec §4.1's many-to-many binding), which also makes the phase record a verdict against the second artefact. **A design question for `demo` and `validator`, not a patch** | +| 4c | **P0 — a validator cannot reach the artefact its target was produced *from*, so five task packages scan the store instead** — *and the route this row previously proposed does not exist* | Was scoped to `examples/demo/logic/store.py`; the file is now `examples/demo/assets/lib/store.py` and **five copies** of it (`demo`, `demo2`, and three under `examples/llm_e2e_performance_optimization/`). See below | | 4d | **`test_a_gate_failure_does_not_deadlock_the_next_dispatch` fails 2 runs in 4** | Green alone and green in its own file. **No cause offered** — and the day's rule applies: a red suite in a shared worktree is not evidence about anyone's change. With `agent-mod-2`. **2026-08-29, end of day: 4 full-suite runs, 4 green** (`1905 passed, 3 skipped, 4 xfailed`, ~64 s each). **Not "fixed" — the worktree was quiet, so the trigger may simply have been absent**, which is the converse of the rule above and the same instrument problem. Running it alone proves nothing and was already known not to; recorded so the next person starts from four data points rather than repeating the isolated run | | 4e | **A hole in the store has no reaper** | §4.14 makes holes permanent and never renumbered by design. Whether they should ever be collected is undecided, not deferred | | 4f | **`check_grounded` has never been observed catching anything** — *ruled parked 2026-08-29, deliberately not worked* | Criterion 10 aims to show a validator catching an ungrounded number; three end-to-end runs showed a good model **declining to fabricate one** instead, so the validator's **failing** direction — what its `strong` claim is about — has never executed. **The user's ruling: not a framework question and not a principle question, this is `check_grounded`'s own business semantics, and it is not worth the time.** The shape they suggested if anyone ever picks it up: **split it in two** — one validator over the other fields, and a second that judges only whether the agent's answer about the missing duration is *reasonable*, passing if it is. **Two measurements bear on any such build:** `check_grounded` matches `\d+`, *"digits, not a parser"*, so `256` reads as grounded via `sha256_prefix` — the grounding set is **wider than what the facts assert**, and a fabricated number landing inside any digit run in the copied facts passes anyway. And `logic/check_grounded/readme.md` named the `UNEXPECTED_SUCCESS`/exit-3 outcome in advance, so **exit 3 is the artefact working, not a fault to repair** | +| 4g | **The backend's `claude` child processes do not exit when their task completes** — *first report, 2026-09-04, measured not inferred* | Seen while watching an `examples/demo2` run for an unrelated reason. Nine `claude` CLI processes alive at once, one per agent task, **elapsed 6 to 26 minutes and holding 5–11 seconds of CPU each, all sleeping (`S`)**. The oldest was `directions`, which the run log showed completing 26 minutes earlier; `ps -o pid=,stat=,etime=,time=` is the whole measurement. So they are not working and not being reaped — they accumulate for the life of a run, one per task. Harmless on `demo2`; a package with many tasks, or a long-lived supervisor, is where it stops being harmless. **Whose it is, is the open part**: it could be `claude-agent-sdk` not closing its transport, or `agent/backends/claude_sdk.py` not disposing the client after the result arrives. Nothing narrows it yet, and nobody has claimed it. What would close it: run one AI task, capture the child pid, and watch whether it exits when the SDK returns — if it does, the leak is in how the runner holds the client, not in the CLI | + +### 4c in full — why the store scan exists, and why declaring the input would not remove it + +**This row said the declared route was `materials.json`, and that reaching a +non-target artefact was a matter of *declaring* it — `inputs: ['summary', +'facts']`. Read against the code, that fix does not work.** A validator's +`inputs` is a **filter over the task's slots on this phase's side, not a +request**. `validator/phase.py:731`: + +```python +return list(task.inputs if kind is PhaseKind.INPUT else task.outputs) +``` + +and `phase.py:657` selects from exactly that: `mine = [t for t in targets if +self._kind_of(t, registry) in spec.inputs]`. `env_mgr/prepare.py:691-695` stages +the same set. So a kind the task does not hold **on that side** is not a target, +is not staged, and cannot be declared into existence. + +**The concrete case.** `check_problems` must verify that the problem set cites a +direction that exists — i.e. that the artefact is faithful to what its producer +consumed. `directions` is on the producing task's **input** side; the validator +runs on that task's **output** phase (and again on the students' input phases, +where the candidate set is `[problems]` too). The two sides never meet in any +phase, so there is no phase in which `directions` is reachable. Its declaration +is `inputs: [problems]` (`examples/demo2/steps/problems.yaml:48`) while its body +reads `directions` — so the schema's own promise for that field, *"DECLARED +rather than discovered, so a reviewer can answer 'what does this actually read' +without running it"*, is already false here. Same shape in +`demo/check_grounded`. + +**What the packages do instead.** `lib/store.py` reads `handoff`'s on-disk +layout through `AGENT_SYS_DEMO_STORE` (`cli/main.py:825`) and scans for *the +newest artefact of that kind anywhere in the store*. Its own docstring calls +that crude and wrong in a graph with more than one producer; it happens to be +right in these packages because there is exactly one. The ~30 +`staged_content(hid) or content_dir(hid)` sites are a different thing and not +this problem — each is commented as the fallback for a run with **no `env_mgr` +wired**, i.e. a validator run standalone. + +**Why it is P0 and why it is quiet.** The scan is only alive because two things +are switched off: `prepare_validation` *"does not confine anything"* +(`env_mgr/prepare.py:686`, and `EnvManager.prepare_validation` at `:751` records +that who confines a validation body *"is a third question that this ruling did +not settle"*), and `AGENT_SYS_NO_PERMISSIONS` defaults to on +(`prepare.py:80-95`). Either one landing kills the route — `env_mgr`'s `p11` +measured **EACCES on the store root from a confined body**, and `store_root()` +is `os.environ[...]` rather than `.get`, so the body dies *before* +`write_verdict` and `PhaseRunner` gets **no `verdict.json` at all** rather than +a `False`. So confining validations — ROADMAP §6.1's P0 — silently converts a +grounding check into a missing file. **Not measured:** whether a confined +*validation* body fails the same way an agent body does; that needs a policy +applied to one validation zone and a run. + +**The fix that removes the knowledge rather than moving it.** Give the output +phase read access to the producer's inputs — stage `task.inputs` read-only +alongside `task.outputs` in `prepare_validation`, and let `inputs:` select from +the union. Then the declaration becomes true, `declared_dir` is the only route a +body needs, and `versions` / `content_dir` / `kind_of` / `latest_of_kind` / +`handoff_dir` delete from all five copies. **A design question for `validator` +and `env_mgr` jointly, not a patch** — it widens what an output validation may +see, which is a criterion-13 (anti-gaming) question and must be argued there +before it is built. + +Raised again 2026-09-04 while labelling runtime directories (PR #156), which is +what made the five duplicated readers visible in one diff. + ## To build in the alpha | # | Item | Note | diff --git a/agent_sys/env_mgr/README.md b/agent_sys/env_mgr/README.md index e4867ab75..af8a8ad5e 100644 --- a/agent_sys/env_mgr/README.md +++ b/agent_sys/env_mgr/README.md @@ -798,6 +798,12 @@ name→path table; it expands nothing.** So no dependency is added, and the reason is recorded rather than assumed. +## O11y + +`agent_sys` starts an [AgentsView](https://github.com/kenn-io/agentsview) panel +at `http://127.0.0.1:18888` over the session transcripts a run produces, resident +across runs and never able to fail one. See `o11y/README.md`. + ## Deviations from the spec, carried from the design `docs/design.md` §16.1 reports seven; the three that change what a test asserts: diff --git a/agent_sys/env_mgr/docs/design.md b/agent_sys/env_mgr/docs/design.md index f0d9da803..4a2615645 100644 --- a/agent_sys/env_mgr/docs/design.md +++ b/agent_sys/env_mgr/docs/design.md @@ -1497,3 +1497,14 @@ The spec set is agreed and a design does not amend it. Each of these is reported | **O5** | **Whether executors nest as processes decides whether task depth is capped at 16.** §8.4 chooses supervisor-spawned executors, which avoids the cap. Nothing outside this document records that the choice has that consequence, and `task_graph` treats depth as unbounded | | **O6** | **§9.3 detects a conflict and refuses; it does not resolve one.** Refusing is right for a one-shot sync at task start. It is not right for whatever eventually wants to sync mid-task, and that caller does not exist yet | | **O7** | **Remote execution is less isolated than local, and now says so.** §10.4 reports it per side rather than resolving it. The moment a validation runs remotely, criterion 13 stops being enforced by anything this document specifies | + +## 17. o11y + +Side-cars that watch a run. **The rule that outranks every feature here: o11y +may never fail the thing it observes.** Every failure is one `log.warning` and a +skip, and there is a test per mode holding that line. + +One component today. Its design is **`../o11y/agentsview/design.md`** — the +panel, the five gates that keep it to `agent_sys`'s own sessions, one project +per run, and the measurements each of those rests on. It lives beside the code +rather than here because it is a component's design, not the module's. diff --git a/agent_sys/env_mgr/fs/layout.py b/agent_sys/env_mgr/fs/layout.py index bb56e2e0f..3d3b541ac 100644 --- a/agent_sys/env_mgr/fs/layout.py +++ b/agent_sys/env_mgr/fs/layout.py @@ -2,11 +2,15 @@ # Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. """The nested layout, and where a validation goes. Design §8. -Spec §5.1, unchanged:: +Spec §5.1, plus a label:: - /task.../ + /task..../ ├── handoffs/ ├── workspace/ ├── playground/ ├── logs/ - └── task.../ ← a subtask, nested + └── task..../ ← a subtask, nested + +The ```` field is for whoever is reading the tree and nothing resolves +through it — see `env_mgr.fs.zone.zone_dirname`. A zone written before it +existed still resolves, because `find_zone_dir` matches the uuid as a field. The nesting is what makes containment answer both *"is this path in the zone"* and *"may this task reach that path"*, because permissions cover the task's own @@ -30,6 +34,7 @@ "copy_out", "create", "find_zone_dir", + "handoff_dir", "handoff_version_dir", "stage", "stage_handoffs", @@ -58,15 +63,21 @@ def _subdirs(domains: DomainRegistry) -> tuple[str, ...]: def find_zone_dir(base: str, task_id: Any) -> str | None: """The directory of `task_id`'s most recent attempt, anywhere under `base`. - A task's uuid is unique, so the ``task..`` prefix identifies it - wherever the tree happens to have put it — which is what lets a subtask be - placed under its parent without the parent's `Task` object being in hand. + A task's uuid is unique, so the ``task.`` prefix plus the uuid as a whole + field identifies it wherever the tree happens to have put it — which is what + lets a subtask be placed under its parent without the parent's `Task` object + being in hand. + + **The uuid is matched as a field, not as a prefix**, because `zone_dirname` + now carries an optional label between the prefix and the uuid. Both shapes + match, so a run resumed against zones written before the label existed still + finds them. """ - prefix = f"{_ZONE_PREFIX}{task_id}." + field = f".{task_id}." best: tuple[int, str] | None = None for dirpath, dirnames, _ in os.walk(base): for name in dirnames: - if not name.startswith(prefix): + if not name.startswith(_ZONE_PREFIX) or field not in name: continue parts = name.split(".") try: @@ -94,7 +105,8 @@ def create(task: Any, execution: Any, domains: DomainRegistry) -> Zone: f"task {task.id} declares parent {parent_id}, which has no zone under {base}" ) base = parent_dir - root = os.path.join(base, zone_dirname(task.id, execution.attempt)) + name = getattr(task, "closure", None) + root = os.path.join(base, zone_dirname(task.id, execution.attempt, name)) for sub in _subdirs(domains): # exist_ok: a resume finds its own playground and keeps the contents. os.makedirs(os.path.join(root, sub), exist_ok=True) @@ -115,7 +127,8 @@ def validation_zone(task: Any, phase: str, domains: DomainRegistry) -> str: base = domains.storage_root() zone_dir = find_zone_dir(base, task.id) parent = os.path.dirname(zone_dir) if zone_dir else base - root = os.path.join(parent, validation_dirname(task.id, phase)) + name = getattr(task, "closure", None) + root = os.path.join(parent, validation_dirname(task.id, phase, name)) os.makedirs(root, exist_ok=True) resolved = resolve_strict(root) if resolved is None: # pragma: no cover - we just created it @@ -139,13 +152,46 @@ def validation_zone(task: Any, phase: str, domains: DomainRegistry) -> str: CONTENT_DIR = "content" +#: `handoff.store.HANDOFF_PREFIX`, spelled again for the same reason +#: `CONTENT_DIR` above is: `env_mgr` may not import `handoff`, and +#: `tests/interfaces/test_handoff_layout.py` is the price that keeps the two +#: spellings honest. +HANDOFF_PREFIX = "handoff" + + +def handoff_dir(store_root: str, handoff_id: Any) -> str: + """``/handoff../`` — `handoff` design §6.2's layout. + + **Resolved by scanning, not composed**, and that is not a style choice: the + label in the middle is the handoff's *kind*, which lives on `task_graph`'s + `Handoff.type` and is not a fact this module has. It does not need it — the + store allocates the directory at dispatch (`task_graph/scheduler.py:470`), + before anything here resolves a grant, so by the time this runs the + directory is on disk and the uuid identifies it. + + A directory is this handoff's when its name *is* the uuid — the shape + written before labels existed, so an older store still resolves — or ends + with ``.``. + """ + wanted = str(handoff_id) + suffix = f".{wanted}" + try: + names = sorted(os.listdir(store_root)) + except OSError: + names = [] + for name in names: + if name == wanted or name.endswith(suffix): + return os.path.join(store_root, name) + return os.path.join(store_root, f"{HANDOFF_PREFIX}.{wanted}") + + def handoff_version_dir(store_root: str, handoff_id: Any, version: int) -> str: - """``//v/`` — `handoff` design §6.2's layout. + """``/handoff../v/`` — `handoff` design §6.2's layout. This module grants access to that directory and computes nothing about its contents (design §1.2). """ - return os.path.join(store_root, str(handoff_id), f"v{version}") + return os.path.join(handoff_dir(store_root, handoff_id), f"v{version}") def stage( @@ -241,11 +287,18 @@ def stage( version = versions.get(hid) if version is None: continue - version_dir = handoff_version_dir(store_root, hid, version) + handoff_root = handoff_dir(store_root, hid) + version_dir = os.path.join(handoff_root, f"v{version}") src = os.path.join(version_dir, CONTENT_DIR) if narrow else version_dir if not os.path.isdir(src): continue - dst = os.path.join(into, str(hid), f"v{version}") + # The staged copy takes the **store directory's own name**, so the + # kind label rides along and a body's `materials/` reads like the store. + # Derived rather than passed: this function is handed slots and versions + # and not kinds, and threading a kinds map through `stage_handoffs` and + # `prepare_validation` would put `task_graph`'s vocabulary in two more + # signatures to compute a string that is already on disk. + dst = os.path.join(into, os.path.basename(handoff_root), f"v{version}") copy_out(src, dst) staged[hid] = dst return staged diff --git a/agent_sys/env_mgr/fs/zone.py b/agent_sys/env_mgr/fs/zone.py index bb0aad957..8abc03007 100644 --- a/agent_sys/env_mgr/fs/zone.py +++ b/agent_sys/env_mgr/fs/zone.py @@ -9,38 +9,75 @@ from env_mgr.fs.path import contained -__all__ = ["Zone", "zone_dirname", "validation_dirname"] +__all__ = ["Zone", "slug", "zone_dirname", "validation_dirname"] #: Readability and accident-avoidance only. Spec §4.1 settled that an #: unguessable prefix is security-by-obscurity — it was recovered three ways by #: the agent that holds it — so this buys no confidence and is not asked to. _HASH_CHARS = 8 +#: Long enough to recognise a closure name, short enough that a zone path still +#: fits in a terminal beside a full uuid. A truncated slug is a label, never a +#: key — nothing resolves a directory through it. +_SLUG_CHARS = 40 + def _tag(*parts: str) -> str: return hashlib.sha256("\x00".join(parts).encode()).hexdigest()[:_HASH_CHARS] -def zone_dirname(task_id: Any, attempt: int) -> str: - """``task...`` — spec §5.1. +def slug(text: Any) -> str: + """A directory-name-safe label, or ``""`` when there is nothing to say. + + **``.`` must not survive**, and that is the whole reason this exists rather + than the name being interpolated raw: ``.`` is this module's field + separator, so a closure called ``a.b`` would silently add a field and + `find_zone_dir`'s ``parts[-2]`` would read the wrong one. + """ + if text is None: + return "" + out: list[str] = [] + for char in str(text): + keep = char if (char.isascii() and (char.isalnum() or char in "_-")) else "-" + if keep == "-" and (not out or out[-1] == "-"): + continue + out.append(keep) + return "".join(out).strip("-")[:_SLUG_CHARS].strip("-") + + +def zone_dirname(task_id: Any, attempt: int, name: Any = None) -> str: + """``task....`` — spec §5.1, plus a label. The zone id is the runtime ``uuid.version``: the task's own identity, not a separate namespace. ``version`` is the attempt, because a zone belongs to an attempt (design §11.3). + + `name` is the task's closure — a label for whoever is reading the tree, and + nothing else. It is **not** in `_tag`'s input and nothing resolves through + it, so an unnamed task still gets today's exact name and a renamed closure + does not move an existing zone. It sits *before* the uuid because that is + what makes ``ls`` and tab-completion useful; the uuid stays whole and stays + the field before the attempt, which is what every lookup keys on. """ uid = str(task_id) - return f"task.{uid}.{attempt}.{_tag(uid, str(attempt))}" + label = slug(name) + prefix = f"task.{label}." if label else "task." + return f"{prefix}{uid}.{attempt}.{_tag(uid, str(attempt))}" -def validation_dirname(task_id: Any, phase: str) -> str: - """``validation...`` — design §8.3, deviation D5. +def validation_dirname(task_id: Any, phase: str, name: Any = None) -> str: + """``validation....`` — design §8.3, deviation D5. A **sibling** of the producing task's zone, never a descendant of it. Anything under the producing task's directory is inside its subtree and therefore reachable, which is exactly what criterion 13 forbids. + + `name` is a label on the same terms as `zone_dirname`'s. """ uid = str(task_id) - return f"validation.{uid}.{phase}.{_tag(uid, phase)}" + label = slug(name) + prefix = f"validation.{label}." if label else "validation." + return f"{prefix}{uid}.{phase}.{_tag(uid, phase)}" class Zone(NamedTuple): diff --git a/agent_sys/env_mgr/material.py b/agent_sys/env_mgr/material.py index a72a1b003..7f1eabf90 100644 --- a/agent_sys/env_mgr/material.py +++ b/agent_sys/env_mgr/material.py @@ -13,15 +13,20 @@ from __future__ import annotations +import logging import os +from pathlib import Path from typing import Any from env_mgr import harness from env_mgr.fs.layout import copy_out from env_mgr.fs.zone import Zone +from env_mgr.prefix import Prefix from env_mgr.protocols import PrepareRefused -__all__ = ["CONFIG_DIR", "MATERIAL_KEYS", "deploy"] +__all__ = ["CONFIG_DIR", "MATERIAL_KEYS", "PROJECTS_DIR", "deploy"] + +log = logging.getLogger("env_mgr.material") #: Placed under a per-attempt config directory rather than ``$HOME``. Measured: #: with ``~/.claude`` granted, a demo agent read the **operator's** personal @@ -30,6 +35,10 @@ #: ``CLAUDE_CONFIG_DIR`` at the zone removes the ``$HOME`` grant entirely. CONFIG_DIR = "config" +#: Claude Code's own name for *where the transcripts go*: one subdirectory per +#: working directory, named by slugifying that path, one JSONL per session. +PROJECTS_DIR = "projects" + #: The three `agent` hands over, in Claude Code's own directory names. MATERIAL_KEYS = ("rules", "hooks", "skills") @@ -50,6 +59,7 @@ def deploy(agent_spec: Any, zone: Zone) -> dict[str, str]: """ config = os.path.join(zone.root, CONFIG_DIR) os.makedirs(config, exist_ok=True) + _share_projects(config) # A temp directory inside the zone: per attempt, and it dies with the zone. # The backend refuses a temp directory it cannot read, and says so well. tmp = os.path.join(zone.root, "tmp") @@ -97,6 +107,41 @@ def deploy(agent_spec: Any, zone: Zone) -> dict[str, str]: return env +def _share_projects(config: str) -> None: + """Point this attempt's ``config/projects`` at the o11y prefix's. + + **Everything else in ``config/`` stays per-attempt.** Only ``projects/`` is + shared: it is Claude Code's *output*, nobody in the zone reads it, and one + physical directory cannot collide because each subdirectory is named after + the slugified cwd. Measured on demo2: nine transcripts in nine zones, and + the panel showed none. **Never raises** — a degraded panel beats a dead run. + """ + link = Path(config) / PROJECTS_DIR + try: + target = Prefix.resolve(os.environ).claude_home / PROJECTS_DIR + target.mkdir(parents=True, exist_ok=True) + if link.is_symlink(): + if link.resolve() == target.resolve(): + return # idempotent: re-running finds its own work and stops + # One named path, and one we placed ourselves: a symlink at exactly + # `/config/projects`. Never a tree, never a variable target. + link.unlink() + elif link.exists(): + # A real directory. `rmdir` **refuses** a non-empty one, which is + # why it is the call used: an empty directory is ours to replace, + # one holding transcripts raises and falls to the warning below. + # Losing a zone from the panel is cheaper than deleting evidence. + os.rmdir(link) + link.symlink_to(target, target_is_directory=True) + except OSError as exc: + log.warning( + "could not share %s with the o11y prefix (%s); this attempt's " + "transcripts stay in its zone and the panel will not show them", + link, + exc, + ) + + def _paths(agent_spec: Any, key: str) -> tuple[str, ...]: value = _get(agent_spec, key) if not value: diff --git a/agent_sys/env_mgr/o11y/__init__.py b/agent_sys/env_mgr/o11y/__init__.py new file mode 100644 index 000000000..e0cc137f4 --- /dev/null +++ b/agent_sys/env_mgr/o11y/__init__.py @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""o11y side-cars: things that watch a run and may never fail one. + +The prefix they install into is `env_mgr.prefix`, not here: it is an `env_mgr` +layout that o11y happens to be the first consumer of. +""" diff --git a/agent_sys/env_mgr/o11y/agentsview/README.md b/agent_sys/env_mgr/o11y/agentsview/README.md new file mode 100644 index 000000000..3b3a8be8f --- /dev/null +++ b/agent_sys/env_mgr/o11y/agentsview/README.md @@ -0,0 +1,56 @@ +# The o11y panel: AgentsView + +`agent_sys` ships an observability panel — [AgentsView](https://github.com/kenn-io/agentsview), +an external Go binary — over the session transcripts a run produces, at +**`http://127.0.0.1:18888`**. Deploying `agent_sys` starts it; it stays resident +across runs and binds loopback only. **AgentsView's own code is never modified.** + +Design, and the measurement behind every claim here: **`design.md`**, beside this +file. + +## Using it + +| | | +|---|---| +| `--agentsview-port N` | the port. Then `AGENTSVIEW_PORT`, then `18888` | +| `--no-agentsview` | do not start it. Not one external call is made | + +`--dry-run` and `--clean` are exempt: a dry run that leaves a resident daemon +behind has broken its only promise. + +You will see two lines on the way past: + +``` + o11y panel at http://127.0.0.1:18888 + o11y this run is project 'run.20260904T121032_b65238' on the panel +``` + +## The four things an operator should be told plainly + +**Your own `~/.claude` is never read, written, or reconfigured.** Measured +against the live daemon: of the 122 session roots AgentsView would scan, 122 are +inside `~/.infera_agent_sys` and 0 outside. + +**Each run is one project.** AgentsView names a project after the session's +deepest path segment, and every attempt runs in its own nested zone, so a run +would otherwise arrive as a dozen unrelated entries. The label lives in a +mapping row, not on the session — so deleting the row un-names that run at the +next full re-sync. There is no automatic cleanup, on purpose. + +**A pre-existing AgentsView of yours is never adopted.** Reuse needs two gates: +it answers `/api/v1/agents` with JSON, **and** a live `daemon..json` in our +own data directory names that port. If you expected reuse and got "port in use", +that is why. + +**The panel cannot fail your run.** Binary missing, port taken, daemon wedged, +health timed out, mapping refused: each is one warning and a skip. Install +failure too, through the recipe's `importance: suggested`. + +## Where things are + +| | | +|---|---| +| `agentsview.py` | the daemon — install, port, launch, ownership, health | +| `mapping.py` | what the panel shows — one project per run | +| `../../recipes/agentsview.o11y.yaml` | the pinned release and its checksum | +| `../../prefix.py` | `~/.infera_agent_sys`, an `env_mgr` layout this is the first consumer of | diff --git a/agent_sys/env_mgr/o11y/agentsview/__init__.py b/agent_sys/env_mgr/o11y/agentsview/__init__.py new file mode 100644 index 000000000..f1f730935 --- /dev/null +++ b/agent_sys/env_mgr/o11y/agentsview/__init__.py @@ -0,0 +1,45 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""AgentsView as `agent_sys`'s o11y panel. Design: `design.md` beside this file. + +Two halves. `agentsview.py` owns the daemon — install, port, launch, ownership, +health. `mapping.py` owns what the panel *shows* — one project per run. + +Re-exported here so a caller writes `from env_mgr.o11y.agentsview import +ensure_running`, unchanged from when this package was a single module. +""" + +from .agentsview import ( + DEFAULT_PORT, + OTHER_PROVIDERS, + RECIPE_PATH, + Status, + check_disabled_agents, + discover_providers, + ensure_installed, + ensure_running, + freshly_installed, + pinned_version, + port_is_free, + resolve_port, + write_config, +) +from .mapping import ensure_run_project, name_for_run + +__all__ = [ + "DEFAULT_PORT", + "OTHER_PROVIDERS", + "RECIPE_PATH", + "Status", + "check_disabled_agents", + "discover_providers", + "ensure_installed", + "ensure_run_project", + "ensure_running", + "freshly_installed", + "name_for_run", + "pinned_version", + "port_is_free", + "resolve_port", + "write_config", +] diff --git a/agent_sys/env_mgr/o11y/agentsview/agentsview.py b/agent_sys/env_mgr/o11y/agentsview/agentsview.py new file mode 100644 index 000000000..344917164 --- /dev/null +++ b/agent_sys/env_mgr/o11y/agentsview/agentsview.py @@ -0,0 +1,618 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""AgentsView, started as a side-car and never allowed to fail a run. + +**The one rule this module exists to enforce:** an observability panel that can +break the thing it observes is worse than no panel. Every function here returns +a `Status` and raises nothing. There is a test per failure mode holding that +line, because the failure mode of a warning-only component is that someone +later "improves" it into a raise. + +**Why we decide the port instead of letting AgentsView decide.** `agentsview +serve` auto-discovers a free port when the requested one is busy — sensible for +a human at a terminal, wrong here: the mission asks for a warning and a skip on +a taken port, and a daemon that quietly moved to 18889 is a panel nobody knows +the address of. So the bind probe happens here, before launch. + +Rationale, measurements and rejected alternatives: `design.md`, beside this file. + +""" + +from __future__ import annotations + +import contextlib +import http.client +import json +import logging +import os +import re +import socket +import subprocess +import time +import urllib.error +import urllib.request +from collections.abc import Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: # pragma: no cover - typing only + from ...prefix import Prefix + +__all__ = [ + "DEFAULT_PORT", + "RECIPE_PATH", + "Status", + "check_disabled_agents", + "discover_providers", + "ensure_installed", + "ensure_running", + "freshly_installed", + "pinned_version", + "port_is_free", + "resolve_port", + "write_config", +] + +log = logging.getLogger("env_mgr.o11y.agentsview") + +#: The mission's number. +DEFAULT_PORT = 18888 + +PORT_ENV_VAR = "AGENTSVIEW_PORT" + + +@dataclass(frozen=True) +class Status: + """What happened. `url` is set only when `running` is true.""" + + running: bool + reason: str + url: str | None = None + + +#: A port we may ask for. **`0` is excluded deliberately**: it binds, then means +#: "any free port", handing the choice to AgentsView's auto-discovery. +_LOWEST_PORT = 1 +_HIGHEST_PORT = 65535 + + +def _in_range(port: int) -> bool: + return _LOWEST_PORT <= port <= _HIGHEST_PORT + + +def resolve_port(flag: int | None, environ: Mapping[str, str]) -> int: + """Flag, then environment, then 18888. + + **An unusable value is the default, not an error** — refusing a deployment + over a typo in a variable nobody needed inverts this module's priority. + Range matters as well as parseability: `bind` answers an out-of-range port + with `OverflowError`, which is not `OSError` and nothing downstream catches. + """ + if flag is not None: + if _in_range(flag): + return int(flag) + log.warning( + "agentsview: --agentsview-port %d is not in %d-%d; using the default %d", + flag, + _LOWEST_PORT, + _HIGHEST_PORT, + DEFAULT_PORT, + ) + return DEFAULT_PORT + raw = environ.get(PORT_ENV_VAR) + if raw is None: + return DEFAULT_PORT + try: + parsed = int(raw) + except ValueError: + parsed = None + if parsed is not None and _in_range(parsed): + return parsed + log.warning( + "%s=%r is not a usable port number; using the default %d", + PORT_ENV_VAR, + raw, + DEFAULT_PORT, + ) + return DEFAULT_PORT + + +def port_is_free(port: int, host: str = "127.0.0.1") -> bool: + """A real bind, not a connect. + + Connecting answers "is someone accepting", a different question: a socket + bound and not listening still makes our own bind fail. `OverflowError` + alongside `OSError` because that is what `bind` raises out of range. + """ + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((host, port)) + except (OSError, OverflowError): + return False + return True + + +#: How long the launch subprocess itself may take. `serve --background` +#: daemonises and returns at once, so anything slower is a hung binary. +LAUNCH_TIMEOUT_S = 20.0 + +#: How long we then wait for the daemon to answer. Cold-start reads the whole +#: session archive, so this is generous. +HEALTH_TIMEOUT_S = 30.0 + +#: How long we spend deciding whether something on the port is *our* panel. +#: Short deliberately: this is on every deployment's path, and the answer either +#: arrives at once or the thing there would not have served us anyway. Named so +#: a test can shrink it. +REUSE_PROBE_TIMEOUT_S = 2.0 + +#: AgentsView's own JSON endpoint (`docs/session-api.md:112`), used as the +#: identity probe. Not `/`: every web server answers that with a 200. +IDENTITY_PATH = "/api/v1/agents" + +#: Cap on the identity response, so a stranger streaming without end cannot +#: hang a deployment on the o11y probe. +IDENTITY_MAX_BYTES = 1 << 20 + +#: One identity request's deadline, as distinct from `REUSE_PROBE_TIMEOUT_S` +#: which bounds the series of them. Equal by coincidence, unrelated. +IDENTITY_PROBE_TIMEOUT_S = 2.0 + +#: **AgentsView's own artefact, not ours.** `serve` writes one per running +#: daemon into its `AGENTSVIEW_DATA_DIR` and removes it on a clean stop +#: (measured, v0.42.0). That directory is the prefix's and nobody else's, so a +#: record here was written by a daemon we configured. Read only. +DAEMON_RECORD_GLOB = "daemon.*.json" + +#: Checked so an unrelated file matching the glob is not read as a daemon. +SERVICE_NAME = "agentsview" + + +def _binary_env(prefix: Prefix) -> dict[str, str]: + """The environment every `agentsview` subprocess gets. One definition. + + **`HOME` is gate 5**: AgentsView derives every provider's default root from + it, so this scopes providers we have never heard of, with no list to go + stale. A replacement, not an overlay — unverified whether any AgentsView + path wants a `TMPDIR` or `LANG` this drops. + """ + return {**prefix.environment(), "PATH": str(prefix.bin), "HOME": str(prefix.root)} + + +#: **The pinned, reviewable statement of intent** — every provider AgentsView +#: can scan, minus Claude Code. Pinned rather than derived at runtime so an +#: upstream change alters the panel through a diff and a review, not silently. +#: Measured against `v0.42.0 doctor sync`'s "Agent roots:"; `check_disabled_agents` +#: re-runs that at install time and warns on drift in either direction. +OTHER_PROVIDERS = ( + "aider", "amp", "antigravity", "antigravity-cli", "codebuff", "codex", + "commandcode", "copilot", "cortex", "cowork", "cursor", "cursor-ide", + "deepseek-harness", "deepseek-tui", "devin", "forge", "gemini", "goose", + "gptme", "grok", "hermes", "icodemate", "iflow", "kilo", "kilo-legacy", + "kimi", "kimi-work", "kiro", "kiro-ide", "mimocode", "omnigent", "omp", + "openclaude", "openclaw", "opencode", "openhands", "pi", "piebald", + "poolside", "posit-assistant", "positron", "prime-agent", "qclaw", + "qoder", "qwen", "qwenpaw", "reasonix", "roocode", "shelley", "trae", + "traex", "vibe", "visualstudio-copilot", "vscode-copilot", "warp", + "windsurf", "workbuddy", "zcode", "zed", "zencoder", +) + +#: The one `OTHER_PROVIDERS` must never contain: gate 3 disables everything +#: except the source `agent_sys` itself writes. +_KEEP_ENABLED = "claude" + +#: `doctor sync`'s report lists every provider the binary recognizes under an +#: "Agent roots:" header, one two-space-indented `name: path (status)` line per +#: root. Measured against a real v0.42.0, not re-derived from the docs' table. +_AGENT_ROOTS_HEADER = "Agent roots:" +_AGENT_ROOT_LINE_RE = re.compile(r"^ ([a-z0-9_-]+):", re.MULTILINE) + +#: The end of that section: the first line that is not indented. +_AGENT_ROOTS_END_RE = re.compile(r"^(?=\S)", re.MULTILINE) + +#: `doctor sync` only stats candidate directories and reads sync metadata. +DISCOVER_PROVIDERS_TIMEOUT_S = 10.0 + + +def _parse_agent_roots(stdout: str) -> tuple[str, ...] | None: + """The "Agent roots:" section of a `doctor sync` report -> sorted names. + + One parser for both consumers, so they cannot drift apart. `None` rather + than `()` when the section is missing or names none — see + `discover_providers` for why the caller needs that distinction. + """ + start = stdout.find(_AGENT_ROOTS_HEADER) + if start == -1: + return None + # Bounded at the next unindented line: scanning past it would read any + # later ` something:` as a phantom provider and warn about drift. + section = _AGENT_ROOTS_END_RE.split( + stdout[start + len(_AGENT_ROOTS_HEADER) :], maxsplit=1 + )[0] + names = {m.group(1) for m in _AGENT_ROOT_LINE_RE.finditer(section)} + names.discard(_KEEP_ENABLED) + return tuple(sorted(names)) if names else None + + +def discover_providers(prefix: Prefix) -> tuple[str, ...] | None: + """Ask the installed binary which session providers it recognizes, now. + + **`doctor sync`, never `health`** — measured: `health`, `projects` and + `session list` each autostart a daemon on a port AgentsView picks. + **`None`, never `()`, on anything untrustworthy**: an empty tuple would read + as "no providers exist", which a probe failure must not assert. To + re-measure `OTHER_PROVIDERS` after an upgrade: + + AGENTSVIEW_DATA_DIR= CLAUDE_PROJECTS_DIR= \\ + agentsview doctor sync | sed -n '/^Agent roots:/,/^Recent/p' \\ + | sed -E 's/^\\s+([a-z0-9_-]+):.*/\\1/' | sort -u + """ + exe = prefix.bin / "agentsview" + try: + proc = subprocess.run( # noqa: S603 + [str(exe), "doctor", "sync"], + env=_binary_env(prefix), + capture_output=True, + text=True, + timeout=DISCOVER_PROVIDERS_TIMEOUT_S, + ) + except (OSError, subprocess.SubprocessError): + return None + if proc.returncode != 0: + return None + return _parse_agent_roots(proc.stdout or "") + + +def write_config(prefix: Prefix, disabled_agents: Sequence[str]) -> None: + """The prefix's `config.toml`. Idempotent, and ours alone. + + Written into `AGENTSVIEW_DATA_DIR`, never `~/.agentsview`, so a user who + already runs AgentsView keeps their archive. `daemon_idle_timeout = "0s"` + because the default 20m empties the panel for anyone opening the URL after + their run — measured, the daemon self-exited twice without it. + """ + cfg = prefix.agentsview_data / "config.toml" + disabled = ", ".join(json.dumps(name) for name in disabled_agents) + cfg.parent.mkdir(parents=True, exist_ok=True) + # Written whole, by rename: a reader catching a truncated `write_text` gets + # a short `disabled_agents`, i.e. every other provider silently re-enabled. + tmp = cfg.with_name(cfg.name + f".{os.getpid()}.tmp") + tmp.write_text( + "# Written by agent_sys. AgentsView itself is unmodified.\n" + f"disabled_agents = [{disabled}]\n" + 'host = "127.0.0.1"\n' + "disable_update_check = true\n" + 'daemon_idle_timeout = "0s"\n' + ) + os.replace(tmp, cfg) + + +#: The substring AgentsView's config parser puts around the offending name +#: (measured: `disabled_agents: unknown session provider "claude-cowork"`). +_UNKNOWN_PROVIDER_RE = re.compile(r'unknown session provider "([^"]+)"') + +#: A cold sync of one `CLAUDE_PROJECTS_DIR`. Generous, not open-ended. +CHECK_DISABLED_AGENTS_TIMEOUT_S = 15.0 + + +def check_disabled_agents(prefix: Prefix) -> tuple[str, ...]: + """Has reality moved past the pinned `OTHER_PROVIDERS`? Checks both ways. + + **Rename or removal:** `doctor sync` exits non-zero naming the entry this + version dropped, and the panel will not start. **Addition, the one that + leaks:** its "Agent roots:" report is diffed against `OTHER_PROVIDERS`, + because a provider we forgot to disable loads with no error at all. Only + one can surface per call — a failing sync never prints the report. Empty + means clean *or* untrusted; a probe failure is evidence of neither. + """ + exe = prefix.bin / "agentsview" + try: + proc = subprocess.run( # noqa: S603 + [str(exe), "doctor", "sync"], + env=_binary_env(prefix), + capture_output=True, + text=True, + timeout=CHECK_DISABLED_AGENTS_TIMEOUT_S, + ) + except (OSError, subprocess.SubprocessError): + return () + + if proc.returncode != 0: + m = _UNKNOWN_PROVIDER_RE.search(proc.stderr or "") + return (m.group(1),) if m else () + + discovered = _parse_agent_roots(proc.stdout or "") + if discovered is None: + return () + return tuple(sorted(set(discovered) - set(OTHER_PROVIDERS))) + + +def _pid_is_alive(pid: int) -> bool: + """`ESRCH` is a no; `EPERM` is a yes — a process we may not signal exists.""" + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except OSError: + return True + return True + + +def _owns_port(prefix: Prefix, port: int) -> bool: + """Did *we* start what is on this port, and is it still alive? + + **A live AgentsView here is not evidence that it is ours**: a user's own + lists every session on the machine. The witness is AgentsView's + `daemon..json` in *our* data directory — a stranger's goes in theirs, + so the isolation is the filesystem's. Removed on a clean stop, so only an + unclean death leaves a stale one and the pid catches that. Anything + missing, unreadable or foreign is a "no". + """ + try: + records = sorted(prefix.agentsview_data.glob(DAEMON_RECORD_GLOB)) + except OSError: + return False + for path in records: + try: + record = json.loads(path.read_text()) + except (OSError, ValueError): + continue + if not isinstance(record, dict) or record.get("service") != SERVICE_NAME: + continue + if not _record_names_port(record, port): + continue + pid = record.get("pid") + if isinstance(pid, int) and _pid_is_alive(pid): + return True + return False + + +def _record_names_port(record: Mapping[str, Any], port: int) -> bool: + """`metadata.port` first, `address` as the fallback. + + Both are in a real v0.42.0 record. Two readings because this is an external + artefact: dropping either field leaves the gate working, dropping both + fails it closed. + """ + metadata = record.get("metadata") + if isinstance(metadata, Mapping) and str(metadata.get("port", "")) == str(port): + return True + address = record.get("address") + return isinstance(address, str) and address.rsplit(":", 1)[-1] == str(port) + + +def _identifies_as_agentsview(url: str) -> bool: + """One request. `200` **and** a JSON body, or it is not AgentsView. + + A status code is not an identity: any web server answers 200, and returning + `Status(True, …)` for one hands the operator a stranger's application + labelled as their panel. + """ + try: + with urllib.request.urlopen( # noqa: S310 + url + IDENTITY_PATH, timeout=IDENTITY_PROBE_TIMEOUT_S + ) as r: + if r.status != 200: + return False + json.loads(r.read(IDENTITY_MAX_BYTES).decode("utf-8", "replace")) + except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException): + # `HTTPException` because `IncompleteRead` — raised on a truncated + # chunked body, the framing a Go server uses with no Content-Length — + # descends from neither `OSError` nor `ValueError`. + return False + return True + + +def _wait_for_health(url: str, timeout: float) -> bool: + """Poll until AgentsView identifies itself, or the deadline passes. + + Always tries once, so a zero timeout still asks, and always sleeps between + attempts *including after a wrong answer* — otherwise a stranger returning + a prompt 200 turns this into a busy loop against someone else's service. + """ + deadline = time.monotonic() + timeout + while True: + if _identifies_as_agentsview(url): + return True + if time.monotonic() >= deadline: + return False + time.sleep(0.5) + + +def ensure_running(prefix: Prefix, port: int) -> Status: + """Start the panel, or say in one line why there is none. + + **Every return is a `Status` and every failure logs exactly one warning.** + One, not two: a caller shown the same problem twice hunts for two problems. + """ + url = f"http://127.0.0.1:{port}" + exe = prefix.bin / "agentsview" + + if not port_is_free(port): + # Two gates, neither enough alone. Ownership first: it is a file read + # rather than a round trip, and a `no` means we must not probe further. + # The two failures are separate warnings because their fixes are + # opposite — "something else has your port" sends an operator hunting + # for a process that does not exist when ours is merely wedged. + if _owns_port(prefix, port): + if _wait_for_health(url, timeout=REUSE_PROBE_TIMEOUT_S): + return Status(True, "already running", url) + log.warning( + "agentsview: our own daemon holds port %d but did not answer %s " + "within %.1fs; skipping the o11y panel. Stop it with " + "`AGENTSVIEW_DATA_DIR=%s agentsview serve stop`.", + port, + url, + REUSE_PROBE_TIMEOUT_S, + prefix.agentsview_data, + ) + return Status(False, f"our daemon on port {port} is not answering") + log.warning( + "agentsview: port %d is in use by something else; skipping the o11y " + "panel. Pass --agentsview-port to choose another.", + port, + ) + return Status(False, f"port {port} in use") + + if not exe.is_file(): + log.warning( + "agentsview: not installed at %s; skipping the o11y panel. " + "Run `env-mgr install env_mgr/recipes/agentsview.o11y.yaml`.", + exe, + ) + return Status(False, "not installed") + + # `--replace`: measured, without it `serve --background --port N` silently + # attaches to any daemon already alive for this data directory and reports + # *its* port, exit 0, `N` ignored. `port_is_free(port)` was true above, so + # anything still alive is necessarily not on the port we asked for and + # there is no legitimate case here where replacing it is wrong. + try: + prefix.create() + write_config(prefix, OTHER_PROVIDERS) + proc = subprocess.run( # noqa: S603 + [str(exe), "serve", "--background", "--no-browser", "--replace", + "--host", "127.0.0.1", "--port", str(port)], + env=_binary_env(prefix), + capture_output=True, + text=True, + timeout=LAUNCH_TIMEOUT_S, + ) + except (OSError, subprocess.SubprocessError) as e: + log.warning("agentsview: could not launch (%s); skipping the o11y panel.", e) + return Status(False, f"launch failed: {e}") + + if proc.returncode != 0: + log.warning( + "agentsview: `serve --background` exited %d; skipping the o11y panel. stderr: %s", + proc.returncode, + (proc.stderr or "").strip()[:400], + ) + return Status(False, f"exit {proc.returncode}") + + if not _wait_for_health(url, HEALTH_TIMEOUT_S): + log.warning( + "agentsview: started but did not answer %s within %.0fs; skipping the o11y panel.", + url, + HEALTH_TIMEOUT_S, + ) + return Status(False, "health check timed out") + + # Nothing is recorded here: AgentsView wrote its own `daemon..json` + # when it started and that is what `_owns_port` reads. A second record of + # ours would only be a thing that can disagree. + return Status(True, "started", url) + + +#: The recipe item `ensure_installed` drives. Fixed and in-package: there is one +#: recipe for this component, and a parameter would be a second way to say so. +RECIPE_PATH = Path(__file__).resolve().parent.parent.parent / "recipes" / "agentsview.o11y.yaml" + +#: `BinInstaller.install` says "installed " when it ran and " +#: already present (skip)" when it did not — the only signal `ensure_installed` +#: passes back about which. `test_the_installers_two_ok_messages_still_discriminate` +#: drives both real branches, so a rephrasing there fails rather than silently +#: muting the first-install notice or firing it every run. +_FRESHLY_INSTALLED_PREFIX = "installed " + + +def freshly_installed(reason: str) -> bool: + """Did `ensure_installed` just download it, or was it already there?""" + return reason.startswith(_FRESHLY_INSTALLED_PREFIX) + + +def pinned_version() -> str: + """The version `RECIPE_PATH` pins, for the first-install notice. + + **Reads the YAML rather than calling `recipe.load_recipe`.** That loader is + below the decoupling wall (spec §9) and nothing under `env_mgr/` may import + it — checked structurally by `tests/env_mgr/test_imports.py`. One field does + not justify an exemption, and `?` rather than a raise because a notice may + not fail the thing it narrates. + """ + try: + import yaml + + spec = yaml.safe_load(RECIPE_PATH.read_text()) + for item in spec.get("items", []): + if item.get("name") == "agentsview": + return str(item.get("version") or "?") + except Exception: # noqa: BLE001 — see the docstring + pass + return "?" + + + +@contextlib.contextmanager +def _patched_environ(extra: Mapping[str, str]) -> Iterator[None]: + """Patch `os.environ` for exactly the duration of the block. + + The recipe references `$AGENT_SYS_HOME` and `installers/base.run_cmd` takes + no `env=`, so the shell expands from the ambient environment or not at all. + **This does mutate the process environment**, unlike the rest of the + feature — but not `CLAUDE_CONFIG_DIR`, which `Prefix.environment()` does not + carry, so the promise about the user's Claude Code holds. Not thread-safe; + the CLI path is single-threaded. + """ + saved = dict(os.environ) + try: + os.environ.update(extra) + yield + finally: + os.environ.clear() + os.environ.update(saved) + + +def ensure_installed(prefix: Prefix, install_item: Callable[[], Sequence[Any]]) -> Status: + """Install the `agentsview` binary via its recipe item, or say why not. + + **`install_item` is injected, not looked up**: spec §9 walls the installer + machinery off from everything under `env_mgr/`, so `cli/main.py` assembles + the call. **Never raises** — one `Status` and one warning. Whether a missing + agentsview is fatal is decided once, by the recipe's `importance:`. + """ + try: + prefix.create() + with _patched_environ(prefix.environment()): + outs = list(install_item()) + except Exception as e: # noqa: BLE001 - see ensure_running's docstring + log.warning("agentsview: install failed (%s); skipping the o11y panel.", e) + return Status(False, f"install error: {e}") + + if not outs: + log.warning( + "agentsview: recipe item 'agentsview' not found in %s; skipping the o11y panel.", + RECIPE_PATH, + ) + return Status(False, "recipe item not found") + + outcome = outs[-1] + if outcome.level == "ok": + # Validated at install time rather than left for `serve` to discover. + # In its own `try`: the binary is installed either way, so a bug in the + # check must not turn a successful install into a reported failure. + try: + write_config(prefix, OTHER_PROVIDERS) + bad = check_disabled_agents(prefix) + except Exception as e: # noqa: BLE001 - see ensure_running's docstring + log.warning( + "agentsview: could not validate OTHER_PROVIDERS against the " + "installed binary (%s); continuing without that check.", + e, + ) + bad = () + if bad: + log.warning( + "agentsview: OTHER_PROVIDERS in env_mgr/o11y/agentsview.py has " + "drifted from the installed agentsview: %s. A name here the " + "binary no longer recognizes will keep the panel from " + "starting; a name the binary recognizes but this list omits " + "means that provider's sessions may appear on the panel.", + ", ".join(bad), + ) + return Status(True, outcome.message) + + log.warning("agentsview: %s; skipping the o11y panel.", outcome.message) + return Status(False, outcome.message) diff --git a/agent_sys/env_mgr/o11y/agentsview/design.md b/agent_sys/env_mgr/o11y/agentsview/design.md new file mode 100644 index 000000000..f43df671e --- /dev/null +++ b/agent_sys/env_mgr/o11y/agentsview/design.md @@ -0,0 +1,164 @@ +# AgentsView as `agent_sys`'s o11y panel — design + +Component design for `env_mgr/o11y/agentsview/`. The module-level design refers +here from `../../docs/design.md` §17. + +**The rule that outranks every feature below:** o11y may never fail the thing it +observes. Every failure is one `log.warning` and a skip, with a test per mode. + +## 1. The panel + +[AgentsView](https://github.com/kenn-io/agentsview) is an external Go binary +that reads Claude Code's JSONL transcripts and serves search, analytics and +token-cost views. `agent_sys` reaches its backend through `claude-agent-sdk`, +which spawns the `claude` CLI, which writes exactly those transcripts — so the +two fit with no glue on either side. The whole integration is *where the +transcripts land* and *which directory the panel reads*. + +**AgentsView's own code is never modified.** Every knob is one it publishes: +`--port`, `CLAUDE_PROJECTS_DIR`, `AGENTSVIEW_DATA_DIR`, `disabled_agents`. + +### The prefix + +`~/.infera_agent_sys`, laid out like `~/.local` (`bin/ share/ state/ run/`), +owned by `env_mgr` and named by the `AGENT_SYS_*` family in `prefix.py`. It +exists because the two obvious alternatives are both wrong: `/usr/local/bin` is +host state we promised not to touch, and `~/.local/bin` is the user's. Upstream's +`install.sh` is not used — it hardcodes `/usr/local/bin` with no override point. +The recipe (`recipes/agentsview.o11y.yaml`, `installer: bin`, +`importance: suggested`) installs a pinned release and verifies its published +`SHA256SUMS`. + +`state/claude` is deliberately not under a run root: the daemon outlives any +single run, so the directory it reads must be a stable path. + +### Session scoping — five gates + +The panel must show only the sessions `agent_sys` produced. **The constraint +that outranks the feature: the user's own Claude Code must be untouched.** + +| | | +|---|---| +| 1 | `CLAUDE_CONFIG_DIR=$AGENT_SYS_CLAUDE_HOME` in the **child's** environment dict, never in our `os.environ`. `material.deploy` sets its own per-attempt value, so `/config/projects` is symlinked into the prefix — credentials and settings stay the zone's, only the output is shared | +| 2 | `CLAUDE_PROJECTS_DIR` points the panel at that one root | +| 3 | `disabled_agents` switches off the other 60 providers. Pinned and hand-maintained; `check_disabled_agents` warns when it has drifted from the installed binary **in either direction** — a name the binary dropped breaks `serve` loudly, a name it gained leaks silently | +| 4 | `AGENTSVIEW_DATA_DIR` is ours, so a user's own archive and settings are untouched | +| 5 | `HOME` is redirected into the prefix for the binary's own subprocesses. AgentsView derives every provider's *default* root from `HOME`, so this needs no list and cannot go stale when upstream adds a provider we have never heard of. Found while measuring, and stronger than gate 3 | + +### Lifecycle, port, failure + +Started at the end of the `env_mgr` deploy path and left **resident** +(`daemon_idle_timeout = "0s"`; the default 20m would empty the panel for anyone +opening the URL after their run). Default port `18888`; resolution order is +`--agentsview-port`, then `AGENTSVIEW_PORT`, then the default, and an unusable +value falls back with a warning rather than failing. `--no-agentsview`, +`--dry-run` and `--clean` make no external call at all. + +**A taken port is a warning and a skip, never a relocation.** We bind-probe +before launching precisely because `serve` would otherwise move quietly to the +next free port, and a panel on 18889 is a panel nobody knows the address of. +`--replace` is passed for the same reason: without it, `serve --background +--port N` silently attaches to any daemon already alive for this data directory +and reports *its* port, exit 0, `N` ignored. + +**Reuse requires proof of ownership.** A live AgentsView on the port is not +evidence it is ours — a user's own daemon lists every session on the machine. +Two gates: it answers `/api/v1/agents` with 200 and JSON (a status code is not +an identity), **and** a live `daemon..json` in our data directory names +that port. That record is AgentsView's own artefact, read never written; because +the data directory is ours alone, one found there was written by a daemon we +configured. It is removed on a clean stop, so only an unclean death leaves a +stale one, and that is caught by checking the pid. + +**No validation path may start a daemon.** Measured: `health`, `projects` and +`session list` all autostart one on a port AgentsView picks — the delegation +this component exists to prevent, happening where nobody is watching. +`doctor sync` does not, and answers the same question. `AGENTSVIEW_NO_DAEMON=1` +does not rescue them; it makes them refuse outright. + +**Success goes to the event stream** (`EventKind.O11Y_PANEL`), not `logging`: +this package never configures `logging`, so an info record reaches nobody while +`log.warning` still reaches stderr through `lastResort`. + +### Known limitation + +The zone symlink's behaviour under an enforcing policy is **untested, because +currently untestable**: `agent_sys` refuses to start any AI task under +`AGENT_SYS_NO_PERMISSIONS=0` today, before the executor runs, so nothing ever +traverses the link. That refusal predates this feature — measured with paired +arms differing in one file, both failing identically. If a confined child ever +does follow it, the prefix is under `$HOME`, which `DEFAULT_SYSTEM_SET` does not +grant; the likely repair is a grant on `$AGENT_SYS_CLAUDE_HOME/projects`, which +is a permissions decision and is deliberately not taken here. Read this as +"untested", never as "safe". + +## 2. One project per run + +AgentsView derives a project from the session's **deepest** path segment. Every +agent attempt runs in its own zone, and zones nest, so one run's sessions arrive +as several unrelated projects — measured on a real nested fixture, four sessions +of one run as `0_11e34171`, `0_f6daeb1b`, `task.main.b869ddf0_…` and +`task.solve_a.8c8fb4c1_…`. + +**Renaming the directories cannot fix this, and that is the whole reason this +section exists.** PR #156 put the closure name into every runtime directory, +which made those strings readable; it did not join them, because a nested child +task fragments off from its parent however prettily both are named. The only +filesystem fix would be putting every attempt of a run under one directory, +which is precisely what zone isolation exists to prevent. + +So `env_mgr/o11y/mapping.py` posts **one `explicit` mapping over the run root** +at run start, through AgentsView's own settings API. The dependency stays +unmodified. + +### What was measured before any of it was written + +Every property below came from a container probe against a real v0.42.0, not +from the API's shape or a field's name. Three of them changed the code. + +| | | +|---|---| +| **`explicit` is the only usable layout** | The other legal value, `repo_dot_worktrees`, matched **zero** sessions across thirteen prefix shapes — including a genuine `/.worktrees/` tree the archive had correctly identified. Why is **unsettled**; upstream source at `ff8fb4e8` would settle it. We do not use it | +| **`explicit` is depth-independent** | One mapping over `runs/` caught that run's sessions at depth 3 *and* 5, and only that run's. Nesting is a non-issue — which is what makes the whole design work | +| **`Origin` is mandatory** | Without it a mutating call answers a plain-text `403 Forbidden`, not the JSON error shape. It reads exactly like a missing endpoint | +| **`machine` must be read, never assembled** | A mapping written with a `machine` the daemon does not recognise matches nothing and says nothing. It comes from the daemon's own `local_machine`. The recon itself ran in a container whose hostname was not the host's — which is how a hard-coded `gethostname()` would have shipped broken | +| **Names normalise `-` → `_`** | Everything else round-trips, including `@ : / + # % .`, spaces and non-ASCII; there is no length cap. We normalise before posting so the string we send is the string the panel shows | +| **No `apply`, `preview`, `reclassify` or token** | A mapping that exists *before* ingest labels the session at sync time. Those calls are only for sessions already in the archive | +| **`409` is success** | `POST` is not idempotent; uniqueness is `(machine, path_prefix)`, not including layout. A re-run of the same run id conflicts with its own row | +| **Prefix boundaries are segment-safe, longest wins** | `run-AAA` does not capture `run-AAAX`; a specific row beats a general one | +| **Classification reads the recorded cwd, not the disk** | A session whose directory never existed classified and mapped normally; one whose directory was moved away survived a `sync --full`. **Zone teardown after a run is harmless to the panel** | +| **AgentsView does not look downward** | A zone is a non-git directory containing `git clone --shared` at `/workspace` whose alternates point outside it. Byte-for-byte identical classification to a bare plain directory: `repository_path=''`, `worktree_relationship='unknown'`. It never reaches `alternates` | + +### Retention, which is a policy and not hygiene + +One row per run, and **no garbage collection** — deliberately. Deleting a row +does not disturb the panel on `apply`, but the next `sync --full` re-derives +labels from the mapping table and that run's sessions revert to their directory +names. **Keeping the label is keeping the row.** So any GC is a decision about +how long old runs stay named, not a tidiness measure, and a GC that silently +un-names last month's runs is worse than a table of small rows. + +**Measured, and it is why no GC is needed.** The cost is `O(rows × sessions)`, +not `O(rows)` — visible only by repeating the whole curve at a different session +count, where the same row counts cost 3.5–4× more. It works out at ~0.75 µs per +(row × session), holding across a 20× range of the product: ~2.6 s added to a +`sync --full` at 5000 rows over 800 sessions, and nothing measurable below ~1000 +rows. + +**But `sync --full` is not the path the daemon runs.** Paired arms over an +identical 800-session archive differing only in the mapping table: idle 0.991 s +against 1.023 s, one new session 1.150 s against 1.189 s. **+32 ms and +39 ms at +5000 rows**, both inside the spread within either arm. Incremental sync — file +watch and poll timer — is unaffected. + +Since runs and sessions grow together the mapping term is quadratic in run +count, but it loses to the linear re-parse cost of a full sync until roughly +**15 000 runs**, by which point a full sync takes ~19 minutes for reasons that +have nothing to do with mappings. If a GC is ever wanted, the trigger is +full-sync latency, not table size. + +**One thing left unsettled, and it is the number a GC would turn on:** whether +`enabled=false` rows still cost a scan. Disabling rather than deleting is the +attractive shape for a GC — deleting un-names the run — and that measurement is +what would decide it. Re-run the scaling harness after flipping the rows via +`PUT /{id}`. diff --git a/agent_sys/env_mgr/o11y/agentsview/mapping.py b/agent_sys/env_mgr/o11y/agentsview/mapping.py new file mode 100644 index 000000000..6522f6dda --- /dev/null +++ b/agent_sys/env_mgr/o11y/agentsview/mapping.py @@ -0,0 +1,165 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""One AgentsView project per run. + +**Why this cannot be solved by naming directories better.** AgentsView derives +a project from the session's **deepest** path segment, so the attempts of one +run arrive as several unrelated projects — measured on a real nested fixture, +four sessions of one run as `0_11e34171`, `0_f6daeb1b`, `task.main.b869ddf0_…` +and `task.solve_a.8c8fb4c1_…`. PR #156 made those strings readable; it did not +and cannot join them, because a nested child task fragments off from its parent +however prettily both are named. The only filesystem fix would be putting every +attempt of a run in one directory, which is what zone isolation exists to +prevent. One mapping over the run root collapses all four, at any depth. + +**AgentsView is not modified.** This uses its published settings API, and every +property relied on below was measured against v0.42.0 rather than assumed — +see `design.md` §2, beside this file. + +Like everything else under `o11y/`: every function returns a `Status` and +raises nothing. +""" + +from __future__ import annotations + +import json +import logging +import urllib.error +import urllib.request +from pathlib import Path + +from .agentsview import Status + +__all__ = ["ensure_run_project", "name_for_run"] + +log = logging.getLogger("env_mgr.o11y.agentsview.mapping") + +#: The settings collection. `GET` lists mappings and names the machine; `POST` +#: creates one. +MAPPINGS_PATH = "/api/v1/settings/worktree-mappings" + +#: The only layout that does what we need. The other legal value, +#: `repo_dot_worktrees`, matched **zero** sessions across thirteen prefix +#: shapes — including a genuine `/.worktrees/` tree the archive had +#: correctly identified — and blanks `project` on write. Why it matches nothing +#: is unsettled and does not matter here: it is not this mechanism. +LAYOUT = "explicit" + +#: Prefix on the project name, so a run is recognisable as one among whatever +#: else a user has in their panel. A dot, because dots round-trip verbatim. +NAME_PREFIX = "run." + +#: The one transformation AgentsView applies to a project name: `-` becomes +#: `_`. Measured across `@ : / + # % .`, spaces, uppercase, leading digits and +#: non-ASCII — everything else round-trips, and there is no length cap. Applied +#: here so that the string we post is the string the panel shows, and a log +#: line cannot disagree with the UI. +_NORMALISE = str.maketrans({"-": "_"}) + +#: Long enough for a local daemon that has already answered a health check, +#: short enough that a wedged one cannot hold up a run. +TIMEOUT_S = 5.0 + + +def name_for_run(run_root: Path) -> str: + """The project name for a run, from its directory name alone. + + Nothing above the run root may change the label: two roots differing only + in `--demo-root` must produce the same name, or the same run reads as two. + """ + return NAME_PREFIX + run_root.name.translate(_NORMALISE) + + +def _get_json(url: str) -> object: + req = urllib.request.Request(url, method="GET") # noqa: S310 + with urllib.request.urlopen(req, timeout=TIMEOUT_S) as r: # noqa: S310 + return json.loads(r.read()) + + +def _post_json(url: str, origin: str, body: dict[str, object]) -> None: + # **`Origin` is mandatory on every mutating call.** Without it the answer is + # a plain-text `403 Forbidden` rather than the JSON error shape, which reads + # exactly like a missing endpoint. Measured. + req = urllib.request.Request( # noqa: S310 + url, + data=json.dumps(body).encode(), + headers={"Content-Type": "application/json", "Origin": origin}, + method="POST", + ) + with urllib.request.urlopen(req, timeout=TIMEOUT_S): # noqa: S310 + return None + + +def ensure_run_project(url: str | None, run_root: Path, name: str | None = None) -> Status: + """Give this run its own project on the panel. Never raises. + + Called once at run start, **before any session exists** — which is the whole + reason there is no `apply`, `preview`, `reclassify` or token here. Measured: + a mapping that exists before ingest is consulted at sync time and the + session arrives already labelled. + + `url` is `None` when the panel did not start. That was already warned about + once, so this says nothing: o11y reports a problem once or not at all. + """ + if url is None: + return Status(False, "no panel") + try: + machine = _machine(url) + if machine is None: + log.warning( + "agentsview: could not read the panel's machine name from %s%s; " + "this run will not get its own project.", + url, + MAPPINGS_PATH, + ) + return Status(False, "no machine name") + project = name or name_for_run(run_root) + _post_json( + url + MAPPINGS_PATH, + origin=url, + body={ + "machine": machine, + "path_prefix": str(run_root), + "project": project, + "layout": LAYOUT, + "enabled": True, + }, + ) + except urllib.error.HTTPError as e: + # **409 is the state we wanted.** `POST` is not idempotent — uniqueness + # is `(machine, path_prefix)` — so a re-run of the same run id conflicts + # with the row it created last time. Nothing to do and nothing to say. + if e.code == 409: + return Status(True, "already mapped") + log.warning( + "agentsview: could not give run %s its own project (HTTP %s); " + "its sessions will appear under their directory names.", + run_root.name, + e.code, + ) + return Status(False, f"http {e.code}") + except Exception as e: # noqa: BLE001 - see the module docstring + log.warning( + "agentsview: could not give run %s its own project (%s); " + "its sessions will appear under their directory names.", + run_root.name, + e, + ) + return Status(False, f"mapping failed: {e}") + return Status(True, project) + + +def _machine(url: str) -> str | None: + """The name the daemon will match against, read from the daemon. + + **Never assembled locally.** A mapping written with a `machine` the daemon + does not recognise matches nothing and says nothing — and the recon that + established all of this ran in a container whose hostname was not the + host's, which is exactly how a hard-coded `socket.gethostname()` would have + shipped broken. + """ + listing = _get_json(url + MAPPINGS_PATH) + if not isinstance(listing, dict): + return None + machine = listing.get("local_machine") + return machine if isinstance(machine, str) and machine else None diff --git a/agent_sys/env_mgr/paths.py b/agent_sys/env_mgr/paths.py index 3b30b5c34..adb7a747b 100644 --- a/agent_sys/env_mgr/paths.py +++ b/agent_sys/env_mgr/paths.py @@ -76,11 +76,17 @@ from env_mgr.fs.zone import Zone __all__ = [ + "BIN_ENV_VAR", + "CLAUDE_HOME_ENV_VAR", "HANDOFFS_ENV_VAR", + "HOME_ENV_VAR", "LOGS_ENV_VAR", "PACKAGE_ENV_VAR", "PLAYGROUND_ENV_VAR", "REMOTE_SUFFIX", + "RUN_ENV_VAR", + "SHARE_ENV_VAR", + "STATE_ENV_VAR", "WORKSPACE_ENV_VAR", "ZONE_ENV_VAR", "remote_name", @@ -137,6 +143,18 @@ #: would otherwise have no name; ``等等`` invited the completion. LOGS_ENV_VAR = "AGENT_SYS_MY_LOGS" +#: **The prefix family, re-exported rather than redefined.** `prefix` owns them +#: because it owns the layout; they are visible here because `paths` is where a +#: reader looks for an ``AGENT_SYS_*`` name. +from .prefix import ( # noqa: E402 + BIN_ENV_VAR, + CLAUDE_HOME_ENV_VAR, + HOME_ENV_VAR, + RUN_ENV_VAR, + SHARE_ENV_VAR, + STATE_ENV_VAR, +) + #: The user's ``_romote``, read as ``_remote``. A suffix rather than a second #: family, so that a name and its counterpart cannot drift apart. REMOTE_SUFFIX = "_REMOTE" diff --git a/agent_sys/env_mgr/prefix.py b/agent_sys/env_mgr/prefix.py new file mode 100644 index 000000000..9eea464bd --- /dev/null +++ b/agent_sys/env_mgr/prefix.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""`~/.infera_agent_sys` — agent_sys's own `~/.local`. + +**Why a prefix at all.** Things `agent_sys` installs have to live somewhere, and +the two obvious somewheres are both wrong: `/usr/local/bin` is host state we +promised not to touch, and `~/.local/bin` is the user's. A prefix we own is the +only place where "install" and "uninstall" are both a directory operation. + +**`resolve` takes its environment as an argument** and never reads `os.environ`. +A component deciding *where the user's Claude transcripts go* must be testable +without a process-global, and the same discipline keeps it out of the ambient +environment at runtime. `o11y` is the first consumer, not the owner. +""" + +from __future__ import annotations + +import os +import tempfile +from collections.abc import Mapping +from dataclasses import dataclass +from pathlib import Path + +__all__ = ["CLAUDE_CONFIG_ENV_VAR", "Prefix", "agent_environment"] + +#: The directory name. Fixed, and deliberately dotted: it is machine state, not +#: something a user browses. +DIRNAME = ".infera_agent_sys" + +HOME_ENV_VAR = "AGENT_SYS_HOME" +BIN_ENV_VAR = "AGENT_SYS_BIN" +SHARE_ENV_VAR = "AGENT_SYS_SHARE" +STATE_ENV_VAR = "AGENT_SYS_STATE" +RUN_ENV_VAR = "AGENT_SYS_RUN" +CLAUDE_HOME_ENV_VAR = "AGENT_SYS_CLAUDE_HOME" + +#: AgentsView's own two names. Not ours to rename — they are the published +#: interface of an external dependency whose code we do not modify. +AGENTSVIEW_DATA_ENV_VAR = "AGENTSVIEW_DATA_DIR" +CLAUDE_PROJECTS_ENV_VAR = "CLAUDE_PROJECTS_DIR" + +#: Claude Code's own name for "where my config, credentials and transcripts +#: live". Also not ours to rename. +CLAUDE_CONFIG_ENV_VAR = "CLAUDE_CONFIG_DIR" + + +def _home(environ: Mapping[str, str]) -> Path: + """`$HOME`, then the passwd entry, then a per-uid directory in `$TMPDIR`. + + The last step is real, not a formality: `resolve` has to be total, and the + cwd would put machine state inside whichever repository is checked out. An + archive that does not survive a reboot is the right degradation here. + """ + home = environ.get("HOME") + if home: + return Path(home).expanduser() + try: + import pwd + + return Path(pwd.getpwuid(os.getuid()).pw_dir) + except Exception: # noqa: BLE001 - see the module docstring; this cannot fail + return Path(tempfile.gettempdir()) / f"infera-agent-sys-{os.getuid()}" + + +@dataclass(frozen=True) +class Prefix: + """One resolved prefix. Every path is derived; none is stored twice.""" + + root: Path + + @classmethod + def resolve(cls, environ: Mapping[str, str]) -> Prefix: + """**Total: it always answers, and never raises.** + + It was `environ["HOME"]` — a `KeyError` under `env -i`, caught at one of + three call sites and fatal at the other two. `expanduser`/`resolve` + because `~/foo` is a literal to `Path` and a relative override moves + with a cwd that changes between zones. + """ + override = environ.get(HOME_ENV_VAR) + if override: + return cls(Path(override).expanduser().resolve()) + return cls(_home(environ) / DIRNAME) + + @property + def bin(self) -> Path: + return self.root / "bin" + + @property + def share(self) -> Path: + return self.root / "share" + + @property + def state(self) -> Path: + return self.root / "state" + + @property + def run(self) -> Path: + return self.root / "run" + + @property + def claude_home(self) -> Path: + """`CLAUDE_CONFIG_DIR` for agent children. + + **Not under a run root**: a reader of these transcripts outlives any + one run, so `runs//` would name a directory that stops existing. + """ + return self.state / "claude" + + @property + def agentsview_data(self) -> Path: + return self.state / "agentsview" + + def environment(self) -> dict[str, str]: + """Every directory, by name, ready to merge into a child's `env`. + + Returned rather than exported: the caller decides whose environment + this joins, and the answer is never this process's. + """ + return { + HOME_ENV_VAR: str(self.root), + BIN_ENV_VAR: str(self.bin), + SHARE_ENV_VAR: str(self.share), + STATE_ENV_VAR: str(self.state), + RUN_ENV_VAR: str(self.run), + CLAUDE_HOME_ENV_VAR: str(self.claude_home), + AGENTSVIEW_DATA_ENV_VAR: str(self.agentsview_data), + CLAUDE_PROJECTS_ENV_VAR: str(self.claude_home / "projects"), + } + + def create(self) -> None: + """Idempotent. Creates only inside `root` — never a parent.""" + for d in ( + self.bin, + self.share, + self.state, + self.run, + self.claude_home / "projects", + self.agentsview_data, + ): + d.mkdir(parents=True, exist_ok=True) + + +def agent_environment( + prefix: Prefix, base: Mapping[str, str], *, bin_on_path: bool = True +) -> dict[str, str]: + """`base`, plus the prefix, plus the one variable that scopes the panel. + + **`CLAUDE_CONFIG_DIR` goes in the returned dict, never into `os.environ`** — + the whole promise to the user, guarded by + `test_agent_environment_does_not_touch_this_process`. + **`bin_on_path=False` under a policy:** `executable_path` derives `PATH` + from the granted set, which excludes `$HOME`. `AGENT_SYS_BIN` still names it. + """ + env = dict(base) + env.update(prefix.environment()) + env[CLAUDE_CONFIG_ENV_VAR] = str(prefix.claude_home) + if bin_on_path: + env["PATH"] = ":".join([str(prefix.bin), base.get("PATH", "")]).rstrip(":") + return env diff --git a/agent_sys/env_mgr/prepare.py b/agent_sys/env_mgr/prepare.py index 7f923f2ed..0e2bccfa9 100644 --- a/agent_sys/env_mgr/prepare.py +++ b/agent_sys/env_mgr/prepare.py @@ -60,6 +60,7 @@ executable_path, ) from env_mgr.isolation.probe import Availability, probe, select +from env_mgr.prefix import Prefix, agent_environment from env_mgr.protocols import Confinement, Context, NoConfinement, PrepareRefused, SyncReport from env_mgr.remote import tools as _tools from env_mgr.sync import Direction @@ -465,7 +466,17 @@ def prepare( # directory the kernel will refuse. A declared `env` may still override it, # because an author saying so outranks a default — but an override naming an # ungranted directory is unreachable, and nothing here can make it otherwise. - environment = {"PATH": executable_path(policy)} + # + # The o11y prefix rides along because this dict is the *child's* + # environment; setting `CLAUDE_CONFIG_DIR` in ours would redirect a Claude + # Code the user started. `bin_on_path=False` because the prefix is under + # `$HOME`, which the default grants exclude, and nothing in a child needs + # to exec the binary. + environment = agent_environment( + Prefix.resolve(os.environ), + base={"PATH": executable_path(policy)}, + bin_on_path=False, + ) # 6a. **The task package: a copy in the zone, not a grant on the root.** # `interfaces.md` §4.16, F19's third position. It sits beside handoff diff --git a/agent_sys/env_mgr/recipes/agentsview.o11y.yaml b/agent_sys/env_mgr/recipes/agentsview.o11y.yaml new file mode 100644 index 000000000..708fc30d1 --- /dev/null +++ b/agent_sys/env_mgr/recipes/agentsview.o11y.yaml @@ -0,0 +1,62 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +# +# AgentsView, agent_sys's o11y panel. An external dependency, installed +# unmodified into agent_sys's own prefix. +# +# `importance: suggested` is load-bearing, not decoration: `level_for_missing` +# turns a failed install of a non-required item into a warning, which is +# exactly the "o11y may never fail a deployment" rule, using the mechanism +# env_mgr already has rather than a second one. +# +# `target.path` below is a placeholder, same convention as +# `recipes/sglang.repo.yaml`: nothing in env_mgr expands `${VAR}` in a YAML +# value. `ensure_installed` sets it in code; a human passes `--path`: +# +# env-mgr install env_mgr/recipes/agentsview.o11y.yaml --path "$AGENT_SYS_HOME" +# +# `check_cmd:` is an absolute path because `run_cmd` takes no `env=` and +# `$AGENT_SYS_HOME/bin` is deliberately not on a child's `PATH`; a bare +# `agentsview --version` would then never report "already present (skip)". +# `ensure_installed` patches `$AGENT_SYS_HOME` in for the duration of the call. +# +# The checksum gate is **two statements, not one `&&` list**: `set -e` is exempt +# for a non-final command in an `&&` list, so the old one-liner carried on and +# installed an unverified binary whenever awk matched no line. Three tests run +# this string for real against a stubbed curl. `${tmp:?}` in the trap because a +# recursive delete on a variable is one unset variable from a larger delete. +# +# Pinned to a release rather than "latest": two machines running different +# panels with no record of which is not an observability story. Bumping is a +# one-line edit of the three places it appears below, plus `version:`. +# amd64 only. +version: 1 + +target: + kind: prefix + name: infera_agent_sys + path: /path/to/your/.infera_agent_sys + +items: + - installer: bin + importance: suggested + layer: system + name: agentsview + version: "0.42.0" + check_cmd: "$AGENT_SYS_HOME/bin/agentsview --version" + install: >- + set -e; + tmp=$(mktemp -d); + trap 'rm -rf "${tmp:?}"' EXIT; + base="https://github.com/kenn-io/agentsview/releases/download/v0.42.0"; + curl -fsSL "$base/agentsview_0.42.0_linux_amd64.tar.gz" -o "$tmp/av.tgz"; + curl -fsSL "$base/SHA256SUMS" -o "$tmp/SHA256SUMS"; + expected=$(awk '$2=="agentsview_0.42.0_linux_amd64.tar.gz"{print $1}' "$tmp/SHA256SUMS"); + [ -n "$expected" ] || exit 1; + actual=$(sha256sum "$tmp/av.tgz" | cut -d' ' -f1); + [ "$expected" = "$actual" ]; + tar -xzf "$tmp/av.tgz" -C "$tmp" agentsview; + mkdir -p "$AGENT_SYS_HOME/bin"; + mv "$tmp/agentsview" "$AGENT_SYS_HOME/bin/agentsview"; + chmod +x "$AGENT_SYS_HOME/bin/agentsview" + tags: [o11y] diff --git a/agent_sys/examples/demo/assets/lib/store.py b/agent_sys/examples/demo/assets/lib/store.py index 941935870..ce95fe497 100644 --- a/agent_sys/examples/demo/assets/lib/store.py +++ b/agent_sys/examples/demo/assets/lib/store.py @@ -53,6 +53,30 @@ def store_root() -> Path: return Path(os.environ["AGENT_SYS_DEMO_STORE"]) +def handoff_dir(hid: str) -> Path: + """This handoff's directory in the store. `handoff.store.handoff_dir`, duplicated. + + The store names a directory ``handoff..`` so that a person + reading a run tree can tell what an artefact is. **Nothing resolves through + the label**: a directory is this handoff's when its name *is* the uuid — the + shape written before labels existed — or ends with ``.``. The same + admissible duplication as `MANIFEST` and `CONTENT` above, and covered by the + same agreement test. + """ + root = store_root() + suffix = f".{hid}" + if root.is_dir(): + for entry in sorted(root.iterdir()): + if entry.name == hid or entry.name.endswith(suffix): + return entry + return root / hid + + +def hid_of(dirname: str) -> str: + """The handoff id in a store directory name — the last field, labelled or not.""" + return dirname.rsplit(".", 1)[-1] + + def versions(hid: str) -> list[int]: """**Published versions only, and the gaps are the point.** @@ -70,7 +94,7 @@ def versions(hid: str) -> list[int]: next. Holes are skipped and never compacted, because renumbering would move an artefact a digest already names. """ - base = store_root() / hid + base = handoff_dir(hid) if not base.is_dir(): return [] found = [ @@ -87,7 +111,7 @@ def content_dir(hid: str, version: int | None = None) -> Path | None: if not numbers: return None chosen = numbers[-1] if version is None else version - path = store_root() / hid / f"v{chosen}" / CONTENT + path = handoff_dir(hid) / f"v{chosen}" / CONTENT return path if path.is_dir() else None @@ -96,7 +120,7 @@ def kind_of(hid: str, version: int | None = None) -> str: if not numbers: return "" chosen = numbers[-1] if version is None else version - manifest = store_root() / hid / f"v{chosen}" / MANIFEST + manifest = handoff_dir(hid) / f"v{chosen}" / MANIFEST if not manifest.is_file(): return "" # **YAML, not JSON**, and this read `json.loads` until the filename was @@ -164,9 +188,10 @@ def latest_of_kind(kind: str) -> Path | None: return None best: tuple[float, Path] | None = None for entry in sorted(root.iterdir()): - if not entry.is_dir() or kind_of(entry.name) != kind: + hid = hid_of(entry.name) + if not entry.is_dir() or kind_of(hid) != kind: continue - found = content_dir(entry.name) + found = content_dir(hid) if found is None: continue stamp = found.stat().st_mtime diff --git a/agent_sys/examples/demo2/assets/lib/store.py b/agent_sys/examples/demo2/assets/lib/store.py index 0fa7cb63d..624d2aef7 100644 --- a/agent_sys/examples/demo2/assets/lib/store.py +++ b/agent_sys/examples/demo2/assets/lib/store.py @@ -58,6 +58,30 @@ def store_root() -> Path: return Path(os.environ["AGENT_SYS_DEMO_STORE"]) +def handoff_dir(hid: str) -> Path: + """This handoff's directory in the store. `handoff.store.handoff_dir`, duplicated. + + The store names a directory ``handoff..`` so that a person + reading a run tree can tell what an artefact is. **Nothing resolves through + the label**: a directory is this handoff's when its name *is* the uuid — the + shape written before labels existed — or ends with ``.``. The same + admissible duplication as `MANIFEST` and `CONTENT` above, and covered by the + same agreement test. + """ + root = store_root() + suffix = f".{hid}" + if root.is_dir(): + for entry in sorted(root.iterdir()): + if entry.name == hid or entry.name.endswith(suffix): + return entry + return root / hid + + +def hid_of(dirname: str) -> str: + """The handoff id in a store directory name — the last field, labelled or not.""" + return dirname.rsplit(".", 1)[-1] + + def versions(hid: str) -> list[int]: """**Published versions only, and the gaps are the point.** @@ -75,7 +99,7 @@ def versions(hid: str) -> list[int]: next. Holes are skipped and never compacted, because renumbering would move an artefact a digest already names. """ - base = store_root() / hid + base = handoff_dir(hid) if not base.is_dir(): return [] found = [ @@ -92,7 +116,7 @@ def content_dir(hid: str, version: int | None = None) -> Path | None: if not numbers: return None chosen = numbers[-1] if version is None else version - path = store_root() / hid / f"v{chosen}" / CONTENT + path = handoff_dir(hid) / f"v{chosen}" / CONTENT return path if path.is_dir() else None @@ -101,7 +125,7 @@ def kind_of(hid: str, version: int | None = None) -> str: if not numbers: return "" chosen = numbers[-1] if version is None else version - manifest = store_root() / hid / f"v{chosen}" / MANIFEST + manifest = handoff_dir(hid) / f"v{chosen}" / MANIFEST if not manifest.is_file(): return "" # **YAML, not JSON**, and this read `json.loads` until the filename was @@ -173,9 +197,10 @@ def latest_of_kind(kind: str) -> Path | None: return None best: tuple[float, Path] | None = None for entry in sorted(root.iterdir()): - if not entry.is_dir() or kind_of(entry.name) != kind: + hid = hid_of(entry.name) + if not entry.is_dir() or kind_of(hid) != kind: continue - found = content_dir(entry.name) + found = content_dir(hid) if found is None: continue stamp = found.stat().st_mtime diff --git a/agent_sys/examples/llm_e2e_performance_optimization/analyze-demo/assets/lib/store.py b/agent_sys/examples/llm_e2e_performance_optimization/analyze-demo/assets/lib/store.py index 941935870..ce95fe497 100644 --- a/agent_sys/examples/llm_e2e_performance_optimization/analyze-demo/assets/lib/store.py +++ b/agent_sys/examples/llm_e2e_performance_optimization/analyze-demo/assets/lib/store.py @@ -53,6 +53,30 @@ def store_root() -> Path: return Path(os.environ["AGENT_SYS_DEMO_STORE"]) +def handoff_dir(hid: str) -> Path: + """This handoff's directory in the store. `handoff.store.handoff_dir`, duplicated. + + The store names a directory ``handoff..`` so that a person + reading a run tree can tell what an artefact is. **Nothing resolves through + the label**: a directory is this handoff's when its name *is* the uuid — the + shape written before labels existed — or ends with ``.``. The same + admissible duplication as `MANIFEST` and `CONTENT` above, and covered by the + same agreement test. + """ + root = store_root() + suffix = f".{hid}" + if root.is_dir(): + for entry in sorted(root.iterdir()): + if entry.name == hid or entry.name.endswith(suffix): + return entry + return root / hid + + +def hid_of(dirname: str) -> str: + """The handoff id in a store directory name — the last field, labelled or not.""" + return dirname.rsplit(".", 1)[-1] + + def versions(hid: str) -> list[int]: """**Published versions only, and the gaps are the point.** @@ -70,7 +94,7 @@ def versions(hid: str) -> list[int]: next. Holes are skipped and never compacted, because renumbering would move an artefact a digest already names. """ - base = store_root() / hid + base = handoff_dir(hid) if not base.is_dir(): return [] found = [ @@ -87,7 +111,7 @@ def content_dir(hid: str, version: int | None = None) -> Path | None: if not numbers: return None chosen = numbers[-1] if version is None else version - path = store_root() / hid / f"v{chosen}" / CONTENT + path = handoff_dir(hid) / f"v{chosen}" / CONTENT return path if path.is_dir() else None @@ -96,7 +120,7 @@ def kind_of(hid: str, version: int | None = None) -> str: if not numbers: return "" chosen = numbers[-1] if version is None else version - manifest = store_root() / hid / f"v{chosen}" / MANIFEST + manifest = handoff_dir(hid) / f"v{chosen}" / MANIFEST if not manifest.is_file(): return "" # **YAML, not JSON**, and this read `json.loads` until the filename was @@ -164,9 +188,10 @@ def latest_of_kind(kind: str) -> Path | None: return None best: tuple[float, Path] | None = None for entry in sorted(root.iterdir()): - if not entry.is_dir() or kind_of(entry.name) != kind: + hid = hid_of(entry.name) + if not entry.is_dir() or kind_of(hid) != kind: continue - found = content_dir(entry.name) + found = content_dir(hid) if found is None: continue stamp = found.stat().st_mtime diff --git a/agent_sys/examples/llm_e2e_performance_optimization/integration-demo/assets/lib/store.py b/agent_sys/examples/llm_e2e_performance_optimization/integration-demo/assets/lib/store.py index 941935870..ce95fe497 100644 --- a/agent_sys/examples/llm_e2e_performance_optimization/integration-demo/assets/lib/store.py +++ b/agent_sys/examples/llm_e2e_performance_optimization/integration-demo/assets/lib/store.py @@ -53,6 +53,30 @@ def store_root() -> Path: return Path(os.environ["AGENT_SYS_DEMO_STORE"]) +def handoff_dir(hid: str) -> Path: + """This handoff's directory in the store. `handoff.store.handoff_dir`, duplicated. + + The store names a directory ``handoff..`` so that a person + reading a run tree can tell what an artefact is. **Nothing resolves through + the label**: a directory is this handoff's when its name *is* the uuid — the + shape written before labels existed — or ends with ``.``. The same + admissible duplication as `MANIFEST` and `CONTENT` above, and covered by the + same agreement test. + """ + root = store_root() + suffix = f".{hid}" + if root.is_dir(): + for entry in sorted(root.iterdir()): + if entry.name == hid or entry.name.endswith(suffix): + return entry + return root / hid + + +def hid_of(dirname: str) -> str: + """The handoff id in a store directory name — the last field, labelled or not.""" + return dirname.rsplit(".", 1)[-1] + + def versions(hid: str) -> list[int]: """**Published versions only, and the gaps are the point.** @@ -70,7 +94,7 @@ def versions(hid: str) -> list[int]: next. Holes are skipped and never compacted, because renumbering would move an artefact a digest already names. """ - base = store_root() / hid + base = handoff_dir(hid) if not base.is_dir(): return [] found = [ @@ -87,7 +111,7 @@ def content_dir(hid: str, version: int | None = None) -> Path | None: if not numbers: return None chosen = numbers[-1] if version is None else version - path = store_root() / hid / f"v{chosen}" / CONTENT + path = handoff_dir(hid) / f"v{chosen}" / CONTENT return path if path.is_dir() else None @@ -96,7 +120,7 @@ def kind_of(hid: str, version: int | None = None) -> str: if not numbers: return "" chosen = numbers[-1] if version is None else version - manifest = store_root() / hid / f"v{chosen}" / MANIFEST + manifest = handoff_dir(hid) / f"v{chosen}" / MANIFEST if not manifest.is_file(): return "" # **YAML, not JSON**, and this read `json.loads` until the filename was @@ -164,9 +188,10 @@ def latest_of_kind(kind: str) -> Path | None: return None best: tuple[float, Path] | None = None for entry in sorted(root.iterdir()): - if not entry.is_dir() or kind_of(entry.name) != kind: + hid = hid_of(entry.name) + if not entry.is_dir() or kind_of(hid) != kind: continue - found = content_dir(entry.name) + found = content_dir(hid) if found is None: continue stamp = found.stat().st_mtime diff --git a/agent_sys/examples/llm_e2e_performance_optimization/profiling-demo/assets/lib/store.py b/agent_sys/examples/llm_e2e_performance_optimization/profiling-demo/assets/lib/store.py index 941935870..ce95fe497 100644 --- a/agent_sys/examples/llm_e2e_performance_optimization/profiling-demo/assets/lib/store.py +++ b/agent_sys/examples/llm_e2e_performance_optimization/profiling-demo/assets/lib/store.py @@ -53,6 +53,30 @@ def store_root() -> Path: return Path(os.environ["AGENT_SYS_DEMO_STORE"]) +def handoff_dir(hid: str) -> Path: + """This handoff's directory in the store. `handoff.store.handoff_dir`, duplicated. + + The store names a directory ``handoff..`` so that a person + reading a run tree can tell what an artefact is. **Nothing resolves through + the label**: a directory is this handoff's when its name *is* the uuid — the + shape written before labels existed — or ends with ``.``. The same + admissible duplication as `MANIFEST` and `CONTENT` above, and covered by the + same agreement test. + """ + root = store_root() + suffix = f".{hid}" + if root.is_dir(): + for entry in sorted(root.iterdir()): + if entry.name == hid or entry.name.endswith(suffix): + return entry + return root / hid + + +def hid_of(dirname: str) -> str: + """The handoff id in a store directory name — the last field, labelled or not.""" + return dirname.rsplit(".", 1)[-1] + + def versions(hid: str) -> list[int]: """**Published versions only, and the gaps are the point.** @@ -70,7 +94,7 @@ def versions(hid: str) -> list[int]: next. Holes are skipped and never compacted, because renumbering would move an artefact a digest already names. """ - base = store_root() / hid + base = handoff_dir(hid) if not base.is_dir(): return [] found = [ @@ -87,7 +111,7 @@ def content_dir(hid: str, version: int | None = None) -> Path | None: if not numbers: return None chosen = numbers[-1] if version is None else version - path = store_root() / hid / f"v{chosen}" / CONTENT + path = handoff_dir(hid) / f"v{chosen}" / CONTENT return path if path.is_dir() else None @@ -96,7 +120,7 @@ def kind_of(hid: str, version: int | None = None) -> str: if not numbers: return "" chosen = numbers[-1] if version is None else version - manifest = store_root() / hid / f"v{chosen}" / MANIFEST + manifest = handoff_dir(hid) / f"v{chosen}" / MANIFEST if not manifest.is_file(): return "" # **YAML, not JSON**, and this read `json.loads` until the filename was @@ -164,9 +188,10 @@ def latest_of_kind(kind: str) -> Path | None: return None best: tuple[float, Path] | None = None for entry in sorted(root.iterdir()): - if not entry.is_dir() or kind_of(entry.name) != kind: + hid = hid_of(entry.name) + if not entry.is_dir() or kind_of(hid) != kind: continue - found = content_dir(entry.name) + found = content_dir(hid) if found is None: continue stamp = found.stat().st_mtime diff --git a/agent_sys/handoff/store.py b/agent_sys/handoff/store.py index 75829f3c4..5087c65a7 100644 --- a/agent_sys/handoff/store.py +++ b/agent_sys/handoff/store.py @@ -50,6 +50,9 @@ "STAGING_PREFIX", "FilesystemStore", "KindSource", + "handoff_dir", + "handoff_dirname", + "slug", "store_name_for", "version_dir", ] @@ -90,14 +93,81 @@ def store_name_for(scope: Scope) -> str: return _STORE_FOR_SCOPE[scope] -def version_dir(root: Path, hid: HandoffId, version: int) -> Path: +#: The kind prefix on a handoff's directory. Every runtime directory in this +#: system now leads with what it *is*, so a user reading a run tree can tell a +#: handoff from a zone without knowing any uuid. +HANDOFF_PREFIX = "handoff" + +#: A label's cap. `env_mgr.fs.zone._SLUG_CHARS`, duplicated across the package +#: boundary on `docs/interfaces.md` §8.1's terms, like `CONTENT_DIR` already is. +_SLUG_CHARS = 40 + + +def slug(text: object) -> str: + """A directory-name-safe label, or ``""`` when there is nothing to say. + + ``.`` is the field separator in a handoff directory name, so it must not + survive — otherwise `handoff_dir`'s *"the uuid is the last field"* rule + stops being true. `env_mgr.fs.zone.slug`'s rule, duplicated; the agreement + is pinned by `tests/interfaces/test_handoff_layout.py`. + """ + if text is None: + return "" + out: list[str] = [] + for char in str(text): + keep = char if (char.isascii() and (char.isalnum() or char in "_-")) else "-" + if keep == "-" and (not out or out[-1] == "-"): + continue + out.append(keep) + return "".join(out).strip("-")[:_SLUG_CHARS].strip("-") + + +def handoff_dirname(hid: HandoffId, kind: object = None) -> str: + """``handoff..``, or ``handoff.`` when the kind is unknown. + + The kind is a **label**: it makes the store readable and nothing resolves + through it, which is why `handoff_dir` can find a directory whose label is + absent, stale, or was written before labels existed. The uuid stays whole + and stays last, because that is what a lookup keys on. + """ + label = slug(kind) + return f"{HANDOFF_PREFIX}.{label}.{hid}" if label else f"{HANDOFF_PREFIX}.{hid}" + + +def handoff_dir(root: Path, hid: HandoffId, kind: object = None) -> Path: + """This handoff's directory under `root` — **the existing one, if there is one**. + + A directory is this handoff's when its name *is* the uuid (the shape written + before labels existed) or *ends with* ``.``. That is the whole + compatibility story: a store written by an earlier run resumes without a + migration, and a label may change without moving an artefact. + + When nothing is on disk yet the name to create is `handoff_dirname`'s. The + scan is over one directory, and only ever the store root. + """ + base = Path(root) + wanted = str(hid) + suffix = f".{wanted}" + try: + entries = sorted(base.iterdir()) + except OSError: + # An unreadable or absent root is not this function's error to raise; + # every caller already handles "the directory is not there". + entries = [] + for entry in entries: + if entry.name == wanted or entry.name.endswith(suffix): + return entry + return base / handoff_dirname(hid, kind) + + +def version_dir(root: Path, hid: HandoffId, version: int, kind: object = None) -> Path: """**The one function that computes a path**, and the on-disk shape is private. Bazel #23576 is the lesson: a path-shape change survived only because consumers use `file.path` rather than composing strings. Every other module asks for a path; none builds one. """ - return Path(root) / str(hid) / f"v{version}" + return handoff_dir(root, hid, kind) / f"v{version}" class KindSource(Protocol): @@ -124,7 +194,7 @@ def kind_for(self, hid: HandoffId) -> HandoffKind | None: ... class FilesystemStore: - """`//v/{content/,validation.yaml,manifest.yaml}`.""" + """`/handoff../v/{content/,validation.yaml,manifest.yaml}`.""" def __init__( self, @@ -144,6 +214,20 @@ def root(self) -> Path: """Read-only. A caller that wants a path inside asks `version_dir`.""" return self._root + def _dir(self, hid: HandoffId) -> Path: + """This handoff's directory, labelled with its kind when one is known. + + The label comes from `self._kinds`, which is exactly the same source + `put` and `seal` already use for the manifest — so a store that can + publish can also name, and a read-only store (`kinds=None`) still finds + every directory, because `handoff_dir` resolves on the uuid. + """ + kind = self._kinds.kind_for(hid) if self._kinds is not None else None + return handoff_dir(self._root, hid, getattr(kind, "name", None)) + + def _version_dir(self, hid: HandoffId, version: int) -> Path: + return self._dir(hid) / f"v{version}" + # ---- reads ---- def list_versions(self, hid: HandoffId) -> list[int]: @@ -210,12 +294,12 @@ def exists(self, hid: HandoffId, version: int | None = None) -> bool: """ if version is None: return bool(self.list_versions(hid)) - return (version_dir(self._root, hid, version) / MANIFEST_FILE).is_file() + return (self._version_dir(hid, version) / MANIFEST_FILE).is_file() def _version_dirs(self, hid: HandoffId) -> list[tuple[int, Path]]: """Every `v/` on disk, published or not. **Internal**: allocation must see the unpublished ones or it would hand out a number in use.""" - base = self._root / str(hid) + base = self._dir(hid) if not base.is_dir(): return [] out = [ @@ -282,7 +366,7 @@ def copy_out(self, hid: HandoffId, version: int, dst: Path) -> Content: want = manifest.digest.get("sha256") if got != want: raise DigestMismatch( - f"{version_dir(self._root, hid, version)}: manifest records " + f"{self._version_dir(hid, version)}: manifest records " f"sha256={want}, the copy at {dst} recomputes to sha256={got} " f"(algorithm {manifest.algorithm})" ) @@ -342,10 +426,10 @@ def allocate(self, hid: HandoffId) -> int: and `seal` are where that refusal lives. Requiring it here would block dispatch on a resolver only the seal can use. """ - base = self._root / str(hid) + base = self._dir(hid) base.mkdir(parents=True, exist_ok=True) for n in itertools.count(self._next_guess(base)): - target = version_dir(self._root, hid, n) + target = self._version_dir(hid, n) try: os.mkdir(target) except FileExistsError: @@ -411,7 +495,7 @@ def seal(self, hid: HandoffId, version: int, *, producer: TaskId) -> str | None: # `NotSealable`, and it raises rather than returning a reason: neither # of these says anything about the content, so neither is an outcome of # the attempt. See `errors.NotSealable` for the re-run case. - target = version_dir(self._root, hid, version) + target = self._version_dir(hid, version) if not target.is_dir(): raise NotSealable( f"cannot seal {hid} v{version}: {target} does not exist. A version " @@ -500,7 +584,7 @@ def put(self, hid: HandoffId, content_dir: Path, *, producer: TaskId) -> int: # disconnected caller, not a deleted module, so re-wiring it is one line. content_mod.check_items(content_mod.load(content_dir), ctype, kind.items_schema) - base = self._root / str(hid) + base = self._dir(hid) base.mkdir(parents=True, exist_ok=True) for n in itertools.count(self._next_guess(base)): stage = base / f"{STAGING_PREFIX}{n}" # a SIBLING of the destination @@ -606,7 +690,7 @@ def _require(self, hid: HandoffId, version: int, *, wanted: str) -> Path: a second home for that fact would pre-empt it. **It returns the day F-D1 moves the seal after output validation, and not before.** """ - path = version_dir(self._root, hid, version) + path = self._version_dir(hid, version) if not (path / MANIFEST_FILE).is_file(): have = self.list_versions(hid) allocated = path.is_dir() diff --git a/agent_sys/tests/cli/conftest.py b/agent_sys/tests/cli/conftest.py index 37c2acbb5..287a98ba2 100644 --- a/agent_sys/tests/cli/conftest.py +++ b/agent_sys/tests/cli/conftest.py @@ -20,10 +20,32 @@ import pytest from cli import build, package +from cli import main as cli_main from cli.stream import Stream from task_graph import Task, build_registry +@pytest.fixture(autouse=True) +def _no_real_agentsview_install(monkeypatch): + """**No test here downloads 45 MB.** + + `main(["run", ...])` reaches `_start_o11y`, and a test that patches only + `ensure_running` leaves `ensure_installed` real — which runs the recipe for + real. `tests/conftest.py` keeps that out of the operator's prefix; this + keeps it off the network, so the suite still passes offline and in seconds. + + It reports "already present", not "missing", so `_start_o11y` goes on to + reach `ensure_running` — which is what the failure-mode tests are about. A + test that is genuinely about installing patches this itself, afterwards. + """ + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr( + cli_main, "ensure_installed", + lambda prefix, install_item: Status(True, "agentsview already present (skip)"), + ) + + @pytest.fixture(scope="session") def package_root() -> Path: """The demo task package, found the way the CLI finds it. diff --git a/agent_sys/tests/cli/test_agentsview_flags.py b/agent_sys/tests/cli/test_agentsview_flags.py new file mode 100644 index 000000000..c71b4f1d5 --- /dev/null +++ b/agent_sys/tests/cli/test_agentsview_flags.py @@ -0,0 +1,466 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""The two flags, and the promise that o11y cannot fail a run.""" + +from __future__ import annotations + +import pytest + +from cli import main as cli_main + + +def test_the_port_flag_parses() -> None: + args = cli_main.parser().parse_args(["run", "--package", "pkg", "--agentsview-port", "9001"]) + assert args.agentsview_port == 9001 + + +def test_the_disable_flag_parses() -> None: + args = cli_main.parser().parse_args(["run", "--package", "pkg", "--no-agentsview"]) + assert args.no_agentsview is True + + +def test_the_default_is_enabled_and_unset() -> None: + args = cli_main.parser().parse_args(["run", "--package", "pkg"]) + assert args.no_agentsview is False + assert args.agentsview_port is None + + +def test_disabled_makes_no_external_call(monkeypatch) -> None: + called = [] + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: called.append(1)) + cli_main._start_o11y(port_flag=None, disabled=True) + assert called == [] + + +def _installed(reason: str): + """Stand in for `ensure_installed`, reporting one of its two ok-reasons.""" + from env_mgr.o11y.agentsview import Status + + return lambda prefix, install_item: Status(True, reason) + + +def test_a_fresh_install_says_so_exactly_once(monkeypatch, caplog) -> None: + """A 45 MB download nobody asked for must be visible when it happens.""" + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr(cli_main, "ensure_installed", _installed("installed agentsview")) + monkeypatch.setattr(cli_main, "ensure_running", lambda prefix, port: Status(False, "x")) + with caplog.at_level("INFO", logger="demo"): + cli_main._start_o11y(port_flag=9009, disabled=False) + notices = [r for r in caplog.records if r.levelname == "INFO" and "fetched" in r.message] + assert len(notices) == 1 + + +def test_an_install_that_was_already_satisfied_is_silent(monkeypatch, caplog) -> None: + """The notice fires on the install, not on the 500 runs after it. + + A line printed every time is noise, and noise is how a real warning gets + scrolled past. + """ + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr( + cli_main, "ensure_installed", _installed("agentsview already present (skip)") + ) + monkeypatch.setattr(cli_main, "ensure_running", lambda prefix, port: Status(False, "x")) + with caplog.at_level("INFO", logger="demo"): + cli_main._start_o11y(port_flag=9009, disabled=False) + assert [r for r in caplog.records if "fetched" in r.message] == [] + + +def test_a_failed_install_does_not_go_on_to_start_a_daemon(monkeypatch) -> None: + from env_mgr.o11y.agentsview import Status + + ran = [] + monkeypatch.setattr(cli_main, "ensure_installed", lambda prefix, install_item: Status(False, "no network")) + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: ran.append(1)) + assert cli_main._start_o11y(port_flag=None, disabled=False) is None + assert ran == [] + + +def test_the_installers_two_ok_messages_still_discriminate() -> None: + """A drift guard on the string `_start_o11y` reads. + + `ensure_installed` passes `Outcome.message` through verbatim, so the notice + can only tell "installed just now" from "already there" by that text. If + `BinInstaller.install` ever rephrases either branch the notice silently + stops firing — or starts firing on every run — and nothing else would + catch it. So both phrasings are taken from the real installer here. + """ + from env_mgr.installers.bin import BinInstaller + from env_mgr.o11y.agentsview import freshly_installed + from env_mgr.recipe import Item, Target + + target = Target(kind="prefix", name="t", path=".") + # A `check_cmd` that really answers a version, so `_satisfied` is true and + # `install` takes its skip branch for real rather than being asserted about. + satisfied = Item( + "bin", + "suggested", + "system", + spec={"name": "agentsview", "check_cmd": "echo agentsview 0.42.0"}, + ) + fresh = Item( + "bin", "suggested", "system", spec={"name": "agentsview", "check_cmd": "", "install": ":"} + ) + # **Both branches are driven for real; neither string is written here.** + # An earlier version fell back to a literal when the skip branch produced + # nothing (`... ] or [f"{name} already present (skip)"]`) and then asserted + # only that the result was truthy — so the test invented the very string it + # was meant to be guarding. Rephrasing `BinInstaller`'s skip message to + # "installed {name} (cached)" — the exact change that makes the 45 MB + # notice fire on *every* run — left it green. `satisfied` therefore gets a + # `check_cmd` that really succeeds, which is what reaches the skip branch. + (fresh_msg,) = [o.message for o in BinInstaller().install(fresh, target) if o.level == "ok"] + (skip_msg,) = [ + o.message for o in BinInstaller().install(satisfied, target) if o.level == "ok" + ] + + assert freshly_installed(fresh_msg) is True + assert freshly_installed(skip_msg) is False + + +def test_the_install_closure_runs_nothing_until_it_is_called(monkeypatch, tmp_path) -> None: + """Laziness is the property, not an implementation detail. + + `ensure_installed` takes a callable so the recipe does not run until it has + decided the install is wanted. A precomputed `Outcome` list would have + downloaded 45 MB before the `--dry-run` check could stop it, so "nothing + happened at construction time" is worth asserting directly. + """ + import env_mgr.runner + from env_mgr.prefix import Prefix + + ran = [] + monkeypatch.setattr( + env_mgr.runner, "run", lambda *a, **k: (ran.append((a, k)), ([], "ok"))[1] + ) + prefix = Prefix.resolve({"HOME": str(tmp_path)}) + + call = cli_main._install_item(prefix) + assert ran == [] # constructing it must not have run the installer + + call() + (args, _kw), = ran + target, _items, stage, filters = args[0], args[1], args[2], args[3] + assert stage == "install" + assert filters.item == "agentsview" + # The checked-in recipe's `target.path` is a placeholder; the caller is + # what points it at this prefix. + assert target.path == str(prefix.root) + + +def test_a_dry_run_installs_nothing_and_starts_nothing(monkeypatch) -> None: + """`--dry-run` downloading 45 MB would be a worse breach than the daemon.""" + called = [] + monkeypatch.setattr(cli_main, "ensure_installed", lambda *a, **k: called.append("install")) + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: called.append("run")) + monkeypatch.setattr(cli_main, "_dry_run", lambda args, stream, panel_url=None: 0) + assert cli_main.main(["run", "--package", "pkg", "--dry-run"]) == 0 + assert called == [] + + +def test_a_dry_run_starts_no_daemon(monkeypatch) -> None: + """`--dry-run` promises *resolve everything, do nothing*. + + Starting a resident daemon, creating `~/.infera_agent_sys` and writing a + `config.toml` are all side effects, and a dry run that leaves a daemon + behind has broken its only contract. + """ + called = [] + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: called.append(1)) + monkeypatch.setattr(cli_main, "_dry_run", lambda args, stream, panel_url=None: 0) + assert cli_main.main(["run", "--package", "pkg", "--dry-run"]) == 0 + assert called == [] + + +def test_clean_starts_no_daemon(monkeypatch) -> None: + """`--clean` removes every run and exits; a panel for it is pointless.""" + called = [] + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: called.append(1)) + monkeypatch.setattr(cli_main, "_clean", lambda args, stream, panel_url=None: 0) + assert cli_main.main(["run", "--package", "pkg", "--clean"]) == 0 + assert called == [] + + +def test_a_raising_side_car_does_not_reach_the_caller(monkeypatch) -> None: + """Belt and braces: even a bug inside ensure_running cannot fail a run.""" + + def boom(*a, **k): + raise RuntimeError("this must never escape") + + monkeypatch.setattr(cli_main, "ensure_running", boom) + assert cli_main._start_o11y(port_flag=None, disabled=False) is None + + +def test_the_side_car_never_exports_into_this_process(monkeypatch) -> None: + """`_start_o11y` reads `os.environ`; it must never write to it. + + The panel's whole isolation story is that `CLAUDE_CONFIG_DIR` lives in a + child's environment dict. A convenience `os.environ[...] = ...` on this + path would redirect a Claude Code the *user* started, which is the one + outcome the integration promised to avoid. + """ + import os + + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr( + cli_main, + "ensure_running", + lambda prefix, port: Status(True, "started", f"http://127.0.0.1:{port}"), + ) + before = dict(os.environ) + assert cli_main._start_o11y(port_flag=9009, disabled=False) == "http://127.0.0.1:9009" + assert dict(os.environ) == before + + +# --------------------------------------------------------------------------- # +# The readiness probe writes a transcript too + + +def test_the_readiness_probe_writes_into_the_prefix_not_the_users_claude_dir( + monkeypatch, +) -> None: + """`preflight_credentials` spawns `claude -p`, so it produces a transcript. + + It is not an agent child, so gate 1 (`assignment.environment`) never + covered it, and it was therefore writing one JSONL into the user's + `~/.claude/projects` on **every** run. The promise was that `agent_sys` + does not write there; the probe is a child like any other and gets the same + `CLAUDE_CONFIG_DIR`. + """ + import os + import subprocess + + from cli import environment as cli_env + from env_mgr.prefix import Prefix + + seen: dict[str, str] = {} + + def spy(cmd, **kw): + seen.update(kw.get("env") or {}) + return subprocess.CompletedProcess(cmd, 0, "ready", "") + + monkeypatch.setattr(cli_env.shutil, "which", lambda c: "/usr/bin/claude") + monkeypatch.setattr(cli_env.subprocess, "run", spy) + before = dict(os.environ) + + assert cli_env.preflight_credentials(cli="claude") == "ready" + + prefix = Prefix.resolve(os.environ) + assert seen["CLAUDE_CONFIG_DIR"] == str(prefix.claude_home) + # Built on top of the ambient environment, not instead of it: the child + # still needs PATH and HOME, and a bare `env={...}` would strip them. + assert seen["PATH"] == os.environ["PATH"] + assert seen["HOME"] == os.environ["HOME"] + # And the same law as everywhere else in this feature. + assert dict(os.environ) == before + assert "CLAUDE_CONFIG_DIR" not in os.environ + + +# --------------------------------------------------------------------------- # +# The wire from `main()` to the panel +# +# Every test above this line drives `_start_o11y` directly, and every test +# below `parser()` drives argparse directly. Neither touches the line that +# joins them -- measured: deleting the `_start_o11y(...)` call from `main()` +# outright left the whole `tests/cli` suite green (179 passed). Acceptance +# criterion 2, "deploying agent_sys starts it automatically", had no test. + + +def _main_with_o11y_spied(monkeypatch, argv: list[str]) -> dict: + """Run `main()` for real, with the run itself and the daemon stubbed.""" + from env_mgr.o11y.agentsview import Status + + seen: dict = {} + + def spy_running(prefix, port): + seen["port"] = port + return Status(True, "started", f"http://127.0.0.1:{port}") + + monkeypatch.setattr(cli_main, "ensure_installed", _installed("agentsview already present")) + monkeypatch.setattr(cli_main, "ensure_running", spy_running) + monkeypatch.setattr(cli_main, "_run", lambda args, stream, panel_url=None: 0) + seen["exit"] = cli_main.main(argv) + return seen + + +def test_a_plain_run_starts_the_panel_on_the_default_port(monkeypatch) -> None: + seen = _main_with_o11y_spied(monkeypatch, ["run", "--package", "pkg"]) + assert seen["port"] == 18888 + assert seen["exit"] == 0 + + +def test_the_port_flag_reaches_the_daemon(monkeypatch) -> None: + """`--agentsview-port` is parsed *and* used. Passing `None` here instead of + `args.agentsview_port` left every other test green.""" + seen = _main_with_o11y_spied( + monkeypatch, ["run", "--package", "pkg", "--agentsview-port", "9001"] + ) + assert seen["port"] == 9001 + + +def test_no_agentsview_reaches_the_call_site(monkeypatch) -> None: + seen = _main_with_o11y_spied(monkeypatch, ["run", "--package", "pkg", "--no-agentsview"]) + assert "port" not in seen + + +def test_a_dry_run_still_starts_nothing_through_main(monkeypatch) -> None: + seen = _main_with_o11y_spied(monkeypatch, ["run", "--package", "pkg", "--dry-run"]) + assert "port" not in seen + + +def test_show_never_reaches_the_panel(monkeypatch) -> None: + """`show` returns before the call site, so it has no flags to consult.""" + from env_mgr.o11y.agentsview import Status + + ran = [] + monkeypatch.setattr(cli_main, "ensure_running", lambda *a, **k: ran.append(1) or Status(False, "x")) + monkeypatch.setattr(cli_main, "_show", lambda args, stream, panel_url=None: 0) + assert cli_main.main(["show", "--package", "pkg"]) == 0 + assert ran == [] + + +# --------------------------------------------------------------------------- # +# The URL has to arrive somewhere the user can see + + +def test_the_panel_url_reaches_the_user(monkeypatch, capsys) -> None: + """Read the artefact, not the exit code. + + Both this and the fresh-install notice were `log.info`, and nothing in this + repository ever configures `logging` -- so the root logger sat at WARNING + with no handler and both lines were discarded. The failure warnings reached + stderr through `logging.lastResort`; the successes reached nobody. The + tests passed only because `caplog.at_level("INFO")` forced the level from + pytest's side, which is precisely the shape of a test that asserts the + program's intent rather than its output. + """ + _main_with_o11y_spied(monkeypatch, ["run", "--package", "pkg", "--agentsview-port", "9001"]) + assert "http://127.0.0.1:9001" in capsys.readouterr().out + + +def test_the_fresh_install_notice_reaches_the_user(monkeypatch, capsys) -> None: + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr(cli_main, "ensure_installed", _installed("installed agentsview")) + monkeypatch.setattr(cli_main, "ensure_running", lambda prefix, port: Status(False, "x")) + monkeypatch.setattr(cli_main, "_run", lambda args, stream, panel_url=None: 0) + cli_main.main(["run", "--package", "pkg"]) + out = capsys.readouterr().out + assert "agentsview" in out and "kenn-io/agentsview" in out + + +def test_a_skipped_panel_says_nothing_to_the_user(monkeypatch, capsys) -> None: + """A warning already went to the log; the event stream is for what *is*.""" + from env_mgr.o11y.agentsview import Status + + monkeypatch.setattr(cli_main, "ensure_installed", _installed("agentsview already present")) + monkeypatch.setattr(cli_main, "ensure_running", lambda prefix, port: Status(False, "port in use")) + monkeypatch.setattr(cli_main, "_run", lambda args, stream, panel_url=None: 0) + cli_main.main(["run", "--package", "pkg"]) + assert "127.0.0.1" not in capsys.readouterr().out + + +def test_the_readiness_probe_runs_from_a_directory_inside_the_prefix(monkeypatch) -> None: + """Where the probe runs decides which project its transcript lands in. + + AgentsView names a project after the session's cwd — resolving the git + *main repository* when there is one. The probe inherited the caller's cwd, + which is the checkout, so ten identical `Reply with exactly one word: + ready` sessions piled into the real `infera` project. A directory of its + own inside the prefix is one argument and no new state. + """ + import os + import subprocess + + from cli import environment as cli_env + from env_mgr.prefix import Prefix + + seen: dict = {} + + def spy(cmd, **kw): + seen.update(kw) + return subprocess.CompletedProcess(cmd, 0, "ready", "") + + monkeypatch.setattr(cli_env.shutil, "which", lambda c: "/usr/bin/claude") + monkeypatch.setattr(cli_env.subprocess, "run", spy) + + assert cli_env.preflight_credentials(cli="claude") == "ready" + + prefix = Prefix.resolve(os.environ) + assert seen["cwd"] == str(cli_env.probe_cwd(prefix)) + assert str(prefix.root) in seen["cwd"] + assert os.path.isdir(seen["cwd"]), "the child refuses a cwd that does not exist" + + +def test_the_probe_still_runs_when_its_directory_cannot_be_made(monkeypatch) -> None: + """A cwd we could not create is not a reason to refuse the whole run. + + `preflight_credentials` failing aborts everything, which is far worse than + a transcript filed under the wrong project. + """ + import subprocess + from pathlib import Path + + from cli import environment as cli_env + + def boom(*a, **k): + raise PermissionError("read-only prefix") + + def spy(cmd, **kw): + assert kw.get("cwd") is None, "no cwd beats a cwd that does not exist" + return subprocess.CompletedProcess(cmd, 0, "ready", "") + + monkeypatch.setattr(cli_env.shutil, "which", lambda c: "/usr/bin/claude") + monkeypatch.setattr(Path, "mkdir", boom) + monkeypatch.setattr(cli_env.subprocess, "run", spy) + + assert cli_env.preflight_credentials(cli="claude") == "ready" + + +# --------------------------------------------------------------------------- # +# The panel URL has to survive the trip from `_start_o11y` to the mapping call + + +def test_the_panel_url_reaches_the_run(monkeypatch) -> None: + """`main` discarded `_start_o11y`'s return value until this feature needed it. + + The mapping call lives in `_real_run` because the run id does not exist when + the panel starts, so the URL has to be threaded through two frames. Both + hops are one keyword each and neither is covered by anything else here. + """ + from env_mgr.o11y.agentsview import Status + + seen: dict = {} + + monkeypatch.setattr(cli_main, "ensure_installed", _installed("agentsview already present")) + monkeypatch.setattr( + cli_main, "ensure_running", + lambda prefix, port: Status(True, "started", f"http://127.0.0.1:{port}"), + ) + monkeypatch.setattr( + cli_main, "_real_run", + lambda args, stream, panel_url=None: seen.setdefault("url", panel_url) and 0 or 0, + ) + + cli_main.main(["run", "--package", "pkg", "--agentsview-port", "9001"]) + + assert seen["url"] == "http://127.0.0.1:9001" + + +def test_a_run_without_a_panel_passes_none_rather_than_failing(monkeypatch) -> None: + """o11y absent is a `None`, not an exception and not a missing argument.""" + from env_mgr.o11y.agentsview import Status + + seen: dict = {} + monkeypatch.setattr(cli_main, "ensure_installed", _installed("agentsview already present")) + monkeypatch.setattr(cli_main, "ensure_running", lambda prefix, port: Status(False, "port in use")) + monkeypatch.setattr( + cli_main, "_real_run", + lambda args, stream, panel_url="MISSING": seen.setdefault("url", panel_url) or 0, + ) + + assert cli_main.main(["run", "--package", "pkg"]) == 0 + assert seen["url"] is None diff --git a/agent_sys/tests/conftest.py b/agent_sys/tests/conftest.py new file mode 100644 index 000000000..52ad1aa42 --- /dev/null +++ b/agent_sys/tests/conftest.py @@ -0,0 +1,32 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""Suite-wide guards. Currently one: **the test suite owns no host state.**""" + +from __future__ import annotations + +import os +from collections.abc import Iterator + +import pytest + + +@pytest.fixture(autouse=True, scope="session") +def _prefix_is_never_the_operators(tmp_path_factory) -> Iterator[None]: + """`AGENT_SYS_HOME` points into `tmp` for the whole session. + + **Measured, and it was not theoretical.** A plain `pytest` wrote **128 MB** + into `$AGENT_SYS_HOME` — which, with no override, is the operator's real + `~/.infera_agent_sys`. Any test reaching `cli.main.main(["run", ...])` + without patching `ensure_installed` runs the actual recipe: a 45 MB download + and a 133 MB binary. Three files did it, and nothing said so. + + Session-scoped and autouse because the next test to do it will be written by + someone who did not know — a per-test patch is a rule people forget, and the + failure is a green suite that happens to have downloaded 45 MB. A test that + wants its own prefix still sets one; function-scoped `monkeypatch` wins. + """ + root = tmp_path_factory.mktemp("agent_sys_home") + with pytest.MonkeyPatch.context() as mp: + mp.setenv("AGENT_SYS_HOME", str(root)) + yield + assert "AGENT_SYS_HOME" not in os.environ or os.environ["AGENT_SYS_HOME"] != str(root) diff --git a/agent_sys/tests/env_mgr/test_grants.py b/agent_sys/tests/env_mgr/test_grants.py index 0565a98a4..d9472fab9 100644 --- a/agent_sys/tests/env_mgr/test_grants.py +++ b/agent_sys/tests/env_mgr/test_grants.py @@ -15,6 +15,7 @@ import pytest from env_mgr.fs.domain import DomainKind, DomainRegistry +from env_mgr.fs.layout import handoff_version_dir from env_mgr.grants import input_env, mode_for, output_env, resolve, resolve_all from env_mgr.protocols import Mode, UnresolvedGrant from task_graph.ids import HandoffId @@ -44,7 +45,7 @@ def test_a_kind_resolves_to_the_version_this_attempt_has(store: str) -> None: # `content/`, **not** `v2/`. Under §4.14 the manifest is the seal, so a # version directory granted whole lets its producer publish its own unsealed # version. A read grant gets one path because a consumer has nothing to claim. - assert granted.path == os.path.join(store, str(task.inputs[0]), "v2", "content") + assert granted.path == os.path.join(handoff_version_dir(store, task.inputs[0], 2), "content") assert granted.mode is Mode.READ_EXEC @@ -157,8 +158,8 @@ def test_resolve_all_flattens_every_grant(tmp_path: Path, store: str) -> None: granted = resolve_all(task, execution, ctx) assert len(granted) == 3 assert {g.path for g in granted} == { - os.path.join(store, str(a), "v0", "content"), - os.path.join(store, str(b), "v0", "content"), + os.path.join(handoff_version_dir(store, a, 0), "content"), + os.path.join(handoff_version_dir(store, b, 0), "content"), "/usr", } @@ -351,7 +352,7 @@ def test_an_output_is_exported_under_its_declared_kind(store: str) -> None: execution = Execution(attempt=0, output_versions={hid: 0}) assert output_env(task, execution, store) == { - "AGENT_SYS_OUTPUT_FACTS": os.path.join(store, str(hid), "v0", "content"), + "AGENT_SYS_OUTPUT_FACTS": os.path.join(handoff_version_dir(store, hid, 0), "content"), } diff --git a/agent_sys/tests/env_mgr/test_layout.py b/agent_sys/tests/env_mgr/test_layout.py index 0b4e60f44..ab2c070af 100644 --- a/agent_sys/tests/env_mgr/test_layout.py +++ b/agent_sys/tests/env_mgr/test_layout.py @@ -171,6 +171,85 @@ def test_find_zone_dir_picks_the_latest_attempt(domains: DomainRegistry) -> None assert layout.find_zone_dir(domains.storage_root(), task.id) == latest.root +# ------------------------------------------------- the label on a directory +# +# A run tree is something a person reads. Every directory `agent_sys` creates +# now leads with what it *is* and carries the name the system already knew, so +# `ls` answers "which task is this" without a lookup. The tests below hold the +# two properties that make the label safe: nothing resolves through it, and a +# tree written before it existed still resolves. + + +def test_a_zone_is_named_after_its_closure(domains: DomainRegistry) -> None: + task = Task(closure="describe") + zone = layout.create(task, task.push_execution(), domains) + assert os.path.basename(zone.root).startswith(f"task.describe.{task.id}.") + + +def test_a_task_with_no_closure_keeps_the_unlabelled_name(domains: DomainRegistry) -> None: + """The label is optional and its absence is not a placeholder: a task that + has no closure gets exactly the name it got before labels existed.""" + task = Task() + zone = layout.create(task, task.push_execution(), domains) + assert os.path.basename(zone.root).startswith(f"task.{task.id}.") + + +def test_a_closure_name_cannot_add_a_field(domains: DomainRegistry) -> None: + """`find_zone_dir` reads the attempt from ``parts[-2]``, so a ``.`` in a + closure name would make it read the wrong field. The slug is what stops it.""" + task = Task(closure="a.b/c d") + execution = task.push_execution() + zone = layout.create(task, execution, domains) + name = os.path.basename(zone.root) + assert name == f"task.a-b-c-d.{task.id}.0.{name.rsplit('.', 1)[-1]}" + assert layout.find_zone_dir(domains.storage_root(), task.id) == zone.root + + +def test_find_zone_dir_still_finds_an_unlabelled_zone(domains: DomainRegistry) -> None: + """The compatibility claim, asserted rather than argued: a zone directory in + the shape written before labels existed is still this task's.""" + task = Task() + legacy = os.path.join(domains.storage_root(), f"task.{task.id}.0.deadbeef") + os.makedirs(legacy) + assert layout.find_zone_dir(domains.storage_root(), task.id) == legacy + + +def test_a_validation_zone_is_named_after_its_closure( + domains: DomainRegistry, +) -> None: + task = Task(closure="describe") + layout.create(task, task.push_execution(), domains) + root = layout.validation_zone(task, "output_validation", domains) + assert os.path.basename(root).startswith(f"validation.describe.{task.id}.output_validation.") + + +def test_a_staged_input_keeps_the_store_directory_s_name( + domains: DomainRegistry, tmp_path: Path +) -> None: + """The staged copy inherits the label instead of recomputing it, so a body's + ``materials/`` reads like the store and no kind map has to be threaded + through `stage`.""" + from task_graph.ids import HandoffId + + from .stubs import context + + store = tmp_path / "store" + hid = HandoffId.new() + published = store / f"handoff.facts.{hid}" / "v1" / "content" + published.mkdir(parents=True) + (published / "result.json").write_text("{}") + + task = Task(inputs=[hid]) + execution = task.push_execution() + execution.input_versions = {hid: 1} + zone = layout.create(task, execution, domains) + + ctx = context(domains=domains, store_root=str(store)) + staged = layout.stage_handoffs(task, execution, zone, ctx) + + assert Path(staged[hid]).parent.name == f"handoff.facts.{hid}" + + def test_a_parent_without_a_zone_is_an_error(domains: DomainRegistry) -> None: orphan = Task(parent=Task().id) with pytest.raises(ValueError, match="has no zone under"): diff --git a/agent_sys/tests/env_mgr/test_material.py b/agent_sys/tests/env_mgr/test_material.py new file mode 100644 index 000000000..2ffefb94a --- /dev/null +++ b/agent_sys/tests/env_mgr/test_material.py @@ -0,0 +1,181 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""`material.deploy` — the zone's `config/`, and the one hole in it. + +The zone-local `CLAUDE_CONFIG_DIR` is deliberate and stays (see `material.py`'s +own comment). But it also relocated the *transcripts*, and the o11y panel reads +one fixed directory. Measured on demo2: nine agent transcripts landed under +`/config/projects/` and none reached the prefix, so the panel showed +nothing. These tests hold the seam that fixes it — `projects` alone is shared — +and the surrounding behaviour it must not disturb. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path + +import pytest + +from env_mgr import material +from env_mgr.fs.zone import Zone +from env_mgr.prefix import Prefix + +from .stubs import AgentSpec + + +@pytest.fixture +def prefix(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Prefix: + """A prefix of our own, so no test touches the operator's `~`.""" + root = tmp_path / "prefix" + monkeypatch.setenv("AGENT_SYS_HOME", str(root)) + return Prefix.resolve(os.environ) + + +def zone_at(tmp_path: Path, name: str) -> Zone: + root = tmp_path / "zones" / name + root.mkdir(parents=True) + return Zone(task_id=name, attempt=0, root=str(root)) + + +def test_config_projects_links_to_the_prefix(tmp_path: Path, prefix: Prefix) -> None: + """Gate 1 of the design's four, in the pipeline rather than only in a probe.""" + zone = zone_at(tmp_path, "a") + + env = material.deploy(AgentSpec(), zone) + + link = Path(env["CLAUDE_CONFIG_DIR"]) / "projects" + assert link.is_symlink(), "a real directory here is the bug this fixes" + assert link.resolve() == (prefix.claude_home / "projects").resolve() + + +def test_the_prefix_target_is_created_when_absent(tmp_path: Path, prefix: Prefix) -> None: + """`env_mgr`'s deploy path creates it, but a task may run without one.""" + assert not (prefix.claude_home / "projects").exists() + + material.deploy(AgentSpec(), zone_at(tmp_path, "a")) + + assert (prefix.claude_home / "projects").is_dir() + + +def test_two_zones_write_under_the_prefix_in_different_slugs( + tmp_path: Path, prefix: Prefix +) -> None: + """Sharing one physical `projects/` cannot collide: Claude Code names each + subdirectory after the slugified cwd, and every attempt has its own zone.""" + envs = [material.deploy(AgentSpec(), zone_at(tmp_path, n)) for n in ("a", "b")] + + for i, env in enumerate(envs): + slug = Path(env["CLAUDE_CONFIG_DIR"]) / "projects" / f"-slug-{i}" + slug.mkdir() + (slug / "session.jsonl").write_text("{}\n") + + landed = sorted(p.parent.name for p in (prefix.claude_home / "projects").glob("*/*.jsonl")) + assert landed == ["-slug-0", "-slug-1"] + + +def test_a_wrong_link_is_repaired(tmp_path: Path, prefix: Prefix) -> None: + """Idempotence has to cover the stale case, not just the correct one.""" + zone = zone_at(tmp_path, "a") + config = Path(zone.root) / material.CONFIG_DIR + config.mkdir() + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (config / "projects").symlink_to(elsewhere) + + material.deploy(AgentSpec(), zone) + + assert (config / "projects").resolve() == (prefix.claude_home / "projects").resolve() + + +def test_deploy_is_idempotent_over_a_correct_link(tmp_path: Path, prefix: Prefix) -> None: + zone = zone_at(tmp_path, "a") + material.deploy(AgentSpec(), zone) + material.deploy(AgentSpec(), zone) + + link = Path(zone.root) / material.CONFIG_DIR / "projects" + assert link.is_symlink() + assert link.resolve() == (prefix.claude_home / "projects").resolve() + + +def test_an_empty_directory_there_is_replaced(tmp_path: Path, prefix: Prefix) -> None: + """`rmdir` refuses a non-empty directory, so this branch cannot lose data.""" + zone = zone_at(tmp_path, "a") + config = Path(zone.root) / material.CONFIG_DIR + (config / "projects").mkdir(parents=True) + + material.deploy(AgentSpec(), zone) + + assert (config / "projects").is_symlink() + + +def test_a_populated_directory_there_is_left_alone_with_a_warning( + tmp_path: Path, prefix: Prefix, caplog: pytest.LogCaptureFixture +) -> None: + """Transcripts already written are somebody's evidence. The panel losing a + zone is cheaper than deleting one, so this warns and proceeds.""" + zone = zone_at(tmp_path, "a") + config = Path(zone.root) / material.CONFIG_DIR + (config / "projects").mkdir(parents=True) + (config / "projects" / "kept.jsonl").write_text("{}\n") + + with caplog.at_level(logging.WARNING): + material.deploy(AgentSpec(), zone) + + assert (config / "projects" / "kept.jsonl").is_file() + assert not (config / "projects").is_symlink() + assert any("projects" in r.message for r in caplog.records) + + +def test_a_failed_link_does_not_raise( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """**The law of this whole feature**: a broken panel never breaks a run. + + The unreachable target is a prefix rooted *inside a regular file*, so + `target.mkdir(parents=True)` raises `NotADirectoryError`. This used to + delete `$HOME` and `AGENT_SYS_HOME` instead, because `Prefix.resolve` then + raised `KeyError` — but that was the bug, not the fixture: two other call + sites did not catch it and a run with no `$HOME` died on a feature it never + asked for. `resolve` is total now, so the failure has to come from the + filesystem, which is where a real one would come from anyway. + """ + blocker = tmp_path / "not-a-directory" + blocker.write_text("") + monkeypatch.setenv("AGENT_SYS_HOME", str(blocker / "prefix")) + + with caplog.at_level(logging.WARNING): + env = material.deploy(AgentSpec(), zone_at(tmp_path, "a")) + + assert env["CLAUDE_CONFIG_DIR"] + assert any("projects" in r.message for r in caplog.records) + + +def test_the_rest_of_the_config_directory_is_untouched( + tmp_path: Path, prefix: Prefix +) -> None: + """The credential/settings mechanism is why the zone-local config exists. + Linking one subdirectory must not disturb it.""" + rule = tmp_path / "rules" / "style.md" + rule.parent.mkdir() + rule.write_text("# house style\n") + zone = zone_at(tmp_path, "a") + + env = material.deploy(AgentSpec(rules=(str(rule),)), zone) + + config = Path(zone.root) / material.CONFIG_DIR + assert env["CLAUDE_CONFIG_DIR"] == str(config) + assert (config / "rules" / "style.md").read_text() == "# house style\n" + assert env["CLAUDE_CODE_TMPDIR"] == os.path.join(zone.root, "tmp") + assert env["TMPDIR"] == os.path.join(zone.root, "tmp") + + +def test_deploy_does_not_mutate_this_process_environment( + tmp_path: Path, prefix: Prefix +) -> None: + before = dict(os.environ) + + material.deploy(AgentSpec(), zone_at(tmp_path, "a")) + + assert dict(os.environ) == before diff --git a/agent_sys/tests/env_mgr/test_o11y_agentsview.py b/agent_sys/tests/env_mgr/test_o11y_agentsview.py new file mode 100644 index 000000000..185da411a --- /dev/null +++ b/agent_sys/tests/env_mgr/test_o11y_agentsview.py @@ -0,0 +1,1093 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""The AgentsView side-car: which port, and every way it is allowed to fail.""" + +from __future__ import annotations + +import contextlib +import http.server +import json +import os +import socket +import subprocess +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest + +from env_mgr.o11y.agentsview import agentsview +from env_mgr.prefix import Prefix + + +def test_the_default_port_is_18888() -> None: + assert agentsview.DEFAULT_PORT == 18888 + assert agentsview.resolve_port(None, {}) == 18888 + + +def test_the_environment_beats_the_default() -> None: + assert agentsview.resolve_port(None, {"AGENTSVIEW_PORT": "9001"}) == 9001 + + +def test_the_flag_beats_the_environment() -> None: + assert agentsview.resolve_port(9002, {"AGENTSVIEW_PORT": "9001"}) == 9002 + + +def test_an_unparseable_environment_value_falls_back_to_the_default() -> None: + assert agentsview.resolve_port(None, {"AGENTSVIEW_PORT": "not-a-port"}) == 18888 + + +@pytest.mark.parametrize("bad", [70000, -1, 0, 65536]) +def test_an_out_of_range_port_falls_back_to_the_default(bad: int, caplog) -> None: + """Range, not just parseability. + + `socket.bind` answers an out-of-range port with `OverflowError`, which is + neither `OSError` nor anything else `port_is_free` catches — so an + unchecked value escapes this module entirely and is caught only by the + CLI's blanket backstop, which is meant to be a fuse and not the mechanism. + `0` is in the list for a different reason: it binds successfully and then + hands the port choice to AgentsView's own auto-discovery, which is exactly + the delegation this module's header says it exists to prevent. + """ + with caplog.at_level("WARNING"): + assert agentsview.resolve_port(bad, {}) == 18888 + assert agentsview.resolve_port(None, {"AGENTSVIEW_PORT": str(bad)}) == 18888 + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 2 + + +@pytest.mark.parametrize("bad", [70000, -1]) +def test_port_is_free_does_not_raise_on_an_impossible_port(bad: int) -> None: + """`OverflowError` is not `OSError`; the module's law covers both.""" + assert agentsview.port_is_free(bad) is False + + +def test_port_is_free_says_no_when_something_is_listening() -> None: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen(1) + taken = s.getsockname()[1] + assert agentsview.port_is_free(taken) is False + + +def test_port_is_free_says_yes_when_nothing_is() -> None: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + assert agentsview.port_is_free(free) is True + + +@pytest.fixture() +def prefix(tmp_path: Path) -> Prefix: + p = Prefix.resolve({"HOME": str(tmp_path)}) + p.create() + return p + + +def _fake_binary(prefix: Prefix, body: str) -> None: + exe = prefix.bin / "agentsview" + exe.write_text("#!/bin/sh\n" + body) + exe.chmod(0o755) + + +def test_a_taken_port_is_one_warning_and_a_skip(prefix, caplog) -> None: + _fake_binary(prefix, "exit 0\n") + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen(1) + taken = s.getsockname()[1] + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=taken) + assert status.running is False + assert "port" in status.reason + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_missing_binary_is_a_warning_and_a_skip(prefix, caplog) -> None: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=free) + assert status.running is False + assert "not installed" in status.reason + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_daemon_that_exits_nonzero_is_a_warning_and_a_skip(prefix, caplog) -> None: + _fake_binary(prefix, "echo boom >&2\nexit 3\n") + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=free) + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_launch_that_times_out_is_a_warning_and_a_skip(prefix, caplog, monkeypatch) -> None: + _fake_binary(prefix, "sleep 30\n") + monkeypatch.setattr(agentsview, "LAUNCH_TIMEOUT_S", 0.2) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=free) + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_no_failure_mode_raises(prefix, monkeypatch) -> None: + """The whole point of the module, asserted directly.""" + monkeypatch.setattr(agentsview, "LAUNCH_TIMEOUT_S", 0.2) + monkeypatch.setattr(agentsview, "HEALTH_TIMEOUT_S", 0.2) + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + for body in ("exit 3\n", "sleep 30\n"): + _fake_binary(prefix, body) + agentsview.ensure_running(prefix, port=1) # privileged port: bind fails + agentsview.ensure_running(prefix, port=0) + + +def test_a_successful_launch_reports_the_url(prefix, monkeypatch) -> None: + _fake_binary(prefix, "exit 0\n") + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + monkeypatch.setattr(agentsview, "_wait_for_health", lambda url, timeout: True) + status = agentsview.ensure_running(prefix, port=free) + assert status.running is True + assert status.url == f"http://127.0.0.1:{free}" + + +def test_the_child_gets_the_prefix_environment_and_os_environ_is_untouched( + prefix, monkeypatch +) -> None: + """`AGENTSVIEW_DATA_DIR` reaches the child; this process never learns it.""" + seen: dict[str, str] = {} + + def spy(cmd, env=None, **kw): # noqa: ANN001 + seen.update(env or {}) + return subprocess.CompletedProcess(cmd, 0, "", "") + + _fake_binary(prefix, "exit 0\n") + monkeypatch.setattr(subprocess, "run", spy) + monkeypatch.setattr(agentsview, "_wait_for_health", lambda url, timeout: True) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + agentsview.ensure_running(prefix, port=free) + assert seen["AGENTSVIEW_DATA_DIR"] == str(prefix.agentsview_data) + assert seen["CLAUDE_PROJECTS_DIR"] == str(prefix.claude_home / "projects") + assert "AGENTSVIEW_DATA_DIR" not in __import__("os").environ + # HOME=$AGENT_SYS_HOME is a second, stronger gate 3 -- not `disabled_agents`, + # an unrelated accident of building the env dict this way. AgentsView + # computes every provider's *default* session root from HOME, so pointing + # it at the prefix means every root it could scan resolves inside the + # prefix regardless of whether that provider is even in OTHER_PROVIDERS. + # Measured (`doctor sync` run with exactly this environment): 122 roots + # listed, 0 outside the prefix. Unlike the denylist, this gate cannot go + # stale when upstream adds a provider we have never heard of -- it needs + # no list at all. A one-line "tidy up the env dict" edit would silently + # restore every default root to the user's real home with nothing else + # here going red; this assertion is the only thing holding that line. + assert seen["HOME"] == str(prefix.root) + + +def test_ensure_running_passes_replace_to_serve(prefix, monkeypatch) -> None: + """Measured directly (scratch/port_repro, reported to team lead): without + `--replace`, `serve --background --port N` silently attaches to any + daemon already running for this `AGENTSVIEW_DATA_DIR` and reports *its* + port, ignoring `N` entirely -- exit 0, no error, and our own health check + on `N` then times out (correctly producing a warning, but only after + burning the full launch+health timeout, and only ever reporting failure, + never actually landing on the port we asked for). `--replace` is the flag + that makes our chosen port actually take effect regardless of any stray + daemon left over from an earlier run. + """ + seen_cmd: list[str] = [] + + def spy(cmd, env=None, **kw): # noqa: ANN001 + seen_cmd.extend(cmd) + return subprocess.CompletedProcess(cmd, 0, "", "") + + _fake_binary(prefix, "exit 0\n") + monkeypatch.setattr(subprocess, "run", spy) + monkeypatch.setattr(agentsview, "_wait_for_health", lambda url, timeout: True) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + agentsview.ensure_running(prefix, port=free) + assert "--replace" in seen_cmd + + +def test_ensure_running_disables_the_other_providers(prefix, monkeypatch) -> None: + """The launch path applies the pinned denylist, not just the writer. + + `write_config` is tested in isolation against an arbitrary list; that says + nothing about whether the one call that matters passes `OTHER_PROVIDERS`. + Changing this call site to `write_config(prefix, ())` left every other + test green — the whole `OTHER_PROVIDERS` / `check_disabled_agents` + apparatus exists to serve this one line. + """ + _fake_binary(prefix, "exit 0\n") + monkeypatch.setattr(agentsview, "_wait_for_health", lambda url, timeout: True) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + agentsview.ensure_running(prefix, port=free) + written = (prefix.agentsview_data / "config.toml").read_text() + for name in agentsview.OTHER_PROVIDERS: + assert f'"{name}"' in written + assert '"claude"' not in written + + +@contextmanager +def _server_on_a_port(body: bytes, content_type: str) -> Iterator[int]: + """A real HTTP server on a real ephemeral port, answering everything alike. + + Real rather than mocked because the thing under test is a decision about a + *stranger's* process, and a mock of the stranger is a mock of exactly the + party we do not control. + """ + + class Handler(http.server.BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + self.send_response(200) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args: object) -> None: + """pytest's captured output is not a web server access log.""" + + srv = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Handler) + thread = threading.Thread(target=srv.serve_forever, daemon=True) + thread.start() + try: + yield srv.server_address[1] + finally: + srv.shutdown() + srv.server_close() + thread.join(timeout=5) + + +def _claim_port(prefix: Prefix, port: int, pid: int | None = None) -> None: + """Forge the evidence that *we* started the daemon on `port`. + + The shape is AgentsView's own, copied from a real + `state/agentsview/daemon..json` written by v0.42.0 — not invented + here, because a test that forges a record the binary does not write proves + only that our parser reads our own fiction. + """ + pid = os.getpid() if pid is None else pid + (prefix.agentsview_data / f"daemon.{pid}.json").write_text( + json.dumps( + { + "pid": pid, + "process_identity_v2": "linux-v1:test:4026531836:1", + "network": "tcp", + "address": f"127.0.0.1:{port}", + "service": "agentsview", + "version": "v0.42.0", + "metadata": {"host": "127.0.0.1", "port": str(port)}, + } + ) + ) + + +def _a_dead_pid() -> int: + """A pid that is certainly not running. + + Found by asking the kernel rather than by picking a large number: pids wrap, + and a hardcoded one is a test that fails on a busy machine in a year. + """ + for candidate in range(_MAX_PID, 1, -1): + try: + os.kill(candidate, 0) + except ProcessLookupError: + return candidate + except OSError: + continue + raise AssertionError("no dead pid available") # pragma: no cover + + +_MAX_PID = 4194304 + + +def test_a_stranger_answering_http_on_our_port_is_not_adopted( + prefix, caplog, monkeypatch +) -> None: + """A 200 is not an identity. + + The port file is written here deliberately, so the *only* thing that can + reject this server is the identity probe. Without it this test would pass + on the ownership check alone and prove nothing about `/api/v1/agents`. + """ + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b"some other service", "text/html") as port: + _claim_port(prefix, port) + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + assert status.url is None + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_someone_elses_agentsview_is_not_adopted(prefix, caplog, monkeypatch) -> None: + """A genuine AgentsView we did not start shows the user's whole machine. + + Adopting it would satisfy the health check and break the one requirement + the panel exists for — that it lists agent_sys's sessions and no others. + """ + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b'[{"name":"claude-code"}]', "application/json") as port: + assert not list(prefix.agentsview_data.glob("daemon.*.json")) # control + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + assert status.url is None + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_our_own_resident_daemon_is_reused(prefix, caplog, monkeypatch) -> None: + """Both gates pass: it answers as AgentsView, and we recorded starting it.""" + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b'[{"name":"claude-code"}]', "application/json") as port: + _claim_port(prefix, port) + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is True + assert status.reason == "already running" + assert status.url == f"http://127.0.0.1:{port}" + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +def test_a_daemon_record_for_a_dead_pid_is_not_ownership( + prefix, caplog, monkeypatch +) -> None: + """The stale-record case, which is the whole reason the pid is checked. + + Our daemon is killed (reboot, OOM, `kill -9`) without the chance to remove + its own record; the user then starts *their* AgentsView on the same port. + Measured on a real v0.42.0: a clean `serve stop` **does** remove + `daemon..json`, so only an unclean death leaves one behind — and in + that case the record names a pid that is gone. Without the liveness check + the operator is handed a URL to a panel listing every session on their + machine, labelled as theirs, with no warning at all. + """ + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b'[{"name":"claude-code"}]', "application/json") as port: + _claim_port(prefix, port, pid=_a_dead_pid()) + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + assert status.url is None + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_daemon_record_for_another_port_is_not_ownership( + prefix, caplog, monkeypatch +) -> None: + """Ownership is per-port. A live daemon of ours on 19000 says nothing + about who holds 19001.""" + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b'[{"name":"claude-code"}]', "application/json") as port: + _claim_port(prefix, port + 1) + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_malformed_daemon_record_is_not_ownership(prefix, caplog, monkeypatch) -> None: + """Unreadable evidence is a `no`: the safe answer to *is this ours* is the + one that declines to adopt a stranger.""" + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with _server_on_a_port(b'[{"name":"claude-code"}]', "application/json") as port: + (prefix.agentsview_data / "daemon.123.json").write_text("{not json") + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_our_own_daemon_that_stops_answering_says_so_and_does_not_blame_a_stranger( + prefix, caplog, monkeypatch +) -> None: + """The two busy-port cases are told apart, because the fixes differ. + + "Something else has your port" sends the operator hunting for a process + that does not exist when the truth is that *our own* daemon is wedged. + """ + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.2) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + s.listen(1) + port = s.getsockname()[1] + _claim_port(prefix, port) + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "something else" not in warnings[0].getMessage() + + +def test_a_health_check_timeout_is_a_warning_and_a_skip( + prefix, caplog, monkeypatch +) -> None: + """The failure mode named in `CLAUDE.md` that had no test. + + The binary launches cleanly and exits 0 (`serve --background` daemonises, + so that is success) but nothing ever answers on the port. + """ + _fake_binary(prefix, "exit 0\n") + monkeypatch.setattr(agentsview, "HEALTH_TIMEOUT_S", 0.2) + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + free = s.getsockname()[1] + with caplog.at_level("WARNING"): + status = agentsview.ensure_running(prefix, port=free) + assert status.running is False + assert status.reason == "health check timed out" + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_a_truncated_identity_response_is_not_an_identity(prefix, monkeypatch) -> None: + """`http.client.IncompleteRead` is neither `OSError` nor `ValueError`. + + urllib wraps connect-time errors in `URLError`, but not errors raised + during `read()`, so without `HTTPException` in the caught tuple this puts + an exception straight through `_identifies_as_agentsview`. + + **Chunked, and measured rather than assumed.** A truncated + `Content-Length` body does *not* reach this: `HTTPResponse.read(amt)` + returns a short read for that case and raises nothing (checked directly — + the first version of this test used it and passed against the unfixed + code). Chunked framing is the path that raises, and it is the framing a Go + HTTP server uses whenever it does not set a length — which is to say, a + realistic stranger. + """ + monkeypatch.setattr(agentsview, "REUSE_PROBE_TIMEOUT_S", 0.0) + + def liar(conn: socket.socket) -> None: + conn.recv(4096) + conn.sendall( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\n" + b"Transfer-Encoding: chunked\r\n\r\n64\r\n[{}]" # promises 0x64, sends 4 + ) + conn.close() + + with _raw_server(liar) as port: + _claim_port(prefix, port) + status = agentsview.ensure_running(prefix, port=port) + assert status.running is False + + +@contextmanager +def _raw_server(answer) -> Iterator[int]: # noqa: ANN001 + """A socket server that answers with bytes we choose, header included. + + `http.server` cannot send a `Content-Length` it does not honour, and a + lying one is exactly the stranger this test needs. + """ + srv = socket.socket() + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", 0)) + srv.listen(5) + stop = threading.Event() + + def loop() -> None: + srv.settimeout(0.2) + while not stop.is_set(): + try: + conn, _ = srv.accept() + except OSError: + continue + with contextlib.suppress(OSError): + answer(conn) + + thread = threading.Thread(target=loop, daemon=True) + thread.start() + try: + yield srv.getsockname()[1] + finally: + stop.set() + thread.join(timeout=5) + srv.close() + + +def test_the_recipe_installs_agentsview_as_an_optional_bin_item() -> None: + """`suggested`, not `required`: install failure must stay a warning.""" + from env_mgr.recipe import load_recipe + + _target, items = load_recipe("env_mgr/recipes/agentsview.o11y.yaml") + (item,) = [i for i in items if i.spec.get("name") == "agentsview"] + assert item.installer == "bin" + assert item.importance == "suggested" + assert item.spec["check_cmd"] == "$AGENT_SYS_HOME/bin/agentsview --version" + assert "o11y" in item.tags + + +def test_the_recipe_pins_a_version_so_check_cmd_is_compared_against_something() -> None: + """`satisfies(actual, None)` accepts anything; a bare `version:` fixes that.""" + from env_mgr.recipe import load_recipe + from env_mgr.versions import satisfies + + _target, items = load_recipe("env_mgr/recipes/agentsview.o11y.yaml") + (item,) = [i for i in items if i.spec.get("name") == "agentsview"] + assert item.version == "0.42.0" + assert satisfies("0.42.0", item.version) is True + assert satisfies("0.41.0", item.version) is False + + +def test_the_recipe_install_command_uses_a_private_tempfile_and_verifies_checksum() -> None: + """The three amendments: no fixed shared tempfile, and a real checksum gate.""" + from env_mgr.recipe import load_recipe + + _target, items = load_recipe("env_mgr/recipes/agentsview.o11y.yaml") + (item,) = [i for i in items if i.spec.get("name") == "agentsview"] + install = item.spec["install"] + assert "mktemp" in install + assert "/tmp/av.tgz" not in install + assert "sha256sum" in install + assert "SHA256SUMS" in install + + +def _recipe_install_command() -> str: + from env_mgr.recipe import load_recipe + + _target, items = load_recipe("env_mgr/recipes/agentsview.o11y.yaml") + (item,) = [i for i in items if i.spec.get("name") == "agentsview"] + return item.spec["install"] + + +def _stub_curl(tmp_path: Path, payload: Path, sums_text: str) -> Path: + """A `curl` that serves two local files by the name being fetched. + + The recipe is a shell string, and the only honest way to test a shell + string is to run it. Everything real stays real — `mktemp`, `tar`, + `sha256sum`, `awk` — and only the network is replaced. + """ + (tmp_path / "SHA256SUMS").write_text(sums_text) + stub = tmp_path / "stub" + stub.mkdir() + curl = stub / "curl" + curl.write_text( + "#!/bin/sh\n" + "url=; out=;\n" + 'while [ $# -gt 0 ]; do\n' + ' case "$1" in\n' + ' -o) out=$2; shift 2 ;;\n' + ' -*) shift ;;\n' + ' *) url=$1; shift ;;\n' + " esac\n" + "done\n" + 'case "$url" in\n' + f' *SHA256SUMS) cp "{tmp_path}/SHA256SUMS" "$out" ;;\n' + f' *.tar.gz) cp "{payload}" "$out" ;;\n' + " *) exit 22 ;;\n" + "esac\n" + ) + curl.chmod(0o755) + return stub + + +@pytest.fixture() +def _tarball(tmp_path: Path) -> Path: + """A real gzipped tar holding one file called `agentsview`.""" + import tarfile + + payload = tmp_path / "agentsview" + payload.write_text("#!/bin/sh\necho v0.42.0\n") + payload.chmod(0o755) + tgz = tmp_path / "release.tar.gz" + with tarfile.open(tgz, "w:gz") as tar: + tar.add(payload, arcname="agentsview") + return tgz + + +def _run_recipe_install(tmp_path: Path, stub: Path) -> subprocess.CompletedProcess[str]: + home = tmp_path / "prefix" + home.mkdir(exist_ok=True) + return subprocess.run( # noqa: S602 - the recipe *is* a shell string + _recipe_install_command(), + shell=True, + cwd=home, + env={"PATH": f"{stub}:{os.environ['PATH']}", "AGENT_SYS_HOME": str(home)}, + capture_output=True, + text=True, + timeout=60, + ) + + +def _sha256(path: Path) -> str: + import hashlib + + return hashlib.sha256(path.read_bytes()).hexdigest() + + +_ASSET = "agentsview_0.42.0_linux_amd64.tar.gz" + + +def test_the_recipe_install_command_installs_when_the_checksum_matches( + tmp_path: Path, _tarball: Path +) -> None: + """The positive control, and it is not decoration. + + Without it the negative test below passes just as well against a recipe + that never installs anything at all. + """ + stub = _stub_curl(tmp_path, _tarball, f"{_sha256(_tarball)} {_ASSET}\n") + done = _run_recipe_install(tmp_path, stub) + assert done.returncode == 0, done.stderr + assert (tmp_path / "prefix" / "bin" / "agentsview").is_file() + + +def test_the_recipe_install_command_refuses_a_wrong_checksum( + tmp_path: Path, _tarball: Path +) -> None: + stub = _stub_curl(tmp_path, _tarball, f"{'0' * 64} {_ASSET}\n") + done = _run_recipe_install(tmp_path, stub) + assert done.returncode != 0 + assert not (tmp_path / "prefix" / "bin" / "agentsview").exists() + + +def test_the_recipe_install_command_refuses_a_checksum_it_could_not_find( + tmp_path: Path, _tarball: Path +) -> None: + """The case the `-n` test was written for, and the one it did not cover. + + `set -e` does not fire for a command that is not the last in an `&&` list, + so `[ -n "$expected" ] && [ "$expected" = "$actual" ]` short-circuits and + *continues* when `awk` matched no line — upstream renaming an asset or + changing the `SHA256SUMS` format would have installed an unverified + binary. A mismatch aborted correctly; the missing-checksum case did not. + """ + stub = _stub_curl(tmp_path, _tarball, f"{_sha256(_tarball)} some-other-file.tar.gz\n") + done = _run_recipe_install(tmp_path, stub) + assert done.returncode != 0 + assert not (tmp_path / "prefix" / "bin" / "agentsview").exists() + + +# --- ensure_installed: the recipe item, actually installed --------------- + + +@pytest.fixture() +def _no_leftover_environ(monkeypatch) -> None: + """`AGENT_SYS_HOME` must not already be ambient, or a leak would go unseen.""" + monkeypatch.delenv("AGENT_SYS_HOME", raising=False) + + +def _fake_run_cmd(*, version_rc: int, install_rc: int, seen_home: list[str | None]): + """Stands in for both `installers.base.subprocess.run` (a shell *string*, + used by `run_cmd`) and `agentsview.py`'s own `subprocess.run` calls (an + argv *list*, used by `check_disabled_agents`/`ensure_running`) -- the same + monkeypatch target (`subprocess.run` is one shared module attribute) + serves both call shapes, so this fake must accept the kwargs either + caller passes (`env=`, `timeout=`) even though it only inspects a few. + + Distinguishes calls by content: the `--version` probe (`_satisfied`) vs. + the recipe's `install:` body vs. a `health` validation probe (always + reports success here -- there is a dedicated test for + `check_disabled_agents` itself; this fake exists to test `ensure_installed` + without that check's outcome contaminating the assertions). Records + `AGENT_SYS_HOME` as seen in `os.environ` *at call time* for the first two + -- the only way to observe whether `_patched_environ` actually reached the + subprocess, since `run_cmd` passes no explicit `env=`. + """ + + def fake(cmd, shell=True, cwd=None, capture_output=True, text=True, env=None, timeout=None): # noqa: ANN001 + cmd_text = cmd if isinstance(cmd, str) else " ".join(cmd) + if "doctor sync" in cmd_text: + return subprocess.CompletedProcess( + cmd, 0, "Agent roots:\n cursor: /fake/path (ok, default)\n", "" + ) + if "health" in cmd_text: + return subprocess.CompletedProcess(cmd, 0, "", "") + seen_home.append(os.environ.get("AGENT_SYS_HOME")) + if "--version" in cmd_text: + rc = version_rc + out = "agentsview v0.42.0\n" if rc == 0 else "" + else: + rc = install_rc + out = "" if rc == 0 else "boom\n" + return subprocess.CompletedProcess(cmd, rc, out, "") + + return fake + + +def _install_item_for(prefix: Prefix): + """Builds the callable `ensure_installed` expects, from the real recipe. + + `ensure_installed` takes this as a dependency rather than loading the + recipe itself, because `env_mgr`'s installer machinery (`recipe`, + `runner`, `installers/…`) is below spec §9's decoupling wall and `o11y` is + not allowed to import it — checked structurally by `test_imports.py`, and + the first draft of `ensure_installed` failed exactly that test by + importing `recipe`/`runner` directly. Test code is not subject to the + wall, so it exercises the real `agentsview.o11y.yaml` recipe end to end + (`subprocess.run` faked aside), the same way a real caller would build + this closure. + """ + from env_mgr.recipe import load_recipe + from env_mgr.runner import Filters, run + + def install_item(): + target, items = load_recipe(agentsview.RECIPE_PATH) + target.path = str(prefix.root) + outs, _status = run(target, items, "install", Filters(item="agentsview")) + return outs + + return install_item + + +def test_ensure_installed_reports_already_present_without_reinstalling( + prefix, monkeypatch, _no_leftover_environ +) -> None: + seen_home: list[str | None] = [] + fake = _fake_run_cmd(version_rc=0, install_rc=1, seen_home=seen_home) + monkeypatch.setattr(subprocess, "run", fake) + + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is True + assert "already present" in status.reason + assert len(seen_home) == 1 # only the --version probe; install never ran + assert seen_home[0] == str(prefix.root) + + +def test_ensure_installed_runs_the_recipe_when_missing( + prefix, monkeypatch, _no_leftover_environ +) -> None: + seen_home: list[str | None] = [] + fake = _fake_run_cmd(version_rc=1, install_rc=0, seen_home=seen_home) + monkeypatch.setattr(subprocess, "run", fake) + + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is True + assert len(seen_home) == 2 # the failed probe, then the install + assert seen_home == [str(prefix.root), str(prefix.root)] + + +def test_ensure_installed_is_one_warning_and_a_skip_when_install_fails( + prefix, monkeypatch, caplog, _no_leftover_environ +) -> None: + fake = _fake_run_cmd(version_rc=1, install_rc=3, seen_home=[]) + monkeypatch.setattr(subprocess, "run", fake) + + with caplog.at_level("WARNING"): + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_ensure_installed_restores_os_environ_even_when_the_installer_raises( + prefix, monkeypatch, _no_leftover_environ +) -> None: + """The exception path is the one that must not leak a half-patched environ.""" + + def boom(*a, **k): # noqa: ANN001, ANN002, ANN003 + raise RuntimeError("this must never escape, and must not leak the environ") + + monkeypatch.setattr(subprocess, "run", boom) + before = dict(os.environ) + + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is False + assert dict(os.environ) == before + assert "AGENT_SYS_HOME" not in os.environ + + +def test_ensure_installed_is_a_warning_and_a_skip_when_the_item_is_absent( + prefix, caplog, _no_leftover_environ +) -> None: + """The `if not outs:` branch — a failure mode with no test until now. + + It fires when the recipe no longer has an item named `agentsview`, e.g. + after a rename. Reached by injecting the empty result directly, because + the point is the branch and not the recipe loader. + """ + with caplog.at_level("WARNING"): + status = agentsview.ensure_installed(prefix, lambda: []) + assert status.running is False + assert status.reason == "recipe item not found" + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_ensure_installed_never_raises_regardless_of_outcome( + prefix, monkeypatch, _no_leftover_environ +) -> None: + for version_rc, install_rc in ((0, 1), (1, 0), (1, 3)): + fake = _fake_run_cmd(version_rc=version_rc, install_rc=install_rc, seen_home=[]) + monkeypatch.setattr(subprocess, "run", fake) + agentsview.ensure_installed(prefix, _install_item_for(prefix)) # must not raise + + +# --- check_disabled_agents: OTHER_PROVIDERS validated against the real binary, +# --- in both directions --- + + +def _fake_agentsview_doctor_sync( + prefix: Prefix, *, rc: int, stdout: str = "", stderr: str = "" +) -> None: + """`check_disabled_agents` and `discover_providers` both only ever run + `doctor sync` -- never `health` -- so one fake, answering that one + subcommand, covers every test for both of them.""" + exe = prefix.bin / "agentsview" + exe.write_text(f"#!/bin/sh\nprintf '%s' '{stdout}'\nprintf '%s' '{stderr}' >&2\nexit {rc}\n") + exe.chmod(0o755) + + +#: One recognized provider ("cursor") already in `OTHER_PROVIDERS`. +_CLEAN_DOCTOR_SYNC_STDOUT = "Agent roots:\n cursor: /fake/path (ok, default)\n" + + +def test_check_disabled_agents_reports_nothing_when_both_directions_are_clean(prefix) -> None: + _fake_agentsview_doctor_sync(prefix, rc=0, stdout=_CLEAN_DOCTOR_SYNC_STDOUT) + assert agentsview.check_disabled_agents(prefix) == () + + +def test_check_disabled_agents_names_a_renamed_or_removed_provider(prefix) -> None: + """Direction 1: OTHER_PROVIDERS lists something the binary no longer knows.""" + _fake_agentsview_doctor_sync( + prefix, + rc=1, + stderr='fatal: loading config: disabled_agents: unknown session provider "claude-cowork"', + ) + assert agentsview.check_disabled_agents(prefix) == ("claude-cowork",) + + +def test_check_disabled_agents_names_a_provider_added_upstream_and_never_listed( + prefix, +) -> None: + """Direction 2, the one that leaks: the binary recognizes a provider + OTHER_PROVIDERS never mentions -- AgentsView would scan its default + directory and put its sessions on the panel with no error at all.""" + _fake_agentsview_doctor_sync( + prefix, + rc=0, + stdout="Agent roots:\n cursor: /fake/path (ok, default)\n" + " brand-new-provider: /fake/other (ok, default)\n", + ) + assert agentsview.check_disabled_agents(prefix) == ("brand-new-provider",) + + +def test_check_disabled_agents_is_empty_not_a_false_accusation_when_the_probe_cannot_run( + prefix, +) -> None: + """A missing/broken binary is 'could not confirm', not 'something is wrong'.""" + assert not (prefix.bin / "agentsview").exists() + assert agentsview.check_disabled_agents(prefix) == () + + +def test_check_disabled_agents_is_empty_on_an_unrelated_failure(prefix) -> None: + """Exit 1 with no recognizable message: still not a provider-name verdict.""" + _fake_agentsview_doctor_sync(prefix, rc=1, stderr="some unrelated crash") + assert agentsview.check_disabled_agents(prefix) == () + + +def test_check_disabled_agents_never_raises_on_a_hanging_binary(prefix, monkeypatch) -> None: + monkeypatch.setattr(agentsview, "CHECK_DISABLED_AGENTS_TIMEOUT_S", 0.2) + exe = prefix.bin / "agentsview" + exe.write_text("#!/bin/sh\nsleep 30\n") + exe.chmod(0o755) + assert agentsview.check_disabled_agents(prefix) == () + + +def test_check_disabled_agents_never_starts_a_daemon(prefix) -> None: + """The bug this whole rewrite exists to close: `health` silently + autostarted a daemon on a port AgentsView picked. `doctor sync` must + not start anything at all, in either the config-valid or + config-invalid case -- checked here by making the fake binary record + every invocation rather than by asserting on a real process, since a + unit test should not depend on a real daemon lifecycle to prove a + negative.""" + exe = prefix.bin / "agentsview" + calls_path = prefix.run / "calls.log" + exe.write_text( + "#!/bin/sh\n" + f"echo \"$@\" >> {calls_path}\n" + f"printf '%s' '{_CLEAN_DOCTOR_SYNC_STDOUT}'\n" + "exit 0\n" + ) + exe.chmod(0o755) + agentsview.check_disabled_agents(prefix) + calls = calls_path.read_text().splitlines() + assert calls == ["doctor sync"] + + +def test_check_disabled_agents_runs_with_home_pointed_into_the_prefix( + prefix, monkeypatch +) -> None: + """Gate 5 (design doc §3): HOME=$AGENT_SYS_HOME makes every provider's + *default* root resolve inside the prefix, not the user's real home -- + unlike OTHER_PROVIDERS, this needs no list and cannot go stale when + upstream adds a provider we have never heard of. See the identical + assertion in test_the_child_gets_the_prefix_environment... for why this + one line matters more than it looks like it should.""" + seen: dict[str, str] = {} + + def spy(cmd, env=None, **kw): # noqa: ANN001 + seen.update(env or {}) + return subprocess.CompletedProcess(cmd, 0, _CLEAN_DOCTOR_SYNC_STDOUT, "") + + monkeypatch.setattr(subprocess, "run", spy) + agentsview.check_disabled_agents(prefix) + assert seen["HOME"] == str(prefix.root) + + +def test_discover_providers_runs_with_home_pointed_into_the_prefix(prefix, monkeypatch) -> None: + """Same gate 5, the other call site that shells out to `doctor sync`.""" + seen: dict[str, str] = {} + + def spy(cmd, env=None, **kw): # noqa: ANN001 + seen.update(env or {}) + return subprocess.CompletedProcess(cmd, 0, _CLEAN_DOCTOR_SYNC_STDOUT, "") + + monkeypatch.setattr(subprocess, "run", spy) + agentsview.discover_providers(prefix) + assert seen["HOME"] == str(prefix.root) + + +# --- discover_providers: the enumeration `check_disabled_agents` uses --- + + +#: A shape modelled directly on a real `agentsview v0.42.0 doctor sync` run +#: (see PHASE0.md §0.3) -- several names repeat across multiple root lines, +#: which is exactly why `discover_providers` dedupes with a `set`. +_REAL_SHAPED_DOCTOR_SYNC_STDOUT = ( + "Sync Diagnostics\n" + "Version: v0.42.0\n" + "Agent roots:\n" + " claude: /home/x/.claude/projects (ok, configured)\n" + " openclaude: /home/x/.openclaude/projects (missing, default)\n" + " cowork: /home/x/Library/Application Support/Claude (missing, default)\n" + " cowork: /home/x/.config/Claude (missing, default)\n" + " cursor: /home/x/.cursor/projects (ok, default)\n" + "Recent debug.log evidence:\n" + " none\n" +) + + +def test_discover_providers_parses_a_real_shaped_report(prefix) -> None: + _fake_agentsview_doctor_sync(prefix, rc=0, stdout=_REAL_SHAPED_DOCTOR_SYNC_STDOUT) + found = agentsview.discover_providers(prefix) + assert found == ("cowork", "cursor", "openclaude") # sorted, deduped, no claude + + +def test_discover_providers_is_none_when_the_binary_is_missing(prefix) -> None: + assert not (prefix.bin / "agentsview").exists() + assert agentsview.discover_providers(prefix) is None + + +def test_discover_providers_is_none_on_a_nonzero_exit(prefix) -> None: + _fake_agentsview_doctor_sync(prefix, rc=1, stdout="") + assert agentsview.discover_providers(prefix) is None + + +def test_discover_providers_is_none_when_the_report_has_no_agent_roots_section(prefix) -> None: + _fake_agentsview_doctor_sync(prefix, rc=0, stdout="Sync Diagnostics\nVersion: v0.42.0\n") + assert agentsview.discover_providers(prefix) is None + + +def test_discover_providers_is_none_rather_than_empty_when_only_claude_is_found( + prefix, +) -> None: + """An empty tuple would read as 'nothing else exists', the single most + permissive way this function could fail `check_disabled_agents`.""" + _fake_agentsview_doctor_sync( + prefix, rc=0, stdout="Agent roots:\n claude: /x/.claude/projects (ok, configured)\n" + ) + assert agentsview.discover_providers(prefix) is None + + +def test_discover_providers_never_raises_on_a_hanging_binary(prefix, monkeypatch) -> None: + monkeypatch.setattr(agentsview, "DISCOVER_PROVIDERS_TIMEOUT_S", 0.2) + exe = prefix.bin / "agentsview" + exe.write_text("#!/bin/sh\nsleep 30\n") + exe.chmod(0o755) + assert agentsview.discover_providers(prefix) is None + + +def test_write_config_writes_exactly_the_providers_it_is_given(prefix) -> None: + """A pure writer: no fake binary, no subprocess, just the list in -> the + list out, in the file `check_disabled_agents`/`serve` will read.""" + agentsview.write_config(prefix, ("foo", "bar")) + text = (prefix.agentsview_data / "config.toml").read_text() + assert 'disabled_agents = ["foo", "bar"]' in text + + +def test_write_config_keeps_the_daemon_alive_indefinitely(prefix) -> None: + """Design §4 promises the panel persists across runs; AgentsView's own + default (20m idle exit) contradicts that unless this key overrides it.""" + agentsview.write_config(prefix, ()) + text = (prefix.agentsview_data / "config.toml").read_text() + assert 'daemon_idle_timeout = "0s"' in text + + +def test_other_providers_excludes_claude_itself() -> None: + """The one provider gate 3 must never disable.""" + assert "claude" not in agentsview.OTHER_PROVIDERS + + +def test_ensure_installed_warns_once_naming_a_renamed_provider( + prefix, monkeypatch, caplog, _no_leftover_environ +) -> None: + """The install-time check, wired end to end through `ensure_installed`, + direction 1: OTHER_PROVIDERS lists something the binary rejects. + `check_disabled_agents` only ever runs `doctor sync` now (never + `health` -- see its docstring), so the rejection comes from that call.""" + seen_home: list[str | None] = [] + + def fake(cmd, shell=True, cwd=None, capture_output=True, text=True, env=None, timeout=None): # noqa: ANN001 + cmd_text = cmd if isinstance(cmd, str) else " ".join(cmd) + if "doctor sync" in cmd_text: + return subprocess.CompletedProcess( + cmd, 1, "", 'fatal: unknown session provider "bogus-provider"' + ) + seen_home.append(os.environ.get("AGENT_SYS_HOME")) + return subprocess.CompletedProcess(cmd, 0, "agentsview v0.42.0\n", "") + + monkeypatch.setattr(subprocess, "run", fake) + + with caplog.at_level("WARNING"): + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is True # the binary install itself still succeeded + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "bogus-provider" in warnings[0].getMessage() + + +def test_ensure_installed_warns_once_naming_a_provider_added_upstream( + prefix, monkeypatch, caplog, _no_leftover_environ +) -> None: + """Direction 2, end to end: a provider `doctor sync` reports that + OTHER_PROVIDERS never listed -- the direction that leaks silently.""" + + def fake(cmd, shell=True, cwd=None, capture_output=True, text=True, env=None, timeout=None): # noqa: ANN001 + cmd_text = cmd if isinstance(cmd, str) else " ".join(cmd) + if "doctor sync" in cmd_text: + return subprocess.CompletedProcess( + cmd, + 0, + "Agent roots:\n cursor: /fake/path (ok, default)\n" + " brand-new-provider: /fake/other (ok, default)\n", + "", + ) + return subprocess.CompletedProcess(cmd, 0, "agentsview v0.42.0\n", "") + + monkeypatch.setattr(subprocess, "run", fake) + + with caplog.at_level("WARNING"): + status = agentsview.ensure_installed(prefix, _install_item_for(prefix)) + + assert status.running is True + warnings = [r for r in caplog.records if r.levelname == "WARNING"] + assert len(warnings) == 1 + assert "brand-new-provider" in warnings[0].getMessage() diff --git a/agent_sys/tests/env_mgr/test_o11y_agentsview_smoke.py b/agent_sys/tests/env_mgr/test_o11y_agentsview_smoke.py new file mode 100644 index 000000000..447915c76 --- /dev/null +++ b/agent_sys/tests/env_mgr/test_o11y_agentsview_smoke.py @@ -0,0 +1,333 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""One smoke test against the **real installed binary**. + +**Why this file exists, and it is not redundant with `test_o11y_agentsview.py`.** +Every test in that file substitutes a fake `agentsview` shell script which +ignores its config file entirely. That approach is structurally blind to any bug +in what the binary does with what we hand it, and two real bugs proved it: + +1. `disabled_agents` naming providers this version does not know — the daemon + exits 1 on the first one, and 626 green tests coexisted with a panel that had + never once come up (`recon/ACCEPTANCE.md`, check 1). +2. `GET /api/v1/sessions` returning `{"sessions":[],"total":0}` for a session + that is present, syncable and visible to `agentsview health`. + +Both are invisible to a fake. So this test writes the config we really produce, +starts the real daemon, and asks it for the sessions we really planted. + +**On the second one, this file was itself wrong first, and that is the lesson +it now encodes.** The empty response is real, but it is the *CLI's* endpoint +applying a documented one-shot exclusion — and we read it as "the panel is +broken". It never was: a real browser loading the plain `/` renders the session, +because the web UI's session list calls a **different endpoint** +(`sessions/sidebar-index`) and sends `include_one_shot=true` in its own request. +Settled by reading a rendered page, twice, after hours spent on a non-bug. + +So the load-bearing assertion is against `UI_SESSIONS` — the request a browser +actually makes — and the CLI surface is pinned separately. **A non-zero session +count, never HTTP 200**: a daemon that starts, answers every health check and +returns an empty list is the failure that must not ship. + +**Nothing here touches the operator's state.** Temporary prefix, temporary +`AGENTSVIEW_DATA_DIR`, ephemeral port — never 18888, never `~/.agentsview`, +never the real `~/.infera_agent_sys/state`. Only the binary is shared, which is +the whole point. Two copies of this test running at once cannot collide, and the +daemon is stopped through its own `serve stop` against *our* data directory, +never a pattern-matched kill: a user's own instance may be on this box. +""" + +from __future__ import annotations + +import contextlib +import json +import os +import re +import shutil +import socket +import subprocess +import time +import urllib.error +import urllib.request +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from env_mgr.o11y import agentsview +from env_mgr.prefix import Prefix + +#: How long the daemon may take to answer after `serve --background` returns. +#: Cold start builds the SQLite archive from the session root; ours holds one +#: file, so this is generous rather than tight. +READY_TIMEOUT_S = 60.0 + +#: `serve --background` daemonises and returns at once. +LAUNCH_TIMEOUT_S = 30.0 + +#: **The request the web UI's session list actually makes**, captured from the +#: network tab of a real headless Chromium loading the plain `/` — twice, by +#: `recon` (`ws.agentsview_o11y/recon/PHASE0.md` §0.9) and again here before +#: this constant was written. The rendered page showed the planted session: +#: `1 SESSION / SIDEBAR-MARKER-9f31`. +#: +#: **This is the surface that matters**, and asserting on it rather than on +#: `/api/v1/sessions` is the correction that this file existed to make and +#: initially got wrong. The two endpoints disagree: `/api/v1/sessions` is the +#: CLI's surface (`session list`) and applies the documented one-shot +#: exclusion, while the browser's session list calls `sessions/sidebar-index` +#: and always sends `include_one_shot=true` itself. Measured here: the bare +#: `sidebar-index` with no parameters also returns nothing, so the parameter +#: comes from the *frontend*, not from a different default on the endpoint. +#: +#: An earlier version of this file asserted only the CLI surface and concluded +#: the panel was broken. It was not. The whole campaign's most expensive +#: mistake was treating an API response as a proxy for what a person sees. +UI_SESSIONS = ( + "/api/v1/sessions/sidebar-index?timezone=UTC&include_one_shot=true&limit=500&order_by=recent" +) + +#: The CLI/API surface, pinned as well. Keeping both means a future release +#: that moves either default is noticed by a test rather than by an operator. +#: `includeOneShot` and `exclude_one_shot=false` were both measured to do +#: nothing; the parameter is snake_case and positive-only. +RAW_SESSIONS = "/api/v1/sessions?include_one_shot=true" + +_PID_RE = re.compile(r"pid (\d+)") + + +def _binary() -> str | None: + """The installed binary, or `None`. Prefix first, then `PATH`.""" + try: + candidate = Prefix.resolve(os.environ).bin / "agentsview" + except KeyError: # no $HOME and no AGENT_SYS_HOME + candidate = None + if candidate is not None and os.access(candidate, os.X_OK): + return str(candidate) + return shutil.which("agentsview") + + +#: **Resolved once, at import.** The guard below runs at collection and the +#: `panel` fixture runs at setup, and `_binary()` reads `AGENT_SYS_HOME` — so +#: anything that redirects the prefix between the two makes them disagree. It +#: does: `tests/conftest.py` points the prefix at `tmp` for the session, which +#: turned the skip into three setup errors. One resolution, shared. +_BINARY = _binary() + +#: **Skip, never fail.** A fresh checkout has no binary and its test run must +#: stay green; this file's job is to catch a bug in the binary we ship with, not +#: to make the absence of one an error. +requires_binary = pytest.mark.skipif( + _BINARY is None, + reason="the agentsview binary is not installed; this smoke test needs the real one", +) + + +def _free_port() -> int: + with socket.socket() as s: + s.bind(("127.0.0.1", 0)) + return int(s.getsockname()[1]) + + +def _plant_session(prefix: Prefix, session_id: str, first_message: str) -> None: + """One Claude Code transcript, in Claude Code's own shape. + + Two turns, because that is the smallest thing AgentsView will call a + session, and a slug subdirectory because that is how the real CLI names + them — one per working directory. + """ + slug = prefix.claude_home / "projects" / "-tmp-agent-sys-smoke" + slug.mkdir(parents=True, exist_ok=True) + turns = [ + { + "parentUuid": None, + "isSidechain": False, + "type": "user", + "message": {"role": "user", "content": first_message}, + "uuid": f"{session_id[:8]}-0000-0000-0000-000000000001", + "timestamp": "2026-09-03T12:00:00.000Z", + "cwd": "/tmp/agent-sys/smoke", + "sessionId": session_id, + "version": "1.0.0", + "userType": "external", + }, + { + "parentUuid": f"{session_id[:8]}-0000-0000-0000-000000000001", + "isSidechain": False, + "type": "assistant", + "message": { + "id": "msg_1", + "role": "assistant", + "model": "claude-opus-5", + "content": [{"type": "text", "text": "acknowledged"}], + "usage": {"input_tokens": 10, "output_tokens": 3}, + }, + "uuid": f"{session_id[:8]}-0000-0000-0000-000000000002", + "timestamp": "2026-09-03T12:00:01.000Z", + "cwd": "/tmp/agent-sys/smoke", + "sessionId": session_id, + "version": "1.0.0", + }, + ] + (slug / f"{session_id}.jsonl").write_text("".join(json.dumps(t) + "\n" for t in turns)) + + +def _get(url: str, timeout: float = 10.0) -> tuple[int, bytes]: + try: + with urllib.request.urlopen(url, timeout=timeout) as r: # noqa: S310 + return int(r.status), r.read() + except urllib.error.HTTPError as exc: + return int(exc.code), exc.read() + + +class Panel: + """A live daemon and the one session we planted in it.""" + + def __init__(self, port: int, session_id: str, first_message: str) -> None: + self.port = port + self.session_id = session_id + self.first_message = first_message + + def sessions(self, path: str = UI_SESSIONS) -> list[dict]: + """Sessions from one endpoint. Defaults to **the one the browser uses**.""" + status, body = _get(f"http://127.0.0.1:{self.port}{path}") + assert status == 200, f"{path} answered {status}" + return list(json.loads(body).get("sessions", [])) + + +@pytest.fixture() +def panel(tmp_path: Path) -> Iterator[Panel]: + binary = _BINARY + assert binary is not None # guarded by `requires_binary` + + prefix = Prefix(tmp_path / "prefix") + prefix.create() + session_id = "5m0ke7e5-0000-4000-8000-00000000000a" + first_message = "planted by the agent_sys smoke test" + _plant_session(prefix, session_id, first_message) + + # **The config we really ship**, produced by the module that ships it — so a + # change to `write_config` or to `OTHER_PROVIDERS` is exercised here rather + # than mirrored into a copy that can drift out of agreement with it. + agentsview.write_config(prefix, agentsview.OTHER_PROVIDERS) + + # `HOME` too: the binary falls back to `~/.agentsview` for anything the + # explicit variables do not cover, and the operator's must stay untouched. + env = { + **prefix.environment(), + "PATH": os.environ.get("PATH", ""), + "HOME": str(prefix.root), + } + port = _free_port() + started = subprocess.run( # noqa: S603 — `binary` is a resolved path + [ + binary, + "serve", + "--background", + "--no-browser", + "--no-update-check", + "--host", + "127.0.0.1", + "--port", + str(port), + ], + env=env, + capture_output=True, + text=True, + timeout=LAUNCH_TIMEOUT_S, + check=False, + ) + if started.returncode != 0: + pytest.fail( + f"`agentsview serve --background` exited {started.returncode}. " + f"This is the failure mode a fake binary cannot reproduce.\n" + f"stdout: {started.stdout}\nstderr: {started.stderr}" + ) + # Recorded now, so teardown can stop **this** process even if `serve stop` + # cannot find its own state. One pid we printed ourselves; never a pattern. + match = _PID_RE.search(started.stdout or "") + pid = int(match.group(1)) if match else None + + try: + deadline = time.monotonic() + READY_TIMEOUT_S + while time.monotonic() < deadline: + status, _ = _get(f"http://127.0.0.1:{port}/", timeout=3.0) + if status == 200: + break + time.sleep(0.5) + else: + pytest.fail(f"the daemon never answered on 127.0.0.1:{port}") + yield Panel(port, session_id, first_message) + finally: + subprocess.run( # noqa: S603 + [binary, "serve", "stop"], + env=env, + capture_output=True, + timeout=LAUNCH_TIMEOUT_S, + check=False, + ) + if pid is not None and not agentsview.port_is_free(port): + # Still up: signal the **recorded** pid and nothing else. An + # `os.kill` on a pid we read from our own launch output cannot + # reach a stranger's daemon the way a name match could. + with contextlib.suppress(ProcessLookupError, PermissionError): + os.kill(pid, 15) + + +@requires_binary +def test_the_panel_a_user_opens_lists_the_session_we_planted(panel: Panel) -> None: + """**The assertion this whole file exists for, on the surface that ships.** + + `UI_SESSIONS` is the request a real browser issues on a plain `/` load, with + no query string of ours added to the address bar. A non-zero count here is + the closest thing to "a person opening the panel sees this run" that a test + without a browser can assert. + + Not 200, and not "the process is alive". A daemon that starts, answers every + health check and returns an empty list is the exact shape of the empty-panel + bug. + """ + sessions = panel.sessions(UI_SESSIONS) + + assert sessions, ( + "the panel's own session-list request answered 200 with zero sessions. " + "A user opening this panel would see an empty page that looks correct." + ) + assert any(s.get("id") == panel.session_id for s in sessions), ( + f"the panel lists sessions but none is ours ({panel.session_id}); " + f"got {[s.get('id') for s in sessions]}" + ) + + +@requires_binary +def test_the_cli_api_surface_serves_it_too(panel: Panel) -> None: + """The other endpoint, pinned deliberately. + + The two disagree today — `/api/v1/sessions` applies the one-shot exclusion + the docs describe, `sessions/sidebar-index` is asked for them by the + frontend — and that gap is now a known fact about this dependency rather + than a discovery waiting to be made again. Pinning both means a release that + moves either default is caught by a test. + """ + sessions = panel.sessions(RAW_SESSIONS) + + assert any(s.get("id") == panel.session_id for s in sessions), ( + f"the CLI/API surface does not serve our session ({panel.session_id}); " + f"got {[s.get('id') for s in sessions]}" + ) + + +@requires_binary +def test_it_is_our_prefix_being_read_and_not_some_other_root(panel: Panel) -> None: + """The count could be non-zero for the wrong reason — a stray archive, or a + provider we failed to disable. Reading the content settles it. + + On the raw endpoint, because `sidebar-index` returns no `first_message` — + which is itself a reason to keep both: the surface a user sees proves + *presence*, and only this one proves *identity*. + """ + ours = [s for s in panel.sessions(RAW_SESSIONS) if s.get("id") == panel.session_id] + + assert len(ours) == 1 + assert ours[0].get("first_message") == panel.first_message + assert ours[0].get("agent") == "claude" diff --git a/agent_sys/tests/env_mgr/test_o11y_mapping.py b/agent_sys/tests/env_mgr/test_o11y_mapping.py new file mode 100644 index 000000000..ff2d0b176 --- /dev/null +++ b/agent_sys/tests/env_mgr/test_o11y_mapping.py @@ -0,0 +1,206 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""One AgentsView project per run: the call, and every way it may fail. + +AgentsView names a project after the session's **deepest** path segment, so one +run's attempts arrive as several unrelated projects — measured on a real nested +fixture, four sessions of one run as four projects. Renaming the directories +cannot join them; only a mapping over the run root can. +""" + +from __future__ import annotations + +import json +import urllib.error +from pathlib import Path + +import pytest + +from env_mgr.o11y.agentsview import mapping + +MACHINE = "test-box" + + +class _Recorder: + """A stand-in for `urlopen` that records requests and replays answers.""" + + def __init__(self, *answers: object) -> None: + self.answers = list(answers) + self.seen: list[dict] = [] + + def __call__(self, req, timeout=None): # noqa: ANN001 + self.seen.append( + { + "url": req.full_url, + "method": req.get_method(), + "headers": {k.lower(): v for k, v in req.header_items()}, + "body": json.loads(req.data.decode()) if req.data else None, + } + ) + answer = self.answers.pop(0) + if isinstance(answer, Exception): + raise answer + return _Response(answer) + + +class _Response: + def __init__(self, payload: object, status: int = 200) -> None: + self._payload = payload + self.status = status + + def read(self, *_a: object) -> bytes: + return json.dumps(self._payload).encode() + + def __enter__(self) -> _Response: + return self + + def __exit__(self, *_a: object) -> None: + return None + + +def _machine_answer() -> dict: + return {"machine": MACHINE, "local_machine": MACHINE, "machines": [MACHINE], "mappings": []} + + +@pytest.fixture() +def urlopen(monkeypatch): + def install(*answers: object) -> _Recorder: + rec = _Recorder(*answers) + monkeypatch.setattr(mapping.urllib.request, "urlopen", rec) + return rec + + return install + + +# --- the name ------------------------------------------------------------- # + + +def test_a_dash_becomes_an_underscore_before_we_send_it() -> None: + """Measured: AgentsView stores `MAPPED-GIT` as `MAPPED_GIT`. + + Normalising on our side means the string we post is the string that comes + back, so a later read or a log line cannot disagree with the panel. + """ + assert mapping.name_for_run(Path("/x/runs/20260903T104807-76274e")) == ( + "run.20260903T104807_76274e" + ) + + +def test_the_name_is_built_from_the_run_directory_alone() -> None: + """Nothing upstream of the run root may change the label.""" + a = mapping.name_for_run(Path("/one/runs/20260903T104807-76274e")) + b = mapping.name_for_run(Path("/another/place/runs/20260903T104807-76274e")) + assert a == b + + +# --- the call ------------------------------------------------------------- # + + +def test_the_mapping_is_posted_for_this_run_only(urlopen) -> None: + rec = urlopen(_machine_answer(), {"id": 1}) + run = Path("/state/runs/20260904T101112-abcdef") + + status = mapping.ensure_run_project("http://127.0.0.1:9001", run) + + assert status.running is True + post = rec.seen[-1] + assert post["method"] == "POST" + assert post["body"] == { + "machine": MACHINE, + "path_prefix": str(run), + "project": "run.20260904T101112_abcdef", + "layout": "explicit", + "enabled": True, + } + + +def test_the_machine_is_read_from_the_daemon_and_never_assumed(urlopen) -> None: + """A wrong `machine` matches nothing, silently. + + The recon ran in a container whose hostname was not the host's, which is + exactly how this would have shipped broken: the value has to come from the + daemon that will do the matching. + """ + rec = urlopen( + {"local_machine": "the-real-one", "machines": ["the-real-one"], "mappings": []}, + {"id": 1}, + ) + + mapping.ensure_run_project("http://127.0.0.1:9001", Path("/state/runs/r-1")) + + assert rec.seen[0]["method"] == "GET" + assert rec.seen[-1]["body"]["machine"] == "the-real-one" + + +def test_every_mutating_call_sends_an_origin_header(urlopen) -> None: + """Measured: without it the answer is a plain-text `403 Forbidden`, not the + JSON error shape — which reads exactly like a missing endpoint.""" + rec = urlopen(_machine_answer(), {"id": 1}) + + mapping.ensure_run_project("http://127.0.0.1:9001", Path("/state/runs/r-1")) + + assert rec.seen[-1]["headers"]["origin"] == "http://127.0.0.1:9001" + + +def test_an_existing_mapping_is_success_not_a_warning(urlopen, caplog) -> None: + """`POST` is not idempotent — uniqueness is `(machine, path_prefix)` — so a + second run of the same id answers `409`. That is the state we wanted.""" + rec = urlopen( + _machine_answer(), + urllib.error.HTTPError("u", 409, "conflict", {}, None), # type: ignore[arg-type] + ) + + with caplog.at_level("WARNING"): + status = mapping.ensure_run_project("http://127.0.0.1:9001", Path("/state/runs/r-1")) + + assert status.running is True + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + assert len(rec.seen) == 2 + + +# --- every failure is one warning and a skip ------------------------------- # + + +@pytest.mark.parametrize( + "answers", + [ + (urllib.error.URLError("connection refused"),), + (OSError("socket blew up"),), + ("not-a-dict",), + ({"machines": []},), # no local_machine + (_machine_answer(), urllib.error.HTTPError("u", 500, "boom", {}, None)), + (_machine_answer(), urllib.error.URLError("died mid-post")), + (_machine_answer(), TimeoutError("too slow")), + ], +) +def test_a_failing_mapping_call_is_one_warning_and_a_skip(urlopen, caplog, answers) -> None: + urlopen(*answers) + + with caplog.at_level("WARNING"): + status = mapping.ensure_run_project("http://127.0.0.1:9001", Path("/state/runs/r-1")) + + assert status.running is False + assert len([r for r in caplog.records if r.levelname == "WARNING"]) == 1 + + +def test_no_panel_means_no_call_and_no_warning(urlopen, caplog) -> None: + """`url=None` is "the panel did not start", which was already warned about.""" + rec = urlopen() + + with caplog.at_level("WARNING"): + status = mapping.ensure_run_project(None, Path("/state/runs/r-1")) + + assert status.running is False + assert rec.seen == [] + assert [r for r in caplog.records if r.levelname == "WARNING"] == [] + + +def test_nothing_escapes_regardless_of_what_the_daemon_says(urlopen) -> None: + """The module's one law, asserted directly.""" + for answers in ( + (RuntimeError("something nobody thought of"),), + (_machine_answer(), ValueError("garbage body")), + (None,), + ): + urlopen(*answers) + mapping.ensure_run_project("http://127.0.0.1:9001", Path("/state/runs/r-1")) diff --git a/agent_sys/tests/env_mgr/test_prefix.py b/agent_sys/tests/env_mgr/test_prefix.py new file mode 100644 index 000000000..0b1ef811a --- /dev/null +++ b/agent_sys/tests/env_mgr/test_prefix.py @@ -0,0 +1,112 @@ +# SPDX-License-Identifier: MIT +# Copyright (c) 2026, Advanced Micro Devices, Inc. All rights reserved. +"""The `~/.infera_agent_sys` prefix: where it is, and what names it publishes.""" + +from __future__ import annotations + +from pathlib import Path + +from env_mgr.prefix import Prefix + + +def test_default_root_is_infera_agent_sys_under_home(tmp_path: Path) -> None: + p = Prefix.resolve({"HOME": str(tmp_path)}) + assert p.root == tmp_path / ".infera_agent_sys" + + +def test_env_var_overrides_home(tmp_path: Path) -> None: + p = Prefix.resolve({"HOME": "/nowhere", "AGENT_SYS_HOME": str(tmp_path / "elsewhere")}) + assert p.root == tmp_path / "elsewhere" + + +def test_resolving_without_home_does_not_raise() -> None: + """`resolve` is total, because two of its three call sites cannot degrade. + + It used to do `environ["HOME"]` and raise `KeyError`. `material.py` caught + that and warned; `prepare.py` and `cli/environment.py` did not, so under a + systemd unit or a stripped cron environment the o11y feature killed a run + that never asked for a panel — the one rule this whole change is built + around. One behaviour at all three call sites is the fix, and the + behaviour is *resolve to somewhere*, never raise. + """ + p = Prefix.resolve({}) + assert p.root.is_absolute() + assert p.root.name.startswith(".infera_agent_sys") or "infera" in p.root.name + + +def test_a_tilde_in_the_override_is_expanded(tmp_path: Path) -> None: + """`AGENT_SYS_HOME=~/foo` is a literal `~/foo` to `Path`, and a relative + override is cwd-dependent — and the cwd changes across zones.""" + p = Prefix.resolve({"HOME": str(tmp_path), "AGENT_SYS_HOME": "~/elsewhere"}) + assert p.root == Path.home() / "elsewhere" + assert p.root.is_absolute() + + +def test_the_layout_is_local_shaped(tmp_path: Path) -> None: + p = Prefix.resolve({"HOME": str(tmp_path)}) + assert p.bin == p.root / "bin" + assert p.share == p.root / "share" + assert p.state == p.root / "state" + assert p.run == p.root / "run" + assert p.claude_home == p.state / "claude" + assert p.agentsview_data == p.state / "agentsview" + + +def test_environment_names_every_directory(tmp_path: Path) -> None: + p = Prefix.resolve({"HOME": str(tmp_path)}) + env = p.environment() + assert env["AGENT_SYS_HOME"] == str(p.root) + assert env["AGENT_SYS_BIN"] == str(p.bin) + assert env["AGENT_SYS_SHARE"] == str(p.share) + assert env["AGENT_SYS_STATE"] == str(p.state) + assert env["AGENT_SYS_RUN"] == str(p.run) + assert env["AGENT_SYS_CLAUDE_HOME"] == str(p.claude_home) + assert env["AGENTSVIEW_DATA_DIR"] == str(p.agentsview_data) + assert env["CLAUDE_PROJECTS_DIR"] == str(p.claude_home / "projects") + + +def test_create_is_idempotent(tmp_path: Path) -> None: + p = Prefix.resolve({"HOME": str(tmp_path)}) + p.create() + p.create() + for d in (p.bin, p.share, p.state, p.run, p.claude_home, p.agentsview_data): + assert d.is_dir() + + +def test_resolve_does_not_read_the_ambient_environment(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("AGENT_SYS_HOME", "/should/be/ignored") + p = Prefix.resolve({"HOME": str(tmp_path)}) + assert p.root == tmp_path / ".infera_agent_sys" + + +def test_prefix_names_are_reachable_from_the_paths_family() -> None: + """`paths` is where a reader looks for an `AGENT_SYS_*` name. All of them.""" + from env_mgr import paths + + assert paths.HOME_ENV_VAR == "AGENT_SYS_HOME" + assert paths.BIN_ENV_VAR == "AGENT_SYS_BIN" + assert paths.CLAUDE_HOME_ENV_VAR == "AGENT_SYS_CLAUDE_HOME" + for name in ("HOME_ENV_VAR", "BIN_ENV_VAR", "CLAUDE_HOME_ENV_VAR"): + assert name in paths.__all__ + + +def test_agent_environment_carries_claude_config_dir(tmp_path: Path) -> None: + from env_mgr.prefix import Prefix, agent_environment + + p = Prefix.resolve({"HOME": str(tmp_path)}) + env = agent_environment(p, base={"PATH": "/usr/bin"}) + assert env["CLAUDE_CONFIG_DIR"] == str(p.claude_home) + assert env["PATH"].startswith(str(p.bin) + ":") + assert "/usr/bin" in env["PATH"] + + +def test_agent_environment_does_not_touch_this_process(tmp_path: Path) -> None: + """The guard on 'the user's own Claude Code is unaffected'.""" + import os + + from env_mgr.prefix import Prefix, agent_environment + + before = dict(os.environ) + agent_environment(Prefix.resolve({"HOME": str(tmp_path)}), base={"PATH": "/usr/bin"}) + assert dict(os.environ) == before + assert "CLAUDE_CONFIG_DIR" not in os.environ diff --git a/agent_sys/tests/env_mgr/test_prepare.py b/agent_sys/tests/env_mgr/test_prepare.py index e3de42af7..901eed1e8 100644 --- a/agent_sys/tests/env_mgr/test_prepare.py +++ b/agent_sys/tests/env_mgr/test_prepare.py @@ -613,6 +613,27 @@ def test_prepared_environment_carries_a_derived_path(ctx) -> None: assert any(contained(d, root) for root in granted), f"{d} is on PATH and not granted" +def test_prepared_environment_scopes_the_agent_to_the_o11y_prefix(ctx) -> None: + """`CLAUDE_CONFIG_DIR` reaches the child, and never this process. + + The write side of the o11y session scoping: the `claude` CLI that a task + spawns writes its transcript under `~/.infera_agent_sys`, so the panel sees + `agent_sys`'s sessions and only those. The second assertion is the one that + matters to a user — a Claude Code they start in their own terminal must + still read `~/.claude`, and it does, because we never set the variable here. + """ + from env_mgr.prefix import Prefix + + before = dict(os.environ) + task = Task() + prepared = prepare(task, task.push_execution(), ctx) + prefix = Prefix.resolve(os.environ) + assert prepared.environment["CLAUDE_CONFIG_DIR"] == str(prefix.claude_home) + assert prepared.environment["AGENT_SYS_BIN"] == str(prefix.bin) + assert dict(os.environ) == before + assert "CLAUDE_CONFIG_DIR" not in os.environ + + def test_a_declared_env_may_override_the_derived_path(ctx) -> None: """An author saying so outranks a default. An override naming an ungranted directory is simply unreachable, and nothing here can make it otherwise — diff --git a/agent_sys/tests/env_mgr/test_task_graph_agreement.py b/agent_sys/tests/env_mgr/test_task_graph_agreement.py index 192db4a90..c5b4f64c7 100644 --- a/agent_sys/tests/env_mgr/test_task_graph_agreement.py +++ b/agent_sys/tests/env_mgr/test_task_graph_agreement.py @@ -21,6 +21,7 @@ import pytest from env_mgr.fs.domain import DomainKind, DomainRegistry +from env_mgr.fs.layout import handoff_version_dir from env_mgr.grants import mode_for, resolve_all from env_mgr.protocols import Context, Mode, Tier, UnresolvedGrant from task_graph import Access, Grant, Permissions @@ -153,7 +154,7 @@ def test_resolve_all_against_real_permissions(store: str, tmp_path: Path) -> Non execution = task.push_execution(AgentId.new(), {hid: 2}) granted = resolve_all(task, execution, _ctx(store, {hid: _handoff(hid, "trace")}, tmp_path)) - assert [g.path for g in granted] == [os.path.join(store, str(hid), "v2", "content")] + assert [g.path for g in granted] == [os.path.join(handoff_version_dir(store, hid, 2), "content")] assert granted[0].mode is Mode.READ_EXEC diff --git a/agent_sys/tests/handoff/conformance.py b/agent_sys/tests/handoff/conformance.py index 0bcb8eebb..89ec19870 100644 --- a/agent_sys/tests/handoff/conformance.py +++ b/agent_sys/tests/handoff/conformance.py @@ -20,7 +20,7 @@ from handoff.digest import tree_digest from handoff.errors import Malformed, NotSealable -from handoff.store import CLAIM_DIR +from handoff.store import CLAIM_DIR, version_dir from task_graph.ids import HandoffId, TaskId from tests.handoff.conftest import FixedKind, make_content, make_kind, open_kind @@ -173,7 +173,7 @@ def test_seal_publishes_what_the_agent_wrote_in_place(self, tmp_path: Path) -> N # does not exist either raises in `prepare` or evaporates. shutil.copytree( tmp_path / "written", - store.root / str(hid) / f"v{version}" / "content", + version_dir(store.root, hid, version) / "content", dirs_exist_ok=True, ) @@ -191,7 +191,7 @@ def test_seal_refuses_a_malformed_version_and_leaves_a_hole(self, tmp_path: Path store, hid = self._store_and_id(tmp_path) version = store.allocate(hid) # Something was written, and it is not a handoff: no README, no items. - (store.root / str(hid) / f"v{version}" / "content" / "stray.txt").write_text("x") + (version_dir(store.root, hid, version) / "content" / "stray.txt").write_text("x") why = store.seal(hid, version, producer=TaskId.new()) assert why, "a refusal reports its reason rather than raising" @@ -210,7 +210,7 @@ def test_allocate_creates_the_directory_the_grant_names(self, tmp_path: Path) -> """ store, hid = self._store_and_id(tmp_path) version = store.allocate(hid) - vdir = store.root / str(hid) / f"v{version}" + vdir = version_dir(store.root, hid, version) for granted in ("content", "claim"): assert (vdir / granted).is_dir(), f"{granted}/ exists before the body runs" assert list((vdir / granted).iterdir()) == [], f"{granted}/ is empty" @@ -235,11 +235,11 @@ def test_the_claim_directory_is_outside_the_digest(self, tmp_path: Path) -> None store, hid = self._store_and_id(tmp_path) plain = store.allocate(hid) - make_content(store.root / str(hid) / f"v{plain}" / "content") + make_content(version_dir(store.root, hid, plain) / "content") store.seal(hid, plain, producer=TaskId.new()) claimed = store.allocate(hid) - vdir = store.root / str(hid) / f"v{claimed}" + vdir = version_dir(store.root, hid, claimed) make_content(vdir / "content") (vdir / CLAIM_DIR / "self_check.yaml").write_text("done: true\n", encoding="utf-8") store.seal(hid, claimed, producer=TaskId.new()) @@ -301,6 +301,6 @@ def test_a_successful_seal_returns_none(self, tmp_path: Path) -> None: rather than `""`, so nothing reads a reason that is not there.""" store, hid = self._store_and_id(tmp_path) version = store.allocate(hid) - make_content(store.root / str(hid) / f"v{version}" / "content") + make_content(version_dir(store.root, hid, version) / "content") assert store.seal(hid, version, producer=TaskId.new()) is None assert store.latest(hid) == version diff --git a/agent_sys/tests/handoff/test_containment.py b/agent_sys/tests/handoff/test_containment.py index d7fb84776..f4296ea51 100644 --- a/agent_sys/tests/handoff/test_containment.py +++ b/agent_sys/tests/handoff/test_containment.py @@ -15,6 +15,7 @@ from handoff import check_contained, version_dir from handoff.errors import NotContained +from handoff.store import handoff_dir from task_graph.ids import HandoffId @@ -89,9 +90,9 @@ def test_a_path_that_does_not_exist_yet_is_still_checkable(tmp_path: Path) -> No def test_the_store_layout_is_what_containment_is_asserted_against(tmp_path: Path) -> None: - """Against the real layout, not a synthetic one: `//v/`.""" + """Against the real layout, not a synthetic one: `/handoff../v/`.""" root = tmp_path / "handoffs" mine, theirs = HandoffId.new(), HandoffId.new() - check_contained(version_dir(root, mine, 0), root / str(mine)) + check_contained(version_dir(root, mine, 0), handoff_dir(root, mine)) with pytest.raises(NotContained): - check_contained(version_dir(root, theirs, 0), root / str(mine)) + check_contained(version_dir(root, theirs, 0), handoff_dir(root, mine)) diff --git a/agent_sys/tests/handoff/test_digest.py b/agent_sys/tests/handoff/test_digest.py index 79f1d6ded..c6550b9c5 100644 --- a/agent_sys/tests/handoff/test_digest.py +++ b/agent_sys/tests/handoff/test_digest.py @@ -19,7 +19,7 @@ import pytest -from handoff import canonical, tree_digest +from handoff import canonical, tree_digest, version_dir from handoff.errors import Malformed # --------------------------------------------------------------------------- # @@ -93,7 +93,7 @@ def test_copy_out_raises_on_a_tampered_version(kinded_store, tmp_path: Path) -> store, hid = kinded_store version = store.put(hid, make_content(tmp_path / "produced"), producer=_task_id()) - (store.root / str(hid) / f"v{version}" / "content" / "items" / "result").write_text("99\n") + (version_dir(store.root, hid, version) / "content" / "items" / "result").write_text("99\n") from handoff.errors import DigestMismatch diff --git a/agent_sys/tests/handoff/test_store.py b/agent_sys/tests/handoff/test_store.py index fb0a1e498..e8168356a 100644 --- a/agent_sys/tests/handoff/test_store.py +++ b/agent_sys/tests/handoff/test_store.py @@ -20,7 +20,7 @@ from handoff import FilesystemStore, Scope, store_name_for, version_dir from handoff.errors import Malformed from handoff.protocols import HandoffStore -from handoff.store import STAGING_PREFIX +from handoff.store import STAGING_PREFIX, handoff_dir from task_graph.ids import HandoffId, TaskId from tests.handoff.conftest import FixedKind, make_content, make_kind, open_kind @@ -181,7 +181,7 @@ def test_a_failed_put_leaves_no_staging_directory(tmp_path: Path) -> None: with pytest.raises(Malformed, match="README.md"): store.put(hid, bad, producer=TaskId.new()) assert store.list_versions(hid) == [] - assert list((store.root / str(hid)).glob(f"{STAGING_PREFIX}*")) == [] + assert list(handoff_dir(store.root, hid).glob(f"{STAGING_PREFIX}*")) == [] def test_a_store_with_no_kind_source_reads_but_does_not_publish(tmp_path: Path) -> None: @@ -294,8 +294,8 @@ def test_knowledge_instance_is_separate(tmp_path: Path) -> None: assert knowledge.exists(hid, 0) assert not handoffs.exists(hid) - assert (tmp_path / "knowledge" / str(hid) / "v0").is_dir() - assert not (tmp_path / "handoffs" / str(hid)).exists() + assert version_dir(tmp_path / "knowledge", hid, 0).is_dir() + assert not handoff_dir(tmp_path / "handoffs", hid).exists() def test_a_store_needs_a_root(tmp_path: Path) -> None: diff --git a/agent_sys/tests/handoff/test_verdict.py b/agent_sys/tests/handoff/test_verdict.py index 04450906b..d42577126 100644 --- a/agent_sys/tests/handoff/test_verdict.py +++ b/agent_sys/tests/handoff/test_verdict.py @@ -16,7 +16,7 @@ import pytest import yaml -from handoff import Manifest, Verdict +from handoff import Manifest, Verdict, version_dir from handoff import verdict as verdict_mod from handoff.errors import Malformed from task_graph.ids import AgentId, TaskId @@ -91,7 +91,7 @@ def test_an_empty_list_and_a_missing_file_mean_different_things( whitelist mean very different things."*""" store, hid = kinded_store version = store.put(hid, make_content(tmp_path / "c"), producer=TaskId.new()) - path = store.root / str(hid) / f"v{version}" / verdict_mod.VERDICT_FILE + path = version_dir(store.root, hid, version) / verdict_mod.VERDICT_FILE assert path.is_file() and store.read_verdicts(hid, version) == [] @@ -136,7 +136,7 @@ def test_a_verdict_with_no_agent_round_trips_as_null(kinded_store, tmp_path: Pat assert got == unattributed, "the whole record round-trips, not just the field" raw = yaml.safe_load( - (store.root / str(hid) / f"v{version}" / verdict_mod.VERDICT_FILE).read_text() + (version_dir(store.root, hid, version) / verdict_mod.VERDICT_FILE).read_text() ) assert raw["verdicts"][0]["agent_id"] is None assert "agent_id" in raw["verdicts"][0], ( diff --git a/agent_sys/tests/interfaces/test_handoff_layout.py b/agent_sys/tests/interfaces/test_handoff_layout.py index 916345601..5699a7e4b 100644 --- a/agent_sys/tests/interfaces/test_handoff_layout.py +++ b/agent_sys/tests/interfaces/test_handoff_layout.py @@ -5,7 +5,7 @@ on-disk shape is private"**, with Bazel #23576 as the reason: a path-shape change survived there only because consumers use `file.path` rather than composing strings. `env_mgr` composes the string anyway — `grants.py` and `meta.py` need -`//v/` to grant access to it. +`/handoff../v/` to grant access to it. **It is duplicated by construction, not by carelessness.** `docs/interfaces.md` §4.6 permits `env_mgr` to import `task_graph` and nothing else of ours, so it @@ -31,6 +31,8 @@ import pytest from env_mgr.fs.layout import handoff_version_dir +from env_mgr.fs.zone import slug as zone_slug +from handoff.store import slug as handoff_slug from handoff.store import version_dir CASES = [ @@ -52,13 +54,46 @@ def test_the_two_writers_of_the_layout_agree(root: str, hid: str, version: int) assert Path(handoff_version_dir(root, hid, version)) == version_dir(Path(root), hid, version) -def test_the_shape_is_root_then_id_then_v_number() -> None: +def test_the_shape_is_root_then_labelled_id_then_v_number() -> None: """Pin the shape itself, so a *matching* change to both still gets read. Without this, the pair could agree on something neither design describes. """ - assert version_dir(Path("/r"), "h-9", 3) == Path("/r/h-9/v3") - assert handoff_version_dir("/r", "h-9", 3) == "/r/h-9/v3" + assert version_dir(Path("/r"), "h-9", 3) == Path("/r/handoff.h-9/v3") + assert handoff_version_dir("/r", "h-9", 3) == "/r/handoff.h-9/v3" + + +def test_the_label_is_the_kind_and_only_handoff_knows_it() -> None: + """`handoff` writes the label; `env_mgr` only ever finds what is already there. + + So the two are *not* symmetric on the write path, and pinning that is the + point: a caller with a kind in hand gets ``handoff..``, and the + one without gets the unlabelled form. + """ + assert version_dir(Path("/r"), "h-9", 3, "trace") == Path("/r/handoff.trace.h-9/v3") + + +@pytest.mark.parametrize( + ("dirname", "hid"), [("h-9", "h-9"), ("handoff.h-9", "h-9"), ("handoff.trace.h-9", "h-9")] +) +def test_both_readers_find_a_directory_whatever_its_label( + tmp_path: Path, dirname: str, hid: str +) -> None: + """Including ```` bare — the shape written before labels existed. + + This is the whole backwards-compatibility claim, and it is asserted on both + sides of the duplication because either could regress alone. + """ + (tmp_path / dirname / "v3").mkdir(parents=True) + assert version_dir(tmp_path, hid, 3) == tmp_path / dirname / "v3" + assert handoff_version_dir(str(tmp_path), hid, 3) == str(tmp_path / dirname / "v3") + + +def test_a_label_never_gains_a_field_separator() -> None: + """A kind with a ``.`` in it must not add a field: the uuid is the last one, + and `handoff_dir`/`find_zone_dir` both key on that.""" + assert version_dir(Path("/r"), "h-9", 0, "a.b c") == Path("/r/handoff.a-b-c.h-9/v0") + assert zone_slug("a.b c") == handoff_slug("a.b c") == "a-b-c" def test_the_two_spellings_of_the_granted_subdirectories_agree() -> None: diff --git a/work.checkpoint.summary.md b/work.checkpoint.summary.md deleted file mode 100644 index 74f3c9a31..000000000 --- a/work.checkpoint.summary.md +++ /dev/null @@ -1,2690 +0,0 @@ -# Checkpoint summary — five-module parallel debug of `llm_e2e_performance_optimization` - -Append-only. One section per 30 minutes of wall clock. Earlier sections are -never revised, including their wrong estimates — the record over time *is* the -value of this file. - -Effort start (T+0) taken as **2026-09-02 08:28 UTC**, the minute the five -deliverable dirs under `/shared_nfs/yihou/agent_sys/debugging/` and the ws2 -runroot were created. - -Reporter reads, cheapest first: the five `*.debug.help.info.md` at the repo -root; `git log`/`git status` in the worktree; the run roots under -`/shared_nfs/yihou/agent_sys_debug/ws2/runroot/`; the deliverable dirs; and -`squeue -u yihou`. - ---- - -## T+0 — 2026-09-02 08:35 UTC (baseline) - -### Walltime countdown (the number that governs everything) - -| job | node | owner | ends (UTC) | remaining at this checkpoint | -|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | **16:17:41** | 7 h 42 m | -| `101053` | `crsuse2-m2m-276` | integration | **16:17:57** | 7 h 42 m | -| `101078` | `crsuse2-m2m-080` | deploy (GPU 0–1), analyze (2–3), kernel-opt (4) | **16:28:21** | 7 h 53 m | - -All three are 8 h holds, `TimeLimit=08:00:00`, confirmed by `scontrol show job`. -The BRIEF's "hard stop ≈ 16:2x UTC" is accurate; the precise wall is **16:17:41** -for the two whole-node jobs and **16:28:21** for the shared node. - -### 1. Progress - -**Effort: ~4 %.** Elapsed 7 minutes. Estimated remaining: unknown — see the -reliability note. - -| module | est. % | basis | -|---|---|---| -| deploy | ~12 % | two notes sections written; package load verified; blocked on image discovery | -| profiling | ~10 % | two notes sections written; transport blocker identified and being localised | -| analyze | 0 % observable | no notes file, no runroot, no deliverable | -| kernel-opt | 0 % observable | no notes file, no runroot, no deliverable | -| integration | 0 % observable | no notes file, no runroot, no deliverable | - -**Reliability of this estimate: very low.** Three of five modules have produced -no observable output at all, which at T+7min means "has not yet written" and not -"is not working" — I cannot distinguish those two states from the outside. The -two percentages I do give are inferred from notes content, not from any run -report; neither module has launched an `agent-sys run` yet as far as I can see. -No `runs/` directory exists under the ws2 runroot, so **zero graph executions -have started across the whole effort.** - -### 2. Current state, per module - -- **deploy** (`crsuse2-m2m-080`, GPU 0–1, ports 8100–8119). Has verified that - the `deploy-demo/` move did not break the package load — `agent-sys show` - reports 2 tasks / 2 closures / 2 validators per output phase, matching - pre-move. Currently investigating whether `infera/engine-sglang:test-local` - (present on the node, built ~7 h ago by another tenant) carries the - `qwen3_5.py` model definition, which would remove an image build from the - critical path. -- **profiling** (`crsuse2-m2m-079`, whole node, ports 8120–8139). Has proved the - package's remote transport unusable on this cluster and is rewriting - `assets/lib/remote.sh::on()` from `srun --overlap` onto `spur exec`, including - hand-serialising the environment because `spur exec` does not carry it. -- **analyze**, **kernel-opt**, **integration**: no observable output. Not - reported as blocked, not reported as running. - -### 3. Code problems (defects in the packages or in `agent_sys`) - -| # | module | problem | state | -|---|---|---|---| -| C1 | profiling | `assets/lib/remote.sh::on()` uses `srun --jobid … --overlap … --export=ALL`. The `srun` on this cluster is a **spur re-implementation**, not Slurm's: `--export` is rejected outright (`unexpected argument`), and even with it dropped the call needs a TTY and exits 128 under agent_sys bodies (no TTY). The transport is unusable as written. | **open** — fix in progress; intent is to make `spur exec` selectable rather than replace `srun` outright, since the other cluster still needs the srun form | -| C2 | profiling | Consequence of C1: `--export=ALL` was load-bearing. The remote side must see `AGENT_SYS_OUTPUT_*` and the whole `PD_*` block; `spur exec` delivers an empty environment (measured: `MARK=hello spur exec … 'echo $MARK'` → empty). `on()` must serialise the environment itself. | **open** — being written | - -Nothing yet reported as a defect in `agent_sys` itself this round. The five -framework limits in the BRIEF (1800 s settle budget, gate-failure-reported-as- -timeout, handoff locality allow-list, no `claude` in a validation zone, single -`--demo-root` knob) are **known and pre-recorded**, not new findings; they are -listed here only so a later reader does not re-derive them. - -One non-defect worth recording, because it looks like one: `agent-sys show` -REJECTs the deploy package without the four site vars -(`deploy.yaml:69:5::$[0].env.E2E_MODEL_NAME: no value for ${model_name}`). That -is the package working as designed. Do not read it as a load failure. - -### 4. Non-code problems (environment / localisation traps) - -| # | problem | state | -|---|---|---| -| E1 | **Docker images are per-node.** `infera/engine-sglang:gfx950-local`, built 2026-09-01 on `crsuse2-m2m-020`/`-188`, is **absent on `crsuse2-m2m-080`**. So is `lmsysorg/sglang:v0.5.17-rocm720-mi35x`, the base the BRIEF names. The node instead carries `infera/engine-sglang:test-local`, `infera/engine-vllm:test-local`, `lmsysorg/sglang:v0.5.12-rocm720-mi35x` (**.12, not .17**), and two other tenants' tags. | **open** — deploy is testing whether `test-local` carries `qwen3_5.py`; if not, an image build enters the critical path | -| E2 | `spur exec` runs at `pwd=/`, `HOME=/opt/spur`, and without `~/.local/bin` on PATH. Every script must `cd` first and export `HOME=/home/yihou` and `PATH="$HOME/.local/bin:$PATH"`. | **known, pre-recorded in BRIEF**; profiling re-measured and confirmed | -| E3 | Data dependencies the three imported packages default to — `/apps/tas/yaoc/...` — **do not exist here**. Specifically absent and not yet located: the AIPerf `conversation_trace.jsonl`, the gsm8k `test.jsonl`, and the analyze seed `gap_analysis.csv`. Synthesising stand-ins is sanctioned but must be declared loudly. | **open** — no module has reported locating or synthesising any of the three | -| E4 | `/shared_nfs` is 98 % full (~7 T free). Image builds and weight copies must be sized against that. | **open, latent** | - -### 5. Undetermined - -1. **`spur exec` identity: root or `yihou`?** The BRIEF states it "runs as `root` - at `pwd=/`". The profiling module's own measurement on `crsuse2-m2m-079` - reports `id -un` → **`yihou`**. Both cannot be right, and which it is - determines whether a container can write where we expect. Not resolved here - by guessing; needs one `spur exec id` per node, and it may genuinely - differ per node or per job. -2. Does `infera/engine-sglang:test-local` carry `qwen3_5.py`? Deploy is checking. - Everything about whether an image build is on the critical path hangs on it. -3. Is `infera/engine-sglang:glm53-flash` present on `crsuse2-m2m-079` or - `-276`? The BRIEF guesses "probably not". No module has reported a - `docker images` from those two nodes yet. A 9m25s build is affordable; not - knowing for another hour is not. -4. Do the profiling / integration packages' **bodies or validators hard-code - GLM**, or are `model_path` / `image` / `served_name` / `tp` genuinely free - variables? The BRIEF's cheap road — run them against Qwen3.6-27B at tp=2 on - an image that already exists — depends entirely on this, and nobody has - reported reading the files. -5. Whether analyze, kernel-opt and integration have started at all. No output is - not evidence of no work at T+7min, but it is also not evidence of work. - -### 6. New commits - -Since the effort began (T+0 baseline, so this is the starting point rather than -a delta): **none.** Worktree HEAD is - -``` -532da57 refactor(llm_e2e): stage 1 moves into `deploy-demo/`, and the root becomes a container -``` - -`git status` shows three untracked entries: `.serena/`, -`deploy.debug.help.info.md`, `profiling.debug.help.info.md`. The two notes files -are expected untracked working output; the leader handles git. - -### 7. Other - -- **No `agent-sys run` has been launched yet by anyone.** The ws2 runroot - `/shared_nfs/yihou/agent_sys_debug/ws2/runroot/` is empty, and all five - deliverable dirs under `/shared_nfs/yihou/agent_sys/debugging/` are empty. - This is the single most important fact in the baseline: at T+7min the effort - is entirely in reconnaissance, and the 1800 s settle budget means a run is not - a small commitment once started. -- A stray `spur-101078.out` (33 bytes) sits in the package root - `agent_sys/examples/llm_e2e_performance_optimization/`. Harmless, but it is - inside the deliverable tree and should not be committed. -- The two modules that have written notes are both following the "append as you - go" instruction, and both notes are already good: each names the file, the - exact error string, and what a later reader should do instead. That is the - behaviour the BRIEF asked for. - ---- - -## T+30 — 2026-09-02 09:02 UTC - -### Walltime countdown — **CHANGED, badly** - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | RUNNING (44 m) | 16:17:41 | 7 h 15 m | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (44 m) | 16:17:57 | 7 h 15 m | -| ~~`101078`~~ | ~~`crsuse2-m2m-080`~~ | deploy, analyze, kernel-opt | **CANCELLED 08:55:44** | — | **gone after 27 m of an 8 h hold** | -| `101146` | (none) | replacement for the above three | **PENDING** | — | **cannot launch** | - -**This is the headline of the checkpoint.** `scontrol show job 101078`: -`JobState=CANCELLED Reason=None`, `RunTime=00:27:22`, `TimeLimit=08:00:00`, -`EndTime=2026-09-02T08:55:44`. Not cancelled by us. An 8 h hold was reaped at 27 -minutes. The replacement `101146` (submitted 08:57:20, `Restarts=1`) is stuck: - -``` -Reason=JobLaunchFailure (dispatch confirmation failed (0/1 confirmed): 1 agent unreachable) -``` - -So **three of five modules currently have no GPU node at all**, and the -mechanism that killed the first one is not understood. The two surviving holds -are the only compute the effort has. - -### 1. Progress - -**Effort: ~40 %.** Elapsed 34 minutes. Estimated remaining: **2–4 h if 101146 -lands soon**; unbounded if it does not. - -| module | est. % | basis | -|---|---|---| -| profiling | ~65 % | image rebuilt, `remote.sh` localised and committed, Qwen substitution proven to work, trace synthesised; run `profiling-a` root exists | -| integration | ~55 % | run `integration-r1` live at 08:54 on a surviving node, 3 handoffs open; six package files modified | -| analyze | ~55 % | run `analyze-dry2` reached `identify: succeeded` → `build_workset: running`; seed CSV synthesised and shown to reproduce the package's documented result; **node lost mid-run** | -| kernel-opt | ~50 % | mock path chosen and documented, install recipe working, run `kernel-opt-mock1` launched, 2 handoffs with a v1; **node lost mid-run** | -| deploy | ~45 % | two runs launched (`deploy-d1`, `deploy-d2`), CPX discovery forced a restart, `deploy-d2` was ~6 min into its agent phase when **the node was cancelled under it** | - -**Reliability: medium.** Upgraded from "very low" because all five modules now -have substantial written notes and five run roots exist with real state -transitions in them — I am reading artefacts, not guessing. But three caveats -pull it down: (a) percent-complete for the three orphaned modules is a measure -of *knowledge acquired*, not of *handoff produced*, and the runs that would have -converted one into the other were killed; (b) **zero handoffs have been -deposited** — all five dirs under `/shared_nfs/yihou/agent_sys/debugging/` are -still empty, so by the delivery contract the effort is at 0 % delivered; (c) I -cannot forecast `101146`. - -### 2. Current state, per module - -- **profiling** (`crsuse2-m2m-079`, alive). The strongest position. Has rebuilt - the image on-node (~25 min), localised `remote.sh` onto a selectable transport - and **committed it** (`6d6b053`), synthesised the missing AIPerf trace, and — - the important one — **proved the Qwen3.6-27B substitution works**: notes - section 9, "the GLM hard-coding is inert, not blocking". That answers - undetermined item 4 from T+0 for this package. -- **integration** (`crsuse2-m2m-276`, alive). Run `integration-r1` - (`20260902T085439-d482be`) launched 08:54 with three handoff slots open. Six - files modified in the worktree, uncommitted. Found that the node's image has - `qwen3_5` but not `glm5_next` — which *decides* the model rather than merely - suggesting it — and that `mix_worker.sh` hard-codes two GLM-only flag groups - that fail as numbers. -- **analyze** (node lost). Run `analyze-dry2` had got `identify` through - `output_validating → succeeded` and `build_workset` into `running` when the - hold died. Four handoff slots open. Committed `4011eb7`. -- **kernel-opt** (node lost). Run `kernel-opt-mock1` launched 08:37 on the mock - path; two handoffs, one already at v1. -- **deploy** (node lost). `deploy-d1` was launched with `tp_size=1` from prior - art, aborted when CPX was discovered at minute 14; `deploy-d2` relaunched - 08:50 with corrected sizing and was ~6 min into the agent phase at - cancellation. Has written a restart procedure for a fresh node. - -### 3. Code problems - -| # | module | problem | state | -|---|---|---|---| -| C1 | profiling | `remote.sh::on()` used `srun --overlap --export=ALL`; this cluster's `srun` is a spur re-implementation that rejects `--export` and needs a TTY (exit 128). | **fixed** — commit `6d6b053` "make the compute-node transport a variable, not srun"; srun form kept selectable | -| C2 | profiling / integration | `spur exec` carries no environment, so `--export=ALL`'s job (delivering `AGENT_SYS_OUTPUT_*` and the `PD_*` block) must be done by hand-serialising it. | **fixed** in profiling (part of `6d6b053`); **open** in integration (`remote.sh` modified, uncommitted) | -| C3 | analyze | `verify_workset` could not reach the GPU as written, same `srun` root cause. | **fixed** — commit `4011eb7` "the GPU transport and the visible cards are parameters" | -| C4 | integration | `mix_worker.sh` hard-codes two GLM-only flag groups; both fail as numbers under a non-GLM model. | **open** — file modified, not committed | -| C5 | kernel-opt | The run refuses to start unless the repo sets `extensions.preciousObjects`; setting it **on a worktree hits every other worktree** (analyze notes §3 independently). A framework-level foot-gun affecting any multi-worktree layout. | **worked around**; underlying behaviour **open** | -| C6 | kernel-opt | Python 3.10 in the only torch image makes `temp/bugs/001` live. | **open**, worked around by the install recipe in notes §6 | -| C7 | analyze | An `identify` handoff declared `usage` naming `'seconds'`, which the task did not declare: `4.596789008937776 is not booked`. Emitted as a console warning, not a failure. Whether it is a package bug or a framework leniency is not settled. | **open**, non-blocking | - -**A BRIEF correction, first-hand and load-bearing.** BRIEF known-limit 1 says -the settle budget is **1800 s, hard-coded at `cli/main.py:790`**. Two modules -checked the actual file: `agent_sys/cli/main.py:903` reads -`_SETTLE_TIMEOUT = 14400.0` (4 h), with a comment recording 300 → 1800 → 14400, -each raise caused by a healthy run being reported as a hang. **The 1800 s figure -is stale.** `temp/bugs/003` and `005` should be re-read against the current -constant. This removes a constraint several modules had designed around. - -### 4. Non-code problems - -| # | problem | state | -|---|---|---| -| **E0** | **An 8 h hold was cancelled at 27 minutes with `Reason=None`, and its replacement cannot dispatch (`1 agent unreachable`).** Three modules lost their node simultaneously. | **OPEN — the effort's top risk.** Cause unknown | -| **E5** | **`crsuse2-m2m-080` is CPX-partitioned: 64 devices × 36 GiB, not 8 × 288 GiB.** Confirmed first-hand: `rocm-smi --showcomputepartition` → CPX on GPU[0],[8],[16],…; `torch.cuda.device_count()` → 64; every device 36.0 GiB; UUIDs identical within each group of eight. **The BRIEF's "8 × MI355X, 288 GiB each" is false on that node.** A 52 GB bf16 model cannot fit TP1 in 36 GiB — this is what aborted `deploy-d1`. Also means "GPU 2–3" in the port/GPU allocation is not HIP device 2 and 3. | **open** — must be checked per node; unknown for `-079` and `-276` | -| **E6** | **A zone on `/shared_nfs` segfaults every ROCm kernel launch.** Measured three times: with `TMPDIR` anywhere under `/shared_nfs`, `torch.ones(4, device="cuda")` exits **139** (SIGSEGV) on the first kernel launch; with `TMPDIR` unset or on `/mnt/m2m_nobackup`, exit 0. Not a hang, not a permission error. Cost kernel-opt 25 minutes of an agent correctly bisecting a fault it could not name. | **understood, worked around** — put scratch on `/mnt/m2m_nobackup`. **Every module using a GPU from an NFS-rooted zone must apply this.** | -| E1 | Images are per-node; no image the BRIEF names existed on the nodes as given. | **resolved by cost** — profiling rebuilt on-node (~25 min); integration found `qwen3_5` present, `glm5_next` absent, which settles its model choice | -| E7 | Two cold starts is integration's budget problem, and the **aiter JIT build** is most of it. | **open** | -| E3 | Missing data dependencies. | **partly resolved** — analyze synthesised the seed CSV and verified it reproduces the package's own documented result; profiling synthesised the AIPerf trace. Both declared in notes. Magpie's kernel finder is absent; analyze records that as a *supported* outcome | -| E8 | `docker commit` freezes an `--entrypoint` override into the image (profiling §10). | **noted** | - -### 5. Undetermined - -1. **Why was `101078` cancelled?** `Reason=None` after 27 m of an 8 h limit, not - by us. Deploy's notes observe the survivors were also at only ~39 m and - suspects "something is reaping these allocations well short of walltime." If - that is systemic, `101052` and `101053` are not safe either and the whole - plan needs shorter, checkpointed units of work. **Unresolved and urgent.** -2. **Will `101146` dispatch?** `1 agent unreachable` is a control-plane fault, not - a queue wait. No estimate. -3. **Are `crsuse2-m2m-079` and `-276` CPX or SPX?** Deploy's restart procedure - opens with this question because it decides `tp_size` "and nothing else - does". Profiling and integration have runs in flight on those nodes; neither - has reported the partition mode. Cheap to answer, expensive to assume. -4. **`spur exec` identity — root or `yihou`?** Still open from T+0. Deploy's - notes use `docker exec -u 50112975:1000`, suggesting a numeric-uid reality - more complicated than either answer. -5. Is the analyze `usage`/`'seconds'` warning a package bug or framework - leniency? Nobody has adjudicated it. -6. **Can the three orphaned runs be resumed, or must they restart?** Deploy has - written a restart procedure, implying restart. If `--resume` cannot recover - an interrupted task's open output slot (BRIEF limit 1), all three lose their - elapsed run time, not just their node. - -### 6. New commits - -Two since T+0: - -``` -4011eb7 feat(analyze-demo): the GPU transport and the visible cards are parameters -6d6b053 fix(profiling-demo): make the compute-node transport a variable, not srun -``` - -- `4011eb7` — analyze's fix for C3: makes the GPU transport and the visible-card - set package variables instead of a baked-in `srun` call, so the CPX/device-id - mismatch and the transport swap are both configurable. -- `6d6b053` — profiling's fix for C1/C2: the compute-node transport becomes a - variable, `spur exec` selectable alongside the original `srun` form. - -Both are correctly scoped to one module each, as the BRIEF requires. -`git status` additionally shows six **uncommitted** modifications under -`integration-demo/` (`shared.yaml`, `assets/lib/remote.sh`, -`assets/serve/{mix_up,mix_worker,round}.sh`, `assets/accept/measure.sh`) — work -in flight, at risk if that node also dies. - -### 7. Other - -- **Delivered: nothing.** All five dirs under - `/shared_nfs/yihou/agent_sys/debugging/` remain empty. Five run roots with - live state exist, but the contract is the sealed handoff plus `PROVENANCE.md`, - and none has been copied out. **The gap between "45–65 % done" and "0 % - delivered" is the number to watch.** -- All five modules are now writing notes as they go, and the cancellation proved - why: deploy's sections 1–8 survived because they were written as measured; the - run did not. That instruction earned its place today. -- Two findings here are worth more than this round — **E5 (CPX)** and **E6 (NFS - segfault)** are cluster facts that will mislead the next effort just as badly - if they stay in a module notes file. They belong in the BRIEF and in - `temp/bugs/`. - ---- - -## T+60 — 2026-09-02 09:32 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | RUNNING (1 h 11 m) | 16:17:41 | 6 h 45 m | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (1 h 11 m) | 16:17:57 | 6 h 45 m | -| `101155` | `crsuse2-m2m-019` | **deploy + kernel-opt** (and analyze's remaining leaves) | RUNNING (22 m) | ~17:07 | ~7 h 35 m | -| ~~`101078`~~ | ~~`-080`~~ | — | CANCELLED 08:55:44 | — | — | -| ~~`101146`~~ | — | — | never dispatched | — | — | - -**The node crisis is resolved.** `101146` never launched; a fresh hold `101155` -on `crsuse2-m2m-019` was obtained ~09:07 and the three orphaned modules moved -onto it. Recovery took roughly 12 minutes from cancellation to a running run on -new hardware. Kernel-opt's notes record `crsuse2-m2m-080` went **`down`** at -~08:58 — so the cancellation was a node failure, not a scheduler reap. That -partly answers T+30 undetermined item 1 and materially lowers the risk to -`101052`/`101053`. - -### 1. Progress - -**Effort: ~70 %.** Elapsed 64 minutes. Estimated remaining: **1.5–3 h.** - -| module | est. % | basis | -|---|---|---| -| **profiling** | **100 % — DELIVERED** | run `20260902T085925-9f72b7`, 7 tasks succeeded, 7 handoffs valid, **6 verdicts PASS**; deliverable + PROVENANCE.md on disk | -| **kernel-opt** | **100 % — DELIVERED** | run `20260902T090855-446cad`, 3 tasks succeeded, **3 verdicts PASS**; deliverable + PROVENANCE.md on disk | -| integration | ~70 % | `integration-r2` in flight; `serve_stock` finished; notes at §9 | -| analyze | ~70 % | all six leaves individually proven; needs one contiguous run or a stitched seal | -| deploy | ~60 % | `deploy-d3` launched 09:14 on a working node with a purpose-built image | - -**Reliability: good, for the first time.** Two modules are done and I verified -them by opening the artefacts, not by reading a claim: both deliverable trees -contain `manifest.yaml` + `validation.yaml` + sealed `content`, and both -PROVENANCE files quote the run report's verdict lines verbatim. The three -remaining estimates are still inference from notes and run roots. - -**Delivered: 2 of 5.** That is the number that was 0 at T+30. - -### 2. Current state, per module - -- **profiling — DONE.** Ran against **Qwen3.6-27B at tp=2** on the rebuilt - `gfx950-local` image, ports 8120–8122, with `--timeout 10800`. Every window - reduced to the cheapest satisfying value (`warmup_s=60`, `window_s=10`, - `stack_window_s=3`, `max_conc=32`), trace synthesised. Seven artefacts - delivered. Its PROVENANCE opens by refusing to let its own numbers be quoted. -- **kernel-opt — DONE.** Mock mode on GPU 4 of `crsuse2-m2m-019`, in a - `rocm/pytorch:rocm7.2.4…py3.12…2.10.0` image (better than the README's, and - it sidesteps the Python 3.10 bug), demo-root on **node-local** - `/mnt/m2m_nobackup` — mandatory because of E6. Its PROVENANCE is the most - self-critical document in the effort: it states in bold that no kernel was - optimised, marks `optimized_kernel.py` byte-identical to the seed (md5 - independently re-checked by the producer), and notes that - `check_speedup_substantiated` PASSed *without measuring anything*, by - documented mock behaviour at `check.py:213-217`. -- **integration** (`-276`, alive). On `integration-r2`. `serve_stock` completed; - `measure_stock` observed live. Committed `fc8699f`. -- **analyze** (moved off the dead node). Has proven **all six leaves** - individually — four in the killed `analyze-dry2` run, two by standalone probes - (`transport_probe.sh`, `packup_probe.sh`, both PASS, locality clean). What it - lacks is one contiguous run. `build_workset` costs ~10 min per operator, which - is its remaining cost driver. -- **deploy** (`-019`, `101155`, SPX). `deploy-d3` started 09:14:19Z, `tp_size=1`, - `mix` mode, on `infera/engine-sglang:gfx950-deploy` **built on-node in ~4 - minutes** (notes §12 carries the whole recipe). Third launch: d1 killed by - CPX, d2 killed by the node, d3 is the run. - -### 3. Code problems - -| # | module | problem | state | -|---|---|---|---| -| C1/C2 | profiling | srun transport + environment serialisation | **fixed**, `6d6b053`, and now **proven in a passing run** | -| C3 | analyze | GPU transport / visible cards as parameters | **fixed**, `2820d47` + `4011eb7` | -| C4 | integration | `mix_worker.sh` GLM-only flag groups; transport seam | **fixed**, `fc8699f` "localise the transport seam and unbind the model" | -| C5 | all | `extensions.preciousObjects` in a **worktree** writes to the SHARED common config (`/home/yihou/dev/git/infera/.git`), hitting four other agents' worktrees. Independently hit by analyze (§3), profiling (§11), integration (§7), kernel-opt (§5). | **worked around** — profiling and kernel-opt both ran from a private clone. The framework requirement is **open** and is the effort's most-repeated foot-gun | -| C8 | framework | **An unparseable `${...}` is passed through, not refused** — committed as `13d1c2b`. Related: integration §6, `${x-default}` is not agent_sys variable syntax and "fails far away from the cause". | **documented**, behaviour **open** | -| C7 | analyze | `usage` naming an undeclared `'seconds'` | **open**, non-blocking | -| C6 | kernel-opt | Python 3.10 / `bugs/001` | **avoided** — a py3.12 image sidesteps it | - -**Two BRIEF facts are now confirmed stale by three independent modules** -(kernel-opt §2, analyze §7, integration §9, profiling implicitly via -`--timeout 10800`): - -1. **The 1800 s settle budget does not exist.** `cli/main.py` has - `_SETTLE_TIMEOUT = 14400.0` and `--timeout` is a real flag (`main.py:166`). - Integration calls this "the single most expensive stale fact in the brief, - because it makes people trade away resolution to fit a ceiling that is not - there." The comment records the 1800 s value once killed a healthy 27 B - bring-up at exactly 1800.0 s and abandoned eight held GPUs. -2. **`Nothing has changed for 20 s` is a diagnostic, not a termination.** - Integration's run printed it during `serve_stock` and finished that task 276 s - later. - -### 4. Non-code problems - -| # | problem | state | -|---|---|---| -| E0 | The `101078` cancellation | **root cause found** — `crsuse2-m2m-080` went **`down`** ~08:58, so this was hardware/node failure. Recovered onto `101155` in ~12 min. `101146` never dispatched and was abandoned rather than waited on — the right call | -| E5 | **Partition mode varies BETWEEN nodes.** `-080` was CPX (64 × 36 GiB); `-019` is **SPX**, so `deploy-d3` runs `tp_size=1`. Deploy §11: "check yours, do not inherit a number." | **understood**; deploy now sets the per-node stanza via a `${GPU_NOTE}` variable and a thin wrapper rather than editing the driver | -| E6 | **NFS `TMPDIR` segfaults every ROCm kernel launch** (exit 139) | **understood, worked around, and committed** as `7016ee5`. Kernel-opt's delivered run put its demo-root on `/mnt/m2m_nobackup` *because* of this | -| E1 | Per-node images | **resolved by building**: profiling rebuilt `gfx950-local` (~25 min); deploy built `gfx950-deploy` in **~4 min** with the recipe in its §12. The 4-minute path is the one to reuse | -| E3 | Missing inputs | **resolved**: profiling synthesised `conversation_trace.jsonl` (generator `make_trace.py` shipped beside the handoff); analyze synthesised the seed CSV and showed it reproduces the package's documented result. Both declared in bold in their PROVENANCE | -| **E9** | **Qwen3.6-27B decodes at ~2.9 tok/s on two MI355X** (~345 ms/token), two orders of magnitude below bandwidth arithmetic for a dense 27 B BF16 at tp=2. Integration §8: "every number in this handoff inherits that." | **open** — does not block a mock sample, but it makes integration's wall-clock budget much worse and is a real anomaly | -| E7 | aiter JIT build dominates cold start | **open** | - -### 5. Undetermined - -1. **Why does Qwen3.6-27B decode at 2.9 tok/s?** (E9.) Integration ruled out - "the model is big" by arithmetic. Nobody has diagnosed it. It is the most - interesting open question in the effort and the one most likely to matter - beyond today. -2. **Can analyze deliver without one contiguous run?** All six leaves are proven, - four in a killed run and two by standalone probe. Whether a handoff stitched - from those is acceptable under "never hand-write a handoff the package did - not produce", or whether a fresh contiguous run is required, is a **judgement - call nobody has made**. At ~10 min per operator for `build_workset`, the - difference is maybe an hour. **This should be settled by the leader, not by - analyze alone.** -3. Was `101078`'s node failure isolated, or is `-080` symptomatic? `101146`'s - `1 agent unreachable` suggests the control plane knew something was wrong. -4. C7, the analyze `usage`/`'seconds'` warning — still unadjudicated. -5. `spur exec` identity — no longer blocking anything (everyone uses - `docker exec -u 50112975:1000`), but still formally unanswered. - -### 6. New commits - -Five since T+30: - -``` -2820d47 docs(analyze-demo): localisation notes — no srun, CPX cards, a synthetic seed, and a settle budget that moved -13d1c2b docs(llm_e2e): an unparseable ${...} is passed through, not refused -7016ee5 docs(kernel-opt-demo): a zone on NFS segfaults every ROCm kernel launch -fc8699f feat(integration-demo): localise the transport seam and unbind the model -e5bf9f7 docs(profiling-demo): localisation notes for the spur cluster -``` - -- `2820d47` — analyze's four localisation findings as package docs. -- `13d1c2b` — the only **framework-level** finding committed so far: agent_sys - passes an unparseable `${...}` through instead of refusing it. Correctly filed - at `llm_e2e` level, not under one module. -- `7016ee5` — the NFS/ROCm segfault, written up where the next reader will hit it. -- `fc8699f` — integration's C4 fix, transport seam + model unbinding. -- `e5bf9f7` — profiling's notes committed into the package. - -Four of the five modules have now committed. `git status` is clean of module -work except `deploy.debug.help.info.md` (untracked) — the four other notes files -have been committed into their packages, which is better than leaving them at -the repo root. - -### 7. Other - -- **The delivery gap from T+30 is closing correctly.** Both delivered modules - shipped `PROVENANCE.md` alongside the sealed handoff, and both PROVENANCE - files lead with what is *not* real. Kernel-opt's goes furthest: it separates - measured from synthetic file by file, and reports that its own A/A null - control makes `mean_case_speedup: 1.0` "ground truth by construction, not a - result". This is exactly the "read the artefact, not the exit code" discipline - the BRIEF asked for, applied by the producers to themselves. -- One genuine incidental finding, from kernel-opt's mock: the workset's baseline - of **55.40 µs was traced on gfx942/MI300X and does not reproduce on gfx950** — - this host measures **50.141 µs**, −9.5 %. The kit marks the cross-check - `DIVERGED` rather than hiding it. Any future gfx950 speedup must be taken - against 50.141 µs. -- The **4-minute image build** in deploy §12 supersedes profiling's 25-minute - rebuild. If a sixth module ever needs an engine image, that is the recipe. -- Still unaddressed from T+30: **E5 (CPX) and E6 (NFS segfault) are now in - package notes and one commit, but not in the BRIEF.** The next effort reads the - BRIEF first. - ---- - -## T+90 — 2026-09-02 10:02 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | RUNNING (1 h 42 m) | 16:17:41 | 6 h 15 m | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (1 h 42 m) | 16:17:57 | 6 h 15 m | -| `101155` | `crsuse2-m2m-019` | deploy, analyze, kernel-opt | RUNNING (52 m) | ~17:07 | ~7 h 05 m | - -Stable. No further node loss. Walltime is **not** the binding constraint on any -module right now. - -### 1. Progress - -**Effort: ~78 %.** Elapsed 94 minutes. Estimated remaining: **1.5–3 h.** - -| module | est. % | basis | -|---|---|---| -| **kernel-opt** | **100 % — DELIVERED (re-delivered)** | `run_mock4.log`; the T+60 delivery moved to `superseded/` and was replaced after a committed fix | -| **profiling** | **100 % — DELIVERED** | run A's handoff set stands; two later runs are hardening, not delivery | -| analyze | ~85 % | `analyze-r1`: **5 of 6 handoffs sealed, 5 verdicts PASS**; the sixth lost in a 20 s window. `analyze-r2` re-running, at `build_workset` | -| integration | ~75 % | `integration-r3` starting; four new findings written since T+60 | -| **deploy** | **~60 %, and it is now the module I know least about** | `deploy-d3` console has not advanced past `running` since **09:14** and its notes have not grown since **09:17** — 45 minutes silent | - -**Reliability: good for four modules, poor for deploy.** Two deliveries verified -on disk. Analyze and integration are both writing detailed notes and have live -run roots. **Deploy is the gap**: a console frozen at `input_validating -> -running` for 45 min is the *expected* appearance of a long agent phase — the -agent works inside the container and the console stays quiet — so this is not -evidence of trouble. But it is also not evidence of progress, and deploy is the -only module whose notes have not grown in this interval. I am not going to score -it from nothing; I am flagging it. - -### 2. Current state, per module - -- **kernel-opt — DELIVERED, second edition.** Moved the T+60 deliverable into - `superseded/` and re-ran (`run_mock4.log`) after committing `45721e6`: the GPU - target is a fact about the host, not about the package. Re-delivering rather - than patching the shipped artefact is the right instinct. -- **profiling — DELIVERED, still hardening.** Run A's seven artefacts stand. - Runs B and C exposed two further faults (below) and produced two commits. -- **analyze.** `analyze-r1` (`20260902T091144-096985`, `top_n=2`) got five - handoffs sealed and **five verdicts PASS**; `build_workset` cost **1419 s, 67 - turns, $13.79**; `verify_workset` measured both operators. The terminal - `analyze_packup` stuck at `generating`. `analyze-r2` is re-running. -- **integration.** `integration-r3`. Four findings since T+60 — a missing eval - module, a hard floor in the eval size, and a constraint on synthetic traces. -- **deploy.** `deploy-d3` in its agent phase since 09:14:19Z. Silent. - -### 3. Code problems - -New this interval: - -| # | module | problem | state | -|---|---|---|---| -| **C9** | **framework** | **A terminal task gets 20 s total for prepare, body and seal.** This is what cost analyze its sixth handoff. Analyze ruled out both usual suspects first-hand: the gate's executable rule (`ls -la` shows the `command` item at **0755**, `packup.py:122`'s `chmod` took) and a slow body (the same body over the same sealed handoffs runs in **0.444 s**). Filed as a bug doc, commit `6e313e8`. | **documented, open** — a real framework defect, and the most consequential code finding since the settle-budget correction | -| **C10** | integration | `sglang.test.run_eval` is **missing from an image that has sglang** | **open/worked around** | -| **C11** | integration | `min_scored_per_eval` is **20, hard-coded**, so `eval_examples` has a floor — the BRIEF's "an eval over 20 questions is fine" is a floor, not a suggestion | **open**, by design | -| **C12** | profiling | `DSA_ARGS`/`PARSER_ARGS` hoisted out of `mix_worker.sh` | **fixed**, `2735e0a` | -| **C13** | profiling | `agent-sys` refuses to start with *"the 'claude' backend is not on PATH"* **even for a package with no AI agent at all** — every closure in profiling-demo is `kind: program`. The check is unconditional. | **open**, worked around by exporting PATH | -| **C14** | framework | **A nested default is a load error**, same family as C8's bare dash | **documented**, `0d6c1b6` | -| C15 | kernel-opt | GPU target was baked into the package | **fixed**, `45721e6` | - -Carried forward: C5 (`preciousObjects` hits sibling worktrees) and C8/C14 -(variable-syntax errors reported far from their cause) remain open. - -### 4. Non-code problems - -**A correction to my own T+30 and T+60 reporting.** I wrote E6 as *"a zone on -`/shared_nfs` segfaults every ROCm kernel launch"* and advised keeping run roots -off NFS. **That rule is too broad.** Profiling ran a full 7-task graph — two -engine bring-ups, two AIPerf replays, four profiler captures, thousands of -kernel launches — with `--demo-root` **on `/shared_nfs`** and saw no segfault. -Kernel-opt accepted the correction and narrowed the rule (its §14): - -> `TMPDIR` on NFS kills ROCm kernel launches **only for processes that run -> inside the agent_sys zone.** - -The two shapes differ: profiling's kernels all run inside a docker container the -zone *starts*, which has its own `/tmp`; kernel-opt runs `driver.py` in the zone -on the host python, which is where the fault bites. So: - -| what | where | -|---|---| -| `--demo-root` (workspace, playground, handoffs) | **`/shared_nfs`** — the user's standing instruction | -| `TMPDIR` for a **zone-launched** GPU process | **node-local disk** | - -Per the append-only rule I have not edited T+30 or T+60; the earlier, broader -statement stands there as written and is corrected here. Anyone reading this -file for the rule should take **this** paragraph. - -New: - -| # | problem | state | -|---|---|---| -| **E10** | **The login node OOM-kills a long run, and the symptom names nothing.** Profiling's run B died with no error line: log stops mid-graph, task sits at `running`, **the body outlived the driver** — it took SIGPIPE writing to the dead parent's stdout just before handoff assembly, leaving a `claim` and an **empty `content/`**. `crs-m2m-cpu-spur-012` was at **1 GB free of 62 GB, load 30.41, 169 users**. Nothing in the package had changed. | **fixed** — run `agent-sys` **on the compute node**; commit `8274a08` adds `PD_TRANSPORT=local`. Deliberately never chosen by `auto`, because "neither transport binary is present" is not the same fact as "I am on the node", and guessing wrong runs every GPU command on the login node | -| **E11** | `pgrep -f "agent-sys run"` **matches your own shell** — a trap. Use `ps -eo pid,cmd \| grep agent-sys \| grep -v grep`. | noted | -| **E12** | A synthetic Mooncake trace must give each `hash_id` **one fixed block size** | **fixed** in integration's generator | -| E9 | Qwen3.6-27B at ~2.9 tok/s | **still open, still undiagnosed** | - -### 5. Undetermined - -1. **What is deploy doing?** 45 minutes of silence on both console and notes. - Consistent with a healthy long agent phase; also consistent with a stall. - Cheap to answer and I would rather ask than score it. -2. **Why does `analyze_packup` need more than 20 s?** C9 says the budget is 20 s - and the body takes 0.444 s. Those two facts do not yet explain a failure — - something between prepare and seal is consuming the rest, and analyze has not - yet named it. **The most interesting open question in the effort right now.** -3. **E9, the 2.9 tok/s decode.** Untouched since T+60. -4. **Is profiling's delivered run A safe from E10?** Run A predates the - discovery. Its verdicts were printed and its content is on disk, so the - empty-`content` signature does not apply — but nobody has re-checked run A's - artefacts against the trap that killed run B. -5. C7 (`usage`/`'seconds'`) — still unadjudicated, three checkpoints on. - -### 6. New commits - -Six since T+60: - -``` -2735e0a fix(profiling-demo): hoist the DSA and parser flag groups out of mix_worker.sh -6e313e8 docs(llm_e2e): a terminal task gets 20 s for prepare, body and seal -8274a08 feat(profiling-demo): add a 'local' transport for driving from the node itself -c502d00 docs(kernel-opt-demo): record the TMPDIR fix and the run that proves it -0d6c1b6 docs(llm_e2e): a nested default is a load error, same family as the bare dash -45721e6 fix(kernel-opt-demo): the GPU target is a fact about the host, not about this package -``` - -- `2735e0a` — GLM-only flag groups become variables (integration's C4 in - profiling's copy). -- `6e313e8` — **the 20-second terminal-task budget**, a 116-line bug doc. The - highest-value commit of this interval. -- `8274a08` — the `local` transport, fixing E10. -- `c502d00` — the TMPDIR fix plus the run proving it. -- `0d6c1b6` — nested-default load error, third in the variable-syntax family. -- `45721e6` — GPU target de-hardcoded. - -Three of six are `docs(llm_e2e)` **framework-level** bug records rather than -module fixes. That ratio is healthy: the effort is now finding defects in -`agent_sys` itself, not just localising packages. - -Uncommitted: one modified `integration-demo/assets/accept/lm_eval.sh`, plus four -notes files modified in place and `deploy.debug.help.info.md` still untracked. - -### 7. Other - -- **Delivered: 2 of 5**, unchanged in count but not in quality — kernel-opt - superseded its own T+60 delivery after finding a fix worth re-running for. - Re-delivering beats patching a sealed artefact. -- **Analyze's 5-of-6 is the near-miss to watch.** It has PASS verdicts on - `check_kernel_table`, `check_worklist_shape`, `check_identity_resolved`, - `check_workset_shape` and `check_workset_runs`. Only the terminal seal is - missing, and the cause is a framework budget, not the package. -- **Cost is now visible**: `build_workset` alone was **$13.79** for 67 turns. - Nobody has aggregated spend across the effort; at five modules with reruns it - is no longer negligible. -- Deploy's 4-minute image recipe (§12) and profiling's `local` transport - (`8274a08`) are the two reusable assets produced today that a later effort will - want first. - ---- - -## T+120 — 2026-09-02 10:33 UTC - -### Walltime countdown — **a second node lost** - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | RUNNING (2 h 12 m) | 16:17:41 | 5 h 45 m | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (2 h 12 m) | 16:17:57 | 5 h 45 m | -| `101576` | `crsuse2-m2m-260` | deploy (+ analyze, kernel-opt) | RUNNING (12 m) | ~18:18 | ~7 h 45 m | -| ~~`101155`~~ | ~~`-019`~~ | — | **CANCELLED 10:18** | — | — | -| ~~`101078`~~ | ~~`-080`~~ | — | CANCELLED 08:55 | — | — | - -**Two of the three short-lived holds have now been reaped mid-run, neither at -walltime, 83 minutes apart.** Deploy's §15 states it plainly: `101155` died -at 10:18 with `deploy-d3` minutes from its validation phase, same signature as -§9. Meanwhile `101052` and `101053` have run **2 h 12 m untouched**. The -mechanism is not hitting every hold equally. - -**Operational rule this now justifies:** any module planning a run longer than -~40 minutes on a freshly issued hold should assume it may not survive, and -should checkpoint the artefact to `/shared_nfs` continuously rather than at the -end. Deploy did exactly that and it saved its deliverable. - -### 1. Progress - -**Effort: ~85 %.** Elapsed 125 minutes. Estimated remaining: **1–2.5 h.** - -| module | est. % | basis | -|---|---|---| -| **profiling** | **100 % — DELIVERED** | 7 handoffs, 6 verdicts PASS, run report | -| **kernel-opt** | **100 % — DELIVERED**, and now self-verified | `digest-selfcheck.txt`: both handoffs' manifest digests recomputed and **VERIFIED True** at 10:15:10Z | -| **deploy** | **~92 % — DELIVERED, with one validator short** | 62-file `content/` + PROVENANCE on disk; `check_deploy_kit` **PASS**, `check_deploy_reproduces` never ran | -| analyze | ~88 % | `analyze-r2`: **5 of 6 sealed, 5 verdicts PASS** — the *same* terminal failure as r1, now diagnosed to a framework line | -| integration | ~75 % | **no notes growth since 09:46** and no new run root — my blind spot this interval | - -**Reliability: good, with one hole.** Three deliverables verified on disk by -opening them. Analyze's two run consoles read directly. **Integration is now the -unknown** — 47 minutes without a note or a new run root. I said last checkpoint -I would rather ask than guess; the same applies here, and I have not asked -integration yet. - -**Delivered: 3 of 5** (2 clean, 1 with a documented gap). - -### 2. Current state, per module - -- **deploy — DELIVERED, and the honesty here is exemplary.** The node died - minutes before validation, so the handoff was **never sealed** and there is - **no run report**. Rather than dress that up, its PROVENANCE leads with *"Read - this before quoting a verdict"* and a two-row table showing one validator ran - and one did not, ending: **"Do not write this up as 'both validators PASS'."** - What it does have is real: `check_deploy_kit` run offline with the package's - own validator body and the exact `args.json` the run would have passed, - over this exact content → `{'…-0001': True}`. 62 files, one packup - `qwen3.6-27b-mix-sglang-gfx950.packup_20260902`. `deploy-d4` is now running on - `-260` to close the gap. -- **kernel-opt — DELIVERED and hardened.** Ten further notes sections (§16–§22) - on whether the delivered handoffs work as **fixtures** — answered "as - delivered, no", then fixed by a scripted relayout (`relayout_handoffs.py`, - commit `f93e982`, failure paths tested). Two findings settled **by experiment - rather than by reading**, including that "staging does not verify, and a - damaged fixture passes silently". -- **analyze — 5 of 6, twice, and now diagnosed.** `analyze-r2` reproduced r1 - exactly: `check_kernel_table`, `check_worklist_shape`, - `check_identity_resolved`, `check_workset_shape`, `check_workset_runs` all - **PASS**; `analyze_packup` stuck at `generating`. See C9 below — the - diagnosis is now precise and it is a framework defect. -- **profiling — DELIVERED.** One commit this interval (`91cb3b9`). -- **integration.** Unknown. Last note 09:46, last run root `integration-r3`. - -### 3. Code problems - -**C9 is now diagnosed, and it is an `agent_sys` defect, not a package one.** -Analyze ruled out every documented cause first-hand: - -- **not** the gate's executable rule — `items/command` is mode **0755**; -- **not** a slow body — the same body over the same handoffs runs in **0.444 s**; -- **not** seal refusal on locality — it ran the *framework's own* - `handoff.locality.check` (not the package's offline copy) over both the lost - `analyze_packup` content and the `operator_workset` content that sealed fine: - **both "locality OK"**. - -> The content was acceptable; **the version simply was never pinned.** - -The remedy is in `agent/runner.py` — `_seal_outputs` has a **silent-skip -branch** — and is therefore not available from a task package. Analyze also -found the institutional memory: `cli/README.md` records this symptom being -investigated once before, attributed to `HandoffStore.put` having no caller, -then "corrected by measurement" when `_seal_outputs` worked on the case tested. -**The two silent branches were left in place. This package's terminal task hits -one of them, two runs out of two.** That is a reopened bug with a reproducer. - -Its advice to a later reader: do not shrink the graph for the settle budget -(four hours now) — shrink so the **terminal task's prepare + body + seal fits in -twenty seconds**. Prepare grows with declared inputs, and `pack_analyze` declares -four, the most in the package. Cheap lever: lower `top_n`. - -| # | module | problem | state | -|---|---|---|---| -| **C9** | **framework** | terminal-task 20 s budget + `_seal_outputs` silent-skip in `agent/runner.py`; content valid, version never pinned | **diagnosed, open** — needs a framework fix | -| C16 | framework | **two `copy_out` functions, and the docstring promises the wrong one** — `dd59cf0` | **documented** | -| C17 | framework | **a chmod on the package source seals into a valid digest** — `5bc148a`; and the exec bits in the delivered handoff are "original to the seal because my chmod got there first" (§21) | **documented** — a supply-chain-shaped observation about what a digest does and does not attest | -| C18 | kernel-opt | delivered handoffs did not work as fixtures | **fixed** — relayout applied (`cae64eb`) and scripted (`f93e982`) | -| C19 | profiling | `REPRODUCE.md`'s machine note must match the transport | **fixed**, `91cb3b9` | - -Carried open: C5 (`preciousObjects`), C8/C14 (variable syntax), C10/C11 -(integration eval), C13 (unconditional claude-on-PATH check), C7. - -### 4. Non-code problems - -| # | problem | state | -|---|---|---| -| **E0′** | **Second unexplained hold cancellation** (`101155`, 10:18, mid-run). Two of three short holds reaped; the two long-lived ones untouched at 2 h 12 m. | **OPEN — recurring, still unexplained.** Now demonstrably a pattern, not an incident | -| **E13** | `deploy` recorded the mitigation that worked: because `--demo-root` was on `/shared_nfs`, the cancellation took *the run*, not *the artefact*. 62 files secured to the deliverable dir **immediately**, before anything else. | **the standing practice** — and note it cuts against a naive reading of the old E6 advice | -| E9 | Qwen3.6-27B ~2.9 tok/s | **still open, still undiagnosed** — three checkpoints | -| E10 | login-node OOM | **fixed** (`8274a08`) | - -### 5. Undetermined - -1. **Why are holds being cancelled?** Two in 83 minutes, mid-run, `Reason=None`, - while two other holds run untouched for over two hours. Deploy has raised it; - nobody can explain it. **This is the effort's top unresolved risk** and it is - outside any module's control. -2. **What is integration doing?** 47 minutes without a note or a run root. - Same blind spot deploy was at T+90 — and that one resolved into "working - hard, just not writing". Not scored from nothing. -3. **Will `deploy-d4` seal, or will it hit C9 too?** Deploy's terminal task is - the same shape as analyze's. If the 20 s budget bites deploy as well, that is - two of five packages blocked on one framework line, and the case for fixing - `_seal_outputs` rather than working around it becomes decisive. -4. **Should C9 be fixed rather than documented?** CLAUDE.md says fix only on - unambiguous evidence. Two reproductions, a named function, a ruled-out - alternative list, and a prior investigation that closed it wrongly is close to - unambiguous. **A leader-level call.** -5. E9 — untouched. - -### 6. New commits - -Eight since T+90: - -``` -dd59cf0 docs(llm_e2e): two copy_out functions, and the docstring promises the wrong one -cae64eb docs(kernel-opt-demo): apply the relayout instead of documenting it -ba0880e docs(llm_e2e): the staging route was disproved by experiment, not only read -9b05691 docs(kernel-opt-demo): the verifying copy_out is the producer side (closes section 18) -5bc148a docs(kernel-opt-demo): a chmod on the package source seals into a valid digest -f93e982 docs(kernel-opt-demo): script the delivery relayout, with its failure paths tested -91cb3b9 fix(profiling-demo): REPRODUCE.md's machine note has to fit the transport -e881e64 docs(deploy): the spur-cluster localisation notes for deploy-demo -``` - -Six of eight are kernel-opt or framework findings about the **handoff/digest/ -staging machinery** — a coherent line of investigation, each step closing a -numbered open question from the one before (`ba0880e` explicitly notes a route -was "disproved by experiment, not only read", and `9b05691` "closes section 18"). -`e881e64` finally commits deploy's notes, so all five modules have now committed. - -Uncommitted: `integration-demo/assets/accept/lm_eval.sh` plus three notes files -modified in place. - -### 7. Other - -- **Delivered 3 of 5**, and the quality bar has held under pressure. All three - PROVENANCE files lead with what is *not* real: profiling ("do not quote a - number"), kernel-opt ("no kernel was optimized", md5-identical, mock validator - passed without measuring), deploy ("do not write this up as both validators - PASS"). Under a node dying minutes before validation, deploy chose to ship the - gap rather than paper it. -- **Kernel-opt independently re-verified its own delivery**: `digest-selfcheck.txt` - recomputes both manifest digests and reports `VERIFIED True` for each, dated - and hostnamed. Nobody asked for that. -- **The framework, not the cluster, is now the main obstacle.** Node loss cost - time but no artefacts; C9 is the one defect standing between analyze and a - complete handoff, and possibly deploy too. -- **I have been blind on one module per checkpoint** — deploy at T+90, - integration at T+120. Both times the module was in fact working. That is worth - noting as a property of *this reporting method*, not of the modules: a module - deep in a long run looks identical to a stalled one from the outside. - ---- - -## T+150 — 2026-09-02 11:03 UTC - -### Walltime countdown — **a third node lost; only the original two remain** - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling | RUNNING (2 h 43 m) | 16:17:41 | 5 h 14 m | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (2 h 43 m) | 16:17:57 | 5 h 14 m | -| ~~`101576`~~ | ~~`-260`~~ | deploy/analyze/kernel-opt | **CANCELLED 10:54**, 34 min in | — | — | -| ~~`101155`~~ | ~~`-019`~~ | — | CANCELLED 10:18, ~70 min in | — | — | -| ~~`101078`~~ | ~~`-080`~~ | — | CANCELLED 08:55, ~30 min in | — | — | - -**Three of three freshly issued holds have been reaped. Zero of the two original -holds have.** Deploy's §16 lays out the pattern and draws the only safe -conclusion: - -> **On this cluster a freshly issued hold is not a resource you can plan a -> 60-minute job around.** - -Deploy, analyze and kernel-opt now have **no GPU node at all**. All three have -already delivered, so this is not currently blocking — but nothing GPU-bound can -be re-run by those three modules without a new hold, and new holds do not last. - -### 1. Progress - -**Effort: ~93 %.** Elapsed 155 minutes. Estimated remaining: **30–90 min**, -essentially all of it integration's. - -| module | est. % | basis | -|---|---|---| -| **profiling** | **100 % — DELIVERED**, relaid out to sealed-handoff shape (`store/`) | run report, 6 verdicts PASS | -| **kernel-opt** | **100 % — DELIVERED**, digests self-verified | 3 verdicts PASS | -| **deploy** | **100 % — DELIVERED**, one validator short and said so | `check_deploy_kit` PASS offline | -| **analyze** | **100 % — DELIVERED**, unsealed and said so | 5 of 6 sealed + 5 verdicts PASS; terminal content real but unsealed | -| **integration** | **~80 %** | alive on `101053`; lost an arm to a validator floor and is re-running | - -**Reliability: high for four modules, medium for integration.** Four -deliverables opened and read. Integration's notes resumed (496 lines, §13 at -10:48) so the T+120 blind spot is closed — it was working, as deploy's was. - -**Delivered: 4 of 5.** - -### 2. Correction to T+120: C9's root cause was *not* the 20-second clock - -At T+120 I reported the terminal-seal failure as a framework defect — a 20 s -budget plus `_seal_outputs`' silent-skip branch. **Analyze has since found the -actual cause and it is the package's own schema** (commit `a449191`, *"the root -cause was our own `items_schema`, not the clock"*). Run directly against the -real unsealed content: - -``` -check_items REFUSED: items $: Additional properties are not allowed - ('REPRODUCE.md', 'environment.md', 'notes.md', 'results' were unexpected) -``` - -`analyze_packup`'s `items_schema` declared six items with -`additionalProperties: false`; `packup.py` writes **ten**; and the four -undeclared ones are exactly the four that `check_analyze_packup_shape` -**requires**. In analyze's words: - -> **The producer, the validator and the kind had drifted apart, and the kind was -> the one nobody ran.** - -Five of six handoffs seal fine, so nothing exercised the mismatch until the -terminal one. **Fixed in `acb8bfe`**, verified both ways against real content: -`check_items` refuses under the old schema, accepts under the new. - -The framework half of C9 survives but is demoted from *cause* to *symptom -amplifier*: `seal` returns the refusal **as a string, not an exception** -(deliberately, so `agent` need not import `handoff`), `_seal_outputs` files it -under `seal_refused`, and `agent/runner.py`'s own docstring admits that key -"has no reader outside these tests yet". So **the reason exists, is correct, is -specific — and is discarded**; the operator sees a task stuck in `running` and a -timeout. That is commit `755e1a4`: *"the stall was the symptom; `seal_refused` -having no reader is the bug."* - -Per the append-only rule, T+120 stands as written. **This paragraph is the -correct account.** My T+120 recommendation ("shrink so the terminal task fits in -20 s") was aimed at the wrong target; the right first move is analyze's: - -```python -from handoff import content as c -c.check_items(c.load(Path(content_dir)), c.content_type("reproducible"), items_schema) -``` - -Any handoff whose producer writes more items than its kind declares is exposed, -and `additionalProperties: false` is house style in these packages. - -### 3. Current state, per module - -- **analyze — DELIVERED, honestly unsealed.** PROVENANCE opens: *"Read the two - warnings in section 1."* The terminal content is **`generating`, not `valid`** — - it predates `acb8bfe`. What is verified rather than assumed: - `check_analyze_packup_shape.check()` called directly returns - *PASS — 4 mandated file(s) present with substance*; the framework's own - `handoff.locality.check` passes; `check_items` refuses under the old schema and - accepts under the new. Its summary is the right one: *"byte-for-byte what a - sealed version would have held, and the run report does not say so. Both facts - are true and neither should be dropped."* -- **deploy — DELIVERED, and step 1 of its assignment answered in full.** Its §17 - answers the original question — did the move to `deploy-demo/` break anything — - **negatively and itemised**: package loads (2 tasks/2 closures/2 validators), - exec bits survived, both validator bodies still resolve and are non-vacuous - (all four controls reproduce), and a real run drove it end to end on two - different nodes. *"Nothing in this module's failures was caused by the move. - Every one was the cluster."* -- **profiling — DELIVERED**, now in sealed-handoff shape (`store/`), plus two - more identifiers turned into parameters (`f4f920c`) and a `check_items` drift - audit **clean for all seven kinds** — the same class of fault analyze was bitten - by, checked for proactively and found absent. -- **kernel-opt — DELIVERED.** No change since T+120. -- **integration.** Live on `101053`, the safest node. Lost an arm to a validator - floor (below) and is re-running. - -### 4. Code problems - -| # | module | problem | state | -|---|---|---|---| -| **C9** | **analyze package** | `items_schema` declared 6 items with `additionalProperties: false`; producer writes 10; the 4 undeclared are the 4 the validator requires | **FIXED**, `acb8bfe`, verified both directions | -| **C9b** | **framework** | `seal_refused` has **no reader** — a correct, specific refusal is discarded and surfaces as a stalled task | **documented, open**, `755e1a4`. Still worth fixing: it turned a one-line schema bug into two lost runs | -| **C20** | integration | `check_bench_report` FAILED on a complete bench handoff: `request_count.avg = 30.0` against `min_requests: 50`. `trace_end_ms=15000` looked free but the replay is **fixed-schedule** — the window truncates the trace, it does not compress it | **fixed**, by lengthening the trace rather than lowering the bar | -| C21 | profiling | container name and context length were hard-coded | **fixed**, `f4f920c` | -| C22 | deploy | README updated for the move | in flight (uncommitted) | - -**Integration's floor audit is the reusable artefact of this interval.** After -C20 it audited every numeric floor out of the step files — *"worth doing before -the first run, not after the third"*: - -| validator | arg | value | overridable? | -|---|---|---|---| -| `check_acceptance` | `min_scored_per_eval` | 20 | **no** | -| | `needle_min_depths_retrieved` / `needle_min_token_ratio` | 1 / 0.95 | **no** | -| `check_bench_report` | `min_requests` | 50 | yes | -| | `max_error_rate` | 0.05 | yes | -| | `expect_rounds` | `${bench_rounds:-2}` | yes, tracks `bench_rounds` | -| `check_packup_shape` | min content lines (README 20, REPRODUCE 15, environment 12, notes 8) | | **no** | -| | `min_command_lines` / `min_result_files` | 8 / 4 | **no** | -| `check_service_live` | `expect_workers` | 1 | **no** | - -This directly qualifies the BRIEF's "cheapest settings win": **there is a floor, -several floors are literals unreachable from the command line, and going under -one costs the whole arm.** Integration measured that cost at **50 minutes**, and -declined the available `--var min_requests=25` on the grounds that *"lowering the -bar to meet the sample changes what the validator means."* That is the right -call and it should be quoted at anyone tempted to tune a threshold to fit. - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| **E0″** | **Three of three freshly issued holds reaped mid-run; both original holds untouched at 2 h 43 m.** | **OPEN, now a confirmed pattern.** The two survivors were issued in the original batch; every later one died | -| **E14** | **The staging lesson — this is the finding of the interval.** `deploy-d3` and `-d4` were killed at similar maturity; one kit survived and one did not, and the difference was *only* where the agent staged it. d3 wrote **directly into the handoff directory** (under `--demo-root`, on `/shared_nfs`) → **62 files survived and pass the shape check**. d4 staged on `$E2E_WORK_ROOT` (`/mnt/m2m_nobackup`, **node-local**) intending to copy at the end → node died first, **4 KB README is all that remains**, including a 19/19 verified deployment. | **open as a package-content decision.** Deploy recorded it rather than changing it, since it is not its call. Recommendation: *write into the handoff as you go; local scratch is for the container's logs, not the deliverable* | -| E9 | Qwen3.6-27B ~2.9 tok/s | **still open**, four checkpoints | - -E14 and the narrowed E6 rule now interlock cleanly: `--demo-root` on -`/shared_nfs` (durable, and what survives a reap), `TMPDIR` on node-local disk -(only for zone-launched GPU processes). - -### 6. Undetermined - -1. **Why are freshly issued holds reaped?** Three for three. Unexplained, and - nobody on the effort can resolve it. It now has a clear operational - workaround, which is why it is no longer blocking. -2. **Should C9b be fixed?** The schema bug is fixed; the framework's discarding - of a correct refusal is not. Two runs and ~90 minutes were spent finding by - hand a reason the framework already had in a variable. **A leader-level call** - — carried from T+120 with a sharper case. -3. **Should the unsealed deliverables be re-run now `acb8bfe` exists?** Analyze's - content is byte-identical to what a sealed one would hold, and the fix is in. - One clean run would convert "unsealed but verified" into "sealed with a run - report" — but analyze has no node, and new holds die. **Weigh against the - BRIEF's "mock samples, not measurements".** -4. E9 — untouched. - -### 7. New commits - -Seven since T+120: - -``` -acb8bfe fix(analyze-demo): declare the four packup items the seal was refusing -755e1a4 docs(llm_e2e): the stall was the symptom; seal_refused having no reader is the bug -aa502e0 docs(profiling-demo): the empty-content diagnosis, the login-node OOM, and the final run -d632b9c docs(deploy-demo): the move is clean; three cancelled holds and what survived them -a449191 docs(analyze-demo): the root cause was our own items_schema, not the clock -f4f920c fix(profiling-demo): the container name and the context length are parameters -4422962 docs(profiling-demo): the sealed-handoff delivery shape, and the last two hoists -``` - -- `acb8bfe` — the real fix for C9. One schema change, two lost runs behind it. -- `755e1a4` — the framework half, correctly separated from the package half. -- `a449191` — **a module publicly correcting its own earlier diagnosis** (232 - lines). This is the behaviour that makes the notes trustworthy. -- `d632b9c` — deploy's answer to step 1, plus the three-cancellation record. -- `f4f920c`, `4422962`, `aa502e0` — profiling's hardening. - -Uncommitted: `deploy-demo/README.md`, `integration-demo/assets/accept/lm_eval.sh`, -`integration.debug.help.info.md`. - -### 8. Other - -- **4 of 5 delivered, and not one deliverable overclaims.** Profiling: don't - quote a number. Kernel-opt: no kernel was optimised, and the validator passed - without measuring. Deploy: not "both validators PASS". Analyze: `generating`, - not `valid`. Every gap is stated by the module that produced it, in bold, at - the top of its own PROVENANCE. -- **Two modules found the same class of bug from opposite ends**: analyze was - bitten by `items_schema` drift; profiling then audited `check_items` across - **all seven** of its kinds and found them clean. The second is only visible - because the first was written down within the hour. -- The effort is now essentially **one module wide** — integration, on the safest - node, with 5 h 14 m of walltime and one arm to re-run. - ---- - -## T+180 — 2026-09-02 11:34 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling — **and now deploy's `deploy-d5`** | RUNNING (3 h 14 m) | 16:17:41 | **4 h 43 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (3 h 14 m) | 16:17:57 | **4 h 43 m** | - -No new holds attempted since `101576` died. **The effort has consolidated onto -the two original holds** — deploy has moved `deploy-d5` onto `-079`, sharing -profiling's node, which is the correct response to "freshly issued holds do not -survive." Both holds are now past 3 h with no sign of the reaping that killed -three consecutive new ones. - -### 1. Progress - -**Effort: ~95 %.** Elapsed 186 minutes. Estimated remaining: **30–75 min**, -all of it integration's. - -| module | est. % | basis | -|---|---|---| -| **profiling** | **100 % — DELIVERED** | sealed store, 6 verdicts PASS | -| **kernel-opt** | **100 % — DELIVERED** | 3 verdicts PASS, digests self-verified | -| **deploy** | **100 % — DELIVERED**; `deploy-d5` running to close the second validator | `check_deploy_kit` PASS | -| **analyze** | **100 % — DELIVERED**, now with a `store/` alongside the raw content | 5 verdicts PASS | -| **integration** | **~85 %** | `integration-r4` (`20260902T104817-8a2995`) launched 10:48, **5 handoff slots open**, still running at 46 min | - -**Reliability: high.** Four deliverables read on disk. Integration's r4 has five -handoff directories and a live store — this is a real run in progress, not a -stall, which settles the T+120/T+150 uncertainty about that module. Its notes -last grew at 10:48, exactly when r4 launched, which is the expected pattern: a -module writes before and after a run, not during. - -**Delivered: 4 of 5**, with the fifth in flight. - -### 2. Current state, per module - -- **integration.** `integration-r4` is the run of record, launched 10:48 right - after the C20 fix (longer trace rather than a lowered bar). Four prior runs - r1–r3 behind it. Five handoff slots open. This is the whole remaining critical - path. -- **deploy.** Delivered, but not stopping there: `deploy-d5` is running on - `-079` to obtain the `check_deploy_reproduces` verdict its delivered kit - lacks. Also corrected a **stale control count** in its own notes (`8fe8c23`) — - a module auditing its own earlier claim without being asked. -- **profiling.** Delivered; added `§22 Node-local paths do not exist on the login - node` and a `per-node versus cluster-wide facts` split (`77a2c13`) — turning - today's environment lessons into a reusable distinction rather than a list. -- **analyze, kernel-opt.** Delivered, no node, no further work possible. - -### 3. The interval's main artefact: an `items_schema` audit of all five packages - -Deploy ran this while `deploy-d5` was in its agent phase, prompted by analyze's -C9. **Result: no drift in any of the five.** - -| package | evidence | finding | -|---|---|---| -| profiling | **all seven kinds sealed** in the delivered store | six `reproducible` carry exactly `[command, env, logs, result, watchout]`; `profile_packup` carries `[codes]` | -| kernel-opt | two sealed kinds | `[codes]` each; **neither declares an `items_schema`** | -| analyze | sealed `analyze_packup` | ten items, matching the widened schema | -| deploy | this module's kit | `[codes]`; `deploy_kit` **declares no `items_schema`** | -| integration | no content yet — producers read | ten kinds, **every one matching** | - -Two things in it are worth more than the result: - -**A method warning.** Grepping for `items/` is **unsound** on these bodies -and produced two false positives before deploy caught them by reading: -`measure.sh` appeared to omit a required `logs`, `seed.py` a required -`watchout`. Neither is true — they build item paths through per-arm shell -variables (`A="$OUT_ACCEPT/items"`, `ITEMS="$OUT/items"`), so the literal string -never appears. **Check real content with `handoff.content.check_items`; fall back -to reading only when no content exists.** A static grep here would have produced -two confident, wrong bug reports. - -**A structural conclusion**, which is the real lesson of analyze's bug: - -> **A closed `items_schema` that merely restates its content type buys nothing -> and carries the whole risk.** - -Four of five were never at risk *because their at-risk kinds declare no -`items_schema` at all* and fall back to the content type's rules, where producer -and type agree by construction. `deploy-demo/steps/deploy.yaml` already argues -this explicitly. `integration-demo`'s ten schemas are all of the risky shape: -correct today, **ten opportunities to drift tomorrow.** - -### 4. Code problems - -No new defects this interval. Standing: - -| # | problem | state | -|---|---|---| -| C9 | analyze `items_schema` drift | **FIXED** `acb8bfe` | -| **C9b** | `seal_refused` has no reader — a correct refusal is discarded, surfacing as a stalled task | **open**, `755e1a4`. Unchanged and still worth a leader decision | -| **C23** | `integration-demo`'s ten closed `items_schema`s restate their content types — latent drift risk | **open, observation only** — nobody has proposed changing them, and the BRIEF's one-module-per-commit rule means it is integration's call | -| C20 | `min_requests` floor | **fixed** by lengthening the trace | -| C5, C8/C14, C10/C11, C13, C7 | carried | **open** | - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| E0″ | Three freshly issued holds reaped; both originals now past **3 h 14 m** untouched | **open, and now routed around** — deploy consolidated onto `-079` rather than requesting a fourth hold. That is the right response and it is working | -| E14 | The staging lesson (write into the handoff as you go; local scratch is not for the deliverable) | **open as a package-content decision** | -| **E15** | **Node-local paths do not exist on the login node** — profiling §22. Obvious once stated, and it invalidates any login-side check of a `/mnt/m2m_nobackup` artefact | noted, `77a2c13` | -| E9 | Qwen3.6-27B ~2.9 tok/s | **still open**, five checkpoints | - -### 6. Undetermined - -1. **Will `integration-r4` complete before it needs another arm?** 46 minutes in, - five slots open, 4 h 43 m of walltime. Comfortable unless it loses an arm - again — and a lost arm costs ~50 min with no resume. -2. **Will `deploy-d5` land the `check_deploy_reproduces` verdict?** It is an AI - validator, so it needs `claude` reachable from the validation zone — the - pre-recorded BRIEF limit 4. Nobody has reported passing `claude_cli` for d5. -3. **C9b** — carried, third checkpoint. Still a leader call. -4. **E9** — untouched, five checkpoints. It will end the effort undiagnosed - unless someone picks it up, and that is a legitimate outcome to record rather - than a gap to hide. -5. **Should `integration-demo`'s ten closed schemas be opened?** (C23.) Deploy - found the risk; only integration can act on it, and it is mid-run. - -### 7. New commits - -Two since T+150: - -``` -8fe8c23 docs(deploy-demo): correct the stale control count, and audit every kind's items_schema -77a2c13 docs(profiling-demo): per-node versus cluster-wide facts -``` - -- `8fe8c23` — the five-package audit above, plus deploy correcting a stale count - in its own earlier notes. -- `77a2c13` — profiling separating per-node facts from cluster-wide ones, so the - next effort inherits a usable distinction rather than a flat list of traps. - -Commit rate has fallen sharply (2 in 30 min, against 8 in the previous interval) -— consistent with four modules done and one mid-run, not with a stall. - -Uncommitted: `integration-demo/assets/accept/lm_eval.sh` and -`integration.debug.help.info.md` — both integration's, both expected to land when -r4 finishes. - -### 8. Other - -- **The effort is one module wide and comfortably inside its walltime.** The - binding risk is no longer time or hardware; it is whether integration's last - arm passes its validators. -- **Deploy's audit is the best example today of a module doing work outside its - own deliverable.** Analyze found a bug at 10:43; by 11:04 deploy had checked - every other package for the same class, found none, and — more useful — - identified *why* four were structurally immune and where the latent risk still - sits. Neither module was asked to do this. -- Two of today's findings now have a general form worth carrying out of this - effort: **"a closed `items_schema` that merely restates its content type buys - nothing and carries the whole risk"**, and **"a static grep for item paths is - unsound; check real content."** -- Still unaddressed across six checkpoints: the cluster facts (CPX per node, the - narrowed NFS/`TMPDIR` rule, hold reaping, the numeric floors, the staging - lesson) live in module notes and commits but **not in the BRIEF**, which is - what the next effort reads first. - ---- - -## T+210 — 2026-09-02 12:05 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy (`deploy-d5`) | RUNNING (3 h 45 m) | 16:17:41 | **4 h 12 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (3 h 45 m) | 16:17:57 | **4 h 12 m** | -| `101753` | `crsuse2-m2m-267` | **analyze** (new) | RUNNING (19 m) | ~16:03 | ~3 h 58 m | - -A **fourth** hold was issued and — unlike the previous three — has survived 19 -minutes so far. Analyze took it to re-run against the `acb8bfe` fix and get a -genuinely sealed terminal handoff. Too early to say the reaping has stopped; one -19-minute survival is not evidence against three deaths at 30/34/70 minutes. - -### 1. Progress - -**Effort: ~97 %.** Elapsed 217 minutes. Estimated remaining: **30–60 min.** - -| module | est. % | basis | -|---|---|---| -| **deploy** | **100 % — DELIVERED, and now fully sealed with BOTH validators PASS** | see below | -| **profiling** | **100 % — DELIVERED** | 6 verdicts PASS | -| **kernel-opt** | **100 % — DELIVERED** | 3 verdicts PASS | -| **analyze** | **100 % delivered**, now **re-running for a sealed version** | `analyze-r5` at `verify_workset` | -| **integration** | **~90 %** | `integration-r4` alive, **8 handoff slots**, last write 11:52 | - -**Reliability: high.** Every claim below was checked by opening a file. - -**Delivered: 4 of 5**, one of them upgraded from "gap documented" to "complete". - -### 2. Deploy closed its gap — verified in the sealed artefact - -At T+120 and T+150 I recorded deploy as delivered with one validator short, on -its own insistence that it not be written up as "both validators PASS". `deploy-d5` -on `-079` has now closed that. From -`/shared_nfs/yihou/agent_sys/debugging/deploy/store/d54829ae-…/v1/validation.yaml`, -read directly: - -```yaml -- validator: check_deploy_kit - result: true strength: strong dimension: completeness - at: '2026-09-02T11:44:05Z' -- validator: check_deploy_reproduces - result: true strength: weak dimension: usability - at: '2026-09-02T11:59:24Z' -``` - -**Both validators ran in the run, both returned true, and the handoff is -sealed** (`claim`, `content`, `manifest.yaml`, `validation.yaml` all present, v1). -`check_deploy_reproduces` is the AI validator — a fresh Claude Code session -following `REPRODUCE.md` and bringing the model up again — so T+180 undetermined -item 2 is answered: it did reach `claude` from the validation zone. The unsealed -d3 content is retained alongside as `unsealed-deploy-d3/`, which is the right -call: it is the kit that survived a node death and it documents a different -lesson. - -**One discrepancy to flag rather than smooth over:** `PROVENANCE.md` in that -directory still has an **mtime of 10:24** and still carries the *"Read this -before quoting a verdict"* table saying `check_deploy_reproduces` did not run, -plus a section headed *"What the run did not get to"*. **The provenance file is -now stale with respect to its own deliverable, and understates it.** Almost -certainly deploy is mid-update — but as it stands on disk, a reader would -under-credit the artefact. Worth confirming it lands. - -### 3. Current state, per module - -- **integration** — the last module. `integration-r4` now shows **8 handoff - slots** (up from 5 at T+180), with writes at 10:48, 10:54, 11:46 and 11:52. - Alive and progressing. Notes have not grown since 10:48, which for this module - has consistently meant "in a run", not "stopped". -- **analyze** — took hold `101753` and launched `analyze-r5`, currently at - `verify_workset: input_validating -> running`. Also ran an `analyze-dryfix` - dry run first (7 tasks resolved, 0 dispatched) to check the schema fix without - spending GPU time. That is the cheap-first discipline the BRIEF asked for. -- **profiling, kernel-opt** — done, no further activity. - -### 4. Code problems - -No new defects. Standing set unchanged from T+180: **C9b** (`seal_refused` has -no reader) and **C23** (integration's ten closed `items_schema`s) are the two -open items anyone would act on; C5, C8/C14, C10/C11, C13, C7 carried. - -Worth recording as **closed by evidence**: the `acb8bfe` schema fix is now being -exercised end-to-end by `analyze-r5`. If r5 seals its terminal handoff, C9 moves -from "fixed, verified offline" to "fixed, verified in a run". - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| E0″ | Hold reaping — three dead (30/34/70 min), two originals now at **3 h 45 m**, a fourth alive at 19 min | **open**; the workaround (consolidate onto surviving holds; treat a new hold as expendable) is holding | -| E14 | The staging lesson | **open as a package-content decision** | -| E9 | Qwen3.6-27B ~2.9 tok/s | **still open**, six checkpoints | -| E15 | Node-local paths absent on the login node | noted | - -### 6. Undetermined - -1. **Will `integration-r4` finish inside the walltime?** 4 h 12 m remain and it - is 77 min in with 8 slots open. Comfortable. The risk is a failed output - validation, which kills the arm with no resume (~50 min, measured). -2. **Will `analyze-r5` seal?** This is the test of `acb8bfe` in a live run. -3. **Deploy's stale PROVENANCE** — will it be refreshed before the effort ends? - The artefact is better than its description right now. -4. **C9b** — fourth checkpoint carrying it. Leader call. -5. **E9** — sixth checkpoint untouched. I now expect this effort to end with it - undiagnosed, and that should be stated as a finding rather than left implicit. - -### 7. New commits - -**None since T+180.** `git log` is unchanged at `77a2c13`. Uncommitted: -`integration-demo/assets/accept/lm_eval.sh` and -`integration.debug.help.info.md`. - -Zero commits in 30 minutes is consistent with the observed state — four modules -finished, two runs in flight, nobody editing packages — and is not itself a -concern. It does mean deploy's d5 result and analyze's r5 are not yet reflected -in any commit or notes file. - -### 8. Other - -- **The deliverable set is now stronger than at any previous checkpoint**: three - fully sealed with all validators PASS (profiling 6/6, kernel-opt 3/3, deploy - 2/2), one delivered-but-unsealed with the gap documented and a sealing run in - flight (analyze), one in progress (integration). -- **Deploy went back for the gap rather than accepting it.** At T+120 it had a - defensible deliverable and a written justification for the missing verdict. - It spent another 90 minutes and two more runs to remove the caveat instead. -- The one thing I would want checked before this effort is called done: **the - stale `PROVENANCE.md` in deploy's deliverable.** Every other provenance file - today has erred toward understating its artefact deliberately; this one now - understates it accidentally, which is a different thing and worth fixing. - ---- - -## T+240 — 2026-09-02 12:35 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (4 h 15 m) | 16:17:41 | **3 h 42 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (4 h 15 m) | 16:17:57 | **3 h 42 m** | -| `101803` | `crsuse2-m2m-050` | **analyze** (fifth hold) | RUNNING (27 m) | ~16:33 | ~3 h 58 m | -| ~~`101753`~~ | ~~`-267`~~ | analyze | **gone** after ~30 min | — | — | - -**Four of four freshly issued holds have now ended early** (`101078` ~30 min, -`101155` ~70, `101576` ~34, `101753` ~30), while the two originals are past -**4 h 15 m** untouched. The pattern from T+150 holds without exception. Analyze -is on its fifth hold and running `analyze-r6`. - -### 1. Progress - -**Effort: ~97 %.** Elapsed 247 minutes. Estimated remaining: **30–75 min.** - -| module | est. % | basis | -|---|---|---| -| **deploy** | **100 % — DELIVERED, sealed, 2/2 PASS, provenance now current** | `e558f16` | -| **profiling** | **100 % — DELIVERED** | 6/6 PASS | -| **kernel-opt** | **100 % — DELIVERED** | 3/3 PASS | -| **analyze** | **100 % delivered**; two further attempts at a *sealed* version have both failed on a new cause | `analyze-r6` running | -| **integration** | **~90 %** | `integration-r4` alive — writing `measure.patched` artefacts at 12:01 | - -Unchanged headline: **delivered 4 of 5**. No regression; the movement this -interval is quality, not count. - -**Reliability: high.** Deploy's stale-provenance flag from T+210 is **resolved** — -`PROVENANCE.md` now has mtime 12:03 and deploy committed `e558f16` -("deploy-d5 is green — both validators PASS, handoff sealed and verified"). - -### 2. Analyze's sealing attempts hit a genuinely new failure - -The `acb8bfe` schema fix **worked** — `analyze-r5` got past the seal problem -entirely. It then failed somewhere new, and the diagnosis is the most -operationally useful finding of this interval: - -``` -check_workset_shape: PASS -workset_evidence slot v0: invalid -check_workset_runs: FAIL -``` - -Both operators ran, both correct, `pass_ratio: 1.0`. The failure is **spread**: - -``` -moe1: per_group_ms [0.1344, 0.1437, 0.1315, 0.1424, 0.1272] rsd 0.047 ok -moe2: per_group_ms [0.1772, 0.4358, 0.1876, 0.1842, 0.1965] rsd 0.423 FAIL -``` - -`rocm-smi` on that node: **every card at 100 % use, VRAM 60–84 %, another -tenant**. So `max_rsd: 0.1` did exactly its job — the machine was not quiet. - -Two properties of that rule are easy to get wrong, and analyze read the -validator rather than assuming: - -- **It is a hard `return False`, not a per-operator note.** In - `check_workset_runs/check.py`, `ran: false` and `correct: false` both - `continue` into notes and are forgiven by `min_pass_ratio`; **an rsd breach - returns immediately.** So `min_pass_ratio: 0.5` does *not* protect you from - noise — one noisy operator fails the whole step no matter how many others were - clean. -- **That inverts the `top_n` advice** analyze itself gave at T+150: - - | risk | `top_n=1` | `top_n=2` | - |---|---|---| - | agent writes one bad driver | fatal | survivable (`min_pass_ratio` 0.5) | - | one operator hits node noise | one chance to be unlucky | **two** chances, either fatal | - - There is no universally right value: prefer 2 on a quiet node, and on a busy - shared node neither is safe. - -**This is a module correcting its own published advice within two hours**, for -the second time today (the first was C9's root cause). It is also the clearest -statement yet of a cost the BRIEF does not mention: **a shared, saturated node -can fail a validator on evidence that is entirely correct.** - -### 3. Current state, per module - -- **integration** — still the last module, and **alive**: files written at - 11:56 and 12:01 under `measure.patched/` (`accept/needle.json`, `steps.tsv`, - `logs/probe.log`, `logs/needle.log`, `logs/smoke.log`). It is in the *patched* - arm, i.e. past the stock arm. 8 handoff slots. Notes still last-written 10:48, - now 107 minutes ago — for this module that has consistently meant "mid-run", - and the artefacts confirm it. -- **analyze** — `analyze-r6` at `verify_workset`, fifth hold, third attempt at a - sealed terminal handoff. Its delivered (unsealed) artefact remains valid and - documented; everything since is upside. -- **deploy** — finished and tidy. Added `§21 Three rules this module earned, - stated as rules` (`f1e495d`) — distilling its day into reusable form rather - than leaving it as narrative. - -### 4. Code problems - -| # | problem | state | -|---|---|---| -| C9 | analyze `items_schema` drift | **FIXED and now proven in a live run** — r5 sealed past it | -| **C24** | `check_workset_runs` treats an rsd breach as a hard `return False` while forgiving `ran`/`correct` failures via `min_pass_ratio` — an inconsistency that makes `min_pass_ratio` misleading | **open, documented.** Arguably correct-as-designed; worth a decision, not a silent fix | -| C9b | `seal_refused` has no reader | **open** — fifth checkpoint | -| C23 | integration's ten closed `items_schema`s | **open** | -| C5, C8/C14, C10/C11, C13, C7, C20 | carried / fixed as previously recorded | — | - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| E0‴ | **Four of four new holds ended early; both originals past 4 h 15 m** | **open**, worked around | -| **E16** | **A saturated shared node fails `max_rsd` on correct evidence.** Analyze's r5 lost to another tenant's 100 %-utilised cards. Not a package fault and not fixable from inside the package | **open** — the real constraint on any timing-based validator here | -| E14 | Staging lesson | **open as a package decision** | -| E9 | Qwen3.6-27B ~2.9 tok/s | **still open**, seven checkpoints | -| E15 | Node-local paths absent on login node | noted | - -### 6. Undetermined - -1. **Will `integration-r4` complete?** It is in the patched arm at 107 min with - 3 h 42 m left. The measured cost of a lost arm is ~50 min with no resume. -2. **Will `analyze-r6` seal?** Third attempt, and the blocker is now node - contention — outside analyze's control. It may simply not get a quiet node. -3. **Is C24 a bug or a design choice?** `min_pass_ratio` forgiving correctness - failures but not noise is defensible, but it is surprising and undocumented. -4. **C9b** — fifth checkpoint. -5. **E9** — seventh checkpoint, untouched. **I now record it as a finding this - effort will not resolve**, rather than as a pending item. - -### 7. New commits - -Two since T+210: - -``` -e558f16 docs(deploy-demo): deploy-d5 is green — both validators PASS, handoff sealed and verified -f1e495d docs(deploy-demo): three rules this module earned -``` - -- `e558f16` — records the green run and refreshes the provenance that T+210 - flagged as stale. The flag is cleared. -- `f1e495d` — deploy converting its findings into stated rules. - -Uncommitted: `integration-demo/assets/accept/lm_eval.sh`, -`analyze.debug.help.info.md`, `integration.debug.help.info.md`. - -### 8. Other - -- **Deploy is the model finish**: closed its validator gap, refreshed its - provenance, then wrote down the transferable rules. Three of its four notes - sections in the last hour are for the *next* reader, not for its own delivery. -- **Two self-corrections in one day from two different modules** (analyze on C9's - cause, analyze again on `top_n`), plus deploy correcting a stale control count - and me correcting the E6 rule at T+90 and the C9 cause at T+150. The notes - are trustworthy *because* of this, not despite it. -- The effort's remaining risk is entirely **integration-r4 completing** and, - secondarily, whether analyze can find a quiet node. Neither is a code problem. - ---- - -## T+270 — 2026-09-02 13:05 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (4 h 45 m) | 16:17:41 | **3 h 12 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (4 h 45 m) | 16:17:57 | **3 h 12 m** | - -`101803` is gone — **five of five freshly issued holds ended early**; the two -originals are past 4 h 45 m. Analyze finished before losing it, so this cost -nothing. Only integration still needs compute. - -### 1. Progress - -**Effort: ~98 %.** Elapsed 277 minutes. Estimated remaining: **20–60 min.** - -| module | est. % | basis | -|---|---|---| -| **analyze** | **100 % — DELIVERED SEALED. Six handoffs, six verdicts PASS** | verified below | -| **deploy** | **100 % — DELIVERED SEALED, 2/2 PASS** | | -| **profiling** | **100 % — DELIVERED SEALED, 6/6 PASS** | | -| **kernel-opt** | **100 % — DELIVERED SEALED, 3/3 PASS** | | -| **integration** | **~95 %** | r4 completed **both arms, 10 handoffs, 9 of 10 validators PASS**; r5 running with recalibrated bars; **r4 preserved as a fallback deliverable** | - -**Delivered: 4 of 5 — and all four are now fully sealed with every validator -PASS.** Analyze's T+150 caveat ("unsealed, and it says so") is gone. - -**Reliability: high.** I read all six of analyze's `validation.yaml` files. - -### 2. Analyze went green — verified - -`/shared_nfs/yihou/agent_sys/debugging/analyze/store/` holds **six sealed -handoffs from one run**, each with its verdict recorded `result: true`: - -``` -check_kernel_table true check_worklist_shape true -check_identity_resolved true check_workset_shape true -check_workset_runs true check_analyze_packup_shape true -``` - -Its PROVENANCE now opens *"Six sealed handoffs from one green run … each `valid`, -each with its validator's PASS recorded"* — and still leads its second paragraph -with the caveat that matters: *"the input profile is synthetic."* The earlier -unsealed delivery is kept as `store-run2-superseded/`. - -That took **six runs** (r1, r2, r4, r5, r6 plus a dryfix), across **three -nodes**, through a schema bug it diagnosed and fixed itself and a node-contention -failure outside its control. - -### 3. Integration: nine of ten, and the tenth is a mis-calibrated bar - -`integration-r4` **completed both arms and produced all ten handoffs**. Nine -validators PASS. The tenth: - -``` -check_no_regression: FAIL usability / strong -"output token throughput (avg): 59.16 -> 46.49, -21.4% against a bar of 5%" -"inter-token latency (avg): 427.23 -> 478.02, +11.9% against a bar of 10%" -``` - -**The validator and the report agreed**; both saw the arms differ, and that -stopped the graph before `packup`. - -Integration's analysis is the strongest reasoning in the effort today. The patch -under test is the mock, whose entire per-call cost is **one boolean branch on a -module global** plus one `logger.warning` at import — next to a full decoder -layer of GPU work. That is not a 21 % effect. So **the 21 % is the arm-to-arm -spread of this deployment**, on a system decoding at 2–3 tok/s (E9) where a -60-request replay is queue-dominated. - -And that is precisely the number the package says it lacks. From its own README, -under *Known gaps*: - -> **The performance bars have no measured basis yet.** 5 % on throughput and -> 10 % on latency are placeholders; the number they want is the natural -> run-to-run spread of one arm, which the first full run will produce. - -r4 **is** that first full run. So r5 sets `max_throughput_regression=0.35`, -`max_ttft_regression=0.30` — the measured spread plus margin — and integration -writes down exactly why, in `run.sh` and `PROVENANCE.md`: - -> widening a bar after seeing the data is exactly the move that hides a real -> regression, and the only thing that separates the two is whether the reasoning -> is written down. - -It then states what is **not** established, unprompted: one pair of arms is an -order-of-magnitude calibration, not a variance estimate; stock always runs first -so part of the gap may be systematic (thermal, page cache, cold allocator) and -this run cannot separate that from noise; the honest fix is `bench_rounds > 1` -or two stock arms back to back, neither of which fits the walltime; and **the -widened bars must not travel back to the GLM deployment** — the defaults stay 5 % -and 10 %. - -**And r4 is preserved as a fallback**: nine handoffs including a complete -`integration_report`, at `ws2/integration/r4_fallback/handoffs/`. If r5 does not -finish, that is the deliverable with §14 as its explanation. Integration cannot -now finish empty-handed. - -### 4. Code problems - -| # | problem | state | -|---|---|---| -| **C25** | **The 5 %/10 % performance bars are placeholders with no measured basis** — the package's README says so, and r4 supplied the missing number (~21 % throughput, ~12 % latency spread on this deployment) | **open by design; now calibrated for this host only.** The defaults are deliberately unchanged | -| **C26** | `handoff`'s **locality check is not called** — the seal does not enforce locality; `redact.py` does. Settled first-hand by analyze and profiling (`23f3d6a`, `561813a`), and documented as **deliberate** | **documented.** Directly contradicts a natural reading of BRIEF limit 3 | -| C27 | analyze's locality helper was merging two different findings | **fixed**, `1a4e8f5` | -| C24 | `check_workset_runs` hard-fails on rsd while forgiving correctness | **open** | -| C9b | `seal_refused` has no reader | **open** — sixth checkpoint | -| C23 | integration's ten closed `items_schema`s | **open** | - -**C26 is a correction to the shared BRIEF.** Known-limit 3 says handoff content -"may not name an absolute path… (`handoff/locality.py`)", which reads as a seal- -time enforcement. Two modules established first-hand that **the seal does not -call it**; `redact.py` is what actually acts, and the disconnection is -intentional. Anyone designing around "the seal will catch my paths" is wrong. - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| E0⁗ | **Five of five new holds ended early; both originals at 4 h 45 m** | **open**, fully worked around | -| E16 | A saturated shared node fails `max_rsd` on correct evidence | **open** | -| **E9** | **Qwen3.6-27B at ~2.9 tok/s** — now *load-bearing*: it is why the replay is queue-dominated and why the arm-to-arm spread is 21 % | **open, and it has propagated.** No longer a curiosity | -| E14 | Staging lesson | **open as a package decision** | - -### 6. Undetermined - -1. **Will `integration-r5` finish inside 3 h 12 m?** r4 took ~2 h for both arms. - Tight but feasible; the fallback removes the downside. -2. **Is integration's 21 % noise or systematic?** Explicitly unresolved by the - module, with the experiment that would settle it named and declined for - walltime. **The right way to leave a question open.** -3. **C9b** — sixth checkpoint. Still unactioned. -4. **E9** — eighth checkpoint. It has now caused a validator failure and a bar - recalibration. **It should be the first item of any follow-up.** -5. Should C25's widened bars be re-tested against GLM before reuse? Integration - says no and gives its reason; nobody has disagreed. - -### 7. New commits - -Five since T+240: - -``` -d8a22fd docs(analyze-demo): the green run, and the max_rsd finding -1a4e8f5 fix(analyze-demo): the locality helper stops merging two different findings -23f3d6a docs(llm_e2e): the locality check is not called, and that is deliberate -81f2c21 docs(analyze-demo): the delivery is a store root and nothing else -561813a docs(profiling-demo): the seal does not enforce locality — redact.py does -``` - -`23f3d6a` and `561813a` are two modules landing the **same** framework finding -from different directions within minutes — the C26 correction above. `81f2c21` -standardises the delivery shape on a store root, which all four delivered -modules now use. - -Uncommitted: `integration-demo/assets/accept/lm_eval.sh`, -`integration.debug.help.info.md`. - -### 8. Other - -- **Four modules delivered, every handoff sealed, every validator PASS.** Counts: - profiling 7 handoffs/6 verdicts, analyze 6/6, kernel-opt 2/3, deploy 1/2. -- **Nobody widened a bar quietly.** Integration hit the only threshold failure of - the day, established the threshold was a documented placeholder, produced the - missing measurement, recalibrated with the reasoning written into two files, - enumerated what its own measurement does not establish, and ring-fenced the new - values from the deployment they were not measured on. That is the single best - piece of work I have observed today. -- **The BRIEF now has three confirmed errors** found by this effort: the 1800 s - settle budget (actually 14400 s), "8 × MI355X 288 GiB" (some nodes are CPX, - 64 × 36 GiB), and the locality check being enforced at seal (it is not called). - All three are recorded in commits; **none is in the BRIEF**, which is what the - next effort reads first. - ---- - -## T+300 — 2026-09-02 13:36 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (5 h 16 m) | 16:17:41 | **2 h 41 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (5 h 16 m) | 16:17:57 | **2 h 41 m** | - -### 1. Progress - -**Effort: ~99 %.** Elapsed 308 minutes. Estimated remaining: **20–60 min**, all -contingent on `integration-r5`. - -| module | est. % | basis | -|---|---|---| -| profiling / kernel-opt / deploy / analyze | **100 % — DELIVERED SEALED, all verdicts PASS** | unchanged | -| **integration** | **~95 % — deliverable now populated, but not by integration** | see below | - -**All five deliverable directories are now non-empty**, which has not been true -before. But the fifth was assembled by other modules on the leader's -authorisation, and that distinction is the substance of this checkpoint. - -**Reliability: high.** I read the leader's note and all ten `validation.yaml` -files in integration's store. - -### 2. Integration's directory was populated out of band — and labelled as such - -`/shared_nfs/yihou/agent_sys/debugging/integration/` now holds: - -| path | what | produced by | -|---|---|---| -| `store/` | **nine sealed handoffs** from run `20260902T104817-8a2995` | integration's run; **copied here by `kernel-opt`** on the leader's authorisation | -| `store/COPIED-BY-KERNEL-OPT.md` | that copy's record + per-handoff verdicts | kernel-opt | -| `packup-out-of-band/` | `integration_packup` content, 47 files, **not sealed** | **produced by `deploy`**, using integration's own unmodified `packup.py` over the nine sealed handoffs | -| `packup-out-of-band/PRODUCED-BY-DEPLOY.md` | that production's record, with an unconditional offer to remove it | deploy | -| `DELIVERY-NOTE-FROM-LEADER.md` | why the directory is readable without integration's own provenance | the leader | - -The leader's note opens: *"This is not `integration`'s own record. Their -`PROVENANCE.md` does not exist at the time of writing and only they can write -it."* Three separate hands touched this directory and **each labelled its own -contribution in a file named after itself.** Nothing here is passed off as -integration's work. - -**Verified independently** — the ten verdicts across the store: - -``` -check_service_live true check_overlay_applies true check_patch_shape true -check_bench_report true check_bench_report true check_patch_live true -check_acceptance true check_acceptance true check_service_live true -check_no_regression FALSE -``` - -**Nine true, one false.** The leader's note flags this in its own section headed -*"The one thing a reader must not miss"*: - -> It is **a sound sample of a refused report and a misleading one if taken for -> a passing example.** - -`check_no_regression` is `strength: strong`, so the false verdict invalidated -the `integration_report` handoff, the graph stopped, and **`integration_packup` -was never dispatched** — which is exactly why the packup here was produced out -of band. The note is careful about the causality: *"The packup step itself is -sound; it is downstream of a validator that correctly refused its input."* - -The refusal is the C25 bar mis-calibration from T+270, unchanged: a declared -no-op patch measured 21 % apart across arms because the deployment decodes at -2–3 tok/s and the bars are documented placeholders. - -**Assessment.** This is a defensible way to populate a directory under time -pressure and it is executed with unusual care — real artefacts from a real run, -the one refused verdict promoted rather than buried, every hand named, and a -standing offer to withdraw the out-of-band piece. What it is **not** is a -handoff of the terminal kind produced by integration's own run, which is what -the BRIEF asks for. `integration-r5` is still the thing that would satisfy that, -and it is still running. - -### 3. Current state - -- **integration** — `integration-r5` (`20260902T125156-4b968b`), 5 handoff slots, - **last write 13:06, now 30 minutes ago.** Its notes have not grown since 12:52. - Given r4 took ~2 h for both arms, a 30-minute quiet stretch mid-arm is - unremarkable; but with 2 h 41 m left this is the number to watch. -- **profiling, kernel-opt, deploy, analyze** — done, and now spending their time - on delivery hygiene and on *each other's* deliverables. - -### 4. Code problems - -No new package defects. The interval's commits are all about **how a delivered -handoff is verified**, which has become a small shared sub-project: - -| # | finding | state | -|---|---|---| -| C28 | `relayout_handoffs.py` needed a **verdict gate and distinct exit codes** — an exit-status disagreement between two modules, resolved with a distinct code rather than a suppression flag, then `--allow-refused` restored with the round-trip cost named | **resolved**, `a9a6ab5` → `3e13a3e` → `bfe56e1`, `fdeb5d6` | -| C29 | **Verifying a delivered handoff takes three checks, not two** (`63ad06b`) | **documented** | -| C25 | placeholder performance bars | **open by design**, calibrated for this host only | -| C26 | locality check not called at seal | **documented** | -| C9b, C23, C24 | carried | **open** | - -C28 is worth noting as process: two modules disagreed about an exit status, and -the resolution went **suppression flag → distinct exit code → flag restored with -its cost documented**, in three commits over ~20 minutes. The disagreement was -settled in the code and the reasoning left behind. - -### 5. Non-code problems - -Unchanged: E0 (five of five new holds died; both originals now 5 h 16 m), E9 -(2.9 tok/s, **ninth checkpoint**, now the acknowledged cause of the only -validator failure in the effort), E14, E15, E16. - -### 6. Undetermined - -1. **Will `integration-r5` finish?** 2 h 41 m left, 30 minutes quiet. This is the - last open question of the effort proper. -2. **Does the out-of-band directory satisfy the delivery contract?** The BRIEF - asks for a handoff of the terminal kind that the package produced. Nine - sealed handoffs qualify; the terminal `integration_packup` was produced out - of band by another module. **A leader call, and the leader has already made - it once by authorising the work** — but it should be stated explicitly in the - final accounting rather than left to a reader of `PROVENANCE.md` files. -3. **Will integration write its own `PROVENANCE.md`?** The leader's note says - only they can, and it is the one document the directory lacks. -4. **C9b** — seventh checkpoint, still unactioned. -5. **E9** — ninth checkpoint. Undiagnosed, and now demonstrably consequential. - -### 7. New commits - -Five since T+270: - -``` -63ad06b docs(profiling-demo): verifying a delivered handoff takes three checks, not two -a9a6ab5 docs(kernel-opt-demo): resolve the exit-status disagreement with a distinct code -fdeb5d6 docs(profiling-demo): record relayout_handoffs.py's verdict gate and exit codes -3e13a3e docs(kernel-opt-demo): drop the suppression flag, keep the distinct exit code -bfe56e1 docs(kernel-opt-demo): restore --allow-refused, and name what the round trip cost -``` - -All five are `docs` on the verification/delivery tooling — no package behaviour -changed. `bfe56e1` explicitly records the cost of the round trip it ends, which -is the honest way to close a reversed decision. - -Uncommitted, unchanged for two hours: `integration-demo/assets/accept/lm_eval.sh` -and `integration.debug.help.info.md`. - -### 8. Other - -- **Four modules that had finished spent this interval on someone else's - problem** — kernel-opt copying integration's store, deploy running - integration's packup, profiling and kernel-opt jointly hardening the shared - relayout tool. None of that was assigned. -- **The one refused verdict was promoted, not buried.** Three separate documents - (integration's §14, the leader's note, kernel-opt's copy record) each state - that `check_no_regression` returned false and why. The easiest thing to do - with a 9-of-10 was to report "nine PASS"; nobody did. -- Carried from every checkpoint since T+120 and still true: **the three BRIEF - errors** (settle budget 14400 s not 1800 s; CPX nodes are 64 × 36 GiB not - 8 × 288 GiB; the locality check is not called at seal) are in commits and - module notes but **not in the BRIEF**. - ---- - -## T+330 — 2026-09-02 14:05 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (5 h 46 m) | 16:17:41 | **2 h 11 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (5 h 46 m) | 16:17:57 | **2 h 11 m** | - -Both originals still untouched. Under two hours remain. - -### 1. Progress - -**Effort: ~99 %.** Elapsed 338 minutes. Estimated remaining: **20–60 min.** - -| module | est. % | basis | -|---|---|---| -| profiling / kernel-opt / deploy / analyze | **100 % — DELIVERED SEALED** | unchanged | -| **integration** | **~95 %** | `integration-r5` alive: **8 handoff slots** (up from 5 at T+300), last write **13:52** | - -Unchanged in count. r5 has grown from 5 slots to 8 in half an hour, so it is -moving and has covered most of a graph that produced ten last time. - -**Reliability: high**, with one caveat I want on the record: **integration has -not written a note since 12:52 (73 min) and has not committed since before -T+270.** Everything I know about that module this interval, I learned from -*other modules' notes about it*. That is a real gap in the record, and it is the -second time today I am reporting on a module through third parties. - -### 2. A fact I reported wrongly by omission: integration's r4 was OOM-killed - -At T+300 I described integration's directory as populated out of band and left -the reason as "under time pressure". Kernel-opt's §23 supplies what I did not -have: integration's **run was OOM-killed at 12:44 and the module was inactive**; -that is why the leader authorised two other modules to recover its artefacts. -This is E10 again — the login-node OOM that killed profiling's run B at T+90 — -now claiming a second victim, five hours later, in a module that had no reason -to expect it. - -I am recording this here rather than editing T+300. **The out-of-band recovery -was a response to a crash, not to slowness**, which is a materially different -thing and reflects better on the module than what I wrote. - -### 3. The interval's finding: "9/9 verified" was true and misleading - -Kernel-opt did the copy of integration's nine handoffs and reported **9/9 -verified** — every copy re-hashed against its own manifest *and* against the -untouched run-store original. Then it wrote §23, headed *"and the check I did -not do"*: - -> I verified **integrity** and **shape** and reported "9/9 verified" — and never -> opened a single `validation.yaml`. - -Deploy did open them, and found `integration_report` carrying -`check_no_regression: result=False, strength: strong`. Kernel-opt then read all -nine itself: **8 pass, 1 fails**, and the failure is on the terminal report of -the measurement chain — which is *why* `integration_packup` never ran. So: - -> **A digest proves the bytes have not changed since sealing; it says nothing -> about whether what was sealed was acceptable.** Those are two independent -> questions and I answered one while sounding like I had answered both. - -It also corrects the leader's expectation by measurement: there is **no tenth -directory** for the unsealed packup — the run's `handoffs/` holds exactly nine — -so the missing tenth is a **consequence of the refused verdict, not of the OOM -kill**. - -And then deploy corrected *kernel-opt's own framing* in turn: the digest check -was not secondary, because deploy's diagnosis of the false verdict **depended** -on it — the bytes being provably the sealed bytes is what let them treat the -verdict as a property of the run rather than of the copy. **A check that rules -out an entire class of confusion is doing real work even when it is silent on -the question you care about.** - -This is the cleanest instance today of the BRIEF's first rule. It produced the -delivery checklist that is the reusable output: **verify the digest, verify the -layout, and read the verdicts. Three checks, not two.** - -### 4. Code problems - -**No new defects, and no commits at all this interval** — `git log` is unchanged -at `bfe56e1`. Standing open items, all carried: - -| # | problem | state | -|---|---|---| -| C9b | `seal_refused` has no reader | **open** — eighth checkpoint | -| C23 | integration's ten closed `items_schema`s | **open** | -| C24 | `check_workset_runs` hard-fails on rsd, forgives correctness | **open** | -| C25 | placeholder performance bars, now calibrated for this host only | **open by design** | -| C26 | locality not enforced at seal | **documented** | - -### 5. Non-code problems - -| # | problem | state | -|---|---|---| -| **E10′** | **The login-node OOM has now killed two runs** — profiling's run B (~09:34) and integration's r4 (12:44). Profiling diagnosed it at T+90 and fixed it *for itself* by moving `agent-sys` onto the compute node (`8274a08`); the fix did not propagate | **open, and the propagation failure is the lesson.** A fix written into one module's package at 11:00 did not reach another module at 12:44 | -| E0 | Five of five new holds died; both originals at 5 h 46 m | **open**, worked around | -| E9 | Qwen3.6-27B ~2.9 tok/s | **open**, tenth checkpoint | -| E14, E15, E16 | carried | **open** | - -E10′ is worth stating plainly: **the single highest-value thing any module -learned today was known for three and a half hours before it claimed its second -victim.** The notes files are per-module by design, and there is no channel that -makes "this will kill your run too" arrive at another module unasked. - -### 6. Undetermined - -1. **Will `integration-r5` finish inside 2 h 11 m?** 8 of an expected ~10 slots, - last write 13 minutes ago. The most likely outcome is yes; the fallback - (r4's nine, already delivered) means the downside is bounded. -2. **Is integration still alive as a module?** 73 minutes without a note, after - an OOM kill that another module reported on its behalf. Its run is - progressing, which is the thing that matters, but I cannot tell whether - anyone is watching it. -3. **Will integration's own `PROVENANCE.md` be written?** Still the one document - its directory lacks. -4. **C9b** — eighth checkpoint. If it is not actioned it should be closed as - "recorded, not fixed" rather than left implying someone will get to it. -5. **E9** — tenth checkpoint, undiagnosed. - -### 7. New commits - -**None.** `git log` unchanged since T+300 at `bfe56e1`. Uncommitted and -unchanged for two and a half hours: `integration-demo/assets/accept/lm_eval.sh`, -`integration.debug.help.info.md`. - -The commit stream stopping is consistent with four modules finished and the -fifth mid-run. It does mean the last hour of work — kernel-opt's §23, the -three-check checklist, deploy's correction — exists in notes files that are -themselves uncommitted for two of the five modules. - -### 8. Other - -- **Four modules delivered and sealed; the fifth has nine sealed handoffs - delivered on its behalf plus a live run that may supersede them.** No module - will finish empty-handed. -- **Three modules corrected themselves or each other this interval alone** — - kernel-opt on its own "9/9 verified", deploy on kernel-opt's framing, and - kernel-opt on the leader's expectation of a tenth directory. Every correction - went into a file rather than into a conversation. -- **My own two corrections today**, both recorded in the section following the - error rather than by editing it: the NFS/`TMPDIR` rule (T+90, too broad) and - C9's root cause (T+150, wrong cause). Add to those the omission corrected in - §2 above. The append-only rule has cost nothing and made all three visible. -- Carried unchanged since T+120: **three confirmed BRIEF errors** (settle budget, - CPX geometry, locality-at-seal) live in commits and notes but not in the BRIEF. - With two hours left, this is the cheapest high-value thing anyone could still do. - ---- - -## T+360 — 2026-09-02 14:35 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (6 h 16 m) | 16:17:41 | **1 h 42 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (6 h 16 m) | 16:17:57 | **1 h 42 m** | - -Under two hours. Both originals have now run 6 h 16 m without interruption, -against five freshly issued holds that all died inside 70 minutes. - -### 1. Progress - -**Effort: ~99 %.** Elapsed 368 minutes. Estimated remaining: **20–50 min**, or -the effort ends at the walltime with what it already has. - -| module | est. % | basis | -|---|---|---| -| profiling / kernel-opt / deploy / analyze | **100 % — DELIVERED SEALED** | unchanged for 90 minutes | -| **integration** | **~97 %** | `integration-r5`: 8 handoff slots, **7 verdicts, all `result: true`**, writing **as of 14:33** | - -**Reliability: high.** I opened r5's `validation.yaml` files directly. - -### 2. `integration-r5` is close, and so far it is clean - -Seven verdicts recorded, **every one true**: - -``` -check_overlay_applies true check_service_live true check_patch_live true -check_service_live true check_acceptance true check_patch_shape true -check_bench_report true -``` - -Compare with r4, which produced ten handoffs and failed exactly one — -`check_no_regression` on the terminal `integration_report`. **r5 has not yet -reached that validator.** The two outstanding pieces are precisely the two that -r4 could not deliver: the `integration_report` carrying `check_no_regression`, -and the `integration_packup` downstream of it that never dispatched. - -So the whole question of whether integration delivers its own terminal handoff -comes down to the one validator that failed last time, with the recalibrated -bars (`max_throughput_regression=0.35`, `max_ttft_regression=0.30`) that -integration derived from r4's own measured spread and documented in two files. -The run was still writing two minutes before this checkpoint. - -**I am not going to predict it.** The bars were set from a single pair of arms, -which integration itself called an order-of-magnitude calibration rather than a -variance estimate, and it noted the stock-arm-first ordering may make part of -the 21 % systematic. If the spread this time exceeds 35 %, it fails again. - -### 3. Current state - -- **integration** — r5 active, 7/7 clean, two handoffs short. Notes still last - written 12:52 (**103 minutes**); no commit since before T+270. The run is - healthy and the record is not being kept. -- **profiling, kernel-opt, deploy, analyze** — all quiet for 60+ minutes. Their - notes last grew at 13:23, 13:27, 12:05 and 12:48. All four are finished and - have stopped, which is the correct behaviour, not a stall. - -### 4. Code problems - -No new defects. No commits. The five open items are unchanged from T+330 — -**C9b** (`seal_refused` has no reader), **C23** (integration's ten closed -`items_schema`s), **C24** (`check_workset_runs` hard-fails on rsd), -**C25** (placeholder bars, now host-calibrated), **C26** (locality not enforced -at seal, documented). - -With 1 h 42 m left and every module either finished or in a terminal run, **none -of these will be fixed in this effort.** They should be handed over as recorded -findings rather than left looking pending. - -### 5. Non-code problems - -Unchanged and all open: **E0** (5/5 new holds died, both originals at 6 h 16 m), -**E9** (2.9 tok/s — eleventh checkpoint, undiagnosed, and the acknowledged cause -of the only validator failure of the day), **E10′** (login-node OOM killed two -runs three and a half hours apart, the fix never propagating between modules), -**E14** (staging), **E15**, **E16** (a saturated node fails `max_rsd` on correct -evidence). - -### 6. Undetermined - -1. **Will `check_no_regression` pass in r5?** The single open question of the - effort. Answered within the hour, one way or the other. -2. **Will integration write its own notes and `PROVENANCE.md`?** 103 minutes - silent. If the walltime arrives first, the record of the fifth module will - consist of its own notes up to 12:52 plus three other parties' accounts of - what happened after — which is a worse outcome than the artefacts deserve. -3. **C9b** — ninth checkpoint. **Recommend closing it as "recorded, not fixed."** -4. **E9** — eleventh checkpoint. Same: it should be handed over as an open - question with the evidence attached, not carried as if someone will get to it. -5. Whether the out-of-band packup stays or is withdrawn if r5 produces a real - one. Deploy offered unconditionally to remove it; nobody has needed to decide. - -### 7. New commits - -**None**, for the second consecutive checkpoint. `git log` unchanged at -`bfe56e1` since ~13:30. Uncommitted for three hours: -`integration-demo/assets/accept/lm_eval.sh`, `integration.debug.help.info.md`. - -An hour of no commits with four modules finished is expected. The thing worth -flagging is narrower: **kernel-opt's §23 and deploy's correction of it — the -best material of the last two hours — are in notes files, and two of the five -notes files are uncommitted.** If the session ends abruptly they are still on -disk in the worktree, so this is a tidiness risk rather than a loss risk. - -### 8. Other - -- **The effort is done except for one validator.** Four modules delivered sealed - with every verdict PASS; the fifth has nine sealed handoffs already delivered - on its behalf, a documented account of the one refused verdict among them, and - a live run that may supersede all of it within the hour. -- **Nothing has regressed at any checkpoint today.** Every interval has been - flat or forward: no deliverable withdrawn, no verdict revoked, no finding - retracted — only narrowed, corrected in place, or superseded by a better run. -- The three BRIEF errors (settle budget 14400 s not 1800 s; CPX nodes 64 × 36 GiB - not 8 × 288 GiB; locality not enforced at seal) remain in commits and notes and - **not in the BRIEF**. This is the ninth checkpoint carrying that line. It is a - ten-minute edit and it is the highest-leverage thing left undone. - ---- - -## T+390 — 2026-09-02 15:05 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (6 h 46 m) | 16:17:41 | **1 h 12 m** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (6 h 46 m) | 16:17:57 | **1 h 12 m** | - -### 1. Progress - -**Effort: 100 % of what it can deliver.** Elapsed 398 minutes. **All five -modules have delivered.** Remaining work is optional hardening. - -| module | state | -|---|---| -| profiling | DELIVERED SEALED, 6/6 PASS | -| kernel-opt | DELIVERED SEALED, 3/3 PASS | -| deploy | DELIVERED SEALED, 2/2 PASS | -| analyze | DELIVERED SEALED, 6/6 PASS | -| **integration** | **DELIVERED** — r5's nine sealed handoffs (**9 verdicts true, 1 false**), terminal packup validated out of band, own `PROVENANCE.md` written | - -**Reliability: high.** All verdicts read from `validation.yaml` on disk. - -**Integration returned and closed out fully**: two commits, 189 new lines of -notes, its own `PROVENANCE.md`, and — the substantial part — **it diagnosed the -anomaly that has been open since T+60.** - -### 2. `integration-r5` failed the same validator, and the failure explains everything - -r5 ran with the widened bars (35 %/30 %) and `check_no_regression` refused -anyway, with numbers of a **completely different order** from r4's: - -``` -output token throughput r1: 193.59 -> 46.70 -75.9% against a 35% bar -time to first token r1: 178.14 -> 2060.77 +1056.8% against a 30% bar -inter-token latency r1: 40.95 -> 475.02 +1060.0% against a 30% bar -``` - -**The stock arm was ten times faster than the patched arm**, visible in raw step -timings at byte-identical settings — `lm_eval` 23 s vs 428 s, `bench_r1` 44 s vs -161 s. Not the patch (one boolean branch on a module global). CUDA-graph config -**identical in both arms' logs**, so the obvious explanation was ruled out by -reading, not assumed. - -### 3. E9 is diagnosed — the deployment is bistable, per bring-up - -Integration had ~12 minutes and the engine container from the finished run was -still up, so it ran the experiment it had named. **Four identical replays against -one still-running instance:** - -``` -v1 out_tps=46.24 itl=480.45 v2 47.38 / 473.72 -v3 47.40 / 471.31 v4 53.05 / 477.15 -``` - -| metric | within-instance spread (n=4) | -|---|---| -| inter-token latency | **2 %** | -| output token throughput | 15 % | -| time to first token | 45 % | - -The instance **never left its slow state**. So: - -> **The flip is decided at bring-up and persists for the life of the container.** -> It is not noise within a measurement window; it is which of two states a fresh -> deployment instantiates into. - -**This is E9** — the ~2.9 tok/s figure I have carried open for eleven -checkpoints. It was never a property of the model or the hardware; it is one of -two states a container draws at start-up, and the fast state (ITL ~41 ms against -~475 ms) is real and was observed in r5's stock arm. - -**Three consequences, and integration got each right:** - -1. **A per-arm bar cannot fix it, and integration retracts its own T+270 - recalibration**: within an instance ITL is stable to 2 %, so the package's - original 10 % bar *"is not too tight — it is well chosen for the thing it can - see. My widening to 30 % in r5 was calibrated against a cross-instance - artefact and was, in hindsight, the wrong response to the wrong number. The - defaults should stay 5 % and 10 %; the r5 bars should not be copied - anywhere."* -2. **The fix belongs in the design.** The package's README says both arms run - back to back in the same session so results are comparable. **That is not - sufficient**: each arm gets a freshly created container and therefore an - independent draw of the state. The design controls for session, node, trace, - order and image — *and not for the one thing that dominates*. What is needed - is a same-state gate; `check_service_live` proves a deployment is *live*, not - that it is *comparable to the other arm's*. -3. **What the two states are is still unexplained.** Candidates named and - untested: aiter kernel selection at first call, the GatedDeltaNet/linear- - attention path, allocator/NUMA placement of a fresh container. *"Whoever picks - this up starts here, and now knows to compare two bring-ups rather than two - measurements."* - -**And the conclusion for the stage**: `check_no_regression` and `compare` are -**not broken** — they recomputed from raw numbers, agreed, and refused, *"which -is exactly right, because a validator that certified a 10× difference as 'no -regression' would be the broken one."* The refusal reports that **this -deployment is not stable enough to support a two-arm comparison at any bar**. -The pipeline is sound; the measurement environment is not; those are different -findings, and the nine PASSing validators are the evidence for the first. - -### 4. How integration's terminal handoff was obtained - -`packup` never dispatched in either run. The terminal artefact was produced out -of band **using the package's own unmodified code** — `packup.py` driven -directly with `AGENT_SYS_INPUT_` per input, then validated by the -package's own `check_packup_shape.validator` in a hand-built zone with -`args.json` copied verbatim from `steps/verdict.yaml:119-128`: - -``` -packup: 45 file(s), verdict REJECTED -check_packup_shape: oob-integration-packup PASS -``` - -With the caveat stated exactly right: - -> It asserts that the content directory the terminal step would have produced -> passes the validator that step's handoff would have faced. It does **not** -> assert a seal. Say it that way round; *"validated out of band"* is easy to read -> as *"sealed"*. - -Its delivery separates its own work from the leader-authorised r4 recovery, -leaving the latter untouched and explaining that r5 supersedes r4 *"as the better -sample, but reaches the same terminal verdict for the more informative reason."* - -### 5. Code problems — final state - -| # | problem | state | -|---|---|---| -| **C25** | performance bars | **resolved as a design finding**: defaults 5 %/10 % are correct; the r5 widening is retracted by its author; the real gap is a missing same-state gate | -| C9b | `seal_refused` has no reader | **open — recorded, not fixed** | -| C23 | integration's ten closed `items_schema`s | **open — recorded, not fixed** | -| C24 | `check_workset_runs` hard-fails on rsd | **open — recorded, not fixed** | -| C26 | locality not enforced at seal | **documented, deliberate** | - -C9, C20, C28 and the transport/parameter fixes were all closed earlier. - -### 6. Non-code problems — final state - -| # | problem | state | -|---|---|---| -| **E9** | Qwen3.6-27B ~2.9 tok/s | **DIAGNOSED** — bistable per bring-up, fast state ~41 ms ITL, slow ~475 ms. Mechanism still unknown; next steps named | -| E0 | 5/5 new holds died; both originals at 6 h 46 m | **open, unexplained**, fully worked around | -| E10′ | login-node OOM killed two runs | **fixed for profiling; propagation failed** | -| E14 | staging: write into the handoff as you go | **open as a package decision** | -| E16 | saturated node fails `max_rsd` on correct evidence | **open** | - -### 7. Undetermined - -1. **What are the two deployment states?** The one genuinely open technical - question, now sharply posed with three candidates and a stated method. -2. **Should the design gain a same-state gate?** Integration's recommendation; - nobody has ruled on it. -3. **E0** — five holds, unexplained. -4. C9b / C23 / C24 — **recommend handing these over as recorded findings.** No - time remains to act on them and they should not read as pending work. - -### 8. New commits - -``` -50a1532 docs(integration-demo): record the spur localisation traps and the bistable deployment -b8a553a docs(integration-demo): the 10x swing is per-deployment, not per-measurement -``` - -**The working tree is now clean** apart from `.serena/` and this file — every -module's notes and fixes are committed. - -### 9. Other - -- **All five modules delivered.** Four sealed with every validator PASS; - integration with nine sealed (one carrying a correctly-refused verdict) plus an - out-of-band terminal artefact validated by the package's own validator. -- **The effort's best work was its last hour.** Integration came back from an OOM - kill, ran a 12-minute experiment against a container that was still up, and - turned "the only validator failure of the day" into a design finding — while - **retracting its own earlier fix** as the wrong response to the wrong number. -- **E9 closed after eleven checkpoints.** It was the right thing to keep carrying - as undetermined rather than guessing at, and it was solved by someone noticing - a still-running container was the cheapest thing in the session to ask. -- Still not done, tenth checkpoint: **the three BRIEF errors** (settle budget - 14400 s not 1800 s; CPX nodes 64 × 36 GiB not 8 × 288 GiB; locality not - enforced at seal) — plus, now, a fourth worth adding: **on this stack a fresh - container draws one of two performance states, so back-to-back arms are not - automatically comparable.** - ---- - -## T+420 — 2026-09-02 15:35 UTC - -### Walltime countdown - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (7 h 16 m) | 16:17:41 | **42 min** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (7 h 16 m) | 16:17:57 | **42 min** | - -Under an hour. Both originals will reach walltime intact — the only two of seven -holds today to do so. - -### 1. Progress - -**Effort: 100 % delivered.** Elapsed 428 minutes. All five deliverables in -place, unchanged since T+390. The remaining time is being spent on -investigation, not delivery. - -**Reliability: high.** - -### 2. The recurring item is closed: the BRIEF has been corrected - -For ten consecutive checkpoints I have recorded that the BRIEF's errors lived -only in commits and module notes. **That is now resolved.** `BRIEF.md` is 998 -lines (from ~200) and carries the corrections directly: - -- *"The 1800 s settle budget does not exist. `cli/main.py:902` is - `_SETTLE_TIMEOUT = 14400.0` — four hours — and `--timeout SECONDS` exposes it. - Packages' READMEs still cite 1800 s; they are stale."* -- SPX/CPX varies per node and changes your `tp_size`; `crsuse2-m2m-080` measured - at **64 devices × 36 GiB**, with the sizing consequence spelled out — *"GLM is - not viable on the CPX node."* - -The next effort will read the corrections rather than rediscover them. - -### 3. Integration corrected itself again — and this one is the most important - -Three further commits and a **control experiment** that overturns section 17, -which I reported at T+390 as the E9 diagnosis. **My T+390 account was built on a -conclusion its own author has since narrowed.** - -At 14:59 on `-276`, under the same co-tenant load, integration brought up a -**stock** container (`patch mounts: 0`, confirmed in the bring-up log) and -replayed the identical trace three times, against the four patched replays from -fifteen minutes earlier: - -``` -patched mean itl 475.7 ms mean out_tps 48.5 -stock mean itl 470.3 ms mean out_tps 47.3 -``` - -**1.1 % apart. Stock reproduces the slow state exactly; the patch has no -measurable cost** — and the patched arm was nominally *faster*. - -It then names both earlier readings as wrong, including its own: - -- *"a declared no-op cannot cost 21 %, therefore it is run-to-run spread"* — read - `"expect": {"speedup": 1.0}` as a fact about what happened; **it is a statement - of what someone expected**; -- *"the patched arm reproduced at 46.49 and 46.70, therefore the patch is - expensive"* — treated **two draws that both landed in the slow state** as a - reproduction of a patch effect. - -> **The missing thing was never a better inference. It was a control: a stock -> deployment measured under the same conditions.** Nobody had one, because in the -> graph the two arms are always separated by an hour of measurement. - -**And it explicitly weakens section 17**: *"the state is drawn at bring-up and -held for the container's life"* — the first half is too strong. What was measured -is that **one container stays in one state for its life**, equally consistent -with *node conditions* being steady across that life. *"The draw language implies -an internal coin-flip, and there is no evidence for one."* - -The leading explanation is external and was visible all along, unmeasured: - -``` -rocm-smi --showuse -> GPU[0..7] 100% -rocm-smi --showpids -> several processes holding 140-149 GB VRAM each - (another tenant, up ~19 h) -``` - -r5's stock arm ran 12:58–13:40 and measured **193.59** tok/s; a stock arm at -15:00 under that load measures **47**. **Node contention at measurement time -explains every observation without the patch doing anything.** - -**Still not established**, and correctly left open: whether contention is the -whole story, or whether something is *additionally* latched per container. -Distinguishing them needs a genuinely quiet node, which this cluster has not -offered today. - -### 4. E9 — the honest final state - -At T+390 I recorded E9 as **diagnosed: bistable per bring-up**. That is now -**too strong**, and I am recording the correction here rather than editing T+390. -The accurate statement: - -> **Qwen3.6-27B's ~10× throughput swing is explained by node contention at -> measurement time.** A stock control under load reproduces the slow state to -> within 1.1 %, so the patch costs nothing. Whether contention is the *entire* -> cause, or whether a container additionally latches a state at bring-up, is -> **not settled** and needs a quiet node. - -E9 goes from "undiagnosed anomaly" through "diagnosed as bistability" to -"explained by contention, with a residual question" — and the middle step was -wrong. It was corrected in 40 minutes by the person who made it, using an -eight-minute experiment. - -### 5. The stage-level finding, which is what survives - -The reject verdict **stands and is correct** — the arms genuinely differed. But: - -> The two-arm design controls for session, node, trace, order and image. It does -> **not** control for node load at measurement time, and on a shared node that is -> the term that dominates. - -`check_service_live` proves a deployment is *live*, not *comparable to the other -arm's*. The fix is a **comparability gate** — record node load at each arm's -measurement and refuse when they differ, or interleave the arms rather than -running them in sequence. **Widening the bar fixes nothing; it moves the point at -which an uncontrolled variable is allowed through.** - -### 6. The method lesson - -Integration's own summary, and the single most transferable line produced today: - -> Three people — me twice, the leader twice, `deploy` once — produced **five -> confident explanations from the same data set** without running the one cheap -> experiment that could discriminate between them. It cost **eight minutes**: one -> bring-up and three replays. -> -> **When the question is "did X cause Y", the answer is a control, not a -> re-reading. A dataset that lacks the control cannot be argued into containing -> it, however many people look at it.** - -I was a sixth reader of that same data set, and I relayed two of those -explanations as findings at T+270 and T+390. - -### 7. Code and non-code problems — final - -Unchanged from T+390. **C9b**, **C23**, **C24** open and recorded-not-fixed; -**C25** resolved as a design finding (defaults 5 %/10 % correct, the widening -retracted); **C26** documented. **E0** (5 of 7 holds died, unexplained) open; -**E14**, **E16** open; **E10′** fixed for one module, propagation failed; -**E9** as restated in §4 above. - -### 8. Undetermined - -1. **Is contention the whole story, or is something latched per container?** - Needs a quiet node. The one open technical question. -2. **Should the design gain a comparability gate?** Integration's - recommendation; unruled. -3. **E0** — five reaped holds, no explanation. -4. C9b / C23 / C24 — hand over as recorded findings. - -### 9. New commits - -``` -4b62b29 docs(integration-demo): a control experiment shows the mock patch costs nothing -d3bb76a docs(integration-demo): how to reproduce a slow instance, and a ranked next-step list -e42491e docs(integration-demo): the co-tenant load is independent of this module -``` - -Working tree clean apart from `.serena/` and this file. - -### 10. Other - -- **Five modules delivered; the last hour has been pure investigation.** Nothing - in the deliverables changed; what changed is the confidence with which their - one refused verdict can be explained. -- **The day's three self-corrections all came from the module that had been - wrong**: analyze on C9's cause, kernel-opt on "9/9 verified", integration - twice on the regression. None was forced by a reviewer. -- **My own corrections, all recorded in the following section rather than by - editing**: the NFS/`TMPDIR` rule (T+90), C9's root cause (T+150), the - out-of-band recovery's reason (T+330), and E9's diagnosis (here). The - append-only rule earned itself four times. - ---- - -## T+450 — 2026-09-02 16:05 UTC - -### Walltime countdown — **13 minutes left** - -| job | node | owner | state | ends (UTC) | remaining | -|---|---|---|---|---|---| -| `101052` | `crsuse2-m2m-079` | profiling + deploy | RUNNING (7 h 46 m) | 16:17:41 | **13 min** | -| `101053` | `crsuse2-m2m-276` | integration | RUNNING (7 h 46 m) | 16:17:57 | **13 min** | - -Both will reach their 8 h walltime intact — **the only two of seven holds today -to do so.** Five freshly issued holds died inside 70 minutes each; the two issued -in the original batch never faltered. That asymmetry was never explained. - -### 1. Progress - -**Effort: complete.** Elapsed 458 minutes (7 h 38 m). All five modules delivered; -no work in flight; nothing at risk from the walltime. - -**Reliability: high** — the table below is a fresh count taken from disk at this -checkpoint, not carried forward. - -### 2. Final verification sweep — counted at 16:04 UTC - -| module | sealed handoffs | verdicts true | verdicts false | PROVENANCE | size | -|---|---|---|---|---|---| -| **profiling** | 7 | 7 | 0 | yes | 362 M | -| **kernel-opt** | 8 | 12 | 0 | yes | 535 K | -| **deploy** | 1 | 2 | 0 | yes | 526 K | -| **analyze** | 11 | 11 | 0 | yes | 796 K | -| **integration** | 18 | 18 | **2** | yes | 2.6 M | -| **total** | **45** | **50** | **2** | 5/5 | ~366 M | - -Method: `manifest.yaml` and `validation.yaml` counted per deliverable tree; -`result:` lines counted directly. Counts exceed earlier per-run figures because -several deliverables carry both a current and a superseded store -(`kernel-opt/superseded/`, `analyze/store-run2-superseded/`, -`integration/store` (r4) alongside `r5-.../handoffs`), each with its own sealed -manifests. **Every module has a `PROVENANCE.md`.** - -The **two false verdicts** are both `check_no_regression`, one each from -integration's r4 and r5 — the same validator, refusing for the reason -established in §18 of its notes. They are labelled as refusals in three separate -documents. Nothing else in 52 recorded verdicts is a failure. - -### 3. Current state - -All five modules quiescent. No note has grown and no commit has landed in -50 minutes: - -| module | last note | last commit | -|---|---|---| -| deploy | 12:05 | ~12:2x | -| analyze | 12:48 | ~13:0x | -| profiling | 13:23 | ~13:2x | -| kernel-opt | 13:27 | ~13:3x | -| integration | 15:13 | 15:2x | - -Working tree clean apart from `.serena/` and this file. Every module's notes and -fixes are committed. - -### 4. Problems — closing state - -**Fixed and proven in a run:** the srun→`spur exec` transport (profiling, -analyze, integration); `items_schema` drift (analyze, `acb8bfe`, verified by a -green run); the login-node OOM via a `local` transport (`8274a08`); the -`min_requests` floor; hard-coded GPU targets, container names, context lengths -and GLM-only flag groups turned into parameters across three packages. - -**Open, recorded, not fixed** — these should be handed over as findings, not as -pending work: - -| # | finding | -|---|---| -| **C9b** | `seal_refused` has no reader in `agent/runner.py`; a correct, specific refusal is discarded and surfaces as a stalled task. Cost analyze two runs | -| **C23** | integration's ten closed `items_schema`s restate their content types — correct today, latent drift tomorrow | -| **C24** | `check_workset_runs` hard-fails on an rsd breach while forgiving `ran`/`correct` via `min_pass_ratio`, which makes that knob misleading | -| **E0** | five of seven holds reaped mid-run, unexplained; the two originals untouched for 7 h 46 m | -| **E14** | staging: write into the handoff as you go — local scratch cost deploy a complete kit | -| **E16** | a saturated shared node fails `max_rsd` on correct evidence | -| **E9′** | the residual: is node contention the whole story, or is something additionally latched per container? Needs a quiet node | - -**Design finding, the most valuable single output:** the two-arm comparison -controls for session, node, trace, order and image, and **not for node load at -measurement time**, which on a shared node dominates. The fix is a comparability -gate, not a wider bar. - -### 5. Undetermined - -1. **E9′** — contention versus a per-container latch. One experiment, needs a - quiet node. -2. **A comparability gate** for the two-arm design — recommended, unruled. -3. **E0** — why five holds died. -4. Whether the out-of-band pieces in integration's directory stay; deploy - offered unconditionally to remove its packup and nobody needed to decide. - -### 6. New commits - -**None** since T+420. Tree clean. - -### 7. Assessment of the effort - -- **5 of 5 modules delivered**, 45 sealed handoffs, 50 PASS verdicts, 2 refusals - that are correct and labelled as such, 5 PROVENANCE files. -- **Every deliverable understates itself.** Profiling: don't quote a number. - Kernel-opt: no kernel was optimised, and the mock validator passed without - measuring. Deploy: not "both validators PASS" (until it was). Analyze: - `generating`, not `valid` (until it wasn't). Integration: "validated out of - band" is not "sealed" — say it that way round. -- **Nothing regressed at any of the sixteen checkpoints.** No deliverable - withdrawn, no verdict revoked, no finding retracted — only narrowed, corrected - in place, or superseded by a better run. -- **Six substantive self-corrections**, every one from the party that had been - wrong and none forced by a reviewer: analyze on C9's cause and on `top_n`, - kernel-opt on "9/9 verified", deploy on kernel-opt's framing, integration twice - on the regression. Plus my own four, each recorded in the following section - rather than by editing the error. -- **The transferable lesson**, integration's: *when the question is "did X cause - Y", the answer is a control, not a re-reading. A dataset that lacks the control - cannot be argued into containing it, however many people look at it.* Six - readers produced five confident explanations of the same data before someone - spent eight minutes on a control. -- **The BRIEF was corrected** (998 lines, from ~200) so the next effort inherits - the settle budget, the CPX geometry and the localisation traps rather than - rediscovering them. - -This is the last checkpoint at which anything can change on a GPU. I will keep -the 30-minute rhythm until told to stop, but absent further instruction the -record from here is expected to be flat. - ---- - -## T+480 — 2026-09-02 16:35 UTC - -### Walltime — **both holds reached their limit and ended cleanly** - -``` -101052 JobState=TIMEOUT Reason=TimeLimit RunTime=08:00:17 EndTime=16:17:58 -101053 JobState=TIMEOUT Reason=TimeLimit RunTime=08:00:15 EndTime=16:18:12 -``` - -`squeue -u yihou` is **empty**. The effort has no compute. - -`RunTime=08:00:17` against `TimeLimit=08:00:00` is the shape of a hold that ran -its full course and was ended by the scheduler for the stated reason — the exact -opposite of the five that died with `JobState=CANCELLED Reason=None` at 30, 34, -70, 30 and ~30 minutes. Final tally: **two of seven holds survived, and they were -the two issued in the original batch.** Still unexplained, and now unfalsifiable -for today. - -Everything the effort needed from a GPU was finished by 15:20, an hour before -the wall. Nothing was lost to the timeout. - -### 1. Progress - -**Complete.** Elapsed 488 minutes (8 h 08 m). No work outstanding, none possible. - -### 2. Final state — unchanged and verified - -| module | deliverable | size | -|---|---|---| -| profiling | 7 sealed handoffs, 7 verdicts PASS, PROVENANCE | 362 M | -| kernel-opt | 8 sealed, 12 PASS, PROVENANCE, digest self-check | 535 K | -| deploy | 1 sealed, 2 PASS, PROVENANCE, + the unsealed d3 kit | 526 K | -| analyze | 11 sealed, 11 PASS, PROVENANCE, + superseded store | 796 K | -| integration | 18 sealed, 18 PASS **and 2 correctly-refused**, PROVENANCE, + leader/kernel-opt/deploy recovery material | 2.6 M | - -**45 sealed handoffs, 50 PASS, 2 documented refusals, 5 of 5 PROVENANCE files.** - -Notes: analyze 878 lines, deploy 1038, integration 983, kernel-opt 1090, -profiling 671 — **4 660 lines of debug notes**, all committed. Working tree -clean apart from `.serena/` and this file. - -### 3. Nothing changed this interval - -No commits since 15:2x. No note since integration's at 15:13. No module active. -The last 80 minutes have been quiescent, which is the correct end state. - -### 4. What remains open, for handover - -Unchanged from T+450 and now final for this effort: - -- **E9′** — is node contention the whole story, or is something additionally - latched per container? Needs a quiet node. One experiment, method written down. -- **The comparability gate** — the two-arm design does not control for node load - at measurement time. Recommended by integration, unruled. -- **C9b** — `seal_refused` has no reader; a correct refusal is discarded. -- **C23** — integration's ten closed `items_schema`s. -- **C24** — `check_workset_runs` hard-fails on rsd while `min_pass_ratio` - forgives correctness. -- **E0** — five of seven holds reaped, unexplained. -- **E14** — write into the handoff as you go; local scratch cost a complete kit. -- **E16** — a saturated node fails `max_rsd` on correct evidence. - -None of these blocked a delivery. All are recorded with a named file, a measured -symptom, and — where one exists — the experiment that would settle them. - -### 5. Closing note on this file - -Sixteen checkpoints over eight hours, appended and never revised. It contains -four estimates I would now write differently and four findings I reported and -later corrected — the NFS/`TMPDIR` rule (T+90), C9's root cause (T+150), the -reason for integration's out-of-band recovery (T+330), and E9's diagnosis -(T+420). Each correction sits in the section after the error, which is the whole -point of the append-only rule: a reader can see not just what was true at the -end, but how long each wrong thing was believed and what dislodged it. - -Two blind spots are also on the record: deploy at T+90 and integration at T+120, -each reported as "unknown, not scored" rather than guessed at. Both were working -hard and writing nothing, which is what a module deep in a long run looks like -from outside. Given the same evidence I would make the same call. - -I will keep the 30-minute rhythm until the leader says stop, but with no compute, -no active module and a clean tree, the record from here is expected to be flat.