diff --git a/.agents/agents/search.md b/.agents/agents/search.md index 491eace99..88be971a9 100644 --- a/.agents/agents/search.md +++ b/.agents/agents/search.md @@ -10,9 +10,6 @@ permission_mode: never_confirm tools: - terminal - file_editor -skills: - - exa-search - - alphaxiv-paper-lookup --- You are Senpai's external research agent. The delegated prompt begins with one diff --git a/.agents/skills/analyze-experiments/SKILL.md b/.agents/skills/analyze-experiments/SKILL.md index 0be3f32d0..ef1c0e929 100644 --- a/.agents/skills/analyze-experiments/SKILL.md +++ b/.agents/skills/analyze-experiments/SKILL.md @@ -1,7 +1,7 @@ --- name: analyze-experiments description: > - Analyzes and categorizes all ML experiment PRs in the senpai research track. + Operator-side analysis of historical ML experiment PRs in Senpai research tracks. Use this skill whenever the user asks to: analyze experiments, categorize PRs, bucket experiments, summarize what's been tried, understand experiment history, review merged vs closed results, or asks "what experiments have we run / worked / failed". @@ -13,9 +13,11 @@ description: > # Analyze Experiments Skill -Fetches fresh experiment PR data via the list-experiments skill, categorizes each PR -using a team of parallel readers, and produces a 5-section report: full catalogue, -category breakdown with merge rates, merged-only wins, closed-only failures, key narratives. +This is a human/operator analysis guide, not a live advisor or student skill. +It fetches fresh experiment PR data via the list-experiments skill, categorizes +each PR using parallel readers, and produces a 5-section report: full catalogue, +category breakdown with merge rates, merged-only wins, closed-only failures, +and key narratives. --- diff --git a/plugins/senpai/skills/bootstrap-target/SKILL.md b/.agents/skills/bootstrap-target/SKILL.md similarity index 100% rename from plugins/senpai/skills/bootstrap-target/SKILL.md rename to .agents/skills/bootstrap-target/SKILL.md diff --git a/plugins/senpai/skills/bootstrap-target/references/benchmark-integrity-patterns.md b/.agents/skills/bootstrap-target/references/benchmark-integrity-patterns.md similarity index 100% rename from plugins/senpai/skills/bootstrap-target/references/benchmark-integrity-patterns.md rename to .agents/skills/bootstrap-target/references/benchmark-integrity-patterns.md diff --git a/plugins/senpai/skills/bootstrap-target/references/interview-question-bank.md b/.agents/skills/bootstrap-target/references/interview-question-bank.md similarity index 100% rename from plugins/senpai/skills/bootstrap-target/references/interview-question-bank.md rename to .agents/skills/bootstrap-target/references/interview-question-bank.md diff --git a/plugins/senpai/skills/bootstrap-target/references/program-template.md b/.agents/skills/bootstrap-target/references/program-template.md similarity index 100% rename from plugins/senpai/skills/bootstrap-target/references/program-template.md rename to .agents/skills/bootstrap-target/references/program-template.md diff --git a/.agents/skills/git-research-log/SKILL.md b/.agents/skills/git-research-log/SKILL.md index 0176670a7..5f2e47362 100644 --- a/.agents/skills/git-research-log/SKILL.md +++ b/.agents/skills/git-research-log/SKILL.md @@ -1,15 +1,16 @@ --- name: git-research-log description: > - How to document and publish ML experiment results to GitHub as a pull request. - Use this skill whenever a student agent has finished running experiments and - needs to create or update a PR — even if the instruction is just "wrap up" or - "log your results" or "open a PR". Also use it mid-session to update an - in-progress PR as trials complete. + Developer reference for Senpai's retired direct-GitHub experiment logging + workflow. Use only when auditing or migrating legacy research tracks. --- # Experiment PR Skill +This guide is not installed into live advisor or student runtimes. Current +students use the plugin's `submit-experiment-results` skill and typed result +tool instead. + The master agent creates an empty PR and hands it to you. Your job is to fill it out as you run trials and finalize it when you're done. The master reads these PRs to decide what to explore next — completeness and honest analysis matter more than polish. ## PR Title Format diff --git a/.agents/skills/list-experiments/SKILL.md b/.agents/skills/list-experiments/SKILL.md index 6a5766bf3..1282dd5bf 100644 --- a/.agents/skills/list-experiments/SKILL.md +++ b/.agents/skills/list-experiments/SKILL.md @@ -1,11 +1,13 @@ --- name: list-experiments -description: Use this skill whenever you need to list all of the experiment ideas tried and in progress for this research programme. It outputs 3 files organized by usefulness — merged winners, a compact results table, and full details for deep dives. Use when generating new experimental ideas to check what has already been tried. +description: Operator-side inventory of experiment PRs for auditing a Senpai research track. It outputs merged winners, a compact results table, and full details for deep dives. --- # List Experiments -Run the script below to fetch all experiment PRs from the advisor branch and organize them into 3 files. +This is a human/operator audit guide, not a live advisor or student skill. Run +the script below to fetch all experiment PRs from the advisor branch and +organize them into 3 files. The `BASE_BRANCH` should be set to the advisor branch (e.g. `noam`). Check the `$ADVISOR_BRANCH` env var or the PR base branch. diff --git a/.agents/skills/plot-experiment-charts/SKILL.md b/.agents/skills/plot-experiment-charts/SKILL.md index 9639fc345..3000cee97 100644 --- a/.agents/skills/plot-experiment-charts/SKILL.md +++ b/.agents/skills/plot-experiment-charts/SKILL.md @@ -5,19 +5,20 @@ name: plot-experiment-charts description: > - Generate a training curve comparison chart and embed it in a GitHub PR description. - Use this skill whenever a student has finished running experiments and is preparing - to submit their PR for advisor review. Triggers on: "plot training curves", "add chart - to PR", "visualize experiment", "training curve comparison", "plot-experiment-charts", - "add chart", "generate comparison chart". Run this before marking the PR ready for review. + Operator-side guide for generating a target-specific training curve chart + for a legacy experiment PR. Use when auditing or presenting historical runs. --- # Plot Experiment Charts -You've just finished one or more training runs. Before submitting for review, generate a -comparison chart so the advisor can see the training dynamics at a glance — not just the -final numbers, but how the experiment got there. A bolded best-run line and a properly -scaled y-axis make the story immediately readable, even if some runs diverged. +This guide is not installed into live advisor or student runtimes. Target +repositories that require experiment charts should provide their own project +skill with the correct metrics and plotting code. + +Generate a comparison chart so a reviewer can see the training dynamics at a +glance—not just the final numbers, but how the experiment got there. A bolded +best-run line and a properly scaled y-axis make the story immediately readable, +even if some runs diverged. This skill takes about 30 seconds. It's worth it. @@ -25,7 +26,7 @@ This skill takes about 30 seconds. It's worth it. - **Baseline W&B run ID**: in the PR body under `## Baseline`, look for the `W&B run: \`xxxxxxxx\`` line. - **Your own run IDs**: the 8-character W&B IDs of the runs you just completed. Find them in the W&B run URLs or in the training output (the run ID is printed at launch). -- **W&B credentials**: `WANDB_ENTITY` and `WANDB_PROJECT` env vars (already set in the pod environment). +- **W&B credentials**: `WANDB_ENTITY` and `WANDB_PROJECT` in the current environment. ## Step 1 — Run the script diff --git a/AGENTS.md b/AGENTS.md index 8a2b36dbb..ad828d7cb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -19,7 +19,7 @@ When helping a user onboard a target repository, inspect an explicitly configure - operational guardrails such as commands, budgets, allowed edits, protected artifacts, and result-reporting expectations; and - optional research avenues, papers, models, and libraries that provide useful starting points without forcing a narrow solution path. -Favor high-level goals and guardrails that let research agents discover the details. Avoid micromanaging methods or over-prompting one idea unless that narrow focus is the user's explicit goal. The [`bootstrap-target`](plugins/senpai/skills/bootstrap-target/SKILL.md) guide and its template can turn the confirmed decisions into the target contract. +Favor high-level goals and guardrails that let research agents discover the details. Avoid micromanaging methods or over-prompting one idea unless that narrow focus is the user's explicit goal. The [`bootstrap-target`](.agents/skills/bootstrap-target/SKILL.md) guide and its template can turn the confirmed decisions into the target contract. Reference examples: diff --git a/README.md b/README.md index 129bca4d0..410349dfd 100644 --- a/README.md +++ b/README.md @@ -303,27 +303,30 @@ Children share the parent workspace, so their process and conversation are isola ## Task guides -OpenHands receives these as progressively disclosed skills; their bodies are loaded only when the task calls for them. +Live advisors and students load Senpai-owned skills only from +`plugins/senpai/skills`; target repositories may also supply project skills. +Guides under `.agents/skills` are for human users and Senpai developers and +are not installed into autoresearch pods. -### Core research workflow +### Live autoresearch | Guide | Purpose | |---|---| -| [Bootstrap a target](plugins/senpai/skills/bootstrap-target/SKILL.md) | Build `program.md` from a new ML repository. | | [Assign an experiment](plugins/senpai/skills/assign-experiment/SKILL.md) | Turn a hypothesis into a typed student branch and draft PR. | | [Delegate subagents](plugins/senpai/skills/delegate-subagents/SKILL.md) | Launch and coordinate bounded parallel research, review, and implementation help. | | [Submit experiment results](plugins/senpai/skills/submit-experiment-results/SKILL.md) | Commit the tested implementation and publish a structured, evidence-backed result. | | [Review an experiment](plugins/senpai/skills/review-experiment/SKILL.md) | Merge a reproducible winner, close a useful negative, or request the missing evidence. | | [Handle human Issues](plugins/senpai/skills/check-human-issues/SKILL.md) | Respond to authenticated human-to-agent messages delivered through GitHub Issues. | +| [Senpai status check](plugins/senpai/skills/senpai-status-check/SKILL.md) | Produce a bounded, read-only GitHub, W&B, and local-controller status report. | +| [Exa search](plugins/senpai/skills/exa-search/SKILL.md) | Search the current web or scholarly publications with mode-specific defaults. | +| [AlphaXiv paper lookup](plugins/senpai/skills/alphaxiv-paper-lookup/SKILL.md) | Get a structured overview before reading a primary paper deeply. | +| [W&B and Weave](plugins/senpai/skills/wandb-primary/SKILL.md) | Inspect runs, metrics, artifacts, evaluations, and agent traces. | -### Evidence and research +### Human and developer guides | Guide | Purpose | |---|---| -| [Senpai status check](.agents/skills/senpai-status-check/SKILL.md) | Produce a bounded, read-only GitHub, W&B, and local-controller status report. | -| [Exa search](.agents/skills/exa-search/SKILL.md) | Search the current web or scholarly publications with mode-specific defaults. | -| [AlphaXiv paper lookup](.agents/skills/alphaxiv-paper-lookup/SKILL.md) | Get a structured overview before reading a primary paper deeply. | -| [W&B and Weave](.agents/skills/wandb-primary/SKILL.md) | Inspect runs, metrics, artifacts, evaluations, and agent traces. | +| [Bootstrap a target](.agents/skills/bootstrap-target/SKILL.md) | Build `program.md` from a new ML repository. | | [Experiment report](.agents/skills/experiment-report/SKILL.md) | Create the project-standard `nn_cfd` W&B comparison report; this guide is target-specific rather than part of the generic runtime. | | [Training code style](literature_and_guidance/TRAINING-CODE-STYLE.md) | Structure expensive ML entrypoints so configuration, artifacts, validation, and failure boundaries stay explicit. | diff --git a/SPEC.md b/SPEC.md index f0283b78b..ac39cb71e 100644 --- a/SPEC.md +++ b/SPEC.md @@ -215,7 +215,10 @@ suffix on every inference, and current time is rendered for every controller wake. Operators must start fresh role state to apply a changed identity, program, or role charter. -File-based subagents are discovered from `.agents/agents`. Skill bodies are not +File-based subagents are discovered from `.agents/agents`. Live advisor and +student skills come only from `plugins/senpai/skills`; `.agents/skills` is for +human operators and Senpai developers and is not installed into pods. Target +repositories may still supply their own project skills. Skill bodies are not concatenated into agent definitions. The OpenHands fork's `main` branch applies each agent definition's `reasoning_effort` override after resolving its inherited LLM or stored model profile. @@ -687,7 +690,8 @@ Removed: Retained intentionally: -- Agent skills and their model/effort metadata under `.agents`; +- runtime skills and their model/effort metadata in the Senpai plugin; +- human and developer guides under `.agents/skills`, outside pod context; - OpenHands Browser, task tracker, Think, and the high-quality default condenser for providers not using stored OpenAI Responses continuation or Anthropic native compaction; diff --git a/k8s/entrypoint-advisor.sh b/k8s/entrypoint-advisor.sh index a666ff7d3..32359e7d4 100644 --- a/k8s/entrypoint-advisor.sh +++ b/k8s/entrypoint-advisor.sh @@ -129,7 +129,7 @@ ls \ "$HOME/.agents/agents/general-purpose.md" \ "$HOME/.agents/agents/explore.md" \ "$HOME/.agents/agents/search.md" \ - "$HOME/.agents/skills/wandb-primary/SKILL.md" + "$SENPAI_PLUGIN/skills/wandb-primary/SKILL.md" # --- Hivemind is intentionally disabled pending its OpenHands rewrite. --- # source "$WORKDIR/k8s/start-hivemind.sh" diff --git a/k8s/entrypoint-student.sh b/k8s/entrypoint-student.sh index e995cc4d3..772bc3dcb 100644 --- a/k8s/entrypoint-student.sh +++ b/k8s/entrypoint-student.sh @@ -79,7 +79,7 @@ ls \ "$HOME/.agents/agents/general-purpose.md" \ "$HOME/.agents/agents/explore.md" \ "$HOME/.agents/agents/search.md" \ - "$HOME/.agents/skills/wandb-primary/SKILL.md" + "$SENPAI_PLUGIN/skills/wandb-primary/SKILL.md" # --- Hivemind is intentionally disabled pending its OpenHands rewrite. --- # source "$WORKDIR/k8s/start-hivemind.sh" diff --git a/plugins/senpai/README.md b/plugins/senpai/README.md index ad6da0c70..ced81e1a6 100644 --- a/plugins/senpai/README.md +++ b/plugins/senpai/README.md @@ -32,7 +32,10 @@ no MCP server. The Python runtime binds the authenticated role and adds the canonical `ADVISOR:` or `STUDENT:` prefix to Senpai-authored GitHub comments; tool payloads contain only the unprefixed message text. -Keep Senpai-owned workflow skills here rather than relying on a provider's user -skill directory. Never commit secret values. The plugin remains the source of -truth for reusable workflow guidance, while Python remains the source of truth -for verified state changes. +Keep every Senpai-owned skill used by a live advisor or student here rather +than relying on a provider's user skill directory. Target repositories may +supply project skills separately. Human onboarding and developer guides stay +under the runner's `.agents/skills` and are not installed into pods. Never +commit secret values. The plugin remains the source of truth for reusable +runtime guidance, while Python remains the source of truth for verified state +changes. diff --git a/plugins/senpai/scripts/agent-context.sh b/plugins/senpai/scripts/agent-context.sh index 998c2d296..59701263d 100644 --- a/plugins/senpai/scripts/agent-context.sh +++ b/plugins/senpai/scripts/agent-context.sh @@ -10,15 +10,10 @@ install_senpai_agent_context() { local runtime_root="$3" local runtime_plugin="$runtime_root/plugin" - mkdir -p "$HOME/.agents/skills" - cp -a "$workdir/.agents/." "$HOME/.agents/" - for marker in "$HOME"/.agents/skills/*/.senpai-developer-only; do - [ -e "$marker" ] || continue - rm -rf -- "${marker%/.senpai-developer-only}" - done + mkdir -p "$HOME/.agents/agents" + cp -a "$workdir/.agents/agents/." "$HOME/.agents/agents/" cp -a "$source_plugin" "$runtime_plugin" "$SENPAI_PYTHON" -m senpai_agent.agent_markdown \ - "$HOME/.agents" "$runtime_plugin" - cp -a "$runtime_plugin/skills/." "$HOME/.agents/skills/" + "$HOME/.agents/agents" "$runtime_plugin" printf '%s\n' "$runtime_plugin" } diff --git a/.agents/skills/alphaxiv-paper-lookup/SKILL.md b/plugins/senpai/skills/alphaxiv-paper-lookup/SKILL.md similarity index 100% rename from .agents/skills/alphaxiv-paper-lookup/SKILL.md rename to plugins/senpai/skills/alphaxiv-paper-lookup/SKILL.md diff --git a/.agents/skills/exa-search/SKILL.md b/plugins/senpai/skills/exa-search/SKILL.md similarity index 95% rename from .agents/skills/exa-search/SKILL.md rename to plugins/senpai/skills/exa-search/SKILL.md index 1586db8a9..856d03f11 100644 --- a/.agents/skills/exa-search/SKILL.md +++ b/plugins/senpai/skills/exa-search/SKILL.md @@ -10,13 +10,13 @@ official `exa_py` client, loads the nearest `.env` through `python-dotenv`, and preserves an `EXA_API_KEY` already set in the environment. ```bash -python "$HOME/.agents/skills/exa-search/scripts/search_exa.py" \ +python "$SENPAI_PLUGIN/skills/exa-search/scripts/search_exa.py" \ general-web \ "current OpenHands SDK file-based agent documentation" ``` ```bash -python "$HOME/.agents/skills/exa-search/scripts/search_exa.py" \ +python "$SENPAI_PLUGIN/skills/exa-search/scripts/search_exa.py" \ research-publications \ "uncertainty calibration for neural networks" ``` diff --git a/.agents/skills/exa-search/scripts/search_exa.py b/plugins/senpai/skills/exa-search/scripts/search_exa.py similarity index 100% rename from .agents/skills/exa-search/scripts/search_exa.py rename to plugins/senpai/skills/exa-search/scripts/search_exa.py diff --git a/.agents/skills/senpai-status-check/SKILL.md b/plugins/senpai/skills/senpai-status-check/SKILL.md similarity index 100% rename from .agents/skills/senpai-status-check/SKILL.md rename to plugins/senpai/skills/senpai-status-check/SKILL.md diff --git a/.agents/skills/wandb-primary/SKILL.md b/plugins/senpai/skills/wandb-primary/SKILL.md similarity index 91% rename from .agents/skills/wandb-primary/SKILL.md rename to plugins/senpai/skills/wandb-primary/SKILL.md index fa14da36b..b904be0b0 100644 --- a/.agents/skills/wandb-primary/SKILL.md +++ b/plugins/senpai/skills/wandb-primary/SKILL.md @@ -59,13 +59,13 @@ This skill covers everything an agent needs to work with Weights & Biases: | I need to... | Use | |---|---| -| Query training runs, loss curves, hyperparameters | **W&B SDK** (`wandb.Api()`) — see `references/WANDB_SDK.md` | -| Query GenAI traces, calls, evaluations | **Weave SDK** (`weave.init()`, `client.get_calls()`) — see `references/WEAVE_SDK.md` | +| Query training runs, loss curves, hyperparameters | **W&B SDK** (`wandb.Api()`) — see `$SENPAI_PLUGIN/skills/wandb-primary/references/WANDB_SDK.md` | +| Query GenAI traces, calls, evaluations | **Weave SDK** (`weave.init()`, `client.get_calls()`) — see `$SENPAI_PLUGIN/skills/wandb-primary/references/WEAVE_SDK.md` | | Convert Weave wrapper types to plain Python | **`weave_helpers.unwrap()`** | | Build a DataFrame from training runs | **`wandb_helpers.runs_to_dataframe()`** | | Extract eval results for analysis | **`weave_helpers.eval_results_to_dicts()`** | -| Need low-level Weave filtering (CallsFilter, Query) | **Raw Weave SDK** (`weave.init()`, `client.get_calls()`) — see `references/WEAVE_SDK.md` | -| Judge curve shape (spikes, smoothness, slope, overfit) | **`training_diagnostics` + `curve_plots`** — use the workflow below, then load `references/TRAINING_DIAGNOSTICS.md` for the heuristics | +| Need low-level Weave filtering (CallsFilter, Query) | **Raw Weave SDK** (`weave.init()`, `client.get_calls()`) — see `$SENPAI_PLUGIN/skills/wandb-primary/references/WEAVE_SDK.md` | +| Judge curve shape (spikes, smoothness, slope, overfit) | **`training_diagnostics` + `curve_plots`** — use the workflow below, then load `$SENPAI_PLUGIN/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md` for the heuristics | --- @@ -74,8 +74,9 @@ This skill covers everything an agent needs to work with Weights & Biases: ### Helper libraries ```python +import os import sys -sys.path.insert(0, ".agents/skills/wandb-primary/scripts") +sys.path.insert(0, f"{os.environ['SENPAI_PLUGIN']}/skills/wandb-primary/scripts") # Weave helpers (traces, evals, GenAI) from weave_helpers import ( @@ -125,9 +126,9 @@ from curve_plots import ( Read these as needed — they contain full API surfaces and recipes: -- **`references/WEAVE_SDK.md`** — Weave SDK for GenAI traces (`client.get_calls()`, `CallsFilter`, `Query`, stats). Start here for Weave queries. -- **`references/WANDB_SDK.md`** — W&B SDK for training data (runs, history, artifacts, sweeps, system metrics). -- **`references/TRAINING_DIAGNOSTICS.md`** — reference heuristics for reading loss / LR / grad-norm / grad-histogram charts. Load this when you are actively interpreting training curves. +- **`$SENPAI_PLUGIN/skills/wandb-primary/references/WEAVE_SDK.md`** — Weave SDK for GenAI traces (`client.get_calls()`, `CallsFilter`, `Query`, stats). Start here for Weave queries. +- **`$SENPAI_PLUGIN/skills/wandb-primary/references/WANDB_SDK.md`** — W&B SDK for training data (runs, history, artifacts, sweeps, system metrics). +- **`$SENPAI_PLUGIN/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md`** — reference heuristics for reading loss / LR / grad-norm / grad-histogram charts. Load this when you are actively interpreting training curves. --- @@ -216,7 +217,7 @@ df = pd.DataFrame(rows) print(df.describe()) ``` -For full W&B SDK reference (filters, history, artifacts, sweeps), read `references/WANDB_SDK.md`. +For full W&B SDK reference (filters, history, artifacts, sweeps), read `$SENPAI_PLUGIN/skills/wandb-primary/references/WANDB_SDK.md`. ### Weave — SDK @@ -226,7 +227,7 @@ client = weave.init(f"{entity}/{project}") # positional string, NOT keyword arg calls = client.get_calls(limit=10) ``` -For raw SDK patterns (CallsFilter, Query, advanced filtering), read `references/WEAVE_SDK.md`. +For raw SDK patterns (CallsFilter, Query, advanced filtering), read `$SENPAI_PLUGIN/skills/wandb-primary/references/WEAVE_SDK.md`. --- @@ -307,7 +308,7 @@ For structured failure analysis on eval results: 3. **Axial coding** — write a second Scorer that classifies notes into a taxonomy 4. **Summarize** — count primary labels with `collections.Counter` -See `references/WEAVE_SDK.md` for the full SDK reference. +See `$SENPAI_PLUGIN/skills/wandb-primary/references/WEAVE_SDK.md` for the full SDK reference. ### W&B Reports @@ -344,7 +345,7 @@ Keep the inline workflow short and load detail on demand: 1. Confirm `step_key` before doing any curve work. Never assume `_step`. 2. Compute features with the bundled helpers instead of hand-rolling spike or slope logic. 3. Render PNGs and inspect them visually. -4. Load `references/TRAINING_DIAGNOSTICS.md` while you interpret the results. +4. Load `$SENPAI_PLUGIN/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md` while you interpret the results. 5. End with a verdict, evidence tied to step ranges, and concrete next actions. ### Required sequence @@ -370,7 +371,7 @@ Next actions: - ``` -Load `references/TRAINING_DIAGNOSTICS.md` for the interpretation heuristics, especially when the numbers and the image disagree. +Load `$SENPAI_PLUGIN/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md` for the interpretation heuristics, especially when the numbers and the image disagree. --- diff --git a/.agents/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md b/plugins/senpai/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md similarity index 100% rename from .agents/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md rename to plugins/senpai/skills/wandb-primary/references/TRAINING_DIAGNOSTICS.md diff --git a/.agents/skills/wandb-primary/references/WANDB_SDK.md b/plugins/senpai/skills/wandb-primary/references/WANDB_SDK.md similarity index 100% rename from .agents/skills/wandb-primary/references/WANDB_SDK.md rename to plugins/senpai/skills/wandb-primary/references/WANDB_SDK.md diff --git a/.agents/skills/wandb-primary/references/WEAVE_SDK.md b/plugins/senpai/skills/wandb-primary/references/WEAVE_SDK.md similarity index 100% rename from .agents/skills/wandb-primary/references/WEAVE_SDK.md rename to plugins/senpai/skills/wandb-primary/references/WEAVE_SDK.md diff --git a/.agents/skills/wandb-primary/scripts/curve_plots.py b/plugins/senpai/skills/wandb-primary/scripts/curve_plots.py similarity index 100% rename from .agents/skills/wandb-primary/scripts/curve_plots.py rename to plugins/senpai/skills/wandb-primary/scripts/curve_plots.py diff --git a/.agents/skills/wandb-primary/scripts/step_axis.py b/plugins/senpai/skills/wandb-primary/scripts/step_axis.py similarity index 100% rename from .agents/skills/wandb-primary/scripts/step_axis.py rename to plugins/senpai/skills/wandb-primary/scripts/step_axis.py diff --git a/.agents/skills/wandb-primary/scripts/training_diagnostics.py b/plugins/senpai/skills/wandb-primary/scripts/training_diagnostics.py similarity index 100% rename from .agents/skills/wandb-primary/scripts/training_diagnostics.py rename to plugins/senpai/skills/wandb-primary/scripts/training_diagnostics.py diff --git a/.agents/skills/wandb-primary/scripts/wandb_helpers.py b/plugins/senpai/skills/wandb-primary/scripts/wandb_helpers.py similarity index 98% rename from .agents/skills/wandb-primary/scripts/wandb_helpers.py rename to plugins/senpai/skills/wandb-primary/scripts/wandb_helpers.py index fda4095ef..a9bdf384a 100644 --- a/.agents/skills/wandb-primary/scripts/wandb_helpers.py +++ b/plugins/senpai/skills/wandb-primary/scripts/wandb_helpers.py @@ -9,8 +9,9 @@ run data into pandas-friendly structures for analysis. Usage (in sandbox): + import os import sys - sys.path.insert(0, ".agents/skills/wandb-primary/scripts") + sys.path.insert(0, f"{os.environ['SENPAI_PLUGIN']}/skills/wandb-primary/scripts") from wandb_helpers import ( runs_to_dataframe, # Convert runs to a clean pandas DataFrame diagnose_run, # Quick diagnostic summary of a training run diff --git a/.agents/skills/wandb-primary/scripts/weave_helpers.py b/plugins/senpai/skills/wandb-primary/scripts/weave_helpers.py similarity index 99% rename from .agents/skills/wandb-primary/scripts/weave_helpers.py rename to plugins/senpai/skills/wandb-primary/scripts/weave_helpers.py index 63f59fc19..99acbdd1d 100644 --- a/.agents/skills/wandb-primary/scripts/weave_helpers.py +++ b/plugins/senpai/skills/wandb-primary/scripts/weave_helpers.py @@ -9,8 +9,9 @@ Python and extract structured data from calls and evals for pandas analysis. Usage (in sandbox): + import os import sys - sys.path.insert(0, ".agents/skills/wandb-primary/scripts") + sys.path.insert(0, f"{os.environ['SENPAI_PLUGIN']}/skills/wandb-primary/scripts") from weave_helpers import ( unwrap, # Recursively convert Weave types -> plain Python get_token_usage, # Extract token counts from a call's summary diff --git a/senpai_agent/openhands_runner.py b/senpai_agent/openhands_runner.py index 7c1d2fca4..15f20f799 100644 --- a/senpai_agent/openhands_runner.py +++ b/senpai_agent/openhands_runner.py @@ -66,7 +66,6 @@ from openhands.sdk.skills import ( Skill, load_skills_from_dir, - load_user_skills, merge_skills_by_name, ) from openhands.sdk.subagent import ( @@ -517,9 +516,7 @@ def resolve_agent_skills( if not definition.skills: return [] - user_skills = (skill for skill in load_user_skills() if is_exposed_skill(skill)) - available = {skill.name: skill for skill in user_skills} - available.update({skill.name: skill for skill in project_skills}) + available = {skill.name: skill for skill in project_skills} missing = [name for name in definition.skills if name not in available] if missing: raise ValueError( @@ -939,6 +936,7 @@ def with_system_instructions( "system_message_suffix": system_suffix, "current_datetime": None, "skills": list(skills.values()), + "load_user_skills": False, "load_project_skills": False, } ) @@ -955,7 +953,7 @@ def build_main_agent_context( system_message_suffix=instructions.prompt, current_datetime=None, load_public_skills=False, - load_user_skills=True, + load_user_skills=False, load_project_skills=False, ) diff --git a/system_instructions/ADVISOR.md b/system_instructions/ADVISOR.md index 5535e3795..8b0535478 100644 --- a/system_instructions/ADVISOR.md +++ b/system_instructions/ADVISOR.md @@ -43,7 +43,7 @@ You are the principal research lead of this lab and you want to see your student Whenever you post a PR comment, issue reply, board message, result, baseline update, or research-state summary that references one or more experiments, always include a direct W&B link for every referenced experiment. Prefer the run URL and include the run id next to the link. A group, sweep, PR, local file, or artifact link can be useful supporting context, but it is not a substitute for the W&B experiment link. -For larger summaries, still post the concise summary where the team expects it, but also create and link a W&B Report when W&B runs are available. Use the `experiment-report` skill if it fits the project. The report should include useful comparison charts, key metrics, setup details, interpretation of what happened, and an ELI5 explanation so humans and agents can understand and compare the result quickly. +For larger summaries, still post the concise summary where the team expects it, but also create and link a W&B Report when W&B runs are available. Include useful comparison charts, key metrics, setup details, interpretation of what happened, and an ELI5 explanation so humans and agents can understand and compare the result quickly. ## Priorities diff --git a/tests/exa_search_support.py b/tests/exa_search_support.py index 2294dec54..bc82aced8 100644 --- a/tests/exa_search_support.py +++ b/tests/exa_search_support.py @@ -4,7 +4,8 @@ SCRIPT = ( Path(__file__).parents[1] - / ".agents" + / "plugins" + / "senpai" / "skills" / "exa-search" / "scripts" diff --git a/tests/test_agent_markdown.py b/tests/test_agent_markdown.py index 1d89abfcf..b9317739e 100644 --- a/tests/test_agent_markdown.py +++ b/tests/test_agent_markdown.py @@ -88,9 +88,11 @@ def test_agent_context_installer_builds_loadable_sanitized_runtime_copies( home.mkdir() runtime_root.mkdir() source_skill = PLUGIN_DIR / "skills" / "review-experiment" / "SKILL.md" + operator_skill = ROOT / ".agents" / "skills" / "experiment-report" / "SKILL.md" source_agent = ROOT / ".agents" / "agents" / "bash-runner.md" originals = { source_skill: source_skill.read_text(encoding="utf-8"), + operator_skill: operator_skill.read_text(encoding="utf-8"), source_agent: source_agent.read_text(encoding="utf-8"), } @@ -115,9 +117,19 @@ def test_agent_context_installer_builds_loadable_sanitized_runtime_copies( plugin = Plugin.load(runtime_plugin) agents = discover_agents(home, include_project=True, include_user=False) - assert "review-experiment" in {skill.name for skill in plugin.skills} + assert {skill.name for skill in plugin.skills} == { + "alphaxiv-paper-lookup", + "assign-experiment", + "check-human-issues", + "delegate-subagents", + "exa-search", + "review-experiment", + "senpai-status-check", + "submit-experiment-results", + "wandb-primary", + } assert "bash-runner" in {agent.name for agent in agents} - assert not (home / ".agents/skills/senpai-tool-telemetry").exists() + assert not (home / ".agents/skills").exists() assert all( strip_spdx_header(text) == text for root in (runtime_plugin, home / ".agents") diff --git a/tests/test_delegation.py b/tests/test_delegation.py index 4aa6c2074..919bed789 100644 --- a/tests/test_delegation.py +++ b/tests/test_delegation.py @@ -182,6 +182,10 @@ def test_child_command_selects_agent_model_effort_and_credential(tmp_path: Path) ) assert "anthropic/claude-opus-4-8" in smart.command assert smart.command[smart.command.index("--reasoning-effort") + 1] == "xhigh" + assert "--child" in smart.command + assert smart.command[smart.command.index("--plugin-dir") + 1] == str( + config.plugin_dir + ) assert "openai/gpt-5.6" in frontier.command assert frontier.command[frontier.command.index("--reasoning-effort") + 1] == "max" assert frontier.command[frontier.command.index("--api-key-env") + 1] == ( diff --git a/tests/test_openhands_config.py b/tests/test_openhands_config.py index 094482697..7195a1abd 100644 --- a/tests/test_openhands_config.py +++ b/tests/test_openhands_config.py @@ -71,7 +71,7 @@ def test_main_agent_context_appends_program_after_harness_and_role(): "# Authoritative launch context\n\nRuntime policy.\n" ) assert context.current_datetime is None - assert context.load_user_skills is True + assert context.load_user_skills is False assert context.load_project_skills is False diff --git a/tests/test_openhands_tools_and_agents.py b/tests/test_openhands_tools_and_agents.py index 0ed5dce71..d3691be64 100644 --- a/tests/test_openhands_tools_and_agents.py +++ b/tests/test_openhands_tools_and_agents.py @@ -6,9 +6,9 @@ from types import SimpleNamespace import pytest -from openhands.sdk import Agent, LLM, Tool +from openhands.sdk import Agent, LLM, LocalConversation, Tool from openhands.sdk.tool import resolve_tool -from openhands.sdk.plugin import Plugin +from openhands.sdk.plugin import Plugin, PluginSource from openhands.sdk.subagent import AgentDefinition, agent_definition_to_factory from openhands.tools.preset.default import register_default_tools from pydantic import SecretStr @@ -204,13 +204,33 @@ def test_native_senpai_plugin_loads_its_runtime_skills(): assert plugin.manifest.name == "senpai" skills = {skill.name: skill for skill in plugin.skills} assert set(skills) == { + "alphaxiv-paper-lookup", "assign-experiment", - "bootstrap-target", "check-human-issues", "delegate-subagents", + "exa-search", "review-experiment", + "senpai-status-check", "submit-experiment-results", + "wandb-primary", } + operator_skills = { + path.parent.name + for path in (REPO_ROOT / ".agents" / "skills").glob("*/SKILL.md") + } + assert { + "analyze-experiments", + "bootstrap-target", + "experiment-report", + "git-research-log", + "list-experiments", + "plot-experiment-charts", + "rlm", + "senpai-tool-telemetry", + "slidev", + } <= operator_skills + assert set(skills).isdisjoint(operator_skills) + assert all(skill.is_agentskills_format for skill in skills.values()) assert "merge_experiment" in skills["review-experiment"].content assert "close_experiment" in skills["review-experiment"].content assert plugin.mcp_config == {} @@ -279,7 +299,10 @@ def test_markdown_agents_register_and_construct_with_the_native_loader(tmp_path) home = tmp_path / "home" workspace = tmp_path / "target" workspace.mkdir() - shutil.copytree(REPO_ROOT / ".agents", home / ".agents") + shutil.copytree( + REPO_ROOT / ".agents" / "agents", + home / ".agents" / "agents", + ) program = textwrap.dedent( """ import os @@ -347,39 +370,40 @@ def test_markdown_agents_register_and_construct_with_the_native_loader(tmp_path) assert result.returncode == 0, result.stdout + result.stderr -def test_search_agent_loads_its_progressive_skills_and_inherits_reasoning_effort( +def test_search_agent_receives_skills_from_the_runtime_plugin( monkeypatch, + tmp_path, ): - import openhands.sdk.skills.skill as skill_module - - monkeypatch.setattr( - skill_module, - "USER_SKILLS_DIRS", - [REPO_ROOT / ".agents" / "skills", PLUGIN_DIR / "skills"], - ) + monkeypatch.setenv("HOME", str(tmp_path / "home")) monkeypatch.setenv("SENPAI_ROLE", "advisor") register_default_tools(enable_browser=False) register_senpai_tools() definition = AgentDefinition.load(AGENT_DIR / "search.md") - agent = agent_definition_to_factory(definition, work_dir=REPO_ROOT)( + agent = agent_definition_to_factory(definition, work_dir=tmp_path)( LLM( model="anthropic/claude-opus-4-8", api_key=SecretStr("test-key"), reasoning_effort="low", ) ) - - assert agent.llm.reasoning_effort == "low" - assert {skill.name for skill in agent.agent_context.skills} == { - "exa-search", - "alphaxiv-paper-lookup", - } - assert all(skill.is_agentskills_format for skill in agent.agent_context.skills) - assert all( - skill.content not in agent.agent_context.system_message_suffix - for skill in agent.agent_context.skills + conversation = LocalConversation( + agent=agent, + workspace=tmp_path, + plugins=[PluginSource(source=str(PLUGIN_DIR))], + visualizer=None, ) + assert definition.skills == [] + assert agent.agent_context.skills == [] + conversation._ensure_plugins_loaded() + try: + assert {skill.name for skill in conversation.agent.agent_context.skills} >= { + "exa-search", + "alphaxiv-paper-lookup", + } + finally: + conversation.close() + @pytest.mark.parametrize( ("filename", "name", "effort", "tools", "skills"), @@ -412,7 +436,7 @@ def test_search_agent_loads_its_progressive_skills_and_inherits_reasoning_effort "search", None, {"terminal", "file_editor"}, - {"exa-search", "alphaxiv-paper-lookup"}, + set(), ), ], ) @@ -506,7 +530,6 @@ def test_core_senpai_prompts_do_not_assume_a_physical_ai_target(): *(REPO_ROOT / "system_instructions").glob("*.md"), *(REPO_ROOT / ".agents" / "agents").glob("*.md"), *(REPO_ROOT / "plugins" / "senpai" / "skills").glob("**/*.md"), - REPO_ROOT / ".agents" / "skills" / "exa-search" / "SKILL.md", ] prompts = "\n".join( path.read_text(encoding="utf-8").lower() for path in prompt_paths