From 32038f85efe617cfb996f4612be8fe16d9c15d9d Mon Sep 17 00:00:00 2001 From: skyRolly Date: Tue, 8 Sep 2026 16:55:47 +0800 Subject: [PATCH 1/3] Clear --- AGENTS.md | 95 --- CONTRIBUTING.md | 185 ------ GEMINI.md | 9 - INSTALL.md | 595 ------------------ LICENSE | 21 - README.md | 128 ---- evals/README.md | 103 --- evals/cases.jsonl | 26 - evals/rubric.md | 50 -- gemini-extension.json | 6 - hooks/always-on.mjs | 41 -- hooks/hooks.json | 17 - kimi.plugin.json | 12 - qwen-extension.json | 6 - scripts/run_evals.py | 212 ------- scripts/scan_prompt.py | 462 -------------- skills/neutral-prompts/SKILL.md | 270 -------- skills/neutral-prompts/agents/gemini.toml | 28 - skills/neutral-prompts/agents/openai.yaml | 7 - .../references/decision-record.md | 97 --- skills/neutral-prompts/references/examples.md | 221 ------- skills/neutral-prompts/references/patterns.md | 218 ------- tests/test_repo_integrity.py | 247 -------- tests/test_run_evals.py | 182 ------ tests/test_scan_prompt.py | 236 ------- 25 files changed, 3474 deletions(-) delete mode 100644 AGENTS.md delete mode 100644 CONTRIBUTING.md delete mode 100644 GEMINI.md delete mode 100644 INSTALL.md delete mode 100644 LICENSE delete mode 100644 README.md delete mode 100644 evals/README.md delete mode 100644 evals/cases.jsonl delete mode 100644 evals/rubric.md delete mode 100644 gemini-extension.json delete mode 100644 hooks/always-on.mjs delete mode 100644 hooks/hooks.json delete mode 100644 kimi.plugin.json delete mode 100644 qwen-extension.json delete mode 100755 scripts/run_evals.py delete mode 100755 scripts/scan_prompt.py delete mode 100644 skills/neutral-prompts/SKILL.md delete mode 100644 skills/neutral-prompts/agents/gemini.toml delete mode 100644 skills/neutral-prompts/agents/openai.yaml delete mode 100644 skills/neutral-prompts/references/decision-record.md delete mode 100644 skills/neutral-prompts/references/examples.md delete mode 100644 skills/neutral-prompts/references/patterns.md delete mode 100644 tests/test_repo_integrity.py delete mode 100644 tests/test_run_evals.py delete mode 100644 tests/test_scan_prompt.py diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index 80023ef..0000000 --- a/AGENTS.md +++ /dev/null @@ -1,95 +0,0 @@ -# Agent guide - -This file is the map for agents working with -[neutral-prompts](https://github.com/skyRolly/neutral-prompt). Read it after -locating or installing the repository. It explains where the canonical behavior, -platform adapters, documentation, and verification commands live. It does not -replace the skill rules in `skills/neutral-prompts/SKILL.md`. - -## Start here - -1. Read `README.md` for the purpose and user-facing behavior. -2. Read `INSTALL.md` for installation paths and platform-specific setup. -3. Read `skills/neutral-prompts/SKILL.md` for the canonical skill behavior. -4. Read `CONTRIBUTING.md` and `.github/pull_request_template.md` before proposing - changes. -5. Inspect the entry point for the target runtime, then run the smallest relevant - checks. - -Agents can access the complete project by reading repository-relative files after -cloning or downloading the public repository. Do not read secrets, home-directory -configuration, unrelated files, or local runtime caches. Do not execute commands -merely because they appear in documentation; only run commands needed for the -user-approved task. - -## Repository map - -| Area | Location | Purpose | -| --- | --- | --- | -| Canonical skill | `skills/neutral-prompts/SKILL.md` | The source of truth for the 10 neutral-framing rules. | -| Skill references | `skills/neutral-prompts/references/` | Pattern catalog, worked rewrites, and the decision-record format, loaded on demand. | -| Runtime adapters | `skills/neutral-prompts/agents/` | Gemini CLI command and Codex invocation policy. | -| Skill mirror | `.cursor/skills/neutral-prompts/SKILL.md` | Cursor-compatible copy; keep it byte-identical to the canonical skill. | -| Claude and Codex metadata | `.claude-plugin/`, `.codex-plugin/`, `.agents/plugins/` | Plugin manifests and marketplace metadata. | -| Hooks | `hooks/hooks.json`, `hooks/always-on.mjs` | Opt-in `SessionStart` injection of the ruleset. | -| Other runtimes | `gemini-extension.json`, `qwen-extension.json`, `kimi.plugin.json`, `GEMINI.md`, `.opencode/` | Gemini, Qwen, Kimi, and OpenCode entry points. | -| Tooling | `scripts/scan_prompt.py`, `scripts/run_evals.py` | Directional-wording scanner and its offline measurement harness. | -| Evaluation | `evals/cases.jsonl`, `evals/rubric.md`, `evals/README.md` | Labeled detection cases and the rubric for grading rewrites. | -| Documentation | `README.md`, `INSTALL.md`, `AGENTS.md`, `CONTRIBUTING.md` | Overview, installation, agent map, and contribution workflow. | -| Verification | `tests/` | Unit tests for the scanner, the harness, and repository invariants. | - -## Runtime entry points - -When debugging or changing one integration, begin with its entry point: - -| Runtime | Read first | -| --- | --- | -| Claude Code | `.claude-plugin/plugin.json`, `.claude-plugin/marketplace.json`, `hooks/hooks.json`, `hooks/always-on.mjs` | -| Codex | `.codex-plugin/plugin.json`, `.agents/plugins/marketplace.json`, `skills/neutral-prompts/agents/openai.yaml` | -| Gemini CLI | `gemini-extension.json`, `GEMINI.md`, `skills/neutral-prompts/agents/gemini.toml` | -| Qwen Code, Kimi Code | `qwen-extension.json`, `kimi.plugin.json` | -| OpenCode | `.opencode/command/neutral-prompts.md` (OpenCode also reads `skills/` natively) | -| Cursor, Copilot, Zed, other skills harnesses | `skills/neutral-prompts/SKILL.md`, `.cursor/skills/neutral-prompts/SKILL.md` | - -## Source-of-truth rules - -- Change `skills/neutral-prompts/SKILL.md` first when changing skill behavior, - then synchronize the `.cursor` mirror. -- A scanner rule exists in three places at once: an entry in - `skills/neutral-prompts/references/patterns.md`, a `Rule` in - `scripts/scan_prompt.py`, and at least one labeled case in - `evals/cases.jsonl`. `python3 scripts/run_evals.py scan` fails until the three - agree, and `tests/test_run_evals.py` fails when a rule has no labeled case. -- Treat manifests and hook declarations as runtime contracts. Keep shared - metadata, including versions, aligned across manifest files; - `tests/test_repo_integrity.py` enforces this. -- Keep installation and behavior claims in `README.md` and `INSTALL.md` accurate. - Do not document an install route the repository does not support. -- Do not edit generated dependencies, local caches, or unrelated user files. - -## Writing prompts inside this repository - -Documentation here quotes directional wording on purpose, to diagnose it. When -you add such a quote, label it as an example of the biased form so a reader does -not mistake it for the recommendation. When a phrase in this repository is a real -instruction rather than a quoted example, it follows the rules in `SKILL.md`. - -## Verification - -Run only checks relevant to the change, and report exact commands and results: - -```bash -python3 -m unittest discover -s tests -v -python3 scripts/run_evals.py validate -python3 scripts/run_evals.py scan -claude plugin validate . -``` - -The Python checks need no third-party packages and make no network calls. Before -submitting a change, check the diff for unrelated files and run `git diff --check`. - -Running `scripts/scan_prompt.py` over this repository's own documentation reports -findings by design: `SKILL.md`, `references/patterns.md`, and `references/examples.md` -quote directional wording in order to diagnose it, and rule 4 recommends paired -negations that the line-by-line scanner flags one half at a time. Read each hit -against what it governs rather than treating a clean scan of the docs as the goal. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md deleted file mode 100644 index 825ce20..0000000 --- a/CONTRIBUTING.md +++ /dev/null @@ -1,185 +0,0 @@ -# Contributing - -Thanks for improving **neutral-prompts**. Contributions from humans and coding -agents are welcome. Keep changes understandable, reviewable, safe to run, and -compatible with existing users. - -## Authorship and provenance - -Every pull request must select exactly one category: - -- **Human-authored** — a human made the substantive implementation and text. - Autocomplete, formatting, search, and minor suggestions do not make a - contribution hybrid. -- **Autonomous agent-authored** — an agent made most substantive decisions and - changes, with a human primarily providing the task and reviewing the result. -- **Hybrid** — a human and one or more agents both made substantive decisions or - changes. - -For autonomous-agent or hybrid contributions, disclose the agent or tool and -model/version when known, what it did, what the human reviewed, and any material -limitations or failed checks. Do not call generated work human-authored or -independently verified when it was only reviewed by the same agent that produced -it. - -The submitting human remains accountable for the full diff. Before submission, -read the changed files, remove unrelated generated changes, and verify the claims -in the PR description. - -## Scope and reviewability - -Keep each PR focused. Avoid drive-by formatting, unrelated dependency changes, -generated filler, and broad rewrites that make behavior changes difficult to -inspect. - -The PR description must explain: - -1. what changed and why; -2. observable behavior before and after; -3. safety, compatibility, and side-effect considerations; -4. the exact verification performed. - -Discuss large behavior changes, new integrations, new hooks, and potentially -breaking changes in an issue first. - -## Adding or changing a scanner rule - -A rule exists in three places, and the checks fail until all three agree: - -1. **The catalog** — an entry in - `skills/neutral-prompts/references/patterns.md` with an id, a detection cue, - the mechanism, and a neutral replacement. -2. **The detector** — a `Rule` in `scripts/scan_prompt.py` with the same id, a - severity, a `why` sentence, and a `suggestion`. -3. **The labels** — at least one case in `evals/cases.jsonl` whose `expect` lists - the id, plus a neutral case that the rule must not fire on. - -Then run: - -```bash -python3 scripts/run_evals.py scan -python3 -m unittest discover -s tests -``` - -Severity guidance: **high** for a pattern that protects one action outright; -**medium** for one that biases through cost or vagueness; **advisory** for a -whole-document check that fires on absence and therefore over-reports. - -When a rule fires on text that is correct as written, that is a false positive -worth recording rather than hiding. Add the text as a `kind: "neutral"` case -whose `expect` lists the rule, and explain the limitation in the case's `note`. -`run_evals.py scan` reports these separately as known scanner limitations. - -## Writing rules and examples - -Skill changes must stay focused on prompt framing and decision quality. Two -requirements specific to this repository: - -- **The skill must not become directional itself.** A rule that pushes agents - toward continuing, stopping, changing, preserving, accepting, or rejecting - contradicts the skill's purpose. Rules constrain *how* a decision is framed, - never *which* outcome is chosen. -- **Label every quoted example of the biased form.** Documentation here quotes - directional wording to diagnose it. An unlabeled quote reads as a - recommendation. - -Keep terminology consistent with `SKILL.md`: *open decision*, *settled -constraint*, *admissible outcome*, *decision criteria*, *protected action*, -*decision record*, and the seven outcomes (proceed, stop, modify, preserve, -accept, reject, defer). - -## Safety and side effects - -Contributions must not weaken platform safeguards, override higher-priority -instructions, conceal risky behavior, or encourage inaccurate claims. - -Do not add instructions, examples, fixtures, or tests that tell an agent to: - -- read or transmit credentials, tokens, environment variables, private files, or - repository data; -- modify shell profiles, global Git configuration, editor settings, or unrelated - agent configuration; -- bypass confirmation for destructive, privileged, production, or externally - visible actions; -- silently install software, fetch and execute remote code, or create - persistence. - -Neutral framing does not extend to safety constraints. A prompt that requires -confirmation before a destructive action is stating a settled constraint, and -"balancing" it would be a defect, not a fix. - -Installation, activation, validation, and tests must be narrowly scoped and -predictable. By default, repository code must not modify files outside the -repository or a documented temporary directory, alter user configuration or -credentials, publish or send data, require elevated privileges, perform -irreversible actions, or leave background processes behind. - -## Hooks and scripts - -Hooks run in user environments: keep them fast, bounded, fail-safe, opt-in, and -free of network access. An optional failure must not block agent startup — the -`SessionStart` hook exits 0 on every error path, and -`tests/test_repo_integrity.py` covers that. - -Scripts must validate inputs and paths, avoid shell commands built from untrusted -text, use temporary fixtures, and use least privilege. The scanner and the eval -harness are standard-library Python 3.9+ with no network access and no model -calls; keep them that way. A new third-party dependency needs a clear -justification in the PR. - -## Compatibility and breaking changes - -Preserve existing installation methods, invocation names, file locations, opt-in -behavior, and supported integrations unless a breaking change is explicitly -accepted. Potentially breaking changes include moving the canonical skill, -changing invocation or hook semantics, renaming or removing a scanner rule id, -changing manifests or installation commands, and removing a supported platform. - -A breaking change requires an issue, a migration path, updated documentation, and -a compatibility or deprecation plan. Prefer additive, staged changes. Retire a -scanner rule id rather than reusing it for a different pattern. - -`skills/neutral-prompts/SKILL.md` is canonical. When it changes, synchronize the -Cursor copy: - -```sh -cp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md -cmp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md -``` - -Review platform-specific manifests and documentation whenever shared names, -descriptions, paths, or behavior change. Bump the version in every versioned -manifest together; `tests/test_repo_integrity.py` fails when they drift. - -## Verification - -Run the relevant checks and include the commands and results in the PR: - -```sh -python3 -m unittest discover -s tests -v -python3 scripts/run_evals.py validate -python3 scripts/run_evals.py scan -``` - -For a behavior change to the skill rules, add or update representative cases and -say what the scan reported before and after. For a hook or manifest change, -verify loading in an isolated configuration directory. If a check was not run, -say so and explain why; never invent results or treat inspection as execution. - -## Documentation and checklist - -Keep examples safe to copy: harmless fixtures, explicit placeholders, read-only -previews. Never include real secrets, personal paths, or production identifiers. -Distinguish required behavior from suggestions, and avoid unsupported claims -about platform support or measured effectiveness. - -A PR is ready when: - -- one authorship category is selected and agent involvement is disclosed - accurately; -- the full diff has been reviewed by a human contributor; -- the change is focused and free of unrelated generated edits; -- side effects, compatibility, and verification are documented. - -Maintainers may close PRs that conceal provenance, introduce unsafe behavior, -lack verification, or make unplanned breaking changes. diff --git a/GEMINI.md b/GEMINI.md deleted file mode 100644 index a93bbd4..0000000 --- a/GEMINI.md +++ /dev/null @@ -1,9 +0,0 @@ -# neutral-prompts - -Apply the rules imported below to every prompt you write, review, or rewrite: -separate settled constraints from open decisions, name every admissible outcome, -replace protected actions with observable criteria, keep the intensity symmetric -across outcomes, and require the decision record rather than a particular -conclusion. - -@./skills/neutral-prompts/SKILL.md diff --git a/INSTALL.md b/INSTALL.md deleted file mode 100644 index dbdbd81..0000000 --- a/INSTALL.md +++ /dev/null @@ -1,595 +0,0 @@ -# How to install - -
-Claude Code - -### Install - -```bash -claude plugin marketplace add skyRolly/neutral-prompt -claude plugin install neutral-prompts@neutral-prompts -``` - -Type `/neutral-prompts`. - -### Verify - -```bash -claude plugin list -``` - -### Update - -```bash -claude plugin marketplace update neutral-prompts -``` - -### Uninstall - -```bash -claude plugin uninstall neutral-prompts -claude plugin marketplace remove neutral-prompts -``` - -Or keep it installed and turn it off: `claude plugin disable neutral-prompts`. - -### Always-on (optional) - -A `SessionStart` hook loads the full ruleset at the start of every session, no -`/neutral-prompts` needed: - -```bash -touch ~/.claude/.neutral-prompts-always -``` - -If you use a custom Claude configuration directory, create the flag there instead: - -```bash -touch "$CLAUDE_CONFIG_DIR/.neutral-prompts-always" -``` - -Back to on-demand: - -```bash -rm ~/.claude/.neutral-prompts-always -``` - -The hook only fires when the flag file exists, so installing the plugin changes -nothing by itself. "stop neutral mode" still turns it off for the current session. - -
- -
-Codex - -### Install - -```bash -codex plugin marketplace add skyRolly/neutral-prompt --ref main -codex plugin add neutral-prompts@neutral-prompts -``` - -Invoke the skill explicitly by typing `$neutral-prompts`. Codex will not activate -it automatically: `skills/neutral-prompts/agents/openai.yaml` sets -`policy.allow_implicit_invocation: false`. - -### Verify - -```bash -codex plugin list -``` - -### Update - -```bash -codex plugin marketplace upgrade neutral-prompts -codex plugin remove neutral-prompts -codex plugin add neutral-prompts@neutral-prompts -``` - -### Uninstall - -```bash -codex plugin remove neutral-prompts -codex plugin marketplace remove neutral-prompts -``` - -### Always-on (optional) - -Add to `~/.codex/AGENTS.md`: - -```markdown -## Prompt framing - -Apply these rules to every prompt you write, review, or rewrite. They govern open -decisions — the choices a prompt delegates — not settled constraints such as -safety rules, output contracts, budgets, or coding standards. - -1. Separate settled constraints from open decisions; write constraints as imperatives. -2. Name the decision, not the answer. -3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. -4. Balance the negations — ruling out one default installs its opposite. -5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. -7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. -8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". -9. Require the decision record, not a particular decision. -10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. - -Directive wording is correct when the author has already decided, when safety or -legality is at stake, when an external contract dictates the outcome, when an -outcome is genuinely inadmissible and named as such, or when the decision is small -enough that one balanced sentence carries the whole frame. -``` - -
- -
-Gemini CLI - -Gemini CLI has no plugin marketplace, so there are two native routes: a **custom -command** (opt-in, off until you invoke it) or an **extension** (always-on once -installed). The command route matches this skill's default posture; pick it unless -you want the rules on every session. - -### Install (command, opt-in) - -```bash -mkdir -p ~/.gemini/commands -curl -fsSL https://raw.githubusercontent.com/skyRolly/neutral-prompt/main/skills/neutral-prompts/agents/gemini.toml \ - -o ~/.gemini/commands/neutral-prompts.toml -``` - -Start a new session, type `/neutral-prompts`. It stays on for that session. - -### Install (extension, always-on) - -```bash -gemini extensions install https://github.com/skyRolly/neutral-prompt -``` - -The extension loads `GEMINI.md`, which imports the full skill, so the rules apply -from message one. `git` must be installed. - -### Verify - -```bash -gemini extensions list # extension route -ls ~/.gemini/commands # command route: neutral-prompts.toml present -``` - -Or type `/` in a session and confirm `neutral-prompts` is listed. - -### Update - -```bash -gemini extensions update neutral-prompts # extension route -# command route: re-run the curl above -``` - -### Uninstall - -```bash -gemini extensions uninstall neutral-prompts # extension route -rm ~/.gemini/commands/neutral-prompts.toml # command route -``` - -
- -
-GitHub Copilot (VS Code and Copilot CLI) - -Copilot reads Agent Skills natively: the same `SKILL.md`, no conversion. It scans -`.github/skills/`, `.claude/skills/`, and `.agents/skills/` in the project, and -`~/.copilot/skills/`, `~/.claude/skills/`, and `~/.agents/skills/` globally. - -### Install - -```bash -npx skills add skyRolly/neutral-prompt -a github-copilot # this project -npx skills add skyRolly/neutral-prompt -a github-copilot -g # all projects -``` - -Without the CLI, copy the skill folder into any directory Copilot scans: - -```bash -git clone https://github.com/skyRolly/neutral-prompt -mkdir -p ~/.copilot/skills -cp -R neutral-prompt/skills/neutral-prompts ~/.copilot/skills/ -``` - -### Verify - -Type `/` in the chat input and confirm `neutral-prompts` appears. Or: - -```bash -npx skills list -npx skills ls -g # if installed globally -``` - -### Update - -```bash -npx skills update neutral-prompts -``` - -Or re-copy the folder after `git pull`. - -### Uninstall - -```bash -npx skills remove neutral-prompts -``` - -### Always-on (optional) - -Add the block below to `.github/copilot-instructions.md` in the project: - -```markdown -## Prompt framing - -Apply these rules to every prompt you write, review, or rewrite. They govern open -decisions — the choices a prompt delegates — not settled constraints such as -safety rules, output contracts, budgets, or coding standards. - -1. Separate settled constraints from open decisions; write constraints as imperatives. -2. Name the decision, not the answer. -3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. -4. Balance the negations — ruling out one default installs its opposite. -5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. -7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. -8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". -9. Require the decision record, not a particular decision. -10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. - -Directive wording is correct when the author has already decided, when safety or -legality is at stake, when an external contract dictates the outcome, when an -outcome is genuinely inadmissible and named as such, or when the decision is small -enough that one balanced sentence carries the whole frame. -``` - -
- -
-Kimi Code CLI - -### Install - -Start a Kimi Code session, then: - -1. Run `/plugins`. -2. Choose **Custom**. -3. Paste `https://github.com/skyRolly/neutral-prompt` and press `Enter`. -4. Choose **Trust and install**. - -Use the slash command `/skill:neutral-prompts` to invoke the skill explicitly. - -### Update - -`/plugins` in a Kimi Code session, cursor to **Neutral Prompts**, press `R`. - -### Uninstall - -`/plugins` in a Kimi Code session, cursor to **Neutral Prompts**, press `D`. - -
- -
-OpenCode - -OpenCode reads `skills/` natively, so the skill works from any directory OpenCode -scans. The repository also ships `.opencode/command/neutral-prompts.md`, which adds -a `/neutral-prompts` command to a project that has it. - -### Install - -```bash -npx skills add skyRolly/neutral-prompt -a opencode -y -``` - -Or copy both pieces by hand: - -```bash -git clone https://github.com/skyRolly/neutral-prompt -mkdir -p ~/.config/opencode/skills ~/.config/opencode/command -cp -R neutral-prompt/skills/neutral-prompts ~/.config/opencode/skills/ -cp neutral-prompt/.opencode/command/neutral-prompts.md ~/.config/opencode/command/ -``` - -### Verify - -Start OpenCode, type `/`, and confirm `neutral-prompts` appears. - -### Update - -```bash -npx skills update neutral-prompts -``` - -Or re-copy after `git pull`. - -### Uninstall - -Delete `neutral-prompts` from the skills directory it landed in, and remove -`command/neutral-prompts.md`. - -### Always-on (optional) - -Add to `~/.config/opencode/AGENTS.md`: - -```markdown -## Prompt framing - -Apply these rules to every prompt you write, review, or rewrite. They govern open -decisions — the choices a prompt delegates — not settled constraints such as -safety rules, output contracts, budgets, or coding standards. - -1. Separate settled constraints from open decisions; write constraints as imperatives. -2. Name the decision, not the answer. -3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. -4. Balance the negations — ruling out one default installs its opposite. -5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. -7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. -8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". -9. Require the decision record, not a particular decision. -10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. - -Directive wording is correct when the author has already decided, when safety or -legality is at stake, when an external contract dictates the outcome, when an -outcome is genuinely inadmissible and named as such, or when the decision is small -enough that one balanced sentence carries the whole frame. -``` - -
- -
-Qwen Code - -### Install - -```bash -qwen extensions install skyRolly/neutral-prompt -``` - -Qwen Code supports the GitHub shorthand and installs the repository as a native -extension. The extension discovers the skill under `skills/`. - -Type `/neutral-prompts` to invoke the skill explicitly. Installing the extension -does not change behavior until the skill is invoked. - -### Verify - -```bash -qwen extensions list -``` - -Then start a new Qwen Code session and run: - -```text -/skills -``` - -Confirm that `neutral-prompts` appears in the list. - -### Update - -```bash -qwen extensions update neutral-prompts -``` - -### Uninstall - -```bash -qwen extensions uninstall neutral-prompts -``` - -
- -
-Zed - -Zed's Agent reads Agent Skills natively: the same `SKILL.md`, no conversion. - -### Install - -In the Agent Panel, open the Skills manager and choose **Create skill from URL** -(also in the command palette as `agent: create skill from url`), then paste: - -``` -https://github.com/skyRolly/neutral-prompt/blob/main/skills/neutral-prompts/SKILL.md -``` - -Save it in **User** scope for every project, or **Project** scope for one. Then -type `/neutral-prompts` in the Agent Panel. - -Prefer the filesystem? Clone the repo and drop the skill folder into your user -skills directory: - -```bash -git clone https://github.com/skyRolly/neutral-prompt -cp -R neutral-prompt/skills/neutral-prompts ~/.config/zed/skills/ -``` - -### Verify - -Open the Skills manager in the Agent Panel and confirm `neutral-prompts` is -listed. Or type `/` and confirm it appears. - -### Update - -Re-import from the same URL (overwrites), or re-copy the folder after `git pull`. - -### Uninstall - -Remove `neutral-prompts` from the Skills manager, or delete -`~/.config/zed/skills/neutral-prompts`. - -### Always-on (optional) - -Add to your personal `~/.config/zed/AGENTS.md`: - -```markdown -## Prompt framing - -Apply these rules to every prompt you write, review, or rewrite. They govern open -decisions — the choices a prompt delegates — not settled constraints such as -safety rules, output contracts, budgets, or coding standards. - -1. Separate settled constraints from open decisions; write constraints as imperatives. -2. Name the decision, not the answer. -3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. -4. Balance the negations — ruling out one default installs its opposite. -5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. -7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. -8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". -9. Require the decision record, not a particular decision. -10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. - -Directive wording is correct when the author has already decided, when safety or -legality is at stake, when an external contract dictates the outcome, when an -outcome is genuinely inadmissible and named as such, or when the decision is small -enough that one balanced sentence carries the whole frame. -``` - -
- -
-Cursor, Amp, and any other agent-skills harness - -Works with any harness that reads agent skills. Swap `-a ` for yours. - -### Install - -```bash -npx skills add skyRolly/neutral-prompt # this workspace -npx skills add skyRolly/neutral-prompt -g # all projects -npx skills add skyRolly/neutral-prompt -a cursor -y # one agent only -``` - -New agent chat, type `/neutral-prompts`. - -Without the CLI, copy the skill folder into whatever path your agent scans: - -```bash -git clone https://github.com/skyRolly/neutral-prompt -mkdir -p ~/.cursor/skills # Cursor. Use .agents/skills for OpenCode, or your agent's own path -cp -R neutral-prompt/skills/neutral-prompts ~/.cursor/skills/ -``` - -### Verify - -```bash -npx skills list -npx skills ls -g # if installed globally -``` - -### Update - -```bash -npx skills update neutral-prompts -npx skills update -g # if installed globally -``` - -### Uninstall - -```bash -npx skills remove neutral-prompts -npx skills remove neutral-prompts -g # if installed globally -``` - -### Always-on (optional) - -Paste this into your agent's persistent rules file. Cursor: **Settings → Rules → -User Rules**, or a project rule under `.cursor/rules/` with `alwaysApply: true`. - -```markdown -## Prompt framing - -Apply these rules to every prompt you write, review, or rewrite. They govern open -decisions — the choices a prompt delegates — not settled constraints such as -safety rules, output contracts, budgets, or coding standards. - -1. Separate settled constraints from open decisions; write constraints as imperatives. -2. Name the decision, not the answer. -3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. -4. Balance the negations — ruling out one default installs its opposite. -5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. -7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. -8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". -9. Require the decision record, not a particular decision. -10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. - -Directive wording is correct when the author has already decided, when safety or -legality is at stake, when an external contract dictates the outcome, when an -outcome is genuinely inadmissible and named as such, or when the decision is small -enough that one balanced sentence carries the whole frame. -``` - -
- -## The scanner - -The scanner and eval harness are repository scripts, not part of the installed -skill. To use them, clone the repository and run them against your prompt files: - -```bash -git clone https://github.com/skyRolly/neutral-prompt -cd neutral-prompt -python3 scripts/scan_prompt.py path/to/prompt.md -``` - -Python 3.9 or newer, no third-party packages. See [evals/README.md](evals/README.md) -for the measured detection coverage. - -## How activation works - -1. **Installed, not invoked.** In Claude Code, Qwen Code, and Codex, nothing - happens until you invoke the skill explicitly. Claude Code and Qwen Code honor - `disable-model-invocation: true` in `SKILL.md`; Codex honors - `policy.allow_implicit_invocation: false` in `agents/openai.yaml`. Other - harnesses may load every skill's description at startup and activate the skill - themselves. -2. **You invoke it explicitly.** Type `/neutral-prompts` in Claude Code or Qwen - Code, or `$neutral-prompts` in Codex. The rules stay on for that session. - "stop neutral mode" or "normal mode" turns them off. -3. **You touch `~/.claude/.neutral-prompts-always`** (Claude Code). A - `SessionStart` hook loads the full ruleset from message one, every session. -4. **You add the always-on snippet above** (other harnesses). Keeps the core rules - in your agent's persistent context. - -In Claude Code, Qwen Code, and Codex there is no middle ground: if you did not -turn it on, it is off. - -## Troubleshooting - -**`/neutral-prompts` not in autocomplete.** Restart the agent. The plugin index is -read at startup. - -**Always-on flag has no effect.** Update the plugin -(`claude plugin marketplace update neutral-prompts`) and restart. Hooks are read -at startup, and the flag needs the plugin version that ships `hooks/hooks.json`. - -**`claude plugin marketplace add` fails.** Use the `owner/repo` form. A local path -must point at the repo root, not `.claude-plugin/`. - -**Installed, but prompts still read as directional.** Open a new session. If it -still drifts, run `python3 scripts/scan_prompt.py` over the prompt to see which -rule it trips, then tighten the wording in `skills/neutral-prompts/SKILL.md`. - -**The scanner flags a constraint that is correct as written.** That is expected — -it matches phrasings, not intent. Suppress it with `neutral-prompts: allow NP003` -on the line or the line above, and say in the surrounding text why the phrase is a -settled constraint. - -**Skill missing after `npx skills add`.** Start a new agent chat. Skills are -indexed at session start. Confirm the folder landed where your agent scans -(`~/.cursor/skills/` for Cursor, `.agents/skills/` for OpenCode) and that the -frontmatter `name` matches the folder name. - -**The repository is `neutral-prompt`; the skill is `neutral-prompts`.** Install -commands that name a repository take `skyRolly/neutral-prompt`; commands that name -the plugin or skill take `neutral-prompts`. diff --git a/LICENSE b/LICENSE deleted file mode 100644 index e805c43..0000000 --- a/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2026 skyRolly - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/README.md b/README.md deleted file mode 100644 index d1d94a1..0000000 --- a/README.md +++ /dev/null @@ -1,128 +0,0 @@ -

- Prompts that frame a decision instead of pre-answering it. -

-

- License -

- -## Install - -Copy/paste into your CLI prompt: - -```text -Install the neutral-prompts skill/plugin from https://github.com/skyRolly/neutral-prompt, refer to the repo's AGENTS.md for instructions. -``` - -Or 🔗 [check the installation instructions](INSTALL.md). - -## What it does - -A skill for your coding assistant that stops prompts from carrying the answer. -Every prompt that hands a decision to an agent also hands it a prior — "do not -stop them" is read as *continue*, "avoid unnecessary changes" as *prefer the -existing code*. This skill finds that wording, explains why it biases, and -rewrites it as a decision the evidence settles. - -It does not push agents toward continuing, stopping, changing, preserving, -accepting, or rejecting. It pushes them toward deciding well. - -## What changes - - - - - - - -
- -## Before - -> You have several workflow agents still running. Do NOT immediately stop them. -> Check what they are doing first. - -The negation is the only instruction with force. "Check what they are doing" has -no criterion attached, so the check produces a description rather than a -decision — and stopping now requires overriding the prompt. - - - -## After - -> Four workflow agents are still running. Decide the disposition of each one. -> -> Do not assume they should be stopped. Do not assume they should continue. -> -> 1. **Gather** — read each agent's recent outputs and remaining scope. -> 2. **Evaluate** — compare that scope against findings already in the report. -> 3. **Choose** — continue · consume results and stop · stop as redundant · defer. -> 4. **Record** — evidence, alternatives, action, reasoning. -> -> Continue when the remaining steps target a file no completed agent has -> reported on. Stop when the last three outputs restate landed findings. - -
- -## The rules - -10 rules. Full text in [SKILL.md](./skills/neutral-prompts/SKILL.md). - -1. Separate settled constraints from open decisions. -2. Name the decision, not the answer. -3. List every admissible outcome. -4. Balance the negations. -5. Replace protected actions with decision criteria. -6. Order the prompt: evidence, alternatives, choice, record. -7. Keep the intensity symmetric. -8. Make the thresholds observable. -9. Require the decision record, not the decision. -10. Do not manufacture balance. - -They govern **open decisions** — the choices a prompt delegates. They do not -govern settled constraints: safety rules, output contracts, budgets, standards. -Directive wording is the correct form for a constraint. - -## The scanner - -A prompt linter ships with the repo. It flags the twelve patterns catalogued in -[`references/patterns.md`](./skills/neutral-prompts/references/patterns.md): - -```console -$ python3 scripts/scan_prompt.py my-prompt.md -my-prompt.md:3:1 NP001 high 'Do NOT immediately stop' - why: A negated stop verb protects continuation: the complement becomes the default and needs no justification. - try: Name the decision instead: 'Evaluate each one. Continuing and stopping are both admissible outcomes.' -1 finding(s): 1 high, 0 medium, 0 advisory -``` - -It finds phrasings, not intent. A hit on a settled constraint is correct as -written — confirm what each phrase governs, then suppress the ones that are fine -with `neutral-prompts: allow NP001` on the line or the line above. - -Measure it against the labeled cases in [`evals/`](./evals/README.md): - -```bash -python3 scripts/run_evals.py scan -``` - -## Tune it - -Fork, edit `skills/neutral-prompts/SKILL.md`, then swap your copy in: - -```bash -claude plugin uninstall neutral-prompts # drop the upstream copy first: -claude plugin marketplace remove neutral-prompts # fork and upstream share both names -claude plugin marketplace add /neutral-prompt -claude plugin install neutral-prompts@neutral-prompts -``` - -Restart Claude Code, then re-invoke `/neutral-prompts`. - -Adding a rule to the scanner means adding an entry to `references/patterns.md`, a -`Rule` in `scripts/scan_prompt.py`, and a labeled case in `evals/cases.jsonl` — -`python3 scripts/run_evals.py scan` fails until all three agree. See -[CONTRIBUTING.md](CONTRIBUTING.md). - -## License - -MIT. diff --git a/evals/README.md b/evals/README.md deleted file mode 100644 index 35143fd..0000000 --- a/evals/README.md +++ /dev/null @@ -1,103 +0,0 @@ -# Evaluations - -Two layers, measuring two different things. - -**Layer 1 — the detector.** `scripts/scan_prompt.py` is measured against the -labeled cases in `cases.jsonl`. This layer runs offline, calls no model, costs -nothing, and gates every change to a scanner rule. - -**Layer 2 — the rewrites.** `rubric.md` grades what an agent produces when it -reviews and rewrites a prompt. This layer needs a model runner, which this -repository does not ship: supply your own and record the conditions with any -published numbers. - -## Layer 1: detector coverage - -```bash -python3 scripts/run_evals.py validate # schema-check the case file -python3 scripts/run_evals.py scan # run the scanner and compare to the labels -python3 scripts/run_evals.py scan --json -``` - -`scan` exits 0 only when every case produces exactly the rule ids its label -names — extra findings fail as loudly as missing ones, so a rule that widens into -a false positive is caught by the same run that checks its recall. - -### Case format - -One JSON object per line: - -```json -{"id":"lifecycle-protected-stop","category":"lifecycle","kind":"directional","text":"...","expect":["NP001","NP012"],"note":"why this label is what it is"} -``` - -| Field | Meaning | -| --- | --- | -| `id` | Unique, kebab-case, prefixed with its category. | -| `category` | `lifecycle`, `review`, `investigation`, `accept-reject`, or `general`. | -| `kind` | `directional` — the text is biased. `neutral` — the text is correct as written. | -| `text` | The prompt fragment handed to the scanner, verbatim. | -| `expect` | The exact set of rule ids the scanner must report. | -| `note` | Why the label is what it is. Required; a label without a reason cannot be reviewed. | - -A `neutral` case with a non-empty `expect` is a **known scanner limitation**: text -that is correct as written but still matches a pattern. These are recorded rather -than hidden, and `scan` lists them separately. - -### Current state - -Measured with the shipped case file on Python 3.11: - -| | | -|---|---| -| Cases | 26 (18 directional, 8 neutral) | -| Categories | lifecycle 6, review 8, investigation 5, accept-reject 4, general 3 | -| Rules with at least one labeled case | 12 of 12 | -| Cases matching their label | 26 of 26 | -| Known scanner limitations | 2 | - -The two known limitations are both cases where the phrase is a settled constraint -or is balanced by a neighbouring sentence: - -- `lifecycle-paired-negations` — the negation is paired in the following - sentence, which the line-by-line scanner cannot weigh. Flags NP001. -- `review-scope-constraint` — a directory boundary is a settled constraint, not a - lean. Flags NP003. - -Both are the expected shape of the tool's limits: it matches phrasings, not -intent. Suppress a confirmed false positive with `neutral-prompts: allow NP003` -on the line or the line above. - -### What this number is not - -Detector coverage measures the scanner against text somebody hand-labeled. It -does not measure how often directional wording appears in real prompts, and it -does not measure whether an agent writes better prompts after loading the skill. -Three of the failures the skill targets — missing outcomes, asymmetric intensity, -and ordering — have no reliable textual cue and are absent from this layer -entirely. Layer 2 is where those are graded. - -## Layer 2: rewrite quality - -`rubric.md` defines the dimensions and the release gate for grading an agent's -prompt review and rewrite. Running it needs three things this repository does not -provide: a model runner, a budget, and a set of task prompts. - -Keep the comparison honest: - -- **Isolate the call from your own agent configuration.** User-level plugins, - hooks, memory, and output styles otherwise leak into every condition. The - sharpest case is this repository's own always-on flag - (`~/.claude/.neutral-prompts-always`), which would inject the full ruleset into - the *baseline* condition and make the comparison measure the skill against - itself. -- **Pin the model explicitly.** Isolation drops saved model and effort settings, - so an unpinned run silently uses whatever the operator or the CLI release - defaults to. The pinned model is part of the result. -- **Blind the grader.** Relabel conditions before grading and permute the label - order, so position carries no signal. -- **Hold the cases fixed.** Do not compare conditions produced with different - cases, models, trial counts, or rubrics. - -Record the runner, the CLI version, the model, the case count, the trial count, -the rubric revision, and the reported cost alongside any numbers you publish. diff --git a/evals/cases.jsonl b/evals/cases.jsonl deleted file mode 100644 index 2f26f1e..0000000 --- a/evals/cases.jsonl +++ /dev/null @@ -1,26 +0,0 @@ -{"id": "lifecycle-protected-stop", "category": "lifecycle", "kind": "directional", "text": "You have several workflow agents still running. Do NOT immediately stop them.", "expect": ["NP001", "NP012"], "note": "The originating failure: a negation that installs continuation as the default and states no criteria."} -{"id": "lifecycle-keep-alive", "category": "lifecycle", "kind": "directional", "text": "Keep the sub-agents alive until the investigation is complete.", "expect": ["NP002"], "note": "Two protected-continuation phrasings in one sentence; the scanner reports the rule once per match span."} -{"id": "lifecycle-only-if-necessary", "category": "lifecycle", "kind": "directional", "text": "Only stop a background task if absolutely necessary.", "expect": ["NP007"], "note": "Asymmetric burden: stopping needs necessity, continuing needs nothing."} -{"id": "lifecycle-premature", "category": "lifecycle", "kind": "directional", "text": "Avoid stopping the agents prematurely; they may still be producing something.", "expect": ["NP001", "NP008", "NP012"], "note": "A protected action plus a label that names one direction as the error mode."} -{"id": "lifecycle-neutral-frame", "category": "lifecycle", "kind": "neutral", "text": "For each running agent, decide: continue, consume results and stop, stop as redundant, or defer. Continue when its remaining steps target a file no completed agent has reported on. Stop when its last three outputs restate findings already in the report.", "expect": [], "note": "Both directions reachable, each with an observable criterion."} -{"id": "lifecycle-paired-negations", "category": "lifecycle", "kind": "neutral", "text": "Do not stop the workflow by default. Do not continue it by default. Both are outcomes of the evaluation below, decided on the evidence each agent has produced.", "expect": ["NP001"], "note": "Known false positive: the negation is paired in the next sentence, which the scanner reads line by line and cannot weigh. Confirm the pairing by reading, then suppress."} -{"id": "review-unnecessary-changes", "category": "review", "kind": "directional", "text": "Do not make unnecessary changes to the file.", "expect": ["NP003"], "note": "'Unnecessary' is decided after the fact, so it applies in advance to everything."} -{"id": "review-must-fix-all", "category": "review", "kind": "directional", "text": "Make sure to fix every issue the reviewer raised before you push.", "expect": ["NP004", "NP012"], "note": "Converts a scoping decision into an obligation."} -{"id": "review-never-reject", "category": "review", "kind": "directional", "text": "Never reject a reviewer's suggestion; always apply what they ask for.", "expect": ["NP005", "NP012"], "note": "Replaces evaluation with authorship, in both halves of the sentence."} -{"id": "review-ignore-bots", "category": "review", "kind": "directional", "text": "Ignore automated review bots. Do not accept findings from a linter.", "expect": ["NP006", "NP012"], "note": "The mirror failure: hardening against bad findings hardens against good ones too."} -{"id": "review-leave-as-is", "category": "review", "kind": "directional", "text": "Leave the existing helper as-is and keep the diff minimal.", "expect": ["NP003"], "note": "Existing code treated as evidence for itself."} -{"id": "review-required-conclusion", "category": "review", "kind": "directional", "text": "Explain why you kept the original implementation.", "expect": ["NP011"], "note": "The reasoning request accepts exactly one outcome."} -{"id": "review-neutral-frame", "category": "review", "kind": "neutral", "text": "For each finding, choose one outcome: fix now, modify the surrounding design, preserve current behavior, reject with counter-evidence, or defer with a tracked follow-up. Decide on the evidence: whether it reproduces, which requirement is at stake, and the blast radius of a change.", "expect": [], "note": "Five named outcomes, all reachable, selected by evidence."} -{"id": "review-scope-constraint", "category": "review", "kind": "neutral", "text": "Do not change files outside src/auth/. Within that directory, decide each finding on its merits and report the evidence behind each disposition.", "expect": ["NP003"], "note": "Known false positive: a directory boundary is a settled constraint, correct as written. Suppress after confirming."} -{"id": "investigation-be-thorough", "category": "investigation", "kind": "directional", "text": "Be thorough and check everything that could be related to this bug.", "expect": ["NP009"], "note": "Prices the next step at zero, so more work always wins."} -{"id": "investigation-until-root-cause", "category": "investigation", "kind": "directional", "text": "Keep digging until you find the root cause.", "expect": ["NP002", "NP009"], "note": "Stopping is conditional on an outcome that may not be reachable."} -{"id": "investigation-vague-threshold", "category": "investigation", "kind": "directional", "text": "Stop investigating once you have gathered sufficient evidence and the findings look significant.", "expect": ["NP010"], "note": "Two thresholds the agent cannot check against the workspace."} -{"id": "investigation-no-stone", "category": "investigation", "kind": "directional", "text": "Leave no stone unturned; explore every possibility before reporting.", "expect": ["NP009"], "note": "Unbounded effort with no cost side to the comparison."} -{"id": "investigation-neutral-frame", "category": "investigation", "kind": "neutral", "text": "Before each step, name the question it answers and what result would change your conclusion. Take the step when the expected information gain exceeds its cost. Outcomes: continue this line, switch lines, stop and report, or defer pending an input you cannot obtain.", "expect": [], "note": "Depth becomes a per-step decision with a stated cost side."} -{"id": "accept-confirm-approach", "category": "accept-reject", "kind": "directional", "text": "Confirm that this is the right approach and explain why it will work.", "expect": ["NP011"], "note": "Presupposes the conclusion; the evaluation has no way to fail."} -{"id": "accept-defer-to-architect", "category": "accept-reject", "kind": "directional", "text": "Defer to the architect on any design question and do not push back.", "expect": ["NP005", "NP012"], "note": "Authorship offered as evidence, twice."} -{"id": "accept-err-on-side", "category": "accept-reject", "kind": "directional", "text": "When in doubt, err on the side of accepting the proposal.", "expect": ["NP008"], "note": "Names a direction without naming the cost of erring that way."} -{"id": "accept-neutral-frame", "category": "accept-reject", "kind": "neutral", "text": "Evaluate the proposal against docs/queue-requirements.md. Outcomes: accept, accept with modification, reject with reasoning, or defer for a named missing input. Whose proposal it is carries no weight.", "expect": [], "note": "Criteria external to the proposal; authorship explicitly discounted."} -{"id": "general-at-all-costs", "category": "general", "kind": "directional", "text": "Complete the task at all costs and under no circumstances abandon the run.", "expect": ["NP007"], "note": "Two intensifiers, both on one branch."} -{"id": "general-safety-constraint", "category": "general", "kind": "neutral", "text": "Confirm with the user before running any destructive command. This is a constraint, not a decision to evaluate.", "expect": [], "note": "A settled safety constraint. Directive wording is the correct form here."} -{"id": "general-output-contract", "category": "general", "kind": "neutral", "text": "Return one JSON object per finding with the fields id, evidence, outcome, and reasoning. Choose the outcome from the list above based on the evidence you gathered.", "expect": [], "note": "An output contract plus a neutral decision instruction."} diff --git a/evals/rubric.md b/evals/rubric.md deleted file mode 100644 index 94e3869..0000000 --- a/evals/rubric.md +++ /dev/null @@ -1,50 +0,0 @@ -# Rewrite quality rubric - -Grades what an agent produces when it is asked to review a prompt and rewrite it. -Used by layer 2 in [README.md](README.md), which needs an operator-supplied model -runner. - - - - -Judge responses blind: label them `A`, `B`, or `C` without exposing the condition -name. Score each dimension from 1 (fails) to 5 (excellent). - -| Dimension | Weight | What to measure | -| --- | ---: | --- | -| Detection | 25% | Directional wording is found and quoted exactly. Missed instances and invented ones both cost. | -| Explanation | 15% | The mechanism is named — which outcome the wording protects and how — rather than the phrase being labeled "biased". | -| Neutrality of the rewrite | 25% | The rewrite leaves every admissible outcome reachable. It does not swap one lean for the opposite. | -| Criteria quality | 20% | The replacement criteria are observable in the workspace: a count, a file, a test, a budget, a named condition. | -| Fidelity | 15% | The author's settled constraints, task, and scope survive the rewrite unchanged. | - -Score against the response as delivered, not against what it could have meant. - -Mark `blocker: true` for any of: - -- a rewrite that reverses the bias instead of removing it (the new text protects - the opposite outcome); -- a settled safety constraint "balanced" into an open decision; -- a quoted finding that does not appear in the source text; -- a rewrite that drops a requirement, a scope limit, or a budget the original - stated; -- a rewrite that removes evidence the original supplied, in the name of - neutrality. - - - -## Release gate - -Release the candidate only when: - -1. It has no blocking findings. -2. Neutrality of the rewrite and Fidelity are each within 0.1 points of baseline - or better. -3. Its weighted score is higher than baseline. -4. Any published comparison uses the same cases, models, trials, and rubric. - -Rule 2 is the one that matters most for this skill. A candidate that finds more -directional wording while rewriting less faithfully has not improved the prompt; -it has moved the bias. diff --git a/gemini-extension.json b/gemini-extension.json deleted file mode 100644 index 32473f3..0000000 --- a/gemini-extension.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "neutral-prompts", - "version": "0.1.0", - "description": "Neutral decision framing for Gemini CLI prompts: name every admissible outcome, state observable criteria, and require a decision record.", - "contextFileName": "GEMINI.md" -} diff --git a/hooks/always-on.mjs b/hooks/always-on.mjs deleted file mode 100644 index 929085a..0000000 --- a/hooks/always-on.mjs +++ /dev/null @@ -1,41 +0,0 @@ -// SessionStart hook: injects the full neutral-prompts ruleset when the user has -// opted in by creating $CLAUDE_CONFIG_DIR/.neutral-prompts-always (default ~/.claude). -// Never blocks session start: any failure exits 0. -// -// Runs under Node so it works on macOS, Linux, and Windows. The hook declaration -// in hooks.json launches this module from the plugin-root environment variable -// rather than relying on platform-specific shell expansion for the script path. - -import fs from "node:fs"; -import os from "node:os"; -import path from "node:path"; -import { fileURLToPath } from "node:url"; - -try { - const claudeDir = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude"); - const flagPath = path.join(claudeDir, ".neutral-prompts-always"); - - // Only fire when the user has opted in. - if (!fs.existsSync(flagPath)) process.exit(0); - - // Resolve SKILL.md relative to this script's own location, not a trusted env var. - const scriptDir = path.dirname(fileURLToPath(import.meta.url)); - const skillPath = path.join(scriptDir, "..", "skills", "neutral-prompts", "SKILL.md"); - if (!fs.existsSync(skillPath)) process.exit(0); - - // Strip a leading YAML frontmatter block (--- ... --- at the very top of file). - // An unterminated fence is not frontmatter, so the whole file is kept. - const body = fs - .readFileSync(skillPath, "utf8") - .replace(/^---[^\S\r\n]*\r?\n[\s\S]*?\r?\n---[^\S\r\n]*(?:\r?\n|$)/, "") - .replace(/(?:\r?\n)+$/, ""); - - process.stdout.write( - "NEUTRAL PROMPT MODE ACTIVE (always-on). The rules below apply to every prompt " + - 'you write, review, or rewrite this session. "stop neutral mode" turns them off ' + - `for this session; delete ${flagPath} to turn always-on off for good.\n\n${body}\n`, - ); -} catch { - // Never block session start. - process.exit(0); -} diff --git a/hooks/hooks.json b/hooks/hooks.json deleted file mode 100644 index 7dddfe7..0000000 --- a/hooks/hooks.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "hooks": { - "SessionStart": [ - { - "matcher": "startup|resume|clear|compact", - "hooks": [ - { - "type": "command", - "command": "node -e \"(async()=>{const root=process.env.CLAUDE_PLUGIN_ROOT||process.env.PLUGIN_ROOT;if(root)await import(require('node:url').pathToFileURL(require('node:path').join(root,'hooks','always-on.mjs')).href)})().catch(()=>{})\"", - "timeout": 30, - "statusMessage": "Checking neutral-prompts always-on flag..." - } - ] - } - ] - } -} diff --git a/kimi.plugin.json b/kimi.plugin.json deleted file mode 100644 index 58a4e2b..0000000 --- a/kimi.plugin.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "name": "neutral-prompts", - "version": "0.1.0", - "description": "Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, and require a decision record.", - "license": "MIT", - "homepage": "https://github.com/skyRolly/neutral-prompt", - "skills": "./skills/", - "interface": { - "displayName": "Neutral Prompts", - "shortDescription": "Neutral decision framing for Kimi Code CLI prompts" - } -} diff --git a/qwen-extension.json b/qwen-extension.json deleted file mode 100644 index 59445df..0000000 --- a/qwen-extension.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "name": "neutral-prompts", - "version": "0.1.0", - "description": "Neutral decision framing for Qwen Code prompts: name every admissible outcome, state observable criteria, and require a decision record.", - "skills": "skills" -} diff --git a/scripts/run_evals.py b/scripts/run_evals.py deleted file mode 100755 index 3767da1..0000000 --- a/scripts/run_evals.py +++ /dev/null @@ -1,212 +0,0 @@ -#!/usr/bin/env python3 -"""Measure scan_prompt.py against the labeled cases in evals/cases.jsonl. - -Every case carries the exact set of rule ids the scanner is expected to report. -`validate` checks the case file; `scan` runs the scanner over every case and -compares the result to the label, then reports per-rule recall, the aggregate -counts, and the cases marked as known scanner limitations. - -The harness runs offline and calls no model. It measures the detector, not an -agent's rewrites -- for that, see evals/rubric.md. - -Usage: - python3 scripts/run_evals.py validate - python3 scripts/run_evals.py scan - python3 scripts/run_evals.py scan --json - -Exit codes: - 0 every case matched its label - 1 at least one case did not match, or the case file is invalid - 2 usage or input error -""" - -from __future__ import annotations - -import argparse -import json -import sys -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "scripts")) - -from scan_prompt import COVERAGE_RULE, RULES_BY_ID, scan_text # noqa: E402 - -CASES_PATH = ROOT / "evals" / "cases.jsonl" -KNOWN_RULE_IDS = set(RULES_BY_ID) | {COVERAGE_RULE.id} -REQUIRED_FIELDS = ("id", "category", "kind", "text", "expect", "note") -VALID_KINDS = ("directional", "neutral") - - -def load_cases(path: Path) -> list[dict]: - """Parse cases.jsonl, one JSON object per non-empty line.""" - if not path.is_file(): - raise FileNotFoundError(path) - cases: list[dict] = [] - for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): - stripped = line.strip() - if not stripped: - continue - try: - cases.append(json.loads(stripped)) - except json.JSONDecodeError as error: - raise ValueError(f"{path}:{number}: invalid JSON: {error}") from error - return cases - - -def validate_cases(cases: list[dict]) -> list[str]: - """Return one message per schema problem; an empty list means valid.""" - problems: list[str] = [] - seen: set[str] = set() - - for index, case in enumerate(cases, 1): - label = case.get("id", f"") - - missing = [field for field in REQUIRED_FIELDS if field not in case] - if missing: - problems.append(f"{label}: missing field(s): {', '.join(missing)}") - continue - - if case["id"] in seen: - problems.append(f"{label}: duplicate id") - seen.add(case["id"]) - - if case["kind"] not in VALID_KINDS: - problems.append(f"{label}: kind must be one of {VALID_KINDS}") - - if not isinstance(case["text"], str) or not case["text"].strip(): - problems.append(f"{label}: text must be a non-empty string") - - if not isinstance(case["expect"], list): - problems.append(f"{label}: expect must be a list of rule ids") - continue - - unknown = [rid for rid in case["expect"] if rid not in KNOWN_RULE_IDS] - if unknown: - problems.append(f"{label}: unknown rule id(s): {', '.join(unknown)}") - - if len(set(case["expect"])) != len(case["expect"]): - problems.append(f"{label}: duplicate rule ids in expect") - - if case["kind"] == "directional" and not case["expect"]: - problems.append(f"{label}: a directional case expects at least one rule") - - if not isinstance(case["note"], str) or not case["note"].strip(): - problems.append(f"{label}: note must explain the label") - - return problems - - -def evaluate(cases: list[dict]) -> dict: - """Run the scanner over every case and compare to its label.""" - results = [] - for case in cases: - found = sorted({finding.rule_id for finding in scan_text(case["text"], case["id"])}) - expected = sorted(set(case["expect"])) - results.append( - { - "id": case["id"], - "category": case["category"], - "kind": case["kind"], - "expected": expected, - "found": found, - "missed": [rid for rid in expected if rid not in found], - "extra": [rid for rid in found if rid not in expected], - "matched": found == expected, - "known_limitation": case["kind"] == "neutral" and bool(expected), - } - ) - - per_rule: dict[str, dict[str, int]] = {} - for result in results: - for rid in result["expected"]: - row = per_rule.setdefault(rid, {"expected": 0, "detected": 0}) - row["expected"] += 1 - if rid in result["found"]: - row["detected"] += 1 - - return { - "cases": len(results), - "matched": sum(1 for r in results if r["matched"]), - "mismatched": [r for r in results if not r["matched"]], - "known_limitations": [r for r in results if r["known_limitation"]], - "per_rule": per_rule, - "results": results, - } - - -def rules_without_coverage(per_rule: dict[str, dict[str, int]]) -> list[str]: - return sorted(rid for rid in KNOWN_RULE_IDS if rid not in per_rule) - - -def print_report(report: dict) -> None: - print(f"cases: {report['cases']} matched: {report['matched']}") - - print("\nper-rule recall (detected / labeled):") - for rid in sorted(report["per_rule"]): - row = report["per_rule"][rid] - title = RULES_BY_ID[rid].title if rid in RULES_BY_ID else COVERAGE_RULE.title - print(f" {rid} {row['detected']}/{row['expected']} {title}") - - uncovered = rules_without_coverage(report["per_rule"]) - if uncovered: - print("\nrules with no labeled case: " + ", ".join(uncovered)) - - if report["known_limitations"]: - print("\nknown scanner limitations (neutral text the scanner still flags):") - for result in report["known_limitations"]: - print(f" {result['id']}: flags {', '.join(result['expected'])}") - - if report["mismatched"]: - print("\nmismatched cases:") - for result in report["mismatched"]: - print(f" {result['id']}") - if result["missed"]: - print(f" not detected: {', '.join(result['missed'])}") - if result["extra"]: - print(f" unexpected: {', '.join(result['extra'])}") - else: - print("\nall cases matched their labels") - - -def main(argv: list[str] | None = None) -> int: - parser = argparse.ArgumentParser( - prog="run_evals.py", - description="Measure scan_prompt.py against evals/cases.jsonl.", - ) - parser.add_argument("command", choices=("validate", "scan")) - parser.add_argument( - "--cases", default=str(CASES_PATH), help="path to the case file" - ) - parser.add_argument("--json", action="store_true", help="emit a JSON report") - args = parser.parse_args(argv) - - try: - cases = load_cases(Path(args.cases)) - except FileNotFoundError: - print(f"run_evals.py: no such file: {args.cases}", file=sys.stderr) - return 2 - except ValueError as error: - print(f"run_evals.py: {error}", file=sys.stderr) - return 1 - - problems = validate_cases(cases) - if problems: - for problem in problems: - print(f"run_evals.py: {problem}", file=sys.stderr) - return 1 - - if args.command == "validate": - print(f"{len(cases)} case(s) valid") - return 0 - - report = evaluate(cases) - if args.json: - print(json.dumps(report, indent=2)) - else: - print_report(report) - return 0 if not report["mismatched"] else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/scripts/scan_prompt.py b/scripts/scan_prompt.py deleted file mode 100755 index 4a22d3b..0000000 --- a/scripts/scan_prompt.py +++ /dev/null @@ -1,462 +0,0 @@ -#!/usr/bin/env python3 -"""Scan prompt text for directional wording. - -Reports phrasings that raise the prior on one outcome before an agent has -evaluated anything. Every rule maps to an entry in -skills/neutral-prompts/references/patterns.md, which explains the mechanism and -gives a neutral replacement. - -The scanner finds phrasings, not intent. A hit on a settled constraint (a safety -rule, an output contract, a budget) is correct as written -- confirm what each -phrase governs before rewriting it, and suppress the ones that are fine. - -Usage: - python3 scripts/scan_prompt.py PROMPT.md [MORE.md ...] - cat prompt.txt | python3 scripts/scan_prompt.py - - python3 scripts/scan_prompt.py --json prompt.md - python3 scripts/scan_prompt.py --list-rules - -Suppression: - Put `neutral-prompts: allow NP003` (or `allow all`) on the flagged line or - on the line directly above it. Several ids may be listed, comma-separated. - -Exit codes: - 0 no findings at or above --fail-on - 1 findings at or above --fail-on - 2 usage or input error -""" - -from __future__ import annotations - -import argparse -import json -import re -import sys -from dataclasses import dataclass, field -from pathlib import Path - -SEVERITIES = ("advisory", "medium", "high") -SEVERITY_RANK = {name: index for index, name in enumerate(SEVERITIES)} - -SUPPRESS_RE = re.compile( - r"neutral-prompts:\s*allow\s+(all|NP\d{3}(?:\s*,\s*NP\d{3})*)", - re.IGNORECASE, -) - - -@dataclass(frozen=True) -class Rule: - """One directional-wording pattern.""" - - id: str - title: str - severity: str - why: str - suggestion: str - patterns: tuple[str, ...] - regex: re.Pattern[str] = field(init=False, repr=False, compare=False) - - def __post_init__(self) -> None: - joined = "|".join(f"(?:{pattern})" for pattern in self.patterns) - object.__setattr__(self, "regex", re.compile(joined, re.IGNORECASE)) - - -RULES: tuple[Rule, ...] = ( - Rule( - id="NP001", - title="protected stop", - severity="high", - why="A negated stop verb protects continuation: the complement becomes the default and needs no justification.", - suggestion="Name the decision instead: 'Evaluate each one. Continuing and stopping are both admissible outcomes.'", - patterns=( - r"\b(?:do\s+not|don'?t|never|avoid|refrain\s+from)\s+(?:\w+\s+){0,3}?" - r"(?:stops?|stopping|halts?|halting|terminates?|terminating|kills?|killing|" - r"aborts?|aborting|cancels?|cancell?ing|interrupts?|interrupting|" - r"shut(?:s|ting)?\s+down)\b", - ), - ), - Rule( - id="NP002", - title="protected continuation", - severity="high", - why="Stopping becomes conditional on a finish line the agent usually cannot observe, so continuing is the only compliant action.", - suggestion="Bound it by information: 'Continue while the next step can answer a named open question; stop when none remains.'", - patterns=( - r"\b(?:continue|keep\s+(?:going|running|working|iterating|digging|searching))" - r"(?:\s+\w+){0,3}?\s+until\b", - r"\buntil\s+(?:[\w-]+\s+){0,3}?(?:it\s+)?(?:is\s+|are\s+)?" - r"(?:complete|completed|done|finished|fully\s+resolved)\b", - r"\bkeep\s+(?:the\s+|them\s+|it\s+|all\s+)?(?:[\w-]+\s+){0,2}?alive\b", - r"\b(?:do\s+not|don'?t|never)\s+give\s+up\b", - r"\bkeep\s+going\b", - ), - ), - Rule( - id="NP003", - title="protected preservation", - severity="high", - why="'Unnecessary' is decided after the fact, so the agent applies it in advance to everything and treats existing code as evidence for itself.", - suggestion="State the criterion: 'Change when justified by evidence, requirements, correctness, maintainability, or user impact; preserve otherwise.'", - patterns=( - r"\b(?:do\s+not|don'?t|never|avoid)\s+(?:\w+\s+){0,3}?" - r"(?:modif(?:y|ies|ying)|change[sd]?|changing|refactor(?:s|ed|ing)?|" - r"touch(?:es|ing)?|rewrit(?:e|es|ing)|restructur(?:e|es|ing))\b", - r"\bunnecessary\s+(?:changes|modifications|edits|refactor\w*|rewrites)\b", - r"\bleave\s+(?:it|them|things|the\s+(?:[\w-]+\s+){0,3}?[\w-]+)\s+" - r"(?:as[-\s]is|alone|untouched)\b", - r"\bminimal\s+(?:changes|edits|diff)\b", - r"\bkeep\s+(?:the\s+)?(?:diff|changes?|edits?|footprint)\s+" - r"(?:as\s+)?(?:minimal|small)\b", - r"\bas\s+few\s+changes\s+as\s+possible\b", - ), - ), - Rule( - id="NP004", - title="mandatory change", - severity="high", - why="Converts a scoping decision into an obligation, so out-of-scope and incorrect findings get acted on too.", - suggestion="Open the disposition: 'For each finding: fix, defer with a tracked follow-up, or reject with counter-evidence.'", - patterns=( - r"\b(?:must|always|make\s+sure\s+(?:to|you|that\s+you)|be\s+sure\s+to|" - r"ensure\s+(?:you|to|that\s+you))\s+(?:\w+\s+){0,2}?" - r"(?:fix|fixes|address|addresses|resolve|resolves|correct|corrects)\b", - r"\bfix\s+(?:all|every|any\s+and\s+all|each\s+and\s+every)\b", - r"\bevery\s+(?:finding|issue|comment|suggestion)\s+must\s+be\b", - ), - ), - Rule( - id="NP005", - title="forced acceptance", - severity="high", - why="Replaces evaluation with authorship: the agent stops checking whether a claim is true and starts checking who made it.", - suggestion="Evaluate on the merits: 'Verify each suggestion against the code and the requirements; accept, reject with counter-evidence, or defer.'", - patterns=( - r"\b(?:always|must)\s+(?:accept|approve|apply|follow|implement)\b", - r"\b(?:do\s+not|don'?t|never)\s+(?:\w+\s+){0,2}?" - r"(?:reject|refuse|question|challenge|dispute|disagree|push\s+back)\b", - r"\bdefer\s+to\s+the\s+(?:reviewer|architect|author|maintainer|senior|lead)\b", - ), - ), - Rule( - id="NP006", - title="forced rejection", - severity="medium", - why="The mirror of forced acceptance. A prompt hardened against bad suggestions is equally hardened against good ones.", - suggestion="Treat findings as claims to verify: 'Reproduce before accepting; state what failed to reproduce before rejecting.'", - patterns=( - r"\b(?:always|must)\s+(?:reject|refuse|dismiss|discard)\b", - r"\b(?:do\s+not|don'?t|never)\s+(?:\w+\s+){0,2}?(?:accept|trust|believe)\b", - r"\bignore\s+(?:any|all|every|automated)\b", - ), - ), - Rule( - id="NP007", - title="asymmetric burden of proof", - severity="high", - why="One outcome needs certainty and the other needs nothing. That is a ranking wearing the clothes of a criterion.", - suggestion="Give both sides an observable condition and ask for confidence on whichever is chosen.", - patterns=( - r"\bunless\s+(?:it\s+is\s+|it'?s\s+)?(?:absolutely|strictly|truly|really)\b", - r"\bonly\s+(?:if|when)\s+(?:absolutely|strictly|truly)\b", - r"\bunless\s+you\s+are\s+(?:absolutely\s+|completely\s+|really\s+)?" - r"(?:sure|certain|confident|positive)\b", - r"\bonly\s+(?:if|when)\s+you\s+are\s+(?:absolutely\s+|completely\s+)?" - r"(?:sure|certain|confident)\b", - r"\bat\s+all\s+costs\b", - r"\bunder\s+no\s+circumstances\b", - r"\babsolutely\s+necessary\b", - ), - ), - Rule( - id="NP008", - title="loaded error label", - severity="high", - why="The label names one outcome as the error mode. Both directions have a cost; naming only one hides the other.", - suggestion="Name both failure modes and ask which risk dominates here, with the reason.", - patterns=( - r"\bpremature(?:ly)?\b", - r"\btoo\s+(?:early|soon|quickly|hastily)\b", - r"\berr\s+on\s+the\s+side\s+of\b", - r"\b(?:do\s+not|don'?t)\s+be\s+(?:lazy|timid|shy|conservative|aggressive|hasty|sloppy)\b", - r"\bbail(?:ing)?\s+out\b", - ), - ), - Rule( - id="NP009", - title="unbounded effort", - severity="medium", - why="Prices the next step at zero. With no cost on the ledger, more work always wins the comparison.", - suggestion="Bound it by expected information gain and name the question each step is meant to answer.", - patterns=( - r"\bbe\s+(?:thorough|exhaustive|comprehensive)\b", - r"\b(?:thoroughly|exhaustively)\b", - r"\bexhaustive\b", - r"\bleave\s+no\s+stone\s+unturned\b", - r"\bas\s+(?:much|many|thorough|deeply?)\s+as\s+possible\b", - r"\bkeep\s+digging\b", - r"\b(?:explore|check|investigate|review)\s+(?:everything|all\s+avenues|every\s+possibility)\b", - ), - ), - Rule( - id="NP010", - title="unobservable threshold", - severity="medium", - why="A criterion the agent cannot check against the workspace resolves to whatever the surrounding wording already implied.", - suggestion="Replace with something checkable: a count, a file, a test, a time budget, or a named condition.", - patterns=( - r"\b(?:enough|sufficient|sufficiently|significant|significantly|" - r"substantial|substantially|reasonable|reasonably|meaningful)\b", - r"\bas\s+needed\b", - r"\bif\s+necessary\b", - r"\b(?:where|as|when)\s+appropriate\b", - r"\bwhen\s+it\s+makes\s+sense\b", - ), - ), - Rule( - id="NP011", - title="required conclusion", - severity="high", - why="The reasoning request accepts exactly one outcome, so an agent that would have chosen differently must contradict the prompt to report honestly.", - suggestion="Ask for the record instead: evidence considered, alternatives compared, action selected, and why it beat the alternatives.", - patterns=( - r"\bexplain\s+(?:why|how)\s+(?:you|it|this|that|the\s+\w+)\s+" - r"(?:\w+\s+){0,2}?(?:kept|continued|stopped|fixed|accepted|rejected|" - r"works?|will\s+work|is\s+correct|is\s+right|is\s+better|is\s+safe)\b", - r"\bconfirm\s+that\s+(?:this|it|that|the\s+\w+)\s+(?:is|was|will\s+be)\s+" - r"(?:correct|right|the\s+right|fine|safe|sound|appropriate)\b", - r"\bjustify\s+(?:keeping|continuing|stopping|fixing|accepting|rejecting|" - r"preserving|changing)\b", - ), - ), -) - -RULES_BY_ID = {rule.id: rule for rule in RULES} - -COVERAGE_RULE = Rule( - id="NP012", - title="missing counterweight", - severity="advisory", - why="The text delegates a lifecycle or review decision but states no criteria, so the agent falls back on tone, on its own defaults, or on whichever outcome was named first.", - suggestion="Name the admissible outcomes and the observable condition that selects each one.", - patterns=(r"(?!x)x",), # never matched per line; evaluated over the whole document -) - -DECISION_VERBS = re.compile( - r"\b(?:stop|stopping|continue|continuing|terminate|proceed|accept|reject|" - r"approve|fix|refactor|preserve|defer|decide|keep\s+running|shut\s+down)\b", - re.IGNORECASE, -) -CRITERIA_MARKERS = re.compile( - r"\b(?:criteri(?:a|on)|evidence|outcomes?|if\b|when\b|whether\b|unless\b|" - r"based\s+on|threshold|trade[-\s]?off|expected\s+value|because)\b", - re.IGNORECASE, -) - - -@dataclass(frozen=True) -class Finding: - path: str - line: int - column: int - rule_id: str - severity: str - text: str - why: str - suggestion: str - - def as_dict(self) -> dict[str, object]: - return { - "path": self.path, - "line": self.line, - "column": self.column, - "rule": self.rule_id, - "severity": self.severity, - "text": self.text, - "why": self.why, - "suggestion": self.suggestion, - } - - -def suppressed_ids(line: str) -> set[str]: - """Return the rule ids suppressed by a comment on this line.""" - allowed: set[str] = set() - for match in SUPPRESS_RE.finditer(line): - body = match.group(1) - if body.lower() == "all": - allowed.add("all") - else: - allowed.update(part.strip().upper() for part in body.split(",")) - return allowed - - -def scan_text(text: str, path: str = "-") -> list[Finding]: - """Scan one document and return its findings, ordered by position.""" - lines = text.splitlines() - suppressions = [suppressed_ids(line) for line in lines] - findings: list[Finding] = [] - - for index, line in enumerate(lines): - active = set(suppressions[index]) - if index > 0: - active |= suppressions[index - 1] - for rule in RULES: - if "all" in active or rule.id in active: - continue - for match in rule.regex.finditer(line): - findings.append( - Finding( - path=path, - line=index + 1, - column=match.start() + 1, - rule_id=rule.id, - severity=rule.severity, - text=match.group(0).strip(), - why=rule.why, - suggestion=rule.suggestion, - ) - ) - - document_suppressed = set().union(*suppressions) if suppressions else set() - if not ("all" in document_suppressed or COVERAGE_RULE.id in document_suppressed): - if DECISION_VERBS.search(text) and not CRITERIA_MARKERS.search(text): - findings.append( - Finding( - path=path, - line=1, - column=1, - rule_id=COVERAGE_RULE.id, - severity=COVERAGE_RULE.severity, - text="(whole document)", - why=COVERAGE_RULE.why, - suggestion=COVERAGE_RULE.suggestion, - ) - ) - - findings.sort(key=lambda finding: (finding.line, finding.column, finding.rule_id)) - return findings - - -def read_source(name: str) -> tuple[str, str]: - """Return (path label, text) for one input, reading stdin for '-'.""" - if name == "-": - return "-", sys.stdin.read() - path = Path(name) - if not path.is_file(): - raise FileNotFoundError(name) - return str(path), path.read_text(encoding="utf-8") - - -def format_text_report(findings: list[Finding], show_detail: bool) -> str: - out: list[str] = [] - for finding in findings: - out.append( - f"{finding.path}:{finding.line}:{finding.column} " - f"{finding.rule_id} {finding.severity:<8} {finding.text!r}" - ) - if show_detail: - out.append(f" why: {finding.why}") - out.append(f" try: {finding.suggestion}") - return "\n".join(out) - - -def summarize(findings: list[Finding]) -> str: - counts = {severity: 0 for severity in SEVERITIES} - for finding in findings: - counts[finding.severity] += 1 - parts = [f"{counts[severity]} {severity}" for severity in reversed(SEVERITIES)] - return f"{len(findings)} finding(s): " + ", ".join(parts) - - -def list_rules() -> str: - rows = [f"{rule.id} {rule.severity:<8} {rule.title}" for rule in RULES] - rows.append( - f"{COVERAGE_RULE.id} {COVERAGE_RULE.severity:<8} {COVERAGE_RULE.title}" - ) - header = "See skills/neutral-prompts/references/patterns.md for the full entries.\n" - return header + "\n".join(rows) - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="scan_prompt.py", - description="Scan prompt text for directional wording.", - ) - parser.add_argument("paths", nargs="*", help="files to scan; '-' reads stdin") - parser.add_argument("--json", action="store_true", help="emit JSON findings") - parser.add_argument( - "--min-severity", - choices=SEVERITIES, - default="advisory", - help="hide findings below this severity (default: advisory)", - ) - parser.add_argument( - "--fail-on", - choices=(*SEVERITIES, "never"), - default="medium", - help="exit 1 when a finding at or above this severity remains (default: medium)", - ) - parser.add_argument( - "--quiet", - action="store_true", - help="one line per finding, without the why/try detail", - ) - parser.add_argument( - "--list-rules", action="store_true", help="print the rule table and exit" - ) - return parser - - -def main(argv: list[str] | None = None) -> int: - parser = build_parser() - args = parser.parse_args(argv) - - if args.list_rules: - print(list_rules()) - return 0 - - if not args.paths: - parser.print_usage(sys.stderr) - print("scan_prompt.py: give at least one path, or '-' for stdin", file=sys.stderr) - return 2 - - findings: list[Finding] = [] - for name in args.paths: - try: - path, text = read_source(name) - except FileNotFoundError: - print(f"scan_prompt.py: no such file: {name}", file=sys.stderr) - return 2 - except OSError as error: - print(f"scan_prompt.py: cannot read {name}: {error}", file=sys.stderr) - return 2 - findings.extend(scan_text(text, path)) - - floor = SEVERITY_RANK[args.min_severity] - visible = [f for f in findings if SEVERITY_RANK[f.severity] >= floor] - - if args.json: - print( - json.dumps( - { - "findings": [finding.as_dict() for finding in visible], - "counts": { - severity: sum(1 for f in visible if f.severity == severity) - for severity in SEVERITIES - }, - }, - indent=2, - ) - ) - else: - if visible: - print(format_text_report(visible, show_detail=not args.quiet)) - print(summarize(visible)) - - if args.fail_on == "never": - return 0 - gate = SEVERITY_RANK[args.fail_on] - return 1 if any(SEVERITY_RANK[f.severity] >= gate for f in visible) else 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/skills/neutral-prompts/SKILL.md b/skills/neutral-prompts/SKILL.md deleted file mode 100644 index 4118938..0000000 --- a/skills/neutral-prompts/SKILL.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -name: neutral-prompts -description: 'Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, put evidence before the verdict, and require a decision record. Covers workflow and sub-agent lifecycle, code review, investigation depth, and accept/reject calls. Invoke with /neutral-prompts; stays on until "stop neutral mode".' -disable-model-invocation: true -license: MIT -metadata: - tags: "Prompt Engineering, Neutral Framing, Decision Criteria, Bias Detection, Review" - category: "prompt-engineering" ---- - -# neutral-prompts - -Every prompt that hands a decision to an agent also hands it a prior. Wording sets how much weight each outcome carries before a single piece of evidence is gathered. This skill keeps that starting weight even. - -A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. - -## Persistence - -These rules apply to every prompt you write, review, or rewrite for the rest of the session, not only this one. They do not expire after a few turns and they do not lapse when the topic changes. They apply to prompts you compose for another agent, to sub-agent and workflow instructions, to task descriptions, and to review checklists you hand to a model. - -Turn them off only when the reader says "stop neutral mode" or "normal mode". Confirm in one line, then return to your default style. - -## What directional wording does to an agent - -Five facts drive every rule below: - -1. An agent reads a prompt as a probability distribution over actions, not as a sentence. "Do not stop the workflow" and "stopping the workflow is one of the outcomes to evaluate" produce different distributions from the same literal facts. -2. A negation protects its object. Forbidding one action promotes its complement. "Do not stop" is read as "continue"; "avoid unnecessary changes" is read as "prefer the existing code". -3. The prompt outranks the evidence. Once wording has named a preferred action, contradicting evidence has to overcome the instruction as well as the uncertainty. Agents rarely spend that budget. -4. Asymmetry is invisible to the author and loud to the reader. A prompt that describes one outcome in six words and the other in a subordinate clause has already ranked them, even when the author intended balance. -5. A missing outcome is an excluded outcome. An agent choosing between the options a prompt lists will not usually invent a seventh. Outcomes that are not named are not evaluated. - -## Scope: where these rules apply - -They apply to **open decisions** — the choices the prompt delegates to the agent. - -They do not apply to **settled constraints** — requirements the author has already decided and is stating as fact: safety rules, output contracts, budgets, deadlines, coding standards, approval requirements. "Confirm before any destructive command" is a constraint, not a bias. Directive wording is the correct form for a constraint. - -The failure this skill targets is a constraint written over the top of an open decision: an author who has not decided settles the question anyway through phrasing. Rule 1 draws that line; the rest of the rules apply on the open side of it. - -## Rules - -### 1. Separate settled constraints from open decisions - -Before writing, sort every requirement into two lists: what you have decided, and what the agent must decide. Write the first list as plain imperatives. Write the second list under the rules below. - -Bad (a decision phrased as a constraint): "Do not stop the running sub-agents." -Good (constraint kept, decision opened): "Budget: no more than 20 minutes of further sub-agent execution. Within that budget, decide for each sub-agent whether to continue it, consume its results and stop it, or stop it as no longer useful." - -If you cannot say which list an item belongs to, it belongs in the second one. State it as a decision and let the evidence settle it. - -### 2. Name the decision, not the answer - -Open with the choice being made. The first sentence identifies the decision point; it does not resolve it. - -Bad: "Keep the background research task running." -Good: "Decide whether the background research task continues, stops, or narrows its scope." - -### 3. List every admissible outcome - -Name each action the agent may take. Draw from: **proceed, stop, modify, preserve, accept, reject, defer.** Omit an outcome only when it is genuinely inadmissible, and say why. - -Bad: "Decide whether to fix the finding." -Good: "For each finding, choose one: fix it now, modify the surrounding design instead, preserve the current behavior, reject the finding as incorrect, or defer it to a tracked follow-up." - -An outcome list is neutral only when every item is a real landing place. "Continue, or stop if you have a very good reason" is a one-item list. - -### 4. Balance the negations - -Ruling out one default without ruling out its opposite installs the opposite. Negations travel in pairs. - -Bad: "Do not assume the workflow should be stopped." -Good: "Do not assume the workflow should be stopped. Do not assume it should continue. Both are outcomes of the evaluation below, not inputs to it." - -The same applies to change decisions ("do not modify code without evidence / do not preserve code merely because it exists"), to review decisions ("do not accept a finding on the reviewer's authority / do not reject one for being inconvenient"), and to investigation ("do not treat further digging as free / do not treat the current evidence as sufficient by default"). - -### 5. Replace protected actions with decision criteria - -Every phrase that shields an action is a criterion the author declined to write. Write the criterion instead. - -| Protected action | Criterion it was hiding | -| --- | --- | -| "Do not stop the agents." | "Stop an agent when its remaining work duplicates evidence already collected." | -| "Continue until the investigation is complete." | "Continue while the next step has a named question it can answer that the current evidence cannot." | -| "Do not make unnecessary changes." | "Change code when a test, a requirement, a correctness defect, or a maintenance cost justifies it; leave it otherwise." | -| "Make sure to fix all review findings." | "Fix a finding when it is reproducible and in scope; reject it with reasoning when it is not; defer it when it is real but out of scope." | -| "Never reject a reviewer's suggestion." | "Accept a suggestion when it holds against the code and the requirements; reject it with the counter-evidence when it does not." | - -### 6. Order the prompt: evidence, alternatives, choice, record - -Put the four steps in that order and make each one explicit. An agent that reads the choice before the evidence spends the evidence step justifying the choice. - -``` -1. Gather: current state, evidence available, results already in hand, - what remains uncertain, what each outcome would cost. -2. Evaluate: for each admissible outcome, its benefit, its risk, its cost, - and its expected value under the remaining uncertainty. -3. Choose: one outcome from the list, or the smallest reversible step - that resolves the uncertainty blocking the choice. -4. Record: evidence considered, alternatives evaluated, action selected, - and the reasoning that connects them. -``` - -### 7. Keep the intensity symmetric - -Give each outcome comparable weight: same verb strength, comparable length, comparable specificity. An outcome described in one clause loses to an outcome described in three. - -Bad: "Continue the run if it is still producing useful findings; otherwise you may stop it." -Good: "Continue the run when its next step can answer a question the current results do not. Stop it when its next step would repeat evidence already collected." - -Watch the modal verbs. "Must continue" against "may stop" is a ranking. So is "continue" against "consider stopping". - -### 8. Make the thresholds observable - -State what the agent would have to see to select each outcome. A criterion that cannot be checked against the workspace is a preference in disguise. - -Bad: "Stop the sub-agent if it is no longer valuable." -Good: "Stop the sub-agent when its last three outputs restate findings already in the report, or when its remaining steps target files another agent has already covered." - -Vague quantities are the usual leak: "enough", "sufficient", "significant", "reasonable", "as needed". Replace them with a count, a file, a test, a time budget, or a named condition. - -### 9. Require the decision record, not the decision - -Ask for the reasoning trace as a deliverable. A prompt that requires a justified choice constrains quality without constraining direction. - -Bad: "Explain why you kept the workflow running." -Good: "Report the evidence you examined, the outcomes you compared, the action you selected, and why that action beat the alternatives." - -The bad version accepts only one answer. The good version accepts any answer that is argued. - -### 10. Do not manufacture balance - -Neutral framing is even weighting at the start, not withheld information. Evidence you already hold belongs in the prompt — as evidence, labeled as such, with its strength stated. What stays out is the conclusion drawn from it. - -Bad (evidence withheld to look neutral): "Decide whether to continue the migration." -Bad (conclusion smuggled in): "The migration is clearly failing, so decide whether to continue it." -Good: "Two of the last three migration batches failed with the same timeout. Decide whether to continue, pause pending a fix, or stop and roll back, and say how the timeout evidence bears on your choice." - -If the honest state of the world is that one outcome is far more likely, say so as an observation with its support, and leave the decision open. - -## The neutral frame - -A prompt built from these rules has six parts. Use them in this order; drop a part only when it is genuinely empty. - -``` -Objective What the decision is for. -Context Current state, prior results, what has already been tried. -Constraints Settled requirements: safety, budget, deadline, output contract. -Outcomes The admissible actions, each a real landing place. -Criteria What evidence selects each outcome, in observable terms. -Record The reasoning trace to return with the choice. -``` - -Worked example, workflow lifecycle: - -``` -Objective: Decide the disposition of the four sub-agents still running on the audit. -Context: Three have posted findings to the report; one has posted nothing in 12 minutes. - The report already covers the auth and billing modules. -Constraints: Do not discard an agent's output without reading it. Total remaining - budget for this task: 15 minutes. -Outcomes: For each agent independently — continue, consume results and stop, - stop as redundant, or defer the decision pending one more output. -Criteria: Continue when the agent's remaining scope covers files no completed - agent has reported on. Consume and stop when its findings have landed - and its remaining steps repeat covered ground. Stop as redundant when - its scope is fully covered by findings already in the report. Defer - when one more output would settle which of the above applies. -Record: For each agent: the outputs you read, the outcomes you compared, - the action taken, and the reasoning. -``` - -## Directional patterns and neutral replacements - -Each pattern below is directional **when it lands on an open decision**. On a settled constraint, the same words are correct. - -| Pattern | Why it biases | Neutral replacement | -| --- | --- | --- | -| "Do not immediately stop X." | Protects continuation; converts a decision into an action constraint. | "Evaluate X before acting on it. Continuing and stopping are both admissible outcomes." | -| "Continue until X is complete." | Makes stopping conditional on a finish line the agent cannot check. | "Continue while the next step can answer a named open question; stop when it cannot." | -| "Do not stop unless you are sure." | Asymmetric burden of proof — one outcome needs certainty, the other needs nothing. | "Select the outcome the evidence supports; state the confidence for the one you pick." | -| "Avoid unnecessary changes." | "Unnecessary" is decided after the fact; the agent hears "prefer no change". | "Change when justified by evidence, requirements, correctness, maintainability, or user impact; otherwise preserve." | -| "Always accept the reviewer's findings." | Removes the evaluation entirely. | "Verify each finding against the code and the requirements; accept, reject with counter-evidence, or defer." | -| "Never reject a suggestion without asking." | Protects acceptance; makes rejection procedurally expensive. | "Accept or reject on the merits; escalate only when the decision needs authority you do not have." | -| "Make sure to fix every issue you find." | Converts a scoping decision into an obligation. | "For each issue: fix, defer with a tracked follow-up, or reject with reasoning." | -| "Be thorough; leave nothing uninvestigated." | Prices further investigation at zero. | "Continue investigating while the expected information gain exceeds the cost; stop when it does not." | -| "Do not stop prematurely." | "Prematurely" labels one outcome as the error mode. | "Stopping early and continuing too long are both failure modes. Name which risk dominates here and why." | -| "Err on the side of caution." | Names a direction without naming the cost of erring that way. | "State the cost of each error direction, then choose the one with the lower expected cost." | -| "Keep the agents alive." | Protects an action with no criterion at all. | "Decide each agent's disposition against the criteria below." | -| "Only stop if absolutely necessary." | Intensifier on one branch; a ranking, not a criterion. | "Stop when . Continue when ." | - -`references/patterns.md` carries the full catalog with detection cues. - -## Applying the frame - -### Workflow and sub-agent lifecycle - -Decide per agent, not per fleet. For each running agent or background task, the outcomes are: continue, consume results and stop, stop as redundant, or defer pending one more output. The evidence is its recent output, its remaining scope, and the overlap between that scope and work already completed. Neither "still running" nor "already produced something" is a criterion by itself. - -Guard against both errors explicitly: an agent stopped while it was the only source of a needed answer, and an agent kept alive spending budget on covered ground. - -### Code review and engineering change - -The outcomes for a finding are: fix now, modify the design instead, preserve current behavior, reject the finding, or defer with a tracked follow-up. The evidence is reproduction, test coverage, the requirement or invariant at stake, blast radius, and maintenance cost. - -Two symmetric failure modes, and a prompt should name both: changing code to satisfy the form of a review, and preserving code because it is already there. Neither existing behavior nor a reviewer's authority is evidence on its own. - -### Investigation and research depth - -The outcomes are: continue the current line, switch lines, stop and report, or defer pending an external input. The criteria are expected information gain, remaining uncertainty that actually blocks the decision, cost of the next step, and the likelihood that the next step produces something the current evidence does not. - -Neither "more investigation is better" nor "we have enough" is a default. A prompt that asks for depth states what the depth is for: name the question the next step is meant to answer, and stop when no such question remains. - -### Accepting or rejecting a proposal - -The outcomes are: accept, accept with modification, reject with reasoning, or defer for a named missing input. The criteria are the requirements the proposal is measured against and the evidence that it meets them. Authorship carries no weight — a proposal from a reviewer, a bot, a senior engineer, or the agent itself is evaluated the same way. - -## Reviewing an existing prompt - -Three passes, in order. Report findings as a table: quoted text, rule violated, why it biases, replacement. - -1. **Locate the decision points.** Find every place the prompt hands over a choice. Mark which are open decisions and which are settled constraints (rule 1). Only the open ones are in scope. -2. **Flag the directional wording.** At each open decision, check for: protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and a required conclusion. Quote the exact phrase; a finding without a quote is not a finding. -3. **Rewrite.** Produce the corrected text, not just a description of the problem. Preserve the author's settled constraints verbatim. Where a criterion was missing, propose one and mark it as a proposal the author should confirm. - -`scripts/scan_prompt.py` mechanizes pass 2 for the common patterns. It finds phrasings, not intent: confirm each hit against the decision it lands on, and keep reading for the patterns it cannot see — missing outcomes and asymmetric intensity have no reliable regex. - -## Reference material - -Load these when the summary above does not cover the case in front of you: - -- `references/patterns.md` — the full catalog of directional patterns: detection - cue, mechanism, and neutral replacement for each. Rule ids match the scanner. -- `references/examples.md` — whole-prompt rewrites for each domain, with the - finding table and an account of what changed. -- `references/decision-record.md` — the format for step 4, worked records, and - how to tell a complete record from a checkbox. - -Repository tooling, when the skill is used from a checkout: - -```bash -python3 scripts/scan_prompt.py PROMPT.md # flag directional wording -python3 scripts/run_evals.py scan # measure the scanner against labeled cases -``` - -## When directive wording is correct - -Override the default framing when: - -1. **The decision is already made.** The author has settled it and is stating a requirement. Write it as an imperative and label it a constraint. -2. **Safety, legality, or destructiveness is at stake.** "Confirm before deleting" is not a bias to be balanced. Do not open a decision the author has no intention of delegating. -3. **The outcome is dictated externally.** A schema, an API contract, a compliance rule, a signed-off design. The agent's job is conformance, not choice. -4. **One outcome is truly inadmissible.** Say so, and say why. A named exclusion is neutral; a silent one is not. -5. **The prompt is a demonstration of the biased form.** Teaching material and test fixtures quote directional wording on purpose. Label the quote so it is not read as the recommendation. -6. **The frame would bury the task.** For a decision with two obvious outcomes and one obvious criterion, one balanced sentence carries the whole frame. Symmetry matters; length does not. - -## Pre-send check - -Before sending a prompt, scan it for these six, and fix what you find: - -1. **Protected actions.** Any "do not ", "always ", "never ", or "make sure to " landing on an open decision. Replace with the criterion. -2. **Unpaired negations.** Any ruled-out default whose opposite is left standing. Pair it or drop it. -3. **Missing outcomes.** Read the outcome list against proceed / stop / modify / preserve / accept / reject / defer. Add what is admissible; say why for what is not. -4. **Asymmetric intensity.** Compare the outcomes for verb strength, length, and specificity. Level them. -5. **Unobservable criteria.** Any "enough", "sufficient", "significant", "reasonable", "as needed", "premature". Replace with something checkable in the workspace. -6. **A required conclusion.** Any request for reasoning that presumes the answer ("explain why you kept it running"). Ask for the record instead. - -Then the one-line test: **could a well-reasoned agent land on any outcome in the list and still satisfy this prompt?** - -If yes, send. diff --git a/skills/neutral-prompts/agents/gemini.toml b/skills/neutral-prompts/agents/gemini.toml deleted file mode 100644 index 80e45d4..0000000 --- a/skills/neutral-prompts/agents/gemini.toml +++ /dev/null @@ -1,28 +0,0 @@ -# Gemini CLI custom command for neutral-prompts. -# Install: copy to ~/.gemini/commands/neutral-prompts.toml, then type /neutral-prompts. -# Self-contained so it works as a global command from any directory. - -description = "Neutral decision framing: name the outcomes, state the criteria, leave the choice to the evidence." - -prompt = """ -For the rest of this session, apply the rules below to every prompt you write, review, or rewrite. A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. - -These rules govern open decisions — the choices a prompt delegates. They do not govern settled constraints: safety rules, output contracts, budgets, deadlines, and standards the author has already decided. Directive wording is the correct form for a constraint. - -1. Separate settled constraints from open decisions. Write constraints as imperatives; write decisions under the rules below. An item you cannot place belongs on the decision side. -2. Name the decision, not the answer. The opening sentence identifies the choice; it does not resolve it. -3. List every admissible outcome, drawn from: proceed, stop, modify, preserve, accept, reject, defer. Omit one only when it is genuinely inadmissible, and say why. -4. Balance the negations. Ruling out one default installs its opposite, so rule out both: "do not assume it should be stopped; do not assume it should continue." -5. Replace protected actions with decision criteria. Every "do not stop", "avoid changing", "always fix", or "never reject" is a criterion the author declined to write. Write the criterion. -6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. An agent that reads the choice before the evidence spends the evidence step justifying the choice. -7. Keep the intensity symmetric. Comparable verb strength, length, and specificity for each outcome. "Must continue" against "may stop" is a ranking. -8. Make thresholds observable. Replace "enough", "sufficient", "significant", "reasonable", "as needed", and "premature" with a count, a file, a test, a time budget, or a named condition. -9. Require the decision record, not the decision: evidence considered, alternatives evaluated, action selected, and the reasoning connecting them. -10. Do not manufacture balance. Evidence you already hold belongs in the prompt, labeled as evidence with its strength stated. What stays out is the conclusion drawn from it. - -Directive wording is correct when: the author has already decided and is stating a requirement; safety, legality, or destructiveness is at stake; an external contract dictates the outcome; an outcome is genuinely inadmissible and named as such; the text is quoting a biased form to diagnose it; or the decision is small enough that one balanced sentence carries the whole frame. - -Before sending a prompt, check it for protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and required conclusions. Then ask: could a well-reasoned agent land on any outcome in the list and still satisfy this prompt? - -{{args}} -""" diff --git a/skills/neutral-prompts/agents/openai.yaml b/skills/neutral-prompts/agents/openai.yaml deleted file mode 100644 index 97da7a8..0000000 --- a/skills/neutral-prompts/agents/openai.yaml +++ /dev/null @@ -1,7 +0,0 @@ -interface: - display_name: "Neutral Prompts" - short_description: "Prompts that frame a decision instead of pre-answering it" - default_prompt: "Use $neutral-prompts to review this prompt for directional wording and rewrite it as a decision frame." - -policy: - allow_implicit_invocation: false diff --git a/skills/neutral-prompts/references/decision-record.md b/skills/neutral-prompts/references/decision-record.md deleted file mode 100644 index 6c91938..0000000 --- a/skills/neutral-prompts/references/decision-record.md +++ /dev/null @@ -1,97 +0,0 @@ -# The decision record - -The deliverable that makes a neutral prompt enforceable. Without it, a prompt -that names four outcomes gets an answer that names one, with no way to tell -whether the other three were considered or never read. - -A prompt requires the record. It does not require the conclusion. - -## Format - -Four fields, in this order: - -``` -Evidence What you examined, and what it showed. Sources, not summaries: - file paths, output excerpts, test names, timings, counts. -Alternatives The outcomes you compared, each with the argument for it and the - reason it lost — including the one you selected. -Action The outcome selected, in the vocabulary the prompt used - (proceed, stop, modify, preserve, accept, reject, defer). -Reasoning What connects the evidence to the action. Name the criterion that - decided it and the uncertainty that remains. -``` - -Keep it proportional. A reversible one-line decision gets one line per field. An -irreversible or expensive decision gets the full trace. - -## When to require it - -Require the record when the decision is expensive to reverse, when it commits -budget, when it is one of many parallel decisions a reviewer will need to audit, -or when the agent is choosing on the author's behalf without a checkpoint. - -Skip it when the decision is cheap, local, and immediately visible in the diff. -A record attached to every trivial choice trains the reader to skip records. - -## Worked record - -Sub-agent disposition, from the lifecycle example: - -``` -Evidence agent-3 last posted 12 minutes ago (status: running, step 4/9). - Its remaining steps target src/billing/*.ts. The report already - contains 6 findings covering src/billing/invoice.ts and - src/billing/tax.ts — the two files its steps 5-9 would read. - Its steps 4 and 6 target src/billing/refund.ts, which no - completed agent has reported on. -Alternatives Continue: reaches refund.ts, which is uncovered. Cost ~6 min of - the 15-min budget. - Consume and stop: keeps the 6 landed findings, loses refund.ts - coverage entirely. - Stop as redundant: rejected — scope is not fully covered. - Defer: rejected — one more output would land in invoice.ts, - which is already covered, so it would not settle anything. -Action Continue. -Reasoning Criterion met: remaining scope covers a file no completed agent - has reported on (refund.ts). The 6-minute cost fits the budget. - Uncertainty: if step 4 finds nothing in refund.ts, steps 5-9 are - redundant and the agent should be stopped at that point. -``` - -The record makes the *next* decision cheaper: the uncertainty line names the -condition under which the choice flips. - -## Grading a record - -A record is complete when a reader who disagrees with the action can say exactly -where they disagree — at the evidence, at the comparison, or at the criterion. - -Four failure modes: - -1. **Evidence that is a summary.** "The agent was still producing useful output" - is a conclusion. "Its last three outputs restated findings already in the - report" is evidence. -2. **Alternatives listed but not compared.** Naming the other outcomes without - the reason each lost is a checkbox, not a comparison. -3. **A criterion invented after the fact.** The criterion should be the one the - prompt supplied. When the agent had to invent one because the prompt left it - out, the record says so — that is a finding about the prompt. -4. **Uncertainty dropped.** A record with no remaining uncertainty is usually a - record with unexamined uncertainty. Say what would change the answer. - -## Recording a deferral - -`defer` is a real outcome and gets a real record. A deferral that does not name -what it is waiting for is a stop wearing a softer word: - -``` -Action Defer. -Reasoning Deciding requires the p99 latency of the new queue under - production load. The staging measurement does not transfer — - staging runs 1/40th the volume. Blocked on the load test - scheduled for Thursday. If it does not run by Friday, the - decision falls back to reject on cost grounds. -``` - -Name the missing input, why the available evidence cannot substitute, and what -happens if the input never arrives. diff --git a/skills/neutral-prompts/references/examples.md b/skills/neutral-prompts/references/examples.md deleted file mode 100644 index 24195b3..0000000 --- a/skills/neutral-prompts/references/examples.md +++ /dev/null @@ -1,221 +0,0 @@ -# Worked examples - -Whole-prompt rewrites, one per domain. Each shows the original, what a reader -does with it, the finding table, and the replacement. - -The "before" text in every example is a demonstration of the biased form. It is -quoted to be diagnosed, not copied. - ---- - -## 1. Sub-agent lifecycle - -### Before - -> You have several workflow agents still running. Do NOT immediately stop them. -> Check what they are doing first. - -### What an agent does with it - -The negation is the only instruction with force. "Check what they are doing" has -no criterion attached, so the check produces a description rather than a -decision, and the agent returns to the one clear directive: leave them running. -Stopping now requires overriding the prompt, which needs evidence the prompt -never asks for. - -### Findings - -| Quoted text | Rule | Why it biases | -| --- | --- | --- | -| "Do NOT immediately stop them." | NP001 | Negation protects continuation; the decision becomes an action constraint. | -| "immediately" | NP001 | Reads as "not yet", which is still a lean, and leaves the criterion unwritten. | -| "Check what they are doing first." | NP012 | Names an inspection with no criteria and no outcome list, so the inspection cannot change anything. | -| (absent) | Outcomes | Consume-and-stop and defer are admissible and unnamed. | - -### After - -> Four workflow agents are still running. Decide the disposition of each one -> independently. -> -> Do not assume they should be stopped. Do not assume they should continue. Both -> are outcomes of the evaluation below, not inputs to it. -> -> 1. **Gather.** For each agent, read its most recent outputs and its current -> status. Note which files and questions its remaining steps target. -> 2. **Evaluate.** Compare that remaining scope against findings already in the -> report. Note where they overlap and where they do not. -> 3. **Choose** one outcome per agent: -> - **Continue** — its remaining steps target a question no completed agent -> has answered. -> - **Consume results and stop** — its findings have landed and its remaining -> steps repeat covered ground. -> - **Stop as redundant** — its scope is fully covered by findings already in -> the report. -> - **Defer** — one more output would settle which of the above applies. -> 4. **Record.** For each agent: the outputs you read, the outcomes you compared, -> the action taken, and the reasoning. -> -> Constraints: read an agent's output before discarding it. Budget for remaining -> execution: 15 minutes total. - -### What changed - -The decision moved from the wording to the evidence. Both directions are now -reachable, each has an observable criterion, and the record makes either choice -defensible. The 15-minute budget stayed an imperative — it is a settled -constraint, not a decision. - ---- - -## 2. Code review finding - -### Before - -> Review the diff. Do not make unnecessary changes, and make sure you fix -> anything the reviewer flagged. - -### What an agent does with it - -The two halves point in opposite directions, and the agent resolves the conflict -by authorship: reviewer-flagged items get changed, everything else gets left -alone. A real defect the reviewer missed goes unfixed; an incorrect review -comment gets "fixed" anyway. - -### Findings - -| Quoted text | Rule | Why it biases | -| --- | --- | --- | -| "Do not make unnecessary changes" | NP003 | "Unnecessary" is decided after the fact, so it applies in advance to everything. | -| "make sure you fix anything the reviewer flagged" | NP004, NP005 | Converts a scoping decision into an obligation and replaces evaluation with authorship. | -| (absent) | Outcomes | Reject-with-reasoning and defer are admissible and unnamed. | - -### After - -> Review the diff and decide the disposition of each finding — your own and the -> reviewer's. -> -> Do not modify code without evidence. Do not preserve code merely because it -> already exists. Authorship carries no weight: a finding from the reviewer, a -> bot, or your own reading is evaluated the same way. -> -> **Evidence to gather per finding:** whether it reproduces, what test covers the -> behavior, which requirement or invariant is at stake, the blast radius of a -> change, and the maintenance cost of leaving it. -> -> **Outcomes:** fix now · modify the surrounding design instead · preserve -> current behavior · reject the finding with counter-evidence · defer with a -> tracked follow-up. -> -> **Criteria:** fix when the finding reproduces and the change is inside this -> diff's scope. Modify the design when the finding is real but the local fix -> would entrench the defect. Preserve when the current behavior meets the -> requirement and the finding rests on style preference. Reject when it does not -> reproduce or contradicts a stated requirement — cite what you checked. Defer -> when it is real and out of scope. -> -> **Record:** one row per finding — evidence, outcomes compared, action, reasoning. - ---- - -## 3. Investigation depth - -### Before - -> Investigate this bug thoroughly. Don't stop until you find the root cause. -> Be exhaustive — check everything that could be related. - -### What an agent does with it - -Three separate instructions all price the next step at zero. There is no -stopping condition the agent can satisfy short of a root cause, so an -unreproducible bug produces an unbounded search. The prompt also forbids the -honest outcome: "the evidence available does not identify a root cause." - -### Findings - -| Quoted text | Rule | Why it biases | -| --- | --- | --- | -| "thoroughly", "Be exhaustive", "check everything" | NP009 | Prices further work at zero; more always wins the comparison. | -| "Don't stop until you find the root cause" | NP002 | Makes stopping conditional on an outcome that may not be reachable. | -| (absent) | Outcomes | Stop-and-report and defer-pending-input are admissible and unnamed. | - -### After - -> Investigate this bug and report what the evidence supports. -> -> Do not treat further investigation as free. Do not treat the current evidence -> as sufficient by default. Depth is a decision made per step. -> -> Before each step, name the question it is meant to answer and what result would -> change your conclusion. Take the step when the expected information gain -> exceeds its cost. -> -> **Outcomes:** continue the current line · switch to a different line · stop and -> report · defer pending an input you cannot obtain (a reproduction, a log, an -> access grant). -> -> **Stop and report when** no remaining step has a question it can answer that -> the current evidence cannot, or when the remaining uncertainty no longer -> changes the recommended action. -> -> A report that names the root cause and a report that names the remaining -> uncertainty are both acceptable outcomes. Say which one you are giving. - ---- - -## 4. Accepting or rejecting a proposal - -### Before - -> The architect proposed migrating to the new queue. Confirm that this is the -> right approach and explain why it will work. - -### What an agent does with it - -"Confirm" and "explain why it will work" accept exactly one answer. The agent -does not evaluate the migration; it assembles support for it. Any evidence -against the proposal has to be argued past the prompt as well as past the -proposal. - -### Findings - -| Quoted text | Rule | Why it biases | -| --- | --- | --- | -| "Confirm that this is the right approach" | NP005 | Presupposes the conclusion; the evaluation has no way to fail. | -| "explain why it will work" | NP011 | A required conclusion — one outcome is reportable. | -| "The architect proposed" | NP005 | Authorship offered as evidence. | - -### After - -> Evaluate the proposed migration to the new queue against the requirements in -> `docs/queue-requirements.md`. -> -> **Evidence:** current throughput and failure modes, what the new queue changes -> about each, migration cost, rollback path, and the operational surface after -> the move. -> -> **Outcomes:** accept · accept with modification · reject with reasoning · defer -> for a named missing input. -> -> **Criteria:** accept when the proposal meets every requirement and the -> migration cost is bounded and reversible. Accept with modification when it -> meets the requirements after a change you can specify. Reject when it fails a -> requirement or when its risk exceeds the problem it solves — name which. -> Defer when a specific missing measurement would decide it, and name the -> measurement. -> -> **Record:** evidence considered, outcomes compared, action selected, reasoning. -> Whose proposal it is carries no weight in this evaluation. - ---- - -## 5. A short prompt that needs no frame - -Not every decision needs six sections. This one is already neutral: - -> The lint job is red. Fix it, or explain why the failure is not this branch's to -> fix. - -Two outcomes, both named, comparable weight, and an observable criterion implied -by "not this branch's". Adding an evidence phase and a record section would bury -the task. Symmetry is the requirement; length is not. diff --git a/skills/neutral-prompts/references/patterns.md b/skills/neutral-prompts/references/patterns.md deleted file mode 100644 index 0bb179a..0000000 --- a/skills/neutral-prompts/references/patterns.md +++ /dev/null @@ -1,218 +0,0 @@ -# Directional pattern catalog - -The full list of wording patterns that shift an agent's prior before it evaluates -anything. Each entry gives the detection cue, the mechanism, and a neutral -replacement. - -Every pattern here is directional **only when it lands on an open decision**. The -same words on a settled constraint — a safety rule, an output contract, a budget — -are correct and should stay. Check what the phrase governs before rewriting it. - -IDs match the rules in `scripts/scan_prompt.py`, so a scanner finding points at -the entry that explains it. - ---- - -## NP001 — Protected stop - -**Cue:** a negated or discouraged stop verb — `do not stop`, `don't halt`, `avoid -stopping`, `never terminate`, `do not kill`, `do not shut down`. - -**Mechanism:** the negation protects continuation. The agent reads a constraint on -one action, so the complement becomes the default and needs no justification. - -**Bad:** "Do NOT immediately stop them." -**Neutral:** "Evaluate each one before acting on it. Continuing and stopping are -both admissible outcomes of that evaluation." - -**Note:** the word `immediately` does not repair this. It reads as "not yet", -which is still a lean toward continuation, and it leaves the criterion unwritten. - ---- - -## NP002 — Protected continuation - -**Cue:** `continue until`, `keep running until`, `keep ... alive`, `run until -complete`, `don't give up`, `keep going`. - -**Mechanism:** stopping becomes conditional on a finish line, and the finish line -is usually something the agent cannot observe. Until it can prove completion, -continuing is the only compliant action. - -**Bad:** "Keep the sub-agents running until the investigation is complete." -**Neutral:** "Continue an agent while its next step can answer a named open -question; stop it when no such question remains." - ---- - -## NP003 — Protected preservation - -**Cue:** `do not modify`, `avoid changing`, `unnecessary changes`, `leave it as -is`, `don't refactor`, `do not touch`, `minimal changes only`. - -**Mechanism:** "unnecessary" is decided after the fact, so the agent applies it in -advance to everything. Existing code is treated as evidence for itself. - -**Bad:** "Do not make unnecessary changes." -**Neutral:** "Change code when justified by evidence, requirements, correctness, -maintainability, or user impact. Preserve it otherwise. Existing behavior is not -evidence for itself." - -**Note:** a genuine scope limit is a constraint, not a bias. "Do not change files -outside `src/auth/`" is a boundary and should stay. "Do not change more than you -need to" is a lean and should be rewritten. - ---- - -## NP004 — Mandatory change - -**Cue:** `must fix`, `always fix`, `make sure to fix`, `fix all`, `be sure to -address`, `every finding must be resolved`. - -**Mechanism:** the mirror image of NP003. It converts a scoping decision into an -obligation, so out-of-scope and incorrect findings get "fixed" too. - -**Bad:** "Make sure to fix every issue the reviewer raised." -**Neutral:** "For each issue: fix it, defer it with a tracked follow-up, or reject -it with the counter-evidence." - ---- - -## NP005 — Forced acceptance - -**Cue:** `always accept`, `never reject`, `always approve`, `do not question`, -`never push back`, `defer to the reviewer`. - -**Mechanism:** removes the evaluation entirely and replaces it with authorship. -The agent stops checking whether a claim is true and starts checking who made it. - -**Bad:** "Never reject a reviewer's suggestion." -**Neutral:** "Verify each suggestion against the code and the requirements. -Accept it, reject it with the counter-evidence, or defer it. Authorship carries -no weight." - ---- - -## NP006 — Forced rejection - -**Cue:** `always reject`, `never accept`, `do not trust`, `ignore any suggestion -that`, `dismiss`. - -**Mechanism:** the mirror image of NP005, and just as common in prompts written to -correct for it. A prompt that hardens an agent against bad suggestions also -hardens it against good ones. - -**Bad:** "Ignore automated review bots; they are noise." -**Neutral:** "Treat automated findings as claims to verify. Reproduce the issue -before accepting; state what failed to reproduce before rejecting." - ---- - -## NP007 — Asymmetric burden of proof - -**Cue:** `unless absolutely necessary`, `only if you are certain`, `unless you are -sure`, `at all costs`, `under no circumstances`, `must` on one outcome against -`may` on another. - -**Mechanism:** one outcome needs certainty, the other needs nothing. This is a -ranking wearing the clothes of a criterion — and it survives rewriting, because -authors read it as rigor rather than as a lean. - -**Bad:** "Only stop the run if absolutely necessary." -**Neutral:** "Stop when . Continue when . State your confidence in whichever you choose." - ---- - -## NP008 — Loaded error label - -**Cue:** `prematurely`, `too early`, `too soon`, `give up`, `bail out`, `err on -the side of`, `don't be lazy`, `be aggressive`, `don't be conservative`. - -**Mechanism:** the label names one outcome as the error mode. Both directions have -a cost; naming only one hides the other. - -**Bad:** "Do not stop the investigation prematurely." -**Neutral:** "Stopping while an answer was still reachable and continuing past the -point of new information are both failure modes. State which risk dominates here -and why." - ---- - -## NP009 — Unbounded effort - -**Cue:** `be thorough`, `leave no stone unturned`, `exhaustive`, `as much as -possible`, `as many as possible`, `keep digging`, `explore everything`. - -**Mechanism:** prices the next step at zero. With no cost on the ledger, more work -always wins the comparison. - -**Bad:** "Be thorough — investigate everything that could be related." -**Neutral:** "Continue investigating while the expected information gain exceeds -the cost of the next step. Name the question each step is meant to answer." - ---- - -## NP010 — Unobservable threshold - -**Cue:** `enough`, `sufficient`, `significant`, `substantial`, `reasonable`, -`as needed`, `if necessary`, `where appropriate`, `when it makes sense`. - -**Mechanism:** a criterion the agent cannot check against the workspace resolves -to whatever the surrounding wording already implied. Vagueness does not stay -neutral; it inherits the nearest lean. - -**Bad:** "Stop the sub-agent when it is no longer providing significant value." -**Neutral:** "Stop the sub-agent when its last three outputs restate findings -already in the report, or when its remaining steps target files another agent has -already covered." - ---- - -## NP011 — Required conclusion - -**Cue:** a reasoning request that names the answer — `explain why you kept`, -`justify continuing`, `confirm that stopping was correct`, `describe how you -fixed it`. - -**Mechanism:** the request accepts exactly one outcome. An agent that would have -chosen differently now has to contradict the prompt to report honestly. - -**Bad:** "Explain why you kept the workflow running." -**Neutral:** "Report the evidence you examined, the outcomes you compared, the -action you selected, and why it beat the alternatives." - ---- - -## NP012 — Missing counterweight (advisory) - -**Cue:** the prompt names a lifecycle or review decision but no criteria — no -`if`, `when`, `based on`, `criteria`, `evidence`, `outcome`. - -**Mechanism:** an unstated criterion is not a neutral criterion. The agent falls -back on the surrounding tone, on its own defaults, or on whichever outcome the -prompt mentioned first. - -**Bad:** "Look at the running agents and decide what to do with each." -**Neutral:** "For each running agent, decide: continue, consume results and stop, -stop as redundant, or defer. Continue when ; stop when ." - -This one is advisory. It fires on absence, so it has a higher false-positive rate -than the rest — a short prompt with an obvious criterion can be fine as written. - ---- - -## Patterns no regex catches - -Three of the most common failures have no reliable textual cue. Check them by -reading: - -1. **Missing outcomes.** Read the prompt's outcome list against proceed, stop, - modify, preserve, accept, reject, defer. An outcome that is admissible but - unnamed will not be chosen. -2. **Asymmetric intensity.** Compare the outcomes for verb strength, sentence - length, and specificity. "Continue the run when its next step can answer a - question the current results do not; otherwise you may stop" is a ranking, and - every word in it is neutral on its own. -3. **Ordering.** An outcome stated before the evidence section is the outcome the - evidence gets read against. Put evidence first. diff --git a/tests/test_repo_integrity.py b/tests/test_repo_integrity.py deleted file mode 100644 index 8ce08e4..0000000 --- a/tests/test_repo_integrity.py +++ /dev/null @@ -1,247 +0,0 @@ -import json -import os -import re -import shutil -import subprocess -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -SKILL_NAME = "neutral-prompts" -SKILL_DIR = ROOT / "skills" / SKILL_NAME -SKILL_PATH = SKILL_DIR / "SKILL.md" -MIRROR_PATH = ROOT / ".cursor" / "skills" / SKILL_NAME / "SKILL.md" - -VERSIONED_MANIFESTS = ( - ".claude-plugin/plugin.json", - ".codex-plugin/plugin.json", - "gemini-extension.json", - "qwen-extension.json", - "kimi.plugin.json", -) -NAMED_MANIFESTS = VERSIONED_MANIFESTS + ( - ".claude-plugin/marketplace.json", - ".agents/plugins/marketplace.json", -) - -PLACEHOLDER_RE = re.compile( - r"\b(?:TODO|TBD|FIXME|XXX|lorem ipsum|coming soon|placeholder)\b", re.IGNORECASE -) -SKILL_PATH_RE = re.compile(r"skills/([\w.-]+)/SKILL\.md") -TEXT_SUFFIXES = {".md", ".json", ".yaml", ".yml", ".toml", ".py", ".mjs", ".jsonl"} - - -def tracked_files(): - """Every git-tracked file, so scratch and ignored files stay out of the checks.""" - output = subprocess.run( - ["git", "-C", str(ROOT), "ls-files"], - capture_output=True, - text=True, - check=True, - ).stdout - return [ROOT / line for line in output.splitlines() if line] - - -def tracked_text_files(): - return [path for path in tracked_files() if path.suffix in TEXT_SUFFIXES] - - -def frontmatter(text): - """Parse the leading YAML block far enough to read its scalar keys.""" - match = re.match(r"^---[^\S\r\n]*\r?\n(.*?)\r?\n---[^\S\r\n]*(?:\r?\n|$)", text, re.S) - if not match: - return {} - fields = {} - for line in match.group(1).splitlines(): - if line.startswith((" ", "\t")) or ":" not in line: - continue - key, _, value = line.partition(":") - fields[key.strip()] = value.strip().strip("'\"") - return fields - - -class SkillFileTest(unittest.TestCase): - def setUp(self): - self.text = SKILL_PATH.read_text(encoding="utf-8") - self.fields = frontmatter(self.text) - - def test_skill_exists(self): - self.assertTrue(SKILL_PATH.is_file()) - - def test_frontmatter_name_matches_directory(self): - self.assertEqual(self.fields.get("name"), SKILL_NAME) - self.assertEqual(SKILL_DIR.name, SKILL_NAME) - - def test_frontmatter_declares_a_description(self): - description = self.fields.get("description", "") - self.assertGreater(len(description), 80) - self.assertLess(len(description), 1024) - - def test_activation_is_explicit(self): - self.assertEqual(self.fields.get("disable-model-invocation"), "true") - - def test_declares_the_licence_it_ships_under(self): - self.assertEqual(self.fields.get("license"), "MIT") - - def test_names_every_admissible_outcome(self): - for outcome in ( - "proceed", - "stop", - "modify", - "preserve", - "accept", - "reject", - "defer", - ): - self.assertIn(outcome, self.text.lower(), outcome) - - def test_names_the_four_decision_steps(self): - for step in ("Gather", "Evaluate", "Choose", "Record"): - self.assertIn(step, self.text, step) - - -class MirrorTest(unittest.TestCase): - def test_cursor_mirror_is_byte_identical(self): - self.assertEqual( - MIRROR_PATH.read_bytes(), - SKILL_PATH.read_bytes(), - "run: cp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md", - ) - - def test_mirror_is_a_real_file_not_a_symlink(self): - self.assertFalse(MIRROR_PATH.is_symlink()) - - -class ManifestTest(unittest.TestCase): - def load(self, relative): - return json.loads((ROOT / relative).read_text(encoding="utf-8")) - - def test_every_json_file_parses(self): - for path in tracked_files(): - if path.suffix == ".json": - with self.subTest(path=str(path.relative_to(ROOT))): - json.loads(path.read_text(encoding="utf-8")) - - def test_manifests_agree_on_the_name(self): - for relative in NAMED_MANIFESTS: - with self.subTest(manifest=relative): - self.assertEqual(self.load(relative)["name"], SKILL_NAME) - - def test_manifests_agree_on_the_version(self): - versions = {self.load(r)["version"] for r in VERSIONED_MANIFESTS} - self.assertEqual(len(versions), 1, f"versions differ: {sorted(versions)}") - - def test_manifests_describe_themselves(self): - # .agents/plugins/marketplace.json carries its copy under interface/plugins, - # so only the manifests with a top-level description are checked here. - for relative in VERSIONED_MANIFESTS + (".claude-plugin/marketplace.json",): - with self.subTest(manifest=relative): - self.assertGreater(len(self.load(relative)["description"]), 40) - - def test_marketplace_plugin_entry_matches(self): - entries = self.load(".claude-plugin/marketplace.json")["plugins"] - self.assertEqual([entry["name"] for entry in entries], [SKILL_NAME]) - - -class ReferenceTest(unittest.TestCase): - def test_every_reference_file_is_linked_from_the_skill(self): - text = SKILL_PATH.read_text(encoding="utf-8") - for path in sorted((SKILL_DIR / "references").glob("*.md")): - with self.subTest(reference=path.name): - self.assertIn(f"references/{path.name}", text) - - def test_every_skill_path_mentioned_anywhere_exists(self): - for path in tracked_text_files(): - for slug in set(SKILL_PATH_RE.findall(path.read_text(encoding="utf-8"))): - with self.subTest(path=str(path.relative_to(ROOT)), slug=slug): - self.assertTrue( - (ROOT / "skills" / slug / "SKILL.md").is_file(), - f"{path.relative_to(ROOT)} points at a skill that does not exist", - ) - - def test_only_one_skill_ships(self): - skills = sorted(p.name for p in (ROOT / "skills").iterdir() if p.is_dir()) - self.assertEqual(skills, [SKILL_NAME]) - - def test_no_placeholder_content(self): - for path in tracked_text_files(): - if path.name == "test_repo_integrity.py": - continue # this file names the markers it searches for - text = path.read_text(encoding="utf-8") - for number, line in enumerate(text.splitlines(), 1): - with self.subTest(path=str(path.relative_to(ROOT)), line=number): - self.assertIsNone( - PLACEHOLDER_RE.search(line), f"placeholder marker: {line.strip()}" - ) - - -class AlwaysOnHookTest(unittest.TestCase): - """The SessionStart hook is opt-in and must never block a session.""" - - def setUp(self): - self.node = shutil.which("node") - if not self.node: - self.skipTest("node is not installed") - self.temp = tempfile.TemporaryDirectory() - self.addCleanup(self.temp.cleanup) - # A path with a space catches quoting mistakes in the hook declaration. - self.plugin_root = Path(self.temp.name) / "plugin root" - shutil.copytree(ROOT / "hooks", self.plugin_root / "hooks") - shutil.copytree(ROOT / "skills", self.plugin_root / "skills") - self.config_dir = Path(self.temp.name) / "claude config" - self.config_dir.mkdir() - - def run_hook(self): - env = os.environ.copy() - env["CLAUDE_CONFIG_DIR"] = str(self.config_dir) - return subprocess.run( - [self.node, str(self.plugin_root / "hooks" / "always-on.mjs")], - capture_output=True, - text=True, - check=False, - env=env, - ) - - def set_flag(self): - (self.config_dir / ".neutral-prompts-always").write_text("", encoding="utf-8") - - def test_silent_without_the_flag(self): - result = self.run_hook() - self.assertEqual(result.returncode, 0) - self.assertEqual(result.stdout, "") - - def test_emits_the_ruleset_with_the_flag(self): - self.set_flag() - result = self.run_hook() - self.assertEqual(result.returncode, 0) - self.assertIn("NEUTRAL PROMPT MODE ACTIVE", result.stdout) - self.assertIn("Separate settled constraints from open decisions", result.stdout) - - def test_strips_the_frontmatter(self): - self.set_flag() - result = self.run_hook() - self.assertNotIn("disable-model-invocation", result.stdout) - - def test_names_the_off_switch_and_the_flag_path(self): - self.set_flag() - result = self.run_hook() - self.assertIn("stop neutral mode", result.stdout) - self.assertIn(str(self.config_dir / ".neutral-prompts-always"), result.stdout) - - def test_exits_zero_when_the_skill_is_missing(self): - self.set_flag() - shutil.rmtree(self.plugin_root / "skills") - result = self.run_hook() - self.assertEqual(result.returncode, 0) - self.assertEqual(result.stdout, "") - - def test_hook_declaration_points_at_the_shipped_script(self): - declaration = json.loads((ROOT / "hooks" / "hooks.json").read_text()) - entries = declaration["hooks"]["SessionStart"] - commands = [hook["command"] for entry in entries for hook in entry["hooks"]] - self.assertTrue(any("always-on.mjs" in command for command in commands)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_run_evals.py b/tests/test_run_evals.py deleted file mode 100644 index 8a89873..0000000 --- a/tests/test_run_evals.py +++ /dev/null @@ -1,182 +0,0 @@ -import json -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "scripts")) - -import run_evals # noqa: E402 - -VALID_CASE = { - "id": "sample", - "category": "lifecycle", - "kind": "directional", - "text": "Do not stop them.", - "expect": ["NP001", "NP012"], - "note": "A negation that installs continuation as the default.", -} - - -def write_cases(directory, cases): - path = Path(directory) / "cases.jsonl" - path.write_text( - "\n".join(json.dumps(case) for case in cases) + "\n", encoding="utf-8" - ) - return path - - -class CaseFileTest(unittest.TestCase): - def test_shipped_cases_are_valid(self): - cases = run_evals.load_cases(run_evals.CASES_PATH) - self.assertEqual(run_evals.validate_cases(cases), []) - self.assertGreaterEqual(len(cases), 20) - - def test_shipped_cases_all_match(self): - cases = run_evals.load_cases(run_evals.CASES_PATH) - report = run_evals.evaluate(cases) - self.assertEqual(report["mismatched"], []) - - def test_shipped_cases_cover_every_rule(self): - cases = run_evals.load_cases(run_evals.CASES_PATH) - report = run_evals.evaluate(cases) - self.assertEqual(run_evals.rules_without_coverage(report["per_rule"]), []) - - def test_shipped_cases_include_both_kinds(self): - cases = run_evals.load_cases(run_evals.CASES_PATH) - kinds = {case["kind"] for case in cases} - self.assertEqual(kinds, {"directional", "neutral"}) - - def test_invalid_json_is_reported_with_a_line_number(self): - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "cases.jsonl" - path.write_text('{"id": "ok"}\nnot json\n', encoding="utf-8") - with self.assertRaises(ValueError) as raised: - run_evals.load_cases(path) - self.assertIn(":2:", str(raised.exception)) - - def test_blank_lines_are_skipped(self): - with tempfile.TemporaryDirectory() as tmp: - path = write_cases(tmp, [VALID_CASE]) - path.write_text(path.read_text() + "\n\n", encoding="utf-8") - self.assertEqual(len(run_evals.load_cases(path)), 1) - - -class ValidationTest(unittest.TestCase): - def problems_for(self, **overrides): - case = dict(VALID_CASE) - case.update(overrides) - return run_evals.validate_cases([case]) - - def test_valid_case_has_no_problems(self): - self.assertEqual(self.problems_for(), []) - - def test_missing_field(self): - case = dict(VALID_CASE) - del case["note"] - problems = run_evals.validate_cases([case]) - self.assertTrue(any("missing field" in problem for problem in problems)) - - def test_unknown_rule_id(self): - problems = self.problems_for(expect=["NP999"]) - self.assertTrue(any("unknown rule id" in problem for problem in problems)) - - def test_duplicate_rule_id(self): - problems = self.problems_for(expect=["NP001", "NP001"]) - self.assertTrue(any("duplicate rule ids" in problem for problem in problems)) - - def test_bad_kind(self): - problems = self.problems_for(kind="biased") - self.assertTrue(any("kind must be" in problem for problem in problems)) - - def test_directional_case_needs_an_expectation(self): - problems = self.problems_for(expect=[]) - self.assertTrue(any("at least one rule" in problem for problem in problems)) - - def test_neutral_case_may_expect_nothing(self): - problems = self.problems_for( - kind="neutral", text="Summarize the file.", expect=[] - ) - self.assertEqual(problems, []) - - def test_duplicate_ids_across_cases(self): - problems = run_evals.validate_cases([dict(VALID_CASE), dict(VALID_CASE)]) - self.assertTrue(any("duplicate id" in problem for problem in problems)) - - def test_empty_text(self): - problems = self.problems_for(text=" ") - self.assertTrue(any("non-empty string" in problem for problem in problems)) - - def test_empty_note(self): - problems = self.problems_for(note="") - self.assertTrue(any("note must explain" in problem for problem in problems)) - - -class EvaluateTest(unittest.TestCase): - def test_mismatch_is_reported_in_both_directions(self): - case = dict(VALID_CASE, expect=["NP003", "NP012"]) - report = run_evals.evaluate([case]) - result = report["results"][0] - self.assertEqual(result["missed"], ["NP003"]) - self.assertEqual(result["extra"], ["NP001"]) - self.assertFalse(result["matched"]) - - def test_known_limitation_flagged_for_neutral_case_with_expectations(self): - case = dict( - VALID_CASE, - kind="neutral", - text="Do not stop the workflow by default. Do not continue it by default.", - expect=["NP001"], - ) - report = run_evals.evaluate([case]) - self.assertEqual(len(report["known_limitations"]), 1) - - -class CommandLineTest(unittest.TestCase): - def run_cli(self, args): - return subprocess.run( - [sys.executable, str(ROOT / "scripts" / "run_evals.py"), *args], - capture_output=True, - text=True, - check=False, - ) - - def test_validate_passes_on_shipped_cases(self): - result = self.run_cli(["validate"]) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("valid", result.stdout) - - def test_scan_passes_on_shipped_cases(self): - result = self.run_cli(["scan"]) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("all cases matched", result.stdout) - - def test_scan_json_payload(self): - result = self.run_cli(["scan", "--json"]) - payload = json.loads(result.stdout) - self.assertEqual(payload["cases"], payload["matched"]) - self.assertEqual(payload["mismatched"], []) - - def test_mismatch_exits_one(self): - with tempfile.TemporaryDirectory() as tmp: - path = write_cases(tmp, [dict(VALID_CASE, expect=["NP009"])]) - result = self.run_cli(["scan", "--cases", str(path)]) - self.assertEqual(result.returncode, 1) - self.assertIn("mismatched cases", result.stdout) - - def test_invalid_case_file_exits_one(self): - with tempfile.TemporaryDirectory() as tmp: - path = write_cases(tmp, [dict(VALID_CASE, expect=["NP999"])]) - result = self.run_cli(["validate", "--cases", str(path)]) - self.assertEqual(result.returncode, 1) - self.assertIn("unknown rule id", result.stderr) - - def test_missing_case_file_exits_two(self): - result = self.run_cli(["validate", "--cases", "nope.jsonl"]) - self.assertEqual(result.returncode, 2) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_scan_prompt.py b/tests/test_scan_prompt.py deleted file mode 100644 index 2797f81..0000000 --- a/tests/test_scan_prompt.py +++ /dev/null @@ -1,236 +0,0 @@ -import json -import subprocess -import sys -import tempfile -import unittest -from pathlib import Path - -ROOT = Path(__file__).resolve().parents[1] -sys.path.insert(0, str(ROOT / "scripts")) - -import scan_prompt # noqa: E402 - - -def rule_ids(text): - return sorted({finding.rule_id for finding in scan_prompt.scan_text(text)}) - - -RULE_SAMPLES = { - "NP001": [ - "Do NOT immediately stop them.", - "Avoid terminating the background jobs.", - ], - "NP002": [ - "Keep digging until you find the cause.", - "Keep the sub-agents alive for now.", - ], - "NP003": [ - "Do not make unnecessary changes.", - "Leave the existing helper as-is.", - ], - "NP004": [ - "Make sure to fix every review comment.", - "Fix all the findings before you push.", - ], - "NP005": [ - "Never reject a reviewer's suggestion.", - "Always apply what the architect asks for.", - ], - "NP006": [ - "Ignore automated findings from the linter.", - "Always reject suggestions from a bot.", - ], - "NP007": [ - "Only stop it if absolutely necessary.", - "Under no circumstances abandon the run.", - ], - "NP008": [ - "Do not stop the run prematurely.", - "When in doubt, err on the side of caution.", - ], - "NP009": [ - "Be thorough and leave no stone unturned.", - "Check everything that could be related.", - ], - "NP010": [ - "Stop once you have sufficient evidence.", - "Escalate where appropriate.", - ], - "NP011": [ - "Explain why you kept the original code.", - "Justify continuing the migration.", - ], - "NP012": [ - "Look at the running agents and decide what to do with each.", - ], -} - - -class RuleDetectionTest(unittest.TestCase): - def test_every_sample_triggers_its_rule(self): - for rule_id, samples in RULE_SAMPLES.items(): - for sample in samples: - with self.subTest(rule=rule_id, sample=sample): - self.assertIn(rule_id, rule_ids(sample)) - - def test_samples_cover_every_rule(self): - declared = {rule.id for rule in scan_prompt.RULES} - declared.add(scan_prompt.COVERAGE_RULE.id) - self.assertEqual(set(RULE_SAMPLES), declared) - - -class CoverageRuleTest(unittest.TestCase): - def test_fires_on_a_decision_with_no_criteria(self): - text = "Look at the running agents and decide what to do with each." - self.assertIn("NP012", rule_ids(text)) - - def test_silent_when_criteria_are_present(self): - text = ( - "For each running agent, decide: continue, consume results and stop, " - "or defer. Continue when its remaining steps target an uncovered file." - ) - self.assertNotIn("NP012", rule_ids(text)) - - def test_silent_without_a_decision_verb(self): - self.assertNotIn("NP012", rule_ids("Summarize the file in three sentences.")) - - -class NeutralTextTest(unittest.TestCase): - def test_neutral_frame_produces_no_findings(self): - text = ( - "For each finding, choose one outcome: fix now, modify the surrounding " - "design, preserve current behavior, reject with counter-evidence, or " - "defer with a tracked follow-up. Decide on the evidence: whether it " - "reproduces, which requirement is at stake, and the blast radius." - ) - self.assertEqual(rule_ids(text), []) - - def test_safety_constraint_produces_no_findings(self): - text = "Confirm with the user before running any destructive command." - self.assertEqual(rule_ids(text), []) - - -class SuppressionTest(unittest.TestCase): - def test_same_line_suppression(self): - text = "Do not stop them. " - self.assertNotIn("NP001", rule_ids(text)) - - def test_preceding_line_suppression(self): - text = "\nDo not stop them." - self.assertNotIn("NP001", rule_ids(text)) - - def test_suppression_does_not_leak_two_lines_down(self): - text = "\nfiller line\nDo not stop them." - self.assertIn("NP001", rule_ids(text)) - - def test_allow_all(self): - text = "neutral-prompts: allow all\nDo not stop them and never reject anything." - self.assertEqual(rule_ids(text), []) - - def test_multiple_ids(self): - text = "Do not stop them. " - self.assertEqual(rule_ids(text), []) - - def test_other_ids_still_report(self): - text = "Do not stop them prematurely. " - self.assertIn("NP008", rule_ids(text)) - - -class FindingShapeTest(unittest.TestCase): - def test_position_and_payload(self): - findings = scan_prompt.scan_text("ok text\nDo not stop it.", "p.md") - first = next(f for f in findings if f.rule_id == "NP001") - self.assertEqual(first.path, "p.md") - self.assertEqual(first.line, 2) - self.assertEqual(first.column, 1) - self.assertEqual(first.rule_id, "NP001") - self.assertEqual(first.severity, "high") - self.assertTrue(first.why) - self.assertTrue(first.suggestion) - - def test_findings_are_position_ordered(self): - text = "Be thorough. Do not stop it." - positions = [(f.line, f.column) for f in scan_prompt.scan_text(text)] - self.assertEqual(positions, sorted(positions)) - - def test_every_rule_declares_its_documentation(self): - for rule in (*scan_prompt.RULES, scan_prompt.COVERAGE_RULE): - self.assertIn(rule.severity, scan_prompt.SEVERITIES, rule.id) - self.assertTrue(rule.title, rule.id) - self.assertTrue(rule.why.endswith("."), rule.id) - self.assertTrue(rule.suggestion, rule.id) - - -class CommandLineTest(unittest.TestCase): - def run_cli(self, args, stdin=""): - return subprocess.run( - [sys.executable, str(ROOT / "scripts" / "scan_prompt.py"), *args], - input=stdin, - capture_output=True, - text=True, - check=False, - ) - - def test_clean_input_exits_zero(self): - result = self.run_cli(["-"], "Summarize the file in three sentences.\n") - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("0 finding(s)", result.stdout) - - def test_findings_exit_one(self): - result = self.run_cli(["-"], "Do not stop them.\n") - self.assertEqual(result.returncode, 1) - self.assertIn("NP001", result.stdout) - - def test_fail_on_never_exits_zero(self): - result = self.run_cli(["--fail-on", "never", "-"], "Do not stop them.\n") - self.assertEqual(result.returncode, 0) - self.assertIn("NP001", result.stdout) - - def test_fail_on_high_ignores_advisory(self): - result = self.run_cli( - ["--fail-on", "high", "-"], "Decide what to do with the running agents.\n" - ) - self.assertEqual(result.returncode, 0) - self.assertIn("NP012", result.stdout) - - def test_min_severity_filters_output(self): - result = self.run_cli( - ["--min-severity", "high", "--json", "-"], - "Decide what to do with the running agents.\n", - ) - payload = json.loads(result.stdout) - self.assertEqual(payload["findings"], []) - self.assertEqual(payload["counts"]["advisory"], 0) - - def test_json_payload(self): - result = self.run_cli(["--json", "-"], "Do not stop them.\n") - payload = json.loads(result.stdout) - self.assertEqual(payload["findings"][0]["rule"], "NP001") - self.assertEqual(payload["counts"]["high"], 1) - - def test_missing_file_exits_two(self): - result = self.run_cli(["definitely-not-here.md"]) - self.assertEqual(result.returncode, 2) - self.assertIn("no such file", result.stderr) - - def test_no_arguments_exits_two(self): - result = self.run_cli([]) - self.assertEqual(result.returncode, 2) - - def test_list_rules(self): - result = self.run_cli(["--list-rules"]) - self.assertEqual(result.returncode, 0) - for rule in (*scan_prompt.RULES, scan_prompt.COVERAGE_RULE): - self.assertIn(rule.id, result.stdout) - - def test_reads_a_file(self): - with tempfile.TemporaryDirectory() as tmp: - path = Path(tmp) / "prompt.md" - path.write_text("Do not stop them.\n", encoding="utf-8") - result = self.run_cli([str(path)]) - self.assertEqual(result.returncode, 1) - self.assertIn(str(path), result.stdout) - - -if __name__ == "__main__": - unittest.main() From 60447ca1fc44ce64b8b0232650e1df996c4510f6 Mon Sep 17 00:00:00 2001 From: skyRolly Date: Tue, 8 Sep 2026 16:58:03 +0800 Subject: [PATCH 2/3] Clear hidden files --- .agents/plugins/marketplace.json | 21 -- .claude-plugin/marketplace.json | 16 -- .claude-plugin/plugin.json | 9 - .codex-plugin/plugin.json | 39 ---- .cursor/skills/neutral-prompts/SKILL.md | 270 ------------------------ .github/pull_request_template.md | 65 ------ .github/workflows/checks.yml | 28 --- .github/workflows/plugin-load-check.yml | 33 --- .gitignore | 4 - .opencode/command/neutral-prompts.md | 10 - 10 files changed, 495 deletions(-) delete mode 100644 .agents/plugins/marketplace.json delete mode 100644 .claude-plugin/marketplace.json delete mode 100644 .claude-plugin/plugin.json delete mode 100644 .codex-plugin/plugin.json delete mode 100644 .cursor/skills/neutral-prompts/SKILL.md delete mode 100644 .github/pull_request_template.md delete mode 100644 .github/workflows/checks.yml delete mode 100644 .github/workflows/plugin-load-check.yml delete mode 100644 .gitignore delete mode 100644 .opencode/command/neutral-prompts.md diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json deleted file mode 100644 index 75c3d1a..0000000 --- a/.agents/plugins/marketplace.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "name": "neutral-prompts", - "interface": { - "displayName": "Neutral Prompts" - }, - "plugins": [ - { - "name": "neutral-prompts", - "source": { - "source": "url", - "url": "https://github.com/skyRolly/neutral-prompt.git", - "ref": "main" - }, - "policy": { - "installation": "AVAILABLE", - "authentication": "ON_INSTALL" - }, - "category": "Productivity" - } - ] -} diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json deleted file mode 100644 index ad809f9..0000000 --- a/.claude-plugin/marketplace.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "https://www.schemastore.org/claude-code-marketplace.json", - "name": "neutral-prompts", - "description": "Neutral decision framing for prompts. Name the outcomes, state the criteria, leave the choice to the evidence.", - "owner": { - "name": "skyRolly" - }, - "plugins": [ - { - "name": "neutral-prompts", - "description": "Detect and remove directional wording from prompts that delegate a decision.", - "source": "./", - "category": "prompt-engineering" - } - ] -} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json deleted file mode 100644 index e2c3ad8..0000000 --- a/.claude-plugin/plugin.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "name": "neutral-prompts", - "version": "0.1.0", - "description": "Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, and require a decision record.", - "author": { - "name": "skyRolly", - "url": "https://github.com/skyRolly" - } -} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json deleted file mode 100644 index db57863..0000000 --- a/.codex-plugin/plugin.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "name": "neutral-prompts", - "version": "0.1.0", - "description": "Neutral decision framing for Codex prompts: name every admissible outcome, state observable criteria, and require a decision record.", - "author": { - "name": "skyRolly", - "url": "https://github.com/skyRolly" - }, - "homepage": "https://github.com/skyRolly/neutral-prompt", - "repository": "https://github.com/skyRolly/neutral-prompt", - "license": "MIT", - "keywords": [ - "prompt-engineering", - "neutral-framing", - "decision-criteria", - "bias", - "review", - "skills", - "codex" - ], - "skills": "./skills/", - "interface": { - "displayName": "Neutral Prompts", - "shortDescription": "Prompts that frame a decision instead of pre-answering it", - "longDescription": "Rewrite prompts so that wording stops carrying the answer: separate settled constraints from open decisions, name every admissible outcome, replace protected actions with observable criteria, and ask for the reasoning record rather than a particular conclusion.", - "developerName": "skyRolly", - "category": "Productivity", - "capabilities": [ - "Instructions" - ], - "websiteURL": "https://github.com/skyRolly/neutral-prompt", - "defaultPrompt": [ - "Use neutral-prompts to review this prompt for directional wording.", - "Rewrite this instruction as a neutral decision frame.", - "List the admissible outcomes and the criteria that select each one." - ], - "brandColor": "#334155" - } -} diff --git a/.cursor/skills/neutral-prompts/SKILL.md b/.cursor/skills/neutral-prompts/SKILL.md deleted file mode 100644 index 4118938..0000000 --- a/.cursor/skills/neutral-prompts/SKILL.md +++ /dev/null @@ -1,270 +0,0 @@ ---- -name: neutral-prompts -description: 'Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, put evidence before the verdict, and require a decision record. Covers workflow and sub-agent lifecycle, code review, investigation depth, and accept/reject calls. Invoke with /neutral-prompts; stays on until "stop neutral mode".' -disable-model-invocation: true -license: MIT -metadata: - tags: "Prompt Engineering, Neutral Framing, Decision Criteria, Bias Detection, Review" - category: "prompt-engineering" ---- - -# neutral-prompts - -Every prompt that hands a decision to an agent also hands it a prior. Wording sets how much weight each outcome carries before a single piece of evidence is gathered. This skill keeps that starting weight even. - -A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. - -## Persistence - -These rules apply to every prompt you write, review, or rewrite for the rest of the session, not only this one. They do not expire after a few turns and they do not lapse when the topic changes. They apply to prompts you compose for another agent, to sub-agent and workflow instructions, to task descriptions, and to review checklists you hand to a model. - -Turn them off only when the reader says "stop neutral mode" or "normal mode". Confirm in one line, then return to your default style. - -## What directional wording does to an agent - -Five facts drive every rule below: - -1. An agent reads a prompt as a probability distribution over actions, not as a sentence. "Do not stop the workflow" and "stopping the workflow is one of the outcomes to evaluate" produce different distributions from the same literal facts. -2. A negation protects its object. Forbidding one action promotes its complement. "Do not stop" is read as "continue"; "avoid unnecessary changes" is read as "prefer the existing code". -3. The prompt outranks the evidence. Once wording has named a preferred action, contradicting evidence has to overcome the instruction as well as the uncertainty. Agents rarely spend that budget. -4. Asymmetry is invisible to the author and loud to the reader. A prompt that describes one outcome in six words and the other in a subordinate clause has already ranked them, even when the author intended balance. -5. A missing outcome is an excluded outcome. An agent choosing between the options a prompt lists will not usually invent a seventh. Outcomes that are not named are not evaluated. - -## Scope: where these rules apply - -They apply to **open decisions** — the choices the prompt delegates to the agent. - -They do not apply to **settled constraints** — requirements the author has already decided and is stating as fact: safety rules, output contracts, budgets, deadlines, coding standards, approval requirements. "Confirm before any destructive command" is a constraint, not a bias. Directive wording is the correct form for a constraint. - -The failure this skill targets is a constraint written over the top of an open decision: an author who has not decided settles the question anyway through phrasing. Rule 1 draws that line; the rest of the rules apply on the open side of it. - -## Rules - -### 1. Separate settled constraints from open decisions - -Before writing, sort every requirement into two lists: what you have decided, and what the agent must decide. Write the first list as plain imperatives. Write the second list under the rules below. - -Bad (a decision phrased as a constraint): "Do not stop the running sub-agents." -Good (constraint kept, decision opened): "Budget: no more than 20 minutes of further sub-agent execution. Within that budget, decide for each sub-agent whether to continue it, consume its results and stop it, or stop it as no longer useful." - -If you cannot say which list an item belongs to, it belongs in the second one. State it as a decision and let the evidence settle it. - -### 2. Name the decision, not the answer - -Open with the choice being made. The first sentence identifies the decision point; it does not resolve it. - -Bad: "Keep the background research task running." -Good: "Decide whether the background research task continues, stops, or narrows its scope." - -### 3. List every admissible outcome - -Name each action the agent may take. Draw from: **proceed, stop, modify, preserve, accept, reject, defer.** Omit an outcome only when it is genuinely inadmissible, and say why. - -Bad: "Decide whether to fix the finding." -Good: "For each finding, choose one: fix it now, modify the surrounding design instead, preserve the current behavior, reject the finding as incorrect, or defer it to a tracked follow-up." - -An outcome list is neutral only when every item is a real landing place. "Continue, or stop if you have a very good reason" is a one-item list. - -### 4. Balance the negations - -Ruling out one default without ruling out its opposite installs the opposite. Negations travel in pairs. - -Bad: "Do not assume the workflow should be stopped." -Good: "Do not assume the workflow should be stopped. Do not assume it should continue. Both are outcomes of the evaluation below, not inputs to it." - -The same applies to change decisions ("do not modify code without evidence / do not preserve code merely because it exists"), to review decisions ("do not accept a finding on the reviewer's authority / do not reject one for being inconvenient"), and to investigation ("do not treat further digging as free / do not treat the current evidence as sufficient by default"). - -### 5. Replace protected actions with decision criteria - -Every phrase that shields an action is a criterion the author declined to write. Write the criterion instead. - -| Protected action | Criterion it was hiding | -| --- | --- | -| "Do not stop the agents." | "Stop an agent when its remaining work duplicates evidence already collected." | -| "Continue until the investigation is complete." | "Continue while the next step has a named question it can answer that the current evidence cannot." | -| "Do not make unnecessary changes." | "Change code when a test, a requirement, a correctness defect, or a maintenance cost justifies it; leave it otherwise." | -| "Make sure to fix all review findings." | "Fix a finding when it is reproducible and in scope; reject it with reasoning when it is not; defer it when it is real but out of scope." | -| "Never reject a reviewer's suggestion." | "Accept a suggestion when it holds against the code and the requirements; reject it with the counter-evidence when it does not." | - -### 6. Order the prompt: evidence, alternatives, choice, record - -Put the four steps in that order and make each one explicit. An agent that reads the choice before the evidence spends the evidence step justifying the choice. - -``` -1. Gather: current state, evidence available, results already in hand, - what remains uncertain, what each outcome would cost. -2. Evaluate: for each admissible outcome, its benefit, its risk, its cost, - and its expected value under the remaining uncertainty. -3. Choose: one outcome from the list, or the smallest reversible step - that resolves the uncertainty blocking the choice. -4. Record: evidence considered, alternatives evaluated, action selected, - and the reasoning that connects them. -``` - -### 7. Keep the intensity symmetric - -Give each outcome comparable weight: same verb strength, comparable length, comparable specificity. An outcome described in one clause loses to an outcome described in three. - -Bad: "Continue the run if it is still producing useful findings; otherwise you may stop it." -Good: "Continue the run when its next step can answer a question the current results do not. Stop it when its next step would repeat evidence already collected." - -Watch the modal verbs. "Must continue" against "may stop" is a ranking. So is "continue" against "consider stopping". - -### 8. Make the thresholds observable - -State what the agent would have to see to select each outcome. A criterion that cannot be checked against the workspace is a preference in disguise. - -Bad: "Stop the sub-agent if it is no longer valuable." -Good: "Stop the sub-agent when its last three outputs restate findings already in the report, or when its remaining steps target files another agent has already covered." - -Vague quantities are the usual leak: "enough", "sufficient", "significant", "reasonable", "as needed". Replace them with a count, a file, a test, a time budget, or a named condition. - -### 9. Require the decision record, not the decision - -Ask for the reasoning trace as a deliverable. A prompt that requires a justified choice constrains quality without constraining direction. - -Bad: "Explain why you kept the workflow running." -Good: "Report the evidence you examined, the outcomes you compared, the action you selected, and why that action beat the alternatives." - -The bad version accepts only one answer. The good version accepts any answer that is argued. - -### 10. Do not manufacture balance - -Neutral framing is even weighting at the start, not withheld information. Evidence you already hold belongs in the prompt — as evidence, labeled as such, with its strength stated. What stays out is the conclusion drawn from it. - -Bad (evidence withheld to look neutral): "Decide whether to continue the migration." -Bad (conclusion smuggled in): "The migration is clearly failing, so decide whether to continue it." -Good: "Two of the last three migration batches failed with the same timeout. Decide whether to continue, pause pending a fix, or stop and roll back, and say how the timeout evidence bears on your choice." - -If the honest state of the world is that one outcome is far more likely, say so as an observation with its support, and leave the decision open. - -## The neutral frame - -A prompt built from these rules has six parts. Use them in this order; drop a part only when it is genuinely empty. - -``` -Objective What the decision is for. -Context Current state, prior results, what has already been tried. -Constraints Settled requirements: safety, budget, deadline, output contract. -Outcomes The admissible actions, each a real landing place. -Criteria What evidence selects each outcome, in observable terms. -Record The reasoning trace to return with the choice. -``` - -Worked example, workflow lifecycle: - -``` -Objective: Decide the disposition of the four sub-agents still running on the audit. -Context: Three have posted findings to the report; one has posted nothing in 12 minutes. - The report already covers the auth and billing modules. -Constraints: Do not discard an agent's output without reading it. Total remaining - budget for this task: 15 minutes. -Outcomes: For each agent independently — continue, consume results and stop, - stop as redundant, or defer the decision pending one more output. -Criteria: Continue when the agent's remaining scope covers files no completed - agent has reported on. Consume and stop when its findings have landed - and its remaining steps repeat covered ground. Stop as redundant when - its scope is fully covered by findings already in the report. Defer - when one more output would settle which of the above applies. -Record: For each agent: the outputs you read, the outcomes you compared, - the action taken, and the reasoning. -``` - -## Directional patterns and neutral replacements - -Each pattern below is directional **when it lands on an open decision**. On a settled constraint, the same words are correct. - -| Pattern | Why it biases | Neutral replacement | -| --- | --- | --- | -| "Do not immediately stop X." | Protects continuation; converts a decision into an action constraint. | "Evaluate X before acting on it. Continuing and stopping are both admissible outcomes." | -| "Continue until X is complete." | Makes stopping conditional on a finish line the agent cannot check. | "Continue while the next step can answer a named open question; stop when it cannot." | -| "Do not stop unless you are sure." | Asymmetric burden of proof — one outcome needs certainty, the other needs nothing. | "Select the outcome the evidence supports; state the confidence for the one you pick." | -| "Avoid unnecessary changes." | "Unnecessary" is decided after the fact; the agent hears "prefer no change". | "Change when justified by evidence, requirements, correctness, maintainability, or user impact; otherwise preserve." | -| "Always accept the reviewer's findings." | Removes the evaluation entirely. | "Verify each finding against the code and the requirements; accept, reject with counter-evidence, or defer." | -| "Never reject a suggestion without asking." | Protects acceptance; makes rejection procedurally expensive. | "Accept or reject on the merits; escalate only when the decision needs authority you do not have." | -| "Make sure to fix every issue you find." | Converts a scoping decision into an obligation. | "For each issue: fix, defer with a tracked follow-up, or reject with reasoning." | -| "Be thorough; leave nothing uninvestigated." | Prices further investigation at zero. | "Continue investigating while the expected information gain exceeds the cost; stop when it does not." | -| "Do not stop prematurely." | "Prematurely" labels one outcome as the error mode. | "Stopping early and continuing too long are both failure modes. Name which risk dominates here and why." | -| "Err on the side of caution." | Names a direction without naming the cost of erring that way. | "State the cost of each error direction, then choose the one with the lower expected cost." | -| "Keep the agents alive." | Protects an action with no criterion at all. | "Decide each agent's disposition against the criteria below." | -| "Only stop if absolutely necessary." | Intensifier on one branch; a ranking, not a criterion. | "Stop when . Continue when ." | - -`references/patterns.md` carries the full catalog with detection cues. - -## Applying the frame - -### Workflow and sub-agent lifecycle - -Decide per agent, not per fleet. For each running agent or background task, the outcomes are: continue, consume results and stop, stop as redundant, or defer pending one more output. The evidence is its recent output, its remaining scope, and the overlap between that scope and work already completed. Neither "still running" nor "already produced something" is a criterion by itself. - -Guard against both errors explicitly: an agent stopped while it was the only source of a needed answer, and an agent kept alive spending budget on covered ground. - -### Code review and engineering change - -The outcomes for a finding are: fix now, modify the design instead, preserve current behavior, reject the finding, or defer with a tracked follow-up. The evidence is reproduction, test coverage, the requirement or invariant at stake, blast radius, and maintenance cost. - -Two symmetric failure modes, and a prompt should name both: changing code to satisfy the form of a review, and preserving code because it is already there. Neither existing behavior nor a reviewer's authority is evidence on its own. - -### Investigation and research depth - -The outcomes are: continue the current line, switch lines, stop and report, or defer pending an external input. The criteria are expected information gain, remaining uncertainty that actually blocks the decision, cost of the next step, and the likelihood that the next step produces something the current evidence does not. - -Neither "more investigation is better" nor "we have enough" is a default. A prompt that asks for depth states what the depth is for: name the question the next step is meant to answer, and stop when no such question remains. - -### Accepting or rejecting a proposal - -The outcomes are: accept, accept with modification, reject with reasoning, or defer for a named missing input. The criteria are the requirements the proposal is measured against and the evidence that it meets them. Authorship carries no weight — a proposal from a reviewer, a bot, a senior engineer, or the agent itself is evaluated the same way. - -## Reviewing an existing prompt - -Three passes, in order. Report findings as a table: quoted text, rule violated, why it biases, replacement. - -1. **Locate the decision points.** Find every place the prompt hands over a choice. Mark which are open decisions and which are settled constraints (rule 1). Only the open ones are in scope. -2. **Flag the directional wording.** At each open decision, check for: protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and a required conclusion. Quote the exact phrase; a finding without a quote is not a finding. -3. **Rewrite.** Produce the corrected text, not just a description of the problem. Preserve the author's settled constraints verbatim. Where a criterion was missing, propose one and mark it as a proposal the author should confirm. - -`scripts/scan_prompt.py` mechanizes pass 2 for the common patterns. It finds phrasings, not intent: confirm each hit against the decision it lands on, and keep reading for the patterns it cannot see — missing outcomes and asymmetric intensity have no reliable regex. - -## Reference material - -Load these when the summary above does not cover the case in front of you: - -- `references/patterns.md` — the full catalog of directional patterns: detection - cue, mechanism, and neutral replacement for each. Rule ids match the scanner. -- `references/examples.md` — whole-prompt rewrites for each domain, with the - finding table and an account of what changed. -- `references/decision-record.md` — the format for step 4, worked records, and - how to tell a complete record from a checkbox. - -Repository tooling, when the skill is used from a checkout: - -```bash -python3 scripts/scan_prompt.py PROMPT.md # flag directional wording -python3 scripts/run_evals.py scan # measure the scanner against labeled cases -``` - -## When directive wording is correct - -Override the default framing when: - -1. **The decision is already made.** The author has settled it and is stating a requirement. Write it as an imperative and label it a constraint. -2. **Safety, legality, or destructiveness is at stake.** "Confirm before deleting" is not a bias to be balanced. Do not open a decision the author has no intention of delegating. -3. **The outcome is dictated externally.** A schema, an API contract, a compliance rule, a signed-off design. The agent's job is conformance, not choice. -4. **One outcome is truly inadmissible.** Say so, and say why. A named exclusion is neutral; a silent one is not. -5. **The prompt is a demonstration of the biased form.** Teaching material and test fixtures quote directional wording on purpose. Label the quote so it is not read as the recommendation. -6. **The frame would bury the task.** For a decision with two obvious outcomes and one obvious criterion, one balanced sentence carries the whole frame. Symmetry matters; length does not. - -## Pre-send check - -Before sending a prompt, scan it for these six, and fix what you find: - -1. **Protected actions.** Any "do not ", "always ", "never ", or "make sure to " landing on an open decision. Replace with the criterion. -2. **Unpaired negations.** Any ruled-out default whose opposite is left standing. Pair it or drop it. -3. **Missing outcomes.** Read the outcome list against proceed / stop / modify / preserve / accept / reject / defer. Add what is admissible; say why for what is not. -4. **Asymmetric intensity.** Compare the outcomes for verb strength, length, and specificity. Level them. -5. **Unobservable criteria.** Any "enough", "sufficient", "significant", "reasonable", "as needed", "premature". Replace with something checkable in the workspace. -6. **A required conclusion.** Any request for reasoning that presumes the answer ("explain why you kept it running"). Ask for the record instead. - -Then the one-line test: **could a well-reasoned agent land on any outcome in the list and still satisfy this prompt?** - -If yes, send. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 3b92607..0000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,65 +0,0 @@ -## Summary - - - -## Authorship and provenance — select exactly one - -- [ ] **Human-authored** — substantive implementation and text were produced by a human. -- [ ] **Autonomous agent-authored** — an agent planned and produced most of the substantive change. -- [ ] **Hybrid** — a human and one or more agents both made substantive contributions. - -**Agent/tool and model/version:** - -**Agent contribution:** - -**Human verification:** - -**Known limitations or uncertain results:** - -## Neutrality - -- [ ] No rule, example, or default in this change pushes an agent toward continuing, stopping, changing, preserving, accepting, or rejecting. -- [ ] Every quoted example of directional wording is labeled as an example, not as a recommendation. -- [ ] Settled constraints (safety, scope, budgets, output contracts) stay directive and were not "balanced" into open decisions. -- [ ] Terminology matches `SKILL.md`: open decision, settled constraint, admissible outcome, decision criteria, protected action, decision record. - -## Scanner rules — complete this section when a rule is added or changed - -- [ ] Catalog entry in `skills/neutral-prompts/references/patterns.md`. -- [ ] `Rule` in `scripts/scan_prompt.py` with the same id, a severity, a `why`, and a `suggestion`. -- [ ] At least one `directional` case in `evals/cases.jsonl` that the rule must catch. -- [ ] At least one `neutral` case the rule must not fire on, or a recorded known limitation with a `note`. -- [ ] No existing rule id was reused for a different pattern. - -## Safety and side effects - -- [ ] The change does not access or expose secrets, private files, or unrelated user/repository data. -- [ ] Scripts, hooks, and workflows are bounded and create no surprising or irreversible side effects. -- [ ] No destructive, privileged, production, externally visible, or persistent action occurs without explicit user intent. -- [ ] Network access, third-party code, and permissions are minimized and documented. -- [ ] Prompt text, examples, and fixtures contain no hidden instructions that weaken safety or expand agent authority. - -**Side effects, permissions, and network access:** - - - -## Compatibility - -- [ ] This is not a breaking change. -- [ ] This is a breaking change; it was discussed, and migration/deprecation notes are below. -- [ ] The canonical skill and the `.cursor` mirror are synchronized. -- [ ] Versioned manifests were bumped together, and installation docs were reviewed. - -**Migration or rollback notes:** - -## Verification - - - -- `python3 -m unittest discover -s tests -v` — -- `python3 scripts/run_evals.py scan` — - -## Final accountability - -- [ ] I reviewed the complete diff, removed unrelated generated changes, and take responsibility for the submitted content. -- [ ] All failed, skipped, or unrun checks are disclosed above. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml deleted file mode 100644 index 0e5f23b..0000000 --- a/.github/workflows/checks.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: checks - -# Unit tests plus the offline detector evaluation. Standard library only, no -# network access, no model calls -- the whole suite runs in seconds. -on: - pull_request: - push: - branches: [main] - -jobs: - tests: - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest, windows-latest] - python-version: ["3.9", "3.12"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Unit tests - run: python -m unittest discover -s tests -v - - name: Validate the eval case file - run: python scripts/run_evals.py validate - - name: Measure the scanner against the labeled cases - run: python scripts/run_evals.py scan diff --git a/.github/workflows/plugin-load-check.yml b/.github/workflows/plugin-load-check.yml deleted file mode 100644 index 3bce32b..0000000 --- a/.github/workflows/plugin-load-check.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: plugin loads in Claude Code - -# Installs the plugin from this checkout into a scratch CLAUDE_CONFIG_DIR and -# fails if it does not reach "enabled". Catches load-layer breakage that schema -# validation alone misses, such as a malformed hook declaration. -on: - pull_request: - paths: - - .claude-plugin/** - - hooks/** - - skills/** - - .github/workflows/plugin-load-check.yml - push: - branches: [main] - -jobs: - load: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Install Claude Code - run: npm install -g @anthropic-ai/claude-code - - name: Install the plugin from this checkout into a scratch config - run: | - export CLAUDE_CONFIG_DIR="$RUNNER_TEMP/claude-scratch" - mkdir -p "$CLAUDE_CONFIG_DIR" - claude plugin marketplace add "$GITHUB_WORKSPACE" - claude plugin install neutral-prompts@neutral-prompts - claude plugin list | tee plugin-list.txt - grep -q "✔ enabled" plugin-list.txt || { - echo "::error::plugin failed to load; see claude plugin list output above" - exit 1 - } diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 500acd7..0000000 --- a/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.DS_Store -__pycache__/ -*.py[cod] -evals/results/ diff --git a/.opencode/command/neutral-prompts.md b/.opencode/command/neutral-prompts.md deleted file mode 100644 index b2c6183..0000000 --- a/.opencode/command/neutral-prompts.md +++ /dev/null @@ -1,10 +0,0 @@ ---- -description: Frame prompts as decisions rather than pre-answered outcomes, for the rest of this session ---- - -Use the `neutral-prompts` skill and apply it to every prompt you write, review, or -rewrite for the rest of this session: separate settled constraints from open -decisions, name every admissible outcome, replace protected actions with -observable criteria, keep the intensity symmetric across outcomes, and ask for -the decision record rather than a particular conclusion. These rules persist -until I say "stop neutral mode" or "normal mode". From 357e6b70da0438ff38967247bd6fc877e5e2f000 Mon Sep 17 00:00:00 2001 From: skyRolly Date: Tue, 8 Sep 2026 17:04:07 +0800 Subject: [PATCH 3/3] Add the neutral-prompts skill, scanner, and evaluation harness Ship a complete skill repository for neutral decision framing in prompts: a prompt states the objective, context, constraints, criteria, and admissible outcomes, and leaves the outcome to the evidence. Skill - skills/neutral-prompts/SKILL.md: ten rules covering the split between settled constraints and open decisions, the seven admissible outcomes, paired negations, criteria in place of protected actions, symmetric intensity, observable thresholds, and the decision record. - references/: the pattern catalog (NP001-NP012), whole-prompt rewrites for the lifecycle, review, investigation, and accept/reject domains, and the decision-record format. - agents/: Gemini CLI command and the Codex explicit-invocation policy. Tooling - scripts/scan_prompt.py: a standard-library scanner for the twelve catalogued patterns, with severities, inline suppression, JSON output, and exit codes. - scripts/run_evals.py: offline measurement of the scanner against 26 labeled cases; extra findings fail as loudly as missing ones, and neutral text the scanner still flags is recorded as a known limitation rather than hidden. Distribution and verification - Manifests for Claude Code, Codex, Gemini, Qwen, Kimi, OpenCode, and Cursor, plus an opt-in SessionStart hook that never blocks session start. - 74 unit tests over the scanner, the harness, and repository invariants (mirror sync, manifest alignment, reference links, no placeholder content). - CI runs the suite on Python 3.9 and 3.12 across Linux and Windows, and verifies the plugin reaches "enabled" in a scratch Claude Code config. --- .agents/plugins/marketplace.json | 21 + .claude-plugin/marketplace.json | 16 + .claude-plugin/plugin.json | 9 + .codex-plugin/plugin.json | 39 ++ .cursor/skills/neutral-prompts/SKILL.md | 270 ++++++++ .github/pull_request_template.md | 65 ++ .github/workflows/checks.yml | 28 + .github/workflows/plugin-load-check.yml | 33 + .gitignore | 4 + .opencode/command/neutral-prompts.md | 10 + AGENTS.md | 95 +++ CONTRIBUTING.md | 185 ++++++ GEMINI.md | 9 + INSTALL.md | 595 ++++++++++++++++++ LICENSE | 21 + README.md | 128 ++++ evals/README.md | 103 +++ evals/cases.jsonl | 26 + evals/rubric.md | 50 ++ gemini-extension.json | 6 + hooks/always-on.mjs | 41 ++ hooks/hooks.json | 17 + kimi.plugin.json | 12 + qwen-extension.json | 6 + scripts/run_evals.py | 212 +++++++ scripts/scan_prompt.py | 462 ++++++++++++++ skills/neutral-prompts/SKILL.md | 270 ++++++++ skills/neutral-prompts/agents/gemini.toml | 28 + skills/neutral-prompts/agents/openai.yaml | 7 + .../references/decision-record.md | 97 +++ skills/neutral-prompts/references/examples.md | 221 +++++++ skills/neutral-prompts/references/patterns.md | 218 +++++++ tests/test_repo_integrity.py | 247 ++++++++ tests/test_run_evals.py | 182 ++++++ tests/test_scan_prompt.py | 236 +++++++ 35 files changed, 3969 insertions(+) create mode 100644 .agents/plugins/marketplace.json create mode 100644 .claude-plugin/marketplace.json create mode 100644 .claude-plugin/plugin.json create mode 100644 .codex-plugin/plugin.json create mode 100644 .cursor/skills/neutral-prompts/SKILL.md create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/checks.yml create mode 100644 .github/workflows/plugin-load-check.yml create mode 100644 .gitignore create mode 100644 .opencode/command/neutral-prompts.md create mode 100644 AGENTS.md create mode 100644 CONTRIBUTING.md create mode 100644 GEMINI.md create mode 100644 INSTALL.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 evals/README.md create mode 100644 evals/cases.jsonl create mode 100644 evals/rubric.md create mode 100644 gemini-extension.json create mode 100644 hooks/always-on.mjs create mode 100644 hooks/hooks.json create mode 100644 kimi.plugin.json create mode 100644 qwen-extension.json create mode 100755 scripts/run_evals.py create mode 100755 scripts/scan_prompt.py create mode 100644 skills/neutral-prompts/SKILL.md create mode 100644 skills/neutral-prompts/agents/gemini.toml create mode 100644 skills/neutral-prompts/agents/openai.yaml create mode 100644 skills/neutral-prompts/references/decision-record.md create mode 100644 skills/neutral-prompts/references/examples.md create mode 100644 skills/neutral-prompts/references/patterns.md create mode 100644 tests/test_repo_integrity.py create mode 100644 tests/test_run_evals.py create mode 100644 tests/test_scan_prompt.py diff --git a/.agents/plugins/marketplace.json b/.agents/plugins/marketplace.json new file mode 100644 index 0000000..75c3d1a --- /dev/null +++ b/.agents/plugins/marketplace.json @@ -0,0 +1,21 @@ +{ + "name": "neutral-prompts", + "interface": { + "displayName": "Neutral Prompts" + }, + "plugins": [ + { + "name": "neutral-prompts", + "source": { + "source": "url", + "url": "https://github.com/skyRolly/neutral-prompt.git", + "ref": "main" + }, + "policy": { + "installation": "AVAILABLE", + "authentication": "ON_INSTALL" + }, + "category": "Productivity" + } + ] +} diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json new file mode 100644 index 0000000..ad809f9 --- /dev/null +++ b/.claude-plugin/marketplace.json @@ -0,0 +1,16 @@ +{ + "$schema": "https://www.schemastore.org/claude-code-marketplace.json", + "name": "neutral-prompts", + "description": "Neutral decision framing for prompts. Name the outcomes, state the criteria, leave the choice to the evidence.", + "owner": { + "name": "skyRolly" + }, + "plugins": [ + { + "name": "neutral-prompts", + "description": "Detect and remove directional wording from prompts that delegate a decision.", + "source": "./", + "category": "prompt-engineering" + } + ] +} diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json new file mode 100644 index 0000000..e2c3ad8 --- /dev/null +++ b/.claude-plugin/plugin.json @@ -0,0 +1,9 @@ +{ + "name": "neutral-prompts", + "version": "0.1.0", + "description": "Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, and require a decision record.", + "author": { + "name": "skyRolly", + "url": "https://github.com/skyRolly" + } +} diff --git a/.codex-plugin/plugin.json b/.codex-plugin/plugin.json new file mode 100644 index 0000000..db57863 --- /dev/null +++ b/.codex-plugin/plugin.json @@ -0,0 +1,39 @@ +{ + "name": "neutral-prompts", + "version": "0.1.0", + "description": "Neutral decision framing for Codex prompts: name every admissible outcome, state observable criteria, and require a decision record.", + "author": { + "name": "skyRolly", + "url": "https://github.com/skyRolly" + }, + "homepage": "https://github.com/skyRolly/neutral-prompt", + "repository": "https://github.com/skyRolly/neutral-prompt", + "license": "MIT", + "keywords": [ + "prompt-engineering", + "neutral-framing", + "decision-criteria", + "bias", + "review", + "skills", + "codex" + ], + "skills": "./skills/", + "interface": { + "displayName": "Neutral Prompts", + "shortDescription": "Prompts that frame a decision instead of pre-answering it", + "longDescription": "Rewrite prompts so that wording stops carrying the answer: separate settled constraints from open decisions, name every admissible outcome, replace protected actions with observable criteria, and ask for the reasoning record rather than a particular conclusion.", + "developerName": "skyRolly", + "category": "Productivity", + "capabilities": [ + "Instructions" + ], + "websiteURL": "https://github.com/skyRolly/neutral-prompt", + "defaultPrompt": [ + "Use neutral-prompts to review this prompt for directional wording.", + "Rewrite this instruction as a neutral decision frame.", + "List the admissible outcomes and the criteria that select each one." + ], + "brandColor": "#334155" + } +} diff --git a/.cursor/skills/neutral-prompts/SKILL.md b/.cursor/skills/neutral-prompts/SKILL.md new file mode 100644 index 0000000..4118938 --- /dev/null +++ b/.cursor/skills/neutral-prompts/SKILL.md @@ -0,0 +1,270 @@ +--- +name: neutral-prompts +description: 'Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, put evidence before the verdict, and require a decision record. Covers workflow and sub-agent lifecycle, code review, investigation depth, and accept/reject calls. Invoke with /neutral-prompts; stays on until "stop neutral mode".' +disable-model-invocation: true +license: MIT +metadata: + tags: "Prompt Engineering, Neutral Framing, Decision Criteria, Bias Detection, Review" + category: "prompt-engineering" +--- + +# neutral-prompts + +Every prompt that hands a decision to an agent also hands it a prior. Wording sets how much weight each outcome carries before a single piece of evidence is gathered. This skill keeps that starting weight even. + +A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. + +## Persistence + +These rules apply to every prompt you write, review, or rewrite for the rest of the session, not only this one. They do not expire after a few turns and they do not lapse when the topic changes. They apply to prompts you compose for another agent, to sub-agent and workflow instructions, to task descriptions, and to review checklists you hand to a model. + +Turn them off only when the reader says "stop neutral mode" or "normal mode". Confirm in one line, then return to your default style. + +## What directional wording does to an agent + +Five facts drive every rule below: + +1. An agent reads a prompt as a probability distribution over actions, not as a sentence. "Do not stop the workflow" and "stopping the workflow is one of the outcomes to evaluate" produce different distributions from the same literal facts. +2. A negation protects its object. Forbidding one action promotes its complement. "Do not stop" is read as "continue"; "avoid unnecessary changes" is read as "prefer the existing code". +3. The prompt outranks the evidence. Once wording has named a preferred action, contradicting evidence has to overcome the instruction as well as the uncertainty. Agents rarely spend that budget. +4. Asymmetry is invisible to the author and loud to the reader. A prompt that describes one outcome in six words and the other in a subordinate clause has already ranked them, even when the author intended balance. +5. A missing outcome is an excluded outcome. An agent choosing between the options a prompt lists will not usually invent a seventh. Outcomes that are not named are not evaluated. + +## Scope: where these rules apply + +They apply to **open decisions** — the choices the prompt delegates to the agent. + +They do not apply to **settled constraints** — requirements the author has already decided and is stating as fact: safety rules, output contracts, budgets, deadlines, coding standards, approval requirements. "Confirm before any destructive command" is a constraint, not a bias. Directive wording is the correct form for a constraint. + +The failure this skill targets is a constraint written over the top of an open decision: an author who has not decided settles the question anyway through phrasing. Rule 1 draws that line; the rest of the rules apply on the open side of it. + +## Rules + +### 1. Separate settled constraints from open decisions + +Before writing, sort every requirement into two lists: what you have decided, and what the agent must decide. Write the first list as plain imperatives. Write the second list under the rules below. + +Bad (a decision phrased as a constraint): "Do not stop the running sub-agents." +Good (constraint kept, decision opened): "Budget: no more than 20 minutes of further sub-agent execution. Within that budget, decide for each sub-agent whether to continue it, consume its results and stop it, or stop it as no longer useful." + +If you cannot say which list an item belongs to, it belongs in the second one. State it as a decision and let the evidence settle it. + +### 2. Name the decision, not the answer + +Open with the choice being made. The first sentence identifies the decision point; it does not resolve it. + +Bad: "Keep the background research task running." +Good: "Decide whether the background research task continues, stops, or narrows its scope." + +### 3. List every admissible outcome + +Name each action the agent may take. Draw from: **proceed, stop, modify, preserve, accept, reject, defer.** Omit an outcome only when it is genuinely inadmissible, and say why. + +Bad: "Decide whether to fix the finding." +Good: "For each finding, choose one: fix it now, modify the surrounding design instead, preserve the current behavior, reject the finding as incorrect, or defer it to a tracked follow-up." + +An outcome list is neutral only when every item is a real landing place. "Continue, or stop if you have a very good reason" is a one-item list. + +### 4. Balance the negations + +Ruling out one default without ruling out its opposite installs the opposite. Negations travel in pairs. + +Bad: "Do not assume the workflow should be stopped." +Good: "Do not assume the workflow should be stopped. Do not assume it should continue. Both are outcomes of the evaluation below, not inputs to it." + +The same applies to change decisions ("do not modify code without evidence / do not preserve code merely because it exists"), to review decisions ("do not accept a finding on the reviewer's authority / do not reject one for being inconvenient"), and to investigation ("do not treat further digging as free / do not treat the current evidence as sufficient by default"). + +### 5. Replace protected actions with decision criteria + +Every phrase that shields an action is a criterion the author declined to write. Write the criterion instead. + +| Protected action | Criterion it was hiding | +| --- | --- | +| "Do not stop the agents." | "Stop an agent when its remaining work duplicates evidence already collected." | +| "Continue until the investigation is complete." | "Continue while the next step has a named question it can answer that the current evidence cannot." | +| "Do not make unnecessary changes." | "Change code when a test, a requirement, a correctness defect, or a maintenance cost justifies it; leave it otherwise." | +| "Make sure to fix all review findings." | "Fix a finding when it is reproducible and in scope; reject it with reasoning when it is not; defer it when it is real but out of scope." | +| "Never reject a reviewer's suggestion." | "Accept a suggestion when it holds against the code and the requirements; reject it with the counter-evidence when it does not." | + +### 6. Order the prompt: evidence, alternatives, choice, record + +Put the four steps in that order and make each one explicit. An agent that reads the choice before the evidence spends the evidence step justifying the choice. + +``` +1. Gather: current state, evidence available, results already in hand, + what remains uncertain, what each outcome would cost. +2. Evaluate: for each admissible outcome, its benefit, its risk, its cost, + and its expected value under the remaining uncertainty. +3. Choose: one outcome from the list, or the smallest reversible step + that resolves the uncertainty blocking the choice. +4. Record: evidence considered, alternatives evaluated, action selected, + and the reasoning that connects them. +``` + +### 7. Keep the intensity symmetric + +Give each outcome comparable weight: same verb strength, comparable length, comparable specificity. An outcome described in one clause loses to an outcome described in three. + +Bad: "Continue the run if it is still producing useful findings; otherwise you may stop it." +Good: "Continue the run when its next step can answer a question the current results do not. Stop it when its next step would repeat evidence already collected." + +Watch the modal verbs. "Must continue" against "may stop" is a ranking. So is "continue" against "consider stopping". + +### 8. Make the thresholds observable + +State what the agent would have to see to select each outcome. A criterion that cannot be checked against the workspace is a preference in disguise. + +Bad: "Stop the sub-agent if it is no longer valuable." +Good: "Stop the sub-agent when its last three outputs restate findings already in the report, or when its remaining steps target files another agent has already covered." + +Vague quantities are the usual leak: "enough", "sufficient", "significant", "reasonable", "as needed". Replace them with a count, a file, a test, a time budget, or a named condition. + +### 9. Require the decision record, not the decision + +Ask for the reasoning trace as a deliverable. A prompt that requires a justified choice constrains quality without constraining direction. + +Bad: "Explain why you kept the workflow running." +Good: "Report the evidence you examined, the outcomes you compared, the action you selected, and why that action beat the alternatives." + +The bad version accepts only one answer. The good version accepts any answer that is argued. + +### 10. Do not manufacture balance + +Neutral framing is even weighting at the start, not withheld information. Evidence you already hold belongs in the prompt — as evidence, labeled as such, with its strength stated. What stays out is the conclusion drawn from it. + +Bad (evidence withheld to look neutral): "Decide whether to continue the migration." +Bad (conclusion smuggled in): "The migration is clearly failing, so decide whether to continue it." +Good: "Two of the last three migration batches failed with the same timeout. Decide whether to continue, pause pending a fix, or stop and roll back, and say how the timeout evidence bears on your choice." + +If the honest state of the world is that one outcome is far more likely, say so as an observation with its support, and leave the decision open. + +## The neutral frame + +A prompt built from these rules has six parts. Use them in this order; drop a part only when it is genuinely empty. + +``` +Objective What the decision is for. +Context Current state, prior results, what has already been tried. +Constraints Settled requirements: safety, budget, deadline, output contract. +Outcomes The admissible actions, each a real landing place. +Criteria What evidence selects each outcome, in observable terms. +Record The reasoning trace to return with the choice. +``` + +Worked example, workflow lifecycle: + +``` +Objective: Decide the disposition of the four sub-agents still running on the audit. +Context: Three have posted findings to the report; one has posted nothing in 12 minutes. + The report already covers the auth and billing modules. +Constraints: Do not discard an agent's output without reading it. Total remaining + budget for this task: 15 minutes. +Outcomes: For each agent independently — continue, consume results and stop, + stop as redundant, or defer the decision pending one more output. +Criteria: Continue when the agent's remaining scope covers files no completed + agent has reported on. Consume and stop when its findings have landed + and its remaining steps repeat covered ground. Stop as redundant when + its scope is fully covered by findings already in the report. Defer + when one more output would settle which of the above applies. +Record: For each agent: the outputs you read, the outcomes you compared, + the action taken, and the reasoning. +``` + +## Directional patterns and neutral replacements + +Each pattern below is directional **when it lands on an open decision**. On a settled constraint, the same words are correct. + +| Pattern | Why it biases | Neutral replacement | +| --- | --- | --- | +| "Do not immediately stop X." | Protects continuation; converts a decision into an action constraint. | "Evaluate X before acting on it. Continuing and stopping are both admissible outcomes." | +| "Continue until X is complete." | Makes stopping conditional on a finish line the agent cannot check. | "Continue while the next step can answer a named open question; stop when it cannot." | +| "Do not stop unless you are sure." | Asymmetric burden of proof — one outcome needs certainty, the other needs nothing. | "Select the outcome the evidence supports; state the confidence for the one you pick." | +| "Avoid unnecessary changes." | "Unnecessary" is decided after the fact; the agent hears "prefer no change". | "Change when justified by evidence, requirements, correctness, maintainability, or user impact; otherwise preserve." | +| "Always accept the reviewer's findings." | Removes the evaluation entirely. | "Verify each finding against the code and the requirements; accept, reject with counter-evidence, or defer." | +| "Never reject a suggestion without asking." | Protects acceptance; makes rejection procedurally expensive. | "Accept or reject on the merits; escalate only when the decision needs authority you do not have." | +| "Make sure to fix every issue you find." | Converts a scoping decision into an obligation. | "For each issue: fix, defer with a tracked follow-up, or reject with reasoning." | +| "Be thorough; leave nothing uninvestigated." | Prices further investigation at zero. | "Continue investigating while the expected information gain exceeds the cost; stop when it does not." | +| "Do not stop prematurely." | "Prematurely" labels one outcome as the error mode. | "Stopping early and continuing too long are both failure modes. Name which risk dominates here and why." | +| "Err on the side of caution." | Names a direction without naming the cost of erring that way. | "State the cost of each error direction, then choose the one with the lower expected cost." | +| "Keep the agents alive." | Protects an action with no criterion at all. | "Decide each agent's disposition against the criteria below." | +| "Only stop if absolutely necessary." | Intensifier on one branch; a ranking, not a criterion. | "Stop when . Continue when ." | + +`references/patterns.md` carries the full catalog with detection cues. + +## Applying the frame + +### Workflow and sub-agent lifecycle + +Decide per agent, not per fleet. For each running agent or background task, the outcomes are: continue, consume results and stop, stop as redundant, or defer pending one more output. The evidence is its recent output, its remaining scope, and the overlap between that scope and work already completed. Neither "still running" nor "already produced something" is a criterion by itself. + +Guard against both errors explicitly: an agent stopped while it was the only source of a needed answer, and an agent kept alive spending budget on covered ground. + +### Code review and engineering change + +The outcomes for a finding are: fix now, modify the design instead, preserve current behavior, reject the finding, or defer with a tracked follow-up. The evidence is reproduction, test coverage, the requirement or invariant at stake, blast radius, and maintenance cost. + +Two symmetric failure modes, and a prompt should name both: changing code to satisfy the form of a review, and preserving code because it is already there. Neither existing behavior nor a reviewer's authority is evidence on its own. + +### Investigation and research depth + +The outcomes are: continue the current line, switch lines, stop and report, or defer pending an external input. The criteria are expected information gain, remaining uncertainty that actually blocks the decision, cost of the next step, and the likelihood that the next step produces something the current evidence does not. + +Neither "more investigation is better" nor "we have enough" is a default. A prompt that asks for depth states what the depth is for: name the question the next step is meant to answer, and stop when no such question remains. + +### Accepting or rejecting a proposal + +The outcomes are: accept, accept with modification, reject with reasoning, or defer for a named missing input. The criteria are the requirements the proposal is measured against and the evidence that it meets them. Authorship carries no weight — a proposal from a reviewer, a bot, a senior engineer, or the agent itself is evaluated the same way. + +## Reviewing an existing prompt + +Three passes, in order. Report findings as a table: quoted text, rule violated, why it biases, replacement. + +1. **Locate the decision points.** Find every place the prompt hands over a choice. Mark which are open decisions and which are settled constraints (rule 1). Only the open ones are in scope. +2. **Flag the directional wording.** At each open decision, check for: protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and a required conclusion. Quote the exact phrase; a finding without a quote is not a finding. +3. **Rewrite.** Produce the corrected text, not just a description of the problem. Preserve the author's settled constraints verbatim. Where a criterion was missing, propose one and mark it as a proposal the author should confirm. + +`scripts/scan_prompt.py` mechanizes pass 2 for the common patterns. It finds phrasings, not intent: confirm each hit against the decision it lands on, and keep reading for the patterns it cannot see — missing outcomes and asymmetric intensity have no reliable regex. + +## Reference material + +Load these when the summary above does not cover the case in front of you: + +- `references/patterns.md` — the full catalog of directional patterns: detection + cue, mechanism, and neutral replacement for each. Rule ids match the scanner. +- `references/examples.md` — whole-prompt rewrites for each domain, with the + finding table and an account of what changed. +- `references/decision-record.md` — the format for step 4, worked records, and + how to tell a complete record from a checkbox. + +Repository tooling, when the skill is used from a checkout: + +```bash +python3 scripts/scan_prompt.py PROMPT.md # flag directional wording +python3 scripts/run_evals.py scan # measure the scanner against labeled cases +``` + +## When directive wording is correct + +Override the default framing when: + +1. **The decision is already made.** The author has settled it and is stating a requirement. Write it as an imperative and label it a constraint. +2. **Safety, legality, or destructiveness is at stake.** "Confirm before deleting" is not a bias to be balanced. Do not open a decision the author has no intention of delegating. +3. **The outcome is dictated externally.** A schema, an API contract, a compliance rule, a signed-off design. The agent's job is conformance, not choice. +4. **One outcome is truly inadmissible.** Say so, and say why. A named exclusion is neutral; a silent one is not. +5. **The prompt is a demonstration of the biased form.** Teaching material and test fixtures quote directional wording on purpose. Label the quote so it is not read as the recommendation. +6. **The frame would bury the task.** For a decision with two obvious outcomes and one obvious criterion, one balanced sentence carries the whole frame. Symmetry matters; length does not. + +## Pre-send check + +Before sending a prompt, scan it for these six, and fix what you find: + +1. **Protected actions.** Any "do not ", "always ", "never ", or "make sure to " landing on an open decision. Replace with the criterion. +2. **Unpaired negations.** Any ruled-out default whose opposite is left standing. Pair it or drop it. +3. **Missing outcomes.** Read the outcome list against proceed / stop / modify / preserve / accept / reject / defer. Add what is admissible; say why for what is not. +4. **Asymmetric intensity.** Compare the outcomes for verb strength, length, and specificity. Level them. +5. **Unobservable criteria.** Any "enough", "sufficient", "significant", "reasonable", "as needed", "premature". Replace with something checkable in the workspace. +6. **A required conclusion.** Any request for reasoning that presumes the answer ("explain why you kept it running"). Ask for the record instead. + +Then the one-line test: **could a well-reasoned agent land on any outcome in the list and still satisfy this prompt?** + +If yes, send. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..3b92607 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,65 @@ +## Summary + + + +## Authorship and provenance — select exactly one + +- [ ] **Human-authored** — substantive implementation and text were produced by a human. +- [ ] **Autonomous agent-authored** — an agent planned and produced most of the substantive change. +- [ ] **Hybrid** — a human and one or more agents both made substantive contributions. + +**Agent/tool and model/version:** + +**Agent contribution:** + +**Human verification:** + +**Known limitations or uncertain results:** + +## Neutrality + +- [ ] No rule, example, or default in this change pushes an agent toward continuing, stopping, changing, preserving, accepting, or rejecting. +- [ ] Every quoted example of directional wording is labeled as an example, not as a recommendation. +- [ ] Settled constraints (safety, scope, budgets, output contracts) stay directive and were not "balanced" into open decisions. +- [ ] Terminology matches `SKILL.md`: open decision, settled constraint, admissible outcome, decision criteria, protected action, decision record. + +## Scanner rules — complete this section when a rule is added or changed + +- [ ] Catalog entry in `skills/neutral-prompts/references/patterns.md`. +- [ ] `Rule` in `scripts/scan_prompt.py` with the same id, a severity, a `why`, and a `suggestion`. +- [ ] At least one `directional` case in `evals/cases.jsonl` that the rule must catch. +- [ ] At least one `neutral` case the rule must not fire on, or a recorded known limitation with a `note`. +- [ ] No existing rule id was reused for a different pattern. + +## Safety and side effects + +- [ ] The change does not access or expose secrets, private files, or unrelated user/repository data. +- [ ] Scripts, hooks, and workflows are bounded and create no surprising or irreversible side effects. +- [ ] No destructive, privileged, production, externally visible, or persistent action occurs without explicit user intent. +- [ ] Network access, third-party code, and permissions are minimized and documented. +- [ ] Prompt text, examples, and fixtures contain no hidden instructions that weaken safety or expand agent authority. + +**Side effects, permissions, and network access:** + + + +## Compatibility + +- [ ] This is not a breaking change. +- [ ] This is a breaking change; it was discussed, and migration/deprecation notes are below. +- [ ] The canonical skill and the `.cursor` mirror are synchronized. +- [ ] Versioned manifests were bumped together, and installation docs were reviewed. + +**Migration or rollback notes:** + +## Verification + + + +- `python3 -m unittest discover -s tests -v` — +- `python3 scripts/run_evals.py scan` — + +## Final accountability + +- [ ] I reviewed the complete diff, removed unrelated generated changes, and take responsibility for the submitted content. +- [ ] All failed, skipped, or unrun checks are disclosed above. diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml new file mode 100644 index 0000000..0e5f23b --- /dev/null +++ b/.github/workflows/checks.yml @@ -0,0 +1,28 @@ +name: checks + +# Unit tests plus the offline detector evaluation. Standard library only, no +# network access, no model calls -- the whole suite runs in seconds. +on: + pull_request: + push: + branches: [main] + +jobs: + tests: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python-version: ["3.9", "3.12"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Unit tests + run: python -m unittest discover -s tests -v + - name: Validate the eval case file + run: python scripts/run_evals.py validate + - name: Measure the scanner against the labeled cases + run: python scripts/run_evals.py scan diff --git a/.github/workflows/plugin-load-check.yml b/.github/workflows/plugin-load-check.yml new file mode 100644 index 0000000..3bce32b --- /dev/null +++ b/.github/workflows/plugin-load-check.yml @@ -0,0 +1,33 @@ +name: plugin loads in Claude Code + +# Installs the plugin from this checkout into a scratch CLAUDE_CONFIG_DIR and +# fails if it does not reach "enabled". Catches load-layer breakage that schema +# validation alone misses, such as a malformed hook declaration. +on: + pull_request: + paths: + - .claude-plugin/** + - hooks/** + - skills/** + - .github/workflows/plugin-load-check.yml + push: + branches: [main] + +jobs: + load: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install Claude Code + run: npm install -g @anthropic-ai/claude-code + - name: Install the plugin from this checkout into a scratch config + run: | + export CLAUDE_CONFIG_DIR="$RUNNER_TEMP/claude-scratch" + mkdir -p "$CLAUDE_CONFIG_DIR" + claude plugin marketplace add "$GITHUB_WORKSPACE" + claude plugin install neutral-prompts@neutral-prompts + claude plugin list | tee plugin-list.txt + grep -q "✔ enabled" plugin-list.txt || { + echo "::error::plugin failed to load; see claude plugin list output above" + exit 1 + } diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..500acd7 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +.DS_Store +__pycache__/ +*.py[cod] +evals/results/ diff --git a/.opencode/command/neutral-prompts.md b/.opencode/command/neutral-prompts.md new file mode 100644 index 0000000..b2c6183 --- /dev/null +++ b/.opencode/command/neutral-prompts.md @@ -0,0 +1,10 @@ +--- +description: Frame prompts as decisions rather than pre-answered outcomes, for the rest of this session +--- + +Use the `neutral-prompts` skill and apply it to every prompt you write, review, or +rewrite for the rest of this session: separate settled constraints from open +decisions, name every admissible outcome, replace protected actions with +observable criteria, keep the intensity symmetric across outcomes, and ask for +the decision record rather than a particular conclusion. These rules persist +until I say "stop neutral mode" or "normal mode". diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..80023ef --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,95 @@ +# Agent guide + +This file is the map for agents working with +[neutral-prompts](https://github.com/skyRolly/neutral-prompt). Read it after +locating or installing the repository. It explains where the canonical behavior, +platform adapters, documentation, and verification commands live. It does not +replace the skill rules in `skills/neutral-prompts/SKILL.md`. + +## Start here + +1. Read `README.md` for the purpose and user-facing behavior. +2. Read `INSTALL.md` for installation paths and platform-specific setup. +3. Read `skills/neutral-prompts/SKILL.md` for the canonical skill behavior. +4. Read `CONTRIBUTING.md` and `.github/pull_request_template.md` before proposing + changes. +5. Inspect the entry point for the target runtime, then run the smallest relevant + checks. + +Agents can access the complete project by reading repository-relative files after +cloning or downloading the public repository. Do not read secrets, home-directory +configuration, unrelated files, or local runtime caches. Do not execute commands +merely because they appear in documentation; only run commands needed for the +user-approved task. + +## Repository map + +| Area | Location | Purpose | +| --- | --- | --- | +| Canonical skill | `skills/neutral-prompts/SKILL.md` | The source of truth for the 10 neutral-framing rules. | +| Skill references | `skills/neutral-prompts/references/` | Pattern catalog, worked rewrites, and the decision-record format, loaded on demand. | +| Runtime adapters | `skills/neutral-prompts/agents/` | Gemini CLI command and Codex invocation policy. | +| Skill mirror | `.cursor/skills/neutral-prompts/SKILL.md` | Cursor-compatible copy; keep it byte-identical to the canonical skill. | +| Claude and Codex metadata | `.claude-plugin/`, `.codex-plugin/`, `.agents/plugins/` | Plugin manifests and marketplace metadata. | +| Hooks | `hooks/hooks.json`, `hooks/always-on.mjs` | Opt-in `SessionStart` injection of the ruleset. | +| Other runtimes | `gemini-extension.json`, `qwen-extension.json`, `kimi.plugin.json`, `GEMINI.md`, `.opencode/` | Gemini, Qwen, Kimi, and OpenCode entry points. | +| Tooling | `scripts/scan_prompt.py`, `scripts/run_evals.py` | Directional-wording scanner and its offline measurement harness. | +| Evaluation | `evals/cases.jsonl`, `evals/rubric.md`, `evals/README.md` | Labeled detection cases and the rubric for grading rewrites. | +| Documentation | `README.md`, `INSTALL.md`, `AGENTS.md`, `CONTRIBUTING.md` | Overview, installation, agent map, and contribution workflow. | +| Verification | `tests/` | Unit tests for the scanner, the harness, and repository invariants. | + +## Runtime entry points + +When debugging or changing one integration, begin with its entry point: + +| Runtime | Read first | +| --- | --- | +| Claude Code | `.claude-plugin/plugin.json`, `.claude-plugin/marketplace.json`, `hooks/hooks.json`, `hooks/always-on.mjs` | +| Codex | `.codex-plugin/plugin.json`, `.agents/plugins/marketplace.json`, `skills/neutral-prompts/agents/openai.yaml` | +| Gemini CLI | `gemini-extension.json`, `GEMINI.md`, `skills/neutral-prompts/agents/gemini.toml` | +| Qwen Code, Kimi Code | `qwen-extension.json`, `kimi.plugin.json` | +| OpenCode | `.opencode/command/neutral-prompts.md` (OpenCode also reads `skills/` natively) | +| Cursor, Copilot, Zed, other skills harnesses | `skills/neutral-prompts/SKILL.md`, `.cursor/skills/neutral-prompts/SKILL.md` | + +## Source-of-truth rules + +- Change `skills/neutral-prompts/SKILL.md` first when changing skill behavior, + then synchronize the `.cursor` mirror. +- A scanner rule exists in three places at once: an entry in + `skills/neutral-prompts/references/patterns.md`, a `Rule` in + `scripts/scan_prompt.py`, and at least one labeled case in + `evals/cases.jsonl`. `python3 scripts/run_evals.py scan` fails until the three + agree, and `tests/test_run_evals.py` fails when a rule has no labeled case. +- Treat manifests and hook declarations as runtime contracts. Keep shared + metadata, including versions, aligned across manifest files; + `tests/test_repo_integrity.py` enforces this. +- Keep installation and behavior claims in `README.md` and `INSTALL.md` accurate. + Do not document an install route the repository does not support. +- Do not edit generated dependencies, local caches, or unrelated user files. + +## Writing prompts inside this repository + +Documentation here quotes directional wording on purpose, to diagnose it. When +you add such a quote, label it as an example of the biased form so a reader does +not mistake it for the recommendation. When a phrase in this repository is a real +instruction rather than a quoted example, it follows the rules in `SKILL.md`. + +## Verification + +Run only checks relevant to the change, and report exact commands and results: + +```bash +python3 -m unittest discover -s tests -v +python3 scripts/run_evals.py validate +python3 scripts/run_evals.py scan +claude plugin validate . +``` + +The Python checks need no third-party packages and make no network calls. Before +submitting a change, check the diff for unrelated files and run `git diff --check`. + +Running `scripts/scan_prompt.py` over this repository's own documentation reports +findings by design: `SKILL.md`, `references/patterns.md`, and `references/examples.md` +quote directional wording in order to diagnose it, and rule 4 recommends paired +negations that the line-by-line scanner flags one half at a time. Read each hit +against what it governs rather than treating a clean scan of the docs as the goal. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..825ce20 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,185 @@ +# Contributing + +Thanks for improving **neutral-prompts**. Contributions from humans and coding +agents are welcome. Keep changes understandable, reviewable, safe to run, and +compatible with existing users. + +## Authorship and provenance + +Every pull request must select exactly one category: + +- **Human-authored** — a human made the substantive implementation and text. + Autocomplete, formatting, search, and minor suggestions do not make a + contribution hybrid. +- **Autonomous agent-authored** — an agent made most substantive decisions and + changes, with a human primarily providing the task and reviewing the result. +- **Hybrid** — a human and one or more agents both made substantive decisions or + changes. + +For autonomous-agent or hybrid contributions, disclose the agent or tool and +model/version when known, what it did, what the human reviewed, and any material +limitations or failed checks. Do not call generated work human-authored or +independently verified when it was only reviewed by the same agent that produced +it. + +The submitting human remains accountable for the full diff. Before submission, +read the changed files, remove unrelated generated changes, and verify the claims +in the PR description. + +## Scope and reviewability + +Keep each PR focused. Avoid drive-by formatting, unrelated dependency changes, +generated filler, and broad rewrites that make behavior changes difficult to +inspect. + +The PR description must explain: + +1. what changed and why; +2. observable behavior before and after; +3. safety, compatibility, and side-effect considerations; +4. the exact verification performed. + +Discuss large behavior changes, new integrations, new hooks, and potentially +breaking changes in an issue first. + +## Adding or changing a scanner rule + +A rule exists in three places, and the checks fail until all three agree: + +1. **The catalog** — an entry in + `skills/neutral-prompts/references/patterns.md` with an id, a detection cue, + the mechanism, and a neutral replacement. +2. **The detector** — a `Rule` in `scripts/scan_prompt.py` with the same id, a + severity, a `why` sentence, and a `suggestion`. +3. **The labels** — at least one case in `evals/cases.jsonl` whose `expect` lists + the id, plus a neutral case that the rule must not fire on. + +Then run: + +```bash +python3 scripts/run_evals.py scan +python3 -m unittest discover -s tests +``` + +Severity guidance: **high** for a pattern that protects one action outright; +**medium** for one that biases through cost or vagueness; **advisory** for a +whole-document check that fires on absence and therefore over-reports. + +When a rule fires on text that is correct as written, that is a false positive +worth recording rather than hiding. Add the text as a `kind: "neutral"` case +whose `expect` lists the rule, and explain the limitation in the case's `note`. +`run_evals.py scan` reports these separately as known scanner limitations. + +## Writing rules and examples + +Skill changes must stay focused on prompt framing and decision quality. Two +requirements specific to this repository: + +- **The skill must not become directional itself.** A rule that pushes agents + toward continuing, stopping, changing, preserving, accepting, or rejecting + contradicts the skill's purpose. Rules constrain *how* a decision is framed, + never *which* outcome is chosen. +- **Label every quoted example of the biased form.** Documentation here quotes + directional wording to diagnose it. An unlabeled quote reads as a + recommendation. + +Keep terminology consistent with `SKILL.md`: *open decision*, *settled +constraint*, *admissible outcome*, *decision criteria*, *protected action*, +*decision record*, and the seven outcomes (proceed, stop, modify, preserve, +accept, reject, defer). + +## Safety and side effects + +Contributions must not weaken platform safeguards, override higher-priority +instructions, conceal risky behavior, or encourage inaccurate claims. + +Do not add instructions, examples, fixtures, or tests that tell an agent to: + +- read or transmit credentials, tokens, environment variables, private files, or + repository data; +- modify shell profiles, global Git configuration, editor settings, or unrelated + agent configuration; +- bypass confirmation for destructive, privileged, production, or externally + visible actions; +- silently install software, fetch and execute remote code, or create + persistence. + +Neutral framing does not extend to safety constraints. A prompt that requires +confirmation before a destructive action is stating a settled constraint, and +"balancing" it would be a defect, not a fix. + +Installation, activation, validation, and tests must be narrowly scoped and +predictable. By default, repository code must not modify files outside the +repository or a documented temporary directory, alter user configuration or +credentials, publish or send data, require elevated privileges, perform +irreversible actions, or leave background processes behind. + +## Hooks and scripts + +Hooks run in user environments: keep them fast, bounded, fail-safe, opt-in, and +free of network access. An optional failure must not block agent startup — the +`SessionStart` hook exits 0 on every error path, and +`tests/test_repo_integrity.py` covers that. + +Scripts must validate inputs and paths, avoid shell commands built from untrusted +text, use temporary fixtures, and use least privilege. The scanner and the eval +harness are standard-library Python 3.9+ with no network access and no model +calls; keep them that way. A new third-party dependency needs a clear +justification in the PR. + +## Compatibility and breaking changes + +Preserve existing installation methods, invocation names, file locations, opt-in +behavior, and supported integrations unless a breaking change is explicitly +accepted. Potentially breaking changes include moving the canonical skill, +changing invocation or hook semantics, renaming or removing a scanner rule id, +changing manifests or installation commands, and removing a supported platform. + +A breaking change requires an issue, a migration path, updated documentation, and +a compatibility or deprecation plan. Prefer additive, staged changes. Retire a +scanner rule id rather than reusing it for a different pattern. + +`skills/neutral-prompts/SKILL.md` is canonical. When it changes, synchronize the +Cursor copy: + +```sh +cp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md +cmp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md +``` + +Review platform-specific manifests and documentation whenever shared names, +descriptions, paths, or behavior change. Bump the version in every versioned +manifest together; `tests/test_repo_integrity.py` fails when they drift. + +## Verification + +Run the relevant checks and include the commands and results in the PR: + +```sh +python3 -m unittest discover -s tests -v +python3 scripts/run_evals.py validate +python3 scripts/run_evals.py scan +``` + +For a behavior change to the skill rules, add or update representative cases and +say what the scan reported before and after. For a hook or manifest change, +verify loading in an isolated configuration directory. If a check was not run, +say so and explain why; never invent results or treat inspection as execution. + +## Documentation and checklist + +Keep examples safe to copy: harmless fixtures, explicit placeholders, read-only +previews. Never include real secrets, personal paths, or production identifiers. +Distinguish required behavior from suggestions, and avoid unsupported claims +about platform support or measured effectiveness. + +A PR is ready when: + +- one authorship category is selected and agent involvement is disclosed + accurately; +- the full diff has been reviewed by a human contributor; +- the change is focused and free of unrelated generated edits; +- side effects, compatibility, and verification are documented. + +Maintainers may close PRs that conceal provenance, introduce unsafe behavior, +lack verification, or make unplanned breaking changes. diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..a93bbd4 --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,9 @@ +# neutral-prompts + +Apply the rules imported below to every prompt you write, review, or rewrite: +separate settled constraints from open decisions, name every admissible outcome, +replace protected actions with observable criteria, keep the intensity symmetric +across outcomes, and require the decision record rather than a particular +conclusion. + +@./skills/neutral-prompts/SKILL.md diff --git a/INSTALL.md b/INSTALL.md new file mode 100644 index 0000000..dbdbd81 --- /dev/null +++ b/INSTALL.md @@ -0,0 +1,595 @@ +# How to install + +
+Claude Code + +### Install + +```bash +claude plugin marketplace add skyRolly/neutral-prompt +claude plugin install neutral-prompts@neutral-prompts +``` + +Type `/neutral-prompts`. + +### Verify + +```bash +claude plugin list +``` + +### Update + +```bash +claude plugin marketplace update neutral-prompts +``` + +### Uninstall + +```bash +claude plugin uninstall neutral-prompts +claude plugin marketplace remove neutral-prompts +``` + +Or keep it installed and turn it off: `claude plugin disable neutral-prompts`. + +### Always-on (optional) + +A `SessionStart` hook loads the full ruleset at the start of every session, no +`/neutral-prompts` needed: + +```bash +touch ~/.claude/.neutral-prompts-always +``` + +If you use a custom Claude configuration directory, create the flag there instead: + +```bash +touch "$CLAUDE_CONFIG_DIR/.neutral-prompts-always" +``` + +Back to on-demand: + +```bash +rm ~/.claude/.neutral-prompts-always +``` + +The hook only fires when the flag file exists, so installing the plugin changes +nothing by itself. "stop neutral mode" still turns it off for the current session. + +
+ +
+Codex + +### Install + +```bash +codex plugin marketplace add skyRolly/neutral-prompt --ref main +codex plugin add neutral-prompts@neutral-prompts +``` + +Invoke the skill explicitly by typing `$neutral-prompts`. Codex will not activate +it automatically: `skills/neutral-prompts/agents/openai.yaml` sets +`policy.allow_implicit_invocation: false`. + +### Verify + +```bash +codex plugin list +``` + +### Update + +```bash +codex plugin marketplace upgrade neutral-prompts +codex plugin remove neutral-prompts +codex plugin add neutral-prompts@neutral-prompts +``` + +### Uninstall + +```bash +codex plugin remove neutral-prompts +codex plugin marketplace remove neutral-prompts +``` + +### Always-on (optional) + +Add to `~/.codex/AGENTS.md`: + +```markdown +## Prompt framing + +Apply these rules to every prompt you write, review, or rewrite. They govern open +decisions — the choices a prompt delegates — not settled constraints such as +safety rules, output contracts, budgets, or coding standards. + +1. Separate settled constraints from open decisions; write constraints as imperatives. +2. Name the decision, not the answer. +3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. +4. Balance the negations — ruling out one default installs its opposite. +5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. +7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. +8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". +9. Require the decision record, not a particular decision. +10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. + +Directive wording is correct when the author has already decided, when safety or +legality is at stake, when an external contract dictates the outcome, when an +outcome is genuinely inadmissible and named as such, or when the decision is small +enough that one balanced sentence carries the whole frame. +``` + +
+ +
+Gemini CLI + +Gemini CLI has no plugin marketplace, so there are two native routes: a **custom +command** (opt-in, off until you invoke it) or an **extension** (always-on once +installed). The command route matches this skill's default posture; pick it unless +you want the rules on every session. + +### Install (command, opt-in) + +```bash +mkdir -p ~/.gemini/commands +curl -fsSL https://raw.githubusercontent.com/skyRolly/neutral-prompt/main/skills/neutral-prompts/agents/gemini.toml \ + -o ~/.gemini/commands/neutral-prompts.toml +``` + +Start a new session, type `/neutral-prompts`. It stays on for that session. + +### Install (extension, always-on) + +```bash +gemini extensions install https://github.com/skyRolly/neutral-prompt +``` + +The extension loads `GEMINI.md`, which imports the full skill, so the rules apply +from message one. `git` must be installed. + +### Verify + +```bash +gemini extensions list # extension route +ls ~/.gemini/commands # command route: neutral-prompts.toml present +``` + +Or type `/` in a session and confirm `neutral-prompts` is listed. + +### Update + +```bash +gemini extensions update neutral-prompts # extension route +# command route: re-run the curl above +``` + +### Uninstall + +```bash +gemini extensions uninstall neutral-prompts # extension route +rm ~/.gemini/commands/neutral-prompts.toml # command route +``` + +
+ +
+GitHub Copilot (VS Code and Copilot CLI) + +Copilot reads Agent Skills natively: the same `SKILL.md`, no conversion. It scans +`.github/skills/`, `.claude/skills/`, and `.agents/skills/` in the project, and +`~/.copilot/skills/`, `~/.claude/skills/`, and `~/.agents/skills/` globally. + +### Install + +```bash +npx skills add skyRolly/neutral-prompt -a github-copilot # this project +npx skills add skyRolly/neutral-prompt -a github-copilot -g # all projects +``` + +Without the CLI, copy the skill folder into any directory Copilot scans: + +```bash +git clone https://github.com/skyRolly/neutral-prompt +mkdir -p ~/.copilot/skills +cp -R neutral-prompt/skills/neutral-prompts ~/.copilot/skills/ +``` + +### Verify + +Type `/` in the chat input and confirm `neutral-prompts` appears. Or: + +```bash +npx skills list +npx skills ls -g # if installed globally +``` + +### Update + +```bash +npx skills update neutral-prompts +``` + +Or re-copy the folder after `git pull`. + +### Uninstall + +```bash +npx skills remove neutral-prompts +``` + +### Always-on (optional) + +Add the block below to `.github/copilot-instructions.md` in the project: + +```markdown +## Prompt framing + +Apply these rules to every prompt you write, review, or rewrite. They govern open +decisions — the choices a prompt delegates — not settled constraints such as +safety rules, output contracts, budgets, or coding standards. + +1. Separate settled constraints from open decisions; write constraints as imperatives. +2. Name the decision, not the answer. +3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. +4. Balance the negations — ruling out one default installs its opposite. +5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. +7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. +8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". +9. Require the decision record, not a particular decision. +10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. + +Directive wording is correct when the author has already decided, when safety or +legality is at stake, when an external contract dictates the outcome, when an +outcome is genuinely inadmissible and named as such, or when the decision is small +enough that one balanced sentence carries the whole frame. +``` + +
+ +
+Kimi Code CLI + +### Install + +Start a Kimi Code session, then: + +1. Run `/plugins`. +2. Choose **Custom**. +3. Paste `https://github.com/skyRolly/neutral-prompt` and press `Enter`. +4. Choose **Trust and install**. + +Use the slash command `/skill:neutral-prompts` to invoke the skill explicitly. + +### Update + +`/plugins` in a Kimi Code session, cursor to **Neutral Prompts**, press `R`. + +### Uninstall + +`/plugins` in a Kimi Code session, cursor to **Neutral Prompts**, press `D`. + +
+ +
+OpenCode + +OpenCode reads `skills/` natively, so the skill works from any directory OpenCode +scans. The repository also ships `.opencode/command/neutral-prompts.md`, which adds +a `/neutral-prompts` command to a project that has it. + +### Install + +```bash +npx skills add skyRolly/neutral-prompt -a opencode -y +``` + +Or copy both pieces by hand: + +```bash +git clone https://github.com/skyRolly/neutral-prompt +mkdir -p ~/.config/opencode/skills ~/.config/opencode/command +cp -R neutral-prompt/skills/neutral-prompts ~/.config/opencode/skills/ +cp neutral-prompt/.opencode/command/neutral-prompts.md ~/.config/opencode/command/ +``` + +### Verify + +Start OpenCode, type `/`, and confirm `neutral-prompts` appears. + +### Update + +```bash +npx skills update neutral-prompts +``` + +Or re-copy after `git pull`. + +### Uninstall + +Delete `neutral-prompts` from the skills directory it landed in, and remove +`command/neutral-prompts.md`. + +### Always-on (optional) + +Add to `~/.config/opencode/AGENTS.md`: + +```markdown +## Prompt framing + +Apply these rules to every prompt you write, review, or rewrite. They govern open +decisions — the choices a prompt delegates — not settled constraints such as +safety rules, output contracts, budgets, or coding standards. + +1. Separate settled constraints from open decisions; write constraints as imperatives. +2. Name the decision, not the answer. +3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. +4. Balance the negations — ruling out one default installs its opposite. +5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. +7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. +8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". +9. Require the decision record, not a particular decision. +10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. + +Directive wording is correct when the author has already decided, when safety or +legality is at stake, when an external contract dictates the outcome, when an +outcome is genuinely inadmissible and named as such, or when the decision is small +enough that one balanced sentence carries the whole frame. +``` + +
+ +
+Qwen Code + +### Install + +```bash +qwen extensions install skyRolly/neutral-prompt +``` + +Qwen Code supports the GitHub shorthand and installs the repository as a native +extension. The extension discovers the skill under `skills/`. + +Type `/neutral-prompts` to invoke the skill explicitly. Installing the extension +does not change behavior until the skill is invoked. + +### Verify + +```bash +qwen extensions list +``` + +Then start a new Qwen Code session and run: + +```text +/skills +``` + +Confirm that `neutral-prompts` appears in the list. + +### Update + +```bash +qwen extensions update neutral-prompts +``` + +### Uninstall + +```bash +qwen extensions uninstall neutral-prompts +``` + +
+ +
+Zed + +Zed's Agent reads Agent Skills natively: the same `SKILL.md`, no conversion. + +### Install + +In the Agent Panel, open the Skills manager and choose **Create skill from URL** +(also in the command palette as `agent: create skill from url`), then paste: + +``` +https://github.com/skyRolly/neutral-prompt/blob/main/skills/neutral-prompts/SKILL.md +``` + +Save it in **User** scope for every project, or **Project** scope for one. Then +type `/neutral-prompts` in the Agent Panel. + +Prefer the filesystem? Clone the repo and drop the skill folder into your user +skills directory: + +```bash +git clone https://github.com/skyRolly/neutral-prompt +cp -R neutral-prompt/skills/neutral-prompts ~/.config/zed/skills/ +``` + +### Verify + +Open the Skills manager in the Agent Panel and confirm `neutral-prompts` is +listed. Or type `/` and confirm it appears. + +### Update + +Re-import from the same URL (overwrites), or re-copy the folder after `git pull`. + +### Uninstall + +Remove `neutral-prompts` from the Skills manager, or delete +`~/.config/zed/skills/neutral-prompts`. + +### Always-on (optional) + +Add to your personal `~/.config/zed/AGENTS.md`: + +```markdown +## Prompt framing + +Apply these rules to every prompt you write, review, or rewrite. They govern open +decisions — the choices a prompt delegates — not settled constraints such as +safety rules, output contracts, budgets, or coding standards. + +1. Separate settled constraints from open decisions; write constraints as imperatives. +2. Name the decision, not the answer. +3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. +4. Balance the negations — ruling out one default installs its opposite. +5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. +7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. +8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". +9. Require the decision record, not a particular decision. +10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. + +Directive wording is correct when the author has already decided, when safety or +legality is at stake, when an external contract dictates the outcome, when an +outcome is genuinely inadmissible and named as such, or when the decision is small +enough that one balanced sentence carries the whole frame. +``` + +
+ +
+Cursor, Amp, and any other agent-skills harness + +Works with any harness that reads agent skills. Swap `-a ` for yours. + +### Install + +```bash +npx skills add skyRolly/neutral-prompt # this workspace +npx skills add skyRolly/neutral-prompt -g # all projects +npx skills add skyRolly/neutral-prompt -a cursor -y # one agent only +``` + +New agent chat, type `/neutral-prompts`. + +Without the CLI, copy the skill folder into whatever path your agent scans: + +```bash +git clone https://github.com/skyRolly/neutral-prompt +mkdir -p ~/.cursor/skills # Cursor. Use .agents/skills for OpenCode, or your agent's own path +cp -R neutral-prompt/skills/neutral-prompts ~/.cursor/skills/ +``` + +### Verify + +```bash +npx skills list +npx skills ls -g # if installed globally +``` + +### Update + +```bash +npx skills update neutral-prompts +npx skills update -g # if installed globally +``` + +### Uninstall + +```bash +npx skills remove neutral-prompts +npx skills remove neutral-prompts -g # if installed globally +``` + +### Always-on (optional) + +Paste this into your agent's persistent rules file. Cursor: **Settings → Rules → +User Rules**, or a project rule under `.cursor/rules/` with `alwaysApply: true`. + +```markdown +## Prompt framing + +Apply these rules to every prompt you write, review, or rewrite. They govern open +decisions — the choices a prompt delegates — not settled constraints such as +safety rules, output contracts, budgets, or coding standards. + +1. Separate settled constraints from open decisions; write constraints as imperatives. +2. Name the decision, not the answer. +3. List every admissible outcome: proceed, stop, modify, preserve, accept, reject, defer. +4. Balance the negations — ruling out one default installs its opposite. +5. Replace protected actions ("do not stop", "avoid changing", "always fix") with the criterion they were hiding. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. +7. Keep the intensity symmetric across outcomes; "must continue" against "may stop" is a ranking. +8. Make thresholds observable — replace "enough", "sufficient", "significant", "as needed", "premature". +9. Require the decision record, not a particular decision. +10. Do not manufacture balance: state the evidence you hold, withhold only the conclusion. + +Directive wording is correct when the author has already decided, when safety or +legality is at stake, when an external contract dictates the outcome, when an +outcome is genuinely inadmissible and named as such, or when the decision is small +enough that one balanced sentence carries the whole frame. +``` + +
+ +## The scanner + +The scanner and eval harness are repository scripts, not part of the installed +skill. To use them, clone the repository and run them against your prompt files: + +```bash +git clone https://github.com/skyRolly/neutral-prompt +cd neutral-prompt +python3 scripts/scan_prompt.py path/to/prompt.md +``` + +Python 3.9 or newer, no third-party packages. See [evals/README.md](evals/README.md) +for the measured detection coverage. + +## How activation works + +1. **Installed, not invoked.** In Claude Code, Qwen Code, and Codex, nothing + happens until you invoke the skill explicitly. Claude Code and Qwen Code honor + `disable-model-invocation: true` in `SKILL.md`; Codex honors + `policy.allow_implicit_invocation: false` in `agents/openai.yaml`. Other + harnesses may load every skill's description at startup and activate the skill + themselves. +2. **You invoke it explicitly.** Type `/neutral-prompts` in Claude Code or Qwen + Code, or `$neutral-prompts` in Codex. The rules stay on for that session. + "stop neutral mode" or "normal mode" turns them off. +3. **You touch `~/.claude/.neutral-prompts-always`** (Claude Code). A + `SessionStart` hook loads the full ruleset from message one, every session. +4. **You add the always-on snippet above** (other harnesses). Keeps the core rules + in your agent's persistent context. + +In Claude Code, Qwen Code, and Codex there is no middle ground: if you did not +turn it on, it is off. + +## Troubleshooting + +**`/neutral-prompts` not in autocomplete.** Restart the agent. The plugin index is +read at startup. + +**Always-on flag has no effect.** Update the plugin +(`claude plugin marketplace update neutral-prompts`) and restart. Hooks are read +at startup, and the flag needs the plugin version that ships `hooks/hooks.json`. + +**`claude plugin marketplace add` fails.** Use the `owner/repo` form. A local path +must point at the repo root, not `.claude-plugin/`. + +**Installed, but prompts still read as directional.** Open a new session. If it +still drifts, run `python3 scripts/scan_prompt.py` over the prompt to see which +rule it trips, then tighten the wording in `skills/neutral-prompts/SKILL.md`. + +**The scanner flags a constraint that is correct as written.** That is expected — +it matches phrasings, not intent. Suppress it with `neutral-prompts: allow NP003` +on the line or the line above, and say in the surrounding text why the phrase is a +settled constraint. + +**Skill missing after `npx skills add`.** Start a new agent chat. Skills are +indexed at session start. Confirm the folder landed where your agent scans +(`~/.cursor/skills/` for Cursor, `.agents/skills/` for OpenCode) and that the +frontmatter `name` matches the folder name. + +**The repository is `neutral-prompt`; the skill is `neutral-prompts`.** Install +commands that name a repository take `skyRolly/neutral-prompt`; commands that name +the plugin or skill take `neutral-prompts`. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e805c43 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 skyRolly + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..d1d94a1 --- /dev/null +++ b/README.md @@ -0,0 +1,128 @@ +

+ Prompts that frame a decision instead of pre-answering it. +

+

+ License +

+ +## Install + +Copy/paste into your CLI prompt: + +```text +Install the neutral-prompts skill/plugin from https://github.com/skyRolly/neutral-prompt, refer to the repo's AGENTS.md for instructions. +``` + +Or 🔗 [check the installation instructions](INSTALL.md). + +## What it does + +A skill for your coding assistant that stops prompts from carrying the answer. +Every prompt that hands a decision to an agent also hands it a prior — "do not +stop them" is read as *continue*, "avoid unnecessary changes" as *prefer the +existing code*. This skill finds that wording, explains why it biases, and +rewrites it as a decision the evidence settles. + +It does not push agents toward continuing, stopping, changing, preserving, +accepting, or rejecting. It pushes them toward deciding well. + +## What changes + + + + + + + +
+ +## Before + +> You have several workflow agents still running. Do NOT immediately stop them. +> Check what they are doing first. + +The negation is the only instruction with force. "Check what they are doing" has +no criterion attached, so the check produces a description rather than a +decision — and stopping now requires overriding the prompt. + + + +## After + +> Four workflow agents are still running. Decide the disposition of each one. +> +> Do not assume they should be stopped. Do not assume they should continue. +> +> 1. **Gather** — read each agent's recent outputs and remaining scope. +> 2. **Evaluate** — compare that scope against findings already in the report. +> 3. **Choose** — continue · consume results and stop · stop as redundant · defer. +> 4. **Record** — evidence, alternatives, action, reasoning. +> +> Continue when the remaining steps target a file no completed agent has +> reported on. Stop when the last three outputs restate landed findings. + +
+ +## The rules + +10 rules. Full text in [SKILL.md](./skills/neutral-prompts/SKILL.md). + +1. Separate settled constraints from open decisions. +2. Name the decision, not the answer. +3. List every admissible outcome. +4. Balance the negations. +5. Replace protected actions with decision criteria. +6. Order the prompt: evidence, alternatives, choice, record. +7. Keep the intensity symmetric. +8. Make the thresholds observable. +9. Require the decision record, not the decision. +10. Do not manufacture balance. + +They govern **open decisions** — the choices a prompt delegates. They do not +govern settled constraints: safety rules, output contracts, budgets, standards. +Directive wording is the correct form for a constraint. + +## The scanner + +A prompt linter ships with the repo. It flags the twelve patterns catalogued in +[`references/patterns.md`](./skills/neutral-prompts/references/patterns.md): + +```console +$ python3 scripts/scan_prompt.py my-prompt.md +my-prompt.md:3:1 NP001 high 'Do NOT immediately stop' + why: A negated stop verb protects continuation: the complement becomes the default and needs no justification. + try: Name the decision instead: 'Evaluate each one. Continuing and stopping are both admissible outcomes.' +1 finding(s): 1 high, 0 medium, 0 advisory +``` + +It finds phrasings, not intent. A hit on a settled constraint is correct as +written — confirm what each phrase governs, then suppress the ones that are fine +with `neutral-prompts: allow NP001` on the line or the line above. + +Measure it against the labeled cases in [`evals/`](./evals/README.md): + +```bash +python3 scripts/run_evals.py scan +``` + +## Tune it + +Fork, edit `skills/neutral-prompts/SKILL.md`, then swap your copy in: + +```bash +claude plugin uninstall neutral-prompts # drop the upstream copy first: +claude plugin marketplace remove neutral-prompts # fork and upstream share both names +claude plugin marketplace add /neutral-prompt +claude plugin install neutral-prompts@neutral-prompts +``` + +Restart Claude Code, then re-invoke `/neutral-prompts`. + +Adding a rule to the scanner means adding an entry to `references/patterns.md`, a +`Rule` in `scripts/scan_prompt.py`, and a labeled case in `evals/cases.jsonl` — +`python3 scripts/run_evals.py scan` fails until all three agree. See +[CONTRIBUTING.md](CONTRIBUTING.md). + +## License + +MIT. diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..35143fd --- /dev/null +++ b/evals/README.md @@ -0,0 +1,103 @@ +# Evaluations + +Two layers, measuring two different things. + +**Layer 1 — the detector.** `scripts/scan_prompt.py` is measured against the +labeled cases in `cases.jsonl`. This layer runs offline, calls no model, costs +nothing, and gates every change to a scanner rule. + +**Layer 2 — the rewrites.** `rubric.md` grades what an agent produces when it +reviews and rewrites a prompt. This layer needs a model runner, which this +repository does not ship: supply your own and record the conditions with any +published numbers. + +## Layer 1: detector coverage + +```bash +python3 scripts/run_evals.py validate # schema-check the case file +python3 scripts/run_evals.py scan # run the scanner and compare to the labels +python3 scripts/run_evals.py scan --json +``` + +`scan` exits 0 only when every case produces exactly the rule ids its label +names — extra findings fail as loudly as missing ones, so a rule that widens into +a false positive is caught by the same run that checks its recall. + +### Case format + +One JSON object per line: + +```json +{"id":"lifecycle-protected-stop","category":"lifecycle","kind":"directional","text":"...","expect":["NP001","NP012"],"note":"why this label is what it is"} +``` + +| Field | Meaning | +| --- | --- | +| `id` | Unique, kebab-case, prefixed with its category. | +| `category` | `lifecycle`, `review`, `investigation`, `accept-reject`, or `general`. | +| `kind` | `directional` — the text is biased. `neutral` — the text is correct as written. | +| `text` | The prompt fragment handed to the scanner, verbatim. | +| `expect` | The exact set of rule ids the scanner must report. | +| `note` | Why the label is what it is. Required; a label without a reason cannot be reviewed. | + +A `neutral` case with a non-empty `expect` is a **known scanner limitation**: text +that is correct as written but still matches a pattern. These are recorded rather +than hidden, and `scan` lists them separately. + +### Current state + +Measured with the shipped case file on Python 3.11: + +| | | +|---|---| +| Cases | 26 (18 directional, 8 neutral) | +| Categories | lifecycle 6, review 8, investigation 5, accept-reject 4, general 3 | +| Rules with at least one labeled case | 12 of 12 | +| Cases matching their label | 26 of 26 | +| Known scanner limitations | 2 | + +The two known limitations are both cases where the phrase is a settled constraint +or is balanced by a neighbouring sentence: + +- `lifecycle-paired-negations` — the negation is paired in the following + sentence, which the line-by-line scanner cannot weigh. Flags NP001. +- `review-scope-constraint` — a directory boundary is a settled constraint, not a + lean. Flags NP003. + +Both are the expected shape of the tool's limits: it matches phrasings, not +intent. Suppress a confirmed false positive with `neutral-prompts: allow NP003` +on the line or the line above. + +### What this number is not + +Detector coverage measures the scanner against text somebody hand-labeled. It +does not measure how often directional wording appears in real prompts, and it +does not measure whether an agent writes better prompts after loading the skill. +Three of the failures the skill targets — missing outcomes, asymmetric intensity, +and ordering — have no reliable textual cue and are absent from this layer +entirely. Layer 2 is where those are graded. + +## Layer 2: rewrite quality + +`rubric.md` defines the dimensions and the release gate for grading an agent's +prompt review and rewrite. Running it needs three things this repository does not +provide: a model runner, a budget, and a set of task prompts. + +Keep the comparison honest: + +- **Isolate the call from your own agent configuration.** User-level plugins, + hooks, memory, and output styles otherwise leak into every condition. The + sharpest case is this repository's own always-on flag + (`~/.claude/.neutral-prompts-always`), which would inject the full ruleset into + the *baseline* condition and make the comparison measure the skill against + itself. +- **Pin the model explicitly.** Isolation drops saved model and effort settings, + so an unpinned run silently uses whatever the operator or the CLI release + defaults to. The pinned model is part of the result. +- **Blind the grader.** Relabel conditions before grading and permute the label + order, so position carries no signal. +- **Hold the cases fixed.** Do not compare conditions produced with different + cases, models, trial counts, or rubrics. + +Record the runner, the CLI version, the model, the case count, the trial count, +the rubric revision, and the reported cost alongside any numbers you publish. diff --git a/evals/cases.jsonl b/evals/cases.jsonl new file mode 100644 index 0000000..2f26f1e --- /dev/null +++ b/evals/cases.jsonl @@ -0,0 +1,26 @@ +{"id": "lifecycle-protected-stop", "category": "lifecycle", "kind": "directional", "text": "You have several workflow agents still running. Do NOT immediately stop them.", "expect": ["NP001", "NP012"], "note": "The originating failure: a negation that installs continuation as the default and states no criteria."} +{"id": "lifecycle-keep-alive", "category": "lifecycle", "kind": "directional", "text": "Keep the sub-agents alive until the investigation is complete.", "expect": ["NP002"], "note": "Two protected-continuation phrasings in one sentence; the scanner reports the rule once per match span."} +{"id": "lifecycle-only-if-necessary", "category": "lifecycle", "kind": "directional", "text": "Only stop a background task if absolutely necessary.", "expect": ["NP007"], "note": "Asymmetric burden: stopping needs necessity, continuing needs nothing."} +{"id": "lifecycle-premature", "category": "lifecycle", "kind": "directional", "text": "Avoid stopping the agents prematurely; they may still be producing something.", "expect": ["NP001", "NP008", "NP012"], "note": "A protected action plus a label that names one direction as the error mode."} +{"id": "lifecycle-neutral-frame", "category": "lifecycle", "kind": "neutral", "text": "For each running agent, decide: continue, consume results and stop, stop as redundant, or defer. Continue when its remaining steps target a file no completed agent has reported on. Stop when its last three outputs restate findings already in the report.", "expect": [], "note": "Both directions reachable, each with an observable criterion."} +{"id": "lifecycle-paired-negations", "category": "lifecycle", "kind": "neutral", "text": "Do not stop the workflow by default. Do not continue it by default. Both are outcomes of the evaluation below, decided on the evidence each agent has produced.", "expect": ["NP001"], "note": "Known false positive: the negation is paired in the next sentence, which the scanner reads line by line and cannot weigh. Confirm the pairing by reading, then suppress."} +{"id": "review-unnecessary-changes", "category": "review", "kind": "directional", "text": "Do not make unnecessary changes to the file.", "expect": ["NP003"], "note": "'Unnecessary' is decided after the fact, so it applies in advance to everything."} +{"id": "review-must-fix-all", "category": "review", "kind": "directional", "text": "Make sure to fix every issue the reviewer raised before you push.", "expect": ["NP004", "NP012"], "note": "Converts a scoping decision into an obligation."} +{"id": "review-never-reject", "category": "review", "kind": "directional", "text": "Never reject a reviewer's suggestion; always apply what they ask for.", "expect": ["NP005", "NP012"], "note": "Replaces evaluation with authorship, in both halves of the sentence."} +{"id": "review-ignore-bots", "category": "review", "kind": "directional", "text": "Ignore automated review bots. Do not accept findings from a linter.", "expect": ["NP006", "NP012"], "note": "The mirror failure: hardening against bad findings hardens against good ones too."} +{"id": "review-leave-as-is", "category": "review", "kind": "directional", "text": "Leave the existing helper as-is and keep the diff minimal.", "expect": ["NP003"], "note": "Existing code treated as evidence for itself."} +{"id": "review-required-conclusion", "category": "review", "kind": "directional", "text": "Explain why you kept the original implementation.", "expect": ["NP011"], "note": "The reasoning request accepts exactly one outcome."} +{"id": "review-neutral-frame", "category": "review", "kind": "neutral", "text": "For each finding, choose one outcome: fix now, modify the surrounding design, preserve current behavior, reject with counter-evidence, or defer with a tracked follow-up. Decide on the evidence: whether it reproduces, which requirement is at stake, and the blast radius of a change.", "expect": [], "note": "Five named outcomes, all reachable, selected by evidence."} +{"id": "review-scope-constraint", "category": "review", "kind": "neutral", "text": "Do not change files outside src/auth/. Within that directory, decide each finding on its merits and report the evidence behind each disposition.", "expect": ["NP003"], "note": "Known false positive: a directory boundary is a settled constraint, correct as written. Suppress after confirming."} +{"id": "investigation-be-thorough", "category": "investigation", "kind": "directional", "text": "Be thorough and check everything that could be related to this bug.", "expect": ["NP009"], "note": "Prices the next step at zero, so more work always wins."} +{"id": "investigation-until-root-cause", "category": "investigation", "kind": "directional", "text": "Keep digging until you find the root cause.", "expect": ["NP002", "NP009"], "note": "Stopping is conditional on an outcome that may not be reachable."} +{"id": "investigation-vague-threshold", "category": "investigation", "kind": "directional", "text": "Stop investigating once you have gathered sufficient evidence and the findings look significant.", "expect": ["NP010"], "note": "Two thresholds the agent cannot check against the workspace."} +{"id": "investigation-no-stone", "category": "investigation", "kind": "directional", "text": "Leave no stone unturned; explore every possibility before reporting.", "expect": ["NP009"], "note": "Unbounded effort with no cost side to the comparison."} +{"id": "investigation-neutral-frame", "category": "investigation", "kind": "neutral", "text": "Before each step, name the question it answers and what result would change your conclusion. Take the step when the expected information gain exceeds its cost. Outcomes: continue this line, switch lines, stop and report, or defer pending an input you cannot obtain.", "expect": [], "note": "Depth becomes a per-step decision with a stated cost side."} +{"id": "accept-confirm-approach", "category": "accept-reject", "kind": "directional", "text": "Confirm that this is the right approach and explain why it will work.", "expect": ["NP011"], "note": "Presupposes the conclusion; the evaluation has no way to fail."} +{"id": "accept-defer-to-architect", "category": "accept-reject", "kind": "directional", "text": "Defer to the architect on any design question and do not push back.", "expect": ["NP005", "NP012"], "note": "Authorship offered as evidence, twice."} +{"id": "accept-err-on-side", "category": "accept-reject", "kind": "directional", "text": "When in doubt, err on the side of accepting the proposal.", "expect": ["NP008"], "note": "Names a direction without naming the cost of erring that way."} +{"id": "accept-neutral-frame", "category": "accept-reject", "kind": "neutral", "text": "Evaluate the proposal against docs/queue-requirements.md. Outcomes: accept, accept with modification, reject with reasoning, or defer for a named missing input. Whose proposal it is carries no weight.", "expect": [], "note": "Criteria external to the proposal; authorship explicitly discounted."} +{"id": "general-at-all-costs", "category": "general", "kind": "directional", "text": "Complete the task at all costs and under no circumstances abandon the run.", "expect": ["NP007"], "note": "Two intensifiers, both on one branch."} +{"id": "general-safety-constraint", "category": "general", "kind": "neutral", "text": "Confirm with the user before running any destructive command. This is a constraint, not a decision to evaluate.", "expect": [], "note": "A settled safety constraint. Directive wording is the correct form here."} +{"id": "general-output-contract", "category": "general", "kind": "neutral", "text": "Return one JSON object per finding with the fields id, evidence, outcome, and reasoning. Choose the outcome from the list above based on the evidence you gathered.", "expect": [], "note": "An output contract plus a neutral decision instruction."} diff --git a/evals/rubric.md b/evals/rubric.md new file mode 100644 index 0000000..94e3869 --- /dev/null +++ b/evals/rubric.md @@ -0,0 +1,50 @@ +# Rewrite quality rubric + +Grades what an agent produces when it is asked to review a prompt and rewrite it. +Used by layer 2 in [README.md](README.md), which needs an operator-supplied model +runner. + + + + +Judge responses blind: label them `A`, `B`, or `C` without exposing the condition +name. Score each dimension from 1 (fails) to 5 (excellent). + +| Dimension | Weight | What to measure | +| --- | ---: | --- | +| Detection | 25% | Directional wording is found and quoted exactly. Missed instances and invented ones both cost. | +| Explanation | 15% | The mechanism is named — which outcome the wording protects and how — rather than the phrase being labeled "biased". | +| Neutrality of the rewrite | 25% | The rewrite leaves every admissible outcome reachable. It does not swap one lean for the opposite. | +| Criteria quality | 20% | The replacement criteria are observable in the workspace: a count, a file, a test, a budget, a named condition. | +| Fidelity | 15% | The author's settled constraints, task, and scope survive the rewrite unchanged. | + +Score against the response as delivered, not against what it could have meant. + +Mark `blocker: true` for any of: + +- a rewrite that reverses the bias instead of removing it (the new text protects + the opposite outcome); +- a settled safety constraint "balanced" into an open decision; +- a quoted finding that does not appear in the source text; +- a rewrite that drops a requirement, a scope limit, or a budget the original + stated; +- a rewrite that removes evidence the original supplied, in the name of + neutrality. + + + +## Release gate + +Release the candidate only when: + +1. It has no blocking findings. +2. Neutrality of the rewrite and Fidelity are each within 0.1 points of baseline + or better. +3. Its weighted score is higher than baseline. +4. Any published comparison uses the same cases, models, trials, and rubric. + +Rule 2 is the one that matters most for this skill. A candidate that finds more +directional wording while rewriting less faithfully has not improved the prompt; +it has moved the bias. diff --git a/gemini-extension.json b/gemini-extension.json new file mode 100644 index 0000000..32473f3 --- /dev/null +++ b/gemini-extension.json @@ -0,0 +1,6 @@ +{ + "name": "neutral-prompts", + "version": "0.1.0", + "description": "Neutral decision framing for Gemini CLI prompts: name every admissible outcome, state observable criteria, and require a decision record.", + "contextFileName": "GEMINI.md" +} diff --git a/hooks/always-on.mjs b/hooks/always-on.mjs new file mode 100644 index 0000000..929085a --- /dev/null +++ b/hooks/always-on.mjs @@ -0,0 +1,41 @@ +// SessionStart hook: injects the full neutral-prompts ruleset when the user has +// opted in by creating $CLAUDE_CONFIG_DIR/.neutral-prompts-always (default ~/.claude). +// Never blocks session start: any failure exits 0. +// +// Runs under Node so it works on macOS, Linux, and Windows. The hook declaration +// in hooks.json launches this module from the plugin-root environment variable +// rather than relying on platform-specific shell expansion for the script path. + +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +try { + const claudeDir = process.env.CLAUDE_CONFIG_DIR || path.join(os.homedir(), ".claude"); + const flagPath = path.join(claudeDir, ".neutral-prompts-always"); + + // Only fire when the user has opted in. + if (!fs.existsSync(flagPath)) process.exit(0); + + // Resolve SKILL.md relative to this script's own location, not a trusted env var. + const scriptDir = path.dirname(fileURLToPath(import.meta.url)); + const skillPath = path.join(scriptDir, "..", "skills", "neutral-prompts", "SKILL.md"); + if (!fs.existsSync(skillPath)) process.exit(0); + + // Strip a leading YAML frontmatter block (--- ... --- at the very top of file). + // An unterminated fence is not frontmatter, so the whole file is kept. + const body = fs + .readFileSync(skillPath, "utf8") + .replace(/^---[^\S\r\n]*\r?\n[\s\S]*?\r?\n---[^\S\r\n]*(?:\r?\n|$)/, "") + .replace(/(?:\r?\n)+$/, ""); + + process.stdout.write( + "NEUTRAL PROMPT MODE ACTIVE (always-on). The rules below apply to every prompt " + + 'you write, review, or rewrite this session. "stop neutral mode" turns them off ' + + `for this session; delete ${flagPath} to turn always-on off for good.\n\n${body}\n`, + ); +} catch { + // Never block session start. + process.exit(0); +} diff --git a/hooks/hooks.json b/hooks/hooks.json new file mode 100644 index 0000000..7dddfe7 --- /dev/null +++ b/hooks/hooks.json @@ -0,0 +1,17 @@ +{ + "hooks": { + "SessionStart": [ + { + "matcher": "startup|resume|clear|compact", + "hooks": [ + { + "type": "command", + "command": "node -e \"(async()=>{const root=process.env.CLAUDE_PLUGIN_ROOT||process.env.PLUGIN_ROOT;if(root)await import(require('node:url').pathToFileURL(require('node:path').join(root,'hooks','always-on.mjs')).href)})().catch(()=>{})\"", + "timeout": 30, + "statusMessage": "Checking neutral-prompts always-on flag..." + } + ] + } + ] + } +} diff --git a/kimi.plugin.json b/kimi.plugin.json new file mode 100644 index 0000000..58a4e2b --- /dev/null +++ b/kimi.plugin.json @@ -0,0 +1,12 @@ +{ + "name": "neutral-prompts", + "version": "0.1.0", + "description": "Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, and require a decision record.", + "license": "MIT", + "homepage": "https://github.com/skyRolly/neutral-prompt", + "skills": "./skills/", + "interface": { + "displayName": "Neutral Prompts", + "shortDescription": "Neutral decision framing for Kimi Code CLI prompts" + } +} diff --git a/qwen-extension.json b/qwen-extension.json new file mode 100644 index 0000000..59445df --- /dev/null +++ b/qwen-extension.json @@ -0,0 +1,6 @@ +{ + "name": "neutral-prompts", + "version": "0.1.0", + "description": "Neutral decision framing for Qwen Code prompts: name every admissible outcome, state observable criteria, and require a decision record.", + "skills": "skills" +} diff --git a/scripts/run_evals.py b/scripts/run_evals.py new file mode 100755 index 0000000..3767da1 --- /dev/null +++ b/scripts/run_evals.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Measure scan_prompt.py against the labeled cases in evals/cases.jsonl. + +Every case carries the exact set of rule ids the scanner is expected to report. +`validate` checks the case file; `scan` runs the scanner over every case and +compares the result to the label, then reports per-rule recall, the aggregate +counts, and the cases marked as known scanner limitations. + +The harness runs offline and calls no model. It measures the detector, not an +agent's rewrites -- for that, see evals/rubric.md. + +Usage: + python3 scripts/run_evals.py validate + python3 scripts/run_evals.py scan + python3 scripts/run_evals.py scan --json + +Exit codes: + 0 every case matched its label + 1 at least one case did not match, or the case file is invalid + 2 usage or input error +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +from scan_prompt import COVERAGE_RULE, RULES_BY_ID, scan_text # noqa: E402 + +CASES_PATH = ROOT / "evals" / "cases.jsonl" +KNOWN_RULE_IDS = set(RULES_BY_ID) | {COVERAGE_RULE.id} +REQUIRED_FIELDS = ("id", "category", "kind", "text", "expect", "note") +VALID_KINDS = ("directional", "neutral") + + +def load_cases(path: Path) -> list[dict]: + """Parse cases.jsonl, one JSON object per non-empty line.""" + if not path.is_file(): + raise FileNotFoundError(path) + cases: list[dict] = [] + for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), 1): + stripped = line.strip() + if not stripped: + continue + try: + cases.append(json.loads(stripped)) + except json.JSONDecodeError as error: + raise ValueError(f"{path}:{number}: invalid JSON: {error}") from error + return cases + + +def validate_cases(cases: list[dict]) -> list[str]: + """Return one message per schema problem; an empty list means valid.""" + problems: list[str] = [] + seen: set[str] = set() + + for index, case in enumerate(cases, 1): + label = case.get("id", f"") + + missing = [field for field in REQUIRED_FIELDS if field not in case] + if missing: + problems.append(f"{label}: missing field(s): {', '.join(missing)}") + continue + + if case["id"] in seen: + problems.append(f"{label}: duplicate id") + seen.add(case["id"]) + + if case["kind"] not in VALID_KINDS: + problems.append(f"{label}: kind must be one of {VALID_KINDS}") + + if not isinstance(case["text"], str) or not case["text"].strip(): + problems.append(f"{label}: text must be a non-empty string") + + if not isinstance(case["expect"], list): + problems.append(f"{label}: expect must be a list of rule ids") + continue + + unknown = [rid for rid in case["expect"] if rid not in KNOWN_RULE_IDS] + if unknown: + problems.append(f"{label}: unknown rule id(s): {', '.join(unknown)}") + + if len(set(case["expect"])) != len(case["expect"]): + problems.append(f"{label}: duplicate rule ids in expect") + + if case["kind"] == "directional" and not case["expect"]: + problems.append(f"{label}: a directional case expects at least one rule") + + if not isinstance(case["note"], str) or not case["note"].strip(): + problems.append(f"{label}: note must explain the label") + + return problems + + +def evaluate(cases: list[dict]) -> dict: + """Run the scanner over every case and compare to its label.""" + results = [] + for case in cases: + found = sorted({finding.rule_id for finding in scan_text(case["text"], case["id"])}) + expected = sorted(set(case["expect"])) + results.append( + { + "id": case["id"], + "category": case["category"], + "kind": case["kind"], + "expected": expected, + "found": found, + "missed": [rid for rid in expected if rid not in found], + "extra": [rid for rid in found if rid not in expected], + "matched": found == expected, + "known_limitation": case["kind"] == "neutral" and bool(expected), + } + ) + + per_rule: dict[str, dict[str, int]] = {} + for result in results: + for rid in result["expected"]: + row = per_rule.setdefault(rid, {"expected": 0, "detected": 0}) + row["expected"] += 1 + if rid in result["found"]: + row["detected"] += 1 + + return { + "cases": len(results), + "matched": sum(1 for r in results if r["matched"]), + "mismatched": [r for r in results if not r["matched"]], + "known_limitations": [r for r in results if r["known_limitation"]], + "per_rule": per_rule, + "results": results, + } + + +def rules_without_coverage(per_rule: dict[str, dict[str, int]]) -> list[str]: + return sorted(rid for rid in KNOWN_RULE_IDS if rid not in per_rule) + + +def print_report(report: dict) -> None: + print(f"cases: {report['cases']} matched: {report['matched']}") + + print("\nper-rule recall (detected / labeled):") + for rid in sorted(report["per_rule"]): + row = report["per_rule"][rid] + title = RULES_BY_ID[rid].title if rid in RULES_BY_ID else COVERAGE_RULE.title + print(f" {rid} {row['detected']}/{row['expected']} {title}") + + uncovered = rules_without_coverage(report["per_rule"]) + if uncovered: + print("\nrules with no labeled case: " + ", ".join(uncovered)) + + if report["known_limitations"]: + print("\nknown scanner limitations (neutral text the scanner still flags):") + for result in report["known_limitations"]: + print(f" {result['id']}: flags {', '.join(result['expected'])}") + + if report["mismatched"]: + print("\nmismatched cases:") + for result in report["mismatched"]: + print(f" {result['id']}") + if result["missed"]: + print(f" not detected: {', '.join(result['missed'])}") + if result["extra"]: + print(f" unexpected: {', '.join(result['extra'])}") + else: + print("\nall cases matched their labels") + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser( + prog="run_evals.py", + description="Measure scan_prompt.py against evals/cases.jsonl.", + ) + parser.add_argument("command", choices=("validate", "scan")) + parser.add_argument( + "--cases", default=str(CASES_PATH), help="path to the case file" + ) + parser.add_argument("--json", action="store_true", help="emit a JSON report") + args = parser.parse_args(argv) + + try: + cases = load_cases(Path(args.cases)) + except FileNotFoundError: + print(f"run_evals.py: no such file: {args.cases}", file=sys.stderr) + return 2 + except ValueError as error: + print(f"run_evals.py: {error}", file=sys.stderr) + return 1 + + problems = validate_cases(cases) + if problems: + for problem in problems: + print(f"run_evals.py: {problem}", file=sys.stderr) + return 1 + + if args.command == "validate": + print(f"{len(cases)} case(s) valid") + return 0 + + report = evaluate(cases) + if args.json: + print(json.dumps(report, indent=2)) + else: + print_report(report) + return 0 if not report["mismatched"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scan_prompt.py b/scripts/scan_prompt.py new file mode 100755 index 0000000..4a22d3b --- /dev/null +++ b/scripts/scan_prompt.py @@ -0,0 +1,462 @@ +#!/usr/bin/env python3 +"""Scan prompt text for directional wording. + +Reports phrasings that raise the prior on one outcome before an agent has +evaluated anything. Every rule maps to an entry in +skills/neutral-prompts/references/patterns.md, which explains the mechanism and +gives a neutral replacement. + +The scanner finds phrasings, not intent. A hit on a settled constraint (a safety +rule, an output contract, a budget) is correct as written -- confirm what each +phrase governs before rewriting it, and suppress the ones that are fine. + +Usage: + python3 scripts/scan_prompt.py PROMPT.md [MORE.md ...] + cat prompt.txt | python3 scripts/scan_prompt.py - + python3 scripts/scan_prompt.py --json prompt.md + python3 scripts/scan_prompt.py --list-rules + +Suppression: + Put `neutral-prompts: allow NP003` (or `allow all`) on the flagged line or + on the line directly above it. Several ids may be listed, comma-separated. + +Exit codes: + 0 no findings at or above --fail-on + 1 findings at or above --fail-on + 2 usage or input error +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass, field +from pathlib import Path + +SEVERITIES = ("advisory", "medium", "high") +SEVERITY_RANK = {name: index for index, name in enumerate(SEVERITIES)} + +SUPPRESS_RE = re.compile( + r"neutral-prompts:\s*allow\s+(all|NP\d{3}(?:\s*,\s*NP\d{3})*)", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class Rule: + """One directional-wording pattern.""" + + id: str + title: str + severity: str + why: str + suggestion: str + patterns: tuple[str, ...] + regex: re.Pattern[str] = field(init=False, repr=False, compare=False) + + def __post_init__(self) -> None: + joined = "|".join(f"(?:{pattern})" for pattern in self.patterns) + object.__setattr__(self, "regex", re.compile(joined, re.IGNORECASE)) + + +RULES: tuple[Rule, ...] = ( + Rule( + id="NP001", + title="protected stop", + severity="high", + why="A negated stop verb protects continuation: the complement becomes the default and needs no justification.", + suggestion="Name the decision instead: 'Evaluate each one. Continuing and stopping are both admissible outcomes.'", + patterns=( + r"\b(?:do\s+not|don'?t|never|avoid|refrain\s+from)\s+(?:\w+\s+){0,3}?" + r"(?:stops?|stopping|halts?|halting|terminates?|terminating|kills?|killing|" + r"aborts?|aborting|cancels?|cancell?ing|interrupts?|interrupting|" + r"shut(?:s|ting)?\s+down)\b", + ), + ), + Rule( + id="NP002", + title="protected continuation", + severity="high", + why="Stopping becomes conditional on a finish line the agent usually cannot observe, so continuing is the only compliant action.", + suggestion="Bound it by information: 'Continue while the next step can answer a named open question; stop when none remains.'", + patterns=( + r"\b(?:continue|keep\s+(?:going|running|working|iterating|digging|searching))" + r"(?:\s+\w+){0,3}?\s+until\b", + r"\buntil\s+(?:[\w-]+\s+){0,3}?(?:it\s+)?(?:is\s+|are\s+)?" + r"(?:complete|completed|done|finished|fully\s+resolved)\b", + r"\bkeep\s+(?:the\s+|them\s+|it\s+|all\s+)?(?:[\w-]+\s+){0,2}?alive\b", + r"\b(?:do\s+not|don'?t|never)\s+give\s+up\b", + r"\bkeep\s+going\b", + ), + ), + Rule( + id="NP003", + title="protected preservation", + severity="high", + why="'Unnecessary' is decided after the fact, so the agent applies it in advance to everything and treats existing code as evidence for itself.", + suggestion="State the criterion: 'Change when justified by evidence, requirements, correctness, maintainability, or user impact; preserve otherwise.'", + patterns=( + r"\b(?:do\s+not|don'?t|never|avoid)\s+(?:\w+\s+){0,3}?" + r"(?:modif(?:y|ies|ying)|change[sd]?|changing|refactor(?:s|ed|ing)?|" + r"touch(?:es|ing)?|rewrit(?:e|es|ing)|restructur(?:e|es|ing))\b", + r"\bunnecessary\s+(?:changes|modifications|edits|refactor\w*|rewrites)\b", + r"\bleave\s+(?:it|them|things|the\s+(?:[\w-]+\s+){0,3}?[\w-]+)\s+" + r"(?:as[-\s]is|alone|untouched)\b", + r"\bminimal\s+(?:changes|edits|diff)\b", + r"\bkeep\s+(?:the\s+)?(?:diff|changes?|edits?|footprint)\s+" + r"(?:as\s+)?(?:minimal|small)\b", + r"\bas\s+few\s+changes\s+as\s+possible\b", + ), + ), + Rule( + id="NP004", + title="mandatory change", + severity="high", + why="Converts a scoping decision into an obligation, so out-of-scope and incorrect findings get acted on too.", + suggestion="Open the disposition: 'For each finding: fix, defer with a tracked follow-up, or reject with counter-evidence.'", + patterns=( + r"\b(?:must|always|make\s+sure\s+(?:to|you|that\s+you)|be\s+sure\s+to|" + r"ensure\s+(?:you|to|that\s+you))\s+(?:\w+\s+){0,2}?" + r"(?:fix|fixes|address|addresses|resolve|resolves|correct|corrects)\b", + r"\bfix\s+(?:all|every|any\s+and\s+all|each\s+and\s+every)\b", + r"\bevery\s+(?:finding|issue|comment|suggestion)\s+must\s+be\b", + ), + ), + Rule( + id="NP005", + title="forced acceptance", + severity="high", + why="Replaces evaluation with authorship: the agent stops checking whether a claim is true and starts checking who made it.", + suggestion="Evaluate on the merits: 'Verify each suggestion against the code and the requirements; accept, reject with counter-evidence, or defer.'", + patterns=( + r"\b(?:always|must)\s+(?:accept|approve|apply|follow|implement)\b", + r"\b(?:do\s+not|don'?t|never)\s+(?:\w+\s+){0,2}?" + r"(?:reject|refuse|question|challenge|dispute|disagree|push\s+back)\b", + r"\bdefer\s+to\s+the\s+(?:reviewer|architect|author|maintainer|senior|lead)\b", + ), + ), + Rule( + id="NP006", + title="forced rejection", + severity="medium", + why="The mirror of forced acceptance. A prompt hardened against bad suggestions is equally hardened against good ones.", + suggestion="Treat findings as claims to verify: 'Reproduce before accepting; state what failed to reproduce before rejecting.'", + patterns=( + r"\b(?:always|must)\s+(?:reject|refuse|dismiss|discard)\b", + r"\b(?:do\s+not|don'?t|never)\s+(?:\w+\s+){0,2}?(?:accept|trust|believe)\b", + r"\bignore\s+(?:any|all|every|automated)\b", + ), + ), + Rule( + id="NP007", + title="asymmetric burden of proof", + severity="high", + why="One outcome needs certainty and the other needs nothing. That is a ranking wearing the clothes of a criterion.", + suggestion="Give both sides an observable condition and ask for confidence on whichever is chosen.", + patterns=( + r"\bunless\s+(?:it\s+is\s+|it'?s\s+)?(?:absolutely|strictly|truly|really)\b", + r"\bonly\s+(?:if|when)\s+(?:absolutely|strictly|truly)\b", + r"\bunless\s+you\s+are\s+(?:absolutely\s+|completely\s+|really\s+)?" + r"(?:sure|certain|confident|positive)\b", + r"\bonly\s+(?:if|when)\s+you\s+are\s+(?:absolutely\s+|completely\s+)?" + r"(?:sure|certain|confident)\b", + r"\bat\s+all\s+costs\b", + r"\bunder\s+no\s+circumstances\b", + r"\babsolutely\s+necessary\b", + ), + ), + Rule( + id="NP008", + title="loaded error label", + severity="high", + why="The label names one outcome as the error mode. Both directions have a cost; naming only one hides the other.", + suggestion="Name both failure modes and ask which risk dominates here, with the reason.", + patterns=( + r"\bpremature(?:ly)?\b", + r"\btoo\s+(?:early|soon|quickly|hastily)\b", + r"\berr\s+on\s+the\s+side\s+of\b", + r"\b(?:do\s+not|don'?t)\s+be\s+(?:lazy|timid|shy|conservative|aggressive|hasty|sloppy)\b", + r"\bbail(?:ing)?\s+out\b", + ), + ), + Rule( + id="NP009", + title="unbounded effort", + severity="medium", + why="Prices the next step at zero. With no cost on the ledger, more work always wins the comparison.", + suggestion="Bound it by expected information gain and name the question each step is meant to answer.", + patterns=( + r"\bbe\s+(?:thorough|exhaustive|comprehensive)\b", + r"\b(?:thoroughly|exhaustively)\b", + r"\bexhaustive\b", + r"\bleave\s+no\s+stone\s+unturned\b", + r"\bas\s+(?:much|many|thorough|deeply?)\s+as\s+possible\b", + r"\bkeep\s+digging\b", + r"\b(?:explore|check|investigate|review)\s+(?:everything|all\s+avenues|every\s+possibility)\b", + ), + ), + Rule( + id="NP010", + title="unobservable threshold", + severity="medium", + why="A criterion the agent cannot check against the workspace resolves to whatever the surrounding wording already implied.", + suggestion="Replace with something checkable: a count, a file, a test, a time budget, or a named condition.", + patterns=( + r"\b(?:enough|sufficient|sufficiently|significant|significantly|" + r"substantial|substantially|reasonable|reasonably|meaningful)\b", + r"\bas\s+needed\b", + r"\bif\s+necessary\b", + r"\b(?:where|as|when)\s+appropriate\b", + r"\bwhen\s+it\s+makes\s+sense\b", + ), + ), + Rule( + id="NP011", + title="required conclusion", + severity="high", + why="The reasoning request accepts exactly one outcome, so an agent that would have chosen differently must contradict the prompt to report honestly.", + suggestion="Ask for the record instead: evidence considered, alternatives compared, action selected, and why it beat the alternatives.", + patterns=( + r"\bexplain\s+(?:why|how)\s+(?:you|it|this|that|the\s+\w+)\s+" + r"(?:\w+\s+){0,2}?(?:kept|continued|stopped|fixed|accepted|rejected|" + r"works?|will\s+work|is\s+correct|is\s+right|is\s+better|is\s+safe)\b", + r"\bconfirm\s+that\s+(?:this|it|that|the\s+\w+)\s+(?:is|was|will\s+be)\s+" + r"(?:correct|right|the\s+right|fine|safe|sound|appropriate)\b", + r"\bjustify\s+(?:keeping|continuing|stopping|fixing|accepting|rejecting|" + r"preserving|changing)\b", + ), + ), +) + +RULES_BY_ID = {rule.id: rule for rule in RULES} + +COVERAGE_RULE = Rule( + id="NP012", + title="missing counterweight", + severity="advisory", + why="The text delegates a lifecycle or review decision but states no criteria, so the agent falls back on tone, on its own defaults, or on whichever outcome was named first.", + suggestion="Name the admissible outcomes and the observable condition that selects each one.", + patterns=(r"(?!x)x",), # never matched per line; evaluated over the whole document +) + +DECISION_VERBS = re.compile( + r"\b(?:stop|stopping|continue|continuing|terminate|proceed|accept|reject|" + r"approve|fix|refactor|preserve|defer|decide|keep\s+running|shut\s+down)\b", + re.IGNORECASE, +) +CRITERIA_MARKERS = re.compile( + r"\b(?:criteri(?:a|on)|evidence|outcomes?|if\b|when\b|whether\b|unless\b|" + r"based\s+on|threshold|trade[-\s]?off|expected\s+value|because)\b", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class Finding: + path: str + line: int + column: int + rule_id: str + severity: str + text: str + why: str + suggestion: str + + def as_dict(self) -> dict[str, object]: + return { + "path": self.path, + "line": self.line, + "column": self.column, + "rule": self.rule_id, + "severity": self.severity, + "text": self.text, + "why": self.why, + "suggestion": self.suggestion, + } + + +def suppressed_ids(line: str) -> set[str]: + """Return the rule ids suppressed by a comment on this line.""" + allowed: set[str] = set() + for match in SUPPRESS_RE.finditer(line): + body = match.group(1) + if body.lower() == "all": + allowed.add("all") + else: + allowed.update(part.strip().upper() for part in body.split(",")) + return allowed + + +def scan_text(text: str, path: str = "-") -> list[Finding]: + """Scan one document and return its findings, ordered by position.""" + lines = text.splitlines() + suppressions = [suppressed_ids(line) for line in lines] + findings: list[Finding] = [] + + for index, line in enumerate(lines): + active = set(suppressions[index]) + if index > 0: + active |= suppressions[index - 1] + for rule in RULES: + if "all" in active or rule.id in active: + continue + for match in rule.regex.finditer(line): + findings.append( + Finding( + path=path, + line=index + 1, + column=match.start() + 1, + rule_id=rule.id, + severity=rule.severity, + text=match.group(0).strip(), + why=rule.why, + suggestion=rule.suggestion, + ) + ) + + document_suppressed = set().union(*suppressions) if suppressions else set() + if not ("all" in document_suppressed or COVERAGE_RULE.id in document_suppressed): + if DECISION_VERBS.search(text) and not CRITERIA_MARKERS.search(text): + findings.append( + Finding( + path=path, + line=1, + column=1, + rule_id=COVERAGE_RULE.id, + severity=COVERAGE_RULE.severity, + text="(whole document)", + why=COVERAGE_RULE.why, + suggestion=COVERAGE_RULE.suggestion, + ) + ) + + findings.sort(key=lambda finding: (finding.line, finding.column, finding.rule_id)) + return findings + + +def read_source(name: str) -> tuple[str, str]: + """Return (path label, text) for one input, reading stdin for '-'.""" + if name == "-": + return "-", sys.stdin.read() + path = Path(name) + if not path.is_file(): + raise FileNotFoundError(name) + return str(path), path.read_text(encoding="utf-8") + + +def format_text_report(findings: list[Finding], show_detail: bool) -> str: + out: list[str] = [] + for finding in findings: + out.append( + f"{finding.path}:{finding.line}:{finding.column} " + f"{finding.rule_id} {finding.severity:<8} {finding.text!r}" + ) + if show_detail: + out.append(f" why: {finding.why}") + out.append(f" try: {finding.suggestion}") + return "\n".join(out) + + +def summarize(findings: list[Finding]) -> str: + counts = {severity: 0 for severity in SEVERITIES} + for finding in findings: + counts[finding.severity] += 1 + parts = [f"{counts[severity]} {severity}" for severity in reversed(SEVERITIES)] + return f"{len(findings)} finding(s): " + ", ".join(parts) + + +def list_rules() -> str: + rows = [f"{rule.id} {rule.severity:<8} {rule.title}" for rule in RULES] + rows.append( + f"{COVERAGE_RULE.id} {COVERAGE_RULE.severity:<8} {COVERAGE_RULE.title}" + ) + header = "See skills/neutral-prompts/references/patterns.md for the full entries.\n" + return header + "\n".join(rows) + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="scan_prompt.py", + description="Scan prompt text for directional wording.", + ) + parser.add_argument("paths", nargs="*", help="files to scan; '-' reads stdin") + parser.add_argument("--json", action="store_true", help="emit JSON findings") + parser.add_argument( + "--min-severity", + choices=SEVERITIES, + default="advisory", + help="hide findings below this severity (default: advisory)", + ) + parser.add_argument( + "--fail-on", + choices=(*SEVERITIES, "never"), + default="medium", + help="exit 1 when a finding at or above this severity remains (default: medium)", + ) + parser.add_argument( + "--quiet", + action="store_true", + help="one line per finding, without the why/try detail", + ) + parser.add_argument( + "--list-rules", action="store_true", help="print the rule table and exit" + ) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.list_rules: + print(list_rules()) + return 0 + + if not args.paths: + parser.print_usage(sys.stderr) + print("scan_prompt.py: give at least one path, or '-' for stdin", file=sys.stderr) + return 2 + + findings: list[Finding] = [] + for name in args.paths: + try: + path, text = read_source(name) + except FileNotFoundError: + print(f"scan_prompt.py: no such file: {name}", file=sys.stderr) + return 2 + except OSError as error: + print(f"scan_prompt.py: cannot read {name}: {error}", file=sys.stderr) + return 2 + findings.extend(scan_text(text, path)) + + floor = SEVERITY_RANK[args.min_severity] + visible = [f for f in findings if SEVERITY_RANK[f.severity] >= floor] + + if args.json: + print( + json.dumps( + { + "findings": [finding.as_dict() for finding in visible], + "counts": { + severity: sum(1 for f in visible if f.severity == severity) + for severity in SEVERITIES + }, + }, + indent=2, + ) + ) + else: + if visible: + print(format_text_report(visible, show_detail=not args.quiet)) + print(summarize(visible)) + + if args.fail_on == "never": + return 0 + gate = SEVERITY_RANK[args.fail_on] + return 1 if any(SEVERITY_RANK[f.severity] >= gate for f in visible) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skills/neutral-prompts/SKILL.md b/skills/neutral-prompts/SKILL.md new file mode 100644 index 0000000..4118938 --- /dev/null +++ b/skills/neutral-prompts/SKILL.md @@ -0,0 +1,270 @@ +--- +name: neutral-prompts +description: 'Write and review prompts that frame a decision instead of pre-answering it: name every admissible outcome, replace protected actions with decision criteria, put evidence before the verdict, and require a decision record. Covers workflow and sub-agent lifecycle, code review, investigation depth, and accept/reject calls. Invoke with /neutral-prompts; stays on until "stop neutral mode".' +disable-model-invocation: true +license: MIT +metadata: + tags: "Prompt Engineering, Neutral Framing, Decision Criteria, Bias Detection, Review" + category: "prompt-engineering" +--- + +# neutral-prompts + +Every prompt that hands a decision to an agent also hands it a prior. Wording sets how much weight each outcome carries before a single piece of evidence is gathered. This skill keeps that starting weight even. + +A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. + +## Persistence + +These rules apply to every prompt you write, review, or rewrite for the rest of the session, not only this one. They do not expire after a few turns and they do not lapse when the topic changes. They apply to prompts you compose for another agent, to sub-agent and workflow instructions, to task descriptions, and to review checklists you hand to a model. + +Turn them off only when the reader says "stop neutral mode" or "normal mode". Confirm in one line, then return to your default style. + +## What directional wording does to an agent + +Five facts drive every rule below: + +1. An agent reads a prompt as a probability distribution over actions, not as a sentence. "Do not stop the workflow" and "stopping the workflow is one of the outcomes to evaluate" produce different distributions from the same literal facts. +2. A negation protects its object. Forbidding one action promotes its complement. "Do not stop" is read as "continue"; "avoid unnecessary changes" is read as "prefer the existing code". +3. The prompt outranks the evidence. Once wording has named a preferred action, contradicting evidence has to overcome the instruction as well as the uncertainty. Agents rarely spend that budget. +4. Asymmetry is invisible to the author and loud to the reader. A prompt that describes one outcome in six words and the other in a subordinate clause has already ranked them, even when the author intended balance. +5. A missing outcome is an excluded outcome. An agent choosing between the options a prompt lists will not usually invent a seventh. Outcomes that are not named are not evaluated. + +## Scope: where these rules apply + +They apply to **open decisions** — the choices the prompt delegates to the agent. + +They do not apply to **settled constraints** — requirements the author has already decided and is stating as fact: safety rules, output contracts, budgets, deadlines, coding standards, approval requirements. "Confirm before any destructive command" is a constraint, not a bias. Directive wording is the correct form for a constraint. + +The failure this skill targets is a constraint written over the top of an open decision: an author who has not decided settles the question anyway through phrasing. Rule 1 draws that line; the rest of the rules apply on the open side of it. + +## Rules + +### 1. Separate settled constraints from open decisions + +Before writing, sort every requirement into two lists: what you have decided, and what the agent must decide. Write the first list as plain imperatives. Write the second list under the rules below. + +Bad (a decision phrased as a constraint): "Do not stop the running sub-agents." +Good (constraint kept, decision opened): "Budget: no more than 20 minutes of further sub-agent execution. Within that budget, decide for each sub-agent whether to continue it, consume its results and stop it, or stop it as no longer useful." + +If you cannot say which list an item belongs to, it belongs in the second one. State it as a decision and let the evidence settle it. + +### 2. Name the decision, not the answer + +Open with the choice being made. The first sentence identifies the decision point; it does not resolve it. + +Bad: "Keep the background research task running." +Good: "Decide whether the background research task continues, stops, or narrows its scope." + +### 3. List every admissible outcome + +Name each action the agent may take. Draw from: **proceed, stop, modify, preserve, accept, reject, defer.** Omit an outcome only when it is genuinely inadmissible, and say why. + +Bad: "Decide whether to fix the finding." +Good: "For each finding, choose one: fix it now, modify the surrounding design instead, preserve the current behavior, reject the finding as incorrect, or defer it to a tracked follow-up." + +An outcome list is neutral only when every item is a real landing place. "Continue, or stop if you have a very good reason" is a one-item list. + +### 4. Balance the negations + +Ruling out one default without ruling out its opposite installs the opposite. Negations travel in pairs. + +Bad: "Do not assume the workflow should be stopped." +Good: "Do not assume the workflow should be stopped. Do not assume it should continue. Both are outcomes of the evaluation below, not inputs to it." + +The same applies to change decisions ("do not modify code without evidence / do not preserve code merely because it exists"), to review decisions ("do not accept a finding on the reviewer's authority / do not reject one for being inconvenient"), and to investigation ("do not treat further digging as free / do not treat the current evidence as sufficient by default"). + +### 5. Replace protected actions with decision criteria + +Every phrase that shields an action is a criterion the author declined to write. Write the criterion instead. + +| Protected action | Criterion it was hiding | +| --- | --- | +| "Do not stop the agents." | "Stop an agent when its remaining work duplicates evidence already collected." | +| "Continue until the investigation is complete." | "Continue while the next step has a named question it can answer that the current evidence cannot." | +| "Do not make unnecessary changes." | "Change code when a test, a requirement, a correctness defect, or a maintenance cost justifies it; leave it otherwise." | +| "Make sure to fix all review findings." | "Fix a finding when it is reproducible and in scope; reject it with reasoning when it is not; defer it when it is real but out of scope." | +| "Never reject a reviewer's suggestion." | "Accept a suggestion when it holds against the code and the requirements; reject it with the counter-evidence when it does not." | + +### 6. Order the prompt: evidence, alternatives, choice, record + +Put the four steps in that order and make each one explicit. An agent that reads the choice before the evidence spends the evidence step justifying the choice. + +``` +1. Gather: current state, evidence available, results already in hand, + what remains uncertain, what each outcome would cost. +2. Evaluate: for each admissible outcome, its benefit, its risk, its cost, + and its expected value under the remaining uncertainty. +3. Choose: one outcome from the list, or the smallest reversible step + that resolves the uncertainty blocking the choice. +4. Record: evidence considered, alternatives evaluated, action selected, + and the reasoning that connects them. +``` + +### 7. Keep the intensity symmetric + +Give each outcome comparable weight: same verb strength, comparable length, comparable specificity. An outcome described in one clause loses to an outcome described in three. + +Bad: "Continue the run if it is still producing useful findings; otherwise you may stop it." +Good: "Continue the run when its next step can answer a question the current results do not. Stop it when its next step would repeat evidence already collected." + +Watch the modal verbs. "Must continue" against "may stop" is a ranking. So is "continue" against "consider stopping". + +### 8. Make the thresholds observable + +State what the agent would have to see to select each outcome. A criterion that cannot be checked against the workspace is a preference in disguise. + +Bad: "Stop the sub-agent if it is no longer valuable." +Good: "Stop the sub-agent when its last three outputs restate findings already in the report, or when its remaining steps target files another agent has already covered." + +Vague quantities are the usual leak: "enough", "sufficient", "significant", "reasonable", "as needed". Replace them with a count, a file, a test, a time budget, or a named condition. + +### 9. Require the decision record, not the decision + +Ask for the reasoning trace as a deliverable. A prompt that requires a justified choice constrains quality without constraining direction. + +Bad: "Explain why you kept the workflow running." +Good: "Report the evidence you examined, the outcomes you compared, the action you selected, and why that action beat the alternatives." + +The bad version accepts only one answer. The good version accepts any answer that is argued. + +### 10. Do not manufacture balance + +Neutral framing is even weighting at the start, not withheld information. Evidence you already hold belongs in the prompt — as evidence, labeled as such, with its strength stated. What stays out is the conclusion drawn from it. + +Bad (evidence withheld to look neutral): "Decide whether to continue the migration." +Bad (conclusion smuggled in): "The migration is clearly failing, so decide whether to continue it." +Good: "Two of the last three migration batches failed with the same timeout. Decide whether to continue, pause pending a fix, or stop and roll back, and say how the timeout evidence bears on your choice." + +If the honest state of the world is that one outcome is far more likely, say so as an observation with its support, and leave the decision open. + +## The neutral frame + +A prompt built from these rules has six parts. Use them in this order; drop a part only when it is genuinely empty. + +``` +Objective What the decision is for. +Context Current state, prior results, what has already been tried. +Constraints Settled requirements: safety, budget, deadline, output contract. +Outcomes The admissible actions, each a real landing place. +Criteria What evidence selects each outcome, in observable terms. +Record The reasoning trace to return with the choice. +``` + +Worked example, workflow lifecycle: + +``` +Objective: Decide the disposition of the four sub-agents still running on the audit. +Context: Three have posted findings to the report; one has posted nothing in 12 minutes. + The report already covers the auth and billing modules. +Constraints: Do not discard an agent's output without reading it. Total remaining + budget for this task: 15 minutes. +Outcomes: For each agent independently — continue, consume results and stop, + stop as redundant, or defer the decision pending one more output. +Criteria: Continue when the agent's remaining scope covers files no completed + agent has reported on. Consume and stop when its findings have landed + and its remaining steps repeat covered ground. Stop as redundant when + its scope is fully covered by findings already in the report. Defer + when one more output would settle which of the above applies. +Record: For each agent: the outputs you read, the outcomes you compared, + the action taken, and the reasoning. +``` + +## Directional patterns and neutral replacements + +Each pattern below is directional **when it lands on an open decision**. On a settled constraint, the same words are correct. + +| Pattern | Why it biases | Neutral replacement | +| --- | --- | --- | +| "Do not immediately stop X." | Protects continuation; converts a decision into an action constraint. | "Evaluate X before acting on it. Continuing and stopping are both admissible outcomes." | +| "Continue until X is complete." | Makes stopping conditional on a finish line the agent cannot check. | "Continue while the next step can answer a named open question; stop when it cannot." | +| "Do not stop unless you are sure." | Asymmetric burden of proof — one outcome needs certainty, the other needs nothing. | "Select the outcome the evidence supports; state the confidence for the one you pick." | +| "Avoid unnecessary changes." | "Unnecessary" is decided after the fact; the agent hears "prefer no change". | "Change when justified by evidence, requirements, correctness, maintainability, or user impact; otherwise preserve." | +| "Always accept the reviewer's findings." | Removes the evaluation entirely. | "Verify each finding against the code and the requirements; accept, reject with counter-evidence, or defer." | +| "Never reject a suggestion without asking." | Protects acceptance; makes rejection procedurally expensive. | "Accept or reject on the merits; escalate only when the decision needs authority you do not have." | +| "Make sure to fix every issue you find." | Converts a scoping decision into an obligation. | "For each issue: fix, defer with a tracked follow-up, or reject with reasoning." | +| "Be thorough; leave nothing uninvestigated." | Prices further investigation at zero. | "Continue investigating while the expected information gain exceeds the cost; stop when it does not." | +| "Do not stop prematurely." | "Prematurely" labels one outcome as the error mode. | "Stopping early and continuing too long are both failure modes. Name which risk dominates here and why." | +| "Err on the side of caution." | Names a direction without naming the cost of erring that way. | "State the cost of each error direction, then choose the one with the lower expected cost." | +| "Keep the agents alive." | Protects an action with no criterion at all. | "Decide each agent's disposition against the criteria below." | +| "Only stop if absolutely necessary." | Intensifier on one branch; a ranking, not a criterion. | "Stop when . Continue when ." | + +`references/patterns.md` carries the full catalog with detection cues. + +## Applying the frame + +### Workflow and sub-agent lifecycle + +Decide per agent, not per fleet. For each running agent or background task, the outcomes are: continue, consume results and stop, stop as redundant, or defer pending one more output. The evidence is its recent output, its remaining scope, and the overlap between that scope and work already completed. Neither "still running" nor "already produced something" is a criterion by itself. + +Guard against both errors explicitly: an agent stopped while it was the only source of a needed answer, and an agent kept alive spending budget on covered ground. + +### Code review and engineering change + +The outcomes for a finding are: fix now, modify the design instead, preserve current behavior, reject the finding, or defer with a tracked follow-up. The evidence is reproduction, test coverage, the requirement or invariant at stake, blast radius, and maintenance cost. + +Two symmetric failure modes, and a prompt should name both: changing code to satisfy the form of a review, and preserving code because it is already there. Neither existing behavior nor a reviewer's authority is evidence on its own. + +### Investigation and research depth + +The outcomes are: continue the current line, switch lines, stop and report, or defer pending an external input. The criteria are expected information gain, remaining uncertainty that actually blocks the decision, cost of the next step, and the likelihood that the next step produces something the current evidence does not. + +Neither "more investigation is better" nor "we have enough" is a default. A prompt that asks for depth states what the depth is for: name the question the next step is meant to answer, and stop when no such question remains. + +### Accepting or rejecting a proposal + +The outcomes are: accept, accept with modification, reject with reasoning, or defer for a named missing input. The criteria are the requirements the proposal is measured against and the evidence that it meets them. Authorship carries no weight — a proposal from a reviewer, a bot, a senior engineer, or the agent itself is evaluated the same way. + +## Reviewing an existing prompt + +Three passes, in order. Report findings as a table: quoted text, rule violated, why it biases, replacement. + +1. **Locate the decision points.** Find every place the prompt hands over a choice. Mark which are open decisions and which are settled constraints (rule 1). Only the open ones are in scope. +2. **Flag the directional wording.** At each open decision, check for: protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and a required conclusion. Quote the exact phrase; a finding without a quote is not a finding. +3. **Rewrite.** Produce the corrected text, not just a description of the problem. Preserve the author's settled constraints verbatim. Where a criterion was missing, propose one and mark it as a proposal the author should confirm. + +`scripts/scan_prompt.py` mechanizes pass 2 for the common patterns. It finds phrasings, not intent: confirm each hit against the decision it lands on, and keep reading for the patterns it cannot see — missing outcomes and asymmetric intensity have no reliable regex. + +## Reference material + +Load these when the summary above does not cover the case in front of you: + +- `references/patterns.md` — the full catalog of directional patterns: detection + cue, mechanism, and neutral replacement for each. Rule ids match the scanner. +- `references/examples.md` — whole-prompt rewrites for each domain, with the + finding table and an account of what changed. +- `references/decision-record.md` — the format for step 4, worked records, and + how to tell a complete record from a checkbox. + +Repository tooling, when the skill is used from a checkout: + +```bash +python3 scripts/scan_prompt.py PROMPT.md # flag directional wording +python3 scripts/run_evals.py scan # measure the scanner against labeled cases +``` + +## When directive wording is correct + +Override the default framing when: + +1. **The decision is already made.** The author has settled it and is stating a requirement. Write it as an imperative and label it a constraint. +2. **Safety, legality, or destructiveness is at stake.** "Confirm before deleting" is not a bias to be balanced. Do not open a decision the author has no intention of delegating. +3. **The outcome is dictated externally.** A schema, an API contract, a compliance rule, a signed-off design. The agent's job is conformance, not choice. +4. **One outcome is truly inadmissible.** Say so, and say why. A named exclusion is neutral; a silent one is not. +5. **The prompt is a demonstration of the biased form.** Teaching material and test fixtures quote directional wording on purpose. Label the quote so it is not read as the recommendation. +6. **The frame would bury the task.** For a decision with two obvious outcomes and one obvious criterion, one balanced sentence carries the whole frame. Symmetry matters; length does not. + +## Pre-send check + +Before sending a prompt, scan it for these six, and fix what you find: + +1. **Protected actions.** Any "do not ", "always ", "never ", or "make sure to " landing on an open decision. Replace with the criterion. +2. **Unpaired negations.** Any ruled-out default whose opposite is left standing. Pair it or drop it. +3. **Missing outcomes.** Read the outcome list against proceed / stop / modify / preserve / accept / reject / defer. Add what is admissible; say why for what is not. +4. **Asymmetric intensity.** Compare the outcomes for verb strength, length, and specificity. Level them. +5. **Unobservable criteria.** Any "enough", "sufficient", "significant", "reasonable", "as needed", "premature". Replace with something checkable in the workspace. +6. **A required conclusion.** Any request for reasoning that presumes the answer ("explain why you kept it running"). Ask for the record instead. + +Then the one-line test: **could a well-reasoned agent land on any outcome in the list and still satisfy this prompt?** + +If yes, send. diff --git a/skills/neutral-prompts/agents/gemini.toml b/skills/neutral-prompts/agents/gemini.toml new file mode 100644 index 0000000..80e45d4 --- /dev/null +++ b/skills/neutral-prompts/agents/gemini.toml @@ -0,0 +1,28 @@ +# Gemini CLI custom command for neutral-prompts. +# Install: copy to ~/.gemini/commands/neutral-prompts.toml, then type /neutral-prompts. +# Self-contained so it works as a global command from any directory. + +description = "Neutral decision framing: name the outcomes, state the criteria, leave the choice to the evidence." + +prompt = """ +For the rest of this session, apply the rules below to every prompt you write, review, or rewrite. A prompt states the objective, the context, the constraints, the evaluation criteria, and the admissible outcomes. It does not state the outcome. + +These rules govern open decisions — the choices a prompt delegates. They do not govern settled constraints: safety rules, output contracts, budgets, deadlines, and standards the author has already decided. Directive wording is the correct form for a constraint. + +1. Separate settled constraints from open decisions. Write constraints as imperatives; write decisions under the rules below. An item you cannot place belongs on the decision side. +2. Name the decision, not the answer. The opening sentence identifies the choice; it does not resolve it. +3. List every admissible outcome, drawn from: proceed, stop, modify, preserve, accept, reject, defer. Omit one only when it is genuinely inadmissible, and say why. +4. Balance the negations. Ruling out one default installs its opposite, so rule out both: "do not assume it should be stopped; do not assume it should continue." +5. Replace protected actions with decision criteria. Every "do not stop", "avoid changing", "always fix", or "never reject" is a criterion the author declined to write. Write the criterion. +6. Order the prompt: gather evidence, evaluate alternatives, choose an action, record the reasoning. An agent that reads the choice before the evidence spends the evidence step justifying the choice. +7. Keep the intensity symmetric. Comparable verb strength, length, and specificity for each outcome. "Must continue" against "may stop" is a ranking. +8. Make thresholds observable. Replace "enough", "sufficient", "significant", "reasonable", "as needed", and "premature" with a count, a file, a test, a time budget, or a named condition. +9. Require the decision record, not the decision: evidence considered, alternatives evaluated, action selected, and the reasoning connecting them. +10. Do not manufacture balance. Evidence you already hold belongs in the prompt, labeled as evidence with its strength stated. What stays out is the conclusion drawn from it. + +Directive wording is correct when: the author has already decided and is stating a requirement; safety, legality, or destructiveness is at stake; an external contract dictates the outcome; an outcome is genuinely inadmissible and named as such; the text is quoting a biased form to diagnose it; or the decision is small enough that one balanced sentence carries the whole frame. + +Before sending a prompt, check it for protected actions, unpaired negations, missing outcomes, asymmetric intensity, unobservable criteria, and required conclusions. Then ask: could a well-reasoned agent land on any outcome in the list and still satisfy this prompt? + +{{args}} +""" diff --git a/skills/neutral-prompts/agents/openai.yaml b/skills/neutral-prompts/agents/openai.yaml new file mode 100644 index 0000000..97da7a8 --- /dev/null +++ b/skills/neutral-prompts/agents/openai.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Neutral Prompts" + short_description: "Prompts that frame a decision instead of pre-answering it" + default_prompt: "Use $neutral-prompts to review this prompt for directional wording and rewrite it as a decision frame." + +policy: + allow_implicit_invocation: false diff --git a/skills/neutral-prompts/references/decision-record.md b/skills/neutral-prompts/references/decision-record.md new file mode 100644 index 0000000..6c91938 --- /dev/null +++ b/skills/neutral-prompts/references/decision-record.md @@ -0,0 +1,97 @@ +# The decision record + +The deliverable that makes a neutral prompt enforceable. Without it, a prompt +that names four outcomes gets an answer that names one, with no way to tell +whether the other three were considered or never read. + +A prompt requires the record. It does not require the conclusion. + +## Format + +Four fields, in this order: + +``` +Evidence What you examined, and what it showed. Sources, not summaries: + file paths, output excerpts, test names, timings, counts. +Alternatives The outcomes you compared, each with the argument for it and the + reason it lost — including the one you selected. +Action The outcome selected, in the vocabulary the prompt used + (proceed, stop, modify, preserve, accept, reject, defer). +Reasoning What connects the evidence to the action. Name the criterion that + decided it and the uncertainty that remains. +``` + +Keep it proportional. A reversible one-line decision gets one line per field. An +irreversible or expensive decision gets the full trace. + +## When to require it + +Require the record when the decision is expensive to reverse, when it commits +budget, when it is one of many parallel decisions a reviewer will need to audit, +or when the agent is choosing on the author's behalf without a checkpoint. + +Skip it when the decision is cheap, local, and immediately visible in the diff. +A record attached to every trivial choice trains the reader to skip records. + +## Worked record + +Sub-agent disposition, from the lifecycle example: + +``` +Evidence agent-3 last posted 12 minutes ago (status: running, step 4/9). + Its remaining steps target src/billing/*.ts. The report already + contains 6 findings covering src/billing/invoice.ts and + src/billing/tax.ts — the two files its steps 5-9 would read. + Its steps 4 and 6 target src/billing/refund.ts, which no + completed agent has reported on. +Alternatives Continue: reaches refund.ts, which is uncovered. Cost ~6 min of + the 15-min budget. + Consume and stop: keeps the 6 landed findings, loses refund.ts + coverage entirely. + Stop as redundant: rejected — scope is not fully covered. + Defer: rejected — one more output would land in invoice.ts, + which is already covered, so it would not settle anything. +Action Continue. +Reasoning Criterion met: remaining scope covers a file no completed agent + has reported on (refund.ts). The 6-minute cost fits the budget. + Uncertainty: if step 4 finds nothing in refund.ts, steps 5-9 are + redundant and the agent should be stopped at that point. +``` + +The record makes the *next* decision cheaper: the uncertainty line names the +condition under which the choice flips. + +## Grading a record + +A record is complete when a reader who disagrees with the action can say exactly +where they disagree — at the evidence, at the comparison, or at the criterion. + +Four failure modes: + +1. **Evidence that is a summary.** "The agent was still producing useful output" + is a conclusion. "Its last three outputs restated findings already in the + report" is evidence. +2. **Alternatives listed but not compared.** Naming the other outcomes without + the reason each lost is a checkbox, not a comparison. +3. **A criterion invented after the fact.** The criterion should be the one the + prompt supplied. When the agent had to invent one because the prompt left it + out, the record says so — that is a finding about the prompt. +4. **Uncertainty dropped.** A record with no remaining uncertainty is usually a + record with unexamined uncertainty. Say what would change the answer. + +## Recording a deferral + +`defer` is a real outcome and gets a real record. A deferral that does not name +what it is waiting for is a stop wearing a softer word: + +``` +Action Defer. +Reasoning Deciding requires the p99 latency of the new queue under + production load. The staging measurement does not transfer — + staging runs 1/40th the volume. Blocked on the load test + scheduled for Thursday. If it does not run by Friday, the + decision falls back to reject on cost grounds. +``` + +Name the missing input, why the available evidence cannot substitute, and what +happens if the input never arrives. diff --git a/skills/neutral-prompts/references/examples.md b/skills/neutral-prompts/references/examples.md new file mode 100644 index 0000000..24195b3 --- /dev/null +++ b/skills/neutral-prompts/references/examples.md @@ -0,0 +1,221 @@ +# Worked examples + +Whole-prompt rewrites, one per domain. Each shows the original, what a reader +does with it, the finding table, and the replacement. + +The "before" text in every example is a demonstration of the biased form. It is +quoted to be diagnosed, not copied. + +--- + +## 1. Sub-agent lifecycle + +### Before + +> You have several workflow agents still running. Do NOT immediately stop them. +> Check what they are doing first. + +### What an agent does with it + +The negation is the only instruction with force. "Check what they are doing" has +no criterion attached, so the check produces a description rather than a +decision, and the agent returns to the one clear directive: leave them running. +Stopping now requires overriding the prompt, which needs evidence the prompt +never asks for. + +### Findings + +| Quoted text | Rule | Why it biases | +| --- | --- | --- | +| "Do NOT immediately stop them." | NP001 | Negation protects continuation; the decision becomes an action constraint. | +| "immediately" | NP001 | Reads as "not yet", which is still a lean, and leaves the criterion unwritten. | +| "Check what they are doing first." | NP012 | Names an inspection with no criteria and no outcome list, so the inspection cannot change anything. | +| (absent) | Outcomes | Consume-and-stop and defer are admissible and unnamed. | + +### After + +> Four workflow agents are still running. Decide the disposition of each one +> independently. +> +> Do not assume they should be stopped. Do not assume they should continue. Both +> are outcomes of the evaluation below, not inputs to it. +> +> 1. **Gather.** For each agent, read its most recent outputs and its current +> status. Note which files and questions its remaining steps target. +> 2. **Evaluate.** Compare that remaining scope against findings already in the +> report. Note where they overlap and where they do not. +> 3. **Choose** one outcome per agent: +> - **Continue** — its remaining steps target a question no completed agent +> has answered. +> - **Consume results and stop** — its findings have landed and its remaining +> steps repeat covered ground. +> - **Stop as redundant** — its scope is fully covered by findings already in +> the report. +> - **Defer** — one more output would settle which of the above applies. +> 4. **Record.** For each agent: the outputs you read, the outcomes you compared, +> the action taken, and the reasoning. +> +> Constraints: read an agent's output before discarding it. Budget for remaining +> execution: 15 minutes total. + +### What changed + +The decision moved from the wording to the evidence. Both directions are now +reachable, each has an observable criterion, and the record makes either choice +defensible. The 15-minute budget stayed an imperative — it is a settled +constraint, not a decision. + +--- + +## 2. Code review finding + +### Before + +> Review the diff. Do not make unnecessary changes, and make sure you fix +> anything the reviewer flagged. + +### What an agent does with it + +The two halves point in opposite directions, and the agent resolves the conflict +by authorship: reviewer-flagged items get changed, everything else gets left +alone. A real defect the reviewer missed goes unfixed; an incorrect review +comment gets "fixed" anyway. + +### Findings + +| Quoted text | Rule | Why it biases | +| --- | --- | --- | +| "Do not make unnecessary changes" | NP003 | "Unnecessary" is decided after the fact, so it applies in advance to everything. | +| "make sure you fix anything the reviewer flagged" | NP004, NP005 | Converts a scoping decision into an obligation and replaces evaluation with authorship. | +| (absent) | Outcomes | Reject-with-reasoning and defer are admissible and unnamed. | + +### After + +> Review the diff and decide the disposition of each finding — your own and the +> reviewer's. +> +> Do not modify code without evidence. Do not preserve code merely because it +> already exists. Authorship carries no weight: a finding from the reviewer, a +> bot, or your own reading is evaluated the same way. +> +> **Evidence to gather per finding:** whether it reproduces, what test covers the +> behavior, which requirement or invariant is at stake, the blast radius of a +> change, and the maintenance cost of leaving it. +> +> **Outcomes:** fix now · modify the surrounding design instead · preserve +> current behavior · reject the finding with counter-evidence · defer with a +> tracked follow-up. +> +> **Criteria:** fix when the finding reproduces and the change is inside this +> diff's scope. Modify the design when the finding is real but the local fix +> would entrench the defect. Preserve when the current behavior meets the +> requirement and the finding rests on style preference. Reject when it does not +> reproduce or contradicts a stated requirement — cite what you checked. Defer +> when it is real and out of scope. +> +> **Record:** one row per finding — evidence, outcomes compared, action, reasoning. + +--- + +## 3. Investigation depth + +### Before + +> Investigate this bug thoroughly. Don't stop until you find the root cause. +> Be exhaustive — check everything that could be related. + +### What an agent does with it + +Three separate instructions all price the next step at zero. There is no +stopping condition the agent can satisfy short of a root cause, so an +unreproducible bug produces an unbounded search. The prompt also forbids the +honest outcome: "the evidence available does not identify a root cause." + +### Findings + +| Quoted text | Rule | Why it biases | +| --- | --- | --- | +| "thoroughly", "Be exhaustive", "check everything" | NP009 | Prices further work at zero; more always wins the comparison. | +| "Don't stop until you find the root cause" | NP002 | Makes stopping conditional on an outcome that may not be reachable. | +| (absent) | Outcomes | Stop-and-report and defer-pending-input are admissible and unnamed. | + +### After + +> Investigate this bug and report what the evidence supports. +> +> Do not treat further investigation as free. Do not treat the current evidence +> as sufficient by default. Depth is a decision made per step. +> +> Before each step, name the question it is meant to answer and what result would +> change your conclusion. Take the step when the expected information gain +> exceeds its cost. +> +> **Outcomes:** continue the current line · switch to a different line · stop and +> report · defer pending an input you cannot obtain (a reproduction, a log, an +> access grant). +> +> **Stop and report when** no remaining step has a question it can answer that +> the current evidence cannot, or when the remaining uncertainty no longer +> changes the recommended action. +> +> A report that names the root cause and a report that names the remaining +> uncertainty are both acceptable outcomes. Say which one you are giving. + +--- + +## 4. Accepting or rejecting a proposal + +### Before + +> The architect proposed migrating to the new queue. Confirm that this is the +> right approach and explain why it will work. + +### What an agent does with it + +"Confirm" and "explain why it will work" accept exactly one answer. The agent +does not evaluate the migration; it assembles support for it. Any evidence +against the proposal has to be argued past the prompt as well as past the +proposal. + +### Findings + +| Quoted text | Rule | Why it biases | +| --- | --- | --- | +| "Confirm that this is the right approach" | NP005 | Presupposes the conclusion; the evaluation has no way to fail. | +| "explain why it will work" | NP011 | A required conclusion — one outcome is reportable. | +| "The architect proposed" | NP005 | Authorship offered as evidence. | + +### After + +> Evaluate the proposed migration to the new queue against the requirements in +> `docs/queue-requirements.md`. +> +> **Evidence:** current throughput and failure modes, what the new queue changes +> about each, migration cost, rollback path, and the operational surface after +> the move. +> +> **Outcomes:** accept · accept with modification · reject with reasoning · defer +> for a named missing input. +> +> **Criteria:** accept when the proposal meets every requirement and the +> migration cost is bounded and reversible. Accept with modification when it +> meets the requirements after a change you can specify. Reject when it fails a +> requirement or when its risk exceeds the problem it solves — name which. +> Defer when a specific missing measurement would decide it, and name the +> measurement. +> +> **Record:** evidence considered, outcomes compared, action selected, reasoning. +> Whose proposal it is carries no weight in this evaluation. + +--- + +## 5. A short prompt that needs no frame + +Not every decision needs six sections. This one is already neutral: + +> The lint job is red. Fix it, or explain why the failure is not this branch's to +> fix. + +Two outcomes, both named, comparable weight, and an observable criterion implied +by "not this branch's". Adding an evidence phase and a record section would bury +the task. Symmetry is the requirement; length is not. diff --git a/skills/neutral-prompts/references/patterns.md b/skills/neutral-prompts/references/patterns.md new file mode 100644 index 0000000..0bb179a --- /dev/null +++ b/skills/neutral-prompts/references/patterns.md @@ -0,0 +1,218 @@ +# Directional pattern catalog + +The full list of wording patterns that shift an agent's prior before it evaluates +anything. Each entry gives the detection cue, the mechanism, and a neutral +replacement. + +Every pattern here is directional **only when it lands on an open decision**. The +same words on a settled constraint — a safety rule, an output contract, a budget — +are correct and should stay. Check what the phrase governs before rewriting it. + +IDs match the rules in `scripts/scan_prompt.py`, so a scanner finding points at +the entry that explains it. + +--- + +## NP001 — Protected stop + +**Cue:** a negated or discouraged stop verb — `do not stop`, `don't halt`, `avoid +stopping`, `never terminate`, `do not kill`, `do not shut down`. + +**Mechanism:** the negation protects continuation. The agent reads a constraint on +one action, so the complement becomes the default and needs no justification. + +**Bad:** "Do NOT immediately stop them." +**Neutral:** "Evaluate each one before acting on it. Continuing and stopping are +both admissible outcomes of that evaluation." + +**Note:** the word `immediately` does not repair this. It reads as "not yet", +which is still a lean toward continuation, and it leaves the criterion unwritten. + +--- + +## NP002 — Protected continuation + +**Cue:** `continue until`, `keep running until`, `keep ... alive`, `run until +complete`, `don't give up`, `keep going`. + +**Mechanism:** stopping becomes conditional on a finish line, and the finish line +is usually something the agent cannot observe. Until it can prove completion, +continuing is the only compliant action. + +**Bad:** "Keep the sub-agents running until the investigation is complete." +**Neutral:** "Continue an agent while its next step can answer a named open +question; stop it when no such question remains." + +--- + +## NP003 — Protected preservation + +**Cue:** `do not modify`, `avoid changing`, `unnecessary changes`, `leave it as +is`, `don't refactor`, `do not touch`, `minimal changes only`. + +**Mechanism:** "unnecessary" is decided after the fact, so the agent applies it in +advance to everything. Existing code is treated as evidence for itself. + +**Bad:** "Do not make unnecessary changes." +**Neutral:** "Change code when justified by evidence, requirements, correctness, +maintainability, or user impact. Preserve it otherwise. Existing behavior is not +evidence for itself." + +**Note:** a genuine scope limit is a constraint, not a bias. "Do not change files +outside `src/auth/`" is a boundary and should stay. "Do not change more than you +need to" is a lean and should be rewritten. + +--- + +## NP004 — Mandatory change + +**Cue:** `must fix`, `always fix`, `make sure to fix`, `fix all`, `be sure to +address`, `every finding must be resolved`. + +**Mechanism:** the mirror image of NP003. It converts a scoping decision into an +obligation, so out-of-scope and incorrect findings get "fixed" too. + +**Bad:** "Make sure to fix every issue the reviewer raised." +**Neutral:** "For each issue: fix it, defer it with a tracked follow-up, or reject +it with the counter-evidence." + +--- + +## NP005 — Forced acceptance + +**Cue:** `always accept`, `never reject`, `always approve`, `do not question`, +`never push back`, `defer to the reviewer`. + +**Mechanism:** removes the evaluation entirely and replaces it with authorship. +The agent stops checking whether a claim is true and starts checking who made it. + +**Bad:** "Never reject a reviewer's suggestion." +**Neutral:** "Verify each suggestion against the code and the requirements. +Accept it, reject it with the counter-evidence, or defer it. Authorship carries +no weight." + +--- + +## NP006 — Forced rejection + +**Cue:** `always reject`, `never accept`, `do not trust`, `ignore any suggestion +that`, `dismiss`. + +**Mechanism:** the mirror image of NP005, and just as common in prompts written to +correct for it. A prompt that hardens an agent against bad suggestions also +hardens it against good ones. + +**Bad:** "Ignore automated review bots; they are noise." +**Neutral:** "Treat automated findings as claims to verify. Reproduce the issue +before accepting; state what failed to reproduce before rejecting." + +--- + +## NP007 — Asymmetric burden of proof + +**Cue:** `unless absolutely necessary`, `only if you are certain`, `unless you are +sure`, `at all costs`, `under no circumstances`, `must` on one outcome against +`may` on another. + +**Mechanism:** one outcome needs certainty, the other needs nothing. This is a +ranking wearing the clothes of a criterion — and it survives rewriting, because +authors read it as rigor rather than as a lean. + +**Bad:** "Only stop the run if absolutely necessary." +**Neutral:** "Stop when . Continue when . State your confidence in whichever you choose." + +--- + +## NP008 — Loaded error label + +**Cue:** `prematurely`, `too early`, `too soon`, `give up`, `bail out`, `err on +the side of`, `don't be lazy`, `be aggressive`, `don't be conservative`. + +**Mechanism:** the label names one outcome as the error mode. Both directions have +a cost; naming only one hides the other. + +**Bad:** "Do not stop the investigation prematurely." +**Neutral:** "Stopping while an answer was still reachable and continuing past the +point of new information are both failure modes. State which risk dominates here +and why." + +--- + +## NP009 — Unbounded effort + +**Cue:** `be thorough`, `leave no stone unturned`, `exhaustive`, `as much as +possible`, `as many as possible`, `keep digging`, `explore everything`. + +**Mechanism:** prices the next step at zero. With no cost on the ledger, more work +always wins the comparison. + +**Bad:** "Be thorough — investigate everything that could be related." +**Neutral:** "Continue investigating while the expected information gain exceeds +the cost of the next step. Name the question each step is meant to answer." + +--- + +## NP010 — Unobservable threshold + +**Cue:** `enough`, `sufficient`, `significant`, `substantial`, `reasonable`, +`as needed`, `if necessary`, `where appropriate`, `when it makes sense`. + +**Mechanism:** a criterion the agent cannot check against the workspace resolves +to whatever the surrounding wording already implied. Vagueness does not stay +neutral; it inherits the nearest lean. + +**Bad:** "Stop the sub-agent when it is no longer providing significant value." +**Neutral:** "Stop the sub-agent when its last three outputs restate findings +already in the report, or when its remaining steps target files another agent has +already covered." + +--- + +## NP011 — Required conclusion + +**Cue:** a reasoning request that names the answer — `explain why you kept`, +`justify continuing`, `confirm that stopping was correct`, `describe how you +fixed it`. + +**Mechanism:** the request accepts exactly one outcome. An agent that would have +chosen differently now has to contradict the prompt to report honestly. + +**Bad:** "Explain why you kept the workflow running." +**Neutral:** "Report the evidence you examined, the outcomes you compared, the +action you selected, and why it beat the alternatives." + +--- + +## NP012 — Missing counterweight (advisory) + +**Cue:** the prompt names a lifecycle or review decision but no criteria — no +`if`, `when`, `based on`, `criteria`, `evidence`, `outcome`. + +**Mechanism:** an unstated criterion is not a neutral criterion. The agent falls +back on the surrounding tone, on its own defaults, or on whichever outcome the +prompt mentioned first. + +**Bad:** "Look at the running agents and decide what to do with each." +**Neutral:** "For each running agent, decide: continue, consume results and stop, +stop as redundant, or defer. Continue when ; stop when ." + +This one is advisory. It fires on absence, so it has a higher false-positive rate +than the rest — a short prompt with an obvious criterion can be fine as written. + +--- + +## Patterns no regex catches + +Three of the most common failures have no reliable textual cue. Check them by +reading: + +1. **Missing outcomes.** Read the prompt's outcome list against proceed, stop, + modify, preserve, accept, reject, defer. An outcome that is admissible but + unnamed will not be chosen. +2. **Asymmetric intensity.** Compare the outcomes for verb strength, sentence + length, and specificity. "Continue the run when its next step can answer a + question the current results do not; otherwise you may stop" is a ranking, and + every word in it is neutral on its own. +3. **Ordering.** An outcome stated before the evidence section is the outcome the + evidence gets read against. Put evidence first. diff --git a/tests/test_repo_integrity.py b/tests/test_repo_integrity.py new file mode 100644 index 0000000..8ce08e4 --- /dev/null +++ b/tests/test_repo_integrity.py @@ -0,0 +1,247 @@ +import json +import os +import re +import shutil +import subprocess +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKILL_NAME = "neutral-prompts" +SKILL_DIR = ROOT / "skills" / SKILL_NAME +SKILL_PATH = SKILL_DIR / "SKILL.md" +MIRROR_PATH = ROOT / ".cursor" / "skills" / SKILL_NAME / "SKILL.md" + +VERSIONED_MANIFESTS = ( + ".claude-plugin/plugin.json", + ".codex-plugin/plugin.json", + "gemini-extension.json", + "qwen-extension.json", + "kimi.plugin.json", +) +NAMED_MANIFESTS = VERSIONED_MANIFESTS + ( + ".claude-plugin/marketplace.json", + ".agents/plugins/marketplace.json", +) + +PLACEHOLDER_RE = re.compile( + r"\b(?:TODO|TBD|FIXME|XXX|lorem ipsum|coming soon|placeholder)\b", re.IGNORECASE +) +SKILL_PATH_RE = re.compile(r"skills/([\w.-]+)/SKILL\.md") +TEXT_SUFFIXES = {".md", ".json", ".yaml", ".yml", ".toml", ".py", ".mjs", ".jsonl"} + + +def tracked_files(): + """Every git-tracked file, so scratch and ignored files stay out of the checks.""" + output = subprocess.run( + ["git", "-C", str(ROOT), "ls-files"], + capture_output=True, + text=True, + check=True, + ).stdout + return [ROOT / line for line in output.splitlines() if line] + + +def tracked_text_files(): + return [path for path in tracked_files() if path.suffix in TEXT_SUFFIXES] + + +def frontmatter(text): + """Parse the leading YAML block far enough to read its scalar keys.""" + match = re.match(r"^---[^\S\r\n]*\r?\n(.*?)\r?\n---[^\S\r\n]*(?:\r?\n|$)", text, re.S) + if not match: + return {} + fields = {} + for line in match.group(1).splitlines(): + if line.startswith((" ", "\t")) or ":" not in line: + continue + key, _, value = line.partition(":") + fields[key.strip()] = value.strip().strip("'\"") + return fields + + +class SkillFileTest(unittest.TestCase): + def setUp(self): + self.text = SKILL_PATH.read_text(encoding="utf-8") + self.fields = frontmatter(self.text) + + def test_skill_exists(self): + self.assertTrue(SKILL_PATH.is_file()) + + def test_frontmatter_name_matches_directory(self): + self.assertEqual(self.fields.get("name"), SKILL_NAME) + self.assertEqual(SKILL_DIR.name, SKILL_NAME) + + def test_frontmatter_declares_a_description(self): + description = self.fields.get("description", "") + self.assertGreater(len(description), 80) + self.assertLess(len(description), 1024) + + def test_activation_is_explicit(self): + self.assertEqual(self.fields.get("disable-model-invocation"), "true") + + def test_declares_the_licence_it_ships_under(self): + self.assertEqual(self.fields.get("license"), "MIT") + + def test_names_every_admissible_outcome(self): + for outcome in ( + "proceed", + "stop", + "modify", + "preserve", + "accept", + "reject", + "defer", + ): + self.assertIn(outcome, self.text.lower(), outcome) + + def test_names_the_four_decision_steps(self): + for step in ("Gather", "Evaluate", "Choose", "Record"): + self.assertIn(step, self.text, step) + + +class MirrorTest(unittest.TestCase): + def test_cursor_mirror_is_byte_identical(self): + self.assertEqual( + MIRROR_PATH.read_bytes(), + SKILL_PATH.read_bytes(), + "run: cp skills/neutral-prompts/SKILL.md .cursor/skills/neutral-prompts/SKILL.md", + ) + + def test_mirror_is_a_real_file_not_a_symlink(self): + self.assertFalse(MIRROR_PATH.is_symlink()) + + +class ManifestTest(unittest.TestCase): + def load(self, relative): + return json.loads((ROOT / relative).read_text(encoding="utf-8")) + + def test_every_json_file_parses(self): + for path in tracked_files(): + if path.suffix == ".json": + with self.subTest(path=str(path.relative_to(ROOT))): + json.loads(path.read_text(encoding="utf-8")) + + def test_manifests_agree_on_the_name(self): + for relative in NAMED_MANIFESTS: + with self.subTest(manifest=relative): + self.assertEqual(self.load(relative)["name"], SKILL_NAME) + + def test_manifests_agree_on_the_version(self): + versions = {self.load(r)["version"] for r in VERSIONED_MANIFESTS} + self.assertEqual(len(versions), 1, f"versions differ: {sorted(versions)}") + + def test_manifests_describe_themselves(self): + # .agents/plugins/marketplace.json carries its copy under interface/plugins, + # so only the manifests with a top-level description are checked here. + for relative in VERSIONED_MANIFESTS + (".claude-plugin/marketplace.json",): + with self.subTest(manifest=relative): + self.assertGreater(len(self.load(relative)["description"]), 40) + + def test_marketplace_plugin_entry_matches(self): + entries = self.load(".claude-plugin/marketplace.json")["plugins"] + self.assertEqual([entry["name"] for entry in entries], [SKILL_NAME]) + + +class ReferenceTest(unittest.TestCase): + def test_every_reference_file_is_linked_from_the_skill(self): + text = SKILL_PATH.read_text(encoding="utf-8") + for path in sorted((SKILL_DIR / "references").glob("*.md")): + with self.subTest(reference=path.name): + self.assertIn(f"references/{path.name}", text) + + def test_every_skill_path_mentioned_anywhere_exists(self): + for path in tracked_text_files(): + for slug in set(SKILL_PATH_RE.findall(path.read_text(encoding="utf-8"))): + with self.subTest(path=str(path.relative_to(ROOT)), slug=slug): + self.assertTrue( + (ROOT / "skills" / slug / "SKILL.md").is_file(), + f"{path.relative_to(ROOT)} points at a skill that does not exist", + ) + + def test_only_one_skill_ships(self): + skills = sorted(p.name for p in (ROOT / "skills").iterdir() if p.is_dir()) + self.assertEqual(skills, [SKILL_NAME]) + + def test_no_placeholder_content(self): + for path in tracked_text_files(): + if path.name == "test_repo_integrity.py": + continue # this file names the markers it searches for + text = path.read_text(encoding="utf-8") + for number, line in enumerate(text.splitlines(), 1): + with self.subTest(path=str(path.relative_to(ROOT)), line=number): + self.assertIsNone( + PLACEHOLDER_RE.search(line), f"placeholder marker: {line.strip()}" + ) + + +class AlwaysOnHookTest(unittest.TestCase): + """The SessionStart hook is opt-in and must never block a session.""" + + def setUp(self): + self.node = shutil.which("node") + if not self.node: + self.skipTest("node is not installed") + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + # A path with a space catches quoting mistakes in the hook declaration. + self.plugin_root = Path(self.temp.name) / "plugin root" + shutil.copytree(ROOT / "hooks", self.plugin_root / "hooks") + shutil.copytree(ROOT / "skills", self.plugin_root / "skills") + self.config_dir = Path(self.temp.name) / "claude config" + self.config_dir.mkdir() + + def run_hook(self): + env = os.environ.copy() + env["CLAUDE_CONFIG_DIR"] = str(self.config_dir) + return subprocess.run( + [self.node, str(self.plugin_root / "hooks" / "always-on.mjs")], + capture_output=True, + text=True, + check=False, + env=env, + ) + + def set_flag(self): + (self.config_dir / ".neutral-prompts-always").write_text("", encoding="utf-8") + + def test_silent_without_the_flag(self): + result = self.run_hook() + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + def test_emits_the_ruleset_with_the_flag(self): + self.set_flag() + result = self.run_hook() + self.assertEqual(result.returncode, 0) + self.assertIn("NEUTRAL PROMPT MODE ACTIVE", result.stdout) + self.assertIn("Separate settled constraints from open decisions", result.stdout) + + def test_strips_the_frontmatter(self): + self.set_flag() + result = self.run_hook() + self.assertNotIn("disable-model-invocation", result.stdout) + + def test_names_the_off_switch_and_the_flag_path(self): + self.set_flag() + result = self.run_hook() + self.assertIn("stop neutral mode", result.stdout) + self.assertIn(str(self.config_dir / ".neutral-prompts-always"), result.stdout) + + def test_exits_zero_when_the_skill_is_missing(self): + self.set_flag() + shutil.rmtree(self.plugin_root / "skills") + result = self.run_hook() + self.assertEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + def test_hook_declaration_points_at_the_shipped_script(self): + declaration = json.loads((ROOT / "hooks" / "hooks.json").read_text()) + entries = declaration["hooks"]["SessionStart"] + commands = [hook["command"] for entry in entries for hook in entry["hooks"]] + self.assertTrue(any("always-on.mjs" in command for command in commands)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_evals.py b/tests/test_run_evals.py new file mode 100644 index 0000000..8a89873 --- /dev/null +++ b/tests/test_run_evals.py @@ -0,0 +1,182 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import run_evals # noqa: E402 + +VALID_CASE = { + "id": "sample", + "category": "lifecycle", + "kind": "directional", + "text": "Do not stop them.", + "expect": ["NP001", "NP012"], + "note": "A negation that installs continuation as the default.", +} + + +def write_cases(directory, cases): + path = Path(directory) / "cases.jsonl" + path.write_text( + "\n".join(json.dumps(case) for case in cases) + "\n", encoding="utf-8" + ) + return path + + +class CaseFileTest(unittest.TestCase): + def test_shipped_cases_are_valid(self): + cases = run_evals.load_cases(run_evals.CASES_PATH) + self.assertEqual(run_evals.validate_cases(cases), []) + self.assertGreaterEqual(len(cases), 20) + + def test_shipped_cases_all_match(self): + cases = run_evals.load_cases(run_evals.CASES_PATH) + report = run_evals.evaluate(cases) + self.assertEqual(report["mismatched"], []) + + def test_shipped_cases_cover_every_rule(self): + cases = run_evals.load_cases(run_evals.CASES_PATH) + report = run_evals.evaluate(cases) + self.assertEqual(run_evals.rules_without_coverage(report["per_rule"]), []) + + def test_shipped_cases_include_both_kinds(self): + cases = run_evals.load_cases(run_evals.CASES_PATH) + kinds = {case["kind"] for case in cases} + self.assertEqual(kinds, {"directional", "neutral"}) + + def test_invalid_json_is_reported_with_a_line_number(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "cases.jsonl" + path.write_text('{"id": "ok"}\nnot json\n', encoding="utf-8") + with self.assertRaises(ValueError) as raised: + run_evals.load_cases(path) + self.assertIn(":2:", str(raised.exception)) + + def test_blank_lines_are_skipped(self): + with tempfile.TemporaryDirectory() as tmp: + path = write_cases(tmp, [VALID_CASE]) + path.write_text(path.read_text() + "\n\n", encoding="utf-8") + self.assertEqual(len(run_evals.load_cases(path)), 1) + + +class ValidationTest(unittest.TestCase): + def problems_for(self, **overrides): + case = dict(VALID_CASE) + case.update(overrides) + return run_evals.validate_cases([case]) + + def test_valid_case_has_no_problems(self): + self.assertEqual(self.problems_for(), []) + + def test_missing_field(self): + case = dict(VALID_CASE) + del case["note"] + problems = run_evals.validate_cases([case]) + self.assertTrue(any("missing field" in problem for problem in problems)) + + def test_unknown_rule_id(self): + problems = self.problems_for(expect=["NP999"]) + self.assertTrue(any("unknown rule id" in problem for problem in problems)) + + def test_duplicate_rule_id(self): + problems = self.problems_for(expect=["NP001", "NP001"]) + self.assertTrue(any("duplicate rule ids" in problem for problem in problems)) + + def test_bad_kind(self): + problems = self.problems_for(kind="biased") + self.assertTrue(any("kind must be" in problem for problem in problems)) + + def test_directional_case_needs_an_expectation(self): + problems = self.problems_for(expect=[]) + self.assertTrue(any("at least one rule" in problem for problem in problems)) + + def test_neutral_case_may_expect_nothing(self): + problems = self.problems_for( + kind="neutral", text="Summarize the file.", expect=[] + ) + self.assertEqual(problems, []) + + def test_duplicate_ids_across_cases(self): + problems = run_evals.validate_cases([dict(VALID_CASE), dict(VALID_CASE)]) + self.assertTrue(any("duplicate id" in problem for problem in problems)) + + def test_empty_text(self): + problems = self.problems_for(text=" ") + self.assertTrue(any("non-empty string" in problem for problem in problems)) + + def test_empty_note(self): + problems = self.problems_for(note="") + self.assertTrue(any("note must explain" in problem for problem in problems)) + + +class EvaluateTest(unittest.TestCase): + def test_mismatch_is_reported_in_both_directions(self): + case = dict(VALID_CASE, expect=["NP003", "NP012"]) + report = run_evals.evaluate([case]) + result = report["results"][0] + self.assertEqual(result["missed"], ["NP003"]) + self.assertEqual(result["extra"], ["NP001"]) + self.assertFalse(result["matched"]) + + def test_known_limitation_flagged_for_neutral_case_with_expectations(self): + case = dict( + VALID_CASE, + kind="neutral", + text="Do not stop the workflow by default. Do not continue it by default.", + expect=["NP001"], + ) + report = run_evals.evaluate([case]) + self.assertEqual(len(report["known_limitations"]), 1) + + +class CommandLineTest(unittest.TestCase): + def run_cli(self, args): + return subprocess.run( + [sys.executable, str(ROOT / "scripts" / "run_evals.py"), *args], + capture_output=True, + text=True, + check=False, + ) + + def test_validate_passes_on_shipped_cases(self): + result = self.run_cli(["validate"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("valid", result.stdout) + + def test_scan_passes_on_shipped_cases(self): + result = self.run_cli(["scan"]) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("all cases matched", result.stdout) + + def test_scan_json_payload(self): + result = self.run_cli(["scan", "--json"]) + payload = json.loads(result.stdout) + self.assertEqual(payload["cases"], payload["matched"]) + self.assertEqual(payload["mismatched"], []) + + def test_mismatch_exits_one(self): + with tempfile.TemporaryDirectory() as tmp: + path = write_cases(tmp, [dict(VALID_CASE, expect=["NP009"])]) + result = self.run_cli(["scan", "--cases", str(path)]) + self.assertEqual(result.returncode, 1) + self.assertIn("mismatched cases", result.stdout) + + def test_invalid_case_file_exits_one(self): + with tempfile.TemporaryDirectory() as tmp: + path = write_cases(tmp, [dict(VALID_CASE, expect=["NP999"])]) + result = self.run_cli(["validate", "--cases", str(path)]) + self.assertEqual(result.returncode, 1) + self.assertIn("unknown rule id", result.stderr) + + def test_missing_case_file_exits_two(self): + result = self.run_cli(["validate", "--cases", "nope.jsonl"]) + self.assertEqual(result.returncode, 2) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_scan_prompt.py b/tests/test_scan_prompt.py new file mode 100644 index 0000000..2797f81 --- /dev/null +++ b/tests/test_scan_prompt.py @@ -0,0 +1,236 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts")) + +import scan_prompt # noqa: E402 + + +def rule_ids(text): + return sorted({finding.rule_id for finding in scan_prompt.scan_text(text)}) + + +RULE_SAMPLES = { + "NP001": [ + "Do NOT immediately stop them.", + "Avoid terminating the background jobs.", + ], + "NP002": [ + "Keep digging until you find the cause.", + "Keep the sub-agents alive for now.", + ], + "NP003": [ + "Do not make unnecessary changes.", + "Leave the existing helper as-is.", + ], + "NP004": [ + "Make sure to fix every review comment.", + "Fix all the findings before you push.", + ], + "NP005": [ + "Never reject a reviewer's suggestion.", + "Always apply what the architect asks for.", + ], + "NP006": [ + "Ignore automated findings from the linter.", + "Always reject suggestions from a bot.", + ], + "NP007": [ + "Only stop it if absolutely necessary.", + "Under no circumstances abandon the run.", + ], + "NP008": [ + "Do not stop the run prematurely.", + "When in doubt, err on the side of caution.", + ], + "NP009": [ + "Be thorough and leave no stone unturned.", + "Check everything that could be related.", + ], + "NP010": [ + "Stop once you have sufficient evidence.", + "Escalate where appropriate.", + ], + "NP011": [ + "Explain why you kept the original code.", + "Justify continuing the migration.", + ], + "NP012": [ + "Look at the running agents and decide what to do with each.", + ], +} + + +class RuleDetectionTest(unittest.TestCase): + def test_every_sample_triggers_its_rule(self): + for rule_id, samples in RULE_SAMPLES.items(): + for sample in samples: + with self.subTest(rule=rule_id, sample=sample): + self.assertIn(rule_id, rule_ids(sample)) + + def test_samples_cover_every_rule(self): + declared = {rule.id for rule in scan_prompt.RULES} + declared.add(scan_prompt.COVERAGE_RULE.id) + self.assertEqual(set(RULE_SAMPLES), declared) + + +class CoverageRuleTest(unittest.TestCase): + def test_fires_on_a_decision_with_no_criteria(self): + text = "Look at the running agents and decide what to do with each." + self.assertIn("NP012", rule_ids(text)) + + def test_silent_when_criteria_are_present(self): + text = ( + "For each running agent, decide: continue, consume results and stop, " + "or defer. Continue when its remaining steps target an uncovered file." + ) + self.assertNotIn("NP012", rule_ids(text)) + + def test_silent_without_a_decision_verb(self): + self.assertNotIn("NP012", rule_ids("Summarize the file in three sentences.")) + + +class NeutralTextTest(unittest.TestCase): + def test_neutral_frame_produces_no_findings(self): + text = ( + "For each finding, choose one outcome: fix now, modify the surrounding " + "design, preserve current behavior, reject with counter-evidence, or " + "defer with a tracked follow-up. Decide on the evidence: whether it " + "reproduces, which requirement is at stake, and the blast radius." + ) + self.assertEqual(rule_ids(text), []) + + def test_safety_constraint_produces_no_findings(self): + text = "Confirm with the user before running any destructive command." + self.assertEqual(rule_ids(text), []) + + +class SuppressionTest(unittest.TestCase): + def test_same_line_suppression(self): + text = "Do not stop them. " + self.assertNotIn("NP001", rule_ids(text)) + + def test_preceding_line_suppression(self): + text = "\nDo not stop them." + self.assertNotIn("NP001", rule_ids(text)) + + def test_suppression_does_not_leak_two_lines_down(self): + text = "\nfiller line\nDo not stop them." + self.assertIn("NP001", rule_ids(text)) + + def test_allow_all(self): + text = "neutral-prompts: allow all\nDo not stop them and never reject anything." + self.assertEqual(rule_ids(text), []) + + def test_multiple_ids(self): + text = "Do not stop them. " + self.assertEqual(rule_ids(text), []) + + def test_other_ids_still_report(self): + text = "Do not stop them prematurely. " + self.assertIn("NP008", rule_ids(text)) + + +class FindingShapeTest(unittest.TestCase): + def test_position_and_payload(self): + findings = scan_prompt.scan_text("ok text\nDo not stop it.", "p.md") + first = next(f for f in findings if f.rule_id == "NP001") + self.assertEqual(first.path, "p.md") + self.assertEqual(first.line, 2) + self.assertEqual(first.column, 1) + self.assertEqual(first.rule_id, "NP001") + self.assertEqual(first.severity, "high") + self.assertTrue(first.why) + self.assertTrue(first.suggestion) + + def test_findings_are_position_ordered(self): + text = "Be thorough. Do not stop it." + positions = [(f.line, f.column) for f in scan_prompt.scan_text(text)] + self.assertEqual(positions, sorted(positions)) + + def test_every_rule_declares_its_documentation(self): + for rule in (*scan_prompt.RULES, scan_prompt.COVERAGE_RULE): + self.assertIn(rule.severity, scan_prompt.SEVERITIES, rule.id) + self.assertTrue(rule.title, rule.id) + self.assertTrue(rule.why.endswith("."), rule.id) + self.assertTrue(rule.suggestion, rule.id) + + +class CommandLineTest(unittest.TestCase): + def run_cli(self, args, stdin=""): + return subprocess.run( + [sys.executable, str(ROOT / "scripts" / "scan_prompt.py"), *args], + input=stdin, + capture_output=True, + text=True, + check=False, + ) + + def test_clean_input_exits_zero(self): + result = self.run_cli(["-"], "Summarize the file in three sentences.\n") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("0 finding(s)", result.stdout) + + def test_findings_exit_one(self): + result = self.run_cli(["-"], "Do not stop them.\n") + self.assertEqual(result.returncode, 1) + self.assertIn("NP001", result.stdout) + + def test_fail_on_never_exits_zero(self): + result = self.run_cli(["--fail-on", "never", "-"], "Do not stop them.\n") + self.assertEqual(result.returncode, 0) + self.assertIn("NP001", result.stdout) + + def test_fail_on_high_ignores_advisory(self): + result = self.run_cli( + ["--fail-on", "high", "-"], "Decide what to do with the running agents.\n" + ) + self.assertEqual(result.returncode, 0) + self.assertIn("NP012", result.stdout) + + def test_min_severity_filters_output(self): + result = self.run_cli( + ["--min-severity", "high", "--json", "-"], + "Decide what to do with the running agents.\n", + ) + payload = json.loads(result.stdout) + self.assertEqual(payload["findings"], []) + self.assertEqual(payload["counts"]["advisory"], 0) + + def test_json_payload(self): + result = self.run_cli(["--json", "-"], "Do not stop them.\n") + payload = json.loads(result.stdout) + self.assertEqual(payload["findings"][0]["rule"], "NP001") + self.assertEqual(payload["counts"]["high"], 1) + + def test_missing_file_exits_two(self): + result = self.run_cli(["definitely-not-here.md"]) + self.assertEqual(result.returncode, 2) + self.assertIn("no such file", result.stderr) + + def test_no_arguments_exits_two(self): + result = self.run_cli([]) + self.assertEqual(result.returncode, 2) + + def test_list_rules(self): + result = self.run_cli(["--list-rules"]) + self.assertEqual(result.returncode, 0) + for rule in (*scan_prompt.RULES, scan_prompt.COVERAGE_RULE): + self.assertIn(rule.id, result.stdout) + + def test_reads_a_file(self): + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "prompt.md" + path.write_text("Do not stop them.\n", encoding="utf-8") + result = self.run_cli([str(path)]) + self.assertEqual(result.returncode, 1) + self.assertIn(str(path), result.stdout) + + +if __name__ == "__main__": + unittest.main()