diff --git a/crates/refinery_cli/src/commands/README.md b/crates/refinery_cli/src/commands/README.md index 2f720f4..f20febc 100644 --- a/crates/refinery_cli/src/commands/README.md +++ b/crates/refinery_cli/src/commands/README.md @@ -135,6 +135,18 @@ The analyzer compares selector counterfactuals over final-round proposals and ev It reports panel quality, quality floor, evaluator disagreement, lexical overlap, and meta-preamble rate. +For whole-panel review, generate a blind packet that hides iteration strategies and model IDs from reviewers while writing a separate answer key: + +```sh +refinery review-brainstorm-panels $(cat run-dirs.txt) \ + --strategies score-only,own-reviews,full-visibility \ + --selector controversy-floor-7 \ + --key-path panel-review-key.json \ + > panel-review-pack.md +``` + +Use the review packet to score useful diversity, non-overlap, novelty, actionability, coverage, overall panel value, and best-answer regret. + ## Planned Verbs | Verb | Status | Idea | diff --git a/crates/refinery_cli/src/commands/mod.rs b/crates/refinery_cli/src/commands/mod.rs index 149c126..c193612 100644 --- a/crates/refinery_cli/src/commands/mod.rs +++ b/crates/refinery_cli/src/commands/mod.rs @@ -1,6 +1,7 @@ pub mod benchmark_brainstorm; pub mod brainstorm; pub mod converge; +pub mod review_brainstorm_panels; pub mod synthesize; mod common; diff --git a/crates/refinery_cli/src/commands/review_brainstorm_panels.rs b/crates/refinery_cli/src/commands/review_brainstorm_panels.rs new file mode 100644 index 0000000..aba7bf8 --- /dev/null +++ b/crates/refinery_cli/src/commands/review_brainstorm_panels.rs @@ -0,0 +1,769 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::{Path, PathBuf}; +use std::process::ExitCode; + +use clap::{Parser, ValueEnum}; +use serde::{Deserialize, Serialize}; + +use refinery_core::brainstorm::BrainstormIterationStrategy; +use refinery_core::scoring; +use refinery_core::types::ModelId; + +use super::common::OutputFormat; + +#[derive(Parser, Debug)] +pub struct ReviewBrainstormPanelsArgs { + /// Brainstorm artifact run directories to turn into a blind panel review pack. + #[arg(value_name = "RUN_DIR")] + run_dirs: Vec, + + /// Selector to use when building each reviewed panel. + #[arg(long, default_value = "controversy-floor-7")] + selector: PanelSelector, + + /// Only include these iteration strategies (comma-separated), e.g. score-only,own-reviews,full-visibility. + #[arg(long, value_delimiter = ',')] + strategies: Vec, + + /// Prompt text mapping in the form `prompt_id=text`. Repeatable. + #[arg(long = "prompt-text", value_name = "ID=TEXT")] + prompt_texts: Vec, + + /// Number of answers per panel [default: 3]. + #[arg(long, default_value = "3", value_parser = clap::value_parser!(u32).range(1..=20))] + panel_size: u32, + + /// Output format [text|json]. Text emits Markdown. + #[arg(short, long, default_value = "text")] + output_format: OutputFormat, + + /// Write a JSON answer key mapping blind labels to strategies and model IDs. + #[arg(long, value_name = "PATH")] + key_path: Option, +} + +#[derive(Clone, Copy, Debug, ValueEnum)] +enum PanelSelector { + Mean, + Stddev, + Controversy, + #[value(name = "controversy-floor-7", alias = "controversy_floor_7")] + ControversyFloor7, + QualityXLexdiv, +} + +impl PanelSelector { + fn as_str(self) -> &'static str { + match self { + Self::Mean => "mean", + Self::Stddev => "stddev", + Self::Controversy => "controversy", + Self::ControversyFloor7 => "controversy_floor_7", + Self::QualityXLexdiv => "quality_x_lexdiv", + } + } +} + +#[derive(Clone, Debug)] +struct Candidate { + model_id: ModelId, + answer: String, + mean_score: f64, + stddev: f64, + controversy_score: f64, +} + +#[derive(Debug)] +struct LoadedRun { + run_dir: PathBuf, + prompt_id: String, + iteration_strategy: BrainstormIterationStrategy, + panel: Vec, +} + +#[derive(Debug, Serialize)] +struct ReviewPack { + status: String, + selector: String, + prompts: Vec, +} + +#[derive(Debug, Serialize)] +struct PromptReview { + prompt_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_text: Option, + panels: Vec, +} + +#[derive(Debug, Serialize)] +struct BlindPanel { + label: String, + answers: Vec, +} + +#[derive(Debug, Serialize)] +struct BlindAnswer { + label: String, + answer: String, +} + +#[derive(Debug, Serialize)] +struct ReviewAnswerKey { + selector: String, + prompts: Vec, +} + +#[derive(Debug, Serialize)] +struct PromptAnswerKey { + prompt_id: String, + panels: Vec, +} + +#[derive(Debug, Serialize)] +struct PanelAnswerKey { + label: String, + iteration_strategy: String, + run_dir: String, + answers: Vec, +} + +#[derive(Debug, Serialize)] +struct AnswerKeyEntry { + label: String, + model_id: String, + mean_score: f64, + stddev: f64, + controversy_score: f64, +} + +#[derive(Debug, Deserialize)] +struct EvalArtifact { + evaluator: String, + evaluatee: String, + score: f64, +} + +#[derive(Debug, Deserialize)] +struct RunMetadata { + iteration_strategy: Option, +} + +pub fn run(args: &ReviewBrainstormPanelsArgs) -> ExitCode { + if args.run_dirs.is_empty() { + eprintln!("Error: at least one brainstorm run directory is required"); + return ExitCode::from(4); + } + + let prompt_texts = match parse_prompt_texts(&args.prompt_texts) { + Ok(texts) => texts, + Err(e) => { + eprintln!("Error: {e}"); + return ExitCode::from(4); + } + }; + + let strategy_filter = match parse_iteration_strategies(&args.strategies) { + Ok(strategies) => strategies, + Err(e) => { + eprintln!("Error: {e}"); + return ExitCode::from(4); + } + }; + let mut runs = Vec::new(); + for run_dir in &args.run_dirs { + match load_run(run_dir, args.selector, args.panel_size as usize) { + Ok(run) + if strategy_filter.is_empty() + || strategy_filter.contains(&run.iteration_strategy) => + { + runs.push(run); + } + Ok(_) => {} + Err(e) => { + eprintln!("Error loading {}: {e}", run_dir.display()); + return ExitCode::from(1); + } + } + } + + if runs.is_empty() { + eprintln!("Error: no runs matched the requested strategy filter"); + return ExitCode::from(4); + } + + let (pack, key) = build_review_pack(runs, args.selector.as_str(), &prompt_texts); + + if let Some(path) = &args.key_path { + if let Err(e) = write_answer_key(path, &key) { + eprintln!("Error writing answer key {}: {e}", path.display()); + return ExitCode::from(1); + } + } + + match args.output_format { + OutputFormat::Json => match serde_json::to_string_pretty(&pack) { + Ok(json) => { + println!("{json}"); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("Failed to serialize review pack: {e}"); + ExitCode::from(1) + } + }, + OutputFormat::Text => { + emit_markdown(&pack); + ExitCode::SUCCESS + } + } +} + +fn parse_prompt_texts(values: &[String]) -> Result, String> { + let mut parsed = BTreeMap::new(); + for value in values { + let (id, text) = value + .split_once('=') + .ok_or_else(|| format!("invalid --prompt-text '{value}', expected ID=TEXT"))?; + if id.trim().is_empty() || text.trim().is_empty() { + return Err(format!( + "invalid --prompt-text '{value}', both ID and TEXT must be non-empty" + )); + } + parsed.insert(id.trim().to_string(), text.trim().to_string()); + } + Ok(parsed) +} + +fn parse_iteration_strategies( + values: &[String], +) -> Result, String> { + values + .iter() + .map(|value| value.parse::()) + .collect() +} + +fn write_answer_key(path: &Path, key: &ReviewAnswerKey) -> Result<(), String> { + if let Some(parent) = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + { + std::fs::create_dir_all(parent) + .map_err(|e| format!("failed to create {}: {e}", parent.display()))?; + } + let json = serde_json::to_string_pretty(key) + .map_err(|e| format!("failed to serialize answer key: {e}"))?; + std::fs::write(path, json).map_err(|e| format!("failed to write {}: {e}", path.display())) +} + +fn load_run( + run_dir: &Path, + selector: PanelSelector, + panel_size: usize, +) -> Result { + let final_round = find_final_round(run_dir)?; + let round_dir = run_dir.join(format!("round-{final_round}")); + let candidates = load_candidates(&round_dir)?; + let panel = select_panel(&candidates, selector, panel_size) + .into_iter() + .cloned() + .collect(); + let metadata = load_run_metadata(run_dir)?; + let iteration_strategy = metadata + .and_then(|metadata| metadata.iteration_strategy) + .ok_or_else(|| "run metadata missing iteration_strategy".to_string())? + .parse::() + .map_err(|e| format!("invalid run metadata iteration_strategy: {e}"))?; + let prompt_id = prompt_id_from_run_dir(run_dir); + + Ok(LoadedRun { + run_dir: run_dir.to_path_buf(), + prompt_id, + iteration_strategy, + panel, + }) +} + +fn load_run_metadata(run_dir: &Path) -> Result, String> { + let path = run_dir.join("metadata.json"); + if !path.exists() { + return Ok(None); + } + let content = std::fs::read_to_string(&path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let metadata = serde_json::from_str(&content) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + Ok(Some(metadata)) +} + +fn prompt_id_from_run_dir(run_dir: &Path) -> String { + run_dir.parent().and_then(Path::file_name).map_or_else( + || run_dir.display().to_string(), + |name| name.to_string_lossy().to_string(), + ) +} + +fn find_final_round(run_dir: &Path) -> Result { + let entries = std::fs::read_dir(run_dir).map_err(|e| e.to_string())?; + let mut max_round = None; + for entry in entries { + let entry = + entry.map_err(|e| format!("failed to read {} entry: {e}", run_dir.display()))?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if let Some(round) = name + .strip_prefix("round-") + .and_then(|suffix| suffix.parse::().ok()) + { + max_round = Some(max_round.map_or(round, |current: u32| current.max(round))); + } + } + max_round.ok_or_else(|| "no round-* directories found".to_string()) +} + +fn load_candidates(round_dir: &Path) -> Result, String> { + let mut scores: BTreeMap> = BTreeMap::new(); + + for entry in std::fs::read_dir(round_dir).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry.file_name(); + let name = name.to_string_lossy(); + if !name.starts_with("evaluate-") || !name.ends_with(".json") { + continue; + } + let path = entry.path(); + let content = std::fs::read_to_string(&path) + .map_err(|e| format!("failed to read {}: {e}", path.display()))?; + let eval: EvalArtifact = serde_json::from_str(&content) + .map_err(|e| format!("failed to parse {}: {e}", path.display()))?; + if eval.evaluator != eval.evaluatee { + scores + .entry(ModelId::new(&eval.evaluatee)) + .or_default() + .push(eval.score); + } + } + + if scores.is_empty() { + return Err("no evaluation artifacts found".to_string()); + } + + scores + .into_iter() + .map(|(model_id, score_values)| { + let safe_id = model_id.to_string().replace('/', "_"); + let proposal_path = round_dir.join(format!("propose-{safe_id}.md")); + let proposal = std::fs::read_to_string(&proposal_path).map_err(|e| { + format!( + "failed to read proposal for {model_id} at {}: {e}", + proposal_path.display() + ) + })?; + let answer = proposal; + let mean_score = scoring::mean(&score_values); + let stddev = scoring::stddev(&score_values, mean_score); + let controversy_score = scoring::controversy_score(&score_values); + Ok(Candidate { + model_id, + answer, + mean_score, + stddev, + controversy_score, + }) + }) + .collect() +} + +fn select_panel( + candidates: &[Candidate], + selector: PanelSelector, + panel_size: usize, +) -> Vec<&Candidate> { + match selector { + PanelSelector::Mean => select_by_mean(candidates, panel_size), + PanelSelector::Stddev => select_by_stddev(candidates, panel_size), + PanelSelector::Controversy => select_by_controversy(candidates, panel_size), + PanelSelector::ControversyFloor7 => { + select_by_controversy_with_quality_floor(candidates, panel_size, 7.0) + } + PanelSelector::QualityXLexdiv => select_by_quality_x_lexdiv(candidates, panel_size), + } +} + +fn select_by_mean(candidates: &[Candidate], panel_size: usize) -> Vec<&Candidate> { + let mut selected: Vec<&Candidate> = candidates.iter().collect(); + selected.sort_by(|a, b| { + b.mean_score + .total_cmp(&a.mean_score) + .then_with(|| b.controversy_score.total_cmp(&a.controversy_score)) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + selected.truncate(panel_size); + selected +} + +fn select_by_stddev(candidates: &[Candidate], panel_size: usize) -> Vec<&Candidate> { + let mut selected: Vec<&Candidate> = candidates.iter().collect(); + selected.sort_by(|a, b| { + b.stddev + .total_cmp(&a.stddev) + .then_with(|| b.mean_score.total_cmp(&a.mean_score)) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + selected.truncate(panel_size); + selected +} + +fn select_by_controversy(candidates: &[Candidate], panel_size: usize) -> Vec<&Candidate> { + let mut selected: Vec<&Candidate> = candidates.iter().collect(); + selected.sort_by(|a, b| { + b.controversy_score + .total_cmp(&a.controversy_score) + .then_with(|| b.mean_score.total_cmp(&a.mean_score)) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + selected.truncate(panel_size); + selected +} + +fn select_by_controversy_with_quality_floor( + candidates: &[Candidate], + panel_size: usize, + quality_floor: f64, +) -> Vec<&Candidate> { + let mut selected: Vec<&Candidate> = candidates + .iter() + .filter(|candidate| candidate.mean_score >= quality_floor) + .collect(); + selected.sort_by(|a, b| { + b.controversy_score + .total_cmp(&a.controversy_score) + .then_with(|| b.mean_score.total_cmp(&a.mean_score)) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + + if selected.len() < panel_size { + let selected_ids: BTreeSet = selected + .iter() + .map(|candidate| candidate.model_id.clone()) + .collect(); + let mut backfill: Vec<&Candidate> = candidates + .iter() + .filter(|candidate| !selected_ids.contains(&candidate.model_id)) + .collect(); + backfill.sort_by(|a, b| { + b.controversy_score + .total_cmp(&a.controversy_score) + .then_with(|| b.mean_score.total_cmp(&a.mean_score)) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + selected.extend(backfill); + } + + selected.truncate(panel_size); + selected +} + +fn select_by_quality_x_lexdiv(candidates: &[Candidate], panel_size: usize) -> Vec<&Candidate> { + if candidates.is_empty() { + return Vec::new(); + } + + let mut selected = Vec::new(); + let mut remaining: Vec<&Candidate> = candidates.iter().collect(); + remaining.sort_by(|a, b| { + b.mean_score + .total_cmp(&a.mean_score) + .then_with(|| a.model_id.to_string().cmp(&b.model_id.to_string())) + }); + selected.push(remaining.remove(0)); + + while selected.len() < panel_size && !remaining.is_empty() { + let best_index = remaining + .iter() + .enumerate() + .max_by(|(_, a), (_, b)| { + quality_x_lexdiv_score(a, &selected) + .total_cmp(&quality_x_lexdiv_score(b, &selected)) + .then_with(|| a.mean_score.total_cmp(&b.mean_score)) + .then_with(|| b.model_id.to_string().cmp(&a.model_id.to_string())) + }) + .map_or(0, |(i, _)| i); + selected.push(remaining.remove(best_index)); + } + + selected +} + +fn quality_x_lexdiv_score(candidate: &Candidate, selected: &[&Candidate]) -> f64 { + let max_similarity = selected + .iter() + .map(|other| lexical_similarity(&candidate.answer, &other.answer)) + .fold(0.0, f64::max); + (candidate.mean_score / 10.0) * (1.0 - max_similarity) +} + +fn lexical_similarity(a: &str, b: &str) -> f64 { + let a_tokens = tokens(a); + let b_tokens = tokens(b); + if a_tokens.is_empty() || b_tokens.is_empty() { + return 0.0; + } + let intersection = a_tokens.intersection(&b_tokens).count(); + let union = a_tokens.union(&b_tokens).count(); + #[allow(clippy::cast_precision_loss)] + { + intersection as f64 / union as f64 + } +} + +fn tokens(text: &str) -> BTreeSet { + text.split(|ch: char| !ch.is_alphanumeric() && ch != '_' && ch != '\'') + .filter_map(|token| { + let token = token.to_lowercase(); + (token.len() >= 3 && !STOPWORDS.contains(&token.as_str())).then_some(token) + }) + .collect() +} + +fn build_review_pack( + runs: Vec, + selector: &str, + prompt_texts: &BTreeMap, +) -> (ReviewPack, ReviewAnswerKey) { + let mut grouped: BTreeMap> = BTreeMap::new(); + for run in runs { + grouped.entry(run.prompt_id.clone()).or_default().push(run); + } + + let mut prompt_reviews = Vec::new(); + let mut prompt_keys = Vec::new(); + + for (prompt_id, mut prompt_runs) in grouped { + prompt_runs.sort_by(|a, b| { + blind_sort_key(&prompt_id, a.iteration_strategy) + .cmp(&blind_sort_key(&prompt_id, b.iteration_strategy)) + .then_with(|| { + a.iteration_strategy + .as_str() + .cmp(b.iteration_strategy.as_str()) + }) + }); + + let mut panels = Vec::new(); + let mut key_panels = Vec::new(); + for (panel_index, run) in prompt_runs.into_iter().enumerate() { + let panel_label = panel_label(panel_index); + let answers: Vec = run + .panel + .iter() + .enumerate() + .map(|(answer_index, candidate)| BlindAnswer { + label: answer_label(answer_index), + answer: candidate.answer.clone(), + }) + .collect(); + let key_answers = run + .panel + .iter() + .enumerate() + .map(|(answer_index, candidate)| AnswerKeyEntry { + label: answer_label(answer_index), + model_id: candidate.model_id.to_string(), + mean_score: candidate.mean_score, + stddev: candidate.stddev, + controversy_score: candidate.controversy_score, + }) + .collect(); + panels.push(BlindPanel { + label: panel_label.clone(), + answers, + }); + key_panels.push(PanelAnswerKey { + label: panel_label, + iteration_strategy: run.iteration_strategy.as_str().to_string(), + run_dir: run.run_dir.display().to_string(), + answers: key_answers, + }); + } + + prompt_reviews.push(PromptReview { + prompt_text: prompt_texts.get(&prompt_id).cloned(), + prompt_id: prompt_id.clone(), + panels, + }); + prompt_keys.push(PromptAnswerKey { + prompt_id, + panels: key_panels, + }); + } + + ( + ReviewPack { + status: "review_pack".to_string(), + selector: selector.to_string(), + prompts: prompt_reviews, + }, + ReviewAnswerKey { + selector: selector.to_string(), + prompts: prompt_keys, + }, + ) +} + +fn blind_sort_key(prompt_id: &str, strategy: BrainstormIterationStrategy) -> u64 { + // FNV-1a for deterministic label shuffling without adding a dependency. + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + for byte in prompt_id + .bytes() + .chain([b'|']) + .chain(strategy.as_str().bytes()) + { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0100_0000_01b3); + } + hash +} + +fn panel_label(index: usize) -> String { + let letter = char::from(b'A' + u8::try_from(index % 26).expect("panel label index fits")); + if index < 26 { + letter.to_string() + } else { + format!("{letter}{}", index / 26) + } +} + +fn answer_label(index: usize) -> String { + format!("Answer {}", index + 1) +} + +fn emit_markdown(pack: &ReviewPack) { + println!("# Brainstorm Panel Review Pack"); + println!(); + println!("Selector: `{}`", pack.selector); + println!(); + println!( + "Review each panel as a complete user-facing brainstorm result. The panel labels are blind: they do not reveal the iteration strategy or model IDs." + ); + println!(); + println!( + "Use a 1-5 scale for: useful diversity, non-overlap, novelty, actionability, coverage, and overall panel value. Also note best-answer regret: whether the panel appears to omit an obviously stronger direction." + ); + + for prompt in &pack.prompts { + println!(); + println!("## Prompt: {}", prompt.prompt_id); + if let Some(text) = &prompt.prompt_text { + println!(); + println!("{text}"); + } + + for panel in &prompt.panels { + println!(); + println!("### Panel {}", panel.label); + for answer in &panel.answers { + println!(); + println!("#### {}", answer.label); + println!(); + println!("{}", answer.answer.trim()); + } + println!(); + println!("#### Review notes for Panel {}", panel.label); + println!(); + println!("- Useful diversity (1-5): "); + println!("- Non-overlap (1-5): "); + println!("- Novelty (1-5): "); + println!("- Actionability (1-5): "); + println!("- Coverage (1-5): "); + println!("- Overall panel value (1-5): "); + println!("- Best-answer regret / omissions: "); + println!("- Notes: "); + } + } +} + +const STOPWORDS: &[&str] = &[ + "the", "and", "for", "that", "this", "with", "from", "into", "onto", "over", "under", "would", + "could", "should", "there", "their", "about", "after", "before", "while", "where", "which", + "what", "when", "then", "than", "they", "them", "these", "those", "your", "ours", "were", + "was", "are", "been", "being", "have", "has", "had", "not", "but", "can", "may", "might", + "will", "just", "only", "also", "very", "more", "most", +]; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_prompt_texts_requires_id_text_pairs() { + let parsed = parse_prompt_texts(&["product=Generate ideas".to_string()]).unwrap(); + assert_eq!(parsed.get("product"), Some(&"Generate ideas".to_string())); + assert!(parse_prompt_texts(&["missing separator".to_string()]).is_err()); + } + + #[test] + fn prompt_id_comes_from_run_parent_directory() { + let path = Path::new("target/bench/score-only/product/20260530_run"); + assert_eq!(prompt_id_from_run_dir(path), "product"); + } + + #[test] + fn parse_iteration_strategies_rejects_unknown_values() { + let parsed = parse_iteration_strategies(&["score-only".to_string()]).unwrap(); + assert_eq!(parsed, vec![BrainstormIterationStrategy::ScoreOnly]); + assert!(parse_iteration_strategies(&["score_only".to_string()]).is_err()); + } + + #[test] + fn write_answer_key_accepts_bare_filename_path() { + let path = PathBuf::from(format!( + "review-brainstorm-panels-key-{}.json", + std::process::id() + )); + let key = ReviewAnswerKey { + selector: "controversy_floor_7".to_string(), + prompts: Vec::new(), + }; + + write_answer_key(&path, &key).unwrap(); + let written = std::fs::read_to_string(&path).unwrap(); + assert!(written.contains("controversy_floor_7")); + std::fs::remove_file(path).unwrap(); + } + + #[test] + fn review_pack_hides_strategy_but_key_reveals_it() { + let runs = vec![ + loaded_run("product", "score-only", "score answer"), + loaded_run("product", "own-reviews", "review answer"), + ]; + let (pack, key) = build_review_pack(runs, "controversy_floor_7", &BTreeMap::new()); + + assert_eq!(pack.prompts.len(), 1); + assert_eq!(pack.prompts[0].panels.len(), 2); + let json = serde_json::to_string(&pack).unwrap(); + assert!(!json.contains("score-only")); + assert!(!json.contains("own-reviews")); + + let key_json = serde_json::to_string(&key).unwrap(); + assert!(key_json.contains("score-only")); + assert!(key_json.contains("own-reviews")); + } + + fn loaded_run(prompt_id: &str, strategy: &str, answer: &str) -> LoadedRun { + LoadedRun { + run_dir: PathBuf::from(format!("target/{strategy}/{prompt_id}/run")), + prompt_id: prompt_id.to_string(), + iteration_strategy: strategy.parse().unwrap(), + panel: vec![Candidate { + model_id: ModelId::new("test/model"), + answer: answer.to_string(), + mean_score: 8.0, + stddev: 0.5, + controversy_score: 4.0, + }], + } + } +} diff --git a/crates/refinery_cli/src/main.rs b/crates/refinery_cli/src/main.rs index a5eeb64..0a5b9b2 100644 --- a/crates/refinery_cli/src/main.rs +++ b/crates/refinery_cli/src/main.rs @@ -39,6 +39,9 @@ enum Command { /// Analyze brainstorm artifacts and compare panel selection strategies. BenchmarkBrainstorm(commands::benchmark_brainstorm::BenchmarkBrainstormArgs), + + /// Create a blind review pack from brainstorm artifact panels. + ReviewBrainstormPanels(commands::review_brainstorm_panels::ReviewBrainstormPanelsArgs), } fn main() -> ExitCode { @@ -73,5 +76,6 @@ async fn async_main() -> ExitCode { Command::Synthesize(args) => commands::synthesize::run(args).await, Command::Brainstorm(args) => commands::brainstorm::run(args).await, Command::BenchmarkBrainstorm(args) => commands::benchmark_brainstorm::run(&args), + Command::ReviewBrainstormPanels(args) => commands::review_brainstorm_panels::run(&args), } } diff --git a/crates/tundish_providers/src/process.rs b/crates/tundish_providers/src/process.rs index 47aa317..9acc760 100644 --- a/crates/tundish_providers/src/process.rs +++ b/crates/tundish_providers/src/process.rs @@ -10,8 +10,12 @@ use tundish_core::error::ProviderError; use tundish_core::progress::ProgressFn; use tundish_core::types::{Message, ModelId, Role}; -/// Maximum response size in bytes (1MB). -const MAX_RESPONSE_SIZE: usize = 1_000_000; +/// Maximum captured stdout size in bytes (64MB). +/// +/// JSON-event CLIs such as pi may emit substantially more transport data than +/// the final assistant text because streamed message updates repeat context. +/// Keep a bounded capture while allowing normal benchmark-sized responses. +const MAX_RESPONSE_SIZE: usize = 64_000_000; static TEMP_FILE_COUNTER: AtomicU64 = AtomicU64::new(0); diff --git a/docs/HANDOFF.md b/docs/HANDOFF.md index 8b874f1..4daf4d6 100644 --- a/docs/HANDOFF.md +++ b/docs/HANDOFF.md @@ -2,7 +2,7 @@ Current state of the project and active work. Read this at session start. Update before compaction or at natural breakpoints. -**Last updated:** 2026-05-31 +**Last updated:** 2026-06-02 ## Project State @@ -39,13 +39,15 @@ See `memory/verb_architecture.md` for full taxonomy with consistent terminology. - **New XML tags in prompts need sanitizers.** See `docs/solutions/security-issues/prompt-injection-prevention-multi-model.md`. - **Rationale before score** in all eval schemas — autoregressive anti-manipulation measure. - **Float score coercion** — `as_u64().or_else(as_f64())` pattern for score parsing. +- **Pi JSON event streams can be much larger than final text.** `tundish_providers::process` now allows 64MB captured stdout so benchmark-sized Pi responses do not trip the transport cap; a future improvement should stream-parse Pi JSON instead of retaining the full event stream. ## Open TODOs Check `todos/` for the full list. Key ones: -- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, and benchmark-only iteration variants completed; next run six-prompt suite across variants using Pi-backed model routing where possible +- **013** — brainstorm strategy benchmarks (in progress): design, analyzer, six-prompt v0 suite, quality-floor follow-up, meta-preamble prompt polish, benchmark-only iteration variants, L2 six-prompt variant suite, blind review pack, and first-pass qualitative L2 panel review completed; next either run human/calibrated judge review or plan L3 prompt-reframing with `score-only` as baseline - **025** — optional brainstorm lineage-reference polish if softer phrases like "builds on..." feel too process-oriented in demos +- **026** — stream-parse Pi JSON events instead of buffering full stdout; current 64MB cap is a benchmark unblocker, not the ideal provider implementation - **018** — brainstorm divergence expansion: each model reframes the initial prompt, all models work all prompt variants; optional future domain collisions - **021** — evaluate TOON (`toon-format/toon`) for prompt-facing artifact export / benchmark fixtures - **011** — evolve verb (designed, not started) @@ -58,6 +60,8 @@ Triage pattern: fix P1/P2 with code, create TODOs for P3/nitpicks, reply to ever ## Recent Context +- 2026-06-01 first-pass brainstorm L2 panel review completed (`todos/013`, `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`): reviewed the blind pack at `target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-pack.md` and unblinded with `l2-panel-review-key.json`. Qualitative result: `score-only` looked strongest on useful diversity/non-overlap, `full-visibility` strongest on actionability/coverage, and `own-reviews` did not dominate globally but produced the strongest debugging/process panel. Recommendation remains: keep production default `score-only`; do not promote `full-visibility` despite higher automated quality scores until stronger human/calibrated judge evidence exists. For L3 prompt-reframing, use `score-only` as baseline and include `own-reviews` only if budget allows. +- 2026-05-30 brainstorm L2 iteration strategy benchmark completed (`todos/013`, `docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md`): ran 24 clean Pi-backed runs (6 prompts × `blind`, `score-only`, `own-reviews`, `full-visibility`) with `pi/openai-codex/gpt-5.4:off`, `pi/zai/glm-5.1:off`, `pi/kimi-coding/kimi-k2-thinking:off`, and `pi/minimax/MiniMax-M2.7:off`. Used `--max-concurrent 1` to avoid Pi local config lock contention and raised bounded stdout capture from 1MB to 64MB because Pi JSON event streams can exceed 1MB. Analyzer outputs live under `target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/` (`run-dirs-clean.txt`, `l2-analysis-clean.json`, `l2-analysis-clean.txt`). Current `controversy_floor_7` aggregate: `full-visibility` highest quality (`mean=8.204`, `min=7.944`) but highest lexical overlap (`0.132`); `score-only` lowest lexical overlap (`0.097`) but lower quality (`mean=7.889`); `own-reviews` middle-ground (`mean=8.019`, disagreement `0.517`). Added `refinery review-brainstorm-panels` and generated blind review artifacts for `score-only`, `own-reviews`, and `full-visibility`: `l2-panel-review-pack.md` plus `l2-panel-review-key.json` in the same logs dir. Recommendation: keep production default `score-only` until whole-panel diversity/human or calibrated model-judge review checks semantic convergence and best-answer regret. Verified with `cargo fmt --all -- --check`, `cargo test -p refinery_cli review_brainstorm_panels`, `cargo clippy -p refinery_cli --all-targets -- -D warnings`, `cargo test -p tundish_providers`, and `cargo clippy -p tundish_providers --all-targets -- -D warnings`. - 2026-05-31 PR #40 (`feat: add Pi provider and brainstorm benchmark variants`) passed final review/checks after follow-up commits. Addressed CodeRabbit/GHA feedback with nested `pi`/`opencode` model-spec validation, plan review-date refresh, and Clippy sort lint fixes; addressed Gemini feedback by comparing `ModelId` directly where compatible, accepting string evaluation scores, and preserving `USERPROFILE`; addressed Codex feedback by forwarding a whitelist of Pi credential/config env vars after `env_clear`. Final observed checks before merge: GitHub Actions Build/Check/Test passed, Buildkite build #31 passed, CodeRabbit approved. Local verification included `cargo fmt --all -- --check`, `cargo clippy --workspace -- -D warnings`, `cargo clippy --workspace --all-targets -- -D warnings`, `cargo build --workspace`, and `cargo test --workspace`. - 2026-05-26 Buildkite baked-image follow-up: PR #39 (`ci: use baked Linux ARM64 Buildkite image`) opened from fork branch `El-Fitz:chore/buildkite-baked-linux-image`. It switches `.buildkite/pipeline.yml` to Tart image `ci-linux-arm64-rust-bazel`, removes per-job apt/rustup bootstrap, keeps HOME/Cargo/Rustup normalization, and adds `/opt/cargo/bin` after review feedback. GitHub Actions checks passed. Buildkite did not appear as a PR check from the fork branch; the upstream Buildkite pipeline may still be using inline pipeline settings, so a maintainer should either update the Buildkite pipeline configuration to upload `.buildkite/pipeline.yml` from the repo or manually run/patch the Buildkite pipeline before merging. - 2026-05-26 Pi provider adapter added: `tundish_providers::pi::PiProvider` supports model specs like `pi/openai/gpt-5.4`, invokes `pi --mode json --no-session --no-context-files --model `, disables tools by default, preserves Pi local config via `HOME`, and extracts assistant text from Pi JSON event streams. Default provider features are now `codex`, `opencode`, and `pi`; `claude`/`gemini` are no longer default options. `opencode` remains supported for users with local OpenCode config, but benchmarks should prefer Pi routing. Docs and `.env.example` updated accordingly. Verified with `cargo test -p tundish_providers`, `cargo clippy -p tundish_providers --all-targets -- -D warnings`, `cargo test -p refinery_cli`, `cargo clippy -p refinery_cli --all-targets -- -D warnings`, and a manual `brainstorm --dry-run` using `pi/openai/gpt-5.4` + `opencode/...`. @@ -90,6 +94,6 @@ Recommended order: 1. If continuing Buildkite migration, review PR #39 and either trigger a real Buildkite run against `ci-linux-arm64-rust-bazel` or update the Buildkite pipeline settings to upload `.buildkite/pipeline.yml` from the repo so PR pipeline changes are exercised. 2. Start from clean `main` and read this handoff plus the valid baseline in `docs/brainstorms/2026-05-23-brainstorm-smoke-baseline.md`. -3. If continuing brainstorm strategy work, run the fixed six-prompt suite for the L2 iteration variants (`blind`, `score-only`, `own-reviews`, `full-visibility`) via hidden `brainstorm --iteration-strategy`, preferably with Pi-backed model specs (`pi//`), then compare analyzer metrics grouped by artifact `iteration_strategy`. -4. If including multiple OpenCode-backed models from local config, use `--max-concurrent 1` and `--idle-timeout 480`; otherwise `todos/022` no longer blocks the main benchmark lane. -5. Do not implement Open Collider-style domain collisions before benchmark budget constraints are explicit. +3. If continuing brainstorm strategy work, read `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`; then either run a human/calibrated model-judge pass over the L2 panel review findings or start L3 prompt-reframing planning from `todos/018` using `score-only` as the baseline. +4. For future Pi-backed benchmark runs, use `--max-concurrent 1` unless Pi config locking is fixed; for OpenCode-backed models use `--max-concurrent 1` and `--idle-timeout 480` until `todos/022` is fixed. +5. Do not implement Open Collider-style domain collisions before benchmark budget constraints are explicit; if moving to L3, start with prompt-reframing expansion from `todos/018`. diff --git a/docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md b/docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md new file mode 100644 index 0000000..c28b93f --- /dev/null +++ b/docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md @@ -0,0 +1,188 @@ +--- +date: 2026-05-30 +topic: brainstorm-l2-iteration-strategy-benchmark +todo: 013-brainstorm-strategy-benchmarks +plan: 2026-05-23-001-research-brainstorm-strategy-benchmarks-plan +--- + +# Brainstorm L2 Iteration Strategy Benchmark + +## Summary + +Ran the fixed six-prompt brainstorm benchmark suite across the four hidden/internal iteration strategies: + +- `blind` +- `score-only` +- `own-reviews` +- `full-visibility` + +Final clean comparison uses 24 non-degraded, peer-evaluated runs: 6 prompts × 4 strategies. Each run used 4 models, 2 rounds, and 32 provider calls. + +High-level result: **`full-visibility` produced the highest judged quality but also the highest lexical overlap**. **`score-only` produced the lowest lexical overlap**, but not the best judged quality. `own-reviews` landed between them and had the highest disagreement under controversy selection. + +The result is not yet enough to change public UX because lexical overlap is only a cheap diversity proxy and full visibility may create semantic convergence not captured by score metrics alone. The next step should be whole-panel human/model-judge review on the saved artifacts. + +## Model Panel + +Pi-backed model routing was used for all final clean runs: + +```text +pi/openai-codex/gpt-5.4:off +pi/zai/glm-5.1:off +pi/kimi-coding/kimi-k2-thinking:off +pi/minimax/MiniMax-M2.7:off +``` + +Notes: + +- `:off` disables Pi model thinking output for benchmark stability and cost/latency control. +- Runs used `--max-concurrent 1` to avoid Pi local config lock contention observed during an initial concurrent attempt. +- `tundish_providers` stdout capture was raised from 1MB to 64MB because Pi JSON event streams can be much larger than the final assistant text. + +## Prompt Suite + +Same six-prompt suite as the 2026-05-23 benchmark: + +1. Product/strategy — privacy-first personal knowledge assistant. +2. Technical/design — secretless multi-model brainstorm artifact format. +3. Architecture — local AI coding tool plugin system with sandboxing. +4. Debugging/process — reduce flaky CI failures in a Rust/Bazel monorepo. +5. Research/science — low-cost indoor air quality experiments. +6. Governance/operations — lightweight governance for AI coding agents. + +## Commands + +Representative run command: + +```sh +cargo run -q -p refinery_cli -- brainstorm "$PROMPT" \ + --models pi/openai-codex/gpt-5.4:off,pi/zai/glm-5.1:off,pi/kimi-coding/kimi-k2-thinking:off,pi/minimax/MiniMax-M2.7:off \ + --max-rounds 2 \ + --panel-size 3 \ + --quality-floor 7.0 \ + --iteration-strategy "$STRATEGY" \ + --output-dir target/brainstorm-benchmark-2026-05-29-l2-pi-serial/$STRATEGY/$PROMPT_SLUG \ + --output-format json \ + --verbose \ + --idle-timeout 480 \ + --timeout 1800 \ + --max-concurrent 1 +``` + +Analyzer command: + +```sh +cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/run-dirs-clean.txt) \ + --output-format json > target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-analysis-clean.json + +cargo run -q -p refinery_cli -- benchmark-brainstorm $(cat target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/run-dirs-clean.txt) \ + --output-format text > target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-analysis-clean.txt +``` + +## Artifact Locations + +```text +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/run-dirs-clean.txt +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-analysis-clean.json +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-analysis-clean.txt +``` + +The clean run list selects the latest successful run for rerun prompts that initially had a single invalid MiniMax evaluation score. + +## Aggregate Selector Results + +Averages across six prompts per strategy. + +| Iteration strategy | Selector | Mean quality | Min quality | Disagreement | Lexical overlap | Meta preamble rate | +|---|---|---:|---:|---:|---:|---:| +| `blind` | `mean` | 8.000 | 7.667 | 0.398 | 0.103 | 0.000 | +| `blind` | `stddev` | 7.926 | 7.444 | 0.477 | 0.105 | 0.000 | +| `blind` | `controversy` | 7.926 | 7.444 | 0.477 | 0.105 | 0.000 | +| `blind` | `controversy_floor_7` | 7.926 | 7.444 | 0.477 | 0.105 | 0.000 | +| `blind` | `quality_x_lexdiv` | 8.000 | 7.667 | 0.398 | 0.103 | 0.000 | +| `score-only` | `mean` | 7.981 | 7.667 | 0.386 | 0.100 | 0.000 | +| `score-only` | `stddev` | 7.889 | 7.500 | 0.464 | 0.097 | 0.000 | +| `score-only` | `controversy` | 7.889 | 7.500 | 0.464 | 0.097 | 0.000 | +| `score-only` | `controversy_floor_7` | 7.889 | 7.500 | 0.464 | 0.097 | 0.000 | +| `score-only` | `quality_x_lexdiv` | 7.981 | 7.667 | 0.386 | 0.100 | 0.000 | +| `own-reviews` | `mean` | 8.111 | 7.889 | 0.386 | 0.109 | 0.000 | +| `own-reviews` | `stddev` | 8.019 | 7.667 | 0.517 | 0.112 | 0.000 | +| `own-reviews` | `controversy` | 8.019 | 7.667 | 0.517 | 0.112 | 0.000 | +| `own-reviews` | `controversy_floor_7` | 8.019 | 7.667 | 0.517 | 0.112 | 0.000 | +| `own-reviews` | `quality_x_lexdiv` | 8.093 | 7.833 | 0.360 | 0.103 | 0.000 | +| `full-visibility` | `mean` | 8.222 | 8.000 | 0.405 | 0.134 | 0.000 | +| `full-visibility` | `stddev` | 8.204 | 7.944 | 0.431 | 0.132 | 0.000 | +| `full-visibility` | `controversy` | 8.204 | 7.944 | 0.431 | 0.132 | 0.000 | +| `full-visibility` | `controversy_floor_7` | 8.204 | 7.944 | 0.431 | 0.132 | 0.000 | +| `full-visibility` | `quality_x_lexdiv` | 8.204 | 7.944 | 0.386 | 0.128 | 0.000 | + +## Production-Selector View + +For the current production-like selector, `controversy_floor_7`: + +| Iteration strategy | Mean quality | Min quality | Disagreement | Lexical overlap | Meta preamble rate | +|---|---:|---:|---:|---:|---:| +| `blind` | 7.926 | 7.444 | 0.477 | 0.105 | 0.000 | +| `score-only` | 7.889 | 7.500 | 0.464 | 0.097 | 0.000 | +| `own-reviews` | 8.019 | 7.667 | 0.517 | 0.112 | 0.000 | +| `full-visibility` | 8.204 | 7.944 | 0.431 | 0.132 | 0.000 | + +## Findings + +### 1. Full visibility wins on score quality but loses on lexical diversity + +`full-visibility` had the highest aggregate mean quality and minimum quality for every selector. Under `controversy_floor_7`, it averaged `8.204` mean quality and `7.944` minimum quality. + +However, its lexical overlap was also highest (`0.132` under `controversy_floor_7`). This supports the expected conformity risk: seeing all prior answers may help models refine toward evaluator-preferred answers, but it may also narrow the panel. + +### 2. Score-only remains the strongest cheap-diversity baseline + +`score-only` had the lowest lexical overlap under controversy-based selectors (`0.097`), but it also had the lowest aggregate quality in this run (`7.889` mean quality under `controversy_floor_7`). It still looks like the safest default if the product promise emphasizes independent divergent ideation. + +### 3. Own-reviews is a plausible middle ground + +`own-reviews` improved quality versus `score-only` and `blind`, and it had the highest panel disagreement under controversy selection (`0.517`). Lexical overlap was higher than `score-only` but lower than `full-visibility`. This may be the best candidate to inspect manually because it gives each model actionable critique without exposing all competing answers. + +### 4. Blind generation is not clearly better than score-only + +`blind` was competitive but did not dominate. It had slightly better quality than `score-only` under controversy selection in this run, but worse lexical overlap. Because blind iteration ignores useful score pressure, it is not an obvious replacement. + +### 5. Meta-preamble polish held across all variants + +Every aggregate row reported `meta_preamble_rate: 0.000`. The 2026-05-25 prompt polish appears robust across the L2 iteration strategies. + +### 6. The quality floor did not alter aggregate selector sets here + +In this Pi-backed run, `controversy` and `controversy_floor_7` produced identical aggregate metrics for every strategy. The candidate pool was generally above the floor; this differs from the 2026-05-23 OpenCode-heavy baseline where raw controversy selected several low-quality, high-disagreement answers. + +## Operational Notes + +An initial Pi-backed concurrent run with `--max-concurrent 4` produced two classes of failures: + +1. Pi local config lock contention, surfaced as transient provider credential/config errors. +2. `ResponseTooLarge` failures from Pi JSON event streams exceeding the previous 1MB stdout cap. + +The clean benchmark used `--max-concurrent 1` and a 64MB bounded stdout capture. A future provider improvement should stream-parse Pi JSON events instead of retaining the whole event stream. + +## Blind Panel Review Pack + +Added `refinery review-brainstorm-panels` to generate a blind review packet from brainstorm artifact directories. The command hides iteration strategies and model IDs in the reviewer-facing output, while writing a separate JSON answer key for later analysis. + +Generated the L2 review pack for `score-only`, `own-reviews`, and `full-visibility` panels selected by `controversy_floor_7`: + +```text +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-pack.md +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-key.json +``` + +The Markdown packet asks reviewers to score each panel on useful diversity, non-overlap, novelty, actionability, coverage, overall panel value, and best-answer regret. + +A first-pass qualitative review is documented in `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`. It found `score-only` strongest on useful diversity/non-overlap and `full-visibility` strongest on actionability/coverage, with no justification to change the production default yet. + +## Recommendations + +1. Use the blind review pack to compare `score-only`, `own-reviews`, and `full-visibility` without exposing strategy labels. +2. Do not promote `full-visibility` to the public default yet despite higher scores; first check semantic convergence in the panel review. +3. Keep `score-only` as the production default for now because it preserves the strongest measured lexical diversity and matches the original brainstorm design goal. +4. Complete `todos/026-stream-parse-pi-json-events.md` to stream-parse Pi JSON mode, avoiding large transport buffers while preserving current event extraction. +5. For L3 prompt-reframing work, use `score-only` as the default baseline and include `own-reviews` as the most interesting L2 challenger if budget allows. diff --git a/docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md b/docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md new file mode 100644 index 0000000..f3027a0 --- /dev/null +++ b/docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md @@ -0,0 +1,136 @@ +--- +date: 2026-06-01 +topic: brainstorm-l2-panel-review +todo: 013-brainstorm-strategy-benchmarks +plan: 2026-05-23-001-research-brainstorm-strategy-benchmarks-plan +review_artifacts: + pack: target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-pack.md + key: target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-key.json +--- + +# Brainstorm L2 Panel Review + +## Summary + +Performed a first-pass qualitative review using the blind L2 brainstorm panel review pack generated from the six-prompt Pi-backed benchmark. The review compared the `score-only`, `own-reviews`, and `full-visibility` iteration strategies using panels selected by `controversy_floor_7`. + +This is **not a replacement for a human panel or calibrated model-judge study**. It is a lightweight agent review to check whether the automated metrics from `docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md` are directionally plausible before changing defaults. + +Main result: + +- `score-only` still looked strongest on useful diversity and non-overlap. +- `full-visibility` looked strongest on actionability and coverage. +- `own-reviews` did not dominate overall, but produced the best debugging/process panel. +- No reviewed evidence justifies changing the production default away from `score-only` yet. + +## Review Method + +Input artifacts: + +```text +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-pack.md +target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/l2-panel-review-key.json +``` + +The review pack hides model IDs and iteration strategies in the reviewer-facing output. I used the reviewer-facing panel labels for scoring and then applied the answer key for strategy-level aggregation. Treat this as a first-pass agent review rather than a rigorously blinded human study. + +Scoring dimensions used the pack's 1-5 rubric: + +- useful diversity, +- non-overlap, +- novelty, +- actionability, +- coverage, +- overall panel value, +- best-answer regret / omissions. + +## Prompt-Level Scores + +| Prompt | Blind panel | Strategy | Useful diversity | Non-overlap | Novelty | Actionability | Coverage | Overall | Best-answer regret / omissions | +|---|---|---|---:|---:|---:|---:|---:|---:|---| +| architecture | A | `own-reviews` | 4 | 3 | 5 | 2 | 4 | 3 | Yes — highly imaginative, but too much abstract trust ecology and not enough implementable baseline architecture. | +| architecture | B | `full-visibility` | 4 | 4 | 5 | 3 | 5 | 4 | No major omission; good breadth across intent VM, codebase-as-oracle, and prompt-influence risk. | +| architecture | C | `score-only` | 4 | 4 | 4 | 4 | 5 | 4 | No major omission; best balance of strong sandbox model plus practical capability checkpoints. | +| debugging | A | `full-visibility` | 3 | 3 | 4 | 4 | 4 | 4 | Minor — duplicated determinism-contract ideas across answers. | +| debugging | B | `score-only` | 4 | 4 | 4 | 4 | 4 | 4 | No major omission; good technical/process spread, but one answer was less complete. | +| debugging | C | `own-reviews` | 4 | 4 | 4 | 5 | 5 | 5 | No major omission; strongest actionable panel across test ABI, failure capsules, isolation, and time-travel replay. | +| governance | A | `own-reviews` | 3 | 3 | 4 | 4 | 5 | 4 | No major omission, but stewardship/reversibility themes overlap. | +| governance | B | `score-only` | 4 | 4 | 4 | 3 | 3 | 3 | Yes — diverse mechanisms, but some are underdeveloped or gimmicky compared with concrete governance lifecycle models. | +| governance | C | `full-visibility` | 4 | 4 | 4 | 4 | 5 | 4 | No major omission; best balanced governance surface across runtime, intent, and risk-budget controls. | +| product | A | `own-reviews` | 5 | 5 | 4 | 3 | 4 | 4 | No major omission; broad set of wedges, though some are farther from a focused PKM startup wedge. | +| product | B | `score-only` | 5 | 5 | 5 | 3 | 4 | 4 | No major omission; very high novelty, but practicality varies across intellectual immune system, executor mode, and ZK recall. | +| product | C | `full-visibility` | 4 | 4 | 4 | 4 | 4 | 4 | Minor — coherent and practical, but over-indexes crisis/caregiving/threat scenarios. | +| research | A | `own-reviews` | 5 | 5 | 5 | 3 | 5 | 4 | No major omission; extremely novel, but includes some fragile biology/physics experiments. | +| research | B | `full-visibility` | 4 | 4 | 4 | 4 | 5 | 4 | No major omission; strong practical coverage, with some overlap around acoustic/resonance methods. | +| research | C | `score-only` | 5 | 5 | 5 | 4 | 5 | 5 | No major omission; best mix of perturbation experiments, soap-film sensing, and bio-integrative approaches. | +| technical | A | `full-visibility` | 3 | 3 | 4 | 4 | 4 | 4 | No major omission; strongest schema/format panel, though answers share the same secretless-shape motif. | +| technical | B | `score-only` | 3 | 3 | 3 | 2 | 2 | 3 | Yes — several entries are too short or schema-stub-like to be directly useful. | +| technical | C | `own-reviews` | 2 | 2 | 3 | 3 | 3 | 3 | Yes — one detailed schema is useful, but the panel is not diverse enough and includes very short answers. | + +## Strategy-Level Aggregate + +Averages across six prompts. + +| Strategy | Useful diversity | Non-overlap | Novelty | Actionability | Coverage | Overall | +|---|---:|---:|---:|---:|---:|---:| +| `score-only` | 4.17 | 4.17 | 4.17 | 3.33 | 3.83 | 3.83 | +| `own-reviews` | 3.83 | 3.67 | 4.17 | 3.33 | 4.33 | 3.83 | +| `full-visibility` | 3.67 | 3.67 | 4.17 | 3.83 | 4.50 | 4.00 | + +Prompt wins by overall qualitative panel value: + +| Prompt | Best panel(s) | Notes | +|---|---|---| +| architecture | `score-only`, `full-visibility` | `score-only` had the best practicality/diversity balance; `full-visibility` had slightly broader threat coverage. | +| debugging | `own-reviews` | The most actionable panel in the review. | +| governance | `full-visibility`, `own-reviews` | `full-visibility` was cleaner; `own-reviews` was also viable. | +| product | all roughly tied | `score-only` and `own-reviews` were more divergent; `full-visibility` was more coherent/practical. | +| research | `score-only` | Strongest combination of novelty, non-overlap, and testability. | +| technical | `full-visibility` | Most complete artifact-format panel; both other strategies had thin schema stubs. | + +## Findings + +### 1. Full visibility's higher score quality was not just evaluator bias + +The automated benchmark found `full-visibility` had the highest mean/min quality and highest lexical overlap. The qualitative review mostly agrees: full visibility often produced panels with better coverage, clearer integration, and more immediately usable proposals. + +This was especially visible in governance and technical artifact-format prompts, where seeing the whole prior round seems to help models fill obvious gaps and converge on complete answer shapes. + +### 2. Score-only still best matches the brainstorm diversity promise + +`score-only` had the highest reviewed useful-diversity and non-overlap averages. It produced the strongest research panel and a strong architecture panel. Its weaker results were not because it lacked imagination; rather, some panels contained underdeveloped or overly short answers. + +For a verb whose public promise is divergent ideation rather than best single answer synthesis, `score-only` remains the safest default. + +### 3. Own-reviews is not an obvious default, but it is a useful challenger + +`own-reviews` produced the best debugging/process panel and generally good coverage. However, it also produced the weakest architecture actionability and a weak technical panel. The review does not support promoting it over `score-only` or `full-visibility` globally. + +It may still be worth keeping as a hidden benchmark variant or eventual advanced option because it can improve practical refinement without fully exposing peer answers. + +### 4. Lexical overlap underestimates some semantic convergence risks + +`full-visibility` did not collapse into identical answers, but the review found repeated framing motifs in several panels: secretless artifact schemas, runtime/governance controls, and determinism contracts. These are not always harmful; sometimes they are the obvious correct abstractions. But they confirm that automated lexical overlap should be treated as a weak proxy, not a final diversity measure. + +### 5. Best-answer regret was concentrated in thin or over-abstract panels + +Major regret cases were mostly: + +- panels with beautiful but hard-to-implement abstractions, +- schema stubs too short to act on, +- panels where all answers leaned into the same framing. + +This suggests a future selection metric should penalize answer underdevelopment and panel redundancy, not just low individual mean scores. + +## Recommendation + +Keep the production default as `score-only`. + +Do not promote `full-visibility` as the default yet. It is a credible option for quality/coverage-oriented brainstorming, but the public `brainstorm` verb should continue prioritizing independent divergence until a larger human/model-judge panel confirms users prefer the higher-coverage, higher-overlap trade-off. + +For the next benchmark phase: + +1. Treat `score-only` as the L3 baseline. +2. Include `own-reviews` as the most interesting challenger only if budget allows. +3. Start L3 with prompt-reframing expansion from `todos/018`; continue deferring domain-collision/Open Collider variants until budget constraints are explicit. +4. Consider a future selector or reranker that adds a panel-level redundancy penalty and answer-completeness floor. diff --git a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md index b0723cb..fdbf405 100644 --- a/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md +++ b/docs/plans/2026-05-23-001-research-brainstorm-strategy-benchmarks-plan.md @@ -11,6 +11,9 @@ todo: 013-brainstorm-strategy-benchmarks **Enhanced:** 2026-05-23 (via `/deepen-plan`) **Reviewed:** 2026-05-31 (via `/coderabbit / review`) **Completed:** TBD +**Addendum:** 2026-05-30 — L2 iteration strategy suite completed with Pi-backed model routing; see `docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md`. +**Addendum:** 2026-05-30 — Added blind panel review pack generator (`refinery review-brainstorm-panels`) and generated the first L2 review packet. +**Addendum:** 2026-06-01 — Completed first-pass qualitative L2 panel review; see `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`. ## Context @@ -143,9 +146,37 @@ Added benchmark-only brainstorm iteration variants behind a hidden CLI flag: The production default remains `score-only`. Brainstorm JSON/text output and dry-run output now expose `iteration_strategy`. Artifact runs now write `metadata.json` with `iteration_strategy`, and `refinery benchmark-brainstorm` reads that metadata so benchmark outputs can group runs by iteration strategy. Evaluation artifacts now include `rationale` while remaining backwards compatible with the analyzer's score loading. +### Completed 2026-05-30 + +Ran the fixed six-prompt benchmark suite across all four L2 iteration variants with Pi-backed model routing: + +- `blind` +- `score-only` +- `own-reviews` +- `full-visibility` + +Final clean comparison uses 24 non-degraded, peer-evaluated runs (6 prompts × 4 strategies). The current production-like `controversy_floor_7` selector showed: + +| Iteration strategy | Mean quality | Min quality | Disagreement | Lexical overlap | +|---|---:|---:|---:|---:| +| `blind` | 7.926 | 7.444 | 0.477 | 0.105 | +| `score-only` | 7.889 | 7.500 | 0.464 | 0.097 | +| `own-reviews` | 8.019 | 7.667 | 0.517 | 0.112 | +| `full-visibility` | 8.204 | 7.944 | 0.431 | 0.132 | + +`full-visibility` had the highest score quality but also the highest lexical overlap, matching the expected conformity risk. `score-only` preserved the lowest lexical overlap and remains the recommended default until whole-panel diversity review says otherwise. `own-reviews` is the strongest middle-ground challenger. + +Operationally, the clean Pi-backed suite needed `--max-concurrent 1` to avoid local config lock contention and a larger bounded stdout capture because Pi JSON event streams can exceed 1MB while streaming normal benchmark-sized answers. + +### Completed 2026-06-01 + +Completed a first-pass qualitative panel review using the generated blind L2 review pack. The review compared `score-only`, `own-reviews`, and `full-visibility` panels on useful diversity, non-overlap, novelty, actionability, coverage, overall panel value, and best-answer regret. + +Result: `score-only` remained strongest on useful diversity/non-overlap; `full-visibility` was strongest on actionability/coverage; `own-reviews` did not dominate overall but produced the strongest debugging/process panel. The review does not justify changing the production default away from `score-only` yet. Treat `score-only` as the L3 baseline, include `own-reviews` as an optional challenger if budget allows, and defer public exposure of iteration strategy variants until stronger human/calibrated judge evidence exists. + ## Next Implementation Step -Run the fixed six-prompt benchmark suite across the four iteration variants (`blind`, `score-only`, `own-reviews`, `full-visibility`) using the hidden `brainstorm --iteration-strategy` flag. Prefer Pi-backed model routing for benchmark panels; if OpenCode-backed models are included from local config, serialize them with `--max-concurrent 1` until `todos/022` is fixed. Compare selector outputs and whole-panel metrics before promoting any variant to public UX. +Continue `todos/013` with either a human/calibrated model-judge pass over `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md` or move to L3 prompt-reframing expansion planning in `todos/018`. Do not change the production default based on the first-pass review alone. ## Verification @@ -157,6 +188,9 @@ Completed: - Artifact analyzer implemented as `refinery benchmark-brainstorm`. - Analyzer run against the two valid 2026-05-23 baseline artifacts and the later six-prompt suite. - Benchmark-only iteration variants implemented behind hidden CLI config. +- L2 six-prompt suite run across `blind`, `score-only`, `own-reviews`, and `full-visibility`; analyzer outputs saved under `target/brainstorm-benchmark-2026-05-29-l2-pi-serial/logs/`. +- Blind panel review pack generator added as `refinery review-brainstorm-panels`; L2 review pack and answer key generated under the same logs directory. +- First-pass L2 panel review documented in `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md`. - `cargo fmt --all -- --check` - `cargo test -p refinery_core brainstorm` - `cargo test -p refinery_cli` diff --git a/todos/013-brainstorm-strategy-benchmarks.md b/todos/013-brainstorm-strategy-benchmarks.md index cd1536b..ccd0ee0 100644 --- a/todos/013-brainstorm-strategy-benchmarks.md +++ b/todos/013-brainstorm-strategy-benchmarks.md @@ -4,7 +4,7 @@ priority: low milestone: v0.4 depends_on: 004-verb-brainstorm status: in_progress -updated: 2026-05-31 +updated: 2026-06-01 --- # Benchmark: Brainstorm Iteration and Selection Strategies @@ -17,6 +17,10 @@ updated: 2026-05-31 **Phase 3 deliverable:** `docs/brainstorms/2026-05-23-six-prompt-brainstorm-benchmark.md` +**Phase 4 deliverable:** `docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md` + +**Phase 5 deliverable:** `docs/brainstorms/2026-06-01-brainstorm-l2-panel-review.md` + ## Goal After brainstorm v0 ships (score-only iteration + controversial selection), benchmark alternative strategies on both axes to find what actually produces the best diverse panels. @@ -94,9 +98,11 @@ Benchmark-only iteration variants are now implemented behind hidden/internal CLI The default production behavior remains score-only. Brainstorm outputs and artifact `metadata.json` now expose `iteration_strategy`, and `refinery benchmark-brainstorm` reads that metadata for grouping. -Implementation PR #40 passed GitHub Actions, Buildkite, CodeRabbit, Gemini, and Codex review after follow-up fixes for nested model-spec validation, Clippy sort linting, string score parsing, Pi credential environment passthrough, and Windows `USERPROFILE` preservation. +The fixed six-prompt suite has now been run for all four L2 variants with Pi-backed model routing. Clean result: 24 non-degraded, peer-evaluated runs. Aggregate `controversy_floor_7` view: `full-visibility` scored highest on mean/min quality but had highest lexical overlap; `score-only` had the lowest lexical overlap but lower judged quality; `own-reviews` is the most interesting middle-ground challenger. + +A first-pass qualitative review over the generated blind panel review pack is complete. Result: `score-only` still looked strongest on useful diversity and non-overlap; `full-visibility` looked strongest on actionability and coverage; `own-reviews` did not dominate globally but produced the strongest debugging/process panel. Keep production default as `score-only` until stronger human/calibrated model-judge evidence says otherwise. -Next concrete step: run the fixed six-prompt suite for each variant using Pi-backed model routing where possible. If OpenCode-backed models are included from local config, serialize them with `--max-concurrent 1` until `todos/022` is fixed. Compare selectors (`mean`, `controversy`, `controversy_floor_7`, `quality_x_lexdiv`) and whole-panel metrics before promoting any variant to public UX. +Next concrete step: either run a human/calibrated model-judge pass over the L2 panel review findings, or move to L3 prompt-reframing expansion planning in `todos/018` with `score-only` as the baseline and `own-reviews` as an optional challenger if budget allows. ## References diff --git a/todos/026-stream-parse-pi-json-events.md b/todos/026-stream-parse-pi-json-events.md new file mode 100644 index 0000000..57a7596 --- /dev/null +++ b/todos/026-stream-parse-pi-json-events.md @@ -0,0 +1,32 @@ +--- +title: "fix: stream-parse Pi JSON events instead of buffering full stdout" +priority: medium +milestone: v0.4 +status: open +created: 2026-05-30 +--- + +# Stream-Parse Pi JSON Events + +## Problem + +Pi `--mode json` can emit JSON event streams that are much larger than the final assistant text, because streamed update events may repeat accumulated content. During the 2026-05-30 L2 brainstorm benchmark, normal benchmark-sized answers exceeded the previous 1MB generic stdout cap and caused `ResponseTooLarge` failures. + +As a tactical unblocker, `tundish_providers::process::MAX_RESPONSE_SIZE` was raised to 64MB. That keeps a bounded capture but is not ideal: it still buffers transport data that the Pi adapter only needs to parse into the latest assistant text. + +## Goal + +Teach the Pi provider path to parse JSONL incrementally and retain only the assistant text / relevant error state, avoiding large transport buffers while keeping existing timeout, process cleanup, and error behavior. + +## Notes + +- Keep generic provider stdout capture bounded for other CLIs. +- Preserve fatal stream-error detection. +- Preserve `extract_pi_response()` unit coverage, or split a streaming parser into a separately testable helper. +- Verify with a real Pi-backed brainstorm smoke run that previously exceeded 1MB. + +## References + +- `crates/tundish_providers/src/pi.rs` +- `crates/tundish_providers/src/process.rs` +- `docs/brainstorms/2026-05-30-brainstorm-l2-iteration-strategy-benchmark.md`