feat: add runbook studio dashboard and universal palette

.github/workflows/phase3-multihost-cancel.yml

@@ -0,0 +1,138 @@
+name: Phase3 Multi-host Cancellation
+
+on:
+  push:
+    branches: [agent/operator-experience]
+
+permissions:
+  contents: write
+
+jobs:
+  patch:
+    if: github.actor != 'github-actions[bot]'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: 1.97.1
+          components: rustfmt
+      - name: Add cooperative cancellation registry
+        run: |
+          python3 - <<'PY'
+          from pathlib import Path
+
+          p = Path('src-tauri/src/multi_host.rs')
+          s = p.read_text()
+          s = s.replace(
+              'use std::thread;\n',
+              'use std::collections::HashMap;\nuse std::sync::atomic::{AtomicBool, Ordering};\nuse std::sync::{Arc, Mutex};\nuse std::thread;\n\nuse once_cell::sync::Lazy;\n'
+          )
+          if 'static CANCELLATIONS' not in s:
+              s = s.replace(
+                  'const MAX_COMMAND_BYTES: usize = 16 * 1024;\n',
+                  'const MAX_COMMAND_BYTES: usize = 16 * 1024;\n\nstatic CANCELLATIONS: Lazy<Mutex<HashMap<String, Arc<AtomicBool>>>> =\n    Lazy::new(|| Mutex::new(HashMap::new()));\n'
+              )
+          s = s.replace(
+              'pub struct MultiHostRequest {\n    pub server_ids: Vec<String>,',
+              'pub struct MultiHostRequest {\n    pub run_id: Option<String>,\n    pub server_ids: Vec<String>,'
+          )
+          validation_marker = '''    if request.server_ids.is_empty() || servers.is_empty() {\n        return Err(anyhow!("select at least one server"));\n    }\n'''
+          validation_add = validation_marker + '''    if let Some(run_id) = request.run_id.as_deref() {\n        let valid = !run_id.is_empty()\n            && run_id.len() <= 64\n            && run_id\n                .chars()\n                .all(|character| character.is_ascii_alphanumeric() || character == '-');\n        if !valid {\n            return Err(anyhow!("run_id must be a UUID-like identifier up to 64 characters"));\n        }\n    }\n'''
+          if 'run_id must be a UUID-like identifier' not in s:
+              if validation_marker not in s:
+                  raise SystemExit('multi-host validation marker missing')
+              s = s.replace(validation_marker, validation_add, 1)
+          if 'pub fn request_cancel' not in s:
+              marker = 'fn failed_output(message: impl ToString) -> CommandOutput {'
+              functions = '''pub fn request_cancel(run_id: &str) -> bool {\n    CANCELLATIONS\n        .lock()\n        .ok()\n        .and_then(|registry| registry.get(run_id).cloned())\n        .map(|flag| {\n            flag.store(true, Ordering::SeqCst);\n            true\n        })\n        .unwrap_or(false)\n}\n\n'''
+              s = s.replace(marker, functions + marker)
+
+          old_start = '''pub fn execute(request: &MultiHostRequest, servers: Vec<Server>) -> Result<MultiHostRun> {\n    validate(request, &servers)?;\n    let started_at = Utc::now().to_rfc3339();\n    let mut results = Vec::with_capacity(servers.len());\n\n    for batch in servers.chunks(request.concurrency) {'''
+          new_start = '''pub fn execute(request: &MultiHostRequest, servers: Vec<Server>) -> Result<MultiHostRun> {\n    validate(request, &servers)?;\n    let run_id = request\n        .run_id\n        .clone()\n        .unwrap_or_else(|| uuid::Uuid::new_v4().to_string());\n    let cancellation = Arc::new(AtomicBool::new(false));\n    CANCELLATIONS\n        .lock()\n        .map_err(|_| anyhow!("multi-host cancellation registry lock poisoned"))?\n        .insert(run_id.clone(), cancellation.clone());\n    let started_at = Utc::now().to_rfc3339();\n    let mut results = Vec::with_capacity(servers.len());\n\n    for batch in servers.chunks(request.concurrency) {\n        if cancellation.load(Ordering::SeqCst) {\n            break;\n        }'''
+          if old_start not in s:
+              raise SystemExit('multi-host execute start marker missing')
+          s = s.replace(old_start, new_start, 1)
+          old_status = '''    let successes = results.iter().filter(|result| result.output.success).count();\n    let status = if successes == results.len() {\n        "success"\n    } else if successes == 0 {\n        "failed"\n    } else {\n        "partial"\n    };\n    Ok(MultiHostRun {\n        id: uuid::Uuid::new_v4().to_string(),'''
+          new_status = '''    let cancelled = cancellation.load(Ordering::SeqCst);\n    if let Ok(mut registry) = CANCELLATIONS.lock() {\n        registry.remove(&run_id);\n    }\n    let successes = results.iter().filter(|result| result.output.success).count();\n    let status = if cancelled {\n        "cancelled"\n    } else if successes == results.len() {\n        "success"\n    } else if successes == 0 {\n        "failed"\n    } else {\n        "partial"\n    };\n    Ok(MultiHostRun {\n        id: run_id,'''
+          if old_status not in s:
+              raise SystemExit('multi-host status marker missing')
+          s = s.replace(old_status, new_status, 1)
+          # Test request literals need run_id.
+          s = s.replace(
+              '        let mut request = MultiHostRequest {\n            server_ids:',
+              '        let mut request = MultiHostRequest {\n            run_id: None,\n            server_ids:'
+          )
+          p.write_text(s)
+
+          p = Path('src-tauri/src/operator_commands.rs')
+          s = p.read_text()
+          old = '''#[tauri::command]\npub fn multi_host_run(\n    state: State<AppState>,\n    request: MultiHostRequest,\n) -> CommandResult<MultiHostRun> {'''
+          new = '''#[tauri::command]\npub async fn multi_host_run(\n    state: State<'_, AppState>,\n    request: MultiHostRequest,\n) -> CommandResult<MultiHostRun> {'''
+          if old not in s:
+              raise SystemExit('multi_host_run command marker missing')
+          s = s.replace(old, new, 1)
+          old_exec = '''    let run = remote(multi_host::execute(&request, servers))?;\n    {\n        let conn = state.db.lock().unwrap();\n        internal(operator_data::save_multi_host_run(&conn, &run))?;\n    }\n    Ok(run)\n}\n'''
+          new_exec = '''    let run = tauri::async_runtime::spawn_blocking(move || multi_host::execute(&request, servers))\n        .await\n        .map_err(|error| DomainError::internal(format!("multi-host worker failed: {error}")))?\n        .map_err(|error| DomainError::remote(error.to_string()))?;\n    {\n        let conn = state.db.lock().unwrap();\n        internal(operator_data::save_multi_host_run(&conn, &run))?;\n    }\n    Ok(run)\n}\n\n#[tauri::command]\npub fn multi_host_cancel(run_id: String) -> CommandResult<()> {\n    if multi_host::request_cancel(&run_id) {\n        Ok(())\n    } else {\n        Err(DomainError::validation(\n            "run_id",\n            "multi-host run is not active or already completed",\n        ))\n    }\n}\n'''
+          if old_exec not in s:
+              raise SystemExit('multi-host execute command marker missing')
+          s = s.replace(old_exec, new_exec, 1)
+          p.write_text(s)
+
+          p = Path('src-tauri/src/lib.rs')
+          s = p.read_text()
+          marker = '            operator_commands::multi_host_run,\n'
+          if 'operator_commands::multi_host_cancel' not in s:
+              if marker not in s:
+                  raise SystemExit('lib multi-host command marker missing')
+              s = s.replace(marker, marker + '            operator_commands::multi_host_cancel,\n')
+          p.write_text(s)
+
+          p = Path('src/operatorTypes.ts')
+          s = p.read_text()
+          s = s.replace(
+              'export interface MultiHostRequest {\n  server_ids: string[];',
+              'export interface MultiHostRequest {\n  run_id?: string | null;\n  server_ids: string[];'
+          )
+          p.write_text(s)
+
+          p = Path('src/operatorApi.ts')
+          s = p.read_text()
+          marker = '''export const multiHostRun = (request: MultiHostRequest) =>\n  invoke<MultiHostRun>("multi_host_run", { request });\n'''
+          addition = marker + '''export const multiHostCancel = (runId: string) =>\n  invoke<void>("multi_host_cancel", { runId });\n'''
+          if 'multiHostCancel' not in s:
+              if marker not in s:
+                  raise SystemExit('operatorApi multiHostRun marker missing')
+              s = s.replace(marker, addition)
+          p.write_text(s)
+
+          p = Path('src/components/OperatorCenter.tsx')
+          s = p.read_text()
+          if 'const [runningId' not in s:
+              s = s.replace(
+                  '  const [run, setRun] = useState<MultiHostRun | null>(null);',
+                  '  const [run, setRun] = useState<MultiHostRun | null>(null);\n  const [runningId, setRunningId] = useState<string | null>(null);\n  const [cancelRequested, setCancelRequested] = useState(false);'
+              )
+          old_execute = '''  async function execute() {\n    try {\n      setRun(await operatorApi.multiHostRun({\n        server_ids: selected,\n        command,\n        concurrency,\n        production_confirmed: productionConfirmed,\n        destructive_confirmed: destructiveConfirmed,\n      }));\n    } catch (reason) { reportError(reason); }\n  }'''
+          new_execute = '''  async function execute() {\n    const runId = crypto.randomUUID();\n    setRunningId(runId);\n    setCancelRequested(false);\n    setRun(null);\n    try {\n      setRun(await operatorApi.multiHostRun({\n        run_id: runId,\n        server_ids: selected,\n        command,\n        concurrency,\n        production_confirmed: productionConfirmed,\n        destructive_confirmed: destructiveConfirmed,\n      }));\n    } catch (reason) {\n      reportError(reason);\n    } finally {\n      setRunningId(null);\n      setCancelRequested(false);\n    }\n  }\n\n  async function cancelRun() {\n    if (!runningId) return;\n    setCancelRequested(true);\n    try {\n      await operatorApi.multiHostCancel(runningId);\n    } catch (reason) {\n      reportError(reason);\n      setCancelRequested(false);\n    }\n  }'''
+          if old_execute not in s:
+              raise SystemExit('OperatorCenter multi-host execute marker missing')
+          s = s.replace(old_execute, new_execute, 1)
+          old_button = '<button className="primary" disabled={!selected.length || !command.trim()} onClick={() => void execute()}>Run on {selected.length} host{selected.length === 1 ? "" : "s"}</button>'
+          new_button = '''<div className="flex">\n        <button className="primary" disabled={Boolean(runningId) || !selected.length || !command.trim()} onClick={() => void execute()}>\n          {runningId ? "Running…" : `Run on ${selected.length} host${selected.length === 1 ? "" : "s"}`}\n        </button>\n        {runningId ? <button className="warn" disabled={cancelRequested} onClick={() => void cancelRun()}>{cancelRequested ? "Cancellation requested…" : "Cancel remaining batches"}</button> : null}\n      </div>\n      {runningId ? <div className="panel-hint">Cancellation stops new batches; SSH commands already in flight are allowed to finish and are still audited.</div> : null}'''
+          if old_button not in s:
+              raise SystemExit('OperatorCenter multi-host button marker missing')
+          s = s.replace(old_button, new_button, 1)
+          p.write_text(s)
+          PY
+          cargo fmt --manifest-path src-tauri/Cargo.toml
+      - name: Remove helper and commit
+        run: |
+          git rm .github/workflows/phase3-multihost-cancel.yml
+          git config user.name github-actions[bot]
+          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
+          git add src-tauri/src src/operatorTypes.ts src/operatorApi.ts src/components/OperatorCenter.tsx
+          git commit -m 'feat: add cancellable multi-host batches'
+          git push origin HEAD:agent/operator-experience

README.md

@@ -8,7 +8,7 @@ RemoteOpsX combines SSH/SFTP/FTP/RDP/VNC access, agentless server-health monitor
 
 ## Project status
 
-RemoteOpsX is a validated MVP undergoing release verification. The core operator workflows and the P0 implementation for explicit SSH host trust, runtime dependency preflight, restart reconciliation, and known-secret redaction are implemented. The remaining public-release blockers are the live SSH integration fixture, packaged-app E2E/security coverage, signed distribution, and repository-side enforcement of the documented branch-protection policy.
+RemoteOpsX is a validated MVP undergoing release verification. The core operator workflows and the P0 implementation for explicit SSH host trust, runtime dependency preflight, restart reconciliation, known-secret redaction, bastion routing, and the live ephemeral-SSH integration fixture are implemented. Operator depth now includes historical health/alerts, persistent transfer queues, Runbook Studio, tunnel resilience, encrypted workspace backup, guarded multi-host operations, a live operations dashboard, and a universal command palette. The remaining public-release blockers are packaged-app E2E/security coverage, signed distribution, and repository-side enforcement of the documented branch-protection policy.
 
 Current automated checks:
 
@@ -57,6 +57,14 @@ Health collection runs over a separate SSH exec path and does not interfere with
 - **Logs / diagnostic bundles** — remote log and journal collection with local export.
 - **Runbooks** — YAML steps, variables, confirmation boundaries, output capture, and persisted history.
 - **SSH tunnels** — local, remote, and dynamic SOCKS forwards with persisted records and live-process reconciliation.
+- **Bastion routing** — key-auth jump hosts with explicit trust on both hops and shared routing across terminal, exec, health, runbooks, SFTP/SCP and tunnels.
+- **Health history + alerts** — bounded local history, threshold/consecutive/cooldown rules, persisted alert acknowledgement and dashboard rollups.
+- **Persistent transfers** — ControlMaster-backed SFTP/SCP queue with cancellation, recursive transfer, byte progress for single files, chmod and drag/drop.
+- **Runbook Studio** — YAML validation/import/export, variable-aware dry-run preview and retry from the first failed step.
+- **Operator Center + Dashboard** — fleet health, alerts, transfers, multi-host operations, tunnel policies and backup/restore in one local-first workspace.
+- **Multi-host operations** — bounded fan-out with production/destructive safety gates and per-host audit results.
+- **Encrypted workspace backup** — versioned encrypted export/import without keyring secret export; restored password profiles require credential re-entry.
+- **Universal command palette** — fuzzy actions across servers, protocols, health, diagnostics, runbooks, snippets and application workflows.
 - **Startup recovery** — stale sessions/runbooks are marked interrupted and stale persisted tunnel state is reconciled after an unclean restart.
 - **Known-secret redaction** — keyring secrets are centrally registered and masked from buffered remote output, backend errors/logging, runbook results, and exported text; known credentials are rejected from persisted profile metadata, snippets, and runbooks.
 - **Session history** and **command snippets**.
@@ -200,7 +208,6 @@ Current protections:
 
 Remaining release blockers:
 
-- Live ephemeral-SSH integration tests must verify key/password auth, PTY/exec/SCP/tunnel behavior, and host-key mismatch rejection against a real SSH server.
 - Packaged desktop E2E/security tests must cover fresh install, upgrade, keyring/dependency failures, destructive confirmation boundaries, and diagnostic-export leakage.
 - Release artifacts have checksums but are not yet signed/notarized.
 - The repository-side `main` branch protection/ruleset must enforce the required CI checks documented in `.github/BRANCH_PROTECTION.md`.

TODO.md

@@ -30,19 +30,23 @@ Status legend: ✅ done · 🚧 partial · ⬜ planned
 - ⬜ **Signed releases** — code-sign/notarize macOS artifacts and sign Linux release artifacts. Checksums are published, but signatures are still required for a strong distribution trust chain.
 
 ## P1 — operator depth
-- ⬜ **Persistent SFTP subsystem** — replace per-operation `ssh`/`scp`; add progress, cancellation, recursive transfers, chmod and drag/drop.
-- ⬜ **Runbook editor** — validated form/YAML editor, variables, dry-run, import/export and partial retry from a failed step.
-- ⬜ **Tunnel resilience** — health checks, auto-reconnect, autostart-on-connect and explicit failure state.
-- ⬜ **Health history** — bounded per-server time-series retention, custom thresholds and desktop/webhook alert routing.
+- ✅ **Persistent SFTP transfer subsystem** — strict per-server OpenSSH ControlMaster reuse, cancellable background upload/download jobs, recursive transfers, single-file byte progress, chmod and Tauri-native drag/drop queueing.
+- ✅ **Runbook Studio** — bounded YAML import/export, validation, variable rendering, non-executing dry-run preview, destructive/confirmation markers and retry from the first failed step.
+- ✅ **Tunnel resilience** — persisted autostart/auto-reconnect desired-state policies, reconciliation, explicit failed state and explicit Stop precedence.
+- 🚧 **Health history + alerts** — bounded 30-second/7-day per-server history, custom threshold/consecutive/cooldown rules, persisted acknowledgement and dashboard visualization are implemented. Desktop/webhook delivery remains planned.
 - ⬜ **App lock** — optional local lock/master credential with keyring-aware unlock behavior.
-- ⬜ **Import/export** — encrypted, versioned backup/restore of profiles, settings, runbooks and snippets without exporting keyring secrets by default.
+- ✅ **Import/export** — encrypted versioned workspace backup/restore for profiles, settings, runbooks, snippets, bastions, alert rules and tunnels; keyring secrets are never exported and password profiles require credential re-entry.
 - ✅ **Jump hosts / bastion routing** — first-class key-auth bastion configuration with strict app-owned SSH config, explicit bastion fingerprint trust, destination fingerprint inspection through the trusted bastion, and shared routing across terminal/exec/health/runbooks/SCP/tunnels.
 
+## Product experience
+- ✅ **Operations Dashboard** — fleet rollup from persisted health, alerts, tunnel state and recent automation, with direct server/SSH drill-down.
+- ✅ **Universal Command Palette** — fuzzy index across application actions, protocol-specific server actions, health/diagnostics, runbooks and snippets; snippet execution requires a focused target and confirmation.
+
 ## P2 — transport and desktop expansion
 - ⬜ Native SSH transport (for example `russh`/`libssh2`) if it materially improves host-key UX, multiplexing, passphrase prompts and portability over the hardened system-OpenSSH backend.
 - ⬜ Embedded RDP and VNC sessions instead of external viewers.
 - ⬜ Flatpak and additional distro packaging targets after the signed release path is stable.
-- ⬜ Broadcast-to-many commands with strong environment/risk safeguards and per-host result tracking.
+- ✅ Broadcast-to-many commands with a 50-host cap, bounded concurrency, independent production/destructive confirmations, known-secret rejection and persisted per-host result tracking.
 
 ## Quality gates
 - ✅ Frontend regression tests