Dogfood fixes: a MiMo provider, native loops on any registry provider, safer source fetches

From D2's dogfooding of real workloads: a built-in MiMo provider (OpenAI- and Anthropic-compatible token plan); native loops fall back to the provider registry; a lock around source checkouts, and abbreviated commits found; Daytona rebuilds an image whose earlier build was canceled; the run, results, and viewer output say more of what happened. The missing-key hint no longer suggests --env.

Author: xdotli

crates/benchflow-agent/providers/mimo.toml

@@ -0,0 +1,24 @@
+# Xiaomi MiMo's token plan (Singapore): MiMo models (mimo-v2.6-flash,
+# mimo-v2.6-pro, mimo-v2.5, mimo-v2.5-pro). Run live on 2026-10-10: the
+# OpenAI-compatible API answered Chat Completions and the Responses API, the
+# Anthropic-compatible one Messages with an x-api-key, and `/models` lists the
+# models for a valid key. A plan key works only on this host (the pay-as-you-go
+# api.xiaomimimo.com and the China plan host reject it), and a plan whose
+# quota is spent answers 429 "quota exhausted".
+name = "mimo"
+description = "Xiaomi MiMo token plan: MiMo models"
+
+[openai]
+base_url = "https://token-plan-sgp.xiaomimimo.com/v1"
+apis = ["chat", "responses"]
+
+[anthropic]
+base_url = "https://token-plan-sgp.xiaomimimo.com/anthropic"
+
+[key]
+env = ["MIMO_API_KEY"]
+
+[check]
+# The model list: needs a valid key, spends nothing.
+endpoint = "openai"
+path = "/models"

crates/benchflow-agent/src/native/endpoint.rs

@@ -4,8 +4,8 @@
 //! A model is named `<provider>/<model>` (`vercel/deepseek/deepseek-v4-flash-0731`,
 //! `anthropic/claude-haiku-4-5`). [`EndpointResolver`] turns that name into
 //! an [`Endpoint`]. [`BuiltinProviders`] knows the providers BenchFlow ships
-//! with until the provider registry (`providers/`) replaces it behind the
-//! same trait.
+//! with, and falls back to the provider registry (`providers/`) for the
+//! rest.
 
 use std::fmt;
 
@@ -97,14 +97,12 @@ impl EndpointError {
         match self {
             EndpointError::NoProvider { .. } | EndpointError::UnknownProvider { .. } => format!(
                 "name the model as <provider>/<model> with one of: {}",
-                BUILTIN
-                    .iter()
-                    .map(|p| p.name)
-                    .collect::<Vec<_>>()
-                    .join(", ")
+                known_providers().join(", ")
             ),
+            // Not `--env`: those values reach the agent's sandbox, and a
+            // native loop's key must stay on this machine.
             EndpointError::NoKey { wanted, .. } => {
-                format!("export {wanted}=<key> (or pass --env {wanted}=<key>), then run again")
+                format!("export {wanted}=<key> in this shell, then run again")
             }
         }
     }
@@ -186,10 +184,7 @@ impl EndpointResolver for BuiltinProviders {
             });
         };
         let Some(spec) = BUILTIN.iter().find(|p| p.name == provider) else {
-            return Err(EndpointError::UnknownProvider {
-                provider: provider.to_string(),
-                model: model.to_string(),
-            });
+            return resolve_from_registry(provider, rest, model, host);
         };
         let found = spec.keys.iter().find_map(|name| {
             host.get(name)
@@ -238,6 +233,65 @@ impl EndpointResolver for BuiltinProviders {
     }
 }
 
+/// A provider the native loops have no entry for, from the provider
+/// registry (`zai/`, `deepseek/`, `mimo/`): its OpenAI-compatible endpoint
+/// when it takes a bearer key, else its Anthropic-compatible one when it takes
+/// `x-api-key`, the way the native clients send keys.
+fn resolve_from_registry(
+    provider: &str,
+    rest: &str,
+    model: &str,
+    host: &HostEnv,
+) -> Result<Endpoint, EndpointError> {
+    use crate::providers::{Auth, ProviderRegistry};
+    let unknown = || EndpointError::UnknownProvider {
+        provider: provider.to_string(),
+        model: model.to_string(),
+    };
+    let registry = ProviderRegistry::builtin();
+    let p = registry.find(provider).ok_or_else(unknown)?;
+    let (api, base_url) = if let Some(e) = p.openai.as_ref().filter(|e| e.auth == Auth::Bearer) {
+        (Api::OpenAiChat, e.base_url.clone())
+    } else if let Some(e) = p.anthropic.as_ref().filter(|e| e.auth == Auth::XApiKey) {
+        // The registry's Anthropic base URL stops before the version.
+        (Api::AnthropicMessages, format!("{}/v1", e.base_url))
+    } else {
+        return Err(unknown());
+    };
+    let key = crate::providers::find_provider_key(p, host)
+        .ok()
+        .filter(|k| !k.value().trim().is_empty())
+        .ok_or_else(|| EndpointError::NoKey {
+            provider: provider.to_string(),
+            wanted: p.key.env.join(" or "),
+        })?;
+    let base_url = host
+        .get(BASE_URL_OVERRIDE_ENV)
+        .map(|(v, _)| v.trim().to_string())
+        .filter(|v| !v.is_empty())
+        .unwrap_or(base_url);
+    Ok(Endpoint {
+        api,
+        base_url: base_url.trim_end_matches('/').to_string(),
+        key: ApiKey::new(key.value()),
+        model: rest.to_string(),
+        provider: provider.to_string(),
+        key_label: key.source.clone(),
+        upstreams: None,
+    })
+}
+
+/// Every provider a native loop can name: its own, then the registry's.
+fn known_providers() -> Vec<String> {
+    let mut names: Vec<String> = BUILTIN.iter().map(|p| p.name.to_string()).collect();
+    for p in crate::providers::ProviderRegistry::builtin().providers() {
+        if !names.contains(&p.name) {
+            names.push(p.name.clone());
+        }
+    }
+    names
+}
+
 #[cfg(test)]
 mod tests {
     use std::collections::BTreeMap;
@@ -332,6 +386,44 @@ mod tests {
         assert!(matches!(err, EndpointError::NoProvider { .. }));
     }
 
+    #[test]
+    fn registry_providers_work_for_native_loops_too() {
+        // MiMo's token plan: OpenAI-compatible with a bearer key.
+        let e = BuiltinProviders
+            .resolve(
+                "mimo/mimo-v2.6-flash",
+                &host(&[("MIMO_API_KEY", "k-123456789")]),
+            )
+            .unwrap();
+        assert_eq!(e.api, Api::OpenAiChat);
+        assert_eq!(
+            e.url(),
+            "https://token-plan-sgp.xiaomimimo.com/v1/chat/completions"
+        );
+        assert_eq!(e.model, "mimo-v2.6-flash");
+        assert_eq!(e.key_label, "MIMO_API_KEY (environment)");
+        // Z.ai and DeepSeek, which were refused before.
+        for (model, var) in [
+            ("zai/glm-5.3", "ZAI_API_KEY"),
+            ("deepseek/deepseek-chat", "DEEPSEEK_API_KEY"),
+        ] {
+            let e = BuiltinProviders
+                .resolve(model, &host(&[(var, "k-123456789")]))
+                .unwrap();
+            assert_eq!(e.api, Api::OpenAiChat, "{model}");
+        }
+        let err = BuiltinProviders
+            .resolve("mimo/mimo-v2.6-flash", &host(&[]))
+            .unwrap_err();
+        assert!(
+            err.next_step().contains("MIMO_API_KEY"),
+            "{}",
+            err.next_step()
+        );
+        let err = BuiltinProviders.resolve("nope/x", &host(&[])).unwrap_err();
+        assert!(err.next_step().contains("mimo"), "{}", err.next_step());
+    }
+
     #[test]
     fn the_override_points_any_provider_at_another_server() {
         let e = BuiltinProviders

crates/benchflow-agent/src/providers.rs

@@ -2,7 +2,7 @@
 //!
 //! A provider is a TOML file that says where its endpoints are, where its key
 //! comes from, and how to check the key. The built-in providers (`vercel`,
-//! `openai`, `anthropic`, `openrouter`, `zai`, `deepseek`) are embedded in
+//! `openai`, `anthropic`, `openrouter`, `zai`, `deepseek`, `mimo`) are embedded in
 //! the binary from `crates/benchflow-agent/providers/`; a project adds or
 //! replaces providers with its own files ([`ProviderRegistry::with_files`]).
 //!
@@ -364,6 +364,7 @@ const BUILTINS: &[(&str, &str)] = &[
     ),
     ("zai.toml", include_str!("../providers/zai.toml")),
     ("deepseek.toml", include_str!("../providers/deepseek.toml")),
+    ("mimo.toml", include_str!("../providers/mimo.toml")),
 ];
 
 /// Every provider a run knows: the built-ins, then the project's.

crates/benchflow-agent/src/providers/tests.rs

@@ -44,7 +44,7 @@ fn host(vars: &[(&str, &str)], home: Option<PathBuf>) -> HostEnv {
 }
 
 #[test]
-fn builtins_are_the_six_providers_with_their_endpoints() {
+fn builtins_are_the_seven_providers_with_their_endpoints() {
     let names: Vec<String> = Provider::builtins().into_iter().map(|p| p.name).collect();
     assert_eq!(
         names,
@@ -54,7 +54,8 @@ fn builtins_are_the_six_providers_with_their_endpoints() {
             "anthropic",
             "openrouter",
             "zai",
-            "deepseek"
+            "deepseek",
+            "mimo"
         ]
     );
     let registry = providers();
@@ -115,6 +116,18 @@ fn builtins_are_the_six_providers_with_their_endpoints() {
             .apis,
         [OpenAiApi::Chat]
     );
+    // MiMo's token plan serves both OpenAI APIs and Anthropic's, on one host.
+    let mimo = registry.find("mimo").unwrap();
+    assert_eq!(
+        mimo.openai.as_ref().unwrap().base_url,
+        "https://token-plan-sgp.xiaomimimo.com/v1"
+    );
+    assert_eq!(
+        mimo.openai.as_ref().unwrap().apis,
+        [OpenAiApi::Chat, OpenAiApi::Responses]
+    );
+    assert_eq!(mimo.anthropic.as_ref().unwrap().auth, Auth::XApiKey);
+    assert_eq!(mimo.key.env, ["MIMO_API_KEY"]);
     assert_eq!(
         ManifestSource::BuiltinProvider("vercel.toml").to_string(),
         "built-in providers/vercel.toml"
@@ -191,7 +204,7 @@ fn a_project_provider_replaces_a_builtin_of_the_same_name() {
     )
     .unwrap();
     let registry = ProviderRegistry::with_files([dir.path()]).unwrap();
-    assert_eq!(registry.providers().len(), 7);
+    assert_eq!(registry.providers().len(), 8);
     let vercel = registry.find("vercel").unwrap();
     assert_eq!(vercel.key.env, ["MY_GATEWAY_KEY"]);
     assert!(vercel.anthropic.is_none());
@@ -365,7 +378,7 @@ fn an_agent_that_cannot_use_the_provider_is_refused_with_its_options() {
     );
     assert_eq!(
         error.next_step(),
-        "pick another agent, or a provider `codex` can use: vercel, openai, openrouter"
+        "pick another agent, or a provider `codex` can use: vercel, openai, openrouter, mimo"
     );
 
     let error = route(&agent("claude"), Some("vercel/"), &providers()).unwrap_err();

crates/benchflow-cli/src/cli.rs

@@ -438,7 +438,7 @@ pub struct RunArgs {
     pub agent: Option<String>,
 
     /// The model, in the agent's own naming, or `<provider>/<model>` to run on a
-    /// provider's key (vercel, openai, anthropic, openrouter, zai, deepseek)
+    /// provider's key (vercel, openai, anthropic, openrouter, zai, deepseek, mimo)
     #[arg(short, long)]
     pub model: Option<String>,
 

crates/benchflow-cli/src/cmd/results.rs

@@ -392,11 +392,10 @@ fn compare(views: &[JobView]) -> Result<Exit> {
             Some(p) => {
                 let parts = [
                     format!(
-                        "  A − {b}: {:+.3} mean reward over {} (95% CI {:+.3} to {:+.3})",
+                        "  A − {b}: {:+.3} mean reward over {} ({})",
                         p.diff,
                         plural(p.tasks, "shared task"),
-                        p.low,
-                        p.high
+                        paired_interval(p.tasks, p.low, p.high)
                     ),
                     format!(
                         "A better on {}, {b} better on {}, tied on {}",
@@ -487,3 +486,28 @@ fn job_json(v: &JobView) -> serde_json::Value {
         "cost_recorded": v.records.iter().filter(|r| r.usage.cost_usd.is_some()).count(),
     })
 }
+
+/// The paired difference's interval, or why there is none: one shared task
+/// has no spread to make one from (the bounds would both be the difference).
+fn paired_interval(tasks: usize, low: f64, high: f64) -> String {
+    if tasks < 2 {
+        "no interval from one task".to_string()
+    } else {
+        format!("95% CI {low:+.3} to {high:+.3}")
+    }
+}
+
+#[cfg(test)]
+mod paired_tests {
+    #[test]
+    fn one_shared_task_gives_no_interval() {
+        assert_eq!(
+            super::paired_interval(1, 0.0, 0.0),
+            "no interval from one task"
+        );
+        assert_eq!(
+            super::paired_interval(3, -0.5, 0.25),
+            "95% CI -0.500 to +0.250"
+        );
+    }
+}

crates/benchflow-cli/src/cmd/run.rs

@@ -188,12 +188,17 @@ pub async fn run(args: RunArgs) -> Result<Exit> {
     }
     let (tasks, bad) = context::load_tasks(&roots)?;
     for r in &bad {
-        let errs = r.diagnostics.iter().filter(|d| d.is_error()).count();
+        let errs: Vec<(&str, &str)> = r
+            .diagnostics
+            .iter()
+            .filter(|d| d.is_error())
+            .map(|d| (d.code, d.message.as_str()))
+            .collect();
         note!(
             "  {} skipping {}: {} ({})",
             warn_mark(),
             ui::path(&r.root),
-            plural(errs, "error"),
+            skip_reason(&errs),
             dim(format!("bench check {}", ui::path(&r.root)))
         );
     }
@@ -414,10 +419,11 @@ pub async fn run(args: RunArgs) -> Result<Exit> {
     let what = format!(
         "{} {} on {}",
         bold("Running"),
-        match &model {
-            Some(m) => format!("{} ({m})", attempt.label()),
-            None => attempt.label(),
-        },
+        attempt_title(
+            &attempt.label(),
+            model.as_deref(),
+            attempt.model().as_deref()
+        ),
         plural(tasks.len() * trials, "trial"),
     );
     let place = format!("{} · {} at a time", provider.name(), st.concurrency);
@@ -1122,3 +1128,60 @@ mod exit_tests {
         assert!(run_exit(&[Status::Scored], 1) == Exit::Failed);
     }
 }
+
+/// Why a task was skipped, on its one line: the first error itself, so a
+/// dataset whose tasks all fail the same way says how without a `bench
+/// check` per task.
+fn skip_reason(errors: &[(&str, &str)]) -> String {
+    match errors {
+        [] => "it did not load".to_string(),
+        [(code, message)] => format!("[{code}] {message}"),
+        [(code, message), rest @ ..] => {
+            let more = if rest.len() == 1 { "error" } else { "errors" };
+            format!("[{code}] {message} (and {} more {more})", rest.len())
+        }
+    }
+}
+
+/// What runs, for the `Running ...` line: the agent and its model. With no
+/// `-m`, the agent's own default model (Claude Code's is Haiku), which
+/// nothing else on screen names.
+fn attempt_title(label: &str, asked: Option<&str>, running: Option<&str>) -> String {
+    match (asked, running) {
+        (Some(m), _) => format!("{label} ({m})"),
+        (None, Some(m)) => format!("{label} ({m}, its default)"),
+        (None, None) => label.to_string(),
+    }
+}
+
+#[cfg(test)]
+mod title_tests {
+    use super::attempt_title;
+
+    #[test]
+    fn a_skipped_task_says_its_first_error() {
+        let renamed = ("X203", "the `environment` key was renamed to `sandbox`");
+        assert_eq!(
+            super::skip_reason(&[renamed]),
+            "[X203] the `environment` key was renamed to `sandbox`"
+        );
+        assert_eq!(
+            super::skip_reason(&[renamed, ("X201", "x"), ("X201", "y")]),
+            "[X203] the `environment` key was renamed to `sandbox` (and 2 more errors)"
+        );
+        assert_eq!(super::skip_reason(&[]), "it did not load");
+    }
+
+    #[test]
+    fn the_running_line_names_a_default_model() {
+        assert_eq!(
+            attempt_title("builtin", Some("mimo/x"), Some("mimo/x")),
+            "builtin (mimo/x)"
+        );
+        assert_eq!(
+            attempt_title("claude", None, Some("claude-haiku-4-5-20251001")),
+            "claude (claude-haiku-4-5-20251001, its default)"
+        );
+        assert_eq!(attempt_title("oracle", None, None), "oracle");
+    }
+}