chi-bench: Actava · openai-agents · moonshotai/kimi-k3

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/README.md

@@ -0,0 +1,13 @@
+# Actava · openai-agents · moonshotai/kimi-k3
+
+Submitted: 2026-08-12 · chi-bench chi-bench-v1.0.0 · pass@1: **25.3%**
+
+| Domain | pass@1 | n_trials |
+|---|---|---|
+| pa_provider | 28.0% | 25 |
+| pa_um | 32.0% | 25 |
+| cm | 16.0% | 25 |
+
+Run executed 2026-07-22 (`frontier_models_seven_model_full_2026_07 experiment matrix`).
+
+See `submission.json` for the full manifest, `provenance.json` for reproducibility info.

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/provenance.json

@@ -0,0 +1,19 @@
+{
+  "chi_bench_git_sha": "2395e9e2ae7d42cdf2750dc3e67daeb0e672b910",
+  "image_digest": null,
+  "judge_model": "claude-opus-4-7",
+  "harness_version": "0.1.0",
+  "code_dirty": false,
+  "dataset_version": "chi-bench-v1.0.0",
+  "environment": "modal",
+  "started_at": "2026-07-22T08:03:23.788878Z",
+  "finished_at": "2026-07-22T14:22:38.269923Z",
+  "source": "frontier_models_seven_model_full_2026_07 experiment matrix",
+  "cost_basis": "All 75 trials have price-table-estimated cost rather than provider-reported cost (cost_reported_trials = 0); 4 pa_um trials with terminal agent exceptions contribute $0.",
+  "filing_note": "Run completed 2026-07-22 as part of the 2026-07-24 frontier release wave and was published in the 4/4 results post on 2026-07-29, but the leaderboard submission was not filed at the time. Filed 2026-08-12 with the original, unmodified aggregation.",
+  "trajectory_repair": {
+    "count": 4,
+    "reason": "4 pa_um trials terminated with agent exceptions and completed verification but emitted no trace.jsonl.",
+    "method": "Generated ATIF-v1.2 error-only trajectories with the official openai_agents_harness._build_atif_trajectory(records=[], ...) builder from each preserved agent/run_result.json; original logs were not modified."
+  }
+}

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/results.csv

@@ -0,0 +1,5 @@
+benchmark,dataset_version,submission_id,team,agent,model,domain,pass_at_1,n_trials,n_tasks,mean_cost_usd,mean_walltime_s,submitted_at
+chi-bench,chi-bench-v1.0.0,kimi-k3-openai-agents,Actava,openai-agents,moonshotai/kimi-k3,overall,0.25333333333333335,75,75,1.27148092,854.36141916,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,kimi-k3-openai-agents,Actava,openai-agents,moonshotai/kimi-k3,pa_provider,0.28,25,25,1.402652808,592.95700092,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,kimi-k3-openai-agents,Actava,openai-agents,moonshotai/kimi-k3,pa_um,0.32,25,25,1.03994136,792.82956592,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,kimi-k3-openai-agents,Actava,openai-agents,moonshotai/kimi-k3,cm,0.16,25,25,1.371848592,1177.29769064,2026-08-12T20:20:00Z

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/sub.yaml

@@ -0,0 +1,28 @@
+schema: chi-bench/submission/v1
+submission:
+  id: kimi-k3-openai-agents
+  team: Actava
+  contact: dark.savi@gmail.com
+  agent: openai-agents
+  model: moonshotai/kimi-k3
+  notes: 'Kimi K3 via the OpenAI Agents SDK harness with a 50-turn limit.
+
+    '
+  agent_kwargs:
+    max_turns: '50'
+    max_retries: '10'
+    max_tool_return_chars: '100000'
+run:
+  environment: modal
+  n_attempts: 1
+  concurrency: 5
+  max_retries: 2
+  timeout_multiplier: 1.0
+  agent_timeout_multiplier: 2.0
+  env_file: .env
+dataset:
+  version: chi-bench-v1.0.0
+  domains:
+  - pa_provider
+  - pa_um
+  - cm

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/submission.json

@@ -0,0 +1,74 @@
+{
+  "schema": "chi-bench/submission/v1",
+  "submission": {
+    "id": "kimi-k3-openai-agents",
+    "team": "Actava",
+    "contact": "dark.savi@gmail.com",
+    "agent": "openai-agents",
+    "model": "moonshotai/kimi-k3",
+    "notes": "Kimi K3 via the OpenAI Agents SDK harness with a 50-turn limit. Cost is derived from the normalized price table rather than provider-reported totals: 71 of 75 trials carry estimated cost and 4 pa_um trials terminated with agent exceptions and report no usage, so leaderboard cost is a lower bound on billed execution cost. Run executed 2026-07-22; filed late (see provenance.filing_note).\n",
+    "submitted_at": "2026-08-12T20:20:00Z"
+  },
+  "dataset": {
+    "name": "chi-bench",
+    "version": "chi-bench-v1.0.0",
+    "domains": [
+      "pa_provider",
+      "pa_um",
+      "cm"
+    ]
+  },
+  "results": {
+    "overall": {
+      "n_trials": 75,
+      "n_tasks": 75,
+      "pass_at_1": 0.25333333333333335,
+      "mean_cost_usd": 1.27148092,
+      "mean_walltime_s": 854.36141916
+    },
+    "per_domain": {
+      "pa_provider": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.28,
+        "mean_cost_usd": 1.402652808,
+        "mean_walltime_s": 592.95700092
+      },
+      "pa_um": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.32,
+        "mean_cost_usd": 1.03994136,
+        "mean_walltime_s": 792.82956592
+      },
+      "cm": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.16,
+        "mean_cost_usd": 1.371848592,
+        "mean_walltime_s": 1177.29769064
+      }
+    },
+    "mean_cost_usd": 1.27148092,
+    "mean_walltime_s": 854.36141916
+  },
+  "provenance": {
+    "chi_bench_git_sha": "2395e9e2ae7d42cdf2750dc3e67daeb0e672b910",
+    "image_digest": null,
+    "judge_model": "claude-opus-4-7",
+    "harness_version": "0.1.0",
+    "code_dirty": false,
+    "dataset_version": "chi-bench-v1.0.0",
+    "environment": "modal",
+    "started_at": "2026-07-22T08:03:23.788878Z",
+    "finished_at": "2026-07-22T14:22:38.269923Z",
+    "source": "frontier_models_seven_model_full_2026_07 experiment matrix",
+    "cost_basis": "All 75 trials have price-table-estimated cost rather than provider-reported cost (cost_reported_trials = 0); 4 pa_um trials with terminal agent exceptions contribute $0.",
+    "filing_note": "Run completed 2026-07-22 as part of the 2026-07-24 frontier release wave and was published in the 4/4 results post on 2026-07-29, but the leaderboard submission was not filed at the time. Filed 2026-08-12 with the original, unmodified aggregation.",
+    "trajectory_repair": {
+      "count": 4,
+      "reason": "4 pa_um trials terminated with agent exceptions and completed verification but emitted no trace.jsonl.",
+      "method": "Generated ATIF-v1.2 error-only trajectories with the official openai_agents_harness._build_atif_trajectory(records=[], ...) builder from each preserved agent/run_result.json; original logs were not modified."
+    }
+  }
+}

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/trials/cm/cm_afib_moderate_anxious_001__VGH2UyV/result.json

@@ -0,0 +1,115 @@
+{
+    "id": "e8875e4c-57b7-4919-b3a7-347902e9c1a9",
+    "task_name": "actava-ai/cm_afib_moderate_anxious_001",
+    "trial_name": "cm_afib_moderate_anxious_001__VGH2UyV",
+    "trial_uri": "file:///Users/haolin.chen/.config/superpowers/worktrees/chi-bench/frontier-models-smoke/logs/experiments/frontier_models_full_2026_07/04_openai-agents_moonshotai-kimi-k3_cm/2026-07-22__01-02-35/cm_afib_moderate_anxious_001__VGH2UyV",
+    "task_id": {
+        "path": "/var/folders/51/f74lzb_9073dvd740hb4qmd80000gn/T/chi_bench-modal-l7vrzqmj/cm_afib_moderate_anxious_001"
+    },
+    "source": "chi_bench-modal-l7vrzqmj",
+    "task_checksum": "ee0491f42e269fa02a2442ddbb574b37d6673ccd3a6c8fbcd403c3d2ff8eccfa",
+    "config": {
+        "task": {
+            "path": "/var/folders/51/f74lzb_9073dvd740hb4qmd80000gn/T/chi_bench-modal-l7vrzqmj/cm_afib_moderate_anxious_001",
+            "git_url": null,
+            "git_commit_id": null,
+            "name": null,
+            "ref": null,
+            "overwrite": false,
+            "download_dir": null,
+            "source": "chi_bench-modal-l7vrzqmj"
+        },
+        "trial_name": "cm_afib_moderate_anxious_001__VGH2UyV",
+        "trials_dir": "logs/experiments/frontier_models_full_2026_07/04_openai-agents_moonshotai-kimi-k3_cm/2026-07-22__01-02-35",
+        "timeout_multiplier": 1.0,
+        "agent_timeout_multiplier": 2.0,
+        "verifier_timeout_multiplier": null,
+        "agent_setup_timeout_multiplier": null,
+        "environment_build_timeout_multiplier": null,
+        "agent": {
+            "name": null,
+            "import_path": "chi_bench.experiment.agents.openai_agents_harness:OpenAIAgentsHarness",
+            "model_name": "moonshotai/kimi-k3",
+            "override_timeout_sec": null,
+            "override_setup_timeout_sec": null,
+            "max_timeout_sec": null,
+            "kwargs": {
+                "max_retries": 10,
+                "max_tool_return_chars": 100000,
+                "max_turns": 50
+            },
+            "env": {
+                "OPENAI_API_KEY": "${OPENAI_API_KEY}",
+                "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
+                "GEMINI_API_KEY": "${GEMINI_API_KEY}",
+                "OPENROUTER_API_KEY": "${OPENROUTER_API_KEY}",
+                "TINKER_API_KEY": "${TINKER_API_KEY}"
+            }
+        },
+        "environment": {
+            "type": null,
+            "import_path": "chi_bench.experiment.modal_env:ChiBenchModalEnvironment",
+            "force_build": false,
+            "delete": true,
+            "override_cpus": null,
+            "override_memory_mb": null,
+            "override_storage_mb": null,
+            "override_gpus": null,
+            "suppress_override_warnings": false,
+            "mounts_json": null,
+            "env": {},
+            "kwargs": {
+                "sandbox_timeout_secs": 86400
+            }
+        },
+        "verifier": {
+            "override_timeout_sec": null,
+            "max_timeout_sec": null,
+            "env": {},
+            "disable": false
+        },
+        "artifacts": [],
+        "job_id": "61097b8b-6dc8-4082-803b-19b9a3e11c3f"
+    },
+    "agent_info": {
+        "name": "openai-agents",
+        "version": "0.13.6",
+        "model_info": {
+            "name": "kimi-k3",
+            "provider": "moonshotai"
+        }
+    },
+    "agent_result": {
+        "n_input_tokens": 1739208,
+        "n_cache_tokens": 1639424,
+        "n_output_tokens": 27243,
+        "cost_usd": null,
+        "rollout_details": null,
+        "metadata": null
+    },
+    "verifier_result": {
+        "rewards": {
+            "reward": 0.0
+        }
+    },
+    "exception_info": null,
+    "started_at": "2026-07-22T09:23:34.235826Z",
+    "finished_at": "2026-07-22T09:48:56.840872Z",
+    "environment_setup": {
+        "started_at": "2026-07-22T09:23:34.237099Z",
+        "finished_at": "2026-07-22T09:23:56.915239Z"
+    },
+    "agent_setup": {
+        "started_at": "2026-07-22T09:23:56.915265Z",
+        "finished_at": "2026-07-22T09:24:01.377505Z"
+    },
+    "agent_execution": {
+        "started_at": "2026-07-22T09:24:01.377601Z",
+        "finished_at": "2026-07-22T09:42:27.904938Z"
+    },
+    "verifier": {
+        "started_at": "2026-07-22T09:42:36.302247Z",
+        "finished_at": "2026-07-22T09:48:54.086734Z"
+    },
+    "step_results": null
+}

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/trials/cm/cm_afib_moderate_anxious_001__VGH2UyV/verifier/reward.json

@@ -0,0 +1 @@
+{"reward": 0.0}

benchmarks/chi-bench/submissions/2026-07-24-kimi-k3-openai-agents/trials/cm/cm_afib_moderate_anxious_001__VGH2UyV/verifier/scorecard.json

@@ -0,0 +1,115 @@
+{
+  "binary_reward": 0.0,
+  "fractional_reward": 0.9473684210526315,
+  "passed_checks": 18,
+  "total_checks": 19,
+  "check_scores": {
+    "cm.assessment.record_exists": 1.0,
+    "cm.assessment.completed": 1.0,
+    "cm.assessment.required_sections_present": 1.0,
+    "cm.care_plan.record_exists": 1.0,
+    "cm.care_plan.finalized": 1.0,
+    "cm.care_plan.problem_count": 1.0,
+    "cm.care_plan.goal_structure": 1.0,
+    "cm.care_plan.intervention_structure": 1.0,
+    "cm.care_plan.escalation_conditions_present": 1.0,
+    "cm.care_plan.follow_up_cadence_present": 1.0,
+    "cm.chart_review.record_exists": 1.0,
+    "cm.cross_stage.target_status": 1.0,
+    "cm.cross_stage.audit_actions": 1.0,
+    "cm.cross_stage.no_forbidden_mutations": 1.0,
+    "judge.cm.chart_review.quality": 1.0,
+    "judge.cm.outreach.quality": 1.0,
+    "judge.cm.assessment.quality": 0.0,
+    "judge.cm.care_plan.quality": 1.0,
+    "judge.cm.stage_coherence": 1.0
+  },
+  "checks": {
+    "cm.assessment.record_exists": true,
+    "cm.assessment.completed": true,
+    "cm.assessment.required_sections_present": true,
+    "cm.care_plan.record_exists": true,
+    "cm.care_plan.finalized": true,
+    "cm.care_plan.problem_count": true,
+    "cm.care_plan.goal_structure": true,
+    "cm.care_plan.intervention_structure": true,
+    "cm.care_plan.escalation_conditions_present": true,
+    "cm.care_plan.follow_up_cadence_present": true,
+    "cm.chart_review.record_exists": true,
+    "cm.cross_stage.target_status": true,
+    "cm.cross_stage.audit_actions": true,
+    "cm.cross_stage.no_forbidden_mutations": true,
+    "judge.cm.chart_review.quality": true,
+    "judge.cm.outreach.quality": true,
+    "judge.cm.assessment.quality": false,
+    "judge.cm.care_plan.quality": true,
+    "judge.cm.stage_coherence": true
+  },
+  "failed_checks": [
+    "judge.cm.assessment.quality"
+  ],
+  "not_applicable_checks": [],
+  "stages": {
+    "cm_assessment": {
+      "passed": true,
+      "checks": {
+        "cm.assessment.record_exists": true,
+        "cm.assessment.completed": true,
+        "cm.assessment.required_sections_present": true
+      },
+      "passed_count": 3,
+      "total_count": 3,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_care_plan": {
+      "passed": true,
+      "checks": {
+        "cm.care_plan.record_exists": true,
+        "cm.care_plan.finalized": true,
+        "cm.care_plan.problem_count": true,
+        "cm.care_plan.goal_structure": true,
+        "cm.care_plan.intervention_structure": true,
+        "cm.care_plan.escalation_conditions_present": true,
+        "cm.care_plan.follow_up_cadence_present": true
+      },
+      "passed_count": 7,
+      "total_count": 7,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_chart_review": {
+      "passed": true,
+      "checks": {
+        "cm.chart_review.record_exists": true
+      },
+      "passed_count": 1,
+      "total_count": 1,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_cross_stage": {
+      "passed": true,
+      "checks": {
+        "cm.cross_stage.target_status": true,
+        "cm.cross_stage.audit_actions": true,
+        "cm.cross_stage.no_forbidden_mutations": true
+      },
+      "passed_count": 3,
+      "total_count": 3,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001"
+      }
+    }
+  }
+}