actava-ai · haolin-chen-actava · Aug 12, 2026 · Aug 12, 2026
diff --git a/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/README.md b/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/README.md
@@ -0,0 +1,13 @@
+# Actava · claude-code · anthropic/claude-opus-5
+
+Submitted: 2026-08-12 · chi-bench chi-bench-v1.0.0 · pass@1: **37.3%**
+
+| Domain | pass@1 | n_trials |
+|---|---|---|
+| pa_provider | 20.0% | 25 |
+| pa_um | 32.0% | 25 |
+| cm | 60.0% | 25 |
+
+Run executed 2026-07-24 on Modal.
+
+See `submission.json` for the full manifest, `provenance.json` for reproducibility info.
diff --git a/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/provenance.json b/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/provenance.json
@@ -0,0 +1,14 @@
+{
+  "chi_bench_git_sha": "2395e9e2ae7d42cdf2750dc3e67daeb0e672b910",
+  "image_digest": null,
+  "judge_model": "claude-opus-4-7",
+  "harness_version": "0.1.0",
+  "code_dirty": false,
+  "dataset_version": "chi-bench-v1.0.0",
+  "environment": "modal",
+  "started_at": "2026-07-24T19:08:02Z",
+  "finished_at": "2026-07-24T23:22:36Z",
+  "source": "logs/submissions/opus-5-claude-code (cb submission run)",
+  "original_run_sha": "c4c11887f95a3909eeb8afc80c4bfa41221785be",
+  "filing_note": "Run completed 2026-07-24T23:22:36Z and an infographic was produced, but the leaderboard submission was never filed. Filed 2026-08-12 from the original, unmodified run manifest; original_run_sha records the code SHA the run actually executed against."
+}
diff --git a/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/results.csv b/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/results.csv
@@ -0,0 +1,5 @@
+benchmark,dataset_version,submission_id,team,agent,model,domain,pass_at_1,n_trials,n_tasks,mean_cost_usd,mean_walltime_s,submitted_at
+chi-bench,chi-bench-v1.0.0,opus-5-claude-code,Actava,claude-code,anthropic/claude-opus-5,overall,0.37333333333333335,75,75,4.790005273333334,770.8856552666666,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,opus-5-claude-code,Actava,claude-code,anthropic/claude-opus-5,pa_provider,0.2,25,25,3.44516798,405.74023172,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,opus-5-claude-code,Actava,claude-code,anthropic/claude-opus-5,pa_um,0.32,25,25,4.3660177,628.4504582,2026-08-12T20:20:00Z
+chi-bench,chi-bench-v1.0.0,opus-5-claude-code,Actava,claude-code,anthropic/claude-opus-5,cm,0.6,25,25,6.55883014,1278.46627588,2026-08-12T20:20:00Z
diff --git a/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/sub.yaml b/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/sub.yaml
@@ -0,0 +1,22 @@
+schema: chi-bench/submission/v1
+
+submission:
+  id: opus-5-claude-code
+  team: Actava
+  contact: dark.savi@gmail.com
+  agent: claude-code
+  model: anthropic/claude-opus-5
+  notes: |
+    Claude Opus 5 via the stock claude-code harness (MCP tools).
+
+run:
+  environment: modal
+  n_attempts: 1
+  concurrency: 5
+  max_retries: 2
+  timeout_multiplier: 2.0
+  env_file: .env
+
+dataset:
+  version: chi-bench-v1.0.0
+  domains: [pa_provider, pa_um, cm]
diff --git a/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/submission.json b/benchmarks/chi-bench/submissions/2026-07-24-opus-5-claude-code/submission.json
@@ -0,0 +1,69 @@
+{
+  "schema": "chi-bench/submission/v1",
+  "submission": {
+    "id": "opus-5-claude-code",
+    "team": "Actava",
+    "contact": "dark.savi@gmail.com",
+    "agent": "claude-code",
+    "model": "anthropic/claude-opus-5",
+    "notes": "Claude Opus 5 via the stock claude-code harness (MCP tools), single attempt per task. Run executed 2026-07-24 alongside the frontier release wave; the packet was produced locally but never filed at the time (see provenance.filing_note).\n",
+    "submitted_at": "2026-08-12T20:20:00Z"
+  },
+  "dataset": {
+    "name": "chi-bench",
+    "version": "chi-bench-v1.0.0",
+    "domains": [
+      "pa_provider",
+      "pa_um",
+      "cm"
+    ]
+  },
+  "results": {
+    "overall": {
+      "n_trials": 75,
+      "n_tasks": 75,
+      "pass_at_1": 0.37333333333333335,
+      "mean_cost_usd": 4.790005273333334,
+      "mean_walltime_s": 770.8856552666666
+    },
+    "per_domain": {
+      "pa_provider": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.2,
+        "mean_cost_usd": 3.44516798,
+        "mean_walltime_s": 405.74023172
+      },
+      "pa_um": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.32,
+        "mean_cost_usd": 4.3660177,
+        "mean_walltime_s": 628.4504582
+      },
+      "cm": {
+        "n_trials": 25,
+        "n_tasks": 25,
+        "pass_at_1": 0.6,
+        "mean_cost_usd": 6.55883014,
+        "mean_walltime_s": 1278.46627588
+      }
+    },
+    "mean_cost_usd": 4.790005273333334,
+    "mean_walltime_s": 770.8856552666666
+  },
+  "provenance": {
+    "chi_bench_git_sha": "2395e9e2ae7d42cdf2750dc3e67daeb0e672b910",
+    "image_digest": null,
+    "judge_model": "claude-opus-4-7",
+    "harness_version": "0.1.0",
+    "code_dirty": false,
+    "dataset_version": "chi-bench-v1.0.0",
+    "environment": "modal",
+    "started_at": "2026-07-24T19:08:02Z",
+    "finished_at": "2026-07-24T23:22:36Z",
+    "source": "logs/submissions/opus-5-claude-code (cb submission run)",
+    "original_run_sha": "c4c11887f95a3909eeb8afc80c4bfa41221785be",
+    "filing_note": "Run completed 2026-07-24T23:22:36Z and an infographic was produced, but the leaderboard submission was never filed. Filed 2026-08-12 from the original, unmodified run manifest; original_run_sha records the code SHA the run actually executed against."
+  }
+}
diff --git a/...-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/agent/trajectory.jsonl.zst b/...-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/agent/trajectory.jsonl.zst
diff --git a/...2026-07-24-opus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/result.json b/...2026-07-24-opus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/result.json
@@ -0,0 +1,121 @@
+{
+    "id": "6e1044d7-5451-406b-a9b5-e48e46f71806",
+    "task_name": "actava-ai/cm_afib_moderate_anxious_001",
+    "trial_name": "cm_afib_moderate_anxious_001__YBzGaU4",
+    "trial_uri": "file:///Users/haolin.chen/repos/chi-bench/logs/submissions/opus-5-claude-code/cm/opus-5-claude-code__cm/cm_afib_moderate_anxious_001__YBzGaU4",
+    "task_id": {
+        "path": "/var/folders/51/f74lzb_9073dvd740hb4qmd80000gn/T/chi_bench-modal-tn459f5x/cm_afib_moderate_anxious_001"
+    },
+    "source": "chi_bench-modal-tn459f5x",
+    "task_checksum": "ee0491f42e269fa02a2442ddbb574b37d6673ccd3a6c8fbcd403c3d2ff8eccfa",
+    "config": {
+        "task": {
+            "path": "/var/folders/51/f74lzb_9073dvd740hb4qmd80000gn/T/chi_bench-modal-tn459f5x/cm_afib_moderate_anxious_001",
+            "git_url": null,
+            "git_commit_id": null,
+            "name": null,
+            "ref": null,
+            "overwrite": false,
+            "download_dir": null,
+            "source": "chi_bench-modal-tn459f5x"
+        },
+        "trial_name": "cm_afib_moderate_anxious_001__YBzGaU4",
+        "trials_dir": "logs/submissions/opus-5-claude-code/cm/opus-5-claude-code__cm",
+        "install_only": false,
+        "timeout_multiplier": 2.0,
+        "agent_timeout_multiplier": null,
+        "verifier_timeout_multiplier": null,
+        "agent_setup_timeout_multiplier": null,
+        "environment_build_timeout_multiplier": null,
+        "agent": {
+            "name": "claude-code",
+            "import_path": null,
+            "model_name": "anthropic/claude-opus-5",
+            "n_concurrent": null,
+            "concurrency_group": null,
+            "skills": [],
+            "override_timeout_sec": null,
+            "override_setup_timeout_sec": null,
+            "max_timeout_sec": null,
+            "resume_trajectory": false,
+            "load_trajectory": null,
+            "extra_allowed_hosts": [],
+            "kwargs": {},
+            "env": {
+                "OPENAI_API_KEY": "${OPENAI_API_KEY}",
+                "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}",
+                "GEMINI_API_KEY": "${GEMINI_API_KEY}",
+                "OPENROUTER_API_KEY": "${OPENROUTER_API_KEY}"
+            },
+            "mcp_servers": []
+        },
+        "environment": {
+            "type": null,
+            "import_path": "chi_bench.experiment.modal_env:ChiBenchModalEnvironment",
+            "force_build": false,
+            "delete": true,
+            "cpu_enforcement_policy": "auto",
+            "memory_enforcement_policy": "auto",
+            "override_cpus": null,
+            "override_memory_mb": null,
+            "override_storage_mb": null,
+            "override_gpus": null,
+            "override_tpu": null,
+            "mounts": null,
+            "extra_docker_compose": [],
+            "kwargs": {
+                "sandbox_timeout_secs": 86400
+            },
+            "extra_allowed_hosts": []
+        },
+        "verifier": {
+            "override_timeout_sec": null,
+            "max_timeout_sec": null,
+            "disable": false
+        },
+        "artifacts": [],
+        "extra_instruction_paths": [],
+        "job_id": "39b7350b-d589-470d-9948-131e4bc7f0e9"
+    },
+    "agent_info": {
+        "name": "claude-code",
+        "version": "2.1.219",
+        "model_info": {
+            "name": "claude-opus-5",
+            "provider": "anthropic"
+        }
+    },
+    "agent_result": {
+        "n_input_tokens": 6308914,
+        "n_cache_tokens": 6027900,
+        "n_output_tokens": 96169,
+        "cost_usd": 7.174407499999999,
+        "rollout_details": null,
+        "metadata": null
+    },
+    "verifier_result": {
+        "rewards": {
+            "reward": 1.0
+        }
+    },
+    "exception_info": null,
+    "started_at": "2026-07-24T22:21:07.798966Z",
+    "finished_at": "2026-07-24T22:45:22.869962Z",
+    "environment_setup": {
+        "started_at": "2026-07-24T22:21:07.800248Z",
+        "finished_at": "2026-07-24T22:21:44.236513Z"
+    },
+    "agent_setup": {
+        "started_at": "2026-07-24T22:21:44.236527Z",
+        "finished_at": "2026-07-24T22:21:46.587517Z"
+    },
+    "agent_execution": {
+        "started_at": "2026-07-24T22:21:46.587554Z",
+        "finished_at": "2026-07-24T22:43:00.439995Z"
+    },
+    "verifier": {
+        "started_at": "2026-07-24T22:43:03.828740Z",
+        "finished_at": "2026-07-24T22:45:22.027109Z"
+    },
+    "step_results": null
+}
diff --git a/...4-opus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/verifier/reward.json b/...4-opus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/verifier/reward.json
@@ -0,0 +1 @@
+{"reward": 1.0}
diff --git a/...pus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/verifier/scorecard.json b/...pus-5-claude-code/trials/cm/cm_afib_moderate_anxious_001__YBzGaU4/verifier/scorecard.json
@@ -0,0 +1,113 @@
+{
+  "binary_reward": 1.0,
+  "fractional_reward": 1.0,
+  "passed_checks": 19,
+  "total_checks": 19,
+  "check_scores": {
+    "cm.assessment.record_exists": 1.0,
+    "cm.assessment.completed": 1.0,
+    "cm.assessment.required_sections_present": 1.0,
+    "cm.care_plan.record_exists": 1.0,
+    "cm.care_plan.finalized": 1.0,
+    "cm.care_plan.problem_count": 1.0,
+    "cm.care_plan.goal_structure": 1.0,
+    "cm.care_plan.intervention_structure": 1.0,
+    "cm.care_plan.escalation_conditions_present": 1.0,
+    "cm.care_plan.follow_up_cadence_present": 1.0,
+    "cm.chart_review.record_exists": 1.0,
+    "cm.cross_stage.target_status": 1.0,
+    "cm.cross_stage.audit_actions": 1.0,
+    "cm.cross_stage.no_forbidden_mutations": 1.0,
+    "judge.cm.chart_review.quality": 1.0,
+    "judge.cm.outreach.quality": 1.0,
+    "judge.cm.assessment.quality": 1.0,
+    "judge.cm.care_plan.quality": 1.0,
+    "judge.cm.stage_coherence": 1.0
+  },
+  "checks": {
+    "cm.assessment.record_exists": true,
+    "cm.assessment.completed": true,
+    "cm.assessment.required_sections_present": true,
+    "cm.care_plan.record_exists": true,
+    "cm.care_plan.finalized": true,
+    "cm.care_plan.problem_count": true,
+    "cm.care_plan.goal_structure": true,
+    "cm.care_plan.intervention_structure": true,
+    "cm.care_plan.escalation_conditions_present": true,
+    "cm.care_plan.follow_up_cadence_present": true,
+    "cm.chart_review.record_exists": true,
+    "cm.cross_stage.target_status": true,
+    "cm.cross_stage.audit_actions": true,
+    "cm.cross_stage.no_forbidden_mutations": true,
+    "judge.cm.chart_review.quality": true,
+    "judge.cm.outreach.quality": true,
+    "judge.cm.assessment.quality": true,
+    "judge.cm.care_plan.quality": true,
+    "judge.cm.stage_coherence": true
+  },
+  "failed_checks": [],
+  "not_applicable_checks": [],
+  "stages": {
+    "cm_assessment": {
+      "passed": true,
+      "checks": {
+        "cm.assessment.record_exists": true,
+        "cm.assessment.completed": true,
+        "cm.assessment.required_sections_present": true
+      },
+      "passed_count": 3,
+      "total_count": 3,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_care_plan": {
+      "passed": true,
+      "checks": {
+        "cm.care_plan.record_exists": true,
+        "cm.care_plan.finalized": true,
+        "cm.care_plan.problem_count": true,
+        "cm.care_plan.goal_structure": true,
+        "cm.care_plan.intervention_structure": true,
+        "cm.care_plan.escalation_conditions_present": true,
+        "cm.care_plan.follow_up_cadence_present": true
+      },
+      "passed_count": 7,
+      "total_count": 7,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_chart_review": {
+      "passed": true,
+      "checks": {
+        "cm.chart_review.record_exists": true
+      },
+      "passed_count": 1,
+      "total_count": 1,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001",
+        "record_count": 1
+      }
+    },
+    "cm_cross_stage": {
+      "passed": true,
+      "checks": {
+        "cm.cross_stage.target_status": true,
+        "cm.cross_stage.audit_actions": true,
+        "cm.cross_stage.no_forbidden_mutations": true
+      },
+      "passed_count": 3,
+      "total_count": 3,
+      "not_applicable_count": 0,
+      "details": {
+        "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001"
+      }
+    }
+  }
+}
diff --git a/...-5-claude-code/trials/cm/cm_anorexia_hard_refuses_001__PXjLSEz/agent/trajectory.jsonl.zst b/...-5-claude-code/trials/cm/cm_anorexia_hard_refuses_001__PXjLSEz/agent/trajectory.jsonl.zst