diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/README.md b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/README.md new file mode 100644 index 00000000..b353d138 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/README.md @@ -0,0 +1,22 @@ +# chi-Bench submission + +- Submission ID: `u2med-u2medfellow-zzq` +- Agent: `U2MedFellow` +- Model: `U2Med` +- Dataset: `chi-bench-v1.0.0` +- Submitted at: `2026-09-03T03:01:09Z` + +## Results + +| Domain | Pass@1 | Trials | Tasks | +|---|---:|---:|---:| +| Overall | 0.586667 | 75 | 75 | +| PA Provider | 0.520000 | 25 | 25 | +| PA UM | 0.520000 | 25 | 25 | +| CM | 0.720000 | 25 | 25 | + +## Selection + +The packet integrates the results for all three benchmark domains in the current submission directory. + +The packet uses the public identity `model=U2Med`, `agent=U2MedFellow`. Trial evidence and benchmark results are included in the submission packet. diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/provenance.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/provenance.json new file mode 100644 index 00000000..d5545218 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/provenance.json @@ -0,0 +1,9 @@ +{ + "chi_bench_git_sha": null, + "image_digest": null, + "judge_model": "claude-opus-4-7", + "harness_version": "0.1.0", + "selection": { + "description": "Integrated results for all three benchmark domains in the current submission directory." + } +} diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/results.csv b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/results.csv new file mode 100644 index 00000000..c257b835 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/results.csv @@ -0,0 +1,5 @@ +benchmark,dataset_version,submission_id,team,agent,model,domain,pass_at_1,n_trials,n_tasks,mean_cost_usd,mean_walltime_s,submitted_at +chi-bench,chi-bench-v1.0.0,u2med-u2medfellow-zzq,U2Med,U2MedFellow,U2Med,overall,0.5866666666666667,75,75,0.0,0.0,2026-09-03T03:01:09Z +chi-bench,chi-bench-v1.0.0,u2med-u2medfellow-zzq,U2Med,U2MedFellow,U2Med,pa_provider,0.52,25,25,0.0,0.0,2026-09-03T03:01:09Z +chi-bench,chi-bench-v1.0.0,u2med-u2medfellow-zzq,U2Med,U2MedFellow,U2Med,pa_um,0.52,25,25,0.0,0.0,2026-09-03T03:01:09Z +chi-bench,chi-bench-v1.0.0,u2med-u2medfellow-zzq,U2Med,U2MedFellow,U2Med,cm,0.72,25,25,0.0,0.0,2026-09-03T03:01:09Z diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/sub.yaml b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/sub.yaml new file mode 100644 index 00000000..74cf5d8b --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/sub.yaml @@ -0,0 +1,29 @@ +schema: chi-bench/submission/v1 + +submission: + id: u2med-u2medfellow-zzq + team: U2Med + contact: zhongzhiqiang-intern@example.com + agent: U2MedFellow + model: U2Med + notes: | + Leaderboard packet integrates the results for all three benchmark domains + in the current submission directory. The public leaderboard identity is + model U2Med with agent U2MedFellow. Trial evidence and benchmark results + are included in the submission packet. + +run: + environment: docker + n_attempts: 1 + concurrency: 4 + max_retries: 0 + timeout_multiplier: 2.0 + env_file: .env + +dataset: + version: chi-bench-v1.0.0 + domains: [pa_provider, pa_um, cm] + +paths: + data_root: data + output_root: logs/submissions/2026-09-03-u2med-u2medfellow-zzq diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/submission.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/submission.json new file mode 100644 index 00000000..942e5f76 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/submission.json @@ -0,0 +1,64 @@ +{ + "schema": "chi-bench/submission/v1", + "submission": { + "id": "u2med-u2medfellow-zzq", + "team": "U2Med", + "contact": "zhongzhiqiang-intern@example.com", + "agent": "U2MedFellow", + "model": "U2Med", + "notes": "Leaderboard packet integrates the results for all three benchmark domains in the current submission directory. The leaderboard identity is model U2Med with agent U2MedFellow. Trial evidence and benchmark results are included in the submission packet.", + "submitted_at": "2026-09-03T03:01:09Z" + }, + "dataset": { + "name": "chi-bench", + "version": "chi-bench-v1.0.0", + "domains": [ + "pa_provider", + "pa_um", + "cm" + ] + }, + "results": { + "overall": { + "pass_at_1": 0.5866666666666667, + "n_trials": 75, + "n_tasks": 75, + "mean_cost_usd": 0.0, + "mean_walltime_s": 0.0 + }, + "per_domain": { + "pa_provider": { + "pass_at_1": 0.52, + "n_trials": 25, + "n_tasks": 25, + "mean_cost_usd": 0.0, + "mean_walltime_s": 0.0 + }, + "pa_um": { + "pass_at_1": 0.52, + "n_trials": 25, + "n_tasks": 25, + "mean_cost_usd": 0.0, + "mean_walltime_s": 0.0 + }, + "cm": { + "pass_at_1": 0.72, + "n_trials": 25, + "n_tasks": 25, + "mean_cost_usd": 0.0, + "mean_walltime_s": 0.0 + } + }, + "mean_cost_usd": 0.0, + "mean_walltime_s": 0.0 + }, + "provenance": { + "chi_bench_git_sha": null, + "image_digest": null, + "judge_model": "claude-opus-4-7", + "harness_version": "0.1.0", + "selection": { + "description": "Integrated results for all three benchmark domains in the current submission directory." + } + } +} diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..4a52e963 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/result.json new file mode 100644 index 00000000..5bd1faa6 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/result.json @@ -0,0 +1,103 @@ +{ + "id": "29bb92f3-097e-4834-abd2-489db18d1cf0", + "task_name": "actava-ai/cm_afib_moderate_anxious_001", + "trial_name": "cm_afib_moderate_anxious_001__SEG6EVC", + "trial_uri": "trials/cm/cm_afib_moderate_anxious_001__SEG6EVC", + "task_id": { + "path": "data/care_management/tasks/cm_afib_moderate_anxious_001" + }, + "source": null, + "task_checksum": "a43c14fadec72f11c1de9af6ee038fd27d3c453ce78dd9de56d454e395e33846", + "config": { + "task": { + "path": "data/care_management/tasks/cm_afib_moderate_anxious_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_afib_moderate_anxious_001__SEG6EVC", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:48:38.555475Z", + "finished_at": "2026-07-19T05:50:18.690223Z", + "environment_setup": { + "started_at": "2026-07-19T05:48:38.625730Z", + "finished_at": "2026-07-19T05:48:48.683974Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:48:48.683988Z", + "finished_at": "2026-07-19T05:48:52.754655Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:48:52.754737Z", + "finished_at": "2026-07-19T05:50:14.897896Z" + }, + "verifier": { + "started_at": "2026-07-19T05:50:15.296800Z", + "finished_at": "2026-07-19T05:50:18.353967Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/scorecard.json new file mode 100644 index 00000000..bc046ab4 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_afib_moderate_anxious_001__SEG6EVC/verifier/scorecard.json @@ -0,0 +1,132 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.3333333333333333, + "passed_checks": 2, + "total_checks": 6, + "check_scores": { + "cm.assessment.record_exists": 0.0, + "cm.assessment.completed": null, + "cm.assessment.required_sections_present": null, + "cm.care_plan.record_exists": 0.0, + "cm.care_plan.finalized": null, + "cm.care_plan.problem_count": null, + "cm.care_plan.goal_structure": null, + "cm.care_plan.intervention_structure": null, + "cm.care_plan.escalation_conditions_present": null, + "cm.care_plan.follow_up_cadence_present": null, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 0.0, + "cm.cross_stage.audit_actions": 0.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable", + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable", + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.assessment.record_exists", + "cm.care_plan.record_exists", + "cm.cross_stage.target_status", + "cm.cross_stage.audit_actions" + ], + "not_applicable_checks": [ + "cm.assessment.completed", + "cm.assessment.required_sections_present", + "cm.care_plan.finalized", + "cm.care_plan.problem_count", + "cm.care_plan.goal_structure", + "cm.care_plan.intervention_structure", + "cm.care_plan.escalation_conditions_present", + "cm.care_plan.follow_up_cadence_present", + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": false, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 2, + "details": { + "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001", + "record_count": 0 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 6, + "details": { + "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001", + "record_count": 0 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": false, + "checks": { + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_AFIB_MODERATE_ANXIOUS_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..56689e50 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/result.json new file mode 100644 index 00000000..50de8e09 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/result.json @@ -0,0 +1,103 @@ +{ + "id": "cdda36ee-0c84-426c-bf79-3ddd94c63f96", + "task_name": "actava-ai/cm_anorexia_hard_refuses_001", + "trial_name": "cm_anorexia_hard_refuses_001__jmsB7HL", + "trial_uri": "trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL", + "task_id": { + "path": "data/care_management/tasks/cm_anorexia_hard_refuses_001" + }, + "source": null, + "task_checksum": "c4f1ce9be85399c3f061364583c88d4fecee4699c90290d070e86002672dc157", + "config": { + "task": { + "path": "data/care_management/tasks/cm_anorexia_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_anorexia_hard_refuses_001__jmsB7HL", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:48:38.628356Z", + "finished_at": "2026-07-19T05:50:31.817850Z", + "environment_setup": { + "started_at": "2026-07-19T05:48:38.629192Z", + "finished_at": "2026-07-19T05:48:48.685175Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:48:48.685184Z", + "finished_at": "2026-07-19T05:48:53.014507Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:48:53.014567Z", + "finished_at": "2026-07-19T05:50:27.749116Z" + }, + "verifier": { + "started_at": "2026-07-19T05:50:28.193779Z", + "finished_at": "2026-07-19T05:50:31.439218Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/scorecard.json new file mode 100644 index 00000000..25801039 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_anorexia_hard_refuses_001__jmsB7HL/verifier/scorecard.json @@ -0,0 +1,132 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.3333333333333333, + "passed_checks": 2, + "total_checks": 6, + "check_scores": { + "cm.assessment.record_exists": 0.0, + "cm.assessment.completed": null, + "cm.assessment.required_sections_present": null, + "cm.care_plan.record_exists": 0.0, + "cm.care_plan.finalized": null, + "cm.care_plan.problem_count": null, + "cm.care_plan.goal_structure": null, + "cm.care_plan.intervention_structure": null, + "cm.care_plan.escalation_conditions_present": null, + "cm.care_plan.follow_up_cadence_present": null, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 0.0, + "cm.cross_stage.audit_actions": 0.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable", + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable", + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.assessment.record_exists", + "cm.care_plan.record_exists", + "cm.cross_stage.target_status", + "cm.cross_stage.audit_actions" + ], + "not_applicable_checks": [ + "cm.assessment.completed", + "cm.assessment.required_sections_present", + "cm.care_plan.finalized", + "cm.care_plan.problem_count", + "cm.care_plan.goal_structure", + "cm.care_plan.intervention_structure", + "cm.care_plan.escalation_conditions_present", + "cm.care_plan.follow_up_cadence_present", + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": false, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 2, + "details": { + "case_id": "CM-CASE-CM_ANOREXIA_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 6, + "details": { + "case_id": "CM-CASE-CM_ANOREXIA_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ANOREXIA_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": false, + "checks": { + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ANOREXIA_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..97eb24d0 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/result.json new file mode 100644 index 00000000..35dc6325 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/result.json @@ -0,0 +1,103 @@ +{ + "id": "2734e6b4-9bdd-4fa2-99d8-fc68c230a693", + "task_name": "actava-ai/cm_asthma_low_coop_001", + "trial_name": "cm_asthma_low_coop_001__bUVc2ji", + "trial_uri": "trials/cm/cm_asthma_low_coop_001__bUVc2ji", + "task_id": { + "path": "data/care_management/tasks/cm_asthma_low_coop_001" + }, + "source": null, + "task_checksum": "4c52fa7e9293de4816d6ee50834802cb42c24bdeb52324d49b19c3661f55cdf3", + "config": { + "task": { + "path": "data/care_management/tasks/cm_asthma_low_coop_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_asthma_low_coop_001__bUVc2ji", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:48:38.631388Z", + "finished_at": "2026-07-19T05:51:24.782818Z", + "environment_setup": { + "started_at": "2026-07-19T05:48:38.633391Z", + "finished_at": "2026-07-19T05:48:48.205850Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:48:48.205889Z", + "finished_at": "2026-07-19T05:48:53.081172Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:48:53.081226Z", + "finished_at": "2026-07-19T05:51:20.673840Z" + }, + "verifier": { + "started_at": "2026-07-19T05:51:21.088048Z", + "finished_at": "2026-07-19T05:51:24.368919Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/scorecard.json new file mode 100644 index 00000000..ec70678e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_asthma_low_coop_001__bUVc2ji/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ASTHMA_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ASTHMA_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ASTHMA_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_ASTHMA_LOW_COOP_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..4a5650b6 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/result.json new file mode 100644 index 00000000..ae16c0b1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/result.json @@ -0,0 +1,103 @@ +{ + "id": "df7efb54-eb6c-4d05-a5ec-4ed69451b133", + "task_name": "actava-ai/cm_ckd_moderate_anxious_001", + "trial_name": "cm_ckd_moderate_anxious_001__gnNeQqU", + "trial_uri": "trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU", + "task_id": { + "path": "data/care_management/tasks/cm_ckd_moderate_anxious_001" + }, + "source": null, + "task_checksum": "6dbb9364be8c8d14cb83d2d630784807b4d775e005b2af4b73d9bfdb505e5576", + "config": { + "task": { + "path": "data/care_management/tasks/cm_ckd_moderate_anxious_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_ckd_moderate_anxious_001__gnNeQqU", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:48:38.635531Z", + "finished_at": "2026-07-19T05:51:44.593616Z", + "environment_setup": { + "started_at": "2026-07-19T05:48:38.636303Z", + "finished_at": "2026-07-19T05:48:48.541594Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:48:48.541616Z", + "finished_at": "2026-07-19T05:48:54.762520Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:48:54.762574Z", + "finished_at": "2026-07-19T05:51:40.439038Z" + }, + "verifier": { + "started_at": "2026-07-19T05:51:40.898930Z", + "finished_at": "2026-07-19T05:51:44.209725Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/scorecard.json new file mode 100644 index 00000000..303b2952 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ckd_moderate_anxious_001__gnNeQqU/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_CKD_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_CKD_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_CKD_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_CKD_MODERATE_ANXIOUS_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..582df361 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/result.json new file mode 100644 index 00000000..4a120b5a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/result.json @@ -0,0 +1,103 @@ +{ + "id": "40a5098e-9efa-47d3-908f-530fc4bf7c89", + "task_name": "actava-ai/cm_complex_esrd_dm_hard_refuses_001", + "trial_name": "cm_complex_esrd_dm_hard_refuses__qa4TtYZ", + "trial_uri": "trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ", + "task_id": { + "path": "data/care_management/tasks/cm_complex_esrd_dm_hard_refuses_001" + }, + "source": null, + "task_checksum": "adf00395838d924695721b0fa2c1a3374d6135c64fb4f307265e563a271641e3", + "config": { + "task": { + "path": "data/care_management/tasks/cm_complex_esrd_dm_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_complex_esrd_dm_hard_refuses__qa4TtYZ", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:50:18.693717Z", + "finished_at": "2026-07-19T05:51:40.431657Z", + "environment_setup": { + "started_at": "2026-07-19T05:50:18.694655Z", + "finished_at": "2026-07-19T05:50:28.289456Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:50:28.289470Z", + "finished_at": "2026-07-19T05:50:32.350829Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:50:32.350884Z", + "finished_at": "2026-07-19T05:51:36.304971Z" + }, + "verifier": { + "started_at": "2026-07-19T05:51:36.718730Z", + "finished_at": "2026-07-19T05:51:40.035552Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/scorecard.json new file mode 100644 index 00000000..25ee67a6 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_esrd_dm_hard_refuses__qa4TtYZ/verifier/scorecard.json @@ -0,0 +1,132 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.3333333333333333, + "passed_checks": 2, + "total_checks": 6, + "check_scores": { + "cm.assessment.record_exists": 0.0, + "cm.assessment.completed": null, + "cm.assessment.required_sections_present": null, + "cm.care_plan.record_exists": 0.0, + "cm.care_plan.finalized": null, + "cm.care_plan.problem_count": null, + "cm.care_plan.goal_structure": null, + "cm.care_plan.intervention_structure": null, + "cm.care_plan.escalation_conditions_present": null, + "cm.care_plan.follow_up_cadence_present": null, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 0.0, + "cm.cross_stage.audit_actions": 0.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable", + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable", + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.assessment.record_exists", + "cm.care_plan.record_exists", + "cm.cross_stage.target_status", + "cm.cross_stage.audit_actions" + ], + "not_applicable_checks": [ + "cm.assessment.completed", + "cm.assessment.required_sections_present", + "cm.care_plan.finalized", + "cm.care_plan.problem_count", + "cm.care_plan.goal_structure", + "cm.care_plan.intervention_structure", + "cm.care_plan.escalation_conditions_present", + "cm.care_plan.follow_up_cadence_present", + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": false, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 2, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_ESRD_DM_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 6, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_ESRD_DM_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_ESRD_DM_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": false, + "checks": { + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_ESRD_DM_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b05bafeb Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/result.json new file mode 100644 index 00000000..37b76861 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/result.json @@ -0,0 +1,103 @@ +{ + "id": "68ad8ce5-3282-40ab-8458-2d532a470366", + "task_name": "actava-ai/cm_complex_hf_afib_ckd_hard_refuses_001", + "trial_name": "cm_complex_hf_afib_ckd_hard_refu__qJZbVgp", + "trial_uri": "trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp", + "task_id": { + "path": "data/care_management/tasks/cm_complex_hf_afib_ckd_hard_refuses_001" + }, + "source": null, + "task_checksum": "1be4202fb4f55d77f1a4fc790d81681858ae66946334447b56819a956f9bd915", + "config": { + "task": { + "path": "data/care_management/tasks/cm_complex_hf_afib_ckd_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_complex_hf_afib_ckd_hard_refu__qJZbVgp", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:50:31.820991Z", + "finished_at": "2026-07-19T05:52:24.581209Z", + "environment_setup": { + "started_at": "2026-07-19T05:50:31.821889Z", + "finished_at": "2026-07-19T05:50:41.400628Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:50:41.400642Z", + "finished_at": "2026-07-19T05:50:45.457911Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:50:45.457963Z", + "finished_at": "2026-07-19T05:52:20.334174Z" + }, + "verifier": { + "started_at": "2026-07-19T05:52:20.757955Z", + "finished_at": "2026-07-19T05:52:24.046320Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/scorecard.json new file mode 100644 index 00000000..9f0cc0a9 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_hf_afib_ckd_hard_refu__qJZbVgp/verifier/scorecard.json @@ -0,0 +1,132 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.3333333333333333, + "passed_checks": 2, + "total_checks": 6, + "check_scores": { + "cm.assessment.record_exists": 0.0, + "cm.assessment.completed": null, + "cm.assessment.required_sections_present": null, + "cm.care_plan.record_exists": 0.0, + "cm.care_plan.finalized": null, + "cm.care_plan.problem_count": null, + "cm.care_plan.goal_structure": null, + "cm.care_plan.intervention_structure": null, + "cm.care_plan.escalation_conditions_present": null, + "cm.care_plan.follow_up_cadence_present": null, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 0.0, + "cm.cross_stage.audit_actions": 0.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable", + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable", + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.assessment.record_exists", + "cm.care_plan.record_exists", + "cm.cross_stage.target_status", + "cm.cross_stage.audit_actions" + ], + "not_applicable_checks": [ + "cm.assessment.completed", + "cm.assessment.required_sections_present", + "cm.care_plan.finalized", + "cm.care_plan.problem_count", + "cm.care_plan.goal_structure", + "cm.care_plan.intervention_structure", + "cm.care_plan.escalation_conditions_present", + "cm.care_plan.follow_up_cadence_present", + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": false, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 2, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_HF_AFIB_CKD_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 6, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_HF_AFIB_CKD_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_HF_AFIB_CKD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": false, + "checks": { + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_HF_AFIB_CKD_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b33e1097 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/result.json new file mode 100644 index 00000000..1024a128 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/result.json @@ -0,0 +1,103 @@ +{ + "id": "238322ee-d5d2-4435-8a17-a1ea40e26f04", + "task_name": "actava-ai/cm_complex_parkinson_dep_moderate_tentative_001", + "trial_name": "cm_complex_parkinson_dep_moderat__zd83trJ", + "trial_uri": "trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ", + "task_id": { + "path": "data/care_management/tasks/cm_complex_parkinson_dep_moderate_tentative_001" + }, + "source": null, + "task_checksum": "53ae0f8f760ccee1db1fc378561a0139c0258bad31ea54caeaf2ed3bf15bb2a5", + "config": { + "task": { + "path": "data/care_management/tasks/cm_complex_parkinson_dep_moderate_tentative_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_complex_parkinson_dep_moderat__zd83trJ", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:51:24.785844Z", + "finished_at": "2026-07-19T05:54:33.983447Z", + "environment_setup": { + "started_at": "2026-07-19T05:51:24.789514Z", + "finished_at": "2026-07-19T05:51:34.374978Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:51:34.374992Z", + "finished_at": "2026-07-19T05:51:39.812937Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:51:39.812991Z", + "finished_at": "2026-07-19T05:54:29.933087Z" + }, + "verifier": { + "started_at": "2026-07-19T05:54:30.335627Z", + "finished_at": "2026-07-19T05:54:33.620311Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/scorecard.json new file mode 100644 index 00000000..bcc440e8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_complex_parkinson_dep_moderat__zd83trJ/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_PARKINSON_DEP_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_PARKINSON_DEP_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_PARKINSON_DEP_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COMPLEX_PARKINSON_DEP_MODERATE_TENTATIVE_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..96eb57d5 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/result.json new file mode 100644 index 00000000..18bd3bb8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/result.json @@ -0,0 +1,103 @@ +{ + "id": "8aa43810-5405-42de-afa6-83d7dde1f743", + "task_name": "actava-ai/cm_copd_hard_refuses_002", + "trial_name": "cm_copd_hard_refuses_002__xSF4XTe", + "trial_uri": "trials/cm/cm_copd_hard_refuses_002__xSF4XTe", + "task_id": { + "path": "data/care_management/tasks/cm_copd_hard_refuses_002" + }, + "source": null, + "task_checksum": "d9f878c654565dc66506460479ebe81eb9fc49339883075cc9bb124b9c9048df", + "config": { + "task": { + "path": "data/care_management/tasks/cm_copd_hard_refuses_002", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_copd_hard_refuses_002__xSF4XTe", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:51:40.434929Z", + "finished_at": "2026-07-19T05:55:16.500762Z", + "environment_setup": { + "started_at": "2026-07-19T05:51:40.437629Z", + "finished_at": "2026-07-19T05:51:50.164881Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:51:50.164900Z", + "finished_at": "2026-07-19T05:51:54.422594Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:51:54.422653Z", + "finished_at": "2026-07-19T05:55:12.356994Z" + }, + "verifier": { + "started_at": "2026-07-19T05:55:12.752161Z", + "finished_at": "2026-07-19T05:55:16.039936Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/scorecard.json new file mode 100644 index 00000000..7e93666f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_copd_hard_refuses_002__xSF4XTe/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COPD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COPD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COPD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_COPD_HARD_REFUSES_002" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..cd1b6352 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/result.json new file mode 100644 index 00000000..ca82a8a1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/result.json @@ -0,0 +1,103 @@ +{ + "id": "96d83d69-fe45-4d20-967f-d8b1663e05fe", + "task_name": "actava-ai/cm_dementia_hard_refuses_001", + "trial_name": "cm_dementia_hard_refuses_001__VDfy4wD", + "trial_uri": "trials/cm/cm_dementia_hard_refuses_001__VDfy4wD", + "task_id": { + "path": "data/care_management/tasks/cm_dementia_hard_refuses_001" + }, + "source": null, + "task_checksum": "a7d37890c6abc2977bce5cff49255ed2586f8a271ea4220b51c99e39fb9060c9", + "config": { + "task": { + "path": "data/care_management/tasks/cm_dementia_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_dementia_hard_refuses_001__VDfy4wD", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:51:44.596605Z", + "finished_at": "2026-07-19T05:53:33.704264Z", + "environment_setup": { + "started_at": "2026-07-19T05:51:44.599295Z", + "finished_at": "2026-07-19T05:51:54.512728Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:51:54.512743Z", + "finished_at": "2026-07-19T05:52:02.280239Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:52:02.280297Z", + "finished_at": "2026-07-19T05:53:29.337285Z" + }, + "verifier": { + "started_at": "2026-07-19T05:53:29.738367Z", + "finished_at": "2026-07-19T05:53:33.022390Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/scorecard.json new file mode 100644 index 00000000..97a5a93f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dementia_hard_refuses_001__VDfy4wD/verifier/scorecard.json @@ -0,0 +1,132 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.3333333333333333, + "passed_checks": 2, + "total_checks": 6, + "check_scores": { + "cm.assessment.record_exists": 0.0, + "cm.assessment.completed": null, + "cm.assessment.required_sections_present": null, + "cm.care_plan.record_exists": 0.0, + "cm.care_plan.finalized": null, + "cm.care_plan.problem_count": null, + "cm.care_plan.goal_structure": null, + "cm.care_plan.intervention_structure": null, + "cm.care_plan.escalation_conditions_present": null, + "cm.care_plan.follow_up_cadence_present": null, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 0.0, + "cm.cross_stage.audit_actions": 0.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable", + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable", + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.assessment.record_exists", + "cm.care_plan.record_exists", + "cm.cross_stage.target_status", + "cm.cross_stage.audit_actions" + ], + "not_applicable_checks": [ + "cm.assessment.completed", + "cm.assessment.required_sections_present", + "cm.care_plan.finalized", + "cm.care_plan.problem_count", + "cm.care_plan.goal_structure", + "cm.care_plan.intervention_structure", + "cm.care_plan.escalation_conditions_present", + "cm.care_plan.follow_up_cadence_present", + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": false, + "checks": { + "cm.assessment.record_exists": false, + "cm.assessment.completed": "not_applicable", + "cm.assessment.required_sections_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 2, + "details": { + "case_id": "CM-CASE-CM_DEMENTIA_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": false, + "cm.care_plan.finalized": "not_applicable", + "cm.care_plan.problem_count": "not_applicable", + "cm.care_plan.goal_structure": "not_applicable", + "cm.care_plan.intervention_structure": "not_applicable", + "cm.care_plan.escalation_conditions_present": "not_applicable", + "cm.care_plan.follow_up_cadence_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 6, + "details": { + "case_id": "CM-CASE-CM_DEMENTIA_HARD_REFUSES_001", + "record_count": 0 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DEMENTIA_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": false, + "checks": { + "cm.cross_stage.target_status": false, + "cm.cross_stage.audit_actions": false, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DEMENTIA_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..638e2851 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/result.json new file mode 100644 index 00000000..34672bc7 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/result.json @@ -0,0 +1,103 @@ +{ + "id": "aeb7c0be-e742-42a1-8a1c-0271ced435be", + "task_name": "actava-ai/cm_dm_hard_refuses_002", + "trial_name": "cm_dm_hard_refuses_002__GGFGdJr", + "trial_uri": "trials/cm/cm_dm_hard_refuses_002__GGFGdJr", + "task_id": { + "path": "data/care_management/tasks/cm_dm_hard_refuses_002" + }, + "source": null, + "task_checksum": "30616160323da379ab8e9e45a77d33e654f196c492b61ffa8b2055682fe7d8e8", + "config": { + "task": { + "path": "data/care_management/tasks/cm_dm_hard_refuses_002", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_dm_hard_refuses_002__GGFGdJr", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:52:24.584236Z", + "finished_at": "2026-07-19T05:56:21.557004Z", + "environment_setup": { + "started_at": "2026-07-19T05:52:24.587060Z", + "finished_at": "2026-07-19T05:52:34.215645Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:52:34.215662Z", + "finished_at": "2026-07-19T05:52:39.027309Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:52:39.027408Z", + "finished_at": "2026-07-19T05:56:17.489834Z" + }, + "verifier": { + "started_at": "2026-07-19T05:56:17.893005Z", + "finished_at": "2026-07-19T05:56:21.175436Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/scorecard.json new file mode 100644 index 00000000..1dc49fef --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_hard_refuses_002__GGFGdJr/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_HARD_REFUSES_002" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..ffca122e Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/result.json new file mode 100644 index 00000000..8678e2f6 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/result.json @@ -0,0 +1,103 @@ +{ + "id": "1ca5aa73-b7f4-4b58-86af-7443883fc71a", + "task_name": "actava-ai/cm_dm_low_coop_001", + "trial_name": "cm_dm_low_coop_001__oBP9V3s", + "trial_uri": "trials/cm/cm_dm_low_coop_001__oBP9V3s", + "task_id": { + "path": "data/care_management/tasks/cm_dm_low_coop_001" + }, + "source": null, + "task_checksum": "7b01a471c2886d188179c311e6ccd6f283e43f199d6d7232886ee4636a41a1c7", + "config": { + "task": { + "path": "data/care_management/tasks/cm_dm_low_coop_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_dm_low_coop_001__oBP9V3s", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:53:33.707580Z", + "finished_at": "2026-07-19T05:56:13.044536Z", + "environment_setup": { + "started_at": "2026-07-19T05:53:33.709860Z", + "finished_at": "2026-07-19T05:53:43.285548Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:53:43.285567Z", + "finished_at": "2026-07-19T05:53:47.623937Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:53:47.624008Z", + "finished_at": "2026-07-19T05:56:09.038550Z" + }, + "verifier": { + "started_at": "2026-07-19T05:56:09.432125Z", + "finished_at": "2026-07-19T05:56:12.693119Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/scorecard.json new file mode 100644 index 00000000..638a643c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_low_coop_001__oBP9V3s/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_LOW_COOP_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_LOW_COOP_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..070940a7 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/result.json new file mode 100644 index 00000000..29c0be44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/result.json @@ -0,0 +1,103 @@ +{ + "id": "af438cf8-bf57-4f9d-b153-c9b016976695", + "task_name": "actava-ai/cm_dm_moderate_anxious_001", + "trial_name": "cm_dm_moderate_anxious_001__LvQg6Ls", + "trial_uri": "trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls", + "task_id": { + "path": "data/care_management/tasks/cm_dm_moderate_anxious_001" + }, + "source": null, + "task_checksum": "f82bf0ab2998949faab581d6fd18ae92c5f23a81e1451a585d42f773c418f4d9", + "config": { + "task": { + "path": "data/care_management/tasks/cm_dm_moderate_anxious_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_dm_moderate_anxious_001__LvQg6Ls", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:54:33.986593Z", + "finished_at": "2026-07-19T05:57:38.687702Z", + "environment_setup": { + "started_at": "2026-07-19T05:54:33.989022Z", + "finished_at": "2026-07-19T05:54:43.547650Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:54:43.547666Z", + "finished_at": "2026-07-19T05:54:48.621119Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:54:48.621190Z", + "finished_at": "2026-07-19T05:57:34.539724Z" + }, + "verifier": { + "started_at": "2026-07-19T05:57:34.965718Z", + "finished_at": "2026-07-19T05:57:38.250087Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/scorecard.json new file mode 100644 index 00000000..de6e4735 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_dm_moderate_anxious_001__LvQg6Ls/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_DM_MODERATE_ANXIOUS_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..0faa5a45 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/result.json new file mode 100644 index 00000000..7b46ec28 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/result.json @@ -0,0 +1,103 @@ +{ + "id": "5c5d7ab9-65e4-4991-acbf-37d1f3e3c16c", + "task_name": "actava-ai/cm_hf_hard_refuses_002", + "trial_name": "cm_hf_hard_refuses_002__XgZPgyC", + "trial_uri": "trials/cm/cm_hf_hard_refuses_002__XgZPgyC", + "task_id": { + "path": "data/care_management/tasks/cm_hf_hard_refuses_002" + }, + "source": null, + "task_checksum": "2d4f133551f4886c51f072434e7186a456a8b0aaf786478566b535b06781cc67", + "config": { + "task": { + "path": "data/care_management/tasks/cm_hf_hard_refuses_002", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_hf_hard_refuses_002__XgZPgyC", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:55:16.504088Z", + "finished_at": "2026-07-19T05:58:59.874663Z", + "environment_setup": { + "started_at": "2026-07-19T05:55:16.506608Z", + "finished_at": "2026-07-19T05:55:26.045696Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:55:26.045709Z", + "finished_at": "2026-07-19T05:55:30.353005Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:55:30.353057Z", + "finished_at": "2026-07-19T05:58:55.775707Z" + }, + "verifier": { + "started_at": "2026-07-19T05:58:56.190607Z", + "finished_at": "2026-07-19T05:58:59.487725Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/scorecard.json new file mode 100644 index 00000000..714ca8dd --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_hf_hard_refuses_002__XgZPgyC/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HF_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HF_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HF_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HF_HARD_REFUSES_002" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..e4c77267 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/result.json new file mode 100644 index 00000000..b4fdae96 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/result.json @@ -0,0 +1,103 @@ +{ + "id": "64894b59-f28a-4bed-8e05-fb656e99a1c5", + "task_name": "actava-ai/cm_htn_low_tentative_001", + "trial_name": "cm_htn_low_tentative_001__G3fmNby", + "trial_uri": "trials/cm/cm_htn_low_tentative_001__G3fmNby", + "task_id": { + "path": "data/care_management/tasks/cm_htn_low_tentative_001" + }, + "source": null, + "task_checksum": "f3d42a19c7d04baa6236bd8594a65167e58ba6dae5cae68d841dc51a6aa3554f", + "config": { + "task": { + "path": "data/care_management/tasks/cm_htn_low_tentative_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_htn_low_tentative_001__G3fmNby", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:56:13.047545Z", + "finished_at": "2026-07-19T05:58:28.101152Z", + "environment_setup": { + "started_at": "2026-07-19T05:56:13.051310Z", + "finished_at": "2026-07-19T05:56:22.663657Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:56:22.663673Z", + "finished_at": "2026-07-19T05:56:27.805904Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:56:27.805973Z", + "finished_at": "2026-07-19T05:58:24.047700Z" + }, + "verifier": { + "started_at": "2026-07-19T05:58:24.453593Z", + "finished_at": "2026-07-19T05:58:27.710715Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/scorecard.json new file mode 100644 index 00000000..e5583606 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_htn_low_tentative_001__G3fmNby/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HTN_LOW_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HTN_LOW_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HTN_LOW_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_HTN_LOW_TENTATIVE_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..e8338943 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/result.json new file mode 100644 index 00000000..c48768f3 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/result.json @@ -0,0 +1,103 @@ +{ + "id": "bca78896-f650-41ee-bb9d-2e4a049036e5", + "task_name": "actava-ai/cm_mdd_hard_refuses_002", + "trial_name": "cm_mdd_hard_refuses_002__fiPfRMC", + "trial_uri": "trials/cm/cm_mdd_hard_refuses_002__fiPfRMC", + "task_id": { + "path": "data/care_management/tasks/cm_mdd_hard_refuses_002" + }, + "source": null, + "task_checksum": "2c6ed5700766b9c338962c0f25e62053e85dde19269bdade287234828657cc39", + "config": { + "task": { + "path": "data/care_management/tasks/cm_mdd_hard_refuses_002", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_mdd_hard_refuses_002__fiPfRMC", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:56:21.560063Z", + "finished_at": "2026-07-19T06:00:07.076120Z", + "environment_setup": { + "started_at": "2026-07-19T05:56:21.566603Z", + "finished_at": "2026-07-19T05:56:31.120492Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:56:31.120509Z", + "finished_at": "2026-07-19T05:56:34.929157Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:56:34.929223Z", + "finished_at": "2026-07-19T06:00:03.010750Z" + }, + "verifier": { + "started_at": "2026-07-19T06:00:03.430759Z", + "finished_at": "2026-07-19T06:00:06.713830Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/scorecard.json new file mode 100644 index 00000000..4366c189 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_hard_refuses_002__fiPfRMC/verifier/scorecard.json @@ -0,0 +1,121 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.9285714285714286, + "passed_checks": 13, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 0.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": false, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.care_plan.goal_structure" + ], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": false, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_HARD_REFUSES_002" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..acf37c16 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/result.json new file mode 100644 index 00000000..3be68f3b --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/result.json @@ -0,0 +1,103 @@ +{ + "id": "4d672815-bf61-49a1-8daa-a42f892b5691", + "task_name": "actava-ai/cm_mdd_moderate_reluctant_001", + "trial_name": "cm_mdd_moderate_reluctant_001__q2kfcpE", + "trial_uri": "trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE", + "task_id": { + "path": "data/care_management/tasks/cm_mdd_moderate_reluctant_001" + }, + "source": null, + "task_checksum": "44738811ef39500c9e6059bb559d49db3d32a512dccaaf0dce3a9ac32cb8dda7", + "config": { + "task": { + "path": "data/care_management/tasks/cm_mdd_moderate_reluctant_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_mdd_moderate_reluctant_001__q2kfcpE", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:57:38.691714Z", + "finished_at": "2026-07-19T06:01:12.114187Z", + "environment_setup": { + "started_at": "2026-07-19T05:57:38.694111Z", + "finished_at": "2026-07-19T05:57:48.240354Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:57:48.240368Z", + "finished_at": "2026-07-19T05:57:52.470633Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:57:52.470684Z", + "finished_at": "2026-07-19T06:01:07.807792Z" + }, + "verifier": { + "started_at": "2026-07-19T06:01:08.238958Z", + "finished_at": "2026-07-19T06:01:11.698875Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/scorecard.json new file mode 100644 index 00000000..b55e89a9 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_mdd_moderate_reluctant_001__q2kfcpE/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_MODERATE_RELUCTANT_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_MODERATE_RELUCTANT_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_MODERATE_RELUCTANT_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_MDD_MODERATE_RELUCTANT_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..d51f08fa Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/result.json new file mode 100644 index 00000000..7148aa5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/result.json @@ -0,0 +1,103 @@ +{ + "id": "45bb3a08-7ab1-4d16-9bf4-a3fb0818b2dc", + "task_name": "actava-ai/cm_metabolic_syndrome_hard_refuses_001", + "trial_name": "cm_metabolic_syndrome_hard_refus__bmFsX2V", + "trial_uri": "trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V", + "task_id": { + "path": "data/care_management/tasks/cm_metabolic_syndrome_hard_refuses_001" + }, + "source": null, + "task_checksum": "65274bb698785d6cc3ab40bcd45690f26cfa95fdbf3e2fa0c341b6ba6417fe83", + "config": { + "task": { + "path": "data/care_management/tasks/cm_metabolic_syndrome_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_metabolic_syndrome_hard_refus__bmFsX2V", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:58:28.104110Z", + "finished_at": "2026-07-19T06:02:44.566241Z", + "environment_setup": { + "started_at": "2026-07-19T05:58:28.106821Z", + "finished_at": "2026-07-19T05:58:37.699913Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:58:37.699927Z", + "finished_at": "2026-07-19T05:58:43.639608Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:58:43.639660Z", + "finished_at": "2026-07-19T06:02:40.401865Z" + }, + "verifier": { + "started_at": "2026-07-19T06:02:40.820503Z", + "finished_at": "2026-07-19T06:02:44.124055Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/scorecard.json new file mode 100644 index 00000000..ba7f918a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_metabolic_syndrome_hard_refus__bmFsX2V/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_METABOLIC_SYNDROME_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_METABOLIC_SYNDROME_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_METABOLIC_SYNDROME_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_METABOLIC_SYNDROME_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..9c924d85 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/result.json new file mode 100644 index 00000000..060e1ed5 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/result.json @@ -0,0 +1,103 @@ +{ + "id": "9118f73a-b2da-46ea-b2a5-4fe2a1507bda", + "task_name": "actava-ai/cm_parkinson_hard_refuses_001", + "trial_name": "cm_parkinson_hard_refuses_001__vc5KXHM", + "trial_uri": "trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM", + "task_id": { + "path": "data/care_management/tasks/cm_parkinson_hard_refuses_001" + }, + "source": null, + "task_checksum": "bbc89db29e4c7f7d85993d26a6da72822c659333f8936da4fd086cccf4befe58", + "config": { + "task": { + "path": "data/care_management/tasks/cm_parkinson_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_parkinson_hard_refuses_001__vc5KXHM", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:58:59.877790Z", + "finished_at": "2026-07-19T06:02:42.831479Z", + "environment_setup": { + "started_at": "2026-07-19T05:58:59.880391Z", + "finished_at": "2026-07-19T05:59:09.591226Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:59:09.591245Z", + "finished_at": "2026-07-19T05:59:13.640746Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:59:13.640816Z", + "finished_at": "2026-07-19T06:02:38.754525Z" + }, + "verifier": { + "started_at": "2026-07-19T06:02:39.169795Z", + "finished_at": "2026-07-19T06:02:42.476571Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/scorecard.json new file mode 100644 index 00000000..556bbea1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_parkinson_hard_refuses_001__vc5KXHM/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PARKINSON_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PARKINSON_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PARKINSON_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PARKINSON_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..2689d97c Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/result.json new file mode 100644 index 00000000..c10fe967 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/result.json @@ -0,0 +1,103 @@ +{ + "id": "677d780e-0b85-481f-ae82-6caedc2e73a6", + "task_name": "actava-ai/cm_post_hip_moderate_anxious_001", + "trial_name": "cm_post_hip_moderate_anxious_001__USSjDon", + "trial_uri": "trials/cm/cm_post_hip_moderate_anxious_001__USSjDon", + "task_id": { + "path": "data/care_management/tasks/cm_post_hip_moderate_anxious_001" + }, + "source": null, + "task_checksum": "bc5594f3fcf6b550dadf29f14143362105d07dd813cac4f358d57c07353f1273", + "config": { + "task": { + "path": "data/care_management/tasks/cm_post_hip_moderate_anxious_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_post_hip_moderate_anxious_001__USSjDon", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:00:07.079598Z", + "finished_at": "2026-07-19T06:03:05.665240Z", + "environment_setup": { + "started_at": "2026-07-19T06:00:07.082796Z", + "finished_at": "2026-07-19T06:00:16.666263Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:00:16.666278Z", + "finished_at": "2026-07-19T06:00:21.137599Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:00:21.137658Z", + "finished_at": "2026-07-19T06:03:01.516753Z" + }, + "verifier": { + "started_at": "2026-07-19T06:03:01.922228Z", + "finished_at": "2026-07-19T06:03:05.274439Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/scorecard.json new file mode 100644 index 00000000..aff2b75f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_hip_moderate_anxious_001__USSjDon/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_HIP_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_HIP_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_HIP_MODERATE_ANXIOUS_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_HIP_MODERATE_ANXIOUS_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..54cca5dc Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/result.json new file mode 100644 index 00000000..569e3476 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/result.json @@ -0,0 +1,103 @@ +{ + "id": "ef7f0d3b-5395-4bb2-9509-e0c9828f4a76", + "task_name": "actava-ai/cm_post_mi_hard_refuses_002", + "trial_name": "cm_post_mi_hard_refuses_002__LLZAqP9", + "trial_uri": "trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9", + "task_id": { + "path": "data/care_management/tasks/cm_post_mi_hard_refuses_002" + }, + "source": null, + "task_checksum": "b1cde60f87bea1a2f14713ffc5706ae9155ad73d2da15af4cfa919db307e12af", + "config": { + "task": { + "path": "data/care_management/tasks/cm_post_mi_hard_refuses_002", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_post_mi_hard_refuses_002__LLZAqP9", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:01:12.117192Z", + "finished_at": "2026-07-19T06:05:18.070143Z", + "environment_setup": { + "started_at": "2026-07-19T06:01:12.119846Z", + "finished_at": "2026-07-19T06:01:21.982046Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:01:21.982060Z", + "finished_at": "2026-07-19T06:01:28.335949Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:01:28.336005Z", + "finished_at": "2026-07-19T06:05:14.006036Z" + }, + "verifier": { + "started_at": "2026-07-19T06:05:14.397319Z", + "finished_at": "2026-07-19T06:05:17.654636Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/scorecard.json new file mode 100644 index 00000000..10802df0 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_mi_hard_refuses_002__LLZAqP9/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_MI_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_MI_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_MI_HARD_REFUSES_002", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_MI_HARD_REFUSES_002" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..49ba05ca Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/result.json new file mode 100644 index 00000000..b379ac35 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/result.json @@ -0,0 +1,103 @@ +{ + "id": "ee536b08-e7b5-4332-8e93-fbb6614ff99c", + "task_name": "actava-ai/cm_post_pna_moderate_tentative_001", + "trial_name": "cm_post_pna_moderate_tentative_0__YcRkwS3", + "trial_uri": "trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3", + "task_id": { + "path": "data/care_management/tasks/cm_post_pna_moderate_tentative_001" + }, + "source": null, + "task_checksum": "428b80298a12eeb83f5ffc5e9edfeb41a9ba003d28dd6c6d59530abd419b9f10", + "config": { + "task": { + "path": "data/care_management/tasks/cm_post_pna_moderate_tentative_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_post_pna_moderate_tentative_0__YcRkwS3", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:02:42.834619Z", + "finished_at": "2026-07-19T06:06:19.293474Z", + "environment_setup": { + "started_at": "2026-07-19T06:02:42.837292Z", + "finished_at": "2026-07-19T06:02:52.730585Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:02:52.730598Z", + "finished_at": "2026-07-19T06:02:57.887867Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:02:57.887920Z", + "finished_at": "2026-07-19T06:06:15.190450Z" + }, + "verifier": { + "started_at": "2026-07-19T06:06:15.594922Z", + "finished_at": "2026-07-19T06:06:18.901810Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/scorecard.json new file mode 100644 index 00000000..e96c2fa1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_pna_moderate_tentative_0__YcRkwS3/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_PNA_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_PNA_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_PNA_MODERATE_TENTATIVE_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_PNA_MODERATE_TENTATIVE_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..7c74e425 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/result.json new file mode 100644 index 00000000..e27f5598 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/result.json @@ -0,0 +1,103 @@ +{ + "id": "4cfce052-5ed2-4dba-ab0a-14a6d8c151c4", + "task_name": "actava-ai/cm_post_stroke_hard_refuses_001", + "trial_name": "cm_post_stroke_hard_refuses_001__CQyAu2U", + "trial_uri": "trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U", + "task_id": { + "path": "data/care_management/tasks/cm_post_stroke_hard_refuses_001" + }, + "source": null, + "task_checksum": "3cd5c049c7f81e235286bc3f507a69fb2fa62422aa859168ec6cd2d44dbe8327", + "config": { + "task": { + "path": "data/care_management/tasks/cm_post_stroke_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_post_stroke_hard_refuses_001__CQyAu2U", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:02:44.569252Z", + "finished_at": "2026-07-19T06:07:02.130688Z", + "environment_setup": { + "started_at": "2026-07-19T06:02:44.572719Z", + "finished_at": "2026-07-19T06:02:54.210187Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:02:54.210201Z", + "finished_at": "2026-07-19T06:03:02.867506Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:03:02.867561Z", + "finished_at": "2026-07-19T06:06:58.083663Z" + }, + "verifier": { + "started_at": "2026-07-19T06:06:58.491534Z", + "finished_at": "2026-07-19T06:07:01.760212Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/scorecard.json new file mode 100644 index 00000000..a56d74e4 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_post_stroke_hard_refuses_001__CQyAu2U/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_STROKE_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_STROKE_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_STROKE_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_POST_STROKE_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..30080c9e Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/result.json new file mode 100644 index 00000000..6654b209 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/result.json @@ -0,0 +1,103 @@ +{ + "id": "b1406e2f-9619-4fd1-ac9d-c8085af2b681", + "task_name": "actava-ai/cm_ptsd_hard_refuses_001", + "trial_name": "cm_ptsd_hard_refuses_001__g2neTRE", + "trial_uri": "trials/cm/cm_ptsd_hard_refuses_001__g2neTRE", + "task_id": { + "path": "data/care_management/tasks/cm_ptsd_hard_refuses_001" + }, + "source": null, + "task_checksum": "1b21d3a81681f6b7e29e2c1fd8436b547d8b34eec1cc2c699c474826aaa541ea", + "config": { + "task": { + "path": "data/care_management/tasks/cm_ptsd_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_ptsd_hard_refuses_001__g2neTRE", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:03:05.668416Z", + "finished_at": "2026-07-19T06:06:46.353797Z", + "environment_setup": { + "started_at": "2026-07-19T06:03:05.671323Z", + "finished_at": "2026-07-19T06:03:15.349873Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:03:15.349887Z", + "finished_at": "2026-07-19T06:03:19.664005Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:03:19.664057Z", + "finished_at": "2026-07-19T06:06:42.288651Z" + }, + "verifier": { + "started_at": "2026-07-19T06:06:42.689241Z", + "finished_at": "2026-07-19T06:06:46.001056Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/scorecard.json new file mode 100644 index 00000000..eb1c3d4e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_ptsd_hard_refuses_001__g2neTRE/verifier/scorecard.json @@ -0,0 +1,121 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.9285714285714286, + "passed_checks": 13, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 0.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": false, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [ + "cm.care_plan.goal_structure" + ], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PTSD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": false, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": false, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PTSD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PTSD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_PTSD_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..553913ae Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/result.json new file mode 100644 index 00000000..dcc2d82a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/result.json @@ -0,0 +1,103 @@ +{ + "id": "c1aebed8-404f-4d57-8fd6-1da53ab9965d", + "task_name": "actava-ai/cm_schizo_hard_refuses_001", + "trial_name": "cm_schizo_hard_refuses_001__EtQtEDC", + "trial_uri": "trials/cm/cm_schizo_hard_refuses_001__EtQtEDC", + "task_id": { + "path": "data/care_management/tasks/cm_schizo_hard_refuses_001" + }, + "source": null, + "task_checksum": "b08372f72c6040a88ddac6f86ff7bc2e71e40f35d2860b195e4549978c2659ed", + "config": { + "task": { + "path": "data/care_management/tasks/cm_schizo_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_schizo_hard_refuses_001__EtQtEDC", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:05:18.073116Z", + "finished_at": "2026-07-19T06:09:12.230613Z", + "environment_setup": { + "started_at": "2026-07-19T06:05:18.075967Z", + "finished_at": "2026-07-19T06:05:27.629046Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:05:27.629058Z", + "finished_at": "2026-07-19T06:05:37.601872Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:05:37.601923Z", + "finished_at": "2026-07-19T06:09:08.179439Z" + }, + "verifier": { + "started_at": "2026-07-19T06:09:08.586204Z", + "finished_at": "2026-07-19T06:09:11.829599Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/scorecard.json new file mode 100644 index 00000000..39bd3294 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_schizo_hard_refuses_001__EtQtEDC/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SCHIZO_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SCHIZO_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SCHIZO_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SCHIZO_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..7ddc844d Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/result.json new file mode 100644 index 00000000..7626f54e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/result.json @@ -0,0 +1,103 @@ +{ + "id": "f004151a-de14-4a7d-b5b5-1b3b82b38109", + "task_name": "actava-ai/cm_sud_hard_refuses_001", + "trial_name": "cm_sud_hard_refuses_001__uZVJRRN", + "trial_uri": "trials/cm/cm_sud_hard_refuses_001__uZVJRRN", + "task_id": { + "path": "data/care_management/tasks/cm_sud_hard_refuses_001" + }, + "source": null, + "task_checksum": "e9377e69811187b43aec45e26f56e92e22a02354dff9b5059a1ae56f99280945", + "config": { + "task": { + "path": "data/care_management/tasks/cm_sud_hard_refuses_001", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "cm_sud_hard_refuses_001__uZVJRRN", + "trials_dir": "trials/cm", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "c1dd117d-2a24-4340-8cac-98389c0ed26d" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T06:06:19.296529Z", + "finished_at": "2026-07-19T06:10:24.640614Z", + "environment_setup": { + "started_at": "2026-07-19T06:06:19.300570Z", + "finished_at": "2026-07-19T06:06:28.932176Z" + }, + "agent_setup": { + "started_at": "2026-07-19T06:06:28.932193Z", + "finished_at": "2026-07-19T06:06:33.483306Z" + }, + "agent_execution": { + "started_at": "2026-07-19T06:06:33.483374Z", + "finished_at": "2026-07-19T06:10:20.425531Z" + }, + "verifier": { + "started_at": "2026-07-19T06:10:20.837021Z", + "finished_at": "2026-07-19T06:10:24.237739Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/scorecard.json new file mode 100644 index 00000000..0bfe787c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/cm/cm_sud_hard_refuses_001__uZVJRRN/verifier/scorecard.json @@ -0,0 +1,119 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 14, + "total_checks": 14, + "check_scores": { + "cm.assessment.record_exists": 1.0, + "cm.assessment.completed": 1.0, + "cm.assessment.required_sections_present": 1.0, + "cm.care_plan.record_exists": 1.0, + "cm.care_plan.finalized": 1.0, + "cm.care_plan.problem_count": 1.0, + "cm.care_plan.goal_structure": 1.0, + "cm.care_plan.intervention_structure": 1.0, + "cm.care_plan.escalation_conditions_present": 1.0, + "cm.care_plan.follow_up_cadence_present": 1.0, + "cm.chart_review.record_exists": 1.0, + "cm.cross_stage.target_status": 1.0, + "cm.cross_stage.audit_actions": 1.0, + "cm.cross_stage.no_forbidden_mutations": 1.0, + "judge.cm.chart_review.quality": null, + "judge.cm.outreach.quality": null, + "judge.cm.assessment.quality": null, + "judge.cm.care_plan.quality": null, + "judge.cm.stage_coherence": null + }, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true, + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true, + "cm.chart_review.record_exists": true, + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true, + "judge.cm.chart_review.quality": "not_applicable", + "judge.cm.outreach.quality": "not_applicable", + "judge.cm.assessment.quality": "not_applicable", + "judge.cm.care_plan.quality": "not_applicable", + "judge.cm.stage_coherence": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.cm.chart_review.quality", + "judge.cm.outreach.quality", + "judge.cm.assessment.quality", + "judge.cm.care_plan.quality", + "judge.cm.stage_coherence" + ], + "stages": { + "cm_assessment": { + "passed": true, + "checks": { + "cm.assessment.record_exists": true, + "cm.assessment.completed": true, + "cm.assessment.required_sections_present": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SUD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_care_plan": { + "passed": true, + "checks": { + "cm.care_plan.record_exists": true, + "cm.care_plan.finalized": true, + "cm.care_plan.problem_count": true, + "cm.care_plan.goal_structure": true, + "cm.care_plan.intervention_structure": true, + "cm.care_plan.escalation_conditions_present": true, + "cm.care_plan.follow_up_cadence_present": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SUD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_chart_review": { + "passed": true, + "checks": { + "cm.chart_review.record_exists": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SUD_HARD_REFUSES_001", + "record_count": 1 + } + }, + "cm_cross_stage": { + "passed": true, + "checks": { + "cm.cross_stage.target_status": true, + "cm.cross_stage.audit_actions": true, + "cm.cross_stage.no_forbidden_mutations": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": { + "case_id": "CM-CASE-CM_SUD_HARD_REFUSES_001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..06a348e8 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/result.json new file mode 100644 index 00000000..6092c86c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/result.json @@ -0,0 +1,109 @@ +{ + "id": "97e5474f-ec9c-4781-8e00-21887e0bc7ef", + "task_name": "actava-ai/pa_t008_t008_o002_p01_new_referral_provider", + "trial_name": "pa_t008_t008_o002_p01_new_referr__nyjA4aN", + "trial_uri": "trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t008_t008_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "259ef5f26831d173025e3517d6fb27756a4d3b67f75806cc54a2429fe547f5f5", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t008_t008_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t008_t008_o002_p01_new_referr__nyjA4aN", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:00:51.322475Z", + "finished_at": "2026-08-06T06:05:42.420527Z", + "environment_setup": { + "started_at": "2026-08-06T06:00:51.481414Z", + "finished_at": "2026-08-06T06:01:13.915516Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:01:13.915532Z", + "finished_at": "2026-08-06T06:01:20.130703Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:01:20.130765Z", + "finished_at": "2026-08-06T06:05:38.460963Z" + }, + "verifier": { + "started_at": "2026-08-06T06:05:38.882594Z", + "finished_at": "2026-08-06T06:05:41.936871Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/scorecard.json new file mode 100644 index 00000000..4153e75a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t008_t008_o002_p01_new_referr__nyjA4aN/verifier/scorecard.json @@ -0,0 +1,90 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 11, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..c057bb11 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/result.json new file mode 100644 index 00000000..0737ed2d --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/result.json @@ -0,0 +1,109 @@ +{ + "id": "4532a7de-81c6-4a4f-9068-a42a56430a25", + "task_name": "actava-ai/pa_t011_t011_o003_p01_new_referral_provider", + "trial_name": "pa_t011_t011_o003_p01_new_referr__AFx6gqi", + "trial_uri": "trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t011_t011_o003_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "9362deccb49c2797b8b2682f403a483ffcdee6f009c4aea21996a0e485292bb4", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t011_t011_o003_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t011_t011_o003_p01_new_referr__AFx6gqi", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:00:51.488880Z", + "finished_at": "2026-08-06T06:06:03.079465Z", + "environment_setup": { + "started_at": "2026-08-06T06:00:51.515199Z", + "finished_at": "2026-08-06T06:01:13.671352Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:01:13.671369Z", + "finished_at": "2026-08-06T06:01:18.289466Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:01:18.289548Z", + "finished_at": "2026-08-06T06:05:58.829787Z" + }, + "verifier": { + "started_at": "2026-08-06T06:05:59.250294Z", + "finished_at": "2026-08-06T06:06:02.643070Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/scorecard.json new file mode 100644 index 00000000..b0b4bd6a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t011_t011_o003_p01_new_referr__AFx6gqi/verifier/scorecard.json @@ -0,0 +1,95 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6363636363636364, + "passed_checks": 7, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 0.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.site_of_service", + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 5, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..0309c94b Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/result.json new file mode 100644 index 00000000..0d5b9c5c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/result.json @@ -0,0 +1,109 @@ +{ + "id": "cede23de-006e-47dd-ad4f-39738aed6f03", + "task_name": "actava-ai/pa_t012_t012_o001_p01_new_referral_provider", + "trial_name": "pa_t012_t012_o001_p01_new_referr__KzDHeft", + "trial_uri": "trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t012_t012_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "3411618ef59171df8613ba856ab55851f04339353349b61d6c74799dd83d1876", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t012_t012_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t012_t012_o001_p01_new_referr__KzDHeft", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:00:51.519041Z", + "finished_at": "2026-08-06T06:09:39.044076Z", + "environment_setup": { + "started_at": "2026-08-06T06:00:51.541933Z", + "finished_at": "2026-08-06T06:01:13.665500Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:01:13.665541Z", + "finished_at": "2026-08-06T06:01:22.820524Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:01:22.820580Z", + "finished_at": "2026-08-06T06:08:04.799643Z" + }, + "verifier": { + "started_at": "2026-08-06T06:08:05.230381Z", + "finished_at": "2026-08-06T06:09:38.561250Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/scorecard.json new file mode 100644 index 00000000..be646fd2 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t012_t012_o001_p01_new_referr__KzDHeft/verifier/scorecard.json @@ -0,0 +1,85 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 10, + "total_checks": 10, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.submission.service_request": 1.0, + "judge.provider_pre_submission.submission.rationale_scope": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true, + "judge.provider_pre_submission.submission.service_request": true, + "judge.provider_pre_submission.submission.rationale_scope": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 0, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..29d56ce4 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/result.json new file mode 100644 index 00000000..653e57a1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/result.json @@ -0,0 +1,109 @@ +{ + "id": "73a074f0-adad-46fa-90fb-d95fcf3596d0", + "task_name": "actava-ai/pa_t013_t013_o002_p01_new_referral_provider", + "trial_name": "pa_t013_t013_o002_p01_new_referr__uza4BAM", + "trial_uri": "trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t013_t013_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "a0f2afe1a57579efcf1387767c4cf4cc11eb31027f18c90353286497343f973a", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t013_t013_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t013_t013_o002_p01_new_referr__uza4BAM", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:00:51.546130Z", + "finished_at": "2026-08-06T06:14:03.536726Z", + "environment_setup": { + "started_at": "2026-08-06T06:00:51.573709Z", + "finished_at": "2026-08-06T06:01:13.672616Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:01:13.672627Z", + "finished_at": "2026-08-06T06:01:20.427806Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:01:20.427862Z", + "finished_at": "2026-08-06T06:07:38.865694Z" + }, + "verifier": { + "started_at": "2026-08-06T06:07:39.274465Z", + "finished_at": "2026-08-06T06:14:03.060055Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/scorecard.json new file mode 100644 index 00000000..1c1e1074 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t013_t013_o002_p01_new_referr__uza4BAM/verifier/scorecard.json @@ -0,0 +1,87 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 13, + "total_checks": 13, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.documentation_review": 1.0, + "judge.provider_pre_submission.evidence.medical_necessity": 1.0, + "judge.provider_pre_submission.submission.service_request": 1.0, + "judge.provider_pre_submission.submission.rationale_scope": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.documentation_review": true, + "judge.provider_pre_submission.evidence.medical_necessity": true, + "judge.provider_pre_submission.submission.service_request": true, + "judge.provider_pre_submission.submission.rationale_scope": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..6bb17590 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/result.json new file mode 100644 index 00000000..a62a258c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/result.json @@ -0,0 +1,109 @@ +{ + "id": "067ca290-8006-44af-99a5-3324f4b61bfe", + "task_name": "actava-ai/pa_t014_t014_o001_p01_new_referral_provider", + "trial_name": "pa_t014_t014_o001_p01_new_referr__MFhEUjE", + "trial_uri": "trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t014_t014_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "1774dfdc3763b724df558c5198a5d510fabd5f9b7edb4f5777f987fd7167bee1", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t014_t014_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t014_t014_o001_p01_new_referr__MFhEUjE", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:05:42.425308Z", + "finished_at": "2026-08-06T06:18:25.341610Z", + "environment_setup": { + "started_at": "2026-08-06T06:05:42.456985Z", + "finished_at": "2026-08-06T06:05:56.142411Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:05:56.142445Z", + "finished_at": "2026-08-06T06:05:59.960513Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:05:59.960590Z", + "finished_at": "2026-08-06T06:13:04.019824Z" + }, + "verifier": { + "started_at": "2026-08-06T06:13:04.446049Z", + "finished_at": "2026-08-06T06:18:24.902474Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/scorecard.json new file mode 100644 index 00000000..be646fd2 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t014_t014_o001_p01_new_referr__MFhEUjE/verifier/scorecard.json @@ -0,0 +1,85 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 10, + "total_checks": 10, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.submission.service_request": 1.0, + "judge.provider_pre_submission.submission.rationale_scope": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true, + "judge.provider_pre_submission.submission.service_request": true, + "judge.provider_pre_submission.submission.rationale_scope": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 0, + "total_count": 0, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..0d265060 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/result.json new file mode 100644 index 00000000..1cc73183 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/result.json @@ -0,0 +1,109 @@ +{ + "id": "283b4930-6d10-4626-ab9b-5198fd902b3a", + "task_name": "actava-ai/pa_t015_t015_o002_p01_new_referral_provider", + "trial_name": "pa_t015_t015_o002_p01_new_referr__LoTBFDn", + "trial_uri": "trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t015_t015_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "af18e41a20183c44bc61a0a2097413502bf5a5a4fada148657052614dc251666", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t015_t015_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t015_t015_o002_p01_new_referr__LoTBFDn", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:06:03.083989Z", + "finished_at": "2026-08-06T06:10:31.852738Z", + "environment_setup": { + "started_at": "2026-08-06T06:06:03.114242Z", + "finished_at": "2026-08-06T06:06:16.813907Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:06:16.813930Z", + "finished_at": "2026-08-06T06:06:20.768797Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:06:20.768870Z", + "finished_at": "2026-08-06T06:10:28.025597Z" + }, + "verifier": { + "started_at": "2026-08-06T06:10:28.434000Z", + "finished_at": "2026-08-06T06:10:31.464545Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/scorecard.json new file mode 100644 index 00000000..a982c8ca --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t015_t015_o002_p01_new_referr__LoTBFDn/verifier/scorecard.json @@ -0,0 +1,93 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 12, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.closed_backed_by_provider_determination": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..59f5e88f Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/result.json new file mode 100644 index 00000000..1f741990 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/result.json @@ -0,0 +1,109 @@ +{ + "id": "6add25f1-7ce5-4f85-b31a-e661959b458b", + "task_name": "actava-ai/pa_t016_t016_o001_p01_new_referral_provider", + "trial_name": "pa_t016_t016_o001_p01_new_referr__a29BRpi", + "trial_uri": "trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t016_t016_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "616f0237e28b100e75a523a19291ae364f1219b41acedfde14184821c2a4f373", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t016_t016_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t016_t016_o001_p01_new_referr__a29BRpi", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:09:39.048857Z", + "finished_at": "2026-08-06T06:17:46.483045Z", + "environment_setup": { + "started_at": "2026-08-06T06:09:39.075965Z", + "finished_at": "2026-08-06T06:09:52.733649Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:09:52.733672Z", + "finished_at": "2026-08-06T06:09:58.684199Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:09:58.684255Z", + "finished_at": "2026-08-06T06:17:42.064796Z" + }, + "verifier": { + "started_at": "2026-08-06T06:17:42.484669Z", + "finished_at": "2026-08-06T06:17:46.099185Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t016_t016_o001_p01_new_referr__a29BRpi/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..a5c7015c Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/result.json new file mode 100644 index 00000000..576a16b2 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/result.json @@ -0,0 +1,109 @@ +{ + "id": "949ae149-8dce-4969-a87d-1d3c1d252405", + "task_name": "actava-ai/pa_t017_t017_o001_p01_new_referral_provider", + "trial_name": "pa_t017_t017_o001_p01_new_referr__T6LaiRq", + "trial_uri": "trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t017_t017_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "01808cf9d5292121f7fb065041b1e66a78ce4d693bf579efb19e44d79d91e5ab", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t017_t017_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t017_t017_o001_p01_new_referr__T6LaiRq", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:10:31.857535Z", + "finished_at": "2026-08-06T06:21:46.317077Z", + "environment_setup": { + "started_at": "2026-08-06T06:10:31.883558Z", + "finished_at": "2026-08-06T06:10:45.532177Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:10:45.532203Z", + "finished_at": "2026-08-06T06:10:50.371852Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:10:50.371907Z", + "finished_at": "2026-08-06T06:14:56.193813Z" + }, + "verifier": { + "started_at": "2026-08-06T06:14:56.625006Z", + "finished_at": "2026-08-06T06:21:45.803967Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/scorecard.json new file mode 100644 index 00000000..145ac1e7 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t017_t017_o001_p01_new_referr__T6LaiRq/verifier/scorecard.json @@ -0,0 +1,91 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 15, + "total_checks": 15, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.osa_diagnostic": 1.0, + "judge.provider_pre_submission.evidence.osa_severity_review": 1.0, + "judge.provider_pre_submission.evidence.pap_therapy_review": 1.0, + "judge.provider_pre_submission.evidence.anatomical_evaluation": 1.0, + "judge.provider_pre_submission.submission.service_request": 1.0, + "judge.provider_pre_submission.submission.rationale_scope": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.osa_diagnostic": true, + "judge.provider_pre_submission.evidence.osa_severity_review": true, + "judge.provider_pre_submission.evidence.pap_therapy_review": true, + "judge.provider_pre_submission.evidence.anatomical_evaluation": true, + "judge.provider_pre_submission.submission.service_request": true, + "judge.provider_pre_submission.submission.rationale_scope": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..fd63da16 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/result.json new file mode 100644 index 00000000..ca4da555 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/result.json @@ -0,0 +1,109 @@ +{ + "id": "5a2236cd-227c-49e4-a039-fec221a8b88e", + "task_name": "actava-ai/pa_t018_t018_o001_p01_new_referral_provider", + "trial_name": "pa_t018_t018_o001_p01_new_referr__ttQKjgT", + "trial_uri": "trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t018_t018_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "c14bd8267824140c38af3fd4d24458845ca4059331362fdde0d15a60529de006", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t018_t018_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t018_t018_o001_p01_new_referr__ttQKjgT", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:14:03.572125Z", + "finished_at": "2026-08-06T06:24:43.447634Z", + "environment_setup": { + "started_at": "2026-08-06T06:14:03.729680Z", + "finished_at": "2026-08-06T06:14:17.559953Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:14:17.559974Z", + "finished_at": "2026-08-06T06:14:25.535068Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:14:25.535142Z", + "finished_at": "2026-08-06T06:23:33.985463Z" + }, + "verifier": { + "started_at": "2026-08-06T06:23:34.404102Z", + "finished_at": "2026-08-06T06:24:43.002226Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/scorecard.json new file mode 100644 index 00000000..c3253a27 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t018_t018_o001_p01_new_referr__ttQKjgT/verifier/scorecard.json @@ -0,0 +1,104 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 9, + "total_checks": 9, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.patient_clinical_conditions": null, + "judge.provider_pre_submission.evidence.facility_accessibility": null, + "judge.provider_pre_submission.evidence.adverse_health_impact": null, + "judge.provider_pre_submission.evidence.patient_clinical_conditions_02": null, + "judge.provider_pre_submission.evidence.facility_accessibility_02": null, + "judge.provider_pre_submission.evidence.adverse_health_impact_02": null, + "judge.provider_pre_submission.submission.service_request": null, + "judge.provider_pre_submission.submission.rationale_scope": null + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.patient_clinical_conditions": "not_applicable", + "judge.provider_pre_submission.evidence.facility_accessibility": "not_applicable", + "judge.provider_pre_submission.evidence.adverse_health_impact": "not_applicable", + "judge.provider_pre_submission.evidence.patient_clinical_conditions_02": "not_applicable", + "judge.provider_pre_submission.evidence.facility_accessibility_02": "not_applicable", + "judge.provider_pre_submission.evidence.adverse_health_impact_02": "not_applicable", + "judge.provider_pre_submission.submission.service_request": "not_applicable", + "judge.provider_pre_submission.submission.rationale_scope": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.provider_pre_submission.evidence.patient_clinical_conditions", + "judge.provider_pre_submission.evidence.facility_accessibility", + "judge.provider_pre_submission.evidence.adverse_health_impact", + "judge.provider_pre_submission.evidence.patient_clinical_conditions_02", + "judge.provider_pre_submission.evidence.facility_accessibility_02", + "judge.provider_pre_submission.evidence.adverse_health_impact_02", + "judge.provider_pre_submission.submission.service_request", + "judge.provider_pre_submission.submission.rationale_scope" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..8881dda8 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/result.json new file mode 100644 index 00000000..d93205f6 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/result.json @@ -0,0 +1,109 @@ +{ + "id": "8fe21dfe-f3f2-42c7-9dd7-fd2ef08174fe", + "task_name": "actava-ai/pa_t019_t019_o001_p01_new_referral_provider", + "trial_name": "pa_t019_t019_o001_p01_new_referr__NmvBBMq", + "trial_uri": "trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t019_t019_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "bdae384fd7e63ead0cbb5214830c5f032e2f64e1249507f4724301deed26cb06", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t019_t019_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t019_t019_o001_p01_new_referr__NmvBBMq", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:17:46.489642Z", + "finished_at": "2026-08-06T06:24:52.717229Z", + "environment_setup": { + "started_at": "2026-08-06T06:17:46.516945Z", + "finished_at": "2026-08-06T06:18:00.180340Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:18:00.180358Z", + "finished_at": "2026-08-06T06:18:05.076383Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:18:05.076469Z", + "finished_at": "2026-08-06T06:23:30.957199Z" + }, + "verifier": { + "started_at": "2026-08-06T06:23:31.384091Z", + "finished_at": "2026-08-06T06:24:52.230509Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/scorecard.json new file mode 100644 index 00000000..fe45bbc8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t019_t019_o001_p01_new_referr__NmvBBMq/verifier/scorecard.json @@ -0,0 +1,104 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 9, + "total_checks": 9, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.pap_therapy_trial_review": null, + "judge.provider_pre_submission.evidence.bmi_apnea_type": null, + "judge.provider_pre_submission.evidence.dise_anatomy": null, + "judge.provider_pre_submission.evidence.dise_anatomy_review": null, + "judge.provider_pre_submission.evidence.dise_anatomy_adverse_determination": null, + "judge.provider_pre_submission.evidence.peer_to_peer_dise_anatomy": null, + "judge.provider_pre_submission.submission.service_request": null, + "judge.provider_pre_submission.submission.rationale_scope": null + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.pap_therapy_trial_review": "not_applicable", + "judge.provider_pre_submission.evidence.bmi_apnea_type": "not_applicable", + "judge.provider_pre_submission.evidence.dise_anatomy": "not_applicable", + "judge.provider_pre_submission.evidence.dise_anatomy_review": "not_applicable", + "judge.provider_pre_submission.evidence.dise_anatomy_adverse_determination": "not_applicable", + "judge.provider_pre_submission.evidence.peer_to_peer_dise_anatomy": "not_applicable", + "judge.provider_pre_submission.submission.service_request": "not_applicable", + "judge.provider_pre_submission.submission.rationale_scope": "not_applicable" + }, + "failed_checks": [], + "not_applicable_checks": [ + "judge.provider_pre_submission.evidence.pap_therapy_trial_review", + "judge.provider_pre_submission.evidence.bmi_apnea_type", + "judge.provider_pre_submission.evidence.dise_anatomy", + "judge.provider_pre_submission.evidence.dise_anatomy_review", + "judge.provider_pre_submission.evidence.dise_anatomy_adverse_determination", + "judge.provider_pre_submission.evidence.peer_to_peer_dise_anatomy", + "judge.provider_pre_submission.submission.service_request", + "judge.provider_pre_submission.submission.rationale_scope" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..ce6bfa95 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/result.json new file mode 100644 index 00000000..da673f4e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/result.json @@ -0,0 +1,109 @@ +{ + "id": "90d2ba12-a6e9-477b-a784-e2c5b1301d83", + "task_name": "actava-ai/pa_t021_t021_o003_p01_new_referral_provider", + "trial_name": "pa_t021_t021_o003_p01_new_referr__xzHecg8", + "trial_uri": "trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t021_t021_o003_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "fc45aff373bb985d5c3e46fc042511645c2dd9b04a588027f948245d09107689", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t021_t021_o003_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t021_t021_o003_p01_new_referr__xzHecg8", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:18:25.348319Z", + "finished_at": "2026-08-06T06:25:49.982486Z", + "environment_setup": { + "started_at": "2026-08-06T06:18:25.373748Z", + "finished_at": "2026-08-06T06:18:39.047795Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:18:39.047821Z", + "finished_at": "2026-08-06T06:18:42.822993Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:18:42.823068Z", + "finished_at": "2026-08-06T06:25:45.625370Z" + }, + "verifier": { + "started_at": "2026-08-06T06:25:46.043339Z", + "finished_at": "2026-08-06T06:25:49.599394Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t021_t021_o003_p01_new_referr__xzHecg8/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..c6245611 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/result.json new file mode 100644 index 00000000..363cc012 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/result.json @@ -0,0 +1,109 @@ +{ + "id": "24047d32-29da-43a5-a819-2c62b57b1d38", + "task_name": "actava-ai/pa_t022_t022_o001_p01_new_referral_provider", + "trial_name": "pa_t022_t022_o001_p01_new_referr__GqZdX4c", + "trial_uri": "trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t022_t022_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "63a168eca6b51490c4149612779a9932b94c75dae06a65311155b912814a1d98", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t022_t022_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t022_t022_o001_p01_new_referr__GqZdX4c", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:21:46.325482Z", + "finished_at": "2026-08-06T06:32:37.217232Z", + "environment_setup": { + "started_at": "2026-08-06T06:21:46.350810Z", + "finished_at": "2026-08-06T06:21:59.989249Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:21:59.989268Z", + "finished_at": "2026-08-06T06:22:03.259147Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:22:03.259205Z", + "finished_at": "2026-08-06T06:29:46.053162Z" + }, + "verifier": { + "started_at": "2026-08-06T06:29:46.466178Z", + "finished_at": "2026-08-06T06:32:36.775420Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/scorecard.json new file mode 100644 index 00000000..516e695c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t022_t022_o001_p01_new_referr__GqZdX4c/verifier/scorecard.json @@ -0,0 +1,85 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 12, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.site_of_care_alternative_availability_review": 1.0, + "judge.provider_pre_submission.submission.service_request": 1.0, + "judge.provider_pre_submission.submission.rationale_scope": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.site_of_care_alternative_availability_review": true, + "judge.provider_pre_submission.submission.service_request": true, + "judge.provider_pre_submission.submission.rationale_scope": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..42d5a2ff Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/result.json new file mode 100644 index 00000000..fb1bb6be --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/result.json @@ -0,0 +1,109 @@ +{ + "id": "f3e6f2b1-4eb2-4903-8164-eb0fc3d81270", + "task_name": "actava-ai/pa_t023_t023_o003_p01_new_referral_provider", + "trial_name": "pa_t023_t023_o003_p01_new_referr__zX9Z5ZW", + "trial_uri": "trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t023_t023_o003_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "be7399962a2566de14c86032472a6c3a16ac1269a04a10c6cf1d3bc857cd4159", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t023_t023_o003_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t023_t023_o003_p01_new_referr__zX9Z5ZW", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:24:43.455977Z", + "finished_at": "2026-08-06T06:30:14.463074Z", + "environment_setup": { + "started_at": "2026-08-06T06:24:43.481461Z", + "finished_at": "2026-08-06T06:24:57.264096Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:24:57.264120Z", + "finished_at": "2026-08-06T06:25:08.191561Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:25:08.191642Z", + "finished_at": "2026-08-06T06:29:23.033522Z" + }, + "verifier": { + "started_at": "2026-08-06T06:29:23.466608Z", + "finished_at": "2026-08-06T06:30:14.023410Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/scorecard.json new file mode 100644 index 00000000..6949ec80 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t023_t023_o003_p01_new_referr__zX9Z5ZW/verifier/scorecard.json @@ -0,0 +1,97 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.8888888888888888, + "passed_checks": 8, + "total_checks": 9, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 0.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.required_documents_present": 1.0, + "cross.forward_transitions": 1.0, + "judge.provider_pre_submission.evidence.indication_workup": null, + "judge.provider_pre_submission.evidence.imaging_protocol": null, + "judge.provider_pre_submission.evidence.site_of_service": null, + "judge.provider_pre_submission.submission.service_request": null, + "judge.provider_pre_submission.submission.rationale_scope": null + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.required_documents_present": true, + "cross.forward_transitions": true, + "judge.provider_pre_submission.evidence.indication_workup": "not_applicable", + "judge.provider_pre_submission.evidence.imaging_protocol": "not_applicable", + "judge.provider_pre_submission.evidence.site_of_service": "not_applicable", + "judge.provider_pre_submission.submission.service_request": "not_applicable", + "judge.provider_pre_submission.submission.rationale_scope": "not_applicable" + }, + "failed_checks": [ + "new_referral.site_of_service" + ], + "not_applicable_checks": [ + "judge.provider_pre_submission.evidence.indication_workup", + "judge.provider_pre_submission.evidence.imaging_protocol", + "judge.provider_pre_submission.evidence.site_of_service", + "judge.provider_pre_submission.submission.service_request", + "judge.provider_pre_submission.submission.rationale_scope" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.required_documents_present": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..496a6bb4 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/result.json new file mode 100644 index 00000000..3d546762 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/result.json @@ -0,0 +1,109 @@ +{ + "id": "cf39e3c5-458a-4f55-b4f5-2fe46ca3d49f", + "task_name": "actava-ai/pa_t024_t024_o001_p01_new_referral_provider", + "trial_name": "pa_t024_t024_o001_p01_new_referr__EHEFjnA", + "trial_uri": "trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t024_t024_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "300e67d38f440527bed50d749c4708a84bb3bb76e29385d6acc953d5959c7319", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t024_t024_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t024_t024_o001_p01_new_referr__EHEFjnA", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:24:52.722487Z", + "finished_at": "2026-08-06T06:33:46.022154Z", + "environment_setup": { + "started_at": "2026-08-06T06:24:52.750588Z", + "finished_at": "2026-08-06T06:25:06.838180Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:25:06.838205Z", + "finished_at": "2026-08-06T06:25:11.486392Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:25:11.486471Z", + "finished_at": "2026-08-06T06:33:41.589936Z" + }, + "verifier": { + "started_at": "2026-08-06T06:33:42.004968Z", + "finished_at": "2026-08-06T06:33:45.582670Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t024_t024_o001_p01_new_referr__EHEFjnA/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..838c95ba Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/result.json new file mode 100644 index 00000000..43b3eb76 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/result.json @@ -0,0 +1,109 @@ +{ + "id": "b72495b8-f4c0-4afb-b66f-3e869004619d", + "task_name": "actava-ai/pa_t026_t026_o003_p01_new_referral_provider", + "trial_name": "pa_t026_t026_o003_p01_new_referr__cUPSsey", + "trial_uri": "trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t026_t026_o003_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "e91a7aaa027ced3475247f08d61641c76a80f265049fb85c582a7faf6449fc34", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t026_t026_o003_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t026_t026_o003_p01_new_referr__cUPSsey", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:25:49.988716Z", + "finished_at": "2026-08-06T06:29:56.515618Z", + "environment_setup": { + "started_at": "2026-08-06T06:25:50.015758Z", + "finished_at": "2026-08-06T06:26:03.823361Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:26:03.823381Z", + "finished_at": "2026-08-06T06:26:13.572492Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:26:13.572552Z", + "finished_at": "2026-08-06T06:29:52.683818Z" + }, + "verifier": { + "started_at": "2026-08-06T06:29:53.101847Z", + "finished_at": "2026-08-06T06:29:56.125387Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/scorecard.json new file mode 100644 index 00000000..a982c8ca --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t026_t026_o003_p01_new_referr__cUPSsey/verifier/scorecard.json @@ -0,0 +1,93 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 12, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.closed_backed_by_provider_determination": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..d3e1e226 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/result.json new file mode 100644 index 00000000..63ba4149 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/result.json @@ -0,0 +1,109 @@ +{ + "id": "3ced76be-aad4-4473-a0be-e04dae7eb79e", + "task_name": "actava-ai/pa_t027_t027_o002_p01_new_referral_provider", + "trial_name": "pa_t027_t027_o002_p01_new_referr__pxmgKF2", + "trial_uri": "trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t027_t027_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "c458c4ebebd7111846d3d1f95820efd04fb8b6723baa8c9736db62194f9f95d5", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t027_t027_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t027_t027_o002_p01_new_referr__pxmgKF2", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:29:56.524480Z", + "finished_at": "2026-08-06T06:35:21.793612Z", + "environment_setup": { + "started_at": "2026-08-06T06:29:56.557764Z", + "finished_at": "2026-08-06T06:30:10.244073Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:30:10.244095Z", + "finished_at": "2026-08-06T06:30:13.516666Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:30:13.516726Z", + "finished_at": "2026-08-06T06:35:17.645616Z" + }, + "verifier": { + "started_at": "2026-08-06T06:35:18.058205Z", + "finished_at": "2026-08-06T06:35:21.369506Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/scorecard.json new file mode 100644 index 00000000..80894921 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t027_t027_o002_p01_new_referr__pxmgKF2/verifier/scorecard.json @@ -0,0 +1,93 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.8181818181818182, + "passed_checks": 9, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 2, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..3e54bd34 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/result.json new file mode 100644 index 00000000..e34736cc --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/result.json @@ -0,0 +1,109 @@ +{ + "id": "d40f9062-89cc-45ed-a5d2-2096f0489a9b", + "task_name": "actava-ai/pa_t028_t028_o001_p01_new_referral_provider", + "trial_name": "pa_t028_t028_o001_p01_new_referr__uzeFJBu", + "trial_uri": "trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t028_t028_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "50395d03b51a478d0a622e0ca6f12431240c18b7508b82f462b682c316f4728d", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t028_t028_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t028_t028_o001_p01_new_referr__uzeFJBu", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:30:14.473755Z", + "finished_at": "2026-08-06T06:35:36.777869Z", + "environment_setup": { + "started_at": "2026-08-06T06:30:14.502794Z", + "finished_at": "2026-08-06T06:30:28.149016Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:30:28.149033Z", + "finished_at": "2026-08-06T06:30:31.377801Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:30:31.377857Z", + "finished_at": "2026-08-06T06:35:32.102801Z" + }, + "verifier": { + "started_at": "2026-08-06T06:35:32.523250Z", + "finished_at": "2026-08-06T06:35:36.370404Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/scorecard.json new file mode 100644 index 00000000..6d164232 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t028_t028_o001_p01_new_referr__uzeFJBu/verifier/scorecard.json @@ -0,0 +1,95 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6363636363636364, + "passed_checks": 7, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 0.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": false, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.required_diagnosis_codes", + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": false, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 5, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b401b46d Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/result.json new file mode 100644 index 00000000..0195f3a8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/result.json @@ -0,0 +1,109 @@ +{ + "id": "984333e6-3ba1-41ac-9844-c1d72be9c4df", + "task_name": "actava-ai/pa_t029_t029_o001_p01_new_referral_provider", + "trial_name": "pa_t029_t029_o001_p01_new_referr__j4wfTCV", + "trial_uri": "trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t029_t029_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "47f3cab0a20728fe8279020f4dd0beecbdd4bf2b329a0cf37314eafbd4f78c55", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t029_t029_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t029_t029_o001_p01_new_referr__j4wfTCV", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:32:37.226247Z", + "finished_at": "2026-08-06T06:37:53.760969Z", + "environment_setup": { + "started_at": "2026-08-06T06:32:37.256455Z", + "finished_at": "2026-08-06T06:32:51.049634Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:32:51.049659Z", + "finished_at": "2026-08-06T06:32:54.839000Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:32:54.839074Z", + "finished_at": "2026-08-06T06:37:49.836072Z" + }, + "verifier": { + "started_at": "2026-08-06T06:37:50.245837Z", + "finished_at": "2026-08-06T06:37:53.346329Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/scorecard.json new file mode 100644 index 00000000..56c7bf48 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t029_t029_o001_p01_new_referr__j4wfTCV/verifier/scorecard.json @@ -0,0 +1,92 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.9090909090909091, + "passed_checks": 10, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 0.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.site_of_service" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": false, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..034bf05d Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/result.json new file mode 100644 index 00000000..f4bdaaaa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/result.json @@ -0,0 +1,109 @@ +{ + "id": "aa472807-402d-4178-8643-59766d878baa", + "task_name": "actava-ai/pa_t030_t030_o002_p01_new_referral_provider", + "trial_name": "pa_t030_t030_o002_p01_new_referr__exChrg9", + "trial_uri": "trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t030_t030_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "a3e3ad32b05feaa55eb1f621a091d658dc0524a8550658509ce6de74a5193a01", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t030_t030_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t030_t030_o002_p01_new_referr__exChrg9", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:33:46.031429Z", + "finished_at": "2026-08-06T06:37:31.302045Z", + "environment_setup": { + "started_at": "2026-08-06T06:33:46.056987Z", + "finished_at": "2026-08-06T06:33:59.854305Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:33:59.854324Z", + "finished_at": "2026-08-06T06:34:08.215774Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:34:08.215832Z", + "finished_at": "2026-08-06T06:37:27.156681Z" + }, + "verifier": { + "started_at": "2026-08-06T06:37:27.583173Z", + "finished_at": "2026-08-06T06:37:30.875596Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t030_t030_o002_p01_new_referr__exChrg9/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..e36ff05d Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/result.json new file mode 100644 index 00000000..d482e38a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/result.json @@ -0,0 +1,109 @@ +{ + "id": "7e7c1441-e0a1-4af3-ae3a-eeb8d4d53048", + "task_name": "actava-ai/pa_t031_t031_o001_p01_new_referral_provider", + "trial_name": "pa_t031_t031_o001_p01_new_referr__hsBbTrY", + "trial_uri": "trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t031_t031_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "f5dc6651c2a3173cd83dc42d68b32de694580b4073502bdb01a98bab329ac55e", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t031_t031_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t031_t031_o001_p01_new_referr__hsBbTrY", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:35:21.802603Z", + "finished_at": "2026-08-06T06:39:15.523774Z", + "environment_setup": { + "started_at": "2026-08-06T06:35:21.829988Z", + "finished_at": "2026-08-06T06:35:35.507900Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:35:35.507923Z", + "finished_at": "2026-08-06T06:35:39.311431Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:35:39.311503Z", + "finished_at": "2026-08-06T06:39:11.031125Z" + }, + "verifier": { + "started_at": "2026-08-06T06:39:11.428979Z", + "finished_at": "2026-08-06T06:39:14.958071Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t031_t031_o001_p01_new_referr__hsBbTrY/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..813ff412 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/result.json new file mode 100644 index 00000000..5c24c656 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/result.json @@ -0,0 +1,109 @@ +{ + "id": "6341fb1c-23fb-4175-b0c1-bee9931df3de", + "task_name": "actava-ai/pa_t032_t032_o002_p01_new_referral_provider", + "trial_name": "pa_t032_t032_o002_p01_new_referr__swYWmqQ", + "trial_uri": "trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t032_t032_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "bd9ddabd7c83456cc7d2122d5ed8c49d5ebad7950ccef1d7652310b544e8e53a", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t032_t032_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t032_t032_o002_p01_new_referr__swYWmqQ", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:35:36.785061Z", + "finished_at": "2026-08-06T06:39:41.473467Z", + "environment_setup": { + "started_at": "2026-08-06T06:35:36.807759Z", + "finished_at": "2026-08-06T06:35:50.444077Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:35:50.444100Z", + "finished_at": "2026-08-06T06:35:53.747037Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:35:53.747111Z", + "finished_at": "2026-08-06T06:39:37.704689Z" + }, + "verifier": { + "started_at": "2026-08-06T06:39:38.110553Z", + "finished_at": "2026-08-06T06:39:41.113964Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/scorecard.json new file mode 100644 index 00000000..a982c8ca --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t032_t032_o002_p01_new_referr__swYWmqQ/verifier/scorecard.json @@ -0,0 +1,93 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 12, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.closed_backed_by_provider_determination": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..9bb4221a Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/result.json new file mode 100644 index 00000000..3d0c8b02 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/result.json @@ -0,0 +1,109 @@ +{ + "id": "fae9d3a8-8a08-4d48-9d76-75b67398d43c", + "task_name": "actava-ai/pa_t033_t033_o001_p01_new_referral_provider", + "trial_name": "pa_t033_t033_o001_p01_new_referr__Na2qz6e", + "trial_uri": "trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t033_t033_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "af9b8ad6467696e15166b97eb6355d33f64dc8842dac97ab2ed66c392d1f35fb", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t033_t033_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t033_t033_o001_p01_new_referr__Na2qz6e", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:37:31.308820Z", + "finished_at": "2026-08-06T06:43:21.484228Z", + "environment_setup": { + "started_at": "2026-08-06T06:37:31.334368Z", + "finished_at": "2026-08-06T06:37:45.068656Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:37:45.068674Z", + "finished_at": "2026-08-06T06:37:56.950624Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:37:56.950696Z", + "finished_at": "2026-08-06T06:43:17.221763Z" + }, + "verifier": { + "started_at": "2026-08-06T06:43:17.641960Z", + "finished_at": "2026-08-06T06:43:21.099782Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/scorecard.json new file mode 100644 index 00000000..050a5d5f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t033_t033_o001_p01_new_referr__Na2qz6e/verifier/scorecard.json @@ -0,0 +1,94 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7272727272727273, + "passed_checks": 8, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..2a11b6a1 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/result.json new file mode 100644 index 00000000..9b049e4d --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/result.json @@ -0,0 +1,109 @@ +{ + "id": "86cdcbe1-23d8-492a-9fd8-c14dfc2941d4", + "task_name": "actava-ai/pa_t034_t034_o002_p01_new_referral_provider", + "trial_name": "pa_t034_t034_o002_p01_new_referr__wWhFgwt", + "trial_uri": "trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t034_t034_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "234e309e5ce38862d7da74eb17324306450ad8a8c6dd46456ee27f73fcaa8c43", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t034_t034_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t034_t034_o002_p01_new_referr__wWhFgwt", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:37:53.767832Z", + "finished_at": "2026-08-06T06:43:39.146199Z", + "environment_setup": { + "started_at": "2026-08-06T06:37:53.795564Z", + "finished_at": "2026-08-06T06:38:07.473639Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:38:07.473661Z", + "finished_at": "2026-08-06T06:38:12.039938Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:38:12.040000Z", + "finished_at": "2026-08-06T06:43:35.300275Z" + }, + "verifier": { + "started_at": "2026-08-06T06:43:35.709269Z", + "finished_at": "2026-08-06T06:43:38.739870Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/scorecard.json new file mode 100644 index 00000000..4153e75a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t034_t034_o002_p01_new_referr__wWhFgwt/verifier/scorecard.json @@ -0,0 +1,90 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 11, + "total_checks": 11, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 1, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b011b100 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/result.json new file mode 100644 index 00000000..8e1dd2f2 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/result.json @@ -0,0 +1,109 @@ +{ + "id": "6cadef67-5283-4c30-a14c-388159a38eac", + "task_name": "actava-ai/pa_t035_t035_o001_p01_new_referral_provider", + "trial_name": "pa_t035_t035_o001_p01_new_referr__YgA8p7G", + "trial_uri": "trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t035_t035_o001_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "754ae6f3967d0efc0c8d8ebd51d4f88f4901afa7f3a8c2e3b54b16c2ee0073ad", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t035_t035_o001_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t035_t035_o001_p01_new_referr__YgA8p7G", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:39:15.531288Z", + "finished_at": "2026-08-06T06:43:18.685212Z", + "environment_setup": { + "started_at": "2026-08-06T06:39:15.559630Z", + "finished_at": "2026-08-06T06:39:29.209246Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:39:29.209268Z", + "finished_at": "2026-08-06T06:39:33.520381Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:39:33.520458Z", + "finished_at": "2026-08-06T06:43:14.525735Z" + }, + "verifier": { + "started_at": "2026-08-06T06:43:14.954746Z", + "finished_at": "2026-08-06T06:43:18.246156Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/scorecard.json new file mode 100644 index 00000000..d5e1cbd9 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t035_t035_o001_p01_new_referr__YgA8p7G/verifier/scorecard.json @@ -0,0 +1,98 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6666666666666666, + "passed_checks": 8, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 0.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 0.0, + "provider_pre_submission.packet_absent": 0.0, + "provider_pre_submission.required_documents_present": null, + "cross.closed_backed_by_provider_determination": 0.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.closed_backed_by_provider_determination": false, + "cross.forward_transitions": true + }, + "failed_checks": [ + "new_referral.case_status", + "provider_pre_submission.final_action", + "provider_pre_submission.packet_absent", + "cross.closed_backed_by_provider_determination" + ], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": false, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": false + }, + "passed_count": 6, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "unique_linked_case" + } + }, + "provider_pre_submission": { + "passed": false, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": false, + "provider_pre_submission.packet_absent": false, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 1, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": false, + "checks": { + "cross.closed_backed_by_provider_determination": false, + "cross.forward_transitions": true + }, + "passed_count": 1, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..1ba260b3 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/result.json new file mode 100644 index 00000000..51ee5c38 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/result.json @@ -0,0 +1,109 @@ +{ + "id": "14005213-2a21-45ce-b84b-e7e38636cc5f", + "task_name": "actava-ai/pa_t036_t036_o002_p01_new_referral_provider", + "trial_name": "pa_t036_t036_o002_p01_new_referr__sSyaEwS", + "trial_uri": "trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS", + "task_id": { + "path": "data/prior_auth_provider/tasks/pa_t036_t036_o002_p01_new_referral_provider" + }, + "source": null, + "task_checksum": "9bb97143710c4a0a8459580cd1926466465a4849c06e8e5aba58adc611de4470", + "config": { + "task": { + "path": "data/prior_auth_provider/tasks/pa_t036_t036_o002_p01_new_referral_provider", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t036_t036_o002_p01_new_referr__sSyaEwS", + "trials_dir": "trials/pa_provider", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "a6913700-9403-4217-a287-87e94e6c0e7c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-06T06:39:41.480367Z", + "finished_at": "2026-08-06T06:43:59.020891Z", + "environment_setup": { + "started_at": "2026-08-06T06:39:41.508896Z", + "finished_at": "2026-08-06T06:39:55.163994Z" + }, + "agent_setup": { + "started_at": "2026-08-06T06:39:55.164013Z", + "finished_at": "2026-08-06T06:40:02.726606Z" + }, + "agent_execution": { + "started_at": "2026-08-06T06:40:02.726665Z", + "finished_at": "2026-08-06T06:43:54.979312Z" + }, + "verifier": { + "started_at": "2026-08-06T06:43:55.392968Z", + "finished_at": "2026-08-06T06:43:58.596676Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/scorecard.json new file mode 100644 index 00000000..a982c8ca --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_provider/pa_t036_t036_o002_p01_new_referr__sSyaEwS/verifier/scorecard.json @@ -0,0 +1,93 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 12, + "total_checks": 12, + "check_scores": { + "new_referral.service_request_created": 1.0, + "new_referral.procedure_code": 1.0, + "new_referral.required_diagnosis_codes": 1.0, + "new_referral.service_type": 1.0, + "new_referral.site_of_service": 1.0, + "new_referral.case_created": 1.0, + "new_referral.case_status": 1.0, + "provider_pre_submission.provider_determination_present": 1.0, + "provider_pre_submission.final_action": 1.0, + "provider_pre_submission.packet_absent": 1.0, + "provider_pre_submission.required_documents_present": null, + "cross.closed_backed_by_provider_determination": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true, + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable", + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "provider_pre_submission.required_documents_present" + ], + "stages": { + "new_referral": { + "passed": true, + "checks": { + "new_referral.service_request_created": true, + "new_referral.procedure_code": true, + "new_referral.required_diagnosis_codes": true, + "new_referral.service_type": true, + "new_referral.site_of_service": true, + "new_referral.case_created": true, + "new_referral.case_status": true + }, + "passed_count": 7, + "total_count": 7, + "not_applicable_count": 0, + "details": { + "service_request_id": "SR-001", + "service_request_candidate_count": 1, + "service_request_resolution": "unique_match", + "case_id": "CASE-ACT-001", + "case_candidate_count": 1, + "case_resolution": "expected_status" + } + }, + "provider_pre_submission": { + "passed": true, + "checks": { + "provider_pre_submission.provider_determination_present": true, + "provider_pre_submission.final_action": true, + "provider_pre_submission.packet_absent": true, + "provider_pre_submission.required_documents_present": "not_applicable" + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "case_id": "CASE-ACT-001" + } + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.closed_backed_by_provider_determination": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-ACT-001" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..ec26ceef Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/result.json new file mode 100644 index 00000000..ebf0cd98 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/result.json @@ -0,0 +1,103 @@ +{ + "id": "f664923b-d25a-4a07-ba0c-ca942818596e", + "task_name": "actava-ai/pa_t008_t008_o002_p01_mdreview_payer", + "trial_name": "pa_t008_t008_o002_p01_mdreview_p__SHRax2d", + "trial_uri": "trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t008_t008_o002_p01_mdreview_payer" + }, + "source": null, + "task_checksum": "5832a9a7faee6a7b4d61dce901d87995fa108e04815e13e0b1c0a552bb65da75", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t008_t008_o002_p01_mdreview_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t008_t008_o002_p01_mdreview_p__SHRax2d", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:48:47.693046Z", + "finished_at": "2026-07-19T04:54:30.510488Z", + "environment_setup": { + "started_at": "2026-07-19T04:48:47.861632Z", + "finished_at": "2026-07-19T04:49:08.953193Z" + }, + "agent_setup": { + "started_at": "2026-07-19T04:49:08.953203Z", + "finished_at": "2026-07-19T04:49:14.094109Z" + }, + "agent_execution": { + "started_at": "2026-07-19T04:49:14.094167Z", + "finished_at": "2026-07-19T04:50:42.253700Z" + }, + "verifier": { + "started_at": "2026-07-19T04:50:42.656619Z", + "finished_at": "2026-07-19T04:54:29.807181Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/scorecard.json new file mode 100644 index 00000000..6990d597 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t008_t008_o002_p01_mdreview_p__SHRax2d/verifier/scorecard.json @@ -0,0 +1,120 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 13, + "total_checks": 13, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.md_review:preop_weight_history": 1.0, + "judge.md_review:preop_psychosocial_eval": 1.0, + "judge.md_review:preop_multidisciplinary_regimen": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.md_review:preop_weight_history": true, + "judge.md_review:preop_psychosocial_eval": true, + "judge.md_review:preop_multidisciplinary_regimen": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination" + ], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable" + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 9, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-3903B8ED" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..5777e28e Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/result.json new file mode 100644 index 00000000..0da4a490 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/result.json @@ -0,0 +1,103 @@ +{ + "id": "9019e1dc-2f34-4e44-815b-8feab43282c4", + "task_name": "actava-ai/pa_t011_t011_o003_p01_intake_payer", + "trial_name": "pa_t011_t011_o003_p01_intake_pay__z4ytRPG", + "trial_uri": "trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t011_t011_o003_p01_intake_payer" + }, + "source": null, + "task_checksum": "755892bb18768fc18e9c06a380d77e475ff89fda0711a9384a718ecabe49729c", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t011_t011_o003_p01_intake_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t011_t011_o003_p01_intake_pay__z4ytRPG", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:48:47.864219Z", + "finished_at": "2026-07-19T04:55:53.105336Z", + "environment_setup": { + "started_at": "2026-07-19T04:48:47.868072Z", + "finished_at": "2026-07-19T04:49:08.108348Z" + }, + "agent_setup": { + "started_at": "2026-07-19T04:49:08.108386Z", + "finished_at": "2026-07-19T04:49:12.134633Z" + }, + "agent_execution": { + "started_at": "2026-07-19T04:49:12.134734Z", + "finished_at": "2026-07-19T04:52:35.453678Z" + }, + "verifier": { + "started_at": "2026-07-19T04:52:35.868360Z", + "finished_at": "2026-07-19T04:55:52.523196Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/scorecard.json new file mode 100644 index 00000000..45116ca1 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t011_t011_o003_p01_intake_pay__z4ytRPG/verifier/scorecard.json @@ -0,0 +1,240 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6, + "passed_checks": 24, + "total_checks": 40, + "check_scores": { + "intake.case_exists": 1.0, + "intake.diagnoses_match": 0.0, + "intake.provider_npi_matches": 0.0, + "intake.line_items_populated": 1.0, + "intake.decision_request_complete": 1.0, + "intake.decision_member_eligible": 1.0, + "intake.decision_service_covered": 1.0, + "intake.audit": 1.0, + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.gold_card_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:crit_test_modality_dna_panel": 0.0, + "judge.nurse_review:crit_indication_personal_hx_breast_le50": 0.0, + "judge.nurse_review:crit_indication_family_hx_criteria": 0.0, + "judge.nurse_review:crit_indication_brca_breast_le65": 0.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:crit_indication_personal_hx_breast_le50": 0.0, + "judge.md_review:crit_indication_family_hx_pattern": 0.0, + "judge.md_review:crit_doc_family_pedigree": 0.0, + "judge.md_review:crit_doc_clinical_records_supporting": 0.0, + "judge.md_review:crit_doc_genetic_counseling": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true, + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:crit_test_modality_dna_panel": false, + "judge.nurse_review:crit_indication_personal_hx_breast_le50": false, + "judge.nurse_review:crit_indication_family_hx_criteria": false, + "judge.nurse_review:crit_indication_brca_breast_le65": false, + "judge.nurse_review:recommendation": false, + "judge.md_review:crit_indication_personal_hx_breast_le50": false, + "judge.md_review:crit_indication_family_hx_pattern": false, + "judge.md_review:crit_doc_family_pedigree": false, + "judge.md_review:crit_doc_clinical_records_supporting": false, + "judge.md_review:crit_doc_genetic_counseling": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "intake.diagnoses_match", + "intake.provider_npi_matches", + "outcome.target_status", + "outcome.letter_types", + "judge.nurse_review:crit_test_modality_dna_panel", + "judge.nurse_review:crit_indication_personal_hx_breast_le50", + "judge.nurse_review:crit_indication_family_hx_criteria", + "judge.nurse_review:crit_indication_brca_breast_le65", + "judge.nurse_review:recommendation", + "judge.md_review:crit_indication_personal_hx_breast_le50", + "judge.md_review:crit_indication_family_hx_pattern", + "judge.md_review:crit_doc_family_pedigree", + "judge.md_review:crit_doc_clinical_records_supporting", + "judge.md_review:crit_doc_genetic_counseling", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination" + ], + "stages": { + "intake": { + "passed": false, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true + }, + "passed_count": 6, + "total_count": 8, + "not_applicable_count": 0, + "details": {} + }, + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 9, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 6, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-6B9F238D" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b46d085d Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/result.json new file mode 100644 index 00000000..e34b57ee --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/result.json @@ -0,0 +1,109 @@ +{ + "id": "a39ea8aa-062a-4576-b16a-e8279b590e94", + "task_name": "actava-ai/pa_t012_t012_o001_p01_triage_payer", + "trial_name": "pa_t012_t012_o001_p01_triage_pay__p84iQRp", + "trial_uri": "trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t012_t012_o001_p01_triage_payer" + }, + "source": null, + "task_checksum": "12b95c9a8b23c95fbfdebc912b87167273fc0f0c0c2070426d91526989372cf4", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t012_t012_o001_p01_triage_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t012_t012_o001_p01_triage_pay__p84iQRp", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:00:35.128207Z", + "finished_at": "2026-08-21T09:04:09.589053Z", + "environment_setup": { + "started_at": "2026-08-21T09:00:35.225154Z", + "finished_at": "2026-08-21T09:00:55.126472Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:00:55.126511Z", + "finished_at": "2026-08-21T09:00:59.201875Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:00:59.201962Z", + "finished_at": "2026-08-21T09:04:05.012699Z" + }, + "verifier": { + "started_at": "2026-08-21T09:04:05.421413Z", + "finished_at": "2026-08-21T09:04:08.990077Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/scorecard.json new file mode 100644 index 00000000..f3434a63 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t012_t012_o001_p01_triage_pay__p84iQRp/verifier/scorecard.json @@ -0,0 +1,106 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 21, + "total_checks": 21, + "check_scores": { + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.gold_card_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 6, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-E33A4F82" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..1c9c46a0 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/result.json new file mode 100644 index 00000000..d7602fe2 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/result.json @@ -0,0 +1,109 @@ +{ + "id": "62eeefaa-03d7-4c7e-b809-f6e7cf3d4266", + "task_name": "actava-ai/pa_t013_t013_o002_p01_nurse_review_payer", + "trial_name": "pa_t013_t013_o002_p01_nurse_revi__g5rcXoC", + "trial_uri": "trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t013_t013_o002_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "7234c3c02860ae4193d2ef3b4683143c0b9cb39c434874f421246f8059dc3208", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t013_t013_o002_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t013_t013_o002_p01_nurse_revi__g5rcXoC", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "2912eb0d-39dc-4306-8bdf-ce4fcfbe337c" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:12:48.325352Z", + "finished_at": "2026-08-21T09:27:26.444145Z", + "environment_setup": { + "started_at": "2026-08-21T09:12:48.388465Z", + "finished_at": "2026-08-21T09:13:08.223640Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:13:08.223681Z", + "finished_at": "2026-08-21T09:13:11.737904Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:13:11.737998Z", + "finished_at": "2026-08-21T09:17:19.416598Z" + }, + "verifier": { + "started_at": "2026-08-21T09:17:19.824263Z", + "finished_at": "2026-08-21T09:27:25.804903Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/scorecard.json new file mode 100644 index 00000000..752e0f0e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t013_t013_o002_p01_nurse_revi__g5rcXoC/verifier/scorecard.json @@ -0,0 +1,128 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 31, + "total_checks": 31, + "check_scores": { + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:doc_eval_report": 1.0, + "judge.nurse_review:doc_plan_of_care": 1.0, + "judge.nurse_review:svc_not_excluded": 1.0, + "judge.nurse_review:svc_codes_match": 1.0, + "judge.nurse_review:svc_no_redundant_modalities": 1.0, + "judge.nurse_review:mn_skilled_care": 1.0, + "judge.nurse_review:mn_rehab_potential": 1.0, + "judge.nurse_review:mn_measurable_goals": 1.0, + "judge.nurse_review:mn_frequency_duration": 1.0, + "judge.nurse_review:recommendation": 1.0 + }, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:doc_eval_report": true, + "judge.nurse_review:doc_plan_of_care": true, + "judge.nurse_review:svc_not_excluded": true, + "judge.nurse_review:svc_codes_match": true, + "judge.nurse_review:svc_no_redundant_modalities": true, + "judge.nurse_review:mn_skilled_care": true, + "judge.nurse_review:mn_rehab_potential": true, + "judge.nurse_review:mn_measurable_goals": true, + "judge.nurse_review:mn_frequency_duration": true, + "judge.nurse_review:recommendation": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 15, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-180824DF" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..dccd4bb4 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/result.json new file mode 100644 index 00000000..0fea73e7 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/result.json @@ -0,0 +1,103 @@ +{ + "id": "48096c80-467c-498a-af11-6f320ed43140", + "task_name": "actava-ai/pa_t014_t014_o001_p01_intake_payer", + "trial_name": "pa_t014_t014_o001_p01_intake_pay__xMqW55D", + "trial_uri": "trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t014_t014_o001_p01_intake_payer" + }, + "source": null, + "task_checksum": "84db6dfdad9f8e2987e41a78ba7fe47a93df935dea8d2bda57d9b273efc3aa04", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t014_t014_o001_p01_intake_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t014_t014_o001_p01_intake_pay__xMqW55D", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:50:01.994981Z", + "finished_at": "2026-07-19T04:52:37.941259Z", + "environment_setup": { + "started_at": "2026-07-19T04:50:01.997869Z", + "finished_at": "2026-07-19T04:50:21.848930Z" + }, + "agent_setup": { + "started_at": "2026-07-19T04:50:21.848947Z", + "finished_at": "2026-07-19T04:50:25.642945Z" + }, + "agent_execution": { + "started_at": "2026-07-19T04:50:25.643015Z", + "finished_at": "2026-07-19T04:52:33.254238Z" + }, + "verifier": { + "started_at": "2026-07-19T04:52:33.668883Z", + "finished_at": "2026-07-19T04:52:37.292765Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/scorecard.json new file mode 100644 index 00000000..b0031ff3 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t014_t014_o001_p01_intake_pay__xMqW55D/verifier/scorecard.json @@ -0,0 +1,146 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7931034482758621, + "passed_checks": 23, + "total_checks": 29, + "check_scores": { + "intake.case_exists": 1.0, + "intake.diagnoses_match": 0.0, + "intake.provider_npi_matches": 0.0, + "intake.line_items_populated": 1.0, + "intake.decision_request_complete": 1.0, + "intake.decision_member_eligible": 1.0, + "intake.decision_service_covered": 1.0, + "intake.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 0.0, + "triage.urgency_matches": 1.0, + "triage.gold_card_matches": 0.0, + "triage.route_matches": 0.0, + "triage.audit": 0.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.disposition_matches": false, + "triage.urgency_matches": true, + "triage.gold_card_matches": false, + "triage.route_matches": false, + "triage.audit": false, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "failed_checks": [ + "intake.diagnoses_match", + "intake.provider_npi_matches", + "triage.disposition_matches", + "triage.gold_card_matches", + "triage.route_matches", + "triage.audit" + ], + "not_applicable_checks": [], + "stages": { + "intake": { + "passed": false, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true + }, + "passed_count": 6, + "total_count": 8, + "not_applicable_count": 0, + "details": {} + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": false, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": false, + "triage.urgency_matches": true, + "triage.gold_card_matches": false, + "triage.route_matches": false, + "triage.audit": false + }, + "passed_count": 2, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-73A33BA7" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..5b8716d2 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/result.json new file mode 100644 index 00000000..261672ad --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/result.json @@ -0,0 +1,103 @@ +{ + "id": "54a9cb1b-1352-404f-acde-c06550e34a0e", + "task_name": "actava-ai/pa_t015_t015_o002_p01_intake_payer", + "trial_name": "pa_t015_t015_o002_p01_intake_pay__BD9CZQq", + "trial_uri": "trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t015_t015_o002_p01_intake_payer" + }, + "source": null, + "task_checksum": "6e45aa07fcf7c26eb6eef91f51028a7c6e8c99c26004ecb9cb03e8fa056a6f84", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t015_t015_o002_p01_intake_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t015_t015_o002_p01_intake_pay__BD9CZQq", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:52:37.944310Z", + "finished_at": "2026-07-19T04:57:20.332599Z", + "environment_setup": { + "started_at": "2026-07-19T04:52:37.946773Z", + "finished_at": "2026-07-19T04:52:57.783360Z" + }, + "agent_setup": { + "started_at": "2026-07-19T04:52:57.783374Z", + "finished_at": "2026-07-19T04:53:02.005518Z" + }, + "agent_execution": { + "started_at": "2026-07-19T04:53:02.005571Z", + "finished_at": "2026-07-19T04:54:38.065649Z" + }, + "verifier": { + "started_at": "2026-07-19T04:54:38.473564Z", + "finished_at": "2026-07-19T04:57:19.818587Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/scorecard.json new file mode 100644 index 00000000..b5e47f4c --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t015_t015_o002_p01_intake_pay__BD9CZQq/verifier/scorecard.json @@ -0,0 +1,225 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7045454545454546, + "passed_checks": 31, + "total_checks": 44, + "check_scores": { + "intake.case_exists": 1.0, + "intake.diagnoses_match": 1.0, + "intake.provider_npi_matches": 0.0, + "intake.line_items_populated": 1.0, + "intake.decision_request_complete": 1.0, + "intake.decision_member_eligible": 1.0, + "intake.decision_service_covered": 1.0, + "intake.audit": 1.0, + "md.decision_exists": 0.0, + "md.rationale_present": null, + "md.signed_off": null, + "md.denial_rationale_present": null, + "md.audit": 0.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 0.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:mmp109_18_qualifying_indication": 1.0, + "judge.nurse_review:mmp109_18_prior_cardiac_workup": 0.0, + "judge.nurse_review:mmp109_18_fda_clearance": 0.0, + "judge.nurse_review:mmp109_18_device_type_appropriateness": 0.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:mmp109_18_device_type_appropriateness_md": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": true, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true, + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.denial_rationale_present": "not_applicable", + "md.audit": false, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:mmp109_18_qualifying_indication": true, + "judge.nurse_review:mmp109_18_prior_cardiac_workup": false, + "judge.nurse_review:mmp109_18_fda_clearance": false, + "judge.nurse_review:mmp109_18_device_type_appropriateness": false, + "judge.nurse_review:recommendation": false, + "judge.md_review:mmp109_18_device_type_appropriateness_md": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "intake.provider_npi_matches", + "md.decision_exists", + "md.audit", + "outcome.target_status", + "outcome.letter_types", + "outcome.determination_decision_matches", + "judge.nurse_review:mmp109_18_prior_cardiac_workup", + "judge.nurse_review:mmp109_18_fda_clearance", + "judge.nurse_review:mmp109_18_device_type_appropriateness", + "judge.nurse_review:recommendation", + "judge.md_review:mmp109_18_device_type_appropriateness_md", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.rationale_present", + "md.signed_off", + "md.denial_rationale_present" + ], + "stages": { + "intake": { + "passed": false, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": true, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true + }, + "passed_count": 7, + "total_count": 8, + "not_applicable_count": 0, + "details": {} + }, + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.denial_rationale_present": "not_applicable", + "md.audit": false + }, + "passed_count": 0, + "total_count": 2, + "not_applicable_count": 3, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 12, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 5, + "total_count": 5, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-31CD4E92" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..98b64ff0 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/result.json new file mode 100644 index 00000000..9367c4af --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/result.json @@ -0,0 +1,109 @@ +{ + "id": "64373c26-65c7-4f13-b8d6-58eb2b5b2d19", + "task_name": "actava-ai/pa_t016_t016_o001_p01_p2p_payer", + "trial_name": "pa_t016_t016_o001_p01_p2p_payer__g5pyEnS", + "trial_uri": "trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t016_t016_o001_p01_p2p_payer" + }, + "source": null, + "task_checksum": "46d869e963bba8b48447312cdc4e1e213ffba288d943f0431c75dfb6dafa9b21", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t016_t016_o001_p01_p2p_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t016_t016_o001_p01_p2p_payer__g5pyEnS", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:00:35.234898Z", + "finished_at": "2026-08-21T09:10:09.784327Z", + "environment_setup": { + "started_at": "2026-08-21T09:00:35.269215Z", + "finished_at": "2026-08-21T09:00:55.129787Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:00:55.129796Z", + "finished_at": "2026-08-21T09:00:58.951376Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:00:58.951504Z", + "finished_at": "2026-08-21T09:05:58.499269Z" + }, + "verifier": { + "started_at": "2026-08-21T09:05:58.917890Z", + "finished_at": "2026-08-21T09:10:09.158635Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/scorecard.json new file mode 100644 index 00000000..8a5e536f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t016_t016_o001_p01_p2p_payer__g5pyEnS/verifier/scorecard.json @@ -0,0 +1,176 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 37, + "total_checks": 37, + "check_scores": { + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.p2p_outcome_matches": 1.0, + "outcome.p2p_scheduled": 1.0, + "outcome.p2p_has_notes": 1.0, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 1.0, + "p2p.session_exists": 1.0, + "p2p.offer_written": 1.0, + "p2p.slot_response_written": 1.0, + "p2p.confirmation_written": 1.0, + "p2p.event_opened": 1.0, + "p2p.summary_written": null, + "p2p.turns_recorded": 1.0, + "p2p.result_recorded": 1.0, + "p2p.audit": 1.0, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": 1.0, + "p2p.required_file_sequence_ordered": 1.0, + "p2p.audit_chronology": 1.0, + "p2p.thread_state_consistent": 1.0, + "p2p.determination_matches_contract": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.p2p:primary_issue": 1.0, + "judge.p2p:resolution_point_01": 1.0, + "judge.p2p:outcome_selection": 1.0, + "judge.p2p:summary_fidelity": 1.0, + "judge.p2p:forbidden_shortcuts": 1.0, + "judge.p2p:payer_policy_alignment": 1.0, + "judge.p2p:provider_policy_alignment": 1.0 + }, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true, + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.p2p:primary_issue": true, + "judge.p2p:resolution_point_01": true, + "judge.p2p:outcome_selection": true, + "judge.p2p:summary_fidelity": true, + "judge.p2p:forbidden_shortcuts": true, + "judge.p2p:payer_policy_alignment": true, + "judge.p2p:provider_policy_alignment": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "p2p.summary_written", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot" + ], + "stages": { + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 0, + "details": {} + }, + "p2p": { + "passed": true, + "checks": { + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 7, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-99826157" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..ee2591b7 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/result.json new file mode 100644 index 00000000..a2c06985 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/result.json @@ -0,0 +1,109 @@ +{ + "id": "5dd13e10-f42f-4cdd-842b-f14cf6255e44", + "task_name": "actava-ai/pa_t017_t017_o001_p01_mdreview_payer", + "trial_name": "pa_t017_t017_o001_p01_mdreview_p__gmXPonq", + "trial_uri": "trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t017_t017_o001_p01_mdreview_payer" + }, + "source": null, + "task_checksum": "359f4a2155973ca04cd31955b2d4a024acffd67db5eef66c5805993d581ea72a", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t017_t017_o001_p01_mdreview_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t017_t017_o001_p01_mdreview_p__gmXPonq", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:04:09.594103Z", + "finished_at": "2026-08-21T09:11:23.681929Z", + "environment_setup": { + "started_at": "2026-08-21T09:04:09.633888Z", + "finished_at": "2026-08-21T09:04:29.440586Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:04:29.440603Z", + "finished_at": "2026-08-21T09:04:32.986759Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:04:32.986814Z", + "finished_at": "2026-08-21T09:09:01.889801Z" + }, + "verifier": { + "started_at": "2026-08-21T09:09:02.299489Z", + "finished_at": "2026-08-21T09:11:23.019586Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/scorecard.json new file mode 100644 index 00000000..246b8124 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t017_t017_o001_p01_mdreview_p__gmXPonq/verifier/scorecard.json @@ -0,0 +1,120 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 25, + "total_checks": 25, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.md_review:osa_severity_psg": 1.0, + "judge.md_review:osa_sleepiness_ess": 1.0, + "judge.md_review:pap_therapy_failure": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.md_review:osa_severity_psg": true, + "judge.md_review:osa_sleepiness_ess": true, + "judge.md_review:pap_therapy_failure": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "md.denial_rationale_present" + ], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 15, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-C9824F3A" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..12928e58 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/result.json new file mode 100644 index 00000000..f7e50d7f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/result.json @@ -0,0 +1,103 @@ +{ + "id": "ae516780-badf-4b80-befd-2eaac5355d4f", + "task_name": "actava-ai/pa_t018_t018_o001_p01_mdreview_payer", + "trial_name": "pa_t018_t018_o001_p01_mdreview_p__xEuA7j7", + "trial_uri": "trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t018_t018_o001_p01_mdreview_payer" + }, + "source": null, + "task_checksum": "1cf8ba6d9d9a8a5cfff2ccf723630a7f204ee39185d985579ef4e54eaf6754c3", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t018_t018_o001_p01_mdreview_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t018_t018_o001_p01_mdreview_p__xEuA7j7", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:55:53.108836Z", + "finished_at": "2026-07-19T04:59:49.824782Z", + "environment_setup": { + "started_at": "2026-07-19T04:55:53.111442Z", + "finished_at": "2026-07-19T04:56:12.940293Z" + }, + "agent_setup": { + "started_at": "2026-07-19T04:56:12.940309Z", + "finished_at": "2026-07-19T04:56:17.275893Z" + }, + "agent_execution": { + "started_at": "2026-07-19T04:56:17.275953Z", + "finished_at": "2026-07-19T04:57:38.530134Z" + }, + "verifier": { + "started_at": "2026-07-19T04:57:38.938205Z", + "finished_at": "2026-07-19T04:59:49.268169Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/scorecard.json new file mode 100644 index 00000000..6c9715f8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t018_t018_o001_p01_mdreview_p__xEuA7j7/verifier/scorecard.json @@ -0,0 +1,158 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6111111111111112, + "passed_checks": 22, + "total_checks": 36, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 0.0, + "md.signed_off": 0.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.md_review:hopd_clinical_trial": 0.0, + "judge.md_review:hopd_24h_specialist_coordination": 0.0, + "judge.md_review:hopd_contrast_allergy": 0.0, + "judge.md_review:hopd_chronic_disease_continuity": 0.0, + "judge.md_review:hopd_cancer_restaging": 0.0, + "judge.md_review:hopd_pre_procedure_imaging": 0.0, + "judge.md_review:hopd_obstetric": 0.0, + "judge.md_review:hopd_perinatology": 0.0, + "judge.md_review:hopd_under_18": 0.0, + "judge.md_review:hopd_sedation_barrier": 0.0, + "judge.md_review:hopd_equipment_size": 0.0, + "judge.md_review:hopd_open_mri_claustrophobia": 0.0, + "judge.md_review:hopd_adverse_health_impact": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.md_review:hopd_clinical_trial": false, + "judge.md_review:hopd_24h_specialist_coordination": false, + "judge.md_review:hopd_contrast_allergy": false, + "judge.md_review:hopd_chronic_disease_continuity": false, + "judge.md_review:hopd_cancer_restaging": false, + "judge.md_review:hopd_pre_procedure_imaging": false, + "judge.md_review:hopd_obstetric": false, + "judge.md_review:hopd_perinatology": false, + "judge.md_review:hopd_under_18": false, + "judge.md_review:hopd_sedation_barrier": false, + "judge.md_review:hopd_equipment_size": false, + "judge.md_review:hopd_open_mri_claustrophobia": false, + "judge.md_review:hopd_adverse_health_impact": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [ + "md.rationale_present", + "md.signed_off", + "judge.md_review:hopd_clinical_trial", + "judge.md_review:hopd_24h_specialist_coordination", + "judge.md_review:hopd_contrast_allergy", + "judge.md_review:hopd_chronic_disease_continuity", + "judge.md_review:hopd_cancer_restaging", + "judge.md_review:hopd_pre_procedure_imaging", + "judge.md_review:hopd_obstetric", + "judge.md_review:hopd_perinatology", + "judge.md_review:hopd_under_18", + "judge.md_review:hopd_sedation_barrier", + "judge.md_review:hopd_equipment_size", + "judge.md_review:hopd_open_mri_claustrophobia" + ], + "not_applicable_checks": [ + "md.denial_rationale_present" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 15, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-89B002CC" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..20d67909 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/result.json new file mode 100644 index 00000000..95c6b890 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/result.json @@ -0,0 +1,109 @@ +{ + "id": "2594d670-f599-4577-8075-88b637423712", + "task_name": "actava-ai/pa_t019_t019_o001_p01_p2p_payer", + "trial_name": "pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z", + "trial_uri": "trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t019_t019_o001_p01_p2p_payer" + }, + "source": null, + "task_checksum": "ff38cc0c7255d813c169efcb30c4052281fbba735f3ccae07f1c97bfe9cc530d", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t019_t019_o001_p01_p2p_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:05:50.758625Z", + "finished_at": "2026-08-21T09:15:55.371677Z", + "environment_setup": { + "started_at": "2026-08-21T09:05:50.785350Z", + "finished_at": "2026-08-21T09:06:11.085172Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:06:11.085194Z", + "finished_at": "2026-08-21T09:06:15.511202Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:06:15.511273Z", + "finished_at": "2026-08-21T09:08:46.210312Z" + }, + "verifier": { + "started_at": "2026-08-21T09:08:46.613594Z", + "finished_at": "2026-08-21T09:15:54.720292Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/scorecard.json new file mode 100644 index 00000000..d75f31a8 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t019_t019_o001_p01_p2p_payer__Y9ojm7Z/verifier/scorecard.json @@ -0,0 +1,182 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 39, + "total_checks": 39, + "check_scores": { + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.p2p_outcome_matches": 1.0, + "outcome.p2p_scheduled": 1.0, + "outcome.p2p_has_notes": 1.0, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 1.0, + "p2p.session_exists": 1.0, + "p2p.offer_written": 1.0, + "p2p.slot_response_written": 1.0, + "p2p.confirmation_written": 1.0, + "p2p.event_opened": 1.0, + "p2p.summary_written": null, + "p2p.turns_recorded": 1.0, + "p2p.result_recorded": 1.0, + "p2p.audit": 1.0, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": 1.0, + "p2p.required_file_sequence_ordered": 1.0, + "p2p.audit_chronology": 1.0, + "p2p.thread_state_consistent": 1.0, + "p2p.determination_matches_contract": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.p2p:primary_issue": 1.0, + "judge.p2p:resolution_point_01": 1.0, + "judge.p2p:outcome_selection": 1.0, + "judge.p2p:summary_fidelity": 1.0, + "judge.p2p:forbidden_shortcuts": 1.0, + "judge.p2p:payer_policy_alignment": 1.0, + "judge.p2p:provider_policy_alignment": 1.0 + }, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true, + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.p2p:primary_issue": true, + "judge.p2p:resolution_point_01": true, + "judge.p2p:outcome_selection": true, + "judge.p2p:summary_fidelity": true, + "judge.p2p:forbidden_shortcuts": true, + "judge.p2p:payer_policy_alignment": true, + "judge.p2p:provider_policy_alignment": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "p2p.summary_written", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot" + ], + "stages": { + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true + }, + "passed_count": 16, + "total_count": 16, + "not_applicable_count": 0, + "details": {} + }, + "p2p": { + "passed": true, + "checks": { + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 7, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-A9891BCD" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..7cac2019 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/result.json new file mode 100644 index 00000000..e9ba9e78 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/result.json @@ -0,0 +1,109 @@ +{ + "id": "b9b9028d-b753-4edd-9629-846334931831", + "task_name": "actava-ai/pa_t021_t021_o003_p01_mdreview_payer", + "trial_name": "pa_t021_t021_o003_p01_mdreview_p__ipPofdm", + "trial_uri": "trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t021_t021_o003_p01_mdreview_payer" + }, + "source": null, + "task_checksum": "65eaad782a7c60e8bef7c0fe70a0ecd19b37ee2d27f22e3c75df6a2daf851121", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t021_t021_o003_p01_mdreview_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t021_t021_o003_p01_mdreview_p__ipPofdm", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:10:09.789052Z", + "finished_at": "2026-08-21T09:17:50.020160Z", + "environment_setup": { + "started_at": "2026-08-21T09:10:09.791843Z", + "finished_at": "2026-08-21T09:10:29.579566Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:10:29.579582Z", + "finished_at": "2026-08-21T09:10:33.344491Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:10:33.344550Z", + "finished_at": "2026-08-21T09:14:54.933332Z" + }, + "verifier": { + "started_at": "2026-08-21T09:14:55.339711Z", + "finished_at": "2026-08-21T09:17:49.392021Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/scorecard.json new file mode 100644 index 00000000..6d29b91e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t021_t021_o003_p01_mdreview_p__ipPofdm/verifier/scorecard.json @@ -0,0 +1,159 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 47, + "total_checks": 47, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": 1.0, + "md.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.md_review:mp1126_cvp_01": 1.0, + "judge.md_review:mp1126_cvp_02": 1.0, + "judge.md_review:mp1126_cvp_03": 1.0, + "judge.md_review:mp1126_cvp_04": 1.0, + "judge.md_review:mp1126_cvp_05": 1.0, + "judge.md_review:mp1126_cvp_06": 1.0, + "judge.md_review:mp1126_cvp_07": 1.0, + "judge.md_review:mp1126_cvp_08": 1.0, + "judge.md_review:mp1126_cvp_09": 1.0, + "judge.md_review:mp1126_cvp_10": 1.0, + "judge.md_review:mp1126_cvp_11": 1.0, + "judge.md_review:mp1126_cvp_12": 1.0, + "judge.md_review:mp1126_mhp_01": 1.0, + "judge.md_review:mp1126_mhp_02": 1.0, + "judge.md_review:mp1126_mhp_03": 1.0, + "judge.md_review:mp1126_mhp_04": 1.0, + "judge.md_review:mp1126_mhp_05": 1.0, + "judge.md_review:mp1126_mhp_06": 1.0, + "judge.md_review:mp1126_mhp_07": 1.0, + "judge.md_review:mp1126_sca_01": 1.0, + "judge.md_review:mp1126_sca_02": 1.0, + "judge.md_review:mp1126_sca_03": 1.0, + "judge.md_review:mp1126_sca_04": 1.0, + "judge.md_review:mp1126_sca_05": 1.0, + "judge.md_review:mp1126_sca_06": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.md_review:mp1126_cvp_01": true, + "judge.md_review:mp1126_cvp_02": true, + "judge.md_review:mp1126_cvp_03": true, + "judge.md_review:mp1126_cvp_04": true, + "judge.md_review:mp1126_cvp_05": true, + "judge.md_review:mp1126_cvp_06": true, + "judge.md_review:mp1126_cvp_07": true, + "judge.md_review:mp1126_cvp_08": true, + "judge.md_review:mp1126_cvp_09": true, + "judge.md_review:mp1126_cvp_10": true, + "judge.md_review:mp1126_cvp_11": true, + "judge.md_review:mp1126_cvp_12": true, + "judge.md_review:mp1126_mhp_01": true, + "judge.md_review:mp1126_mhp_02": true, + "judge.md_review:mp1126_mhp_03": true, + "judge.md_review:mp1126_mhp_04": true, + "judge.md_review:mp1126_mhp_05": true, + "judge.md_review:mp1126_mhp_06": true, + "judge.md_review:mp1126_mhp_07": true, + "judge.md_review:mp1126_sca_01": true, + "judge.md_review:mp1126_sca_02": true, + "judge.md_review:mp1126_sca_03": true, + "judge.md_review:mp1126_sca_04": true, + "judge.md_review:mp1126_sca_05": true, + "judge.md_review:mp1126_sca_06": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true + }, + "passed_count": 5, + "total_count": 5, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-BB186B55" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..67316389 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/result.json new file mode 100644 index 00000000..b0e9a420 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/result.json @@ -0,0 +1,103 @@ +{ + "id": "1473ef1f-5275-441a-b782-c62496d3ba10", + "task_name": "actava-ai/pa_t022_t022_o001_p01_nurse_review_payer", + "trial_name": "pa_t022_t022_o001_p01_nurse_revi__92wAj4u", + "trial_uri": "trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t022_t022_o001_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "9e4f3c528f0c32d0e13a0ebd958a7844c7d0f2a8f67cfbc6de44d6589891f0d1", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t022_t022_o001_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t022_t022_o001_p01_nurse_revi__92wAj4u", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T04:59:49.827632Z", + "finished_at": "2026-07-19T05:07:24.422233Z", + "environment_setup": { + "started_at": "2026-07-19T04:59:49.830192Z", + "finished_at": "2026-07-19T05:00:09.807785Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:00:09.807803Z", + "finished_at": "2026-07-19T05:00:20.023123Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:00:20.023178Z", + "finished_at": "2026-07-19T05:04:26.213894Z" + }, + "verifier": { + "started_at": "2026-07-19T05:04:26.631834Z", + "finished_at": "2026-07-19T05:07:23.868088Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/scorecard.json new file mode 100644 index 00000000..c629905e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t022_t022_o001_p01_nurse_revi__92wAj4u/verifier/scorecard.json @@ -0,0 +1,211 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.2727272727272727, + "passed_checks": 12, + "total_checks": 44, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": 0.0, + "outcome.review_decision_exists": 0.0, + "outcome.determination_stage_trace": 0.0, + "outcome.determination_finalize_audit": 0.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 0.0, + "outcome.source_recommendation_consistent": 0.0, + "outcome.review_decision_matches": 0.0, + "outcome.determination_decision_matches": null, + "outcome.determination_has_rationale": null, + "outcome.clean_determination": 0.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:SOC_MED_STABILITY_01": 0.0, + "judge.nurse_review:SOC_MED_STABILITY_02": 0.0, + "judge.nurse_review:SOC_MED_STABILITY_03": 0.0, + "judge.nurse_review:SOC_INFUSION_SAFETY_01": 0.0, + "judge.nurse_review:SOC_INFUSION_SAFETY_02": 0.0, + "judge.nurse_review:SOC_TREATMENT_INIT_01": 0.0, + "judge.nurse_review:SOC_TREATMENT_INIT_02": 0.0, + "judge.nurse_review:SOC_SITE_AVAIL_01": 0.0, + "judge.nurse_review:SOC_SITE_AVAIL_02": 0.0, + "judge.nurse_review:SOC_SITE_AVAIL_03": 0.0, + "judge.nurse_review:recommendation": 1.0, + "judge.md_review:SOC_MED_STABILITY_01": 0.0, + "judge.md_review:SOC_MED_STABILITY_02": 0.0, + "judge.md_review:SOC_MED_STABILITY_03": 0.0, + "judge.md_review:SOC_INFUSION_SAFETY_01": 0.0, + "judge.md_review:SOC_INFUSION_SAFETY_02": 0.0, + "judge.md_review:SOC_TREATMENT_INIT_01": 0.0, + "judge.md_review:SOC_TREATMENT_INIT_02": 0.0, + "judge.md_review:SOC_SITE_AVAIL_01": 0.0, + "judge.md_review:SOC_SITE_AVAIL_02": 0.0, + "judge.md_review:SOC_SITE_AVAIL_03": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": false, + "outcome.review_decision_exists": false, + "outcome.determination_stage_trace": false, + "outcome.determination_finalize_audit": false, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": false, + "outcome.source_recommendation_consistent": false, + "outcome.review_decision_matches": false, + "outcome.determination_decision_matches": "not_applicable", + "outcome.determination_has_rationale": "not_applicable", + "outcome.clean_determination": false, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:SOC_MED_STABILITY_01": false, + "judge.nurse_review:SOC_MED_STABILITY_02": false, + "judge.nurse_review:SOC_MED_STABILITY_03": false, + "judge.nurse_review:SOC_INFUSION_SAFETY_01": false, + "judge.nurse_review:SOC_INFUSION_SAFETY_02": false, + "judge.nurse_review:SOC_TREATMENT_INIT_01": false, + "judge.nurse_review:SOC_TREATMENT_INIT_02": false, + "judge.nurse_review:SOC_SITE_AVAIL_01": false, + "judge.nurse_review:SOC_SITE_AVAIL_02": false, + "judge.nurse_review:SOC_SITE_AVAIL_03": false, + "judge.nurse_review:recommendation": true, + "judge.md_review:SOC_MED_STABILITY_01": false, + "judge.md_review:SOC_MED_STABILITY_02": false, + "judge.md_review:SOC_MED_STABILITY_03": false, + "judge.md_review:SOC_INFUSION_SAFETY_01": false, + "judge.md_review:SOC_INFUSION_SAFETY_02": false, + "judge.md_review:SOC_TREATMENT_INIT_01": false, + "judge.md_review:SOC_TREATMENT_INIT_02": false, + "judge.md_review:SOC_SITE_AVAIL_01": false, + "judge.md_review:SOC_SITE_AVAIL_02": false, + "judge.md_review:SOC_SITE_AVAIL_03": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "outcome.target_status", + "outcome.letter_types", + "outcome.determination_exists", + "outcome.review_decision_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_matches", + "outcome.clean_determination", + "judge.nurse_review:SOC_MED_STABILITY_01", + "judge.nurse_review:SOC_MED_STABILITY_02", + "judge.nurse_review:SOC_MED_STABILITY_03", + "judge.nurse_review:SOC_INFUSION_SAFETY_01", + "judge.nurse_review:SOC_INFUSION_SAFETY_02", + "judge.nurse_review:SOC_TREATMENT_INIT_01", + "judge.nurse_review:SOC_TREATMENT_INIT_02", + "judge.nurse_review:SOC_SITE_AVAIL_01", + "judge.nurse_review:SOC_SITE_AVAIL_02", + "judge.nurse_review:SOC_SITE_AVAIL_03", + "judge.md_review:SOC_MED_STABILITY_01", + "judge.md_review:SOC_MED_STABILITY_02", + "judge.md_review:SOC_MED_STABILITY_03", + "judge.md_review:SOC_INFUSION_SAFETY_01", + "judge.md_review:SOC_INFUSION_SAFETY_02", + "judge.md_review:SOC_TREATMENT_INIT_01", + "judge.md_review:SOC_TREATMENT_INIT_02", + "judge.md_review:SOC_SITE_AVAIL_01", + "judge.md_review:SOC_SITE_AVAIL_02", + "judge.md_review:SOC_SITE_AVAIL_03", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.determination_decision_matches", + "outcome.determination_has_rationale" + ], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": false, + "outcome.review_decision_exists": false, + "outcome.determination_stage_trace": false, + "outcome.determination_finalize_audit": false, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": false, + "outcome.source_recommendation_consistent": false, + "outcome.review_decision_matches": false, + "outcome.determination_decision_matches": "not_applicable", + "outcome.determination_has_rationale": "not_applicable", + "outcome.clean_determination": false + }, + "passed_count": 1, + "total_count": 11, + "not_applicable_count": 2, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-27EFB677" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..e735b5d6 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/result.json new file mode 100644 index 00000000..b2bd7cbd --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/result.json @@ -0,0 +1,109 @@ +{ + "id": "5184eeb9-8a8c-46b9-863f-3021b7eba6b6", + "task_name": "actava-ai/pa_t023_t023_o003_p01_triage_payer", + "trial_name": "pa_t023_t023_o003_p01_triage_pay__aFE74LC", + "trial_uri": "trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t023_t023_o003_p01_triage_payer" + }, + "source": null, + "task_checksum": "25c03f624bb9bf1aec91269c1c10288aaf02d6c9b89ca1a57e90434186233266", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t023_t023_o003_p01_triage_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t023_t023_o003_p01_triage_pay__aFE74LC", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:11:23.687122Z", + "finished_at": "2026-08-21T09:24:15.519569Z", + "environment_setup": { + "started_at": "2026-08-21T09:11:23.718780Z", + "finished_at": "2026-08-21T09:11:43.570882Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:11:43.570901Z", + "finished_at": "2026-08-21T09:11:48.163500Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:11:48.163574Z", + "finished_at": "2026-08-21T09:17:46.898007Z" + }, + "verifier": { + "started_at": "2026-08-21T09:17:47.329016Z", + "finished_at": "2026-08-21T09:24:14.859383Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/scorecard.json new file mode 100644 index 00000000..aa3ce484 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t023_t023_o003_p01_triage_pay__aFE74LC/verifier/scorecard.json @@ -0,0 +1,145 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 32, + "total_checks": 32, + "check_scores": { + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.gold_card_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:evicore_rad_brain_indication": 1.0, + "judge.nurse_review:evicore_rad_brain_prior_workup": 1.0, + "judge.nurse_review:evicore_rad_brain_protocol": 1.0, + "judge.nurse_review:evicore_rad_brain_site_of_service": 1.0, + "judge.nurse_review:recommendation": 1.0 + }, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:evicore_rad_brain_indication": true, + "judge.nurse_review:evicore_rad_brain_prior_workup": true, + "judge.nurse_review:evicore_rad_brain_protocol": true, + "judge.nurse_review:evicore_rad_brain_site_of_service": true, + "judge.nurse_review:recommendation": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 15, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 6, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-7D075BE2" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..304ba8e3 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/result.json new file mode 100644 index 00000000..04026b1b --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/result.json @@ -0,0 +1,103 @@ +{ + "id": "c6393591-d799-4460-8e2c-585d2002548a", + "task_name": "actava-ai/pa_t024_t024_o001_p01_nurse_review_payer", + "trial_name": "pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm", + "trial_uri": "trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t024_t024_o001_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "b9216104534a63986353dbcc735c965ea4273940648c794a3e60bbc15f2c82cf", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t024_t024_o001_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:03:20.948661Z", + "finished_at": "2026-07-19T05:06:10.335486Z", + "environment_setup": { + "started_at": "2026-07-19T05:03:20.951243Z", + "finished_at": "2026-07-19T05:03:40.795574Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:03:40.795590Z", + "finished_at": "2026-07-19T05:03:46.510131Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:03:46.510201Z", + "finished_at": "2026-07-19T05:05:43.073167Z" + }, + "verifier": { + "started_at": "2026-07-19T05:05:43.471228Z", + "finished_at": "2026-07-19T05:06:09.774486Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/scorecard.json new file mode 100644 index 00000000..b5d9170f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t024_t024_o001_p01_nurse_revi__6pBa8Xm/verifier/scorecard.json @@ -0,0 +1,176 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.7857142857142857, + "passed_checks": 11, + "total_checks": 14, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 0.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:crit_ra_diagnosis": null, + "judge.nurse_review:crit_disease_activity": null, + "judge.nurse_review:crit_tnf_step_therapy": null, + "judge.nurse_review:crit_methotrexate": null, + "judge.nurse_review:crit_combination_exclusion": null, + "judge.nurse_review:crit_dosing_attestation": null, + "judge.nurse_review:recommendation": null, + "judge.md_review:crit_combination_exclusion": null, + "judge.md_review:crit_dosing_compliance": null, + "judge.md_review:final_decision": null, + "judge.md_review:decision_rationale": null + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:crit_ra_diagnosis": "not_applicable", + "judge.nurse_review:crit_disease_activity": "not_applicable", + "judge.nurse_review:crit_tnf_step_therapy": "not_applicable", + "judge.nurse_review:crit_methotrexate": "not_applicable", + "judge.nurse_review:crit_combination_exclusion": "not_applicable", + "judge.nurse_review:crit_dosing_attestation": "not_applicable", + "judge.nurse_review:recommendation": "not_applicable", + "judge.md_review:crit_combination_exclusion": "not_applicable", + "judge.md_review:crit_dosing_compliance": "not_applicable", + "judge.md_review:final_decision": "not_applicable", + "judge.md_review:decision_rationale": "not_applicable" + }, + "failed_checks": [ + "md.rationale_present", + "outcome.target_status", + "outcome.letter_types" + ], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination", + "judge.nurse_review:crit_ra_diagnosis", + "judge.nurse_review:crit_disease_activity", + "judge.nurse_review:crit_tnf_step_therapy", + "judge.nurse_review:crit_methotrexate", + "judge.nurse_review:crit_combination_exclusion", + "judge.nurse_review:crit_dosing_attestation", + "judge.nurse_review:recommendation", + "judge.md_review:crit_combination_exclusion", + "judge.md_review:crit_dosing_compliance", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 3, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 9, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-3F5CB30F" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..e0eef51f Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/result.json new file mode 100644 index 00000000..69fc671e --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/result.json @@ -0,0 +1,103 @@ +{ + "id": "7d918701-f309-42f1-9b98-145195561736", + "task_name": "actava-ai/pa_t026_t026_o003_p01_nurse_review_payer", + "trial_name": "pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx", + "trial_uri": "trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t026_t026_o003_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "182e183d98ab4989058eaeececb2d03eb36527ef27bd6a820149a289f790969b", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t026_t026_o003_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:03:37.268631Z", + "finished_at": "2026-07-19T05:06:09.196478Z", + "environment_setup": { + "started_at": "2026-07-19T05:03:37.271122Z", + "finished_at": "2026-07-19T05:03:57.137215Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:03:57.137232Z", + "finished_at": "2026-07-19T05:04:02.800233Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:04:02.800301Z", + "finished_at": "2026-07-19T05:05:59.867876Z" + }, + "verifier": { + "started_at": "2026-07-19T05:06:00.274569Z", + "finished_at": "2026-07-19T05:06:08.653345Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/scorecard.json new file mode 100644 index 00000000..3c058d0f --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t026_t026_o003_p01_nurse_revi__Pv2fyjx/verifier/scorecard.json @@ -0,0 +1,309 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.8076923076923077, + "passed_checks": 21, + "total_checks": 26, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 0.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 0.0, + "outcome.determination_has_rationale": 1.0, + "outcome.p2p_outcome_matches": null, + "outcome.p2p_scheduled": null, + "outcome.p2p_has_notes": null, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 0.0, + "p2p.session_exists": null, + "p2p.offer_written": null, + "p2p.slot_response_written": null, + "p2p.confirmation_written": null, + "p2p.event_opened": null, + "p2p.summary_written": null, + "p2p.turns_recorded": null, + "p2p.result_recorded": null, + "p2p.audit": null, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": null, + "p2p.required_file_sequence_ordered": null, + "p2p.audit_chronology": null, + "p2p.thread_state_consistent": null, + "p2p.determination_matches_contract": null, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:mp15_13_advanced_liver_disease": null, + "judge.nurse_review:mp15_13_bleeding_disorder": null, + "judge.nurse_review:mp15_13_cardiac_arrhythmia": null, + "judge.nurse_review:mp15_13_copd": null, + "judge.nurse_review:mp15_13_cad_pvd": null, + "judge.nurse_review:mp15_13_resistant_htn": null, + "judge.nurse_review:mp15_13_remaining_comorbidities": null, + "judge.nurse_review:mp15_13_asc_weight_health_restriction": null, + "judge.nurse_review:mp15_13_asc_geographic_access": null, + "judge.nurse_review:recommendation": null, + "judge.md_review:mp15_13_high_risk_comorbidity": null, + "judge.md_review:mp15_13_asc_access_barrier": null, + "judge.md_review:final_decision": null, + "judge.md_review:decision_rationale": null, + "judge.p2p:primary_issue": null, + "judge.p2p:resolution_point_01": null, + "judge.p2p:outcome_selection": null, + "judge.p2p:summary_fidelity": null, + "judge.p2p:forbidden_shortcuts": null, + "judge.p2p:payer_policy_alignment": null, + "judge.p2p:provider_policy_alignment": null + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.p2p_outcome_matches": "not_applicable", + "outcome.p2p_scheduled": "not_applicable", + "outcome.p2p_has_notes": "not_applicable", + "outcome.clean_determination": true, + "p2p.request_exists": false, + "p2p.session_exists": "not_applicable", + "p2p.offer_written": "not_applicable", + "p2p.slot_response_written": "not_applicable", + "p2p.confirmation_written": "not_applicable", + "p2p.event_opened": "not_applicable", + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": "not_applicable", + "p2p.result_recorded": "not_applicable", + "p2p.audit": "not_applicable", + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": "not_applicable", + "p2p.required_file_sequence_ordered": "not_applicable", + "p2p.audit_chronology": "not_applicable", + "p2p.thread_state_consistent": "not_applicable", + "p2p.determination_matches_contract": "not_applicable", + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:mp15_13_advanced_liver_disease": "not_applicable", + "judge.nurse_review:mp15_13_bleeding_disorder": "not_applicable", + "judge.nurse_review:mp15_13_cardiac_arrhythmia": "not_applicable", + "judge.nurse_review:mp15_13_copd": "not_applicable", + "judge.nurse_review:mp15_13_cad_pvd": "not_applicable", + "judge.nurse_review:mp15_13_resistant_htn": "not_applicable", + "judge.nurse_review:mp15_13_remaining_comorbidities": "not_applicable", + "judge.nurse_review:mp15_13_asc_weight_health_restriction": "not_applicable", + "judge.nurse_review:mp15_13_asc_geographic_access": "not_applicable", + "judge.nurse_review:recommendation": "not_applicable", + "judge.md_review:mp15_13_high_risk_comorbidity": "not_applicable", + "judge.md_review:mp15_13_asc_access_barrier": "not_applicable", + "judge.md_review:final_decision": "not_applicable", + "judge.md_review:decision_rationale": "not_applicable", + "judge.p2p:primary_issue": "not_applicable", + "judge.p2p:resolution_point_01": "not_applicable", + "judge.p2p:outcome_selection": "not_applicable", + "judge.p2p:summary_fidelity": "not_applicable", + "judge.p2p:forbidden_shortcuts": "not_applicable", + "judge.p2p:payer_policy_alignment": "not_applicable", + "judge.p2p:provider_policy_alignment": "not_applicable" + }, + "failed_checks": [ + "md.signed_off", + "outcome.target_status", + "outcome.letter_types", + "outcome.determination_decision_matches", + "p2p.request_exists" + ], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.p2p_outcome_matches", + "outcome.p2p_scheduled", + "outcome.p2p_has_notes", + "p2p.session_exists", + "p2p.offer_written", + "p2p.slot_response_written", + "p2p.confirmation_written", + "p2p.event_opened", + "p2p.summary_written", + "p2p.turns_recorded", + "p2p.result_recorded", + "p2p.audit", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot", + "p2p.required_file_sequence_present", + "p2p.required_file_sequence_ordered", + "p2p.audit_chronology", + "p2p.thread_state_consistent", + "p2p.determination_matches_contract", + "judge.nurse_review:mp15_13_advanced_liver_disease", + "judge.nurse_review:mp15_13_bleeding_disorder", + "judge.nurse_review:mp15_13_cardiac_arrhythmia", + "judge.nurse_review:mp15_13_copd", + "judge.nurse_review:mp15_13_cad_pvd", + "judge.nurse_review:mp15_13_resistant_htn", + "judge.nurse_review:mp15_13_remaining_comorbidities", + "judge.nurse_review:mp15_13_asc_weight_health_restriction", + "judge.nurse_review:mp15_13_asc_geographic_access", + "judge.nurse_review:recommendation", + "judge.md_review:mp15_13_high_risk_comorbidity", + "judge.md_review:mp15_13_asc_access_barrier", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale", + "judge.p2p:primary_issue", + "judge.p2p:resolution_point_01", + "judge.p2p:outcome_selection", + "judge.p2p:summary_fidelity", + "judge.p2p:forbidden_shortcuts", + "judge.p2p:payer_policy_alignment", + "judge.p2p:provider_policy_alignment" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 3, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.p2p_outcome_matches": "not_applicable", + "outcome.p2p_scheduled": "not_applicable", + "outcome.p2p_has_notes": "not_applicable", + "outcome.clean_determination": true + }, + "passed_count": 12, + "total_count": 15, + "not_applicable_count": 3, + "details": {} + }, + "p2p": { + "passed": false, + "checks": { + "p2p.request_exists": false, + "p2p.session_exists": "not_applicable", + "p2p.offer_written": "not_applicable", + "p2p.slot_response_written": "not_applicable", + "p2p.confirmation_written": "not_applicable", + "p2p.event_opened": "not_applicable", + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": "not_applicable", + "p2p.result_recorded": "not_applicable", + "p2p.audit": "not_applicable", + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": "not_applicable", + "p2p.required_file_sequence_ordered": "not_applicable", + "p2p.audit_chronology": "not_applicable", + "p2p.thread_state_consistent": "not_applicable", + "p2p.determination_matches_contract": "not_applicable" + }, + "passed_count": 0, + "total_count": 1, + "not_applicable_count": 20, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-8B8FB0AC" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..89e393fb Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/result.json new file mode 100644 index 00000000..8b74d59a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/result.json @@ -0,0 +1,103 @@ +{ + "id": "6a9d2725-c35f-4dea-b051-535426b4842d", + "task_name": "actava-ai/pa_t027_t027_o002_p01_nurse_review_payer", + "trial_name": "pa_t027_t027_o002_p01_nurse_revi__NfYYbqm", + "trial_uri": "trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t027_t027_o002_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "8ecdd8c71568f0c6aeb363aa472421f15102b23970f195a0e481f523536ab5ca", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t027_t027_o002_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t027_t027_o002_p01_nurse_revi__NfYYbqm", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:06:09.199485Z", + "finished_at": "2026-07-19T05:12:40.284224Z", + "environment_setup": { + "started_at": "2026-07-19T05:06:09.202240Z", + "finished_at": "2026-07-19T05:06:29.014841Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:06:29.014853Z", + "finished_at": "2026-07-19T05:06:33.083854Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:06:33.083926Z", + "finished_at": "2026-07-19T05:08:11.361372Z" + }, + "verifier": { + "started_at": "2026-07-19T05:08:11.768897Z", + "finished_at": "2026-07-19T05:12:39.668811Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/scorecard.json new file mode 100644 index 00000000..c20018ea --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t027_t027_o002_p01_nurse_revi__NfYYbqm/verifier/scorecard.json @@ -0,0 +1,180 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.5526315789473685, + "passed_checks": 21, + "total_checks": 38, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 0.0, + "md.signed_off": 0.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 0.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:sinus_dx_crs_duration": 0.0, + "judge.nurse_review:sinus_dx_cardinal_symptoms": 0.0, + "judge.nurse_review:sinus_dx_inflammation": 1.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:sinus_dx_type": 0.0, + "judge.md_review:sinus_dx_laterality": 0.0, + "judge.md_review:sinus_intranasal_steroids": 0.0, + "judge.md_review:sinus_antibiotics": 0.0, + "judge.md_review:sinus_nasal_lavage": 0.0, + "judge.md_review:sinus_ct_timing": 0.0, + "judge.md_review:sinus_ct_qualifying_findings": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:sinus_dx_crs_duration": false, + "judge.nurse_review:sinus_dx_cardinal_symptoms": false, + "judge.nurse_review:sinus_dx_inflammation": true, + "judge.nurse_review:recommendation": false, + "judge.md_review:sinus_dx_type": false, + "judge.md_review:sinus_dx_laterality": false, + "judge.md_review:sinus_intranasal_steroids": false, + "judge.md_review:sinus_antibiotics": false, + "judge.md_review:sinus_nasal_lavage": false, + "judge.md_review:sinus_ct_timing": false, + "judge.md_review:sinus_ct_qualifying_findings": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "md.rationale_present", + "md.signed_off", + "outcome.target_status", + "outcome.letter_types", + "outcome.determination_decision_matches", + "judge.nurse_review:sinus_dx_crs_duration", + "judge.nurse_review:sinus_dx_cardinal_symptoms", + "judge.nurse_review:recommendation", + "judge.md_review:sinus_dx_type", + "judge.md_review:sinus_dx_laterality", + "judge.md_review:sinus_intranasal_steroids", + "judge.md_review:sinus_antibiotics", + "judge.md_review:sinus_nasal_lavage", + "judge.md_review:sinus_ct_timing", + "judge.md_review:sinus_ct_qualifying_findings", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.denial_rationale_present" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 12, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-70FB6F70" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..4045f237 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/result.json new file mode 100644 index 00000000..c639ba51 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/result.json @@ -0,0 +1,103 @@ +{ + "id": "c701812d-5d29-4386-afee-a844f1763905", + "task_name": "actava-ai/pa_t028_t028_o001_p01_nurse_review_payer", + "trial_name": "pa_t028_t028_o001_p01_nurse_revi__DpYdoXL", + "trial_uri": "trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t028_t028_o001_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "c9ae4564813e1bda7af150b9e15c5fb86b0b5294b47389e5a8741d55d3ecd0dc", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t028_t028_o001_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t028_t028_o001_p01_nurse_revi__DpYdoXL", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:06:10.339268Z", + "finished_at": "2026-07-19T05:10:47.607775Z", + "environment_setup": { + "started_at": "2026-07-19T05:06:10.343941Z", + "finished_at": "2026-07-19T05:06:30.109289Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:06:30.109302Z", + "finished_at": "2026-07-19T05:06:33.934372Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:06:33.934461Z", + "finished_at": "2026-07-19T05:08:37.719649Z" + }, + "verifier": { + "started_at": "2026-07-19T05:08:38.133157Z", + "finished_at": "2026-07-19T05:10:47.048797Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/scorecard.json new file mode 100644 index 00000000..a7c48515 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t028_t028_o001_p01_nurse_revi__DpYdoXL/verifier/scorecard.json @@ -0,0 +1,178 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.46153846153846156, + "passed_checks": 12, + "total_checks": 26, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 0.0, + "md.signed_off": 0.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:nurse_psg_indication_01": 1.0, + "judge.nurse_review:nurse_pathway_01": 0.0, + "judge.nurse_review:nurse_non_osa_01": 0.0, + "judge.nurse_review:nurse_protocol_01": 1.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:md_psg_indication_01": 0.0, + "judge.md_review:md_pathway_01": 0.0, + "judge.md_review:md_psg_necessity_hsat": 0.0, + "judge.md_review:md_psg_necessity_pediatric": 0.0, + "judge.md_review:md_psg_necessity_comorbidity": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:nurse_psg_indication_01": true, + "judge.nurse_review:nurse_pathway_01": false, + "judge.nurse_review:nurse_non_osa_01": false, + "judge.nurse_review:nurse_protocol_01": true, + "judge.nurse_review:recommendation": false, + "judge.md_review:md_psg_indication_01": false, + "judge.md_review:md_pathway_01": false, + "judge.md_review:md_psg_necessity_hsat": false, + "judge.md_review:md_psg_necessity_pediatric": false, + "judge.md_review:md_psg_necessity_comorbidity": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "md.rationale_present", + "md.signed_off", + "outcome.target_status", + "outcome.letter_types", + "judge.nurse_review:nurse_pathway_01", + "judge.nurse_review:nurse_non_osa_01", + "judge.nurse_review:recommendation", + "judge.md_review:md_psg_indication_01", + "judge.md_review:md_pathway_01", + "judge.md_review:md_psg_necessity_hsat", + "judge.md_review:md_psg_necessity_pediatric", + "judge.md_review:md_psg_necessity_comorbidity", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.denial_rationale_present", + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": true, + "md.rationale_present": false, + "md.signed_off": false, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 9, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-0EAA0F45" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..fb6289aa Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/result.json new file mode 100644 index 00000000..c02d8d08 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/result.json @@ -0,0 +1,109 @@ +{ + "id": "38e1687f-c6fa-4109-a631-7d378c7257d1", + "task_name": "actava-ai/pa_t029_t029_o001_p01_nurse_review_payer", + "trial_name": "pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf", + "trial_uri": "trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t029_t029_o001_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "3cd024875255776b30fda47435a92275f2de099f4b778fb4a0e49559594ab6fc", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t029_t029_o001_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:15:55.377131Z", + "finished_at": "2026-08-21T09:22:25.934962Z", + "environment_setup": { + "started_at": "2026-08-21T09:15:55.417811Z", + "finished_at": "2026-08-21T09:16:15.234799Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:16:15.234816Z", + "finished_at": "2026-08-21T09:16:18.747543Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:16:18.747602Z", + "finished_at": "2026-08-21T09:20:42.241339Z" + }, + "verifier": { + "started_at": "2026-08-21T09:20:42.659784Z", + "finished_at": "2026-08-21T09:22:25.279533Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/scorecard.json new file mode 100644 index 00000000..db8a3959 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t029_t029_o001_p01_nurse_revi__uy8y2Xf/verifier/scorecard.json @@ -0,0 +1,142 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 31, + "total_checks": 31, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": 1.0, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:surgical_plan_proven_procedure": 1.0, + "judge.nurse_review:recommendation": 1.0, + "judge.md_review:spinal_fusion_proven_procedure": 1.0, + "judge.md_review:qualifying_diagnosis_imaging_correlation": 1.0, + "judge.md_review:radicular_signs": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:surgical_plan_proven_procedure": true, + "judge.nurse_review:recommendation": true, + "judge.md_review:spinal_fusion_proven_procedure": true, + "judge.md_review:qualifying_diagnosis_imaging_correlation": true, + "judge.md_review:radicular_signs": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true + }, + "passed_count": 5, + "total_count": 5, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-4BCF1471" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..d4fb6427 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/result.json new file mode 100644 index 00000000..621adc3a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/result.json @@ -0,0 +1,103 @@ +{ + "id": "0531b99d-f236-4774-94ba-9217be18e1a1", + "task_name": "actava-ai/pa_t030_t030_o002_p01_triage_payer", + "trial_name": "pa_t030_t030_o002_p01_triage_pay__ZdUMdis", + "trial_uri": "trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t030_t030_o002_p01_triage_payer" + }, + "source": null, + "task_checksum": "569595a3233cffcbc06931b14d118b0454ed2fa2317608e1457bd29e0d2da44c", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t030_t030_o002_p01_triage_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t030_t030_o002_p01_triage_pay__ZdUMdis", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:07:24.425207Z", + "finished_at": "2026-07-19T05:11:27.365859Z", + "environment_setup": { + "started_at": "2026-07-19T05:07:24.427474Z", + "finished_at": "2026-07-19T05:07:44.249197Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:07:44.249210Z", + "finished_at": "2026-07-19T05:07:48.286598Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:07:48.286652Z", + "finished_at": "2026-07-19T05:09:45.311025Z" + }, + "verifier": { + "started_at": "2026-07-19T05:09:45.712064Z", + "finished_at": "2026-07-19T05:11:26.779802Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/scorecard.json new file mode 100644 index 00000000..d05ac742 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t030_t030_o002_p01_triage_pay__ZdUMdis/verifier/scorecard.json @@ -0,0 +1,187 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6571428571428571, + "passed_checks": 23, + "total_checks": 35, + "check_scores": { + "md.decision_exists": 0.0, + "md.rationale_present": null, + "md.signed_off": null, + "md.denial_rationale_present": null, + "md.audit": 0.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.determination_decision_matches": 0.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:nurse_dx_imaging": 1.0, + "judge.nurse_review:nurse_severity_match": 0.0, + "judge.nurse_review:nurse_conservative_tx": 1.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:md_dx_imaging_confirmation": 0.0, + "judge.md_review:md_severity_procedure_match": 0.0, + "judge.md_review:md_conservative_treatment_adequacy": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.denial_rationale_present": "not_applicable", + "md.audit": false, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:nurse_dx_imaging": true, + "judge.nurse_review:nurse_severity_match": false, + "judge.nurse_review:nurse_conservative_tx": true, + "judge.nurse_review:recommendation": false, + "judge.md_review:md_dx_imaging_confirmation": false, + "judge.md_review:md_severity_procedure_match": false, + "judge.md_review:md_conservative_treatment_adequacy": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "md.decision_exists", + "md.audit", + "outcome.target_status", + "outcome.letter_types", + "outcome.determination_decision_matches", + "judge.nurse_review:nurse_severity_match", + "judge.nurse_review:recommendation", + "judge.md_review:md_dx_imaging_confirmation", + "judge.md_review:md_severity_procedure_match", + "judge.md_review:md_conservative_treatment_adequacy", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.rationale_present", + "md.signed_off", + "md.denial_rationale_present" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.denial_rationale_present": "not_applicable", + "md.audit": false + }, + "passed_count": 0, + "total_count": 2, + "not_applicable_count": 3, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.determination_decision_matches": false, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 12, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 3, + "total_count": 3, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-226FC81E" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..908504ab Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/result.json new file mode 100644 index 00000000..5f70ade6 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/result.json @@ -0,0 +1,109 @@ +{ + "id": "b5cfd859-4b22-4b89-a762-7224cebdf3e6", + "task_name": "actava-ai/pa_t031_t031_o001_p01_p2p_payer", + "trial_name": "pa_t031_t031_o001_p01_p2p_payer__EQniiEZ", + "trial_uri": "trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t031_t031_o001_p01_p2p_payer" + }, + "source": null, + "task_checksum": "aa96b597aa62b858b7fae16e143809cf14ba99845b66cd0c868faefa6007f64c", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t031_t031_o001_p01_p2p_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t031_t031_o001_p01_p2p_payer__EQniiEZ", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:17:50.025671Z", + "finished_at": "2026-08-21T09:24:23.428303Z", + "environment_setup": { + "started_at": "2026-08-21T09:17:50.029460Z", + "finished_at": "2026-08-21T09:18:09.860470Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:18:09.860490Z", + "finished_at": "2026-08-21T09:18:13.385045Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:18:13.385120Z", + "finished_at": "2026-08-21T09:21:19.472378Z" + }, + "verifier": { + "started_at": "2026-08-21T09:21:19.870025Z", + "finished_at": "2026-08-21T09:24:22.763418Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/scorecard.json new file mode 100644 index 00000000..08dfcee4 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t031_t031_o001_p01_p2p_payer__EQniiEZ/verifier/scorecard.json @@ -0,0 +1,179 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 38, + "total_checks": 38, + "check_scores": { + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "outcome.p2p_scheduled": 1.0, + "outcome.p2p_has_notes": 1.0, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 1.0, + "p2p.session_exists": 1.0, + "p2p.offer_written": 1.0, + "p2p.slot_response_written": 1.0, + "p2p.confirmation_written": 1.0, + "p2p.event_opened": 1.0, + "p2p.summary_written": null, + "p2p.turns_recorded": 1.0, + "p2p.result_recorded": 1.0, + "p2p.audit": 1.0, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": 1.0, + "p2p.required_file_sequence_ordered": 1.0, + "p2p.audit_chronology": 1.0, + "p2p.thread_state_consistent": 1.0, + "p2p.determination_matches_contract": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.p2p:primary_issue": 1.0, + "judge.p2p:resolution_point_01": 1.0, + "judge.p2p:outcome_selection": 1.0, + "judge.p2p:summary_fidelity": 1.0, + "judge.p2p:forbidden_shortcuts": 1.0, + "judge.p2p:payer_policy_alignment": 1.0, + "judge.p2p:provider_policy_alignment": 1.0 + }, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true, + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.p2p:primary_issue": true, + "judge.p2p:resolution_point_01": true, + "judge.p2p:outcome_selection": true, + "judge.p2p:summary_fidelity": true, + "judge.p2p:forbidden_shortcuts": true, + "judge.p2p:payer_policy_alignment": true, + "judge.p2p:provider_policy_alignment": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "p2p.summary_written", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot" + ], + "stages": { + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true + }, + "passed_count": 15, + "total_count": 15, + "not_applicable_count": 0, + "details": {} + }, + "p2p": { + "passed": true, + "checks": { + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 7, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-4F2B1688" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..b88496f1 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/result.json new file mode 100644 index 00000000..ccbcee0b --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/result.json @@ -0,0 +1,109 @@ +{ + "id": "9cb3455b-203c-4f9a-94ab-f5b489367268", + "task_name": "actava-ai/pa_t032_t032_o002_p01_nurse_review_payer", + "trial_name": "pa_t032_t032_o002_p01_nurse_revi__TeqsHMe", + "trial_uri": "trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t032_t032_o002_p01_nurse_review_payer" + }, + "source": null, + "task_checksum": "ec5343539e1a4e0903fee7ff46e81885d32be6e71b7333804c34b33ab13758df", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t032_t032_o002_p01_nurse_review_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t032_t032_o002_p01_nurse_revi__TeqsHMe", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:22:25.940269Z", + "finished_at": "2026-08-21T09:29:37.775207Z", + "environment_setup": { + "started_at": "2026-08-21T09:22:25.969490Z", + "finished_at": "2026-08-21T09:22:45.857605Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:22:45.857621Z", + "finished_at": "2026-08-21T09:22:49.384442Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:22:49.384498Z", + "finished_at": "2026-08-21T09:27:21.031205Z" + }, + "verifier": { + "started_at": "2026-08-21T09:27:21.436324Z", + "finished_at": "2026-08-21T09:29:37.150311Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/scorecard.json new file mode 100644 index 00000000..e4f43525 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t032_t032_o002_p01_nurse_revi__TeqsHMe/verifier/scorecard.json @@ -0,0 +1,239 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 52, + "total_checks": 52, + "check_scores": { + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": null, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.p2p_outcome_matches": 1.0, + "outcome.p2p_has_notes": 1.0, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 1.0, + "p2p.session_exists": 1.0, + "p2p.offer_written": 1.0, + "p2p.slot_response_written": 1.0, + "p2p.confirmation_written": 1.0, + "p2p.event_opened": 1.0, + "p2p.summary_written": null, + "p2p.turns_recorded": 1.0, + "p2p.result_recorded": 1.0, + "p2p.audit": 1.0, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": 1.0, + "p2p.required_file_sequence_ordered": 1.0, + "p2p.audit_chronology": 1.0, + "p2p.thread_state_consistent": 1.0, + "p2p.determination_matches_contract": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:fda_device_compliance": 1.0, + "judge.nurse_review:clinical_diagnosis_eligibility": 1.0, + "judge.nurse_review:conservative_treatment_interqual": 1.0, + "judge.nurse_review:hybrid_exclusion_nurse": 1.0, + "judge.nurse_review:recommendation": 1.0, + "judge.md_review:hybrid_exclusion_md": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0, + "judge.p2p:primary_issue": 1.0, + "judge.p2p:resolution_point_01": 1.0, + "judge.p2p:outcome_selection": 1.0, + "judge.p2p:summary_fidelity": 1.0, + "judge.p2p:forbidden_shortcuts": 1.0, + "judge.p2p:payer_policy_alignment": 1.0, + "judge.p2p:provider_policy_alignment": 1.0 + }, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true, + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:fda_device_compliance": true, + "judge.nurse_review:clinical_diagnosis_eligibility": true, + "judge.nurse_review:conservative_treatment_interqual": true, + "judge.nurse_review:hybrid_exclusion_nurse": true, + "judge.nurse_review:recommendation": true, + "judge.md_review:hybrid_exclusion_md": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true, + "judge.p2p:primary_issue": true, + "judge.p2p:resolution_point_01": true, + "judge.p2p:outcome_selection": true, + "judge.p2p:summary_fidelity": true, + "judge.p2p:forbidden_shortcuts": true, + "judge.p2p:payer_policy_alignment": true, + "judge.p2p:provider_policy_alignment": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "md.denial_rationale_present", + "p2p.summary_written", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot" + ], + "stages": { + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": "not_applicable", + "md.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 1, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "p2p": { + "passed": true, + "checks": { + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 7, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-8116C2B4" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..50587013 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/result.json new file mode 100644 index 00000000..bd46c3ef --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/result.json @@ -0,0 +1,103 @@ +{ + "id": "d9fe2c80-3d1d-47f0-9155-909949380616", + "task_name": "actava-ai/pa_t033_t033_o001_p01_triage_payer", + "trial_name": "pa_t033_t033_o001_p01_triage_pay__YRumf83", + "trial_uri": "trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t033_t033_o001_p01_triage_payer" + }, + "source": null, + "task_checksum": "6bd13d06c009256d3056a360d0b50a7ac8ad7de3f456ea6201d387b7f923e3f7", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t033_t033_o001_p01_triage_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t033_t033_o001_p01_triage_pay__YRumf83", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:12:40.288018Z", + "finished_at": "2026-07-19T05:17:13.739090Z", + "environment_setup": { + "started_at": "2026-07-19T05:12:40.290520Z", + "finished_at": "2026-07-19T05:13:00.138221Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:13:00.138239Z", + "finished_at": "2026-07-19T05:13:04.067256Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:13:04.067325Z", + "finished_at": "2026-07-19T05:14:50.374351Z" + }, + "verifier": { + "started_at": "2026-07-19T05:14:50.770935Z", + "finished_at": "2026-07-19T05:17:13.135548Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/scorecard.json new file mode 100644 index 00000000..16430e98 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t033_t033_o001_p01_triage_pay__YRumf83/verifier/scorecard.json @@ -0,0 +1,208 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.45161290322580644, + "passed_checks": 14, + "total_checks": 31, + "check_scores": { + "md.decision_exists": 0.0, + "md.rationale_present": null, + "md.signed_off": null, + "md.audit": 0.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.letter_types": 0.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.gold_card_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:nurse_qualifying_amputation": 0.0, + "judge.nurse_review:nurse_prosthetic_replaces_limb": 0.0, + "judge.nurse_review:nurse_functional_restoration": 0.0, + "judge.nurse_review:recommendation": 0.0, + "judge.md_review:md_indication_qualifying_amputation": 0.0, + "judge.md_review:md_amputation_level_classification": 0.0, + "judge.md_review:md_functional_assessment": 0.0, + "judge.md_review:md_training_willingness": 0.0, + "judge.md_review:md_myoelectric_operability": 0.0, + "judge.md_review:md_device_prosthetist_eval": 0.0, + "judge.md_review:md_device_physician_order": 0.0, + "judge.md_review:final_decision": 0.0, + "judge.md_review:decision_rationale": 0.0 + }, + "checks": { + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.audit": false, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:nurse_qualifying_amputation": false, + "judge.nurse_review:nurse_prosthetic_replaces_limb": false, + "judge.nurse_review:nurse_functional_restoration": false, + "judge.nurse_review:recommendation": false, + "judge.md_review:md_indication_qualifying_amputation": false, + "judge.md_review:md_amputation_level_classification": false, + "judge.md_review:md_functional_assessment": false, + "judge.md_review:md_training_willingness": false, + "judge.md_review:md_myoelectric_operability": false, + "judge.md_review:md_device_prosthetist_eval": false, + "judge.md_review:md_device_physician_order": false, + "judge.md_review:final_decision": false, + "judge.md_review:decision_rationale": false + }, + "failed_checks": [ + "md.decision_exists", + "md.audit", + "outcome.target_status", + "outcome.letter_types", + "judge.nurse_review:nurse_qualifying_amputation", + "judge.nurse_review:nurse_prosthetic_replaces_limb", + "judge.nurse_review:nurse_functional_restoration", + "judge.nurse_review:recommendation", + "judge.md_review:md_indication_qualifying_amputation", + "judge.md_review:md_amputation_level_classification", + "judge.md_review:md_functional_assessment", + "judge.md_review:md_training_willingness", + "judge.md_review:md_myoelectric_operability", + "judge.md_review:md_device_prosthetist_eval", + "judge.md_review:md_device_physician_order", + "judge.md_review:final_decision", + "judge.md_review:decision_rationale" + ], + "not_applicable_checks": [ + "md.rationale_present", + "md.signed_off", + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination" + ], + "stages": { + "md_review": { + "passed": false, + "checks": { + "md.decision_exists": false, + "md.rationale_present": "not_applicable", + "md.signed_off": "not_applicable", + "md.audit": false + }, + "passed_count": 0, + "total_count": 2, + "not_applicable_count": 2, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.letter_types": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true + }, + "passed_count": 2, + "total_count": 4, + "not_applicable_count": 9, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.gold_card_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 6, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-7C158C7A" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..158d17fd Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/result.json new file mode 100644 index 00000000..69a28037 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/result.json @@ -0,0 +1,109 @@ +{ + "id": "02b6b181-bf0d-485b-9ec0-7db221f3958a", + "task_name": "actava-ai/pa_t034_t034_o002_p01_intake_payer", + "trial_name": "pa_t034_t034_o002_p01_intake_pay__R9K5RNy", + "trial_uri": "trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t034_t034_o002_p01_intake_payer" + }, + "source": null, + "task_checksum": "3e4e61d189a090c0dd8e57caeecc6cb2ffcec304ce2c796a8f19e525596fff0a", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t034_t034_o002_p01_intake_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t034_t034_o002_p01_intake_pay__R9K5RNy", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "08713c15-56ea-4e45-b51e-762c279d7a01" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:37:10.325436Z", + "finished_at": "2026-08-21T09:42:29.531030Z", + "environment_setup": { + "started_at": "2026-08-21T09:37:10.413894Z", + "finished_at": "2026-08-21T09:37:31.317112Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:37:31.317152Z", + "finished_at": "2026-08-21T09:37:35.129572Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:37:35.129661Z", + "finished_at": "2026-08-21T09:41:13.370547Z" + }, + "verifier": { + "started_at": "2026-08-21T09:41:13.773860Z", + "finished_at": "2026-08-21T09:42:28.924193Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/scorecard.json new file mode 100644 index 00000000..7cf4060b --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t034_t034_o002_p01_intake_pay__R9K5RNy/verifier/scorecard.json @@ -0,0 +1,197 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 44, + "total_checks": 44, + "check_scores": { + "intake.case_exists": 1.0, + "intake.diagnoses_match": 1.0, + "intake.line_items_populated": 1.0, + "intake.decision_member_eligible": 1.0, + "intake.decision_service_covered": 1.0, + "intake.audit": 1.0, + "md.decision_exists": 1.0, + "md.rationale_present": 1.0, + "md.signed_off": 1.0, + "md.denial_rationale_present": 1.0, + "md.audit": 1.0, + "nurse.reviews_exist": 1.0, + "nurse.recommendation_exists": 1.0, + "nurse.no_unauthorized_denial": 1.0, + "nurse.audit": 1.0, + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.determination_decision_matches": 1.0, + "outcome.determination_has_rationale": 1.0, + "outcome.clean_determination": 1.0, + "triage.record_exists": 1.0, + "triage.disposition_matches": 1.0, + "triage.urgency_matches": 1.0, + "triage.route_matches": 1.0, + "triage.audit": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.nurse_review:vns_aed_trial_failure": 1.0, + "judge.nurse_review:recommendation": 1.0, + "judge.md_review:vns_aed_trial_failure": 1.0, + "judge.md_review:vns_surgical_candidacy": 1.0, + "judge.md_review:vns_no_vagotomy": 1.0, + "judge.md_review:vns_all_criteria_check": 1.0, + "judge.md_review:vns_disputed_criterion_identification": 1.0, + "judge.md_review:final_decision": 1.0, + "judge.md_review:decision_rationale": 1.0 + }, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": true, + "intake.line_items_populated": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true, + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true, + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true, + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true, + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.route_matches": true, + "triage.audit": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.nurse_review:vns_aed_trial_failure": true, + "judge.nurse_review:recommendation": true, + "judge.md_review:vns_aed_trial_failure": true, + "judge.md_review:vns_surgical_candidacy": true, + "judge.md_review:vns_no_vagotomy": true, + "judge.md_review:vns_all_criteria_check": true, + "judge.md_review:vns_disputed_criterion_identification": true, + "judge.md_review:final_decision": true, + "judge.md_review:decision_rationale": true + }, + "failed_checks": [], + "not_applicable_checks": [], + "stages": { + "intake": { + "passed": true, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": true, + "intake.line_items_populated": true, + "intake.decision_member_eligible": true, + "intake.decision_service_covered": true, + "intake.audit": true + }, + "passed_count": 6, + "total_count": 6, + "not_applicable_count": 0, + "details": {} + }, + "md_review": { + "passed": true, + "checks": { + "md.decision_exists": true, + "md.rationale_present": true, + "md.signed_off": true, + "md.denial_rationale_present": true, + "md.audit": true + }, + "passed_count": 5, + "total_count": 5, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "nurse_review": { + "passed": true, + "checks": { + "nurse.reviews_exist": true, + "nurse.recommendation_exists": true, + "nurse.no_unauthorized_denial": true, + "nurse.audit": true + }, + "passed_count": 4, + "total_count": 4, + "not_applicable_count": 0, + "details": { + "criteria": [] + } + }, + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.determination_decision_matches": true, + "outcome.determination_has_rationale": true, + "outcome.clean_determination": true + }, + "passed_count": 13, + "total_count": 13, + "not_applicable_count": 0, + "details": {} + }, + "triage": { + "passed": true, + "checks": { + "triage.record_exists": true, + "triage.disposition_matches": true, + "triage.urgency_matches": true, + "triage.route_matches": true, + "triage.audit": true + }, + "passed_count": 5, + "total_count": 5, + "not_applicable_count": 0, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-50D9ECA8" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..61f97d9b Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/result.json new file mode 100644 index 00000000..1165ab98 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/result.json @@ -0,0 +1,103 @@ +{ + "id": "103e9155-0990-41fb-8908-9fd885283da9", + "task_name": "actava-ai/pa_t035_t035_o001_p01_intake_payer", + "trial_name": "pa_t035_t035_o001_p01_intake_pay__biZYjWd", + "trial_uri": "trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t035_t035_o001_p01_intake_payer" + }, + "source": null, + "task_checksum": "ef43b4ecb4356fbf9494b3e96e8ba33488420b336f628bc6da3ae6550be366cf", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t035_t035_o001_p01_intake_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t035_t035_o001_p01_intake_pay__biZYjWd", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0609", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": {} + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "07050be1-36ef-480e-896f-a6b03982bc45" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0609", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 0.0 + } + }, + "exception_info": null, + "started_at": "2026-07-19T05:16:45.725603Z", + "finished_at": "2026-07-19T05:18:58.421177Z", + "environment_setup": { + "started_at": "2026-07-19T05:16:45.727912Z", + "finished_at": "2026-07-19T05:17:05.517422Z" + }, + "agent_setup": { + "started_at": "2026-07-19T05:17:05.517446Z", + "finished_at": "2026-07-19T05:17:10.655966Z" + }, + "agent_execution": { + "started_at": "2026-07-19T05:17:10.656019Z", + "finished_at": "2026-07-19T05:18:53.901623Z" + }, + "verifier": { + "started_at": "2026-07-19T05:18:54.292668Z", + "finished_at": "2026-07-19T05:18:57.830795Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/reward.json new file mode 100644 index 00000000..3da0d9aa --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 0.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/scorecard.json new file mode 100644 index 00000000..494462f7 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t035_t035_o001_p01_intake_pay__biZYjWd/verifier/scorecard.json @@ -0,0 +1,121 @@ +{ + "binary_reward": 0.0, + "fractional_reward": 0.6666666666666666, + "passed_checks": 8, + "total_checks": 12, + "check_scores": { + "intake.case_exists": 1.0, + "intake.diagnoses_match": 0.0, + "intake.provider_npi_matches": 0.0, + "intake.line_items_populated": 1.0, + "intake.decision_request_complete": 1.0, + "intake.decision_member_eligible": 0.0, + "intake.audit": 1.0, + "outcome.target_status": 0.0, + "outcome.determination_exists": null, + "outcome.determination_source_exists": null, + "outcome.source_recommendation_consistent": null, + "outcome.review_decision_exists": null, + "outcome.review_decision_matches": null, + "outcome.terminal_transition_exists": null, + "outcome.determination_stage_trace": null, + "outcome.determination_finalize_audit": null, + "outcome.clean_determination": null, + "outcome.audit_auth_number": 1.0, + "outcome.authorization_number": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0 + }, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": false, + "intake.audit": true, + "outcome.target_status": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "failed_checks": [ + "intake.diagnoses_match", + "intake.provider_npi_matches", + "intake.decision_member_eligible", + "outcome.target_status" + ], + "not_applicable_checks": [ + "outcome.determination_exists", + "outcome.determination_source_exists", + "outcome.source_recommendation_consistent", + "outcome.review_decision_exists", + "outcome.review_decision_matches", + "outcome.terminal_transition_exists", + "outcome.determination_stage_trace", + "outcome.determination_finalize_audit", + "outcome.clean_determination" + ], + "stages": { + "intake": { + "passed": false, + "checks": { + "intake.case_exists": true, + "intake.diagnoses_match": false, + "intake.provider_npi_matches": false, + "intake.line_items_populated": true, + "intake.decision_request_complete": true, + "intake.decision_member_eligible": false, + "intake.audit": true + }, + "passed_count": 4, + "total_count": 7, + "not_applicable_count": 0, + "details": {} + }, + "outcome": { + "passed": false, + "checks": { + "outcome.target_status": false, + "outcome.determination_exists": "not_applicable", + "outcome.determination_source_exists": "not_applicable", + "outcome.source_recommendation_consistent": "not_applicable", + "outcome.review_decision_exists": "not_applicable", + "outcome.review_decision_matches": "not_applicable", + "outcome.terminal_transition_exists": "not_applicable", + "outcome.determination_stage_trace": "not_applicable", + "outcome.determination_finalize_audit": "not_applicable", + "outcome.clean_determination": "not_applicable", + "outcome.audit_auth_number": true, + "outcome.authorization_number": true + }, + "passed_count": 2, + "total_count": 3, + "not_applicable_count": 9, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-516EF322" + } + } + } +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/agent/trajectory.jsonl.zst b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/agent/trajectory.jsonl.zst new file mode 100644 index 00000000..c8fcffd3 Binary files /dev/null and b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/agent/trajectory.jsonl.zst differ diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/result.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/result.json new file mode 100644 index 00000000..f19f060a --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/result.json @@ -0,0 +1,109 @@ +{ + "id": "ac26d73f-a6d7-430c-8827-e93f92d888e4", + "task_name": "actava-ai/pa_t036_t036_o002_p01_p2p_payer", + "trial_name": "pa_t036_t036_o002_p01_p2p_payer__RetbaNp", + "trial_uri": "trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp", + "task_id": { + "path": "data/prior_auth_um/tasks/pa_t036_t036_o002_p01_p2p_payer" + }, + "source": null, + "task_checksum": "8ddf6ed257293450c50b88a1f5227978052edf2e65effe7339543c159f71c195", + "config": { + "task": { + "path": "data/prior_auth_um/tasks/pa_t036_t036_o002_p01_p2p_payer", + "git_url": null, + "git_commit_id": null, + "name": null, + "ref": null, + "overwrite": false, + "download_dir": null, + "source": null + }, + "trial_name": "pa_t036_t036_o002_p01_p2p_payer__RetbaNp", + "trials_dir": "trials/pa_um", + "timeout_multiplier": 2.0, + "agent_timeout_multiplier": null, + "verifier_timeout_multiplier": null, + "agent_setup_timeout_multiplier": null, + "environment_build_timeout_multiplier": null, + "agent": { + "name": null, + "import_path": "chi_bench.experiment.agents.hermes_harness:HermesHarness", + "model_name": "openai/qwen3.5_27b_sft_0620", + "override_timeout_sec": null, + "override_setup_timeout_sec": null, + "max_timeout_sec": null, + "kwargs": {}, + "env": { + "OPENAI_API_KEY": "${OPENAI_API_KEY}", + "OPENAI_BASE_URL": "https://solotvvrh0bnp02qhk7g8.apigateway-cn-beijing.volceapi.com/v1", + "ANTHROPIC_API_KEY": "${ANTHROPIC_API_KEY}", + "CHI_BENCH_PATIENT_SIM_MODEL": "claude-opus-4-7", + "CHI_BENCH_PATIENT_SIM_BASE_URL": "https://api.gpugeek.com" + } + }, + "environment": { + "type": null, + "import_path": "chi_bench.experiment.docker_env:ChiBenchDockerEnvironment", + "force_build": false, + "delete": true, + "override_cpus": null, + "override_memory_mb": null, + "override_storage_mb": null, + "override_gpus": null, + "suppress_override_warnings": false, + "mounts_json": null, + "env": {}, + "kwargs": {} + }, + "verifier": { + "override_timeout_sec": null, + "max_timeout_sec": null, + "env": {}, + "disable": false + }, + "artifacts": [], + "job_id": "babfe0f6-897d-448d-b6f3-87fa9dfcbf9f" + }, + "agent_info": { + "name": "hermes", + "version": "Hermes Agent v0.17.0 (2026.6.19)\nProject: /usr/local/lib/hermes-agent/venv/lib/python3.12/site-packages\nPython: 3.12.13\nOpenAI SDK: 2.24.0\nUpdate available: 1 commit behind — run 'uv pip install --upgrade hermes-agent'", + "model_info": { + "name": "qwen3.5_27b_sft_0620", + "provider": "openai" + } + }, + "agent_result": { + "n_input_tokens": 0, + "n_cache_tokens": null, + "n_output_tokens": 0, + "cost_usd": null, + "rollout_details": null, + "metadata": null + }, + "verifier_result": { + "rewards": { + "reward": 1.0 + } + }, + "exception_info": null, + "started_at": "2026-08-21T09:24:23.433184Z", + "finished_at": "2026-08-21T09:29:37.080774Z", + "environment_setup": { + "started_at": "2026-08-21T09:24:23.439154Z", + "finished_at": "2026-08-21T09:24:43.289994Z" + }, + "agent_setup": { + "started_at": "2026-08-21T09:24:43.290017Z", + "finished_at": "2026-08-21T09:24:47.457641Z" + }, + "agent_execution": { + "started_at": "2026-08-21T09:24:47.457718Z", + "finished_at": "2026-08-21T09:27:35.497950Z" + }, + "verifier": { + "started_at": "2026-08-21T09:27:35.902147Z", + "finished_at": "2026-08-21T09:29:36.328469Z" + }, + "step_results": null +} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/reward.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/reward.json new file mode 100644 index 00000000..d9a8cb44 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/reward.json @@ -0,0 +1 @@ +{"reward": 1.0} \ No newline at end of file diff --git a/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/scorecard.json b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/scorecard.json new file mode 100644 index 00000000..0b2d7791 --- /dev/null +++ b/benchmarks/chi-bench/submissions/2026-09-03-u2med-u2medfellow-zzq/trials/pa_um/pa_t036_t036_o002_p01_p2p_payer__RetbaNp/verifier/scorecard.json @@ -0,0 +1,176 @@ +{ + "binary_reward": 1.0, + "fractional_reward": 1.0, + "passed_checks": 37, + "total_checks": 37, + "check_scores": { + "outcome.target_status": 1.0, + "outcome.letter_types": 1.0, + "outcome.determination_exists": 1.0, + "outcome.review_decision_exists": 1.0, + "outcome.determination_stage_trace": 1.0, + "outcome.determination_finalize_audit": 1.0, + "outcome.terminal_transition_exists": 1.0, + "outcome.determination_source_exists": 1.0, + "outcome.source_recommendation_consistent": 1.0, + "outcome.review_decision_matches": 1.0, + "outcome.p2p_outcome_matches": 1.0, + "outcome.p2p_scheduled": 1.0, + "outcome.p2p_has_notes": 1.0, + "outcome.clean_determination": 1.0, + "p2p.request_exists": 1.0, + "p2p.session_exists": 1.0, + "p2p.offer_written": 1.0, + "p2p.slot_response_written": 1.0, + "p2p.confirmation_written": 1.0, + "p2p.event_opened": 1.0, + "p2p.summary_written": null, + "p2p.turns_recorded": 1.0, + "p2p.result_recorded": 1.0, + "p2p.audit": 1.0, + "p2p.offer_slots_match_contract": null, + "p2p.offer_artifact_slots_match_contract": null, + "p2p.selected_slot_present": null, + "p2p.selected_slot_in_offered_slots": null, + "p2p.selected_slot_matches_rule": null, + "p2p.confirmation_matches_selected_slot": null, + "p2p.required_file_sequence_present": 1.0, + "p2p.required_file_sequence_ordered": 1.0, + "p2p.audit_chronology": 1.0, + "p2p.thread_state_consistent": 1.0, + "p2p.determination_matches_contract": 1.0, + "cross.forbidden_mutations": 1.0, + "cross.forward_transitions": 1.0, + "judge.p2p:primary_issue": 1.0, + "judge.p2p:resolution_point_01": 1.0, + "judge.p2p:outcome_selection": 1.0, + "judge.p2p:summary_fidelity": 1.0, + "judge.p2p:forbidden_shortcuts": 1.0, + "judge.p2p:payer_policy_alignment": 1.0, + "judge.p2p:provider_policy_alignment": 1.0 + }, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true, + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true, + "cross.forbidden_mutations": true, + "cross.forward_transitions": true, + "judge.p2p:primary_issue": true, + "judge.p2p:resolution_point_01": true, + "judge.p2p:outcome_selection": true, + "judge.p2p:summary_fidelity": true, + "judge.p2p:forbidden_shortcuts": true, + "judge.p2p:payer_policy_alignment": true, + "judge.p2p:provider_policy_alignment": true + }, + "failed_checks": [], + "not_applicable_checks": [ + "p2p.summary_written", + "p2p.offer_slots_match_contract", + "p2p.offer_artifact_slots_match_contract", + "p2p.selected_slot_present", + "p2p.selected_slot_in_offered_slots", + "p2p.selected_slot_matches_rule", + "p2p.confirmation_matches_selected_slot" + ], + "stages": { + "outcome": { + "passed": true, + "checks": { + "outcome.target_status": true, + "outcome.letter_types": true, + "outcome.determination_exists": true, + "outcome.review_decision_exists": true, + "outcome.determination_stage_trace": true, + "outcome.determination_finalize_audit": true, + "outcome.terminal_transition_exists": true, + "outcome.determination_source_exists": true, + "outcome.source_recommendation_consistent": true, + "outcome.review_decision_matches": true, + "outcome.p2p_outcome_matches": true, + "outcome.p2p_scheduled": true, + "outcome.p2p_has_notes": true, + "outcome.clean_determination": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 0, + "details": {} + }, + "p2p": { + "passed": true, + "checks": { + "p2p.request_exists": true, + "p2p.session_exists": true, + "p2p.offer_written": true, + "p2p.slot_response_written": true, + "p2p.confirmation_written": true, + "p2p.event_opened": true, + "p2p.summary_written": "not_applicable", + "p2p.turns_recorded": true, + "p2p.result_recorded": true, + "p2p.audit": true, + "p2p.offer_slots_match_contract": "not_applicable", + "p2p.offer_artifact_slots_match_contract": "not_applicable", + "p2p.selected_slot_present": "not_applicable", + "p2p.selected_slot_in_offered_slots": "not_applicable", + "p2p.selected_slot_matches_rule": "not_applicable", + "p2p.confirmation_matches_selected_slot": "not_applicable", + "p2p.required_file_sequence_present": true, + "p2p.required_file_sequence_ordered": true, + "p2p.audit_chronology": true, + "p2p.thread_state_consistent": true, + "p2p.determination_matches_contract": true + }, + "passed_count": 14, + "total_count": 14, + "not_applicable_count": 7, + "details": {} + }, + "cross_stage": { + "passed": true, + "checks": { + "cross.forbidden_mutations": true, + "cross.forward_transitions": true + }, + "passed_count": 2, + "total_count": 2, + "not_applicable_count": 0, + "details": { + "case_id": "CASE-39B10A18" + } + } + } +} \ No newline at end of file