diff --git a/.changeset/pass-rate-excludes-sampled-out-items.md b/.changeset/pass-rate-excludes-sampled-out-items.md new file mode 100644 index 000000000..3c44a998a --- /dev/null +++ b/.changeset/pass-rate-excludes-sampled-out-items.md @@ -0,0 +1,5 @@ +--- +"@voltagent/evals": patch +--- + +Fixed experiment `summary.passRate` counting items whose scorers were all sampled out as passes. Those items have no evaluation evidence, but `evaluateItemStatus()` returns `passed` for them, so lowering scorer sampling pushed the pass rate toward 1 and a `{ type: "passRate" }` criterion could pass on results that were never scored. Sampled-out items are now excluded from the pass rate, matching how skipped scorer results are already excluded from a scorer's own pass rate; a run with nothing evaluated reports `passRate: null`. `successCount`, `failureCount`, `errorCount`, and `skippedCount` are unchanged. diff --git a/packages/evals/src/experiment/aggregator.ts b/packages/evals/src/experiment/aggregator.ts index 74faf5e0d..684ed5fa0 100644 --- a/packages/evals/src/experiment/aggregator.ts +++ b/packages/evals/src/experiment/aggregator.ts @@ -32,6 +32,8 @@ export interface ExperimentAggregatorState { failureCount: number; errorCount: number; skippedCount: number; + evaluatedCount: number; + evaluatedSuccessCount: number; globalScoreSum: number; globalScoreCount: number; scorers: Map; @@ -47,6 +49,8 @@ export function createAggregatorState(totalHint?: number): ExperimentAggregatorS failureCount: 0, errorCount: 0, skippedCount: 0, + evaluatedCount: 0, + evaluatedSuccessCount: 0, globalScoreSum: 0, globalScoreCount: 0, scorers: new Map(), @@ -76,8 +80,16 @@ export function recordAggregatorResult( const scores = Object.values(item.scores); const allSkipped = scores.length > 0 && scores.every((score) => score.status === "skipped"); + // An item whose every scorer was skipped carries no evaluation evidence, so + // it stays out of the pass rate, the same way skipped scorer results stay + // out of a scorer's own pass rate. if (allSkipped) { state.skippedCount += 1; + } else { + state.evaluatedCount += 1; + if (item.status === "passed") { + state.evaluatedSuccessCount += 1; + } } for (const score of scores) { @@ -141,7 +153,7 @@ export function buildAggregatorSummary( errorCount: state.errorCount, skippedCount: state.skippedCount, meanScore: state.globalScoreCount > 0 ? state.globalScoreSum / state.globalScoreCount : null, - passRate: completedCount > 0 ? state.successCount / completedCount : null, + passRate: state.evaluatedCount > 0 ? state.evaluatedSuccessCount / state.evaluatedCount : null, startedAt: state.startedAt, scorers: buildScorerAggregates(state), criteria: [], diff --git a/packages/evals/src/experiment/run-experiment.spec.ts b/packages/evals/src/experiment/run-experiment.spec.ts index b69337b36..aef342339 100644 --- a/packages/evals/src/experiment/run-experiment.spec.ts +++ b/packages/evals/src/experiment/run-experiment.spec.ts @@ -9,6 +9,7 @@ const DATASET_ID = "dataset-integration"; const DATASET_VERSION_ID = "dataset-version-integration"; const DATASET_ITEM_1_ID = "11111111-1111-4111-8111-111111111111"; const DATASET_ITEM_2_ID = "22222222-2222-4222-8222-222222222222"; +const DATASET_ITEM_3_ID = "33333333-3333-4333-8333-333333333333"; function createDatasetItems(): ExperimentDatasetItem[] { return [ @@ -142,4 +143,81 @@ describe("runExperiment integration", () => { expect(result.summary.scorers.hede?.passRate).toBe(1); expect(result.summary.scorers).not.toHaveProperty("original-id"); }); + + it("keeps sampled-out items out of the pass rate", async () => { + const experiment = createExperiment({ + id: "run-sampled-out", + dataset: { + items: [ + { id: DATASET_ITEM_1_ID, input: "scored", expected: "scored" }, + { id: DATASET_ITEM_2_ID, input: "sampled-out", expected: "sampled-out" }, + { id: DATASET_ITEM_3_ID, input: "boom", expected: "boom" }, + ], + }, + runner: async ({ item }) => ({ output: item.input }), + scorers: [ + { + id: "sampled", + threshold: 0.5, + scorer: { + id: "sampled", + name: "sampled", + // One item is skipped, exactly as scorer sampling skips it, one + // scores below the threshold and one errors. + scorer: ({ payload }) => { + if (payload.input === "sampled-out") { + return { status: "skipped" }; + } + if (payload.input === "boom") { + return { status: "error", error: new Error("scorer failed") }; + } + return { status: "success", score: 0 }; + }, + }, + }, + ], + }); + + const result = await runExperiment(experiment); + + // Nothing passed: one item scored below its threshold and one errored, so + // the pass rate is 0 of the 2 evaluated items. Before this fix the + // sampled-out item counted as a success and reported passRate 0.33. + expect(result.summary.passRate).toBe(0); + expect(result.summary.skippedCount).toBe(1); + expect(result.summary.errorCount).toBe(1); + }); + + it("reports a null pass rate when every item is sampled out", async () => { + const experiment = createExperiment({ + id: "run-fully-sampled-out", + dataset: { + items: [{ id: DATASET_ITEM_1_ID, input: "hello", expected: "hello" }], + }, + runner: async ({ item }) => ({ output: item.input }), + scorers: [ + { + id: "sampled", + threshold: 0.5, + scorer: { + id: "sampled", + name: "sampled", + sampling: { type: "never" }, + scorer: () => ({ status: "success", score: 1 }), + }, + }, + ], + passCriteria: [ + { type: "passRate", min: 1 }, + { type: "passRate", min: 1, scorerId: "sampled" }, + ], + }); + + const result = await runExperiment(experiment); + + // Both spellings of the same criterion must agree: with no evaluated + // evidence there is no pass rate to meet, so neither can pass. + expect(result.summary.passRate).toBeNull(); + expect(result.summary.criteria.map((entry) => entry.passed)).toEqual([false, false]); + }); }); diff --git a/website/evaluation-docs/offline-evaluations.md b/website/evaluation-docs/offline-evaluations.md index 05aa9871e..83c62612e 100644 --- a/website/evaluation-docs/offline-evaluations.md +++ b/website/evaluation-docs/offline-evaluations.md @@ -164,7 +164,7 @@ While individual scorers determine if each item passes or fails, **pass criteria There are two types of criteria: - **`meanScore`**: Average score across all items must meet a minimum -- **`passRate`**: Percentage of passed items must meet a minimum +- **`passRate`**: Percentage of passed items must meet a minimum. Items whose scorers were all sampled out are not counted, since they carry no evaluation evidence ```ts import { createExperiment } from "@voltagent/evals"; @@ -785,7 +785,7 @@ interface ExperimentSummary { errorCount: number; // items with status "error" skippedCount: number; // items with status "skipped" meanScore?: number | null; - passRate?: number | null; + passRate?: number | null; // passed / evaluated items; null if nothing was evaluated startedAt: number; // Unix timestamp completedAt?: number; // Unix timestamp durationMs?: number;