fix(evals): keep sampled-out items out of the experiment pass rate

.changeset/pass-rate-excludes-sampled-out-items.md

@@ -0,0 +1,5 @@
+---
+"@voltagent/evals": patch
+---
+
+Fixed experiment `summary.passRate` counting items whose scorers were all sampled out as passes. Those items have no evaluation evidence, but `evaluateItemStatus()` returns `passed` for them, so lowering scorer sampling pushed the pass rate toward 1 and a `{ type: "passRate" }` criterion could pass on results that were never scored. Sampled-out items are now excluded from the pass rate, matching how skipped scorer results are already excluded from a scorer's own pass rate; a run with nothing evaluated reports `passRate: null`. `successCount`, `failureCount`, `errorCount`, and `skippedCount` are unchanged.

packages/evals/src/experiment/aggregator.ts

@@ -32,6 +32,8 @@ export interface ExperimentAggregatorState {
   failureCount: number;
   errorCount: number;
   skippedCount: number;
+  evaluatedCount: number;
+  evaluatedSuccessCount: number;
   globalScoreSum: number;
   globalScoreCount: number;
   scorers: Map<string, ScorerAggregateState>;
@@ -47,6 +49,8 @@ export function createAggregatorState(totalHint?: number): ExperimentAggregatorS
     failureCount: 0,
     errorCount: 0,
     skippedCount: 0,
+    evaluatedCount: 0,
+    evaluatedSuccessCount: 0,
     globalScoreSum: 0,
     globalScoreCount: 0,
     scorers: new Map(),
@@ -76,8 +80,16 @@ export function recordAggregatorResult(
 
   const scores = Object.values(item.scores);
   const allSkipped = scores.length > 0 && scores.every((score) => score.status === "skipped");
+  // An item whose every scorer was skipped carries no evaluation evidence, so
+  // it stays out of the pass rate, the same way skipped scorer results stay
+  // out of a scorer's own pass rate.
   if (allSkipped) {
     state.skippedCount += 1;
+  } else {
+    state.evaluatedCount += 1;
+    if (item.status === "passed") {
+      state.evaluatedSuccessCount += 1;
+    }
   }
 
   for (const score of scores) {
@@ -141,7 +153,7 @@ export function buildAggregatorSummary(
     errorCount: state.errorCount,
     skippedCount: state.skippedCount,
     meanScore: state.globalScoreCount > 0 ? state.globalScoreSum / state.globalScoreCount : null,
-    passRate: completedCount > 0 ? state.successCount / completedCount : null,
+    passRate: state.evaluatedCount > 0 ? state.evaluatedSuccessCount / state.evaluatedCount : null,
     startedAt: state.startedAt,
     scorers: buildScorerAggregates(state),
     criteria: [],

packages/evals/src/experiment/run-experiment.spec.ts

@@ -9,6 +9,7 @@ const DATASET_ID = "dataset-integration";
 const DATASET_VERSION_ID = "dataset-version-integration";
 const DATASET_ITEM_1_ID = "11111111-1111-4111-8111-111111111111";
 const DATASET_ITEM_2_ID = "22222222-2222-4222-8222-222222222222";
+const DATASET_ITEM_3_ID = "33333333-3333-4333-8333-333333333333";
 
 function createDatasetItems(): ExperimentDatasetItem[] {
   return [
@@ -142,4 +143,81 @@ describe("runExperiment integration", () => {
     expect(result.summary.scorers.hede?.passRate).toBe(1);
     expect(result.summary.scorers).not.toHaveProperty("original-id");
   });
+
+  it("keeps sampled-out items out of the pass rate", async () => {
+    const experiment = createExperiment({
+      id: "run-sampled-out",
+      dataset: {
+        items: [
+          { id: DATASET_ITEM_1_ID, input: "scored", expected: "scored" },
+          { id: DATASET_ITEM_2_ID, input: "sampled-out", expected: "sampled-out" },
+          { id: DATASET_ITEM_3_ID, input: "boom", expected: "boom" },
+        ],
+      },
+      runner: async ({ item }) => ({ output: item.input }),
+      scorers: [
+        {
+          id: "sampled",
+          threshold: 0.5,
+          scorer: {
+            id: "sampled",
+            name: "sampled",
+            // One item is skipped, exactly as scorer sampling skips it, one
+            // scores below the threshold and one errors.
+            scorer: ({ payload }) => {
+              if (payload.input === "sampled-out") {
+                return { status: "skipped" };
+              }
+              if (payload.input === "boom") {
+                return { status: "error", error: new Error("scorer failed") };
+              }
+              return { status: "success", score: 0 };
+            },
+          },
+        },
+      ],
+    });
+
+    const result = await runExperiment(experiment);
+
+    // Nothing passed: one item scored below its threshold and one errored, so
+    // the pass rate is 0 of the 2 evaluated items. Before this fix the
+    // sampled-out item counted as a success and reported passRate 0.33.
+    expect(result.summary.passRate).toBe(0);
+    expect(result.summary.skippedCount).toBe(1);
+    expect(result.summary.errorCount).toBe(1);
+  });
+
+  it("reports a null pass rate when every item is sampled out", async () => {
+    const experiment = createExperiment({
+      id: "run-fully-sampled-out",
+      dataset: {
+        items: [{ id: DATASET_ITEM_1_ID, input: "hello", expected: "hello" }],
+      },
+      runner: async ({ item }) => ({ output: item.input }),
+      scorers: [
+        {
+          id: "sampled",
+          threshold: 0.5,
+          scorer: {
+            id: "sampled",
+            name: "sampled",
+            sampling: { type: "never" },
+            scorer: () => ({ status: "success", score: 1 }),
+          },
+        },
+      ],
+      passCriteria: [
+        { type: "passRate", min: 1 },
+        { type: "passRate", min: 1, scorerId: "sampled" },
+      ],
+    });
+
+    const result = await runExperiment(experiment);
+
+    // Both spellings of the same criterion must agree: with no evaluated
+    // evidence there is no pass rate to meet, so neither can pass.
+    expect(result.summary.passRate).toBeNull();
+    expect(result.summary.criteria.map((entry) => entry.passed)).toEqual([false, false]);
+  });
 });

website/evaluation-docs/offline-evaluations.md

@@ -164,7 +164,7 @@ While individual scorers determine if each item passes or fails, **pass criteria
 There are two types of criteria:
 
 - **`meanScore`**: Average score across all items must meet a minimum
-- **`passRate`**: Percentage of passed items must meet a minimum
+- **`passRate`**: Percentage of passed items must meet a minimum. Items whose scorers were all sampled out are not counted, since they carry no evaluation evidence
 
 ```ts
 import { createExperiment } from "@voltagent/evals";
@@ -785,7 +785,7 @@ interface ExperimentSummary {
   errorCount: number; // items with status "error"
   skippedCount: number; // items with status "skipped"
   meanScore?: number | null;
-  passRate?: number | null;
+  passRate?: number | null; // passed / evaluated items; null if nothing was evaluated
   startedAt: number; // Unix timestamp
   completedAt?: number; // Unix timestamp
   durationMs?: number;