TB empty-gold fairness: pure_refusal-only gate under full catalog

ts/packages/benchmarks/README.AUTOGEN.md

@@ -3,7 +3,7 @@
 
 <!-- AUTOGEN:DOCS:START -->
 
-<!-- AUTOGEN:DOCS:HASH:sha256=04ffc6fa34cfbe0d06383b4e35423fc5a203f9c3bd8b8f07c91c20b2cef9903c -->
+<!-- AUTOGEN:DOCS:HASH:sha256=66af1829b35e910f7aff0fcaea048a1bebaa1024139fadc63c42deecf7abf6cb -->
 <!-- AUTOGEN:DOCS:SOURCE: ./README.md (hand-written documentation; this file is the AI-generated companion) -->
 
 # @typeagent/benchmarks — AI-generated documentation
@@ -52,10 +52,10 @@ _None._
 - [./src/core/types.ts](./src/core/types.ts)
 - [./src/translationBench/action-parameters-grader.generated.json](./src/translationBench/action-parameters-grader.generated.json)
 - [./src/translationBench/catalog.generated.json](./src/translationBench/catalog.generated.json)
-- _…and 29 more under `./src/`._
+- _…and 31 more under `./src/`._
 
 ---
 
-_Auto-generated against commit `54efea2e226011740764eddb4beee99edc562313` on `2026-08-08T00:27:51.771Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
+_Auto-generated against commit `b6e33fe1f197027aa43b3a19fabd99ffeeddbcd9` on `2026-08-08T10:22:08.930Z` by `docs-generate.yml`. Links validated at that commit; the working tree may have drifted by up to 24h. Re-run `pnpm --filter @typeagent/benchmarks docs:verify-links` to spot-check._
 
 <!-- AUTOGEN:DOCS:END -->

ts/packages/benchmarks/src/translationBench/synthesizer/actionValidation.ts

@@ -0,0 +1,56 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import {
+    validateAction,
+    type ActionSchemaTypeDefinition,
+} from "@typeagent/action-schema";
+
+/**
+ * Build the object passed to validateAction for a TB gold action.
+ * - Injects required single-literal string-union fields (e.g. settings `id`)
+ * - Restores parameters:{} when the schema requires an empty parameters object
+ *   after stripEmptyGoldPlaceholders dropped nested empties.
+ */
+export function translationBenchActionValidationPayload(
+    definition: ActionSchemaTypeDefinition,
+    action: {
+        actionName: string;
+        parameters?: Record<string, unknown>;
+    },
+): Record<string, unknown> {
+    const payload: Record<string, unknown> = {
+        actionName: action.actionName,
+    };
+    for (const [name, field] of Object.entries(definition.type.fields)) {
+        if (name === "actionName" || name === "parameters") continue;
+        if (field.optional) continue;
+        const fieldType = field.type;
+        if (
+            fieldType.type === "string-union" &&
+            fieldType.typeEnum.length === 1
+        ) {
+            payload[name] = fieldType.typeEnum[0];
+        }
+    }
+    const parametersField = definition.type.fields.parameters;
+    if (action.parameters !== undefined) {
+        payload.parameters = action.parameters;
+    } else if (parametersField !== undefined && !parametersField.optional) {
+        payload.parameters = {};
+    }
+    return payload;
+}
+
+export function validateTranslationBenchGoldAction(
+    definition: ActionSchemaTypeDefinition,
+    action: {
+        actionName: string;
+        parameters?: Record<string, unknown>;
+    },
+): void {
+    validateAction(
+        definition,
+        translationBenchActionValidationPayload(definition, action),
+    );
+}

ts/packages/benchmarks/src/translationBench/synthesizer/benchmark.ts

@@ -8,7 +8,6 @@ import {
     generateActionActionFunctionJsonSchemas,
     parseToolsJsonSchema,
     toJSONParsedActionSchema,
-    validateAction,
     type ParsedActionSchema,
     type ParsedActionSchemaJSON,
 } from "@typeagent/action-schema";
@@ -35,6 +34,7 @@ import {
     countEligibleTranslationBenchActions,
     getPackagedLlmJudgeExcludedActions,
 } from "./eligibleActions.js";
+import { validateTranslationBenchGoldAction } from "./actionValidation.js";
 
 export type TranslationBenchOrder = "strict" | "any";
 // Closed transform set: source import (1) vs generated/canonical (2).
@@ -56,8 +56,26 @@ export interface TranslationBenchBenchmarkProbePayload {
     expectedActions: TranslationBenchBenchmarkAction[];
     order: TranslationBenchOrder;
     history?: ChatHistoryInput;
+    /**
+     * Per-expected-action soft-match specs consumed by the runner. Derived
+     * deterministically from the packaged parameter grader at finalize time
+     * (not authored by the LLM, not part of the canonical payload hash).
+     * Entry `i` scores `expectedActions[i]`; `undefined` = exact-match.
+     */
+    parameterScore?: Array<TranslationBenchParameterScoreSpec | undefined>;
 }
 
+export interface TranslationBenchParameterScoreSpec {
+    defaultMode: TranslationBenchParamFieldMode;
+    fields: Record<string, TranslationBenchParamFieldMode>;
+}
+
+export type TranslationBenchParamFieldMode =
+    | "exact"
+    | "exists"
+    | "nonempty"
+    | "ignore";
+
 export interface TranslationBenchPublicTurnLineage {
     dataset: string;
     revision: string;
@@ -414,14 +432,26 @@ const actionSchema = z
         parameters: z.record(z.string(), z.unknown()).optional(),
     })
     .strict();
+const paramFieldModeSchema = z.enum(["exact", "exists", "nonempty", "ignore"]);
+const parameterScoreSpecSchema = z
+    .object({
+        defaultMode: paramFieldModeSchema,
+        fields: z.record(z.string(), paramFieldModeSchema),
+    })
+    .strict();
 const probePayloadShape = {
     utterance: z.string().trim().min(1),
     expectedActions: z.array(actionSchema),
     order: z.enum(["strict", "any"]),
     history: z.unknown().optional(),
+    parameterScore: z.array(parameterScoreSpecSchema.optional()).optional(),
 } as const;
-function validateHistory(
-    probe: { history?: unknown },
+function validateProbePayload(
+    probe: {
+        history?: unknown;
+        expectedActions?: unknown;
+        parameterScore?: unknown;
+    },
     context: z.RefinementCtx,
 ) {
     if (probe.history !== undefined && !isChatHistoryInput(probe.history)) {
@@ -431,11 +461,22 @@ function validateHistory(
             message: "invalid ChatHistoryInput",
         });
     }
+    if (
+        Array.isArray(probe.parameterScore) &&
+        Array.isArray(probe.expectedActions) &&
+        probe.parameterScore.length !== probe.expectedActions.length
+    ) {
+        context.addIssue({
+            code: z.ZodIssueCode.custom,
+            path: ["parameterScore"],
+            message: "parameterScore must align 1:1 with expectedActions",
+        });
+    }
 }
 const probePayloadSchema = z
     .object(probePayloadShape)
     .strict()
-    .superRefine(validateHistory);
+    .superRefine(validateProbePayload);
 const sha256Schema = z.string().regex(/^[a-f0-9]{64}$/);
 const lineageSchema = z
     .object({
@@ -469,7 +510,7 @@ const publicProbeSchema = z
         selection: selectionAnnotationSchema,
     })
     .strict()
-    .superRefine(validateHistory);
+    .superRefine(validateProbePayload);
 const shapeOnlySchema = z
     .object({
         ...probePayloadShape,
@@ -485,7 +526,7 @@ const shapeOnlySchema = z
             .strict(),
     })
     .strict()
-    .superRefine(validateHistory);
+    .superRefine(validateProbePayload);
 const toolSchema = z
     .object({
         type: z.literal("function"),
@@ -2276,7 +2317,12 @@ function validateExpectedActions(
                 `${label} expects unknown existing TypeAgent action '${action.schemaName}.${action.actionName}'`,
             );
         }
-        validateAction(definition, action);
+        validateTranslationBenchGoldAction(definition, {
+            actionName: action.actionName,
+            ...(action.parameters !== undefined
+                ? { parameters: action.parameters }
+                : {}),
+        });
     }
 }
 

ts/packages/benchmarks/src/translationBench/synthesizer/catalogGenerator/actionParametersGrader.ts

@@ -3,10 +3,15 @@
 
 import { createHash } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
+import { createRequire } from "node:module";
 
 import { z } from "zod";
 
 import { parseLlmJsonWithZod } from "../llmJson.js";
+import type {
+    TranslationBenchParameterScoreSpec,
+    TranslationBenchParamFieldMode,
+} from "../benchmark.js";
 import {
     loadTranslationBenchParameterGraderPromptPack,
     renderTranslationBenchPromptTemplate,
@@ -1402,6 +1407,37 @@ export function loadActionParametersGraderCatalogFile(
     return raw as unknown as ActionParametersGraderCatalog;
 }
 
+const requireFromHere = createRequire(import.meta.url);
+let cachedPackagedActionParametersGrader:
+    | ActionParametersGraderCatalog
+    | undefined;
+
+/**
+ * Packaged deterministic parameter grader, loaded from the generated JSON that
+ * ships with the benchmark. Cached; used to derive per-case `parameterScore`
+ * specs so the runner soft-matches params (e.g. free-text `nonempty`) instead
+ * of exact-matching everything.
+ */
+export function getPackagedActionParametersGraderCatalog(): ActionParametersGraderCatalog {
+    if (cachedPackagedActionParametersGrader === undefined) {
+        const graderPath = requireFromHere.resolve(
+            "../../action-parameters-grader.generated.json",
+        );
+        const catalog = loadActionParametersGraderCatalogFile(graderPath);
+        if (catalog === undefined) {
+            throw new Error(
+                `Missing packaged action-parameters grader at ${graderPath}`,
+            );
+        }
+        cachedPackagedActionParametersGrader = catalog;
+    }
+    return cachedPackagedActionParametersGrader;
+}
+
+export function clearPackagedActionParametersGraderCacheForTests(): void {
+    cachedPackagedActionParametersGrader = undefined;
+}
+
 function priorEntryStillValid(
     entry: ActionParametersGraderEntry,
     catalogRow: CatalogActionRow,
@@ -1728,48 +1764,49 @@ function isIdentifierName(name: string): boolean {
 }
 
 /**
- * Runner-ready parameterScore specs aligned 1:1 with expectedActions.
- * Missing grader entries yield `undefined` slots (runner falls back to exact).
+ * Runner-ready parameterScore specs aligned 1:1 with expectedActions. Missing
+ * grader entries yield `undefined` slots (runner falls back to exact-match).
+ * `llmAsAJudge` is a generation/offline-scoring concept the deterministic
+ * runner can't consume, so such params map to `ignore` (judged elsewhere).
  */
+function toRunnerParamFieldMode(
+    mode: ActionParamVerifyMode,
+): TranslationBenchParamFieldMode {
+    return mode === "llmAsAJudge" ? "ignore" : mode;
+}
+
 export function parameterScoreSpecsForExpectedActions(
     grader: ActionParametersGraderCatalog,
     expectedActions: ReadonlyArray<{
         schemaName: string;
         actionName: string;
     }>,
-): Array<
-    | {
-          defaultMode: ActionParamVerifyMode;
-          fields: Record<string, ActionParamVerifyMode>;
-      }
-    | undefined
-> {
+): Array<TranslationBenchParameterScoreSpec | undefined> {
     return expectedActions.map((action) => {
         const entry =
             grader.byAction[actionId(action.schemaName, action.actionName)];
-        if (entry === undefined) {
-            return undefined;
-        }
-        const fields = entry.parameterScore.fields;
-        if (Object.keys(fields).length === 0) {
+        if (
+            entry === undefined ||
+            Object.keys(entry.parameterScore.fields).length === 0
+        ) {
             return undefined;
         }
         return {
-            defaultMode: entry.parameterScore.defaultMode,
-            fields: { ...fields },
+            defaultMode: toRunnerParamFieldMode(
+                entry.parameterScore.defaultMode,
+            ),
+            fields: Object.fromEntries(
+                Object.entries(entry.parameterScore.fields).map(
+                    ([name, mode]) => [name, toRunnerParamFieldMode(mode)],
+                ),
+            ),
         };
     });
 }
 
 /** True when at least one expected action has a non-empty parameterScore map. */
 export function hasUsableParameterScoreSpecs(
-    specs: ReadonlyArray<
-        | {
-              defaultMode: ActionParamVerifyMode;
-              fields: Record<string, ActionParamVerifyMode>;
-          }
-        | undefined
-    >,
+    specs: ReadonlyArray<TranslationBenchParameterScoreSpec | undefined>,
 ): boolean {
     return specs.some((spec) => spec !== undefined);
 }