From ff9302f13d6c231a641c9ccc40c0ae8badca2cc1 Mon Sep 17 00:00:00 2001 From: "wei.zhong1" Date: Fri, 10 Jul 2026 10:42:21 -0700 Subject: [PATCH 1/4] Extend key entities metric to audio-native models Extend transcription_accuracy_key_entities to support S2S / AUDIO_LLM pipelines. Audio-native models expose no reliable STT transcript, so the agent's tool-call arguments (from conversation_trace) become the evidence of what it understood: if the user says "my id is EM123" and the agent calls a tool with "EMP124", the entity is flagged as misheard. The metric now branches on pipeline type: - CASCADE: compares intended vs transcribed user turns (unchanged). - Audio-native: compares intended user entities vs tool-call arguments, using a new audio_native_prompt judge template. Bumps the metric to v0.4, metrics_version to 2.3.0, and regenerates the metric signature fixture. --- configs/prompts/judge.yaml | 166 ++++++++++++++++++ src/eva/__init__.py | 2 +- .../transcription_accuracy_key_entities.py | 81 +++++++-- tests/fixtures/metric_signatures.json | 4 +- ...est_transcription_accuracy_key_entities.py | 89 ++++++++++ 5 files changed, 324 insertions(+), 18 deletions(-) diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml index e4b4ba14..b29949dc 100644 --- a/configs/prompts/judge.yaml +++ b/configs/prompts/judge.yaml @@ -1071,3 +1071,169 @@ judge: "summary": "<1-2 sentence summary for this turn>" }} ] + + audio_native_prompt: | + You are an expert evaluator analyzing whether an AI voice agent correctly **understood** the key entities a user provided during a spoken conversation. + + This is a speech-to-speech / audio-native agent: there is no reliable speech-to-text transcript of the user to inspect. Instead, the evidence of what the agent actually understood is what it did with the information — primarily the **arguments it passed to its tool calls**, and secondarily any values it read back to the user. When a user says "my id is EM123" and the agent then calls a tool with "EMP124", the agent misheard the entity. + + Your task: + 1. For EACH user turn, identify all key entities in what the USER SAID (the "User said" text). + 2. For each entity, find where the agent used it — a tool call argument (see the Conversation below) or a value the agent read back — and check whether the agent captured it CORRECTLY. + 3. Mark each entity as correct or incorrect. + 4. If the agent never used a given entity anywhere (no tool call argument and no read-back reference to it), there is no evidence to judge understanding — include the entity but mark it as skipped. + 5. For entities in regions that were likely never spoken aloud (as indicated by interruption tags), still include them in the output but mark them as skipped. + + ## What Counts as an Entity + An entity must have a **specific, concrete value** — something that could be passed as an input to a program or tool (not an AI, but a script or database lookup). Ask yourself: could this value be stored in a variable and used programmatically? + + - Names (people, places, organizations): e.g. "John Smith", "Austin", "Delta Airlines" + - Specific dates and times: e.g. "December 15th", "3:45 PM" — NOT vague references like "tomorrow morning" or "later today" + - Confirmation codes / reference numbers: e.g. "ABC123", "ZK3FFW" + - Flight numbers: e.g. "UA 204" + - Amounts and prices: use the specific value only, e.g. "$120" — for qualifier phrases like "under $120", only use the specific value + - Addresses: e.g. "123 Main Street" + - Phone numbers: e.g. "555-867-5309" + - Email addresses: e.g. "john@example.com" + - Other specific identifiers: seat numbers, loyalty numbers, booking IDs, employee IDs, etc. + + **Not an entity:** vague temporal words ("tomorrow", "next week", "morning"), general descriptors ("the cheap flight", "a long trip"), or open-ended qualifiers ("less than an hour", "around noon"). + + ## IMPORTANT: What This Metric Measures + This measures whether the agent **understood** the entity the user provided, as evidenced by how it used it. It is NOT a faithfulness or tool-correctness metric. Do NOT penalize: + - The agent choosing the wrong tool, or omitting a tool call it should have made + - The agent using an entity that came from a tool RESPONSE rather than from the user (only user-provided entities are evaluated here) + - Correct values that simply never got reused (mark those skipped, not incorrect) + + Only mark an entity incorrect when the agent demonstrably used a **different value** than the user provided (e.g. user "EM123" → tool call "EMP124", user "March 25th" → tool call "2026-03-26"). + + ## Understanding Tags in the User Text + The user text may contain non-spoken tags and markers. These are metadata — they were never said aloud and must not be treated as entities or evaluated. + + ### Audio-Direction Tags + Tags like [slow], [firm], [annoyed] describe how the words were meant to be spoken. Ignore them entirely. + + ### Interruption Tags + {interruption_tags_reference} + + These markers indicate that parts of the user text may never have been spoken aloud, because the user was interrupted or talked over. An entity that was never spoken cannot have been understood, so you must NOT penalize for entities in regions that were likely not said — mark them as skipped. + + **Key principle:** Only evaluate entities that were reasonably expected to have been spoken aloud AND that the agent had an occasion to use. If a tag indicates the user was interrupted before or during an entity, still include the entity but set `skipped: true` and explain why. + + ## User Turns to Evaluate + {user_turns} + + ## Conversation (with agent turns and tool calls/responses) + Use this to find where each user-provided entity was used by the agent. Tool call arguments are the primary evidence of what the agent understood. + {conversation} + + ## Correctness Criteria + - Entity must have been used correctly by the agent — the value passed to the tool call (or read back) must match what the user said. + - Numbers: "150" and "one hundred fifty" are equivalent + - Dates: "December 15th" and "Dec 15" are equivalent + - Names: Case-insensitive match. Phonetically equivalent spelling variants are acceptable on the **first mention** of a name (e.g., German Meyer/Meier/Maier/Mayer — all sound the same when spoken). Once the user has explicitly disambiguated the spelling (e.g., spelled it out letter by letter, corrected a prior spelling, or confirmed a specific spelling after the assistant read it back), subsequent usage must match that clarified spelling exactly. Flag if: + - The value the agent used clearly differs (different sound, not just different spelling) + - The agent used a different spelling variant **after** the user has explicitly clarified the correct spelling + - Non-Latin scripts: A name or identifier rendered in its native writing system (e.g. Kanji, Hangul, Arabic) is equivalent to its romanized/phonetic form if they represent the same spoken entity. For example, "山田" and "Yamada" should be treated as matching. + - Phonetic letter spellings: In languages that do not use the Latin alphabet, Roman/Latin letters are spoken using language-specific phonetic equivalents. These are correct, not errors (e.g., in Korean "에이" for "A" or "비" for "B"). + - Number surface variants: Many languages have systematic spoken-form differences that do not change the underlying value and must not be penalized: + - Reversed digit order in spoken form (e.g., German "vierundzwanzig" = four-and-twenty = 24) + - Grammatical gender/agreement variants (e.g., Spanish "veintiún" vs "veintiuna" for 21) + - Large-unit grouping differences (e.g., Japanese 万-system: "ichi-man go-sen" / 一万五千 = 15,000) + - Honorific/title variants: "Ms." vs "Miss", "Mr." vs "Mister", and language equivalents (e.g., French "Madame" vs "Mme", Japanese "様" vs "-san") are acceptable variants of the same title. + + Important note: The user text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate semantic equivalence — these are considered equivalent if the value matches what the agent used. + + ## Examples + + **Example Input:** + User Turns to Evaluate: + Turn 1: + User said: `My employee id is E M one two three and my name is John Smith.` + + Turn 2: + User said: `Book it for December 15th.` + + Turn 3: + User said: `[slow] The code is X X F six O H, with the letter O, [assistant interrupts] not zero.` + + Conversation: + Turn 1: + user: My employee id is E M one two three and my name is John Smith. + tool_call: lookup_employee({{'employee_id': 'EMP124'}}) + Turn 2: + user: Book it for December 15th. + tool_call: create_booking({{'date': '2026-12-15'}}) + + **Example Response:** + [ + {{ + "turn_id": 1, + "entities": [ + {{ + "type": "employee_id", + "value": "E M one two three", + "captured_value": "EMP124", + "analysis": "User said EM123 but the agent called lookup_employee with EMP124 — misheard.", + "correct": false, + "skipped": false + }}, + {{ + "type": "name", + "value": "John Smith", + "captured_value": "not used", + "analysis": "The name was never passed to a tool call or read back, so there is no evidence of how the agent understood it.", + "correct": false, + "skipped": true + }} + ], + "summary": "1 entity used (employee_id, incorrect). Name never used — skipped." + }}, + {{ + "turn_id": 2, + "entities": [ + {{ + "type": "date", + "value": "December 15th", + "captured_value": "2026-12-15", + "analysis": "Agent passed 2026-12-15 to create_booking — matches December 15th.", + "correct": true, + "skipped": false + }} + ], + "summary": "1 entity correct." + }}, + {{ + "turn_id": 3, + "entities": [ + {{ + "type": "confirmation_code", + "value": "X X F six O H", + "captured_value": "not used", + "analysis": "Code appears before the [assistant interrupts] tag, but the agent never used it in a tool call — no evidence to judge, skipped.", + "correct": false, + "skipped": true + }} + ], + "summary": "1 entity found; never used by the agent — skipped." + }} + ] + + ## Response Format + Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the User Turns to Evaluate section above: + [ + {{ + "turn_id": , + "entities": [ + {{ + "type": "", + "value": "", + "captured_value": "", + "analysis": "", + "correct": , + "skipped": + }} + ], + "summary": "<1-2 sentence summary for this turn>" + }} + ] diff --git a/src/eva/__init__.py b/src/eva/__init__.py index eb2eaa6a..5a7690b0 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -11,4 +11,4 @@ # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.2.0" +metrics_version = "2.3.0" diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py index 8647d7f4..806974ea 100644 --- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py +++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py @@ -1,4 +1,16 @@ -"""Transcription accuracy key entities metric using LLM-as-judge (entire conversation). +"""Key entity accuracy metric using LLM-as-judge (entire conversation). + +Measures whether the user's key entities (names, dates, confirmation codes, +amounts, etc.) survived intact through the agent's pipeline. The comparison +target depends on the pipeline type: + +- **CASCADE**: compares what the user was instructed to say (``intended_user_turns``) + against what the agent's STT transcribed (``transcribed_user_turns``) — i.e. + transcription accuracy. +- **Audio-native (S2S / AUDIO_LLM)**: there is no reliable STT output to inspect, + so the agent's **tool-call arguments** (from ``conversation_trace``) become the + evidence of what it understood. If the user says "my id is EM123" and the agent + then calls a tool with "EMP124", the agent misheard the entity. Debug metric for diagnosing model performance issues, not directly used in final evaluation scores. @@ -10,23 +22,29 @@ from eva.metrics.registry import register_metric from eva.metrics.utils import ( aggregate_per_turn_scores, + format_transcript_with_tools, make_rate_sub_metric, parse_judge_response_list, resolve_turn_id, ) -from eva.models.config import PipelineType from eva.models.results import MetricScore @register_metric class TranscriptionAccuracyKeyEntitiesMetric(TextJudgeMetric): - """LLM-based transcription accuracy metric for key entities only (entire conversation). + """LLM-based key entity accuracy metric (entire conversation). - Evaluates STT transcription accuracy by comparing key entities (names, dates, - confirmation codes, amounts, etc.) between what the user was supposed to say - (intended_user_turns) and what STT transcribed (transcribed_user_turns). + Evaluates whether the user's key entities (names, dates, confirmation codes, + amounts, etc.) survived intact through the agent's pipeline. The comparison + target depends on the pipeline type: - Computes the ratio of correctly transcribed entities per turn: + - **CASCADE**: compares ``intended_user_turns`` against ``transcribed_user_turns`` + (STT output) — transcription accuracy. + - **Audio-native (S2S / AUDIO_LLM)**: no reliable STT output exists, so the agent's + tool-call arguments (from ``conversation_trace``) are used as the evidence of + what it understood. + + Computes the ratio of correctly captured entities per turn: score = correct_entities / total_entities Entity types evaluated: @@ -48,11 +66,12 @@ class TranscriptionAccuracyKeyEntitiesMetric(TextJudgeMetric): """ name = "transcription_accuracy_key_entities" - version = "v0.3" - description = "Debug metric: LLM judge evaluation of STT key entity transcription accuracy for entire conversation" + version = "v0.4" + description = ( + "Debug metric: LLM judge evaluation of user key entity accuracy (STT in cascade, tool calls in audio-native)" + ) category = "diagnostic" exclude_from_pass_at_k = True - supported_pipeline_types = frozenset({PipelineType.CASCADE}) rating_scale = None # Custom scoring (not 1-3 scale) default_aggregation = "mean" @@ -70,9 +89,7 @@ async def compute(self, context: MetricContext) -> MetricScore: MetricScore with aggregated score and per-turn details """ try: - turns_to_evaluate = self._get_turns_to_evaluate(context) - user_turns_text = self._format_user_turns(turns_to_evaluate, context) - prompt = self.get_judge_prompt(user_turns=user_turns_text) + prompt, turns_to_evaluate = self._build_prompt_and_turns(context) response_text = await self._call_judge_raw(prompt, context) turn_evaluations = parse_judge_response_list(response_text) @@ -206,14 +223,43 @@ def _build_per_entity_type_sub_metrics( ) return sub_metrics + def _build_prompt_and_turns(self, context: MetricContext) -> tuple[str, list[int]]: + """Select the comparison source and judge prompt based on pipeline type. + + CASCADE compares intended vs STT-transcribed user turns. Audio-native + pipelines (S2S / AUDIO_LLM) have no reliable STT, so the user's intended + turns are compared against the agent's tool-call arguments (from + ``conversation_trace``), which are the evidence of what it understood. + + Returns: + Tuple of (rendered judge prompt, sorted turn IDs to evaluate). + """ + if context.is_audio_native: + turn_ids = self._get_audio_native_turns(context) + prompt = self.get_judge_prompt( + prompt_key="audio_native_prompt", + user_turns=self._format_intended_user_turns(turn_ids, context), + conversation=format_transcript_with_tools(context.conversation_trace), + ) + return prompt, turn_ids + + turn_ids = self._get_turns_to_evaluate(context) + prompt = self.get_judge_prompt(user_turns=self._format_user_turns(turn_ids, context)) + return prompt, turn_ids + @staticmethod def _get_turns_to_evaluate(context: MetricContext) -> list[int]: - """Return sorted turn IDs present in both tts_text_user and transcript_user.""" + """Return sorted turn IDs present in both intended and transcribed user turns.""" return sorted(context.intended_user_turns.keys() & context.transcribed_user_turns.keys()) + @staticmethod + def _get_audio_native_turns(context: MetricContext) -> list[int]: + """Return sorted intended user turn IDs (ground truth for audio-native pipelines).""" + return sorted(context.intended_user_turns.keys()) + @staticmethod def _format_user_turns(turn_ids: list[int], context: MetricContext) -> str: - """Format user turns for the prompt.""" + """Format expected-vs-transcribed user turn pairs for the cascade prompt.""" return "\n\n".join( f"Turn {tid}:\n" f'Expected: "{context.intended_user_turns[tid]}"\n' @@ -221,6 +267,11 @@ def _format_user_turns(turn_ids: list[int], context: MetricContext) -> str: for tid in turn_ids ) + @staticmethod + def _format_intended_user_turns(turn_ids: list[int], context: MetricContext) -> str: + """Format the user's intended utterances for the audio-native prompt.""" + return "\n\n".join(f'Turn {tid}:\nUser said: "{context.intended_user_turns[tid]}"' for tid in turn_ids) + async def _call_judge_raw(self, prompt: str, context: MetricContext) -> str | None: """Call LLM judge and return raw response text. diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 3b95ed18..280ba360 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -86,8 +86,8 @@ "TranscriptionAccuracyKeyEntitiesMetric": { "name": "transcription_accuracy_key_entities", "prompt_hash": "877dfa61f232", - "source_hash": "70e92b2f7221", - "version": "v0.3" + "source_hash": "1be843cc964d", + "version": "v0.4" }, "TurnTakingMetric": { "name": "turn_taking", diff --git a/tests/unit/metrics/test_transcription_accuracy_key_entities.py b/tests/unit/metrics/test_transcription_accuracy_key_entities.py index 654c7ed7..d8de0216 100644 --- a/tests/unit/metrics/test_transcription_accuracy_key_entities.py +++ b/tests/unit/metrics/test_transcription_accuracy_key_entities.py @@ -9,6 +9,7 @@ TranscriptionAccuracyKeyEntitiesMetric, ) from eva.metrics.utils import aggregate_per_turn_scores +from eva.models.config import PipelineType from .conftest import make_judge_metric, make_metric_context @@ -375,3 +376,91 @@ async def test_no_response_from_judge(self, metric): assert result.error == "No response from judge" assert result.score == 0.0 + + +class TestPipelineSupport: + def test_supports_cascade_and_audio_native(self): + """Metric supports cascade and both audio-native pipeline types.""" + supported = TranscriptionAccuracyKeyEntitiesMetric.supported_pipeline_types + assert PipelineType.CASCADE in supported + assert PipelineType.S2S in supported + assert PipelineType.AUDIO_LLM in supported + + +class TestAudioNativeCompute: + """Audio-native pipelines compare intended user entities against tool-call arguments.""" + + @pytest.mark.asyncio + async def test_uses_audio_native_prompt_and_intended_turns(self, metric): + """Audio-native path builds the tool-call prompt from intended turns + conversation trace.""" + context = make_metric_context( + pipeline_type=PipelineType.S2S, + intended_user_turns={1: "My employee id is E M one two three"}, + transcribed_user_turns={}, # unreliable / absent for audio-native + conversation_trace=[ + {"role": "user", "content": "My employee id is E M one two three", "turn_id": 1}, + { + "type": "tool_call", + "tool_name": "lookup_employee", + "parameters": {"employee_id": "EMP124"}, + "turn_id": 1, + }, + ], + ) + response = _make_judge_response( + [ + { + "turn_id": 1, + "summary": "Misheard employee id", + "entities": [ + {"type": "employee_id", "value": "EM123", "correct": False, "skipped": False}, + ], + } + ] + ) + generate_text = AsyncMock(return_value=(response, None)) + metric.llm_client.generate_text = generate_text + + result = await metric.compute(context) + + assert result.error is None + assert result.score == 0.0 + assert result.normalized_score == 0.0 + # Prompt must be the audio-native variant fed with tool-call evidence, not STT. + sent_prompt = generate_text.call_args[0][0][0]["content"] + assert "tool call" in sent_prompt.lower() + assert "EMP124" in sent_prompt + assert "Transcribed:" not in sent_prompt + + @pytest.mark.asyncio + async def test_correct_entity_in_tool_call_scores_one(self, metric): + """Entity correctly captured in the tool call → score 1.0.""" + context = make_metric_context( + pipeline_type=PipelineType.AUDIO_LLM, + intended_user_turns={1: "Book it for December 15th"}, + conversation_trace=[ + {"role": "user", "content": "Book it for December 15th", "turn_id": 1}, + { + "type": "tool_call", + "tool_name": "create_booking", + "parameters": {"date": "2026-12-15"}, + "turn_id": 1, + }, + ], + ) + response = _make_judge_response( + [ + { + "turn_id": 1, + "summary": "Date captured correctly", + "entities": [{"type": "date", "value": "December 15th", "correct": True, "skipped": False}], + } + ] + ) + metric.llm_client.generate_text = AsyncMock(return_value=(response, None)) + + result = await metric.compute(context) + + assert result.error is None + assert result.score == 1.0 + assert result.normalized_score == 1.0 From e7025135264b1a70b4bfb6c1fee9cf8ff090a8ab Mon Sep 17 00:00:00 2001 From: "wei.zhong1" Date: Fri, 17 Jul 2026 13:27:36 -0700 Subject: [PATCH 2/4] Address review: redact assistant turns, single conversation, prompt-key signatures Review feedback from @gabegma on the audio-native key-entity path: - Redact assistant turns in the judge input (show [Assistant speaks]) so a mis-transcribed agent read-back can't be used to penalize a user entity (e.g. user 'INC462', agent 'Did you say INC463?'). Judge now sees only user turns + tool calls/responses. - Collapse the duplicated user-turns section into a single redacted Conversation block (user turns shown inline), mirroring speech_fidelity. - Drop standalone honorific/title variants from the entity criteria (not entities). - Merge the two non-spoken-tag subsections into one. - signatures.py: hash ALL prompt keys under judge.{name}, not just user_prompt, so edits to audio_native_prompt are caught by the drift test. Bumps metric v0.4 -> v0.5, metrics_version 2.3.0 -> 2.4.0, regenerates signatures. --- configs/prompts/judge.yaml | 75 ++++++++----------- src/eva/__init__.py | 2 +- .../transcription_accuracy_key_entities.py | 36 +++++++-- src/eva/metrics/signatures.py | 15 ++-- tests/fixtures/metric_signatures.json | 6 +- ...est_transcription_accuracy_key_entities.py | 37 +++++++++ 6 files changed, 110 insertions(+), 61 deletions(-) diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml index b29949dc..b4acbc02 100644 --- a/configs/prompts/judge.yaml +++ b/configs/prompts/judge.yaml @@ -1075,13 +1075,15 @@ judge: audio_native_prompt: | You are an expert evaluator analyzing whether an AI voice agent correctly **understood** the key entities a user provided during a spoken conversation. - This is a speech-to-speech / audio-native agent: there is no reliable speech-to-text transcript of the user to inspect. Instead, the evidence of what the agent actually understood is what it did with the information — primarily the **arguments it passed to its tool calls**, and secondarily any values it read back to the user. When a user says "my id is EM123" and the agent then calls a tool with "EMP124", the agent misheard the entity. + This is a speech-to-speech / audio-native agent: there is no reliable speech-to-text transcript of the user to inspect. Instead, the evidence of what the agent actually understood is the **arguments it passed to its tool calls**. When a user says "my id is EM123" and the agent then calls a tool with "EMP124", the agent misheard the entity. + + You are given a Conversation containing user turns, tool calls, and tool responses. **The assistant's own spoken turns are redacted** (shown as `[Assistant speaks]`) and deliberately withheld: the agent's speech is itself imperfectly transcribed, so it must never be used as evidence. For example, if the user says "INC462" and the agent asks "Did you say INC463?", that read-back may just be a transcription artifact — do not treat it as the agent mishearing the entity. Judge only what the user said (user turns) against what the agent did (tool-call arguments). Your task: - 1. For EACH user turn, identify all key entities in what the USER SAID (the "User said" text). - 2. For each entity, find where the agent used it — a tool call argument (see the Conversation below) or a value the agent read back — and check whether the agent captured it CORRECTLY. + 1. For EACH user turn in the Conversation, identify all key entities in what the USER SAID. + 2. For each entity, find whether the agent used it in a tool-call argument and check whether it was captured CORRECTLY. 3. Mark each entity as correct or incorrect. - 4. If the agent never used a given entity anywhere (no tool call argument and no read-back reference to it), there is no evidence to judge understanding — include the entity but mark it as skipped. + 4. If the agent never used a given entity in any tool call, there is no evidence to judge understanding — include the entity but mark it as skipped. 5. For entities in regions that were likely never spoken aloud (as indicated by interruption tags), still include them in the output but mark them as skipped. ## What Counts as an Entity @@ -1097,41 +1099,34 @@ judge: - Email addresses: e.g. "john@example.com" - Other specific identifiers: seat numbers, loyalty numbers, booking IDs, employee IDs, etc. - **Not an entity:** vague temporal words ("tomorrow", "next week", "morning"), general descriptors ("the cheap flight", "a long trip"), or open-ended qualifiers ("less than an hour", "around noon"). + **Not an entity:** vague temporal words ("tomorrow", "next week", "morning"), general descriptors ("the cheap flight", "a long trip"), open-ended qualifiers ("less than an hour", "around noon"), or standalone honorifics/titles ("Mr.", "Ms."). ## IMPORTANT: What This Metric Measures - This measures whether the agent **understood** the entity the user provided, as evidenced by how it used it. It is NOT a faithfulness or tool-correctness metric. Do NOT penalize: + This measures whether the agent **understood** the entity the user provided, as evidenced by the value it passed to a tool call. It is NOT a faithfulness or tool-correctness metric. Do NOT penalize: - The agent choosing the wrong tool, or omitting a tool call it should have made - The agent using an entity that came from a tool RESPONSE rather than from the user (only user-provided entities are evaluated here) - Correct values that simply never got reused (mark those skipped, not incorrect) Only mark an entity incorrect when the agent demonstrably used a **different value** than the user provided (e.g. user "EM123" → tool call "EMP124", user "March 25th" → tool call "2026-03-26"). - ## Understanding Tags in the User Text - The user text may contain non-spoken tags and markers. These are metadata — they were never said aloud and must not be treated as entities or evaluated. + ## Non-Spoken Tags in the User Text + The user text may contain non-spoken tags and markers — metadata that was never said aloud and must not be treated as entities. There are two kinds, and they are handled differently: - ### Audio-Direction Tags - Tags like [slow], [firm], [annoyed] describe how the words were meant to be spoken. Ignore them entirely. + - **Audio-direction tags** like [slow], [firm], [annoyed] describe how the words were meant to be spoken. Ignore them entirely; they have no effect on scoring. + - **Interruption tags** indicate that part of the user text was likely never spoken, because the user was interrupted or talked over. An entity that was never spoken cannot have been understood, so do NOT penalize entities in these regions — include them but mark them as skipped. - ### Interruption Tags {interruption_tags_reference} - These markers indicate that parts of the user text may never have been spoken aloud, because the user was interrupted or talked over. An entity that was never spoken cannot have been understood, so you must NOT penalize for entities in regions that were likely not said — mark them as skipped. - - **Key principle:** Only evaluate entities that were reasonably expected to have been spoken aloud AND that the agent had an occasion to use. If a tag indicates the user was interrupted before or during an entity, still include the entity but set `skipped: true` and explain why. + **Key principle:** Only evaluate entities that were reasonably expected to have been spoken aloud AND that the agent used in a tool call. If a tag indicates the user was interrupted before or during an entity, mark it skipped and explain why. - ## User Turns to Evaluate - {user_turns} - - ## Conversation (with agent turns and tool calls/responses) - Use this to find where each user-provided entity was used by the agent. Tool call arguments are the primary evidence of what the agent understood. + ## Conversation {conversation} ## Correctness Criteria - - Entity must have been used correctly by the agent — the value passed to the tool call (or read back) must match what the user said. + - Entity must have been used correctly by the agent — the value passed to the tool call must match what the user said. - Numbers: "150" and "one hundred fifty" are equivalent - Dates: "December 15th" and "Dec 15" are equivalent - - Names: Case-insensitive match. Phonetically equivalent spelling variants are acceptable on the **first mention** of a name (e.g., German Meyer/Meier/Maier/Mayer — all sound the same when spoken). Once the user has explicitly disambiguated the spelling (e.g., spelled it out letter by letter, corrected a prior spelling, or confirmed a specific spelling after the assistant read it back), subsequent usage must match that clarified spelling exactly. Flag if: + - Names: Case-insensitive match. Phonetically equivalent spelling variants are acceptable on the **first mention** of a name (e.g., German Meyer/Meier/Maier/Mayer — all sound the same when spoken). Once the user has explicitly disambiguated the spelling (e.g., spelled it out letter by letter, or corrected a prior spelling), subsequent usage must match that clarified spelling exactly. Flag if: - The value the agent used clearly differs (different sound, not just different spelling) - The agent used a different spelling variant **after** the user has explicitly clarified the correct spelling - Non-Latin scripts: A name or identifier rendered in its native writing system (e.g. Kanji, Hangul, Arabic) is equivalent to its romanized/phonetic form if they represent the same spoken entity. For example, "山田" and "Yamada" should be treated as matching. @@ -1140,30 +1135,20 @@ judge: - Reversed digit order in spoken form (e.g., German "vierundzwanzig" = four-and-twenty = 24) - Grammatical gender/agreement variants (e.g., Spanish "veintiún" vs "veintiuna" for 21) - Large-unit grouping differences (e.g., Japanese 万-system: "ichi-man go-sen" / 一万五千 = 15,000) - - Honorific/title variants: "Ms." vs "Miss", "Mr." vs "Mister", and language equivalents (e.g., French "Madame" vs "Mme", Japanese "様" vs "-san") are acceptable variants of the same title. Important note: The user text will often feature things formatted like "one two three" instead of "123". Your goal is to evaluate semantic equivalence — these are considered equivalent if the value matches what the agent used. ## Examples - **Example Input:** - User Turns to Evaluate: - Turn 1: - User said: `My employee id is E M one two three and my name is John Smith.` - - Turn 2: - User said: `Book it for December 15th.` - - Turn 3: - User said: `[slow] The code is X X F six O H, with the letter O, [assistant interrupts] not zero.` - - Conversation: - Turn 1: - user: My employee id is E M one two three and my name is John Smith. - tool_call: lookup_employee({{'employee_id': 'EMP124'}}) - Turn 2: - user: Book it for December 15th. - tool_call: create_booking({{'date': '2026-12-15'}}) + **Example Input (Conversation):** + Turn 1 - User: My employee id is E M one two three and my name is John Smith. + Turn 1 - [Assistant speaks] + Turn 1 - Tool Call (lookup_employee): {{'employee_id': 'EMP124'}} + Turn 2 - User: Book it for December 15th. + Turn 2 - [Assistant speaks] + Turn 2 - Tool Call (create_booking): {{'date': '2026-12-15'}} + Turn 3 - User: [slow] The code is X X F six O H, with the letter O, [assistant interrupts] not zero. + Turn 3 - [Assistant speaks] **Example Response:** [ @@ -1182,7 +1167,7 @@ judge: "type": "name", "value": "John Smith", "captured_value": "not used", - "analysis": "The name was never passed to a tool call or read back, so there is no evidence of how the agent understood it.", + "analysis": "The name was never passed to a tool call, so there is no evidence of how the agent understood it.", "correct": false, "skipped": true }} @@ -1210,7 +1195,7 @@ judge: "type": "confirmation_code", "value": "X X F six O H", "captured_value": "not used", - "analysis": "Code appears before the [assistant interrupts] tag, but the agent never used it in a tool call — no evidence to judge, skipped.", + "analysis": "The agent never used this code in a tool call — no evidence to judge, skipped.", "correct": false, "skipped": true }} @@ -1220,15 +1205,15 @@ judge: ] ## Response Format - Respond with a JSON object. Each turn entry must include the turn_id matching the turn number shown in the User Turns to Evaluate section above: + Respond with a JSON object. Each turn entry must include the turn_id matching the user turn number shown in the Conversation above: [ {{ - "turn_id": , + "turn_id": , "entities": [ {{ "type": "", "value": "", - "captured_value": "", + "captured_value": "", "analysis": "", "correct": , "skipped": diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 5a7690b0..5306d24a 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -11,4 +11,4 @@ # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.3.0" +metrics_version = "2.4.0" diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py index 806974ea..2a61f39b 100644 --- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py +++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py @@ -22,7 +22,6 @@ from eva.metrics.registry import register_metric from eva.metrics.utils import ( aggregate_per_turn_scores, - format_transcript_with_tools, make_rate_sub_metric, parse_judge_response_list, resolve_turn_id, @@ -66,7 +65,7 @@ class TranscriptionAccuracyKeyEntitiesMetric(TextJudgeMetric): """ name = "transcription_accuracy_key_entities" - version = "v0.4" + version = "v0.5" description = ( "Debug metric: LLM judge evaluation of user key entity accuracy (STT in cascade, tool calls in audio-native)" ) @@ -238,8 +237,7 @@ def _build_prompt_and_turns(self, context: MetricContext) -> tuple[str, list[int turn_ids = self._get_audio_native_turns(context) prompt = self.get_judge_prompt( prompt_key="audio_native_prompt", - user_turns=self._format_intended_user_turns(turn_ids, context), - conversation=format_transcript_with_tools(context.conversation_trace), + conversation=self._format_redacted_conversation(context), ) return prompt, turn_ids @@ -268,9 +266,33 @@ def _format_user_turns(turn_ids: list[int], context: MetricContext) -> str: ) @staticmethod - def _format_intended_user_turns(turn_ids: list[int], context: MetricContext) -> str: - """Format the user's intended utterances for the audio-native prompt.""" - return "\n\n".join(f'Turn {tid}:\nUser said: "{context.intended_user_turns[tid]}"' for tid in turn_ids) + def _format_redacted_conversation(context: MetricContext) -> str: + """Format a redacted conversation trace for the audio-native judge. + + User turns and tool calls/responses are shown verbatim — the user turns are + the entity source, the tool calls are the evidence of what the agent understood. + Assistant turns are redacted to ``[Assistant speaks]`` so the judge cannot use + the agent's own (possibly mis-transcribed) speech to penalize an entity: e.g. if + the user says "INC462" and the agent asks "Did you say INC463?", that read-back + is an artifact of the agent's transcription, not evidence the user was misheard. + Only tool-call arguments and explicit user turns should drive the score. + """ + lines: list[str] = [] + for entry in context.conversation_trace or []: + role = entry.get("role") + entry_type = entry.get("type") + turn_id = entry.get("turn_id", "?") + if role == "user": + lines.append(f"Turn {turn_id} - User: {entry.get('content', '')}") + elif role == "assistant": + lines.append(f"Turn {turn_id} - [Assistant speaks]") + elif entry_type == "tool_call": + params = entry.get("parameters", {}) + lines.append(f"Turn {turn_id} - Tool Call ({entry.get('tool_name', 'unknown')}): {params}") + elif entry_type == "tool_response": + resp = entry.get("tool_response", {}) + lines.append(f"Turn {turn_id} - Tool Response ({entry.get('tool_name', 'unknown')}): {resp}") + return "\n".join(lines) async def _call_judge_raw(self, prompt: str, context: MetricContext) -> str | None: """Call LLM judge and return raw response text. diff --git a/src/eva/metrics/signatures.py b/src/eva/metrics/signatures.py index c16e4692..bdcae19f 100644 --- a/src/eva/metrics/signatures.py +++ b/src/eva/metrics/signatures.py @@ -64,14 +64,19 @@ def _source_hash(cls: type) -> str: def _prompt_hash_for_metric(cls: type[BaseMetric]) -> str | None: """Return the prompt template hash for judge metrics, or None. - All judge metrics in this codebase use `judge.{name}.user_prompt`. - A judge metric without a corresponding template raises KeyError — - that's a configuration bug we want surfaced. + Hashes *every* prompt template under ``judge.{name}`` (sorted by key), not just + ``user_prompt``, so a metric that ships multiple prompts — e.g. a per-pipeline + variant like ``audio_native_prompt`` — has all of them covered by drift detection. + A judge metric without any templates raises KeyError — that's a configuration + bug we want surfaced. """ if not issubclass(cls, TextJudgeMetric | AudioJudgeMetric): return None - template = get_prompt_manager().get_template(f"judge.{cls.name}.user_prompt") - return hash_prompt_template(template) + templates = get_prompt_manager().prompts.get("judge", {}).get(cls.name) + if not isinstance(templates, dict): + raise KeyError(f"No judge prompts found for metric {cls.name!r}") + combined = "\n".join(templates[key] for key in sorted(templates) if isinstance(templates[key], str)) + return hash_prompt_template(combined) def compute_all_metric_signatures() -> dict[str, dict[str, str | None]]: diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 280ba360..92148d4f 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -85,9 +85,9 @@ }, "TranscriptionAccuracyKeyEntitiesMetric": { "name": "transcription_accuracy_key_entities", - "prompt_hash": "877dfa61f232", - "source_hash": "1be843cc964d", - "version": "v0.4" + "prompt_hash": "3ed3a2fe7dcb", + "source_hash": "2926af63f4da", + "version": "v0.5" }, "TurnTakingMetric": { "name": "turn_taking", diff --git a/tests/unit/metrics/test_transcription_accuracy_key_entities.py b/tests/unit/metrics/test_transcription_accuracy_key_entities.py index d8de0216..ce4a8921 100644 --- a/tests/unit/metrics/test_transcription_accuracy_key_entities.py +++ b/tests/unit/metrics/test_transcription_accuracy_key_entities.py @@ -431,6 +431,43 @@ async def test_uses_audio_native_prompt_and_intended_turns(self, metric): assert "tool call" in sent_prompt.lower() assert "EMP124" in sent_prompt assert "Transcribed:" not in sent_prompt + # User turn is shown inline in the single Conversation block (not a separate section). + assert "User: My employee id is E M one two three" in sent_prompt + + @pytest.mark.asyncio + async def test_assistant_turns_are_redacted(self, metric): + """Assistant speech is redacted so a mis-transcribed read-back can't penalize an entity. + + If the user says "QX7710" and the agent's (imperfectly transcribed) reply reads + it back as "QX7711", the judge must not see that text and treat it as a mishear. + (Sentinel values chosen to not collide with any example in the prompt template.) + """ + context = make_metric_context( + pipeline_type=PipelineType.S2S, + intended_user_turns={1: "My incident is QX7710"}, + conversation_trace=[ + {"role": "user", "content": "My incident is QX7710", "turn_id": 1}, + {"role": "assistant", "content": "Did you say QX7711?", "turn_id": 1}, + { + "type": "tool_call", + "tool_name": "get_incident", + "parameters": {"incident_id": "QX7710"}, + "turn_id": 1, + }, + ], + ) + response = _make_judge_response( + [{"turn_id": 1, "summary": "ok", "entities": [{"type": "incident", "value": "QX7710", "correct": True}]}] + ) + generate_text = AsyncMock(return_value=(response, None)) + metric.llm_client.generate_text = generate_text + + await metric.compute(context) + + sent_prompt = generate_text.call_args[0][0][0]["content"] + assert "[Assistant speaks]" in sent_prompt # assistant turn present but redacted + assert "QX7711" not in sent_prompt # the mis-transcribed read-back is withheld + assert "QX7710" in sent_prompt # user turn + tool call are visible @pytest.mark.asyncio async def test_correct_entity_in_tool_call_scores_one(self, metric): From 3c0ae36877d92fe15bf29a183d567c0c622a5724 Mon Sep 17 00:00:00 2001 From: "wei.zhong1" Date: Mon, 20 Jul 2026 13:33:13 -0700 Subject: [PATCH 3/4] Review round 2: rate per tool call, trim prompt, per-call test coverage Address @gabegma's follow-up comments: - Rate each tool call individually (keyed by a per-call id) instead of per user turn. A user may state an entity in one turn and the agent use it several turns later; a single turn may contain multiple tool calls; and the same entity may appear in more than one call. Each labelled 'Tool Call [N]' is now its own rated unit, so a value can be wrong in one call and right in another, and no tool call is skipped when a turn has several. - Drop the now-redundant 'For example' read-back sentence (assistant turns are redacted, so the judge never sees it). - Ignore tool-call args sourced from tool responses / generated ids. - Tests: add correction-across-turns and multiple-tool-calls-in-one-turn cases. Bumps metric v0.5 -> v0.6, metrics_version 2.4.0 -> 2.5.0, regenerates signatures. Validated on a live AUDIO_LLM record: recovers entities that per-turn keying missed and rates each tool call separately. --- configs/prompts/judge.yaml | 103 ++++++------- src/eva/__init__.py | 2 +- .../transcription_accuracy_key_entities.py | 62 ++++---- tests/fixtures/metric_signatures.json | 6 +- ...est_transcription_accuracy_key_entities.py | 136 ++++++++++++++++-- 5 files changed, 215 insertions(+), 94 deletions(-) diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml index b4acbc02..8ec18ca6 100644 --- a/configs/prompts/judge.yaml +++ b/configs/prompts/judge.yaml @@ -1077,14 +1077,16 @@ judge: This is a speech-to-speech / audio-native agent: there is no reliable speech-to-text transcript of the user to inspect. Instead, the evidence of what the agent actually understood is the **arguments it passed to its tool calls**. When a user says "my id is EM123" and the agent then calls a tool with "EMP124", the agent misheard the entity. - You are given a Conversation containing user turns, tool calls, and tool responses. **The assistant's own spoken turns are redacted** (shown as `[Assistant speaks]`) and deliberately withheld: the agent's speech is itself imperfectly transcribed, so it must never be used as evidence. For example, if the user says "INC462" and the agent asks "Did you say INC463?", that read-back may just be a transcription artifact — do not treat it as the agent mishearing the entity. Judge only what the user said (user turns) against what the agent did (tool-call arguments). + You are given a Conversation containing user turns, tool calls, and tool responses. **The assistant's own spoken turns are redacted** (shown as `[Assistant speaks]`) and deliberately withheld: the agent's speech is itself imperfectly transcribed, so it must never be used as evidence. Judge only what the user said (user turns) against what the agent did (tool-call arguments). - Your task: - 1. For EACH user turn in the Conversation, identify all key entities in what the USER SAID. - 2. For each entity, find whether the agent used it in a tool-call argument and check whether it was captured CORRECTLY. - 3. Mark each entity as correct or incorrect. - 4. If the agent never used a given entity in any tool call, there is no evidence to judge understanding — include the entity but mark it as skipped. - 5. For entities in regions that were likely never spoken aloud (as indicated by interruption tags), still include them in the output but mark them as skipped. + Your task — evaluate each of the agent's TOOL CALLS individually: + 1. Each tool call in the Conversation is labelled with a stable id: `Tool Call [N]`. Produce **one output entry per tool call** that uses at least one user-provided entity, keyed by that id (`tool_call_id: N`). Be exhaustive — a single turn often contains MORE THAN ONE tool call (e.g. a lookup followed by a search, or a rebook followed by a seat assignment); rate each labelled tool call separately, never merge two of them or skip one. + 2. Within each tool call, look at **every argument**. For any argument whose value traces back to something the USER said — a user-provided key entity (name, confirmation code, date, amount, seat preference, etc.) — work out what the user actually said for it. + 3. Rate whether the value in that tool call matches what the user said: correct or incorrect. + 4. A user often states an entity in one turn and the agent uses it in a tool call several turns later — always rate it at the tool call, never at the user turn. If the same user value is used in more than one tool call, rate it in EACH one (so it can be correct in one tool call and wrong in another). + 5. If a tool call uses no user-provided entities (every argument came from a tool response or was generated by the agent), omit it from your output entirely. + 6. Only evaluate arguments that trace back to a value the USER provided. Ignore arguments the agent got from a tool RESPONSE or generated itself (e.g. a journey/record id returned by a prior search, or a passenger id) — those are not user entities. + 7. If a user-provided entity falls in a region that was likely never spoken aloud (interruption tags), mark it skipped and explain why. ## What Counts as an Entity An entity must have a **specific, concrete value** — something that could be passed as an input to a program or tool (not an AI, but a script or database lookup). Ask yourself: could this value be stored in a variable and used programmatically? @@ -1102,12 +1104,12 @@ judge: **Not an entity:** vague temporal words ("tomorrow", "next week", "morning"), general descriptors ("the cheap flight", "a long trip"), open-ended qualifiers ("less than an hour", "around noon"), or standalone honorifics/titles ("Mr.", "Ms."). ## IMPORTANT: What This Metric Measures - This measures whether the agent **understood** the entity the user provided, as evidenced by the value it passed to a tool call. It is NOT a faithfulness or tool-correctness metric. Do NOT penalize: + This measures whether the agent **understood** the entities the user provided, as evidenced by the values it passed to its tool calls. It is NOT a faithfulness or tool-correctness metric. Do NOT penalize: - The agent choosing the wrong tool, or omitting a tool call it should have made - - The agent using an entity that came from a tool RESPONSE rather than from the user (only user-provided entities are evaluated here) - - Correct values that simply never got reused (mark those skipped, not incorrect) + - Values in a tool call that came from a tool RESPONSE or that the agent generated itself (only user-provided values are evaluated) + - A user entity that was never used in any tool call — it simply is not rated (do not report it) - Only mark an entity incorrect when the agent demonstrably used a **different value** than the user provided (e.g. user "EM123" → tool call "EMP124", user "March 25th" → tool call "2026-03-26"). + Only mark a value incorrect when a tool call used a **different value** than the user provided (e.g. user "EM123" → tool call "EMP124", user "March 25th" → tool call "2026-03-26"). ## Non-Spoken Tags in the User Text The user text may contain non-spoken tags and markers — metadata that was never said aloud and must not be treated as entities. There are two kinds, and they are handled differently: @@ -1140,85 +1142,72 @@ judge: ## Examples + The user speaks the id in turns 1–2 (with a correction) but nothing is rated at those turns — each **tool call** is rated on its own. Tool Call [1] is wrong and [2] is correct for the same entity. Tool Call [3] uses only a `record_id` that came from a tool response, so it is omitted entirely. + **Example Input (Conversation):** - Turn 1 - User: My employee id is E M one two three and my name is John Smith. + Turn 1 - User: My employee id is E M one two three. Turn 1 - [Assistant speaks] - Turn 1 - Tool Call (lookup_employee): {{'employee_id': 'EMP124'}} - Turn 2 - User: Book it for December 15th. + Turn 2 - User: Actually, sorry — the id is E M four five six. Turn 2 - [Assistant speaks] - Turn 2 - Tool Call (create_booking): {{'date': '2026-12-15'}} - Turn 3 - User: [slow] The code is X X F six O H, with the letter O, [assistant interrupts] not zero. - Turn 3 - [Assistant speaks] + Turn 4 - [Assistant speaks] + Turn 4 - Tool Call [1] (lookup_employee): {{'employee_id': 'EM455'}} + Turn 4 - Tool Response (lookup_employee): {{'employee_id': 'EM455', 'record_id': 'R-9012'}} + Turn 6 - [Assistant speaks] + Turn 6 - Tool Call [2] (get_record): {{'employee_id': 'EM456', 'record_id': 'R-9012'}} + Turn 6 - Tool Call [3] (send_summary): {{'record_id': 'R-9012'}} **Example Response:** [ {{ - "turn_id": 1, + "tool_call_id": 1, + "tool_name": "lookup_employee", "entities": [ {{ "type": "employee_id", - "value": "E M one two three", - "captured_value": "EMP124", - "analysis": "User said EM123 but the agent called lookup_employee with EMP124 — misheard.", + "value": "E M four five six (user corrected it in turn 2)", + "captured_value": "EM455", + "analysis": "The user corrected the id to EM456 in turn 2; this tool call used EM455 — one character off, incorrect.", "correct": false, "skipped": false - }}, - {{ - "type": "name", - "value": "John Smith", - "captured_value": "not used", - "analysis": "The name was never passed to a tool call, so there is no evidence of how the agent understood it.", - "correct": false, - "skipped": true }} ], - "summary": "1 entity used (employee_id, incorrect). Name never used — skipped." + "summary": "Tool call used EM455; the user's (corrected) id is EM456 — incorrect." }}, {{ - "turn_id": 2, + "tool_call_id": 2, + "tool_name": "get_record", "entities": [ {{ - "type": "date", - "value": "December 15th", - "captured_value": "2026-12-15", - "analysis": "Agent passed 2026-12-15 to create_booking — matches December 15th.", + "type": "employee_id", + "value": "E M four five six", + "captured_value": "EM456", + "analysis": "Matches the user's corrected id. (record_id R-9012 came from the turn-4 tool response, not the user, so it is not rated.)", "correct": true, "skipped": false }} ], - "summary": "1 entity correct." - }}, - {{ - "turn_id": 3, - "entities": [ - {{ - "type": "confirmation_code", - "value": "X X F six O H", - "captured_value": "not used", - "analysis": "The agent never used this code in a tool call — no evidence to judge, skipped.", - "correct": false, - "skipped": true - }} - ], - "summary": "1 entity found; never used by the agent — skipped." + "summary": "Tool call used EM456 — correct." }} ] + Note Tool Call [3] does not appear: its only argument (record_id) came from a tool response, not the user. + ## Response Format - Respond with a JSON object. Each turn entry must include the turn_id matching the user turn number shown in the Conversation above: + Respond with a JSON array. Output **one entry per tool call that uses at least one user-provided entity**, keyed by the tool call's `[N]` id. Tool calls with no user-provided entities, and turns where the user only speaks, must NOT appear. [ {{ - "turn_id": , + "tool_call_id": , + "tool_name": "", "entities": [ {{ "type": "", - "value": "", - "captured_value": "", - "analysis": "", + "value": "", + "captured_value": "", + "analysis": "", "correct": , - "skipped": + "skipped": }} ], - "summary": "<1-2 sentence summary for this turn>" + "summary": "<1-2 sentence summary for this tool call>" }} ] diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 5306d24a..157f1dfa 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -11,4 +11,4 @@ # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.4.0" +metrics_version = "2.5.0" diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py index 2a61f39b..507a7667 100644 --- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py +++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py @@ -65,7 +65,7 @@ class TranscriptionAccuracyKeyEntitiesMetric(TextJudgeMetric): """ name = "transcription_accuracy_key_entities" - version = "v0.5" + version = "v0.6" description = ( "Debug metric: LLM judge evaluation of user key entity accuracy (STT in cascade, tool calls in audio-native)" ) @@ -114,7 +114,14 @@ async def compute(self, context: MetricContext) -> MetricScore: error=error, ) - # Compute scores for each turn, keyed by turn_id + # Audio-native rates one entry per tool call: normalize the judge's + # ``tool_call_id`` onto the shared ``turn_id`` key used by the loop below. + if context.is_audio_native: + for item in turn_evaluations: + if isinstance(item, dict) and "turn_id" not in item and "tool_call_id" in item: + item["turn_id"] = item["tool_call_id"] + + # Compute scores for each unit (user turn in cascade, tool call in audio-native), keyed by turn_id per_turn_ratings: dict[int, float | None] = {} per_turn_normalized: dict[int, float | None] = {} per_turn_explanations: dict[int, str] = {} @@ -225,21 +232,18 @@ def _build_per_entity_type_sub_metrics( def _build_prompt_and_turns(self, context: MetricContext) -> tuple[str, list[int]]: """Select the comparison source and judge prompt based on pipeline type. - CASCADE compares intended vs STT-transcribed user turns. Audio-native - pipelines (S2S / AUDIO_LLM) have no reliable STT, so the user's intended - turns are compared against the agent's tool-call arguments (from - ``conversation_trace``), which are the evidence of what it understood. + CASCADE compares intended vs STT-transcribed user turns, keyed on user + turns. Audio-native pipelines (S2S / AUDIO_LLM) have no reliable STT, so the + judge instead rates the agent's tool-call arguments against what the user + said — keyed on the tool-call turns (see ``_get_audio_native_turns``). Returns: Tuple of (rendered judge prompt, sorted turn IDs to evaluate). """ if context.is_audio_native: - turn_ids = self._get_audio_native_turns(context) - prompt = self.get_judge_prompt( - prompt_key="audio_native_prompt", - conversation=self._format_redacted_conversation(context), - ) - return prompt, turn_ids + conversation, tool_call_ids = self._build_audio_native_conversation(context) + prompt = self.get_judge_prompt(prompt_key="audio_native_prompt", conversation=conversation) + return prompt, tool_call_ids turn_ids = self._get_turns_to_evaluate(context) prompt = self.get_judge_prompt(user_turns=self._format_user_turns(turn_ids, context)) @@ -250,11 +254,6 @@ def _get_turns_to_evaluate(context: MetricContext) -> list[int]: """Return sorted turn IDs present in both intended and transcribed user turns.""" return sorted(context.intended_user_turns.keys() & context.transcribed_user_turns.keys()) - @staticmethod - def _get_audio_native_turns(context: MetricContext) -> list[int]: - """Return sorted intended user turn IDs (ground truth for audio-native pipelines).""" - return sorted(context.intended_user_turns.keys()) - @staticmethod def _format_user_turns(turn_ids: list[int], context: MetricContext) -> str: """Format expected-vs-transcribed user turn pairs for the cascade prompt.""" @@ -266,18 +265,29 @@ def _format_user_turns(turn_ids: list[int], context: MetricContext) -> str: ) @staticmethod - def _format_redacted_conversation(context: MetricContext) -> str: - """Format a redacted conversation trace for the audio-native judge. - - User turns and tool calls/responses are shown verbatim — the user turns are - the entity source, the tool calls are the evidence of what the agent understood. + def _build_audio_native_conversation(context: MetricContext) -> tuple[str, list[int]]: + """Build the redacted conversation for the audio-native judge and the tool-call ids. + + Audio-native scoring is keyed per **tool call** — the point where the agent + commits to a value — not per user turn: a user may state an entity in one turn + and the agent may use it in a tool call several turns later, a single turn may + contain several tool calls, and the same entity can appear (correctly or not) in + more than one. Each tool call is therefore labelled with a stable 1-based id + (``Tool Call [N]``) that the judge rates individually. + + User turns and tool calls/responses are shown verbatim — the user turns are the + entity source, the tool calls are the evidence of what the agent understood. Assistant turns are redacted to ``[Assistant speaks]`` so the judge cannot use the agent's own (possibly mis-transcribed) speech to penalize an entity: e.g. if the user says "INC462" and the agent asks "Did you say INC463?", that read-back is an artifact of the agent's transcription, not evidence the user was misheard. - Only tool-call arguments and explicit user turns should drive the score. + + Returns: + Tuple of (formatted conversation, ordered list of tool-call ids). """ lines: list[str] = [] + tool_call_ids: list[int] = [] + idx = 0 for entry in context.conversation_trace or []: role = entry.get("role") entry_type = entry.get("type") @@ -287,12 +297,14 @@ def _format_redacted_conversation(context: MetricContext) -> str: elif role == "assistant": lines.append(f"Turn {turn_id} - [Assistant speaks]") elif entry_type == "tool_call": + idx += 1 + tool_call_ids.append(idx) params = entry.get("parameters", {}) - lines.append(f"Turn {turn_id} - Tool Call ({entry.get('tool_name', 'unknown')}): {params}") + lines.append(f"Turn {turn_id} - Tool Call [{idx}] ({entry.get('tool_name', 'unknown')}): {params}") elif entry_type == "tool_response": resp = entry.get("tool_response", {}) lines.append(f"Turn {turn_id} - Tool Response ({entry.get('tool_name', 'unknown')}): {resp}") - return "\n".join(lines) + return "\n".join(lines), tool_call_ids async def _call_judge_raw(self, prompt: str, context: MetricContext) -> str | None: """Call LLM judge and return raw response text. diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 208966a0..946255f4 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -85,9 +85,9 @@ }, "TranscriptionAccuracyKeyEntitiesMetric": { "name": "transcription_accuracy_key_entities", - "prompt_hash": "3ed3a2fe7dcb", - "source_hash": "2926af63f4da", - "version": "v0.5" + "prompt_hash": "778c999bc0e5", + "source_hash": "adf7083be05b", + "version": "v0.6" }, "TurnTakingMetric": { "name": "turn_taking", diff --git a/tests/unit/metrics/test_transcription_accuracy_key_entities.py b/tests/unit/metrics/test_transcription_accuracy_key_entities.py index ce4a8921..01ea0112 100644 --- a/tests/unit/metrics/test_transcription_accuracy_key_entities.py +++ b/tests/unit/metrics/test_transcription_accuracy_key_entities.py @@ -388,11 +388,11 @@ def test_supports_cascade_and_audio_native(self): class TestAudioNativeCompute: - """Audio-native pipelines compare intended user entities against tool-call arguments.""" + """Audio-native pipelines rate the agent's tool-call arguments against what the user said.""" @pytest.mark.asyncio - async def test_uses_audio_native_prompt_and_intended_turns(self, metric): - """Audio-native path builds the tool-call prompt from intended turns + conversation trace.""" + async def test_uses_audio_native_prompt_with_tool_call_evidence(self, metric): + """Audio-native path builds the tool-call prompt from the redacted conversation trace.""" context = make_metric_context( pipeline_type=PipelineType.S2S, intended_user_turns={1: "My employee id is E M one two three"}, @@ -410,7 +410,8 @@ async def test_uses_audio_native_prompt_and_intended_turns(self, metric): response = _make_judge_response( [ { - "turn_id": 1, + "tool_call_id": 1, + "tool_name": "lookup_employee", "summary": "Misheard employee id", "entities": [ {"type": "employee_id", "value": "EM123", "correct": False, "skipped": False}, @@ -431,8 +432,9 @@ async def test_uses_audio_native_prompt_and_intended_turns(self, metric): assert "tool call" in sent_prompt.lower() assert "EMP124" in sent_prompt assert "Transcribed:" not in sent_prompt - # User turn is shown inline in the single Conversation block (not a separate section). + # User turn is shown inline in the single Conversation block, tool call is labelled [1]. assert "User: My employee id is E M one two three" in sent_prompt + assert "Tool Call [1] (lookup_employee)" in sent_prompt @pytest.mark.asyncio async def test_assistant_turns_are_redacted(self, metric): @@ -457,7 +459,13 @@ async def test_assistant_turns_are_redacted(self, metric): ], ) response = _make_judge_response( - [{"turn_id": 1, "summary": "ok", "entities": [{"type": "incident", "value": "QX7710", "correct": True}]}] + [ + { + "tool_call_id": 1, + "summary": "ok", + "entities": [{"type": "incident", "value": "QX7710", "correct": True}], + } + ] ) generate_text = AsyncMock(return_value=(response, None)) metric.llm_client.generate_text = generate_text @@ -467,7 +475,9 @@ async def test_assistant_turns_are_redacted(self, metric): sent_prompt = generate_text.call_args[0][0][0]["content"] assert "[Assistant speaks]" in sent_prompt # assistant turn present but redacted assert "QX7711" not in sent_prompt # the mis-transcribed read-back is withheld - assert "QX7710" in sent_prompt # user turn + tool call are visible + # Both the user turn AND the tool call must be visible to the judge. + assert "User: My incident is QX7710" in sent_prompt + assert "Tool Call [1] (get_incident): {'incident_id': 'QX7710'}" in sent_prompt @pytest.mark.asyncio async def test_correct_entity_in_tool_call_scores_one(self, metric): @@ -488,7 +498,7 @@ async def test_correct_entity_in_tool_call_scores_one(self, metric): response = _make_judge_response( [ { - "turn_id": 1, + "tool_call_id": 1, "summary": "Date captured correctly", "entities": [{"type": "date", "value": "December 15th", "correct": True, "skipped": False}], } @@ -501,3 +511,113 @@ async def test_correct_entity_in_tool_call_scores_one(self, metric): assert result.error is None assert result.score == 1.0 assert result.normalized_score == 1.0 + + @pytest.mark.asyncio + async def test_rates_per_tool_call_across_turns_with_correction(self, metric): + """Each tool call is rated on its own; a corrected entity is wrong in one call, right in another. + + User states an id in turn 1, corrects it in turn 2; the agent then makes two + tool calls (turns 4 and 6). Nothing is rated at the user turns — the two tool + calls are keyed 1 and 2 and rated independently. + """ + context = make_metric_context( + pipeline_type=PipelineType.S2S, + intended_user_turns={1: "My employee id is E M one two three", 2: "Sorry, it's E M four five six"}, + conversation_trace=[ + {"role": "user", "content": "My employee id is E M one two three", "turn_id": 1}, + {"role": "user", "content": "Sorry, it's E M four five six", "turn_id": 2}, + { + "type": "tool_call", + "tool_name": "lookup_employee", + "parameters": {"employee_id": "EM455"}, + "turn_id": 4, + }, + { + "type": "tool_call", + "tool_name": "lookup_employee", + "parameters": {"employee_id": "EM456"}, + "turn_id": 6, + }, + ], + ) + response = _make_judge_response( + [ + { + "tool_call_id": 1, + "summary": "wrong", + "entities": [{"type": "employee_id", "value": "EM456", "correct": False}], + }, + { + "tool_call_id": 2, + "summary": "right", + "entities": [{"type": "employee_id", "value": "EM456", "correct": True}], + }, + ] + ) + generate_text = AsyncMock(return_value=(response, None)) + metric.llm_client.generate_text = generate_text + + result = await metric.compute(context) + + assert result.error is None + # Rated units are the two tool calls (ids 1, 2), never the user turns (1, 2 as turns). + assert sorted(result.details["per_turn_ratings"].keys()) == [1, 2] + assert result.score == 0.5 # one tool call wrong, one right + sent_prompt = generate_text.call_args[0][0][0]["content"] + assert "Turn 4 - Tool Call [1]" in sent_prompt + assert "Turn 6 - Tool Call [2]" in sent_prompt + + @pytest.mark.asyncio + async def test_multiple_tool_calls_in_one_turn_rated_separately(self, metric): + """Two tool calls in the SAME turn are each labelled and rated individually.""" + context = make_metric_context( + pipeline_type=PipelineType.S2S, + intended_user_turns={1: "Confirmation is six V O R J U, last name Thompson"}, + conversation_trace=[ + {"role": "user", "content": "Confirmation is six V O R J U, last name Thompson", "turn_id": 1}, + { + "type": "tool_call", + "tool_name": "get_reservation", + "parameters": {"confirmation_number": "6VORJU", "last_name": "Thompson"}, + "turn_id": 1, + }, + { + "type": "tool_call", + "tool_name": "search_options", + "parameters": {"confirmation_number": "6VORXX"}, + "turn_id": 1, + }, + ], + ) + response = _make_judge_response( + [ + { + "tool_call_id": 1, + "tool_name": "get_reservation", + "summary": "both correct", + "entities": [ + {"type": "confirmation_code", "value": "6VORJU", "correct": True}, + {"type": "name", "value": "Thompson", "correct": True}, + ], + }, + { + "tool_call_id": 2, + "tool_name": "search_options", + "summary": "code wrong", + "entities": [{"type": "confirmation_code", "value": "6VORJU", "correct": False}], + }, + ] + ) + generate_text = AsyncMock(return_value=(response, None)) + metric.llm_client.generate_text = generate_text + + result = await metric.compute(context) + + assert result.error is None + # Both tool calls in turn 1 are rated as separate units (ids 1 and 2). + assert sorted(result.details["per_turn_ratings"].keys()) == [1, 2] + # Tool call 1: 2/2 correct = 1.0; tool call 2: 0/1 = 0.0; mean = 0.5. + assert result.score == 0.5 + sent_prompt = generate_text.call_args[0][0][0]["content"] + assert "Tool Call [1] (get_reservation)" in sent_prompt + assert "Tool Call [2] (search_options)" in sent_prompt From 777cee4f76ffc475517811e799f35508378d6394 Mon Sep 17 00:00:00 2001 From: "wei.zhong1" Date: Tue, 21 Jul 2026 13:54:55 -0700 Subject: [PATCH 4/4] Review round 3: per-tool-call prompt polish, single version bump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address @gabegma's follow-up comments: - Merge the two duplicate task items about ignoring tool-response/generated args. - Drop the interruption 'skipped' handling and the 'skipped' output field — an unspoken entity can't appear in a tool call, so there is nothing to skip. - Example now uses the pre-correction value (EM123) to show it is wrong once the user has corrected it; keep the entity 'value' clean (reasoning goes in analysis). - Replace the tool_call_id->turn_id normalization block with a setdefault fallback at the point of use. - Collapse to a single version bump for the whole PR (metric v0.4, metrics_version 2.3.0) instead of bumping each review round. --- configs/prompts/judge.yaml | 46 ++++++++----------- src/eva/__init__.py | 2 +- .../transcription_accuracy_key_entities.py | 12 ++--- tests/fixtures/metric_signatures.json | 6 +-- 4 files changed, 28 insertions(+), 38 deletions(-) diff --git a/configs/prompts/judge.yaml b/configs/prompts/judge.yaml index 8ec18ca6..8a56c32d 100644 --- a/configs/prompts/judge.yaml +++ b/configs/prompts/judge.yaml @@ -1084,9 +1084,7 @@ judge: 2. Within each tool call, look at **every argument**. For any argument whose value traces back to something the USER said — a user-provided key entity (name, confirmation code, date, amount, seat preference, etc.) — work out what the user actually said for it. 3. Rate whether the value in that tool call matches what the user said: correct or incorrect. 4. A user often states an entity in one turn and the agent uses it in a tool call several turns later — always rate it at the tool call, never at the user turn. If the same user value is used in more than one tool call, rate it in EACH one (so it can be correct in one tool call and wrong in another). - 5. If a tool call uses no user-provided entities (every argument came from a tool response or was generated by the agent), omit it from your output entirely. - 6. Only evaluate arguments that trace back to a value the USER provided. Ignore arguments the agent got from a tool RESPONSE or generated itself (e.g. a journey/record id returned by a prior search, or a passenger id) — those are not user entities. - 7. If a user-provided entity falls in a region that was likely never spoken aloud (interruption tags), mark it skipped and explain why. + 5. Only rate arguments that trace back to a value the USER provided. Ignore any argument sourced from a tool response or generated by the agent (e.g. a journey/record id returned by a prior search, or a passenger id). If a tool call has no user-provided arguments at all, omit it from your output entirely. ## What Counts as an Entity An entity must have a **specific, concrete value** — something that could be passed as an input to a program or tool (not an AI, but a script or database lookup). Ask yourself: could this value be stored in a variable and used programmatically? @@ -1112,15 +1110,13 @@ judge: Only mark a value incorrect when a tool call used a **different value** than the user provided (e.g. user "EM123" → tool call "EMP124", user "March 25th" → tool call "2026-03-26"). ## Non-Spoken Tags in the User Text - The user text may contain non-spoken tags and markers — metadata that was never said aloud and must not be treated as entities. There are two kinds, and they are handled differently: + The user text may contain non-spoken tags — metadata that was never said aloud and must never be treated as entities: - - **Audio-direction tags** like [slow], [firm], [annoyed] describe how the words were meant to be spoken. Ignore them entirely; they have no effect on scoring. - - **Interruption tags** indicate that part of the user text was likely never spoken, because the user was interrupted or talked over. An entity that was never spoken cannot have been understood, so do NOT penalize entities in these regions — include them but mark them as skipped. + - **Audio-direction tags** like [slow], [firm], [annoyed] describe how the words were meant to be spoken. Ignore them entirely. + - **Interruption tags** mark text the user likely never finished saying. No special handling is needed: anything that was never spoken cannot appear in a tool call, so there is simply nothing to rate. {interruption_tags_reference} - **Key principle:** Only evaluate entities that were reasonably expected to have been spoken aloud AND that the agent used in a tool call. If a tag indicates the user was interrupted before or during an entity, mark it skipped and explain why. - ## Conversation {conversation} @@ -1142,7 +1138,7 @@ judge: ## Examples - The user speaks the id in turns 1–2 (with a correction) but nothing is rated at those turns — each **tool call** is rated on its own. Tool Call [1] is wrong and [2] is correct for the same entity. Tool Call [3] uses only a `record_id` that came from a tool response, so it is omitted entirely. + The user gives an id in turn 1 and corrects it in turn 2; nothing is rated at those user turns — each **tool call** is rated on its own. Tool Call [1] used the pre-correction value EM123 (wrong, because the user had already corrected it) and Tool Call [2] used the corrected value EM456 (right). Tool Call [3] uses only a `record_id` that came from a tool response, so it is omitted entirely. **Example Input (Conversation):** Turn 1 - User: My employee id is E M one two three. @@ -1150,10 +1146,11 @@ judge: Turn 2 - User: Actually, sorry — the id is E M four five six. Turn 2 - [Assistant speaks] Turn 4 - [Assistant speaks] - Turn 4 - Tool Call [1] (lookup_employee): {{'employee_id': 'EM455'}} - Turn 4 - Tool Response (lookup_employee): {{'employee_id': 'EM455', 'record_id': 'R-9012'}} + Turn 4 - Tool Call [1] (lookup_employee): {{'employee_id': 'EM123'}} + Turn 4 - Tool Response (lookup_employee): {{'error': 'not_found'}} Turn 6 - [Assistant speaks] - Turn 6 - Tool Call [2] (get_record): {{'employee_id': 'EM456', 'record_id': 'R-9012'}} + Turn 6 - Tool Call [2] (lookup_employee): {{'employee_id': 'EM456'}} + Turn 6 - Tool Response (lookup_employee): {{'employee_id': 'EM456', 'record_id': 'R-9012'}} Turn 6 - Tool Call [3] (send_summary): {{'record_id': 'R-9012'}} **Example Response:** @@ -1164,26 +1161,24 @@ judge: "entities": [ {{ "type": "employee_id", - "value": "E M four five six (user corrected it in turn 2)", - "captured_value": "EM455", - "analysis": "The user corrected the id to EM456 in turn 2; this tool call used EM455 — one character off, incorrect.", - "correct": false, - "skipped": false + "value": "E M four five six", + "captured_value": "EM123", + "analysis": "The user first said EM123 but corrected it to EM456 in turn 2, so EM456 is the value they provided. This tool call used the pre-correction value EM123 — incorrect.", + "correct": false }} ], - "summary": "Tool call used EM455; the user's (corrected) id is EM456 — incorrect." + "summary": "Tool call used the pre-correction id EM123; the user's provided id is EM456 — incorrect." }}, {{ "tool_call_id": 2, - "tool_name": "get_record", + "tool_name": "lookup_employee", "entities": [ {{ "type": "employee_id", "value": "E M four five six", "captured_value": "EM456", - "analysis": "Matches the user's corrected id. (record_id R-9012 came from the turn-4 tool response, not the user, so it is not rated.)", - "correct": true, - "skipped": false + "analysis": "Matches the id the user provided (after their turn-2 correction).", + "correct": true }} ], "summary": "Tool call used EM456 — correct." @@ -1201,11 +1196,10 @@ judge: "entities": [ {{ "type": "", - "value": "", + "value": "", "captured_value": "", - "analysis": "", - "correct": , - "skipped": + "analysis": "", + "correct": }} ], "summary": "<1-2 sentence summary for this tool call>" diff --git a/src/eva/__init__.py b/src/eva/__init__.py index 157f1dfa..5a7690b0 100644 --- a/src/eva/__init__.py +++ b/src/eva/__init__.py @@ -11,4 +11,4 @@ # Bump metrics_version when changes affect metric computation (metrics code, # judge prompts, pricing tables, postprocessor). -metrics_version = "2.5.0" +metrics_version = "2.3.0" diff --git a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py index 507a7667..387fb8ab 100644 --- a/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py +++ b/src/eva/metrics/diagnostic/transcription_accuracy_key_entities.py @@ -65,7 +65,7 @@ class TranscriptionAccuracyKeyEntitiesMetric(TextJudgeMetric): """ name = "transcription_accuracy_key_entities" - version = "v0.6" + version = "v0.4" description = ( "Debug metric: LLM judge evaluation of user key entity accuracy (STT in cascade, tool calls in audio-native)" ) @@ -114,13 +114,6 @@ async def compute(self, context: MetricContext) -> MetricScore: error=error, ) - # Audio-native rates one entry per tool call: normalize the judge's - # ``tool_call_id`` onto the shared ``turn_id`` key used by the loop below. - if context.is_audio_native: - for item in turn_evaluations: - if isinstance(item, dict) and "turn_id" not in item and "tool_call_id" in item: - item["turn_id"] = item["tool_call_id"] - # Compute scores for each unit (user turn in cascade, tool call in audio-native), keyed by turn_id per_turn_ratings: dict[int, float | None] = {} per_turn_normalized: dict[int, float | None] = {} @@ -128,6 +121,9 @@ async def compute(self, context: MetricContext) -> MetricScore: per_turn_entity_details: dict[int, dict] = {} for turn_eval in turn_evaluations: + # Cascade keys results on ``turn_id``, audio-native on ``tool_call_id`` — accept either. + if isinstance(turn_eval, dict): + turn_eval.setdefault("turn_id", turn_eval.get("tool_call_id")) turn_id = resolve_turn_id(turn_eval, turns_to_evaluate, self.name) if turn_id is None: continue diff --git a/tests/fixtures/metric_signatures.json b/tests/fixtures/metric_signatures.json index 946255f4..0543a4b1 100644 --- a/tests/fixtures/metric_signatures.json +++ b/tests/fixtures/metric_signatures.json @@ -85,9 +85,9 @@ }, "TranscriptionAccuracyKeyEntitiesMetric": { "name": "transcription_accuracy_key_entities", - "prompt_hash": "778c999bc0e5", - "source_hash": "adf7083be05b", - "version": "v0.6" + "prompt_hash": "8492d1f0d168", + "source_hash": "d3c32f2497b1", + "version": "v0.4" }, "TurnTakingMetric": { "name": "turn_taking",