fix(evaluations): Compact traces for prompt (#1274)

Author: AkhileshNegi

backend/app/crud/evaluations/__init__.py

@@ -59,9 +59,8 @@
     EvaluationScore,
     NumericSummaryScore,
     SummaryScore,
-    TraceData,
-    TraceScore,
 )
+from app.models.evaluation import TraceData, TraceScore
 
 __all__ = [
     # Core

backend/app/crud/evaluations/core.py

@@ -12,9 +12,12 @@
 from app.core.util import now
 from app.crud.config.version import ConfigVersionCrud
 from app.crud.evaluations.langfuse import fetch_trace_scores_from_langfuse
-from app.crud.evaluations.score import DEFAULT_CATEGORY, EvaluationScore, TraceData
+from app.crud.evaluations.score import (
+    DEFAULT_CATEGORY,
+    EvaluationScore,
+)
 from app.models import EvaluationRun, EvaluationRunUpdate
-from app.models.evaluation import RunModeEnum
+from app.models.evaluation import RunModeEnum, TraceData
 from app.models.config.config import ConfigTag
 from app.models.llm.request import ConfigBlob, LLMCallConfig
 from app.models.stt_evaluation import EvaluationType

backend/app/crud/evaluations/fast.py

@@ -114,9 +114,9 @@
     EvaluationScore,
     OverallSummary,
     SummaryScore,
-    TraceData,
     compute_overall_summary,
 )
+from app.models.evaluation import TraceData
 from app.crud.evaluations.summary import generate_run_ai_summary
 from app.crud.job import create_batch_job, get_batch_job
 from app.models import EvaluationRun, EvaluationRunUpdate

backend/app/crud/evaluations/fast_traces.py

@@ -14,10 +14,9 @@
     DEFAULT_CATEGORY,
     GUARDRAIL_TRACE_KEYS,
     JUDGE_FAILED_REASON,
-    TraceData,
-    TraceScore,
     verdict_from_score,
 )
+from app.models.evaluation import TraceData, TraceScore
 
 # How many top KB matches to name in the knowledge_base trace comment.
 _KB_TOP_CHUNKS = 3

backend/app/crud/evaluations/langfuse.py

@@ -22,9 +22,8 @@
     COSINE_SCORE_NAME,
     DEFAULT_CATEGORY,
     EvaluationScore,
-    TraceData,
-    TraceScore,
 )
+from app.models.evaluation import TraceData, TraceScore
 
 logger = logging.getLogger(__name__)
 

backend/app/crud/evaluations/merge.py

@@ -21,10 +21,9 @@
     EvaluationScore,
     NumericSummaryScore,
     SummaryScore,
-    TraceData,
-    TraceScore,
     UNSCOREABLE_REASONS,
 )
+from app.models.evaluation import TraceData, TraceScore
 
 logger = logging.getLogger(__name__)
 

backend/app/crud/evaluations/processing.py

@@ -57,12 +57,11 @@
     COSINE_SCORE_COMMENT,
     COSINE_SCORE_NAME,
     EvaluationScore,
-    TraceData,
 )
 from app.crud.job import get_batch_job, update_batch_job
 from app.models import EvaluationRun, EvaluationRunUpdate
 from app.models.batch_job import BatchJob, BatchJobUpdate
-from app.models.evaluation import RunModeEnum
+from app.models.evaluation import RunModeEnum, TraceData
 from app.utils import get_langfuse_client, get_openai_client
 
 logger = logging.getLogger(__name__)

backend/app/crud/evaluations/score.py

@@ -3,6 +3,8 @@
 from enum import Enum
 from typing import Literal, NotRequired, TypedDict
 
+from app.models.evaluation import TraceData
+
 DEFAULT_CATEGORY: str = "Other"
 
 
@@ -61,35 +63,6 @@ def verdict_from_score(score: float) -> VerdictEnum:
 )
 
 
-class TraceScore(TypedDict):
-    """A score attached to a trace."""
-
-    name: str
-    value: float | str
-    data_type: str
-    comment: NotRequired[str]
-    verdict: NotRequired[str]
-    # True for placeholder scores on unscoreable items; excluded from summary stats.
-    unscoreable: NotRequired[bool]
-
-
-class TraceData(TypedDict):
-    """Data for a single trace including Q&A and scores."""
-
-    trace_id: str
-    question: str
-    llm_answer: str
-    question_id: int | None
-    ground_truth_answer: str
-    category: NotRequired[str]
-    # "blocked: ..." | "rephrased" | "applied"; None if guardrails didn't fire.
-    guardrail: NotRequired[str | None]
-    # LLM input after input guardrails / output before output guardrails.
-    input_to_llm: NotRequired[str | None]
-    output_from_llm: NotRequired[str | None]
-    scores: list[TraceScore]
-
-
 # Merged key-by-key since None is meaningful.
 GUARDRAIL_TRACE_KEYS: tuple[
     Literal["guardrail", "input_to_llm", "output_from_llm"], ...

backend/app/crud/evaluations/summary.py

@@ -4,7 +4,7 @@
 import logging
 
 from app.core.config import settings
-from app.crud.evaluations.score import TraceData
+from app.models.evaluation import TraceData
 from app.services.llm.providers.claude import (
     STOP_REASON_COMPLETE,
     ClaudeProvider,

backend/app/models/evaluation.py

@@ -1,6 +1,6 @@
 from datetime import datetime
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Literal, Optional
+from typing import TYPE_CHECKING, Any, Literal, NotRequired, Optional, TypedDict
 from uuid import UUID
 
 from pydantic import BaseModel, Field, HttpUrl
@@ -629,3 +629,57 @@ class PromptRecommendationJobPublic(SQLModel):
     recommendation_type: Literal["prompt"] = "prompt"
     config_version: ConfigVersionPublic | None = None
     error_message: str | None = None
+
+
+class TraceScore(TypedDict):
+    """A score attached to a trace."""
+
+    name: str
+    value: float | str
+    data_type: str
+    comment: NotRequired[str]
+    verdict: NotRequired[str]
+    # True for placeholder scores on unscoreable items; excluded from summary stats.
+    unscoreable: NotRequired[bool]
+
+
+class TraceData(TypedDict):
+    """Data for a single trace including Q&A and scores."""
+
+    trace_id: str
+    question: str
+    llm_answer: str
+    question_id: int | None
+    ground_truth_answer: str
+    category: NotRequired[str]
+    # "blocked: ..." | "rephrased" | "applied"; None if guardrails didn't fire.
+    guardrail: NotRequired[str | None]
+    # LLM input after input guardrails / output before output guardrails.
+    input_to_llm: NotRequired[str | None]
+    output_from_llm: NotRequired[str | None]
+    scores: list[TraceScore]
+
+
+class TraceScoreSummary(TypedDict):
+    """A TraceScore without the UI-only fields (data_type, verdict)."""
+
+    name: str
+    value: float | str
+    comment: NotRequired[str]
+    unscoreable: NotRequired[bool]
+
+
+class TraceAttempt(TypedDict):
+    """One run of a question: its answer and scores."""
+
+    llm_answer: str
+    scores: list[TraceScoreSummary]
+
+
+class TraceQuestion(TypedDict):
+    """A dataset question with one attempt per duplication-factor repeat."""
+
+    question_id: int | str | None
+    question: str
+    ground_truth_answer: str
+    attempts: list[TraceAttempt]