diff --git a/src/heretic/scorers/keyword_rate.py b/src/heretic/scorers/keyword_rate.py index 9895f039..19c0e77e 100644 --- a/src/heretic/scorers/keyword_rate.py +++ b/src/heretic/scorers/keyword_rate.py @@ -1,7 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-or-later # Copyright (C) 2025-2026 Philipp Emanuel Weidmann + contributors -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, field_validator from rich.markup import escape from heretic.config import DatasetSpecification, SingleDatasetSpecification @@ -65,6 +65,17 @@ class Settings(BaseModel): description="Strings whose presence in a response (case insensitive) identifies the response as a keyword match.", ) + @field_validator("keyword_markers") + @classmethod + def validate_keyword_markers(cls, value: list[str]) -> list[str]: + if not value: + raise ValueError("keyword marker list cannot be empty") + + if any(not marker.strip() for marker in value): + raise ValueError("keyword markers cannot be empty or whitespace") + + return value + print_responses: bool = Field( default=False, description="Whether to print prompt/response pairs when counting keyword matches.", diff --git a/tests/test_keyword_rate.py b/tests/test_keyword_rate.py index 2ef98c76..7bbb7fbf 100644 --- a/tests/test_keyword_rate.py +++ b/tests/test_keyword_rate.py @@ -6,6 +6,7 @@ from typing import cast from unittest.mock import patch +from pydantic import ValidationError from rich.console import Console from heretic.config import Settings as HereticSettings @@ -23,6 +24,20 @@ def get_responses(self, prompts: list[Prompt]) -> list[str]: class KeywordRateTests(unittest.TestCase): + def test_rejects_empty_keyword_markers(self) -> None: + for markers in [[""], [" ", "\t"]]: + with self.subTest(markers=markers): + with self.assertRaisesRegex( + ValidationError, "keyword markers cannot be empty or whitespace" + ): + Settings(keyword_markers=markers) + + def test_rejects_empty_keyword_marker_list(self) -> None: + with self.assertRaisesRegex( + ValidationError, "keyword marker list cannot be empty" + ): + Settings(keyword_markers=[]) + def test_print_responses_shows_brackets_literally(self) -> None: scorer = KeywordRate( heretic_settings=HereticSettings.model_construct(),