diff --git a/pyproject.toml b/pyproject.toml index 09be12e..bdfb879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "sinonym" -version = "0.3.0" +version = "0.3.1" description = "Chinese Name Detection and Normalization Module" readme = "README.md" requires-python = ">=3.10" diff --git a/sinonym/utils/string_manipulation.py b/sinonym/utils/string_manipulation.py index d50ecd6..7355f11 100644 --- a/sinonym/utils/string_manipulation.py +++ b/sinonym/utils/string_manipulation.py @@ -590,6 +590,11 @@ def _normalize_and_capitalize_single_part(part: str) -> str: # Normalize Unicode and remove diacritical marks normalized = unicodedata.normalize("NFD", part) without_diacritics = "".join(c for c in normalized if unicodedata.category(c) != "Mn") + if not without_diacritics: + # Token was made up entirely of combining marks / variation selectors (all + # category Mn), so stripping them leaves nothing to capitalize. Return the + # empty result rather than indexing [0] into an empty string (IndexError). + return without_diacritics return without_diacritics[0].upper() + without_diacritics[1:].lower() @staticmethod diff --git a/tests/test_bug_report_fixes.py b/tests/test_bug_report_fixes.py index 903de71..0ec4981 100644 --- a/tests/test_bug_report_fixes.py +++ b/tests/test_bug_report_fixes.py @@ -539,3 +539,23 @@ def __getattr__(self, name): def test_name_order_routing_priors_are_imported_from_canonical_data(): assert name_order_routing.NAME_PRIOR_COMMON_CHINESE_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_COMMON_CHINESE_SURNAMES assert name_order_routing.NAME_PRIOR_KOREAN_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_KOREAN_SURNAMES + + +def test_capitalize_name_part_all_combining_marks_does_not_crash(): + """Regression: a name part consisting solely of combining marks / variation selectors + (all Unicode category Mn) strips to '' after NFD + Mn removal. capitalize_name_part then + indexed [0] into the empty string and raised IndexError. It must now return '' instead.""" + from sinonym.utils.string_manipulation import StringManipulationUtils + + assert StringManipulationUtils.capitalize_name_part("́") == "" # lone combining acute + assert StringManipulationUtils.capitalize_name_part("\U000E0100") == "" # ideographic variation selector + # real names are unaffected (still stripped + capitalized) + assert StringManipulationUtils.capitalize_name_part("josé") == "Jose" + assert StringManipulationUtils.capitalize_name_part("ou-yang") == "Ou-Yang" + + +def test_normalize_name_with_embedded_variation_selector_does_not_crash(detector): + """Integration regression: a corpus name with an embedded ideographic variation selector + (U+E0100, seen on paper_id 274957019) crashed normalize_name via the capitalize step.""" + result = detector.normalize_name("彬人 樽\U000E0100井") # 彬人 樽󠄀井 + assert result is not None # no exception raised; the classification value is out of scope here