From 89a5a117b0a95f91e8d1a1cc021c30df1a5fb626 Mon Sep 17 00:00:00 2001 From: Atalya Alon Date: Tue, 14 Jul 2026 12:01:33 -0700 Subject: [PATCH] Fix IndexError on name parts that are all combining marks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit capitalize_name_part strips Unicode Mn characters (combining marks, variation selectors) after NFD normalization. A part composed entirely of Mn chars (e.g. a lone combining acute, or an ideographic variation selector like U+E0100 embedded in a CJK name) became empty, and the capitalizer then indexed [0] into the empty string, raising IndexError and crashing normalize_name. Return the empty result instead. Seen in production on paper_id 274957019 ('彬人 樽󠄀井', with U+E0100). Bump 0.3.0 -> 0.3.1. --- pyproject.toml | 2 +- sinonym/utils/string_manipulation.py | 5 +++++ tests/test_bug_report_fixes.py | 20 ++++++++++++++++++++ 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 09be12e..bdfb879 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "sinonym" -version = "0.3.0" +version = "0.3.1" description = "Chinese Name Detection and Normalization Module" readme = "README.md" requires-python = ">=3.10" diff --git a/sinonym/utils/string_manipulation.py b/sinonym/utils/string_manipulation.py index d50ecd6..7355f11 100644 --- a/sinonym/utils/string_manipulation.py +++ b/sinonym/utils/string_manipulation.py @@ -590,6 +590,11 @@ def _normalize_and_capitalize_single_part(part: str) -> str: # Normalize Unicode and remove diacritical marks normalized = unicodedata.normalize("NFD", part) without_diacritics = "".join(c for c in normalized if unicodedata.category(c) != "Mn") + if not without_diacritics: + # Token was made up entirely of combining marks / variation selectors (all + # category Mn), so stripping them leaves nothing to capitalize. Return the + # empty result rather than indexing [0] into an empty string (IndexError). + return without_diacritics return without_diacritics[0].upper() + without_diacritics[1:].lower() @staticmethod diff --git a/tests/test_bug_report_fixes.py b/tests/test_bug_report_fixes.py index 903de71..0ec4981 100644 --- a/tests/test_bug_report_fixes.py +++ b/tests/test_bug_report_fixes.py @@ -539,3 +539,23 @@ def __getattr__(self, name): def test_name_order_routing_priors_are_imported_from_canonical_data(): assert name_order_routing.NAME_PRIOR_COMMON_CHINESE_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_COMMON_CHINESE_SURNAMES assert name_order_routing.NAME_PRIOR_KOREAN_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_KOREAN_SURNAMES + + +def test_capitalize_name_part_all_combining_marks_does_not_crash(): + """Regression: a name part consisting solely of combining marks / variation selectors + (all Unicode category Mn) strips to '' after NFD + Mn removal. capitalize_name_part then + indexed [0] into the empty string and raised IndexError. It must now return '' instead.""" + from sinonym.utils.string_manipulation import StringManipulationUtils + + assert StringManipulationUtils.capitalize_name_part("́") == "" # lone combining acute + assert StringManipulationUtils.capitalize_name_part("\U000E0100") == "" # ideographic variation selector + # real names are unaffected (still stripped + capitalized) + assert StringManipulationUtils.capitalize_name_part("josé") == "Jose" + assert StringManipulationUtils.capitalize_name_part("ou-yang") == "Ou-Yang" + + +def test_normalize_name_with_embedded_variation_selector_does_not_crash(detector): + """Integration regression: a corpus name with an embedded ideographic variation selector + (U+E0100, seen on paper_id 274957019) crashed normalize_name via the capitalize step.""" + result = detector.normalize_name("彬人 樽\U000E0100井") # 彬人 樽󠄀井 + assert result is not None # no exception raised; the classification value is out of scope here