Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ build-backend = "hatchling.build"

[project]
name = "sinonym"
version = "0.3.0"
version = "0.3.1"
description = "Chinese Name Detection and Normalization Module"
readme = "README.md"
requires-python = ">=3.10"
Expand Down
5 changes: 5 additions & 0 deletions sinonym/utils/string_manipulation.py
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,11 @@ def _normalize_and_capitalize_single_part(part: str) -> str:
# Normalize Unicode and remove diacritical marks
normalized = unicodedata.normalize("NFD", part)
without_diacritics = "".join(c for c in normalized if unicodedata.category(c) != "Mn")
if not without_diacritics:
# Token was made up entirely of combining marks / variation selectors (all
# category Mn), so stripping them leaves nothing to capitalize. Return the
# empty result rather than indexing [0] into an empty string (IndexError).
return without_diacritics
return without_diacritics[0].upper() + without_diacritics[1:].lower()

@staticmethod
Expand Down
20 changes: 20 additions & 0 deletions tests/test_bug_report_fixes.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,3 +539,23 @@ def __getattr__(self, name):
def test_name_order_routing_priors_are_imported_from_canonical_data():
assert name_order_routing.NAME_PRIOR_COMMON_CHINESE_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_COMMON_CHINESE_SURNAMES
assert name_order_routing.NAME_PRIOR_KOREAN_SURNAMES is chinese_names_data.NAME_ORDER_ROUTING_KOREAN_SURNAMES


def test_capitalize_name_part_all_combining_marks_does_not_crash():
"""Regression: a name part consisting solely of combining marks / variation selectors
(all Unicode category Mn) strips to '' after NFD + Mn removal. capitalize_name_part then
indexed [0] into the empty string and raised IndexError. It must now return '' instead."""
from sinonym.utils.string_manipulation import StringManipulationUtils

assert StringManipulationUtils.capitalize_name_part("́") == "" # lone combining acute
assert StringManipulationUtils.capitalize_name_part("\U000E0100") == "" # ideographic variation selector
# real names are unaffected (still stripped + capitalized)
assert StringManipulationUtils.capitalize_name_part("josé") == "Jose"
assert StringManipulationUtils.capitalize_name_part("ou-yang") == "Ou-Yang"


def test_normalize_name_with_embedded_variation_selector_does_not_crash(detector):
"""Integration regression: a corpus name with an embedded ideographic variation selector
(U+E0100, seen on paper_id 274957019) crashed normalize_name via the capitalize step."""
result = detector.normalize_name("彬人 樽\U000E0100井") # 彬人 樽󠄀井
assert result is not None # no exception raised; the classification value is out of scope here
Loading