diff --git a/README.md b/README.md index 5f0356c..9ed59d3 100644 --- a/README.md +++ b/README.md @@ -195,7 +195,9 @@ if result.success: result = detector.normalize_name("John Smith") if not result.success: print(f"Error: {result.error_message}") + print(f"Canonical person name: {result.canonical_name.text}") # Expected Output: Error: name not recognised as Chinese + # Canonical person name: John Smith # --- Example 5: Japanese name in Chinese characters (ML-enhanced detection) --- result = detector.normalize_name("山田太郎") @@ -251,6 +253,76 @@ When you call `normalize_name`, you get a `ParseResult` with helpful structured - `parsed_original_order`: A `ParsedName` with the same semantic `surname` and `given_name` labels as `parsed`, plus an `order` list that records how those components appeared in the input. +- `canonical_name`: an all-person canonical representation. This is populated + for Chinese and non-Chinese people, while the legacy `success`, `result`, and + `parsed` fields remain Chinese-recognition fields. + +### Canonical names for all people + +`canonical_name.text` is the fully normalized display form. Its `normalized` +components expose `given_name`, `middle_name`, `surname`, and `suffix`, plus +immutable token tuples and their display order. Name dashes and apostrophes are +standardized to ASCII `-` and `'`; obvious titles and credentials are removed; +and true generational suffixes are kept in the suffix field. + +Periods are treated by role and shape rather than removed globally. Known +leading titles and trailing credentials are consumed; generational suffixes are +canonicalized; pure dotted initial clusters are uppercased while preserving the +source dot positions; transliteration abbreviations such as ``M.Yu.`` retain +their mixed casing; and a terminal full stop on an ordinary word is removed as +sentence punctuation. + +```python +western = detector.normalize_name("Dr. Ana–Maria O’Neill PhD") +assert not western.success # unchanged: not recognized as Chinese +assert western.parsed is None +assert western.canonical_name.text == "Ana-Maria O'Neill" +assert western.canonical_name.normalized.given_name == "Ana-Maria" +assert western.canonical_name.normalized.surname == "O'Neill" + +suffixed = detector.normalize_person_name("Steve Blando IV") +assert suffixed.text == "Steve Blando IV" +assert suffixed.normalized.suffix == "IV" + +repaired = detector.normalize_person_name_components( + first_name="dr steve", + middle_name="marsh", + last_name="phd", +) +assert repaired.text == "Steve Marsh" +assert repaired.normalized.given_name == "Steve" +assert repaired.normalized.middle_name == "" +assert repaired.normalized.surname == "Marsh" +``` + +For an undelimited raw name, token order alone cannot always distinguish middle +names from multi-token family names. Passing structured components makes the +caller's first/middle/last assignment the default normalized contract. Cultural +convention alone does not move a source middle token into the given or family +field. The same conservative East Asian router used for raw names may reinterpret +a structured source sequence when its directional evidence is decisive; the +structured ``canonical_name.source`` still retains the caller's original roles, +order, and token lineage. Otherwise, roles are re-inferred only when mechanical +cleanup, such as removing a title or credential, empties a required boundary. + +Raw parsing preserves visible order by default. A separate conservative router +assigns semantic family-first components only for evidence combinations that +held the existing non-Chinese benchmark constant: three-syllable compact +Hangul; Japanese native text supported by the Chinese/Japanese classifier and +component dictionaries; strict Korean romanized shapes; diacritic-bearing +Vietnamese with a supported initial surname; and two-token Japanese +romanizations whose surname/given dictionaries support only the family-first +direction. Ambiguous or unsupported names retain the generic input-order +normalization. + +The East Asian component assets contain no complete-person exceptions. Their +sources, hashes, licenses, and regeneration command are documented in +`sinonym/data/EAST_ASIAN_NAME_LEXICONS.md` and +`scripts/build_east_asian_name_lexicons.py`. + +TIMO clients can opt into `sinonym_v2` or `sinonym_routing_v2` to receive the +same nested canonical payload. The existing `sinonym_v1` and +`sinonym_routing_v1` response schemas remain unchanged. Notes: - The tokens in `parsed` and `parsed_original_order` are the same normalized tokens; only the conceptual ordering differs via the `order` list. diff --git a/docs/canonical_name_evaluation.md b/docs/canonical_name_evaluation.md new file mode 100644 index 0000000..1ac99bd --- /dev/null +++ b/docs/canonical_name_evaluation.md @@ -0,0 +1,115 @@ +# Canonical name normalization evaluation + +## Result + +The final rules were evaluated on 1,000 manually reviewed, real non-Chinese +author names: + +| Measure | Exact | Accuracy | +|---|---:|---:| +| Person outcome | 1,000 / 1,000 | 100% | +| Canonical display text | 1,000 / 1,000 | 100% | +| First/given component | 1,000 / 1,000 | 100% | +| Suffix component | 1,000 / 1,000 | 100% | +| Middle component | 982 / 1,000 | 98.2% | +| Last/surname component | 982 / 1,000 | 98.2% | +| All five fields together | 982 / 1,000 | 98.2% | + +The 18 component-only differences are reported as semantic-boundary +differences, separately from display accuracy. Raw undelimited strings cannot +identify those middle/surname boundaries reliably. When callers provide +structured first/middle/last fields, those source roles are authoritative by +default and are repaired only after mechanical cleanup empties a boundary. + +The originally frozen 200-name holdout scores 196/200 (98.0%) on semantic +components and 200/200 on canonical display. The four component differences +remain visible under the source-preserving policy rather than being closed by +authority-specific family-span rules. + +## Data and manual review + +The sampling pool combined: + +- 978 selected names from the internal S2 scholarly-author Parquet. The pinned + source file contains 41,551,414 rows and has SHA-256 + `E4050B4832DAAF2F0B8464F0B78E6CA0D35412897BDDA5AF148F7BACBEE9D363`. +- 22 selected names from the repository's ACL 2025 author list. + +Candidate sampling was deterministic, source-balanced, deduplicated, and +stratified by visible name shape. The final 1,000 contain: + +| Shape | Count | +|---|---:| +| Ordinary | 498 | +| Initials | 130 | +| Non-ASCII Latin | 94 | +| Hyphen | 69 | +| Apostrophe | 57 | +| Family particle | 42 | +| Comma form | 40 | +| Suffix | 39 | +| Four-plus tokens | 31 | + +Every initial output was assigned manually from the raw string under the +written rubric. Parser, model, and normalizer outputs were not shown to or used +by that initial labeling step. Ambiguous, corrupt, Chinese, and non-person rows +were excluded from gold rather than guessed. Review produced 1,053 usable gold +rows; a salted hash selected 1,000, then a separate salted hash fixed an 800/200 +development/holdout split. The remaining 53 rows were a development reserve. + +After the final evaluation exposed 23 ambiguous middle/surname boundaries, the +user explicitly requested a web-source adjudication of every mismatch. Six +manual labels were corrected from structured institutional or bibliographic +authority records. The remaining 18 boundaries are retained as semantic +benchmark differences; they are not treated as sufficient evidence to override +structured source roles. Each changed review row records its external source +URLs, and the full adjudication ledger is retained in ignored scratch artifacts. + +The selected JSONL has SHA-256 +`B367254A7282DA3B30A37B478A679202E5C65D465541A3A0E772AA649D30B8DE`. +It contains 977 high-confidence and 23 medium-confidence reviews. + +Raw S2-derived names and manual review files remain under ignored +`scratch/canonical_names/`; they are not committed because they contain +internal personal data with upstream redistribution constraints. The ACL +source's redistribution terms are also not documented locally. + +## Evaluation protocol + +Rules were iterated only against the 800-name development split and the +53-name reserve. The holdout was evaluated once after the tuning rules were +frozen. A later independent code review found specification-level edge cases +not represented in the gold packet; those fixes were developed only against +synthetic regression strings, without consulting holdout labels. The full +packet was then rerun as a regression check and the reported holdout score was +unchanged. Exact match requires the person outcome, canonical text, first, +middle, last, and suffix to all agree with the manual record. + +Reproduction commands, when the ignored source/review artifacts are present: + +```powershell +uv run python scratch\canonical_names\assemble_gold.py ` + --acl-packet scratch\canonical_names\packets\acl_pilot_60.jsonl ` + --s2-packet scratch\canonical_names\packets\s2_candidates_1300.jsonl ` + --reviews-dir scratch\canonical_names\reviews ` + --output scratch\canonical_names\gold\canonical_non_chinese_gold_1000.jsonl ` + --reserve-output scratch\canonical_names\gold\canonical_non_chinese_reserve.jsonl ` + --manifest scratch\canonical_names\gold\manifest.json ` + --size 1000 --holdout-size 200 + +uv run python scratch\canonical_names\evaluate_gold.py ` + --input scratch\canonical_names\gold\canonical_non_chinese_gold_1000.jsonl ` + --split all --limit 1000 ` + --report scratch\canonical_names\metrics\all_final.json ` + --mismatches scratch\canonical_names\metrics\all_final_mismatches.jsonl +``` + +## Boundary policy + +An undelimited raw string still cannot identify every middle/surname boundary +from shape alone. The normalizer therefore keeps the conservative final-token +surname rule, family-particle rules, and initial-shape rules as its default. +It does not use person-specific or authority-specific family-span exceptions. +Structured input preserves the supplied first/middle/last roles unless +mechanical cleanup empties a required boundary. Semantic boundary evaluation is +reported separately rather than changing that storage contract. diff --git a/pyproject.toml b/pyproject.toml index bdfb879..658acc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "sinonym" -version = "0.3.1" +version = "0.4.0" description = "Chinese Name Detection and Normalization Module" readme = "README.md" requires-python = ">=3.10" @@ -67,6 +67,7 @@ include = [ "scripts/verify_multiprocess.py", "scripts/generate_surname_romanizations.py", "scripts/generate_name_statistics.py", + "scripts/build_east_asian_name_lexicons.py", "scripts/train_ml_classifier_for_chinese_vs_japanese.py", "README.md", "pyproject.toml", diff --git a/scripts/build_east_asian_name_lexicons.py b/scripts/build_east_asian_name_lexicons.py new file mode 100644 index 0000000..ef29e0c --- /dev/null +++ b/scripts/build_east_asian_name_lexicons.py @@ -0,0 +1,267 @@ +# ruff: noqa: INP001 +"""Build deterministic East Asian name-order lexicons from pinned open data.""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import hashlib +import io +import json +import sys +import time +import unicodedata +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +COUNTRY_COMMIT = "eb62e13d4d62dd96cdfae79d293a02066352205f" +JAPANESE_COMMIT = "c5220278652e7bae05b06cfaf527f1b09a100de6" +USER_AGENT = "sinonym-east-asian-lexicon-builder/1.0" +MAX_ATTEMPTS = 4 +RETRYABLE_HTTP_CODES = frozenset({429, 502, 503, 504}) +MIN_GIVEN_FIELDS = 2 + + +@dataclass(frozen=True) +class Source: + """One hash-pinned upstream CSV.""" + + name: str + url: str + sha256: str + license: str + repository: str + + +SOURCES = ( + Source( + name="country_surnames", + url=( + "https://raw.githubusercontent.com/sigpwned/" + f"popular-names-by-country-dataset/{COUNTRY_COMMIT}/common-surnames-by-country.csv" + ), + sha256="32cb28bea558a9d353feeef097da03c6488a4e7ba9398e5983cee2f0b9caa91c", + license="CC0-1.0", + repository="https://github.com/sigpwned/popular-names-by-country-dataset", + ), + Source( + name="japanese_surnames", + url=( + "https://raw.githubusercontent.com/shuheilocale/" + f"japanese-personal-name-dataset/{JAPANESE_COMMIT}/" + "japanese_personal_name_dataset/dataset/last_name_org.csv" + ), + sha256="02706bd06932d6bde121e6fbaf9bc2a88c43e6f015128215eecb213223523d30", + license="MIT", + repository="https://github.com/shuheilocale/japanese-personal-name-dataset", + ), + Source( + name="japanese_male_given_names", + url=( + "https://raw.githubusercontent.com/shuheilocale/" + f"japanese-personal-name-dataset/{JAPANESE_COMMIT}/" + "japanese_personal_name_dataset/dataset/first_name_man_org.csv" + ), + sha256="c14a67047f6101fa3234e87c3f643f68ec1807f3fdb545855e57ab9bd5cd5a3b", + license="MIT", + repository="https://github.com/shuheilocale/japanese-personal-name-dataset", + ), + Source( + name="japanese_female_given_names", + url=( + "https://raw.githubusercontent.com/shuheilocale/" + f"japanese-personal-name-dataset/{JAPANESE_COMMIT}/" + "japanese_personal_name_dataset/dataset/first_name_woman_org.csv" + ), + sha256="de8ce6f8f430e14e0036ca209656a1ec06e9bcd83e9defffe1b0654dc636f9a9", + license="MIT", + repository="https://github.com/shuheilocale/japanese-personal-name-dataset", + ), +) + + +def parse_args() -> argparse.Namespace: + """Parse explicit output paths.""" + parser = argparse.ArgumentParser() + parser.add_argument("--roman-output", required=True, type=Path) + parser.add_argument("--native-output", required=True, type=Path) + return parser.parse_args() + + +def fetch(source: Source) -> bytes: + """Fetch and verify one pinned source with bounded retries.""" + for attempt in range(1, MAX_ATTEMPTS + 1): + request = urllib.request.Request(source.url, headers={"User-Agent": USER_AGENT}) # noqa: S310 + try: + with urllib.request.urlopen(request, timeout=60) as response: # noqa: S310 + payload = response.read() + except urllib.error.HTTPError as error: + if error.code not in RETRYABLE_HTTP_CODES or attempt == MAX_ATTEMPTS: + raise + delay_seconds = 5 * attempt + print( + f"retry source={source.name} attempt={attempt}/{MAX_ATTEMPTS} http={error.code} delay={delay_seconds}s", + file=sys.stderr, + ) + time.sleep(delay_seconds) + continue + digest = hashlib.sha256(payload).hexdigest() + if digest != source.sha256: + message = f"source hash mismatch for {source.name}: expected={source.sha256} actual={digest}" + raise ValueError(message) + return payload + message = f"bounded attempts exhausted for {source.name}" + raise AssertionError(message) + + +def fold(value: str) -> str: + """Return a lowercase accent-insensitive lookup key.""" + translated = value.translate(str.maketrans({"\u0110": "D", "\u0111": "d"})) + return "".join( + character for character in unicodedata.normalize("NFD", translated).casefold() if not unicodedata.combining(character) + ) + + +def japanese_roman_keys(value: str) -> set[str]: + """Return exact and common long-vowel-neutral Japanese keys.""" + exact = fold(value) + return {exact, exact.replace("ou", "o").replace("oo", "o").replace("uu", "u")} + + +def slash_variants(value: str) -> list[str]: + """Expand slash-separated source variants.""" + return [part.strip() for part in value.split("/") if part.strip()] + + +def is_hangul(value: str) -> bool: + """Return whether a source form is entirely modern Hangul.""" + return bool(value) and all("\uac00" <= character <= "\ud7a3" for character in value) + + +def source_metadata() -> list[dict[str, str]]: + """Return serializable provenance for every input.""" + return [ + { + "name": source.name, + "url": source.url, + "sha256": source.sha256, + "license": source.license, + "repository": source.repository, + } + for source in SOURCES + ] + + +def encode(payload: dict[str, object]) -> bytes: + """Return deterministic gzip-compressed JSON.""" + serialized = json.dumps( + payload, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ).encode("utf-8") + return gzip.compress(serialized, compresslevel=9, mtime=0) + + +def main() -> None: + """Build the two runtime assets and print their counts and hashes.""" + args = parse_args() + fetched = {source.name: fetch(source) for source in SOURCES} + country_rows = list( + csv.DictReader(io.StringIO(fetched["country_surnames"].decode("utf-8-sig"))), + ) + japanese_surname_rows = list( + csv.reader(io.StringIO(fetched["japanese_surnames"].decode("utf-8-sig"))), + ) + japanese_given_rows = [ + row + for source_name in ( + "japanese_male_given_names", + "japanese_female_given_names", + ) + for row in csv.reader(io.StringIO(fetched[source_name].decode("utf-8-sig"))) + if len(row) >= MIN_GIVEN_FIELDS and row[0].strip() and row[1].strip() + ] + + japanese_surnames_roman = sorted( + { + key + for row in japanese_surname_rows + if len(row) == 4 and row[3].strip() # noqa: PLR2004 + for key in japanese_roman_keys(row[3].strip()) + }, + ) + japanese_surnames_native = sorted( + { + row[0].strip() + for row in japanese_surname_rows + if len(row) == 4 and row[0].strip() # noqa: PLR2004 + }, + ) + japanese_given_roman = sorted( + {key for row in japanese_given_rows for key in japanese_roman_keys(row[1].strip())}, + ) + japanese_given_native = sorted( + {native.strip() for row in japanese_given_rows for native in row[2:] if native.strip()}, + ) + korean_surnames_roman = sorted( + { + fold(romanized) + for row in country_rows + if row["Country"] == "KR" and is_hangul(row["Localized Name"]) + for romanized in slash_variants(row["Romanized Name"]) + }, + ) + vietnamese_surnames_roman = sorted( + { + fold(romanized) + for row in country_rows + if row["Country"] == "VN" + for romanized in slash_variants(row["Romanized Name"]) + }, + ) + + provenance = source_metadata() + roman_payload: dict[str, object] = { + "schema_version": 1, + "sources": provenance, + "japanese_surnames": japanese_surnames_roman, + "japanese_given_names": japanese_given_roman, + "korean_surnames": korean_surnames_roman, + "vietnamese_surnames": vietnamese_surnames_roman, + } + native_payload: dict[str, object] = { + "schema_version": 1, + "sources": provenance, + "japanese_surnames": japanese_surnames_native, + "japanese_given_names": japanese_given_native, + } + outputs = ( + (args.roman_output, roman_payload), + (args.native_output, native_payload), + ) + report: dict[str, object] = {"counts": {}} + for path, payload in outputs: + path.parent.mkdir(parents=True, exist_ok=True) + encoded = encode(payload) + path.write_bytes(encoded) + report[path.name] = { + "bytes": len(encoded), + "sha256": hashlib.sha256(encoded).hexdigest(), + } + report["counts"] = { + "japanese_surnames_roman": len(japanese_surnames_roman), + "japanese_given_roman": len(japanese_given_roman), + "japanese_surnames_native": len(japanese_surnames_native), + "japanese_given_native": len(japanese_given_native), + "korean_surnames_roman": len(korean_surnames_roman), + "vietnamese_surnames_roman": len(vietnamese_surnames_roman), + } + print(json.dumps(report, ensure_ascii=False, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/sinonym/chinese_names_data.py b/sinonym/chinese_names_data.py index ed3eeed..ec90d34 100644 --- a/sinonym/chinese_names_data.py +++ b/sinonym/chinese_names_data.py @@ -573,6 +573,10 @@ def _assert_no_duplicate_keys(*layers): "teo": ("zhang", "张"), # 张 - Teochew/Hokkien Teo = Mandarin Zhang "goh": ("wu", "吴"), # 吴 - Teochew/Hokkien Goh = Mandarin Wu "khoo": ("qiu", "邱"), # 邱 - Teochew/Hokkien Khoo = Mandarin Qiu + # Reviewed Taiwanese/Wade-Giles surname spellings. Conservative + # as-written frequency shares already live in surname_romanizations.csv. + "horng": ("hong", "洪"), + "hsien": ("xian", "冼/先"), "soo": ("su", "苏"), # 苏 - Korean Soo = Mandarin Su # Korean surnames with Chinese equivalents "jang": ("zhang", "张"), # 张 - Korean Jang = Mandarin Zhang @@ -583,6 +587,8 @@ def _assert_no_duplicate_keys(*layers): "kyeong": ("jing", "京"), # 京 - Korean Kyeong = Mandarin Jing } +ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES = frozenset({"horng", "hsien"}) + # Sanity check: Ensure no inconsistencies between SYLLABLE_RULES and CANTONESE_SURNAMES def _check_surname_mapping_consistency(): @@ -1058,6 +1064,8 @@ def _check_surname_mapping_consistency(): # Korean name pairs: Common Korean given name combinations KOREAN_GIVEN_PAIRS = frozenset( { + ("in", "sun"), + ("jin", "uk"), ("soo", "jin"), # 수진 - very common Korean name ("min", "soo"), # 민수 - very common Korean name ("min", "jung"), # 민정 - very common Korean name @@ -1086,6 +1094,11 @@ def _check_surname_mapping_consistency(): }, ) +# Directional evidence only. ``Ok`` also occurs in Korean given names, so it +# must never be added to the anywhere-in-name Korean surname sets. +KOREAN_DIRECTIONAL_FAMILY_FIRST_SURNAMES = frozenset({"ok"}) +KOREAN_DIRECTIONAL_SINGLE_GIVEN_NAMES = frozenset({"jeung"}) + # Overlapping Korean surnames (exist in both Korean and Chinese) OVERLAPPING_KOREAN_SURNAMES = frozenset( @@ -1256,7 +1269,6 @@ def _check_surname_mapping_consistency(): }, ) - NAME_ORDER_ROUTING_COMMON_CHINESE_SURNAMES = frozenset( { "bai", diff --git a/sinonym/coretypes/__init__.py b/sinonym/coretypes/__init__.py index d40f77f..372005e 100644 --- a/sinonym/coretypes/__init__.py +++ b/sinonym/coretypes/__init__.py @@ -10,7 +10,9 @@ BatchFormatPattern, BatchParseResult, CacheInfo, + CanonicalName, IndividualAnalysis, + NameComponents, NameFormat, NameOrderEvidence, ParseCandidate, @@ -22,8 +24,10 @@ "BatchFormatPattern", "BatchParseResult", "CacheInfo", + "CanonicalName", "ChineseNameConfig", "IndividualAnalysis", + "NameComponents", "NameFormat", "NameOrderEvidence", "ParseCandidate", diff --git a/sinonym/coretypes/results.py b/sinonym/coretypes/results.py index 80426c2..62dfe2f 100644 --- a/sinonym/coretypes/results.py +++ b/sinonym/coretypes/results.py @@ -11,6 +11,44 @@ from enum import Enum +@dataclass(frozen=True) +class NameComponents: + """Assigned components and token lineage for a person's name. + + Component strings provide the convenient public representation while the + token tuples preserve the corresponding token boundaries. ``order`` records + component roles in display order; roles may repeat when tokens from the same + component are non-contiguous. + """ + + given_name: str = "" + middle_name: str = "" + surname: str = "" + suffix: str = "" + given_tokens: tuple[str, ...] = () + middle_tokens: tuple[str, ...] = () + surname_tokens: tuple[str, ...] = () + suffix_tokens: tuple[str, ...] = () + order: tuple[str, ...] = () + + +@dataclass(frozen=True) +class CanonicalName: + """Canonical representation of a person's name and its source components. + + ``source`` records the component assignment before normalization, while + ``normalized`` records the final component assignment used to render + ``text``. For structured input, surviving source roles remain authoritative; + normalization repairs roles only when mechanical cleanup empties a required + boundary. Both use immutable tuples so the complete value is immutable. + """ + + source_text: str + text: str + source: NameComponents + normalized: NameComponents + + @dataclass(frozen=True) class ParsedName: """Parsed name with surname and given name components. @@ -55,6 +93,8 @@ class ParseResult: parsed: ParsedName | None = None # Structured parsed components in the original input order parsed_original_order: ParsedName | None = None + # Canonical representation for person names, including non-Chinese names + canonical_name: CanonicalName | None = None @classmethod def success_with_name( @@ -63,6 +103,7 @@ def success_with_name( original_compound_surname: str | None = None, parsed: ParsedName | None = None, parsed_original_order: ParsedName | None = None, + canonical_name: CanonicalName | None = None, ) -> ParseResult: """Create a successful result with final formatted name. @@ -76,6 +117,7 @@ def success_with_name( original_compound_surname=original_compound_surname, parsed=parsed, parsed_original_order=parsed_original_order, + canonical_name=canonical_name, ) @classmethod @@ -84,6 +126,7 @@ def success_with_parse( surname_tokens: list[str], given_tokens: list[str], original_compound_surname: str | None = None, + canonical_name: CanonicalName | None = None, ) -> ParseResult: """Create a successful intermediate parse with raw tokens. @@ -104,11 +147,19 @@ def success_with_parse( original_compound_surname=original_compound_surname, parsed=parsed, parsed_original_order=None, + canonical_name=canonical_name, ) @classmethod - def failure(cls, error_message: str) -> ParseResult: - return cls(success=False, result="", error_message=error_message, original_compound_surname=None) + def failure(cls, error_message: str, canonical_name: CanonicalName | None = None) -> ParseResult: + """Create a failed Chinese-name parse with optional person metadata.""" + return cls( + success=False, + result="", + error_message=error_message, + original_compound_surname=None, + canonical_name=canonical_name, + ) def map(self, f) -> ParseResult: """Functor map operation - Scala-like transformation""" @@ -119,9 +170,10 @@ def map(self, f) -> ParseResult: self.original_compound_surname, self.parsed, self.parsed_original_order, + self.canonical_name, ) except Exception as e: # noqa: BLE001 - user-provided callbacks may raise arbitrary exceptions. - return ParseResult.failure(str(e)) + return ParseResult.failure(str(e), canonical_name=self.canonical_name) return self def flat_map(self, f) -> ParseResult: @@ -130,17 +182,21 @@ def flat_map(self, f) -> ParseResult: try: result = f(self.result) except Exception as e: # noqa: BLE001 - user-provided callbacks may raise arbitrary exceptions. - return ParseResult.failure(str(e)) + return ParseResult.failure(str(e), canonical_name=self.canonical_name) else: - # Preserve the original compound surname if the result doesn't already have one - if result.success and result.original_compound_surname is None: + preserve_original_compound = result.success and result.original_compound_surname is None + preserve_canonical_name = result.canonical_name is None and self.canonical_name is not None + if preserve_original_compound or preserve_canonical_name: return ParseResult( - result.success, - result.result, - result.error_message, - self.original_compound_surname, - result.parsed, - result.parsed_original_order, + success=result.success, + result=result.result, + error_message=result.error_message, + original_compound_surname=( + self.original_compound_surname if preserve_original_compound else result.original_compound_surname + ), + parsed=result.parsed, + parsed_original_order=result.parsed_original_order, + canonical_name=self.canonical_name if preserve_canonical_name else result.canonical_name, ) return result return self diff --git a/sinonym/data/EAST_ASIAN_NAME_LEXICONS.md b/sinonym/data/EAST_ASIAN_NAME_LEXICONS.md new file mode 100644 index 0000000..d7d1ce7 --- /dev/null +++ b/sinonym/data/EAST_ASIAN_NAME_LEXICONS.md @@ -0,0 +1,34 @@ +# East Asian name-order lexicon notices + +The runtime assets `east_asian_roman_lexicons.json.gz` and +`japanese_native_lexicons.json.gz` are deterministic derived lookup lists used +only by the conservative Japanese, Korean, and Vietnamese family-first router. +They contain component names, not complete people. + +## Japanese Personal Name Dataset + +Source: + +Pinned commit: `c5220278652e7bae05b06cfaf527f1b09a100de6` + +Copyright (c) 2022 shuheilocale. Distributed under the MIT License. The source +repository's `LICENSE` file contains the full notice and permission terms. + +The derived assets use the repository's surname and male/female given-name CSV +files. The build script records and verifies the SHA-256 of every input. + +## Popular Names by Country Dataset + +Source: + +Pinned commit: `eb62e13d4d62dd96cdfae79d293a02066352205f` + +Distributed under CC0-1.0. The derived assets use only the Korean and +Vietnamese surname rows whose provenance is recorded in the generated JSON. + +## Excluded research source + +JMnedict/ENAMDICT was evaluated in the scratch experiment but is deliberately +not included in these package assets. Its EDRDG/CC BY-SA terms include +attribution and ongoing data-update requirements that are outside this static +asset's release contract. diff --git a/sinonym/data/README.md b/sinonym/data/README.md index 82216d5..efc6952 100644 --- a/sinonym/data/README.md +++ b/sinonym/data/README.md @@ -4,6 +4,19 @@ This directory contains the data files and trained models used by the Sinonym li ## Data Files +### Conservative East Asian name-order lexicons + +- **`east_asian_roman_lexicons.json.gz`**: sorted Romanized Japanese surname + and given-name keys plus Korean and Vietnamese surname keys. +- **`japanese_native_lexicons.json.gz`**: sorted Japanese native-script surname + and given-name forms used to score compact-name boundaries. + +These assets contain component names rather than complete people. They are +generated deterministically by `scripts/build_east_asian_name_lexicons.py` +from hash-pinned MIT and CC0 sources. Full attribution, source commits, and the +reason the broader research-only JMnedict data is not bundled are recorded in +`EAST_ASIAN_NAME_LEXICONS.md`. + ### Name Frequency Data - **`familyname_orcid.csv`**: Chinese surnames with frequency data (parts per million) derived from ORCID records. Contains the most common Chinese surnames like 王, 李, 张, etc., with their frequency statistics. - **`givenname_orcid.csv`**: Chinese given name characters with usage statistics from ORCID records. Includes character, pinyin romanization, and frequency (ppm). diff --git a/sinonym/data/east_asian_roman_lexicons.json.gz b/sinonym/data/east_asian_roman_lexicons.json.gz new file mode 100644 index 0000000..04c3750 Binary files /dev/null and b/sinonym/data/east_asian_roman_lexicons.json.gz differ diff --git a/sinonym/data/japanese_native_lexicons.json.gz b/sinonym/data/japanese_native_lexicons.json.gz new file mode 100644 index 0000000..01e92ce Binary files /dev/null and b/sinonym/data/japanese_native_lexicons.json.gz differ diff --git a/sinonym/detector.py b/sinonym/detector.py index 0e07527..f40e776 100644 --- a/sinonym/detector.py +++ b/sinonym/detector.py @@ -165,13 +165,16 @@ import logging import platform import string +from dataclasses import replace from typing import Literal from sinonym.chinese_names_data import COMPOUND_VARIANTS from sinonym.coretypes import ( BatchFormatPattern, BatchParseResult, + CanonicalName, IndividualAnalysis, + NameComponents, NameFormat, NameOrderEvidence, ) @@ -195,7 +198,16 @@ ServiceContext, SurnameResolver, ) +from sinonym.services.east_asian_name_order import ( + EastAsianNameOrderDecision, + EastAsianNameOrderService, +) from sinonym.services.order_metadata import original_component_order +from sinonym.services.person_name_normalization import ( + DropReason, + PersonNameNormalizationService, + PersonNameOutcome, +) from sinonym.services.process_pool import PersistentMultiprocessNormalizer LOGGER = logging.getLogger(__name__) @@ -208,10 +220,12 @@ BILINGUAL_SURNAME_STRENGTH_RATIO_MIN = 5.0 BILINGUAL_ROMAN_HAN_SOURCE_ORDER_RATIO_MAX = 12.0 BILINGUAL_ENDPOINT_PAIR_COUNT = 2 +TWO_TOKEN_NAME_COUNT = 2 SPACED_ALL_CHINESE_GROUP_COUNT = 2 THREE_CHARACTER_ALL_CHINESE_TOKEN_COUNT = 3 SPACED_HAN_PREFIX_SURNAME_RATIO_MIN = 5.0 CAMEL_CASE_LAST_SURNAME_RATIO_MIN = 5.0 +LEADING_ET_AL_TOKEN_COUNT = 2 CURATED_COMPOUND_SURNAME_FORMS = frozenset((*COMPOUND_VARIANTS.keys(), *COMPOUND_VARIANTS.values())) CompactHanRomanCandidate = tuple[list[str], list[str], list[str], float, bool] @@ -261,6 +275,8 @@ def __init__(self, config: ChineseNameConfig | None = None, weights: list[float] self._config = config or ChineseNameConfig.create_default() self._cache_service = PinyinCacheService(self._config) self._normalizer = NormalizationService(self._config, self._cache_service) + self._person_name_normalizer = PersonNameNormalizationService() + self._east_asian_name_order = EastAsianNameOrderService() self._data_service = DataInitializationService(self._config, self._cache_service, self._normalizer) self._data: NameDataStructures | None = None self._surname_resolver: SurnameResolver | None = None @@ -844,9 +860,9 @@ def _initial_input_failure(self, raw_name: str) -> ParseResult | None: return None return ParseResult.failure(non_person_reason) - def normalize_name(self, raw_name: str) -> ParseResult: + def _normalize_chinese_name(self, raw_name: str) -> ParseResult: """ - Main API method: Detect if a name is Chinese and normalize it. + Run the legacy Chinese detection and normalization pipeline unchanged. Returns ParseResult with: - success=True, result=formatted_name if Chinese name detected @@ -856,6 +872,8 @@ def normalize_name(self, raw_name: str) -> ParseResult: if initial_failure is not None: return initial_failure + raw_name = self._chinese_classification_input(raw_name) + # Use new normalization service for cleaner pipeline normalized_input = self._normalizer.apply(raw_name) @@ -863,7 +881,14 @@ def normalize_name(self, raw_name: str) -> ParseResult: return ParseResult.failure(f"needs at least {self._config.min_tokens_required} Roman tokens") # Check if this is an all-Chinese input first - is_all_chinese = self._normalizer._text_preprocessor.is_all_chinese_input(raw_name) + is_all_chinese = not raw_name.isascii() and self._normalizer._text_preprocessor.is_all_chinese_input(raw_name) + + # Exact alternating Roman/Han alignment is stronger evidence than the + # Roman-only ethnicity gate, especially for polyphonic Han surnames. + if self._config.cjk_pattern.search(raw_name) and self._config.ascii_alpha_pattern.search(raw_name): + aligned_bilingual_result = self._normalize_aligned_bilingual_name(normalized_input) + if aligned_bilingual_result is not None: + return aligned_bilingual_result # Check for non-Chinese ethnicity using normalized tokens (consistent for all inputs) non_chinese_result = self._ethnicity_service.classify_ethnicity( @@ -875,9 +900,9 @@ def normalize_name(self, raw_name: str) -> ParseResult: if non_chinese_result.success is False: return non_chinese_result - aligned_bilingual_result = self._normalize_aligned_bilingual_name(normalized_input) - if aligned_bilingual_result is not None: - return aligned_bilingual_result + contextual_taiwan_result = self._normalize_contextual_taiwan_name(normalized_input) + if contextual_taiwan_result is not None: + return contextual_taiwan_result compact_han_roman_result = self._normalize_compact_han_roman_name(normalized_input) if compact_han_roman_result is not None: @@ -1033,46 +1058,66 @@ def normalize_name(self, raw_name: str) -> ParseResult: original_tokens = list(normalized_input.roman_tokens) best_candidate = None - for order in (normalized_input.roman_tokens, normalized_input.roman_tokens[::-1]): - order_tokens = list(order) - parse_result = self._parsing_service.parse_name_order_tokens( - order_tokens, - normalized_input.norm_map, - normalized_input.compound_metadata, - ) - if parse_result is None: - continue - - surname_tokens, given_tokens, original_compound_surname = parse_result - score = self._parsing_service.calculate_parse_score( - surname_tokens, - given_tokens, + # For two tokens the parser already evaluates both endpoint surname + # candidates. Reversing repeats the same candidate set; uncertain + # fallback parses and parenthetical-order hints retain the full path. + if len(original_tokens) == TWO_TOKEN_NAME_COUNT and not normalized_input.surname_first_parenthetical_hint: + direct_parse = self._parsing_service._best_parse_tokens( original_tokens, normalized_input.norm_map, - is_all_chinese=False, - original_compound_format=original_compound_surname, - surname_first_parenthetical_hint=normalized_input.surname_first_parenthetical_hint, + normalized_input.compound_metadata, ) - used_original = order_tokens == original_tokens - - candidate = { - "surname_tokens": surname_tokens, - "given_tokens": given_tokens, - "score": score, - "order_tokens": order_tokens, - "used_original": used_original, - } - - if ( - best_candidate is None - or candidate["score"] > best_candidate["score"] - or ( - candidate["score"] == best_candidate["score"] - and candidate["used_original"] - and not best_candidate["used_original"] + if direct_parse is not None: + surname_tokens, given_tokens, _original_compound_surname = direct_parse + best_candidate = { + "surname_tokens": surname_tokens, + "given_tokens": given_tokens, + "score": 0.0, + "order_tokens": original_tokens, + "used_original": True, + } + + if best_candidate is None: + for order in (normalized_input.roman_tokens, normalized_input.roman_tokens[::-1]): + order_tokens = list(order) + parse_result = self._parsing_service.parse_name_order_tokens( + order_tokens, + normalized_input.norm_map, + normalized_input.compound_metadata, + ) + if parse_result is None: + continue + + surname_tokens, given_tokens, original_compound_surname = parse_result + score = self._parsing_service.calculate_parse_score( + surname_tokens, + given_tokens, + original_tokens, + normalized_input.norm_map, + is_all_chinese=False, + original_compound_format=original_compound_surname, + surname_first_parenthetical_hint=normalized_input.surname_first_parenthetical_hint, ) - ): - best_candidate = candidate + used_original = order_tokens == original_tokens + + candidate = { + "surname_tokens": surname_tokens, + "given_tokens": given_tokens, + "score": score, + "order_tokens": order_tokens, + "used_original": used_original, + } + + if ( + best_candidate is None + or candidate["score"] > best_candidate["score"] + or ( + candidate["score"] == best_candidate["score"] + and candidate["used_original"] + and not best_candidate["used_original"] + ) + ): + best_candidate = candidate if best_candidate is not None: surname_tokens = best_candidate["surname_tokens"] @@ -1136,6 +1181,310 @@ def normalize_name(self, raw_name: str) -> ParseResult: return ParseResult.failure("name not recognised as Chinese") + def _normalize_contextual_taiwan_name(self, normalized_input: NormalizedInput) -> ParseResult | None: + """Parse the narrowly admitted Taiwan romanization contexts.""" + given_parts = self._ethnicity_service.contextual_taiwan_given_parts(normalized_input.roman_tokens) + if given_parts is None: + return None + + norm_map = dict(normalized_input.norm_map) + # BOCA's ``Jr`` corresponds to the Mandarin syllable ``Zhi``. Keep the + # visible Taiwan spelling while supplying internal given-name evidence. + if given_parts == ("tsung", "jr"): + norm_map["jr"] = "zhi" + contextual_input = replace(normalized_input, norm_map=norm_map) + try: + return self._format_parse_result( + [normalized_input.roman_tokens[-1]], + list(given_parts), + contextual_input, + ["given", "surname"], + ) + except ValueError as error: + return ParseResult.failure(str(error)) + + def _chinese_classification_input(self, raw_name: str) -> str: + """Remove only an audited leading ``Et al.`` citation contaminant.""" + prefix = raw_name.lstrip() + marker = "et al." + if len(prefix) <= len(marker) or prefix[: len(marker)].casefold() != marker or not prefix[len(marker)].isspace(): + return raw_name + normalized = self._person_name_normalizer.normalize_text(raw_name) + if normalized.canonical_name is None or len(normalized.dropped_tokens) < LEADING_ET_AL_TOKEN_COUNT: + return raw_name + first, second = normalized.dropped_tokens[:LEADING_ET_AL_TOKEN_COUNT] + has_exact_prefix = ( + first.text.casefold() == "et" + and second.text.casefold() == "al." + and first.reason is DropReason.CONNECTOR + and second.reason is DropReason.CONNECTOR + ) + return normalized.canonical_name.text if has_exact_prefix else raw_name + + @staticmethod + def _canonical_components_from_parsed(parsed: ParsedName) -> NameComponents: + """Convert legacy parsed components to immutable canonical components.""" + if parsed.order == ["given", "middle", "surname"]: + expanded_order = ( + ("given",) * len(parsed.given_tokens) + + ("middle",) * len(parsed.middle_tokens) + + ("surname",) * len(parsed.surname_tokens) + ) + else: + counts = { + "given": len(parsed.given_tokens), + "middle": len(parsed.middle_tokens), + "surname": len(parsed.surname_tokens), + } + occurrences = {role: parsed.order.count(role) for role in counts} + expanded: list[str] = [] + for role in parsed.order: + count = counts.get(role, 0) + if occurrences.get(role) == 1: + expanded.extend([role] * count) + elif count: + expanded.append(role) + expanded_order = tuple(expanded) + return NameComponents( + given_name=parsed.given_name, + middle_name=parsed.middle_name, + surname=parsed.surname, + given_tokens=tuple(parsed.given_tokens), + middle_tokens=tuple(parsed.middle_tokens), + surname_tokens=tuple(parsed.surname_tokens), + order=expanded_order, + ) + + def _canonical_name_from_chinese_result(self, raw_name: str, result: ParseResult) -> CanonicalName | None: + """Build canonical metadata from the final selected Chinese parse.""" + if not result.success or result.parsed is None or not isinstance(result.result, str): + return None + normalized = self._canonical_components_from_parsed(result.parsed) + source_parsed = result.parsed_original_order or result.parsed + source = self._canonical_source_components(raw_name, source_parsed, normalized) + return CanonicalName( + source_text=raw_name, + text=result.result, + source=source, + normalized=normalized, + ) + + @staticmethod + def _component_token_key(token: str) -> str: + """Return a comparison key for source-to-normalized token lineage.""" + if token.isalnum(): + return token.casefold() + return "".join(character.casefold() for character in token if character.isalnum()) + + @staticmethod + def _ordered_component_tokens(components: NameComponents) -> list[str]: + """Rebuild a component token stream from its positional role order.""" + queues = { + "given": iter(components.given_tokens), + "middle": iter(components.middle_tokens), + "surname": iter(components.surname_tokens), + "suffix": iter(components.suffix_tokens), + } + return [next(queues[role]) for role in components.order] + + def _canonical_source_components( + self, + raw_name: str, + parsed_original: ParsedName, + normalized: NameComponents, + ) -> NameComponents: + """Preserve raw Latin token boundaries while retaining parsed roles.""" + simple_tokens = self._simple_source_tokens(raw_name) + if simple_tokens is not None: + simple_source = self._source_components_from_tokens( + simple_tokens, + parsed_original, + normalized, + require_unique_roles=True, + ) + if simple_source is not None: + return simple_source + + source_result = self._person_name_normalizer.normalize_text(raw_name) + if source_result.canonical_name is None: + return self._canonical_components_from_parsed(parsed_original) + + generic_source = source_result.canonical_name.source + ordered_tokens = self._ordered_component_tokens(generic_source) + source = self._source_components_from_tokens( + ordered_tokens, + parsed_original, + normalized, + require_unique_roles=False, + ) + assert source is not None + return source + + def _simple_source_tokens(self, raw_name: str) -> tuple[str, ...] | None: + """Return already-clean ASCII source tokens, or abstain.""" + return self._normalizer.simple_latin_tokens(raw_name) + + def _source_components_from_tokens( + self, + ordered_tokens: list[str] | tuple[str, ...], + parsed_original: ParsedName, + normalized: NameComponents, + *, + require_unique_roles: bool, + ) -> NameComponents | None: + """Assign source tokens to normalized roles without changing token text.""" + normalized_by_role = { + "given": normalized.given_tokens, + "middle": normalized.middle_tokens, + "surname": normalized.surname_tokens, + "suffix": normalized.suffix_tokens, + } + role_keys = { + role: { + *(self._component_token_key(token) for token in tokens), + self._component_token_key("".join(tokens)), + } + for role, tokens in normalized_by_role.items() + if tokens + } + fallback_roles = [role for role in parsed_original.order if role in normalized_by_role] + assigned: list[tuple[str, str]] = [] + for index, token in enumerate(ordered_tokens): + key = self._component_token_key(token) + candidates = [role for role, keys in role_keys.items() if key and key in keys] + if require_unique_roles and len(candidates) != 1: + return None + fallback = fallback_roles[index] if index < len(fallback_roles) else "given" + role = candidates[0] if len(candidates) == 1 else fallback + assigned.append((role, token)) + + tokens_by_role = { + role: tuple(token for assigned_role, token in assigned if assigned_role == role) for role in normalized_by_role + } + return NameComponents( + given_name=" ".join(tokens_by_role["given"]), + middle_name=" ".join(tokens_by_role["middle"]), + surname=" ".join(tokens_by_role["surname"]), + suffix=" ".join(tokens_by_role["suffix"]), + given_tokens=tokens_by_role["given"], + middle_tokens=tokens_by_role["middle"], + surname_tokens=tokens_by_role["surname"], + suffix_tokens=tokens_by_role["suffix"], + order=tuple(role for role, _token in assigned), + ) + + def normalize_person_name(self, raw_name: str) -> CanonicalName | None: + """Return a canonical name for one person-like raw string. + + This method is independent of Chinese recognition and returns ``None`` + for invalid or obvious non-person inputs. + """ + if not raw_name or len(raw_name) > self._config.max_name_length: + return None + if all(character in string.punctuation + string.whitespace for character in raw_name): + return None + self._ensure_initialized() + if self._non_person_input_service is not None: + non_person_reason = self._non_person_input_service.failure_reason(raw_name) + if non_person_reason is not None: + return None + normalized = self._person_name_normalizer.normalize_text(raw_name) + if normalized.outcome is not PersonNameOutcome.PERSON or normalized.canonical_name is None: + return None + routed = self._canonical_name_from_east_asian_order(raw_name, normalized.canonical_name) + return routed or normalized.canonical_name + + def _canonical_name_from_east_asian_order( + self, + raw_name: str, + baseline: CanonicalName, + ) -> CanonicalName | None: + """Return a conservatively family-first canonical name or abstain.""" + if self._ethnicity_service is None: + return None + try: + decision = self._east_asian_name_order.infer( + raw_name, + japanese_probability=self._ethnicity_service.japanese_probability, + ) + except RuntimeError as error: + LOGGER.warning( + "East Asian name-order routing abstained after classifier failure for %r: %s", + raw_name, + error, + ) + return None + if decision is None: + return None + return self._canonical_name_from_order_decision(baseline, decision) + + def _canonical_name_from_order_decision( + self, + baseline: CanonicalName, + decision: EastAsianNameOrderDecision, + ) -> CanonicalName: + """Normalize routed components while retaining original source-role lineage.""" + routed = self._person_name_normalizer.normalize_components( + first_name=decision.first_name, + middle_name=decision.middle_name, + last_name=decision.last_name, + ) + if routed.outcome is not PersonNameOutcome.PERSON or routed.canonical_name is None: + message = f"East Asian route {decision.reason!r} produced invalid components" + raise RuntimeError(message) + return replace( + routed.canonical_name, + source_text=baseline.source_text, + source=decision.source_components(), + ) + + def normalize_person_name_components( + self, + *, + first_name: str | None = None, + middle_name: str | None = None, + last_name: str | None = None, + suffix: str | None = None, + ) -> CanonicalName | None: + """Normalize structured fields, including conservative family-first routing. + + The normalized assignment may follow high-confidence East Asian order + evidence, while ``source`` retains the caller-provided roles and order. + """ + normalized = self._person_name_normalizer.normalize_components( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + suffix=suffix, + ) + if normalized.outcome is not PersonNameOutcome.PERSON or normalized.canonical_name is None: + return None + baseline = normalized.canonical_name + routed = self._canonical_name_from_east_asian_order(baseline.source_text, baseline) + if routed is None: + return baseline + return replace(routed, source=baseline.source) + + def _attach_canonical_name(self, raw_name: str, result: ParseResult) -> ParseResult: + """Attach canonical metadata without changing legacy recognition fields.""" + if result.canonical_name is not None: + return result + canonical_name = self._canonical_name_from_chinese_result(raw_name, result) + if canonical_name is None: + canonical_name = self.normalize_person_name(raw_name) + if canonical_name is None: + return result + return replace(result, canonical_name=canonical_name) + + def normalize_name(self, raw_name: str) -> ParseResult: + """Detect/normalize Chinese names and surface canonical data for people. + + Legacy ``success``, ``result``, ``parsed``, and error semantics remain + Chinese-specific. ``canonical_name`` is the all-person representation. + """ + result = self._normalize_chinese_name(raw_name) + return self._attach_canonical_name(raw_name, result) + # Backwards compatibility alias def is_chinese_name(self, raw_name: str) -> ParseResult: # pragma: no cover - thin wrapper """Deprecated: use normalize_name(). Maintained for compatibility.""" @@ -1175,7 +1524,7 @@ def analyze_name_batch( individual_results = [self.normalize_name(name) for name in names] return self._create_fallback_batch_result(names, individual_results) - return self._batch_analysis_service.analyze_name_batch( + batch_result = self._batch_analysis_service.analyze_name_batch( names, self._normalizer, self._formatting_service, @@ -1184,6 +1533,11 @@ def analyze_name_batch( format_threshold=format_threshold, ), ) + canonical_results = [ + self._attach_canonical_name(name, result) + for name, result in zip(batch_result.names, batch_result.results, strict=True) + ] + return replace(batch_result, results=canonical_results) def detect_batch_format( self, diff --git a/sinonym/pipeline/name_order_routing.py b/sinonym/pipeline/name_order_routing.py index 77d45c1..884d5f5 100644 --- a/sinonym/pipeline/name_order_routing.py +++ b/sinonym/pipeline/name_order_routing.py @@ -17,9 +17,9 @@ abstain + "unknown" as a contract violation and raise. - PP-only regime: there is no second run to pick from; emit the as-typed reading via `input_order_parsed(result)` (trailing token is the surname, everything - else keeps its position), except spaced Han surname-first rows, where source - spacing already marks the surname boundary and the PP parse is the input-order - reading. If the as-typed reading cannot be materialized (e.g. the + else keeps its position), except spaced Han surname-first rows and internal + compound-surname spans, where the PP parse preserves the established surname + boundary. If the as-typed reading cannot be materialized (e.g. the original-order parse ends in a middle initial), the abstain surfaces as an explicit failure (no parsed person) — never fall back to the reordered PP parse the abstain declined to trust. Do NOT re-parse the name standalone: the @@ -847,8 +847,9 @@ def pp_abstain_parsed(result: ParseResult, route_row: Row) -> ParsedName | None: """Return the final parsed person components for a PP-only abstain row. Latin-only abstain keeps the as-typed given-first reading. Spaced Han rows - guarded by `spaced_cjk_zero_batch_surname_first` already have a source - surname boundary, so the PP parse is the stable input-order reading. + guarded by `spaced_cjk_zero_batch_surname_first` and corrected internal + compound-surname spans already have a stable surname boundary, so the PP + parse is the stable component reading. Returns None when the as-typed reading cannot be materialized (e.g. the original-order parse ends in a middle initial): an abstain must surface as an explicit failure, never fall back to the reordered PP parse it declined @@ -964,7 +965,13 @@ def _pp_abstain_failed_parse(row: Row) -> bool: def _pp_abstain_uses_pp_parse_for_abstain(row: Row) -> bool: - return row.get("router_reason") == "spaced_cjk_zero_batch_surname_first" + if row.get("router_reason") == "spaced_cjk_zero_batch_surname_first": + return True + return ( + row.get("selected_surname_position") == "internal" + and "selected_surname_token_count" in row + and _required_float(row, "selected_surname_token_count") > 1 + ) def _pp_abstain_predicates(row: Row) -> dict[str, bool]: diff --git a/sinonym/services/__init__.py b/sinonym/services/__init__.py index caab591..17fa812 100644 --- a/sinonym/services/__init__.py +++ b/sinonym/services/__init__.py @@ -15,6 +15,13 @@ from sinonym.services.non_person import NonPersonInputDetectionService from sinonym.services.normalization import LazyNormalizationMap, NormalizationService, NormalizedInput from sinonym.services.parsing import NameParsingService +from sinonym.services.person_name_normalization import ( + DroppedNameToken, + DropReason, + PersonNameNormalizationResult, + PersonNameNormalizationService, + PersonNameOutcome, +) class ServiceContext: @@ -37,6 +44,8 @@ def __init__(self, config, normalizer, data): "CacheInfo", "ChineseNameConfig", "DataInitializationService", + "DropReason", + "DroppedNameToken", "EthnicityClassificationService", "LazyNormalizationMap", # Data structures @@ -47,6 +56,9 @@ def __init__(self, config, normalizer, data): "NormalizationService", "NormalizedInput", "ParseResult", + "PersonNameNormalizationResult", + "PersonNameNormalizationService", + "PersonNameOutcome", # Services "PinyinCacheService", "ServiceContext", diff --git a/sinonym/services/cache.py b/sinonym/services/cache.py index 3516253..b90497d 100644 --- a/sinonym/services/cache.py +++ b/sinonym/services/cache.py @@ -7,6 +7,7 @@ from __future__ import annotations from functools import cache +from itertools import product from typing import TYPE_CHECKING import pypinyin @@ -20,6 +21,13 @@ def _char_to_pinyin(ch: str) -> str: return pypinyin.lazy_pinyin(ch, style=pypinyin.Style.NORMAL)[0] +@cache # one entry per unique Han character +def _char_to_pinyin_alternatives(ch: str) -> tuple[str, ...]: + """Return every dictionary pronunciation for one Han character.""" + readings = pypinyin.pinyin(ch, style=pypinyin.Style.NORMAL, heteronym=True)[0] + return tuple(dict.fromkeys(readings)) + + class PinyinCacheService: """ * deterministic, thread‑safe, O(1) repeated look‑ups @@ -34,6 +42,10 @@ def han_to_pinyin_fast(self, han_str: str) -> list[str]: """Return pinyin for every character, memoising on first sight.""" return [_char_to_pinyin(c) for c in han_str] + def han_to_pinyin_alternatives_fast(self, han_str: str) -> tuple[tuple[str, ...], ...]: + """Return pronunciation combinations for an explicitly aligned Han token.""" + return tuple(product(*(_char_to_pinyin_alternatives(character) for character in han_str))) + # (optional) diagnostics you were using elsewhere @property def cache_size(self) -> int: diff --git a/sinonym/services/east_asian_name_order.py b/sinonym/services/east_asian_name_order.py new file mode 100644 index 0000000..5a54d27 --- /dev/null +++ b/sinonym/services/east_asian_name_order.py @@ -0,0 +1,355 @@ +"""Conservative family-first routing for Japanese, Korean, and Vietnamese names.""" + +from __future__ import annotations + +import gzip +import json +import re +import unicodedata +from bisect import bisect_left +from dataclasses import dataclass +from functools import lru_cache +from itertools import pairwise +from typing import TYPE_CHECKING, Any + +from sinonym.chinese_names_data import ( + KOREAN_AMBIGUOUS_PATTERNS, + KOREAN_GIVEN_PATTERNS, + KOREAN_ONLY_SURNAMES, + KOREAN_SPECIFIC_PATTERNS, + NAME_ORDER_ROUTING_KOREAN_GIVEN_SYLLABLES, + NAME_ORDER_ROUTING_KOREAN_SURNAMES, + OVERLAPPING_KOREAN_SURNAMES, + VIETNAMESE_ONLY_SURNAMES, +) +from sinonym.coretypes import NameComponents +from sinonym.resources import read_bytes + +if TYPE_CHECKING: + from collections.abc import Callable + +JAPANESE_ML_THRESHOLD = 0.8 +KOREAN_NATIVE_TOKEN_LENGTH = 3 +MAX_ROMANIZED_TOKENS = 5 +MAX_KOREAN_ROMANIZED_TOKENS = 3 +MIN_ROMANIZED_TOKENS = 2 +ROMAN_ASSET = "east_asian_roman_lexicons.json.gz" +NATIVE_ASSET = "japanese_native_lexicons.json.gz" +WHITESPACE_RE = re.compile(r"\s+") + + +@dataclass(frozen=True) +class EastAsianNameOrderDecision: + """One high-confidence semantic component assignment.""" + + surface: str + given_tokens: tuple[str, ...] + middle_tokens: tuple[str, ...] + surname_tokens: tuple[str, ...] + source_order: tuple[str, ...] + reason: str + + @property + def first_name(self) -> str: + """Return the semantic given component.""" + return " ".join(self.given_tokens) + + @property + def middle_name(self) -> str: + """Return the semantic middle component.""" + return " ".join(self.middle_tokens) + + @property + def last_name(self) -> str: + """Return the semantic surname component.""" + return " ".join(self.surname_tokens) + + def source_components(self) -> NameComponents: + """Return source-role lineage in the original family-first order.""" + return NameComponents( + given_name=self.first_name, + middle_name=self.middle_name, + surname=self.last_name, + suffix="", + given_tokens=self.given_tokens, + middle_tokens=self.middle_tokens, + surname_tokens=self.surname_tokens, + suffix_tokens=(), + order=self.source_order, + ) + + +@dataclass(frozen=True) +class _RomanLexicons: + japanese_surnames: tuple[str, ...] + japanese_given_names: tuple[str, ...] + korean_surnames: tuple[str, ...] + vietnamese_surnames: tuple[str, ...] + + +@dataclass(frozen=True) +class _NativeLexicons: + japanese_surnames: tuple[str, ...] + japanese_given_names: tuple[str, ...] + + +def _load_payload(name: str) -> dict[str, Any]: + """Load one strict gzip JSON package resource.""" + payload = json.loads(gzip.decompress(read_bytes(name)).decode("utf-8")) + if not isinstance(payload, dict) or payload.get("schema_version") != 1: + message = f"unsupported East Asian lexicon schema in {name}" + raise ValueError(message) + return payload + + +def _validated_values(payload: dict[str, Any], key: str, asset: str) -> tuple[str, ...]: + """Validate one sorted, unique string list at the package boundary.""" + values = payload.get(key) + if not isinstance(values, list) or not all(isinstance(value, str) and value for value in values): + message = f"invalid {key} in {asset}" + raise ValueError(message) + if any(left >= right for left, right in pairwise(values)): + message = f"{key} must be sorted and unique in {asset}" + raise ValueError(message) + return tuple(values) + + +@lru_cache(maxsize=1) +def _roman_lexicons() -> _RomanLexicons: + payload = _load_payload(ROMAN_ASSET) + external_korean = _validated_values(payload, "korean_surnames", ROMAN_ASSET) + korean = sorted( + set(external_korean) | NAME_ORDER_ROUTING_KOREAN_SURNAMES | KOREAN_ONLY_SURNAMES | OVERLAPPING_KOREAN_SURNAMES, + ) + vietnamese = sorted( + set(_validated_values(payload, "vietnamese_surnames", ROMAN_ASSET)) + | {_fold(value) for value in VIETNAMESE_ONLY_SURNAMES}, + ) + return _RomanLexicons( + japanese_surnames=_validated_values(payload, "japanese_surnames", ROMAN_ASSET), + japanese_given_names=_validated_values(payload, "japanese_given_names", ROMAN_ASSET), + korean_surnames=tuple(korean), + vietnamese_surnames=tuple(vietnamese), + ) + + +@lru_cache(maxsize=1) +def _native_lexicons() -> _NativeLexicons: + payload = _load_payload(NATIVE_ASSET) + return _NativeLexicons( + japanese_surnames=_validated_values(payload, "japanese_surnames", NATIVE_ASSET), + japanese_given_names=_validated_values(payload, "japanese_given_names", NATIVE_ASSET), + ) + + +def _contains(values: tuple[str, ...], key: str) -> bool: + index = bisect_left(values, key) + return index < len(values) and values[index] == key + + +def _fold(value: str) -> str: + if value.isascii(): + return value.casefold() + translated = value.translate(str.maketrans({"\u0110": "D", "\u0111": "d"})) + return "".join( + character for character in unicodedata.normalize("NFD", translated).casefold() if not unicodedata.combining(character) + ) + + +KOREAN_ROUTING_GIVEN_PARTS = frozenset( + _fold(value) + for value in ( + NAME_ORDER_ROUTING_KOREAN_GIVEN_SYLLABLES | KOREAN_GIVEN_PATTERNS | KOREAN_SPECIFIC_PATTERNS | KOREAN_AMBIGUOUS_PATTERNS + ) +) + + +def _japanese_roman_keys(value: str) -> tuple[str, ...]: + exact = _fold(value) + collapsed = exact.replace("ou", "o").replace("oo", "o").replace("uu", "u") + return (exact,) if exact == collapsed else (exact, collapsed) + + +def _contains_any(values: tuple[str, ...], keys: tuple[str, ...]) -> bool: + return any(_contains(values, key) for key in keys) + + +def _is_han(character: str) -> bool: + return "\u3400" <= character <= "\u4dbf" or "\u4e00" <= character <= "\u9fff" or "\uf900" <= character <= "\ufaff" + + +def _is_kana(character: str) -> bool: + return "\u3040" <= character <= "\u30ff" or "\u31f0" <= character <= "\u31ff" + + +def _is_hangul(value: str) -> bool: + return bool(value) and all("\uac00" <= character <= "\ud7a3" for character in value) + + +def _is_compact_japanese(value: str) -> bool: + return bool(value) and " " not in value and all(_is_han(character) or _is_kana(character) for character in value) + + +def _han_to_kana_boundary(value: str) -> int | None: + index = 0 + while index < len(value) and _is_han(value[index]): + index += 1 + if index and index < len(value) and all(_is_kana(character) for character in value[index:]): + return index + return None + + +class EastAsianNameOrderService: + """Infer only the family-first cases supported by conservative evidence.""" + + def infer( + self, + raw_name: str, + *, + japanese_probability: Callable[[str], float], + ) -> EastAsianNameOrderDecision | None: + """Return semantic components or abstain without changing visible order.""" + surface = WHITESPACE_RE.sub(" ", unicodedata.normalize("NFKC", raw_name)).strip() + if not surface or "," in surface: + return None + + native = self._infer_native(surface, japanese_probability) + if native is not None: + return native + return self._infer_romanized(surface) + + def _infer_native( + self, + surface: str, + japanese_probability: Callable[[str], float], + ) -> EastAsianNameOrderDecision | None: + if _is_hangul(surface): + if len(surface) != KOREAN_NATIVE_TOKEN_LENGTH: + return None + return EastAsianNameOrderDecision( + surface=surface, + given_tokens=(surface[1:],), + middle_tokens=(), + surname_tokens=(surface[:1],), + source_order=("surname", "given"), + reason="korean_native_three_syllable", + ) + if not _is_compact_japanese(surface) or japanese_probability(surface) < JAPANESE_ML_THRESHOLD: + return None + boundary = self._japanese_native_boundary(surface) + return EastAsianNameOrderDecision( + surface=surface, + given_tokens=(surface[boundary:],), + middle_tokens=(), + surname_tokens=(surface[:boundary],), + source_order=("surname", "given"), + reason="japanese_native_dictionary", + ) + + @staticmethod + def _japanese_native_boundary(surface: str) -> int: + transition = _han_to_kana_boundary(surface) + if transition is not None: + return transition + lexicons = _native_lexicons() + default_boundary = 1 if len(surface) == 2 else 2 # noqa: PLR2004 + candidates: list[tuple[int, int, int, int]] = [] + for boundary in range(1, len(surface)): + score = ( + 4 * _contains(lexicons.japanese_surnames, surface[:boundary]) + + 2 * _contains(lexicons.japanese_given_names, surface[boundary:]) + + (boundary == default_boundary) + ) + candidates.append((score, -abs(boundary - default_boundary), -boundary, boundary)) + return max(candidates)[-1] + + def _infer_romanized(self, surface: str) -> EastAsianNameOrderDecision | None: + tokens = surface.split() + if not MIN_ROMANIZED_TOKENS <= len(tokens) <= MAX_ROMANIZED_TOKENS: + return None + if not all(all(character.isalpha() or character in "-'" for character in token) for token in tokens): + return None + lexicons = _roman_lexicons() + + vietnamese = self._infer_vietnamese(tokens, surface, lexicons) + if vietnamese is not None: + return vietnamese + korean = self._infer_korean(tokens, lexicons) + if korean is not None: + return korean + return self._infer_japanese_romanized(tokens, lexicons) + + @staticmethod + def _infer_vietnamese( + tokens: list[str], + surface: str, + lexicons: _RomanLexicons, + ) -> EastAsianNameOrderDecision | None: + if surface.isascii() or not _contains(lexicons.vietnamese_surnames, _fold(tokens[0])): + return None + middle_tokens = tuple(tokens[1:-1]) + return EastAsianNameOrderDecision( + surface=surface, + given_tokens=(tokens[-1],), + middle_tokens=middle_tokens, + surname_tokens=(tokens[0],), + source_order=("surname", *("middle" for _ in middle_tokens), "given"), + reason="vietnamese_unicode_surname_first", + ) + + @staticmethod + def _infer_korean( + tokens: list[str], + lexicons: _RomanLexicons, + ) -> EastAsianNameOrderDecision | None: + if len(tokens) > MAX_KOREAN_ROMANIZED_TOKENS: + return None + if not _contains(lexicons.korean_surnames, _fold(tokens[0])): + return None + if _contains(lexicons.korean_surnames, _fold(tokens[-1])): + return None + given_parts = [_fold(part) for token in tokens[1:] for part in token.split("-") if part] + has_hyphen = any("-" in token for token in tokens[1:]) + all_known = bool(given_parts) and all(part in KOREAN_ROUTING_GIVEN_PARTS for part in given_parts) + if not (has_hyphen or (len(tokens) == MAX_KOREAN_ROMANIZED_TOKENS and all_known)): + return None + given_tokens = tuple(tokens[1:]) + return EastAsianNameOrderDecision( + surface=" ".join(tokens), + given_tokens=given_tokens, + middle_tokens=(), + surname_tokens=(tokens[0],), + source_order=("surname", *("given" for _ in given_tokens)), + reason="korean_romanized_strict", + ) + + @staticmethod + def _infer_japanese_romanized( + tokens: list[str], + lexicons: _RomanLexicons, + ) -> EastAsianNameOrderDecision | None: + if len(tokens) != 2: # noqa: PLR2004 + return None + first_keys = _japanese_roman_keys(tokens[0]) + last_keys = _japanese_roman_keys(tokens[1]) + surname_first = _contains_any(lexicons.japanese_surnames, first_keys) and _contains_any( + lexicons.japanese_given_names, + last_keys, + ) + reverse_plausible = _contains_any(lexicons.japanese_given_names, first_keys) and _contains_any( + lexicons.japanese_surnames, + last_keys, + ) + if not surname_first or reverse_plausible: + return None + return EastAsianNameOrderDecision( + surface=" ".join(tokens), + given_tokens=(tokens[1],), + middle_tokens=(), + surname_tokens=(tokens[0],), + source_order=("surname", "given"), + reason="japanese_romanized_directional_dictionary", + ) + + +__all__ = ["EastAsianNameOrderDecision", "EastAsianNameOrderService"] diff --git a/sinonym/services/ethnicity.py b/sinonym/services/ethnicity.py index 0036b2a..21ebb02 100644 --- a/sinonym/services/ethnicity.py +++ b/sinonym/services/ethnicity.py @@ -9,15 +9,20 @@ from __future__ import annotations import logging +from functools import lru_cache from sinonym.chinese_names_data import ( COMPOUND_VARIANTS, + ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES, JAPANESE_SURNAMES, KOREAN_AMBIGUOUS_PATTERNS, + KOREAN_DIRECTIONAL_FAMILY_FIRST_SURNAMES, + KOREAN_DIRECTIONAL_SINGLE_GIVEN_NAMES, KOREAN_GIVEN_PAIRS, KOREAN_GIVEN_PATTERNS, KOREAN_ONLY_SURNAMES, KOREAN_SPECIFIC_PATTERNS, + NAME_ORDER_ROUTING_KOREAN_SURNAMES, OVERLAPPING_KOREAN_SURNAMES, OVERLAPPING_VIETNAMESE_SURNAMES, VIETNAMESE_GIVEN_PATTERNS, @@ -32,11 +37,23 @@ MIN_COMPOUND_SURNAME_TOKEN_COUNT = 3 JAPANESE_CLASSIFIER_REJECTION = "japanese" JAPANESE_CLASSIFIER_RUNTIME_ERROR = "ML Japanese classifier failed" +MIN_DIRECTIONAL_KOREAN_TOKENS = 2 +MAX_DIRECTIONAL_KOREAN_TOKENS = 3 +MIN_CONTEXTUAL_TAIWAN_SURNAME_FREQUENCY = 100.0 +MIN_CHINESE_SURNAME_STRENGTH = 0.5 +CONTEXTUAL_TAIWAN_GIVEN_PARTS = { + "jungting": ("jung", "ting"), + "tsung-jr": ("tsung", "jr"), +} +KOREAN_DIRECTIONAL_SURNAMES = frozenset( + NAME_ORDER_ROUTING_KOREAN_SURNAMES | KOREAN_ONLY_SURNAMES | OVERLAPPING_KOREAN_SURNAMES, +) # Optional ML Japanese classifier imports - consolidated from separate service try: # Ensure custom model components are importable when deserializing import sinonym.ml_model_components # noqa: F401 + ML_AVAILABLE = True except ImportError: ML_AVAILABLE = False @@ -63,8 +80,7 @@ def __init__(self, confidence_threshold: float = 0.8): self._model = load_skops("chinese_japanese_classifier.skops") except Exception as skops_err: # noqa: BLE001 - skops may raise several deserialization errors. LOGGER.info( - "SKOPS model not available or failed to load (%s); " - "falling back to legacy joblib artifact.", + "SKOPS model not available or failed to load (%s); falling back to legacy joblib artifact.", skops_err, ) self._model = load_joblib("chinese_japanese_classifier.joblib") @@ -195,7 +211,6 @@ def classify_ethnicity( # Check if this is an all-Chinese character input that could be Japanese compact_chinese_text = self._normalizer._text_preprocessor.compact_all_chinese_input(original_text) if compact_chinese_text and self._ml_classifier.is_available(): - # Use ML classifier to check for Japanese names in Chinese characters ml_result = self._ml_classifier.classify_all_chinese_name(compact_chinese_text) @@ -224,10 +239,22 @@ def get_normalized(token: str) -> str: original_keys_normalized = [get_normalized(t) for t in expanded_tokens] expanded_keys = list(set(original_keys_raw + original_keys_normalized)) + # Directional Korean structure must be evaluated before every + # affirmative Chinese shortcut. Surname/given roles are essential: + # many individual syllables and surnames overlap with Chinese. + if self._has_directional_korean_structure(tokens): + return ParseResult.failure("Korean structural patterns detected") + + if self.contextual_taiwan_given_parts(tokens) is not None: + return ParseResult.success_with_name("") + # ================================================================= # TIER 1: DEFINITIVE EVIDENCE (High Confidence) # ================================================================= + if self._has_wade_giles_apostrophe_surname(tokens): + return ParseResult.success_with_name("") + # Single loop for all definitive evidence checks (short-circuit optimization) for key in expanded_keys: # Check for Western names first (most common case, faster lookup) @@ -249,6 +276,11 @@ def get_normalized(token: str) -> str: if clean_key in VIETNAMESE_ONLY_SURNAMES: return ParseResult.failure("appears to be Vietnamese name") + if self._has_reviewed_chinese_surname_alias(tokens): + return ParseResult.success_with_name("") + if self._has_dominant_surname_compact_initial(tokens): + return ParseResult.success_with_name("") + # ================================================================= # TIER 2: CULTURAL CONTEXT (Medium Confidence) # ================================================================= @@ -256,11 +288,8 @@ def get_normalized(token: str) -> str: # Optimized validation chain: calculate overlapping surname evidence once with reduced string ops def check_overlapping_surname(token): clean_token_lower = StringManipulationUtils.remove_spaces(token).lower() - return ( - clean_token_lower in self._data.surnames and ( - clean_token_lower in OVERLAPPING_KOREAN_SURNAMES or - clean_token_lower in OVERLAPPING_VIETNAMESE_SURNAMES - ) + return clean_token_lower in self._data.surnames and ( + clean_token_lower in OVERLAPPING_KOREAN_SURNAMES or clean_token_lower in OVERLAPPING_VIETNAMESE_SURNAMES ) has_overlapping_chinese_surname = any(check_overlapping_surname(token) for token in tokens) @@ -278,15 +307,86 @@ def check_overlapping_surname(token): # TIER 3: CHINESE DEFAULT (Low Confidence) # ================================================================= - chinese_surname_strength = self._calculate_chinese_surname_strength(expanded_keys, normalized_cache) - # Chinese default: Accept if we have any reasonable Chinese surname evidence - if chinese_surname_strength >= 0.5: + if self._has_sufficient_chinese_surname_strength(expanded_keys, normalized_cache): return ParseResult.success_with_name("") # No Chinese evidence found return ParseResult.failure("no Chinese evidence found") + @staticmethod + def _has_reviewed_chinese_surname_alias(tokens: tuple[str, ...]) -> bool: + """Return whether a name edge is a reviewed Chinese surname spelling.""" + return bool(tokens) and ( + tokens[0].lower() in ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES + or tokens[-1].lower() in ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES + ) + + def _has_wade_giles_apostrophe_surname(self, tokens: tuple[str, ...]) -> bool: + """Return whether an exact name edge has backed Wade-Giles surname evidence.""" + return bool(tokens) and ( + self._surname_resolver.evidence_is_wade_giles_apostrophe_surname(tokens[0]) + or self._surname_resolver.evidence_is_wade_giles_apostrophe_surname(tokens[-1]) + ) + + def _has_dominant_surname_compact_initial(self, tokens: tuple[str, ...]) -> bool: + """Return dominant family-first surname plus a vowelless initial bundle.""" + if len(tokens) != 2: + return False + abbreviation = tokens[1] + return bool( + self._surname_resolver.evidence_is_dominant_surname(tokens[0]) + and abbreviation.isalpha() + and 2 <= len(abbreviation) <= 3 + and not any(character.lower() in "aeiou" for character in abbreviation), + ) + + @staticmethod + def _split_roman_components(tokens: tuple[str, ...]) -> list[str]: + """Return lowercase Roman components from spaced or hyphenated tokens.""" + return [part.lower() for token in tokens for part in token.split("-") if part and part.isalpha()] + + @classmethod + @lru_cache(maxsize=4096) + def _has_directional_korean_structure(cls, tokens: tuple[str, ...]) -> bool: + """Return whether surname and given evidence align as a Korean name.""" + if not MIN_DIRECTIONAL_KOREAN_TOKENS <= len(tokens) <= MAX_DIRECTIONAL_KOREAN_TOKENS: + return False + + lowered = tuple(StringManipulationUtils.remove_spaces(token).lower() for token in tokens) + given_first_parts = cls._split_roman_components(lowered[:-1]) + family_first_parts = cls._split_roman_components(lowered[1:]) + + given_first = lowered[-1] in KOREAN_DIRECTIONAL_SURNAMES and tuple(given_first_parts) in KOREAN_GIVEN_PAIRS + if given_first: + return True + + directional_single = ( + lowered[0] in KOREAN_DIRECTIONAL_FAMILY_FIRST_SURNAMES + and len(family_first_parts) == 1 + and family_first_parts[0] in KOREAN_DIRECTIONAL_SINGLE_GIVEN_NAMES + ) + if directional_single: + return True + + return bool( + lowered[0] in KOREAN_DIRECTIONAL_SURNAMES + and lowered[-1] not in KOREAN_DIRECTIONAL_SURNAMES + and tuple(family_first_parts) in KOREAN_GIVEN_PAIRS, + ) + + def contextual_taiwan_given_parts(self, tokens: tuple[str, ...]) -> tuple[str, str] | None: + """Return an exact contextual Taiwan given-name segmentation.""" + if len(tokens) != 2 or self._has_directional_korean_structure(tokens): + return None + given_key = StringManipulationUtils.remove_spaces(tokens[0]).lower() + parts = CONTEXTUAL_TAIWAN_GIVEN_PARTS.get(given_key) + if parts is None: + return None + if self._surname_resolver.evidence_frequency(tokens[-1]) < MIN_CONTEXTUAL_TAIWAN_SURNAME_FREQUENCY: + return None + return parts + def _classify_first_token_surname(self, tokens: tuple[str, ...]) -> str: """Classify the first token's surname type for ethnicity detection.""" if not tokens: @@ -367,6 +467,7 @@ def _korean_given_pairs(self, tokens: tuple[str, ...], surname_type: str) -> lis @staticmethod def _candidate_korean_given_sequences(tokens: tuple[str, ...], surname_type: str) -> list[list[str]]: """Return plausible given-token spans for Korean pair detection.""" + def split_given_tokens(given_tokens: tuple[str, ...]) -> list[str]: split_tokens: list[str] = [] for token in given_tokens: @@ -387,8 +488,7 @@ def split_given_tokens(given_tokens: tuple[str, ...]) -> list[str]: def _has_trailing_overlapping_korean_surname(tokens: tuple[str, ...]) -> bool: """Return whether a final token can be a Korean surname.""" return bool( - tokens - and StringManipulationUtils.remove_spaces(tokens[-1]).lower() in OVERLAPPING_KOREAN_SURNAMES, + tokens and StringManipulationUtils.remove_spaces(tokens[-1]).lower() in OVERLAPPING_KOREAN_SURNAMES, ) def _first_token_has_dominant_chinese_surname(self, tokens: tuple[str, ...]) -> bool: @@ -409,12 +509,17 @@ def _calculate_non_chinese_patterns_unified( # Calculate Korean score korean_score = self._calculate_korean_score_from_analysis( - analysis, tokens, expanded_keys, normalized_cache, + analysis, + tokens, + expanded_keys, + normalized_cache, ) # Calculate Vietnamese score vietnamese_score = self._calculate_vietnamese_score_from_analysis( - analysis, tokens, expanded_keys, + analysis, + tokens, + expanded_keys, ) return korean_score, vietnamese_score @@ -438,17 +543,12 @@ def _calculate_korean_score_from_analysis( score = 0.0 # Overlapping Korean surname anywhere in the name - overlapping_any = any( - StringManipulationUtils.remove_spaces(t).lower() in OVERLAPPING_KOREAN_SURNAMES - for t in tokens - ) + overlapping_any = any(StringManipulationUtils.remove_spaces(t).lower() in OVERLAPPING_KOREAN_SURNAMES for t in tokens) # Helper functions (reused from original) def is_chinese_given_strict(tok: str) -> bool: normalized = normalized_cache[tok] if normalized_cache and tok in normalized_cache else self._normalizer.norm(tok) - return ( - normalized in self._data.given_names_normalized or tok.lower() in self._data.given_names_normalized - ) + return normalized in self._data.given_names_normalized or tok.lower() in self._data.given_names_normalized def has_korean_signature(tok: str) -> bool: t = tok.lower() @@ -464,10 +564,7 @@ def has_korean_signature(tok: str) -> bool: first_cn = is_chinese_given_strict(first) second_cn = is_chinese_given_strict(second) if overlapping_any: - if ( - (has_korean_signature(first) or has_korean_signature(second)) - and not (first_cn and second_cn) - ): + if (has_korean_signature(first) or has_korean_signature(second)) and not (first_cn and second_cn): score += 3.0 elif (not (first_cn and second_cn)) and (has_korean_signature(first) or has_korean_signature(second)): score += 3.0 @@ -509,9 +606,7 @@ def _calculate_vietnamese_score_from_analysis( if vietnamese_surname_count >= 1 and vietnamese_given_count >= 1: # Check if any token is a Chinese surname - has_chinese_surname = any( - StringManipulationUtils.remove_spaces(key) in self._data.surnames for key in expanded_keys - ) + has_chinese_surname = any(StringManipulationUtils.remove_spaces(key) in self._data.surnames for key in expanded_keys) if not has_chinese_surname: score += 2.0 # Strong Vietnamese pattern @@ -522,8 +617,12 @@ def _calculate_vietnamese_score_from_analysis( return score - def _calculate_chinese_surname_strength(self, expanded_keys: list[str], normalized_cache: dict[str, str]) -> float: - """Calculate Chinese surname strength (simplified from original).""" + def _has_sufficient_chinese_surname_strength( + self, + expanded_keys: list[str], + normalized_cache: dict[str, str], + ) -> bool: + """Return whether nonnegative surname evidence reaches the Chinese threshold.""" chinese_surname_strength = 0.0 # Local memoization for repeated split/component checks @@ -555,6 +654,8 @@ def _calculate_chinese_surname_strength(self, expanded_keys: list[str], normaliz base_strength = 0.2 chinese_surname_strength += base_strength + if chinese_surname_strength >= MIN_CHINESE_SURNAME_STRENGTH: + return True # Check for compact compound surnames in COMPOUND_VARIANTS elif clean_key_lower in COMPOUND_VARIANTS: # This is a compact compound surname - give it good strength @@ -566,7 +667,7 @@ def _calculate_chinese_surname_strength(self, expanded_keys: list[str], normaliz part1, part2 = compound_parts if part1 in self._data.surnames_normalized and part2 in self._data.surnames_normalized: # Both parts are valid Chinese surnames, give high confidence - chinese_surname_strength += 1.0 + return True else: # NEW: Check if this could be a compound Chinese given name if clean_key_lower in split_result_cache: @@ -604,5 +705,7 @@ def _calculate_chinese_surname_strength(self, expanded_keys: list[str], normaliz if all_chinese_components: # Add modest boost for compound given names (helps cases like "Beining") chinese_surname_strength += 0.3 + if chinese_surname_strength >= MIN_CHINESE_SURNAME_STRENGTH: + return True - return chinese_surname_strength + return False diff --git a/sinonym/services/formatting.py b/sinonym/services/formatting.py index 25ca821..eb18b4b 100644 --- a/sinonym/services/formatting.py +++ b/sinonym/services/formatting.py @@ -17,6 +17,8 @@ from typing import TYPE_CHECKING +from sinonym.chinese_names_data import ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES +from sinonym.services.name_lookup import DOMINANT_CHINESE_SURNAME_FREQ_MIN, SurnameResolver from sinonym.utils.string_manipulation import StringManipulationUtils if TYPE_CHECKING: @@ -38,6 +40,7 @@ def __init__(self, context_or_config, normalizer=None, data=None): self._config = context_or_config self._normalizer = normalizer self._data = data + self._surname_resolver = SurnameResolver(self._data, self._normalizer) def format_name_output( self, @@ -58,7 +61,15 @@ def format_name_output( - NO structural/pattern-based splitting (that belongs in TextPreprocessor) """ # First validate that given tokens could plausibly be Chinese - if not self._normalizer.validate_given_tokens(given_tokens, normalized_cache): + compact_initial = self._accepts_compact_initial(surname_tokens, given_tokens) + alias_given_parts = self._reviewed_alias_compact_given_parts(surname_tokens, given_tokens) + wade_giles_single_given = self._accepts_wade_giles_single_given(surname_tokens, given_tokens) + if ( + not self._normalizer.validate_given_tokens(given_tokens, normalized_cache) + and not compact_initial + and not alias_given_parts + and not wade_giles_single_given + ): msg = "given name tokens are not plausibly Chinese" raise ValueError(msg) @@ -75,6 +86,18 @@ def format_name_output( parts.append(token) continue + if compact_initial and len(given_tokens) == 1 and token == given_tokens[0]: + parts.append(token) + continue + + if alias_given_parts and len(given_tokens) == 1 and token == given_tokens[0]: + parts.extend(alias_given_parts) + continue + + if wade_giles_single_given and len(given_tokens) == 1 and token == given_tokens[0]: + parts.append(token) + continue + # Check if token is already a valid Chinese syllable (no splitting needed) if self._normalizer.is_valid_chinese_phonetics(token): parts.append(token) @@ -128,9 +151,7 @@ def format_name_output( # Determine separator based on part lengths # Use spaces when we have mixed-length parts (some single chars, some multi-char) if len(formatted_parts) > 1: - part_lengths = [ - len(part.replace("-", "")) for part in formatted_parts - ] # Count chars, ignoring internal hyphens + part_lengths = [len(part.replace("-", "")) for part in formatted_parts] # Count chars, ignoring internal hyphens has_single_char = any(length == 1 for length in part_lengths) has_multi_char = any(length > 1 for length in part_lengths) @@ -208,7 +229,15 @@ def format_name_output_with_tokens( - surname_str / given_str: component strings as used in full_formatted_name """ # Validate given name tokens first - if not self._normalizer.validate_given_tokens(given_tokens, normalized_cache): + compact_initial = self._accepts_compact_initial(surname_tokens, given_tokens) + alias_given_parts = self._reviewed_alias_compact_given_parts(surname_tokens, given_tokens) + wade_giles_single_given = self._accepts_wade_giles_single_given(surname_tokens, given_tokens) + if ( + not self._normalizer.validate_given_tokens(given_tokens, normalized_cache) + and not compact_initial + and not alias_given_parts + and not wade_giles_single_given + ): msg = "given name tokens are not plausibly Chinese" raise ValueError(msg) @@ -224,6 +253,18 @@ def format_name_output_with_tokens( parts.append(token) continue + if compact_initial and len(given_tokens) == 1 and token == given_tokens[0]: + parts.append(token) + continue + + if alias_given_parts and len(given_tokens) == 1 and token == given_tokens[0]: + parts.extend(alias_given_parts) + continue + + if wade_giles_single_given and len(given_tokens) == 1 and token == given_tokens[0]: + parts.append(token) + continue + if self._normalizer.is_valid_chinese_phonetics(token): parts.append(token) continue @@ -356,6 +397,53 @@ def format_name_output_with_tokens( return full_formatted, given_tokens_final, surname_tokens_final, surname_str, given_str, middle_tokens_final + def _accepts_compact_initial(self, surname_tokens: list[str], given_tokens: list[str]) -> bool: + """Return dominant surname plus a single vowelless 2-3 letter bundle.""" + if len(surname_tokens) != 1 or len(given_tokens) != 1: + return False + abbreviation = given_tokens[0] + surname_key = self._normalizer.norm_light(surname_tokens[0]) + return bool( + self._data.get_surname_freq_as_written(surname_key) >= DOMINANT_CHINESE_SURNAME_FREQ_MIN + and abbreviation.isalpha() + and 2 <= len(abbreviation) <= 3 # noqa: PLR2004 + and not any(character.lower() in "aeiou" for character in abbreviation), + ) + + def _accepts_wade_giles_single_given( + self, + surname_tokens: list[str], + given_tokens: list[str], + ) -> bool: + """Allow one alphabetic given token behind exact Wade-Giles surname evidence.""" + return bool( + len(surname_tokens) == 1 + and len(given_tokens) == 1 + and given_tokens[0].isalpha() + and self._surname_resolver.evidence_is_wade_giles_apostrophe_surname(surname_tokens[0]), + ) + + def _reviewed_alias_compact_given_parts( + self, + surname_tokens: list[str], + given_tokens: list[str], + ) -> list[str]: + """Split one compact given token only under reviewed surname-alias evidence.""" + if ( + len(surname_tokens) != 1 + or len(given_tokens) != 1 + or surname_tokens[0].lower() not in ETHNICITY_CHINESE_SURNAME_ROMANIZATION_ALIASES + ): + return [] + token = given_tokens[0] + candidates = [ + [token[:index], token[index:]] + for index in range(1, len(token)) + if self._normalizer.is_valid_chinese_phonetics(token[:index]) + and self._normalizer.is_valid_chinese_phonetics(token[index:]) + ] + return candidates[0] if len(candidates) == 1 else [] + def capitalize_name_part(self, part: str) -> str: """Properly capitalize a name part - delegated to centralized utility.""" return StringManipulationUtils.capitalize_name_part(part) @@ -374,6 +462,7 @@ def _format_compound_with_metadata( Returns: Formatted surname string using metadata-driven formatting """ + def _get_meta_for_token(token: str) -> CompoundMetadata | None: meta = compound_metadata.get(token) if meta is not None: diff --git a/sinonym/services/name_lookup.py b/sinonym/services/name_lookup.py index b581205..7bff809 100644 --- a/sinonym/services/name_lookup.py +++ b/sinonym/services/name_lookup.py @@ -2,11 +2,26 @@ from __future__ import annotations +import re from typing import TYPE_CHECKING from sinonym.utils.string_manipulation import StringManipulationUtils DOMINANT_CHINESE_SURNAME_FREQ_MIN = 10_000.0 +_WADE_GILES_APOSTROPHE_SURNAME_RE = re.compile(r"(?:ch|ts|tz|k|p|t)'[a-z]+", re.IGNORECASE) +_APOSTROPHE_TRANSLATION = str.maketrans( + { + "\u02b9": "'", + "\u02bb": "'", + "\u02bc": "'", + "\u2018": "'", + "\u2019": "'", + "\u2032": "'", + "\uff07": "'", + "`": "'", + "\u00b4": "'", + }, +) if TYPE_CHECKING: from collections.abc import Sequence @@ -63,6 +78,13 @@ def evidence_is_dominant_surname(self, token: str) -> bool: """Return whether ``token`` carries dominant as-written Chinese surname evidence.""" return self.evidence_frequency(token) >= DOMINANT_CHINESE_SURNAME_FREQ_MIN + def evidence_is_wade_giles_apostrophe_surname(self, token: str) -> bool: + """Return exact-token Wade-Giles spelling backed by existing surname data.""" + normalized_apostrophe = token.translate(_APOSTROPHE_TRANSLATION) + return bool( + _WADE_GILES_APOSTROPHE_SURNAME_RE.fullmatch(normalized_apostrophe) and self.evidence_is_surname(token), + ) + def evidence_span_key(self, token: str) -> str: """Return the evidence key for batch span assembly only.""" return self._evidence_key(token) @@ -90,8 +112,10 @@ def _parser_key(self, surname_tokens: Sequence[str]) -> str: """Derive the parser-strict surname key used by parse scoring.""" self._require_tokens(surname_tokens) cache_key = tuple(surname_tokens) - if cache_key in self._parser_key_cache: + try: return self._parser_key_cache[cache_key] + except KeyError: + pass if len(cache_key) == 1: parser_key = self._single_parser_key(cache_key[0]) @@ -124,12 +148,15 @@ def _contains_cjk(self, token: str) -> bool: def _evidence_key(self, token: str) -> str: """Derive the as-written surname evidence key.""" - if token not in self._evidence_key_cache: - self._evidence_key_cache[token] = self._data.surname_lookup_key( + try: + return self._evidence_key_cache[token] + except KeyError: + evidence_key = self._data.surname_lookup_key( self._normalizer.norm_light(token), self._normalizer.norm(token), ) - return self._evidence_key_cache[token] + self._evidence_key_cache[token] = evidence_key + return evidence_key def _is_wade_giles_initial_remapped_surname_token(self, token: str) -> bool: """Return whether a direct surname was remapped by a Wade-Giles initial rule.""" diff --git a/sinonym/services/non_person.py b/sinonym/services/non_person.py index 86aab9f..2ff9996 100644 --- a/sinonym/services/non_person.py +++ b/sinonym/services/non_person.py @@ -59,6 +59,8 @@ def __init__( def failure_reason(self, raw_name: str) -> str | None: """Return a failure reason when the input is clearly not one personal name.""" + if raw_name.isascii(): + return NON_PERSON_FAILURE_REASON if self._has_latin_author_list_shape(raw_name) else None if ( self._has_cjk_non_person_marker(raw_name) or self._has_latin_author_list_shape(raw_name) @@ -177,6 +179,8 @@ def _is_trailing_latin_initial_suffix(raw_name: str, letter_index: int) -> bool: def _cjk_chunks(self, raw_name: str) -> list[str]: """Return contiguous CJK runs split by non-CJK separators.""" + if raw_name.isascii(): + return [] chunks: list[str] = [] current: list[str] = [] @@ -196,7 +200,7 @@ def _cjk_chunks(self, raw_name: str) -> list[str]: def _has_latin_author_list_shape(self, raw_name: str) -> bool: """Return whether a Latin string looks like several Chinese author names collapsed together.""" - if self._config.cjk_pattern.search(raw_name): + if not raw_name.isascii() and self._config.cjk_pattern.search(raw_name): return False tokens = LATIN_WORD_RE.findall(raw_name) diff --git a/sinonym/services/normalization.py b/sinonym/services/normalization.py index d484d3c..45a089c 100644 --- a/sinonym/services/normalization.py +++ b/sinonym/services/normalization.py @@ -9,6 +9,7 @@ import string from dataclasses import dataclass +from functools import lru_cache from typing import TYPE_CHECKING, Literal from sinonym.chinese_names_data import VALID_CHINESE_RIMES @@ -142,7 +143,9 @@ def get_normalized(self, token: str, norm_cache: dict[str, str]) -> str: Consolidates the common pattern: normalized_cache.get(token, self.norm(token)) """ - return norm_cache.get(token, self.norm(token)) + if token in norm_cache: + return norm_cache[token] + return self.norm(token) def apply(self, raw_name: str) -> NormalizedInput: """ @@ -152,6 +155,19 @@ def apply(self, raw_name: str) -> NormalizedInput: if not raw_name or not raw_name.strip(): return NormalizedInput.empty(raw_name) + simple_tokens = self.simple_latin_tokens(raw_name) + if simple_tokens is not None: + norm_map = {token: self._text_normalizer.normalize_token(token) for token in simple_tokens} + compound_metadata = self._compound_detector.generate_compound_metadata(simple_tokens, self._data) + return NormalizedInput( + raw=raw_name, + cleaned=raw_name, + tokens=simple_tokens, + roman_tokens=simple_tokens, + norm_map=norm_map, + compound_metadata=compound_metadata, + ) + # Phase 1: Clean input (single regex pass) cleaned, from_camel_case_pair, surname_first_parenthetical_hint = self._text_preprocessor.preprocess_input( raw_name, @@ -197,6 +213,21 @@ def apply(self, raw_name: str) -> NormalizedInput: surname_first_parenthetical_hint=surname_first_parenthetical_hint, ) + @staticmethod + @lru_cache(maxsize=4096) + def simple_latin_tokens(raw_name: str) -> tuple[str, ...] | None: + """Return clean space-delimited ASCII name tokens, or abstain.""" + if not raw_name.isascii() or raw_name != raw_name.strip(): + return None + tokens = tuple(raw_name.split(" ")) + if len(tokens) < 2 or any(not token for token in tokens): + return None + for token in tokens: + parts = token.replace("'", "-").split("-") + if any(not part.isalpha() for part in parts): + return None + return tokens + def _process_mixed_tokens(self, tokens: list[str], is_all_chinese: bool = False) -> list[str]: """Extract existing mixed token processing logic with enhanced all-Chinese support.""" mix = [] @@ -323,8 +354,8 @@ def aligned_bilingual_pairs(self, normalized_input: NormalizedInput) -> tuple[Bi else: return None - han_pinyin = tuple(self._cache_service.han_to_pinyin_fast(han_token)) - if not roman_token or not han_pinyin or not self._roman_matches_han_token(roman_token, han_pinyin): + han_pinyin = self._matching_aligned_han_pinyin(roman_token, han_token) + if not roman_token or han_pinyin is None: return None pairs.append(BilingualTokenPair(roman_token=roman_token, han_token=han_token, han_pinyin=han_pinyin)) @@ -332,6 +363,22 @@ def aligned_bilingual_pairs(self, normalized_input: NormalizedInput) -> tuple[Bi return tuple(pairs) if len(pairs) >= MIN_BILINGUAL_ALIGNMENT_PAIRS else None + def _matching_aligned_han_pinyin(self, roman_token: str, han_token: str) -> tuple[str, ...] | None: + """Match one explicit pair, allowing heteronyms only for recognized Han surnames.""" + primary = tuple(self._cache_service.han_to_pinyin_fast(han_token)) + if primary and self._roman_matches_han_token(roman_token, primary): + return primary + if self._data is None or self._data.surname_frequencies.get(han_token, 0.0) <= 0: + return None + return next( + ( + alternative + for alternative in self._cache_service.han_to_pinyin_alternatives_fast(han_token) + if self._roman_matches_han_token(roman_token, alternative) + ), + None, + ) + def _is_han_token(self, token: str) -> bool: """Return whether the whole token is CJK characters.""" return bool(token) and all(self._config.cjk_pattern.search(char) for char in token) diff --git a/sinonym/services/parsing.py b/sinonym/services/parsing.py index 1047fd6..e65a262 100644 --- a/sinonym/services/parsing.py +++ b/sinonym/services/parsing.py @@ -345,9 +345,13 @@ def _best_parse_tokens( normalized_cache, compound_metadata, ) - parses = [(surname, given) for surname, given, _ in parses_with_format] - if not parses: + if not parses_with_format: return None + if len(parses_with_format) == 1: + surname_tokens, given_tokens, original_compound_format = parses_with_format[0] + needs_initial_guard = any(len(token) == 1 for token in given_tokens) and any(len(token) > 3 for token in tokens) + if not needs_initial_guard: + return surname_tokens, given_tokens, original_compound_format scored_parses = [] has_multi_syllable_tokens = any(len(token) > 3 for token in tokens) @@ -468,7 +472,7 @@ def _generate_all_parses_with_format( ): # This is a multi-token compound in the middle original_format = self._get_compound_original_format(second_meta, tokens[1:3]) - parses.append((tokens[1:3], tokens[0:1], original_format)) + parses.append((tokens[1:3], [tokens[0], *tokens[3:]], original_format)) # 2. Single-token surnames - only at beginning or end (contiguous sequences only) # Surname-first pattern: surname + given_names diff --git a/sinonym/services/person_name_normalization.py b/sinonym/services/person_name_normalization.py new file mode 100644 index 0000000..95d5cb7 --- /dev/null +++ b/sinonym/services/person_name_normalization.py @@ -0,0 +1,1449 @@ +"""Dependency-free canonical normalization for personal names. + +This module deliberately does not decide whether a name is Chinese and does not +call the Chinese parser. It provides a small boundary-normalization contract +that detector and batch entry points can invoke after their existing routing +decisions have been made. +""" + +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass, replace +from enum import Enum + +from sinonym.coretypes import CanonicalName, NameComponents + + +class PersonNameOutcome(str, Enum): + """Typed classification returned by canonical name normalization.""" + + PERSON = "person" + NON_PERSON = "non_person" + INVALID = "invalid" + + +class DropReason(str, Enum): + """Reason that a source token was intentionally omitted.""" + + TITLE = "title" + CREDENTIAL = "credential" + AFFILIATION = "affiliation" + CONNECTOR = "connector" + DUPLICATE = "duplicate" + + +@dataclass(frozen=True) +class DroppedNameToken: + """One source token omitted from the canonical name.""" + + text: str + source_role: str + reason: DropReason + + +@dataclass(frozen=True) +class PersonNameNormalizationResult: + """Typed outcome of one canonical name normalization request.""" + + outcome: PersonNameOutcome + canonical_name: CanonicalName | None = None + reason: str | None = None + dropped_tokens: tuple[DroppedNameToken, ...] = () + + +@dataclass(frozen=True) +class _Token: + text: str + source_role: str + position: int + source_text: str | None = None + + +@dataclass(frozen=True) +class _DroppedToken: + token: _Token + reason: DropReason + + +_WHITESPACE_RE = re.compile(r"\s+") +_JOINER_SPACING_RE = re.compile(r"\s*([-'])\s*") +_DUPLICATE_APOSTROPHE_RE = re.compile(r"'{2,}") +_LEADING_STRAY_JOINER_RE = re.compile(r"^[-']\s+") +_WORD_AND_RE = re.compile(r"\band\b", re.IGNORECASE) +_TOKEN_RE = re.compile(r"\S+") +_TRAILING_DIGITS_RE = re.compile(r"\d+$") +_INITIAL_RE = re.compile(r"([^\W\d_])\.", re.UNICODE) +_MULTI_INITIAL_RE = re.compile(r"(?:[^\W\d_]\.)+[^\W\d_]\.?", re.UNICODE) +_ABBREVIATED_TOKEN_RE = re.compile(r"(?:[^\W\d_]+[-'])*[^\W\d_]{1,3}\.", re.UNICODE) +_COMPOUND_INITIAL_RE = re.compile(r"[^\W\d_]\.-[^\W\d_]\.", re.UNICODE) +_LEADING_HYPHEN_INITIAL_RE = re.compile(r"^-([^\W\d_])\.$", re.UNICODE) +_FUSED_INITIAL_SURNAME_RE = re.compile(r"^([^\W\d_])\.(.+)$", re.UNICODE) +_LOWER_MARKER_INITIAL_RE = re.compile(r"^([a-z])([A-Z])\.$") +_PARENTHETICAL_DUPLICATE_RE = re.compile(r"^(\S+)\s+\(([^)]+)\)\s+(.+)$") +_ASCII_JOINER_TRANSLATION = str.maketrans({"`": "'"}) +_PRE_NFKC_JOINER_TRANSLATION = str.maketrans({"\u00b4": "'"}) + +_APOSTROPHE_LIKE = frozenset( + { + "'", + "\u02b9", # modifier letter prime + "\u02bb", # modifier letter turned comma + "\u02bc", # modifier letter apostrophe + "\u2018", # left single quotation mark + "\u2019", # right single quotation mark + "\u201a", # single low-9 quotation mark + "\u201b", # single high-reversed-9 quotation mark + "\u2032", # prime + "\u2035", # reversed prime + "\ua78b", # Latin capital letter saltillo + "\ua78c", # Latin small letter saltillo + "\uff07", # fullwidth apostrophe + "`", # grave accent used as an apostrophe + "\uff40", # fullwidth grave accent + "\u00b4", # acute accent used as an apostrophe + }, +) +_DASH_LIKE_EXTRAS = frozenset({"\u2043", "\u2212", "\u208b"}) + +_TITLE_KEYS = frozenset( + { + "capt", + "captain", + "chaplain", + "dame", + "doctor", + "dr", + "father", + "frau", + "hon", + "honorable", + "lord", + "miss", + "mr", + "mrs", + "ms", + "pastor", + "prof", + "professor", + "rabbi", + "rev", + "reverend", + "sir", + "univprof", + }, +) +_TITLE_QUALIFIER_KEYS = frozenset({"hc", "habil", "honoraire", "med", "nat", "rer"}) +_LEADING_NAME_ABBREVIATION_KEYS = frozenset({"md"}) +_LONG_NAME_ABBREVIATION_KEYS = frozenset({"mohd", "most"}) +_CREDENTIAL_KEYS = frozenset( + { + "ba", + "bs", + "bsc", + "dds", + "dmd", + "do", + "dphil", + "dvm", + "edd", + "esq", + "facp", + "facr", + "frcp", + "jd", + "llb", + "llm", + "ma", + "mba", + "md", + "meng", + "mph", + "mpa", + "ms", + "msc", + "pharmd", + "phd", + "psyd", + "rn", + }, +) +_AMBIGUOUS_CREDENTIAL_KEYS = frozenset({"ba", "bs", "do", "jd", "ma", "mba", "md", "meng", "mpa", "ms", "rn"}) +_MIXED_CASE_CREDENTIALS = {"meng": "MEng"} +_FAMILY_PARTICLES = frozenset( + { + "al", + "ap", + "ben", + "bin", + "da", + "das", + "dal", + "de", + "del", + "della", + "den", + "der", + "di", + "dos", + "du", + "el", + "ibn", + "la", + "las", + "le", + "los", + "st", + "ter", + "van", + "von", + "zu", + "zum", + "zur", + }, +) +_LOWERCASE_RELATIONAL_TOKENS = frozenset({"b.", "d.", "e", "kizi", "kyzy", "oglu", "o'g'li", "oğlu", "qizi"}) +_STRONG_FAMILY_PARTICLE_SPANS = ( + ("de", "la"), + ("de", "las"), + ("de", "los"), + ("van", "der"), + ("von", "der"), + ("da",), + ("das",), + ("dos",), +) +_ORGANIZATION_WORDS = frozenset( + { + "association", + "center", + "centre", + "committee", + "company", + "consortium", + "corporation", + "department", + "hospital", + "inc", + "institute", + "laboratory", + "ltd", + "society", + "team", + "university", + }, +) +_STANDARD_SUFFIXES = { + "jr": "Jr.", + "junior": "Jr.", + "sr": "Sr.", + "senior": "Sr.", + "2nd": "2nd", + "3rd": "3rd", + "4th": "4th", + "5th": "5th", + "6th": "6th", +} +_RAW_ROMAN_SUFFIXES = frozenset({"II", "III", "IV", "VI", "VII", "VIII", "IX", "X"}) +_EXPLICIT_ROMAN_SUFFIXES = _RAW_ROMAN_SUFFIXES | {"I", "V", "X"} +_CASE_INSENSITIVE_ROMAN_SUFFIXES = _RAW_ROMAN_SUFFIXES - {"II"} +_SIMPLE_TWO_TOKEN_POLICY_KEYS = frozenset( + _TITLE_KEYS + | _TITLE_QUALIFIER_KEYS + | _CREDENTIAL_KEYS + | _ORGANIZATION_WORDS + | _STANDARD_SUFFIXES.keys() + | {suffix.casefold() for suffix in _RAW_ROMAN_SUFFIXES} + | {"and"}, +) +_TWO_COMPONENTS = 2 +_THREE_COMPONENTS = 3 +_FOUR_COMPONENTS = 4 +_MAX_SPACED_CREDENTIAL_TOKENS = 3 +_MAX_DOTTED_ABBREVIATION_LETTERS = 3 +_MIN_PARENTHESIZED_TOKEN_LENGTH = 3 +_MC_PREFIX_LENGTH = 2 + + +class PersonNameNormalizationService: + """Normalize raw or structured personal names without Chinese routing.""" + + def normalize_text(self, raw_name: str | None) -> PersonNameNormalizationResult: # noqa: C901, PLR0911, PLR0912 + """Normalize one raw name string into semantic canonical components.""" + if not isinstance(raw_name, str): + return self._invalid("name must be a string") + + simple_result = self._normalize_simple_two_token_text(raw_name) + if simple_result is not None: + return simple_result + + source_text = raw_name + normalized_input, leading_markers = self._strip_leading_superscript_affiliation(raw_name) + surface = self._normalize_surface(normalized_input) + if not surface: + return self._invalid("name is empty") + if not any(character.isalpha() for character in surface): + return self._invalid("name has no letters") + + dropped: list[_DroppedToken] = [] + if leading_markers: + dropped.append(_DroppedToken(_Token(leading_markers, "", 0), DropReason.AFFILIATION)) + surface, affiliation_dropped = self._strip_trailing_affiliation_surface(surface) + dropped.extend(affiliation_dropped) + surface = self._collapse_parenthetical_duplicate_surface(surface) + non_person_reason = self._non_person_reason(surface) + if non_person_reason is not None: + return self._non_person(non_person_reason) + + segment_matches = list(re.finditer(r"[^,]+", surface)) + segments = [self._tokens(match.group(), "", match.start()) for match in segment_matches] + segments = [segment for segment in segments if segment] + if not segments: + return self._invalid("name has no usable tokens") + if leading_markers: + first = segments[0][0] + segments[0][0] = replace(first, source_text=f"{leading_markers}{first.text}") + + suffix = "" + suffix_token: _Token | None = None + while len(segments) > 1: + ambiguous_given_segment = ( + len(segments) == _TWO_COMPONENTS + and len(segments[-1]) == 1 + and self._is_ambiguous_credential(segments[-1][0].text) + and ( + len(segments[0]) == 1 + or any(self._particle_key(token.text) in _FAMILY_PARTICLES for token in segments[0][:-1]) + ) + ) + if ambiguous_given_segment: + break + boundary = self._consume_boundary_segment(segments[-1]) + if boundary is None: + break + segment_suffix, segment_suffix_token, segment_dropped = boundary + segments.pop() + dropped.extend(segment_dropped) + if segment_suffix: + if suffix: + return self._invalid("name has multiple suffixes", dropped) + suffix = segment_suffix + suffix_token = segment_suffix_token + + if len(segments) > _TWO_COMPONENTS: + return self._non_person("multiple comma-separated names") + if len(segments) == _TWO_COMPONENTS: + return self._normalize_comma_name( + source_text, + segments[0], + segments[1], + suffix, + suffix_token, + dropped, + ) + return self._normalize_regular_name(source_text, segments[0], suffix, suffix_token, dropped) + + def _normalize_simple_two_token_text(self, raw_name: str) -> PersonNameNormalizationResult | None: + """Normalize policy-neutral two-token ASCII names without preprocessing.""" + if not raw_name.isascii() or raw_name != raw_name.strip(): + return None + raw_tokens = raw_name.split(" ") + if len(raw_tokens) != _TWO_COMPONENTS or any(not token.isalpha() for token in raw_tokens): + return None + if any(token.casefold() in _SIMPLE_TWO_TOKEN_POLICY_KEYS for token in raw_tokens): + return None + + source_tokens = [ + _Token(raw_tokens[0], "", 0), + _Token(raw_tokens[1], "", len(raw_tokens[0]) + 1), + ] + given = [replace(source_tokens[0], source_role="given")] + surname = [replace(source_tokens[1], source_role="surname")] + source = self._components(given, [], surname, []) + return self._person(raw_name, source, given, [], surname, "", []) + + def normalize_components( # noqa: C901, PLR0911 + self, + *, + first_name: str | None = None, + middle_name: str | None = None, + last_name: str | None = None, + suffix: str | None = None, + ) -> PersonNameNormalizationResult: + """Normalize already structured first, middle, last, and suffix fields.""" + values = { + "given": first_name, + "middle": middle_name, + "surname": last_name, + "suffix": suffix, + } + if any(value is not None and not isinstance(value, str) for value in values.values()): + return self._invalid("name components must be strings or null") + + stripped_values: dict[str, str] = {} + leading_markers_by_role: dict[str, str] = {} + for role, value in values.items(): + stripped_values[role], leading_markers_by_role[role] = self._strip_leading_superscript_affiliation(value or "") + surfaces = {role: self._normalize_surface(value) for role, value in stripped_values.items()} + source_surfaces = { + role: f"{leading_markers_by_role[role]}{surface}" if surface else "" for role, surface in surfaces.items() + } + source_text = " ".join(surface for surface in source_surfaces.values() if surface) + if not source_text: + return self._invalid("name is empty") + if not any(character.isalpha() for character in source_text): + return self._invalid("name has no letters") + + non_person_reason = self._non_person_reason(source_text) + if non_person_reason is not None: + return self._non_person(non_person_reason) + + position = 0 + source_by_role: dict[str, list[_Token]] = {} + for role in ("given", "middle", "surname", "suffix"): + source_by_role[role] = self._tokens(surfaces[role], role, position) + if leading_markers_by_role[role] and source_by_role[role]: + first = source_by_role[role][0] + source_by_role[role][0] = replace( + first, + source_text=f"{leading_markers_by_role[role]}{first.text}", + ) + position += len(surfaces[role]) + 1 + source = self._components( + source_by_role["given"], + source_by_role["middle"], + source_by_role["surname"], + source_by_role["suffix"], + order=self._structured_source_order(source_by_role), + ) + + dropped = [ + _DroppedToken( + _Token(markers, role, source_by_role[role][0].position), + DropReason.AFFILIATION, + ) + for role, markers in leading_markers_by_role.items() + if markers and source_by_role[role] + ] + middle_tokens = self._strip_structured_middle_surname_artifact( + source_by_role["middle"], + source_by_role["surname"], + dropped, + ) + name_tokens = [ + *source_by_role["given"], + *middle_tokens, + *source_by_role["surname"], + ] + name_tokens = self._repair_leading_marker_initial(name_tokens, dropped) + name_tokens = self._join_separated_compound_initials(name_tokens) + name_tokens = self._strip_leading_titles(name_tokens, dropped) + name_tokens = self._strip_standalone_periods(name_tokens, dropped) + name_tokens = self._strip_boundary_markers(name_tokens, dropped) + + canonical_suffix, _, explicit_dropped = self._consume_explicit_suffix( + source_by_role["suffix"], + ) + dropped.extend(explicit_dropped) + name_tokens, boundary_suffix, _ = self._strip_trailing_boundaries(name_tokens, dropped) + if boundary_suffix: + if canonical_suffix: + return self._invalid("name has multiple suffixes", dropped) + canonical_suffix = boundary_suffix + + name_tokens = self._strip_attached_affiliations(name_tokens, dropped) + invalid_reason = self._invalid_token_reason(name_tokens) + if invalid_reason is not None: + return self._invalid(invalid_reason, dropped) + + cleaned_by_role = { + role: [token for token in name_tokens if token.source_role == role] for role in ("given", "middle", "surname") + } + given, middle, surname = self._repair_structured_roles(cleaned_by_role) + if not given and not middle and not surname: + return self._invalid("no personal-name tokens remain", dropped) + + return self._person(source_text, source, given, middle, surname, canonical_suffix, dropped) + + def _normalize_regular_name( + self, + source_text: str, + tokens: list[_Token], + suffix: str, + suffix_token: _Token | None, + dropped: list[_DroppedToken], + ) -> PersonNameNormalizationResult: + original_tokens = list(tokens) + tokens = self._repair_leading_marker_initial(tokens, dropped) + tokens = self._join_separated_compound_initials(tokens) + tokens = self._strip_leading_titles(tokens, dropped) + tokens = self._strip_standalone_periods(tokens, dropped) + tokens = self._strip_boundary_markers(tokens, dropped) + tokens, boundary_suffix, boundary_suffix_token = self._strip_trailing_boundaries(tokens, dropped) + if boundary_suffix: + if suffix: + return self._invalid("name has multiple suffixes", dropped) + suffix = boundary_suffix + suffix_token = boundary_suffix_token + + tokens = self._strip_attached_affiliations(tokens, dropped) + invalid_reason = self._invalid_token_reason(tokens) + if invalid_reason is not None: + return self._invalid(invalid_reason, dropped) + if not tokens: + return self._invalid("no personal-name tokens remain", dropped) + + packed_surname_first = self._is_packed_surname_first_initials(tokens) + given, middle, surname = self._infer_regular_roles(tokens) + assigned = [*given, *middle, *surname] + assigned_by_position = {token.position: token for token in assigned} + prefix_source = self._regular_prefix_source( + original_tokens, + assigned_by_position, + tokens[0], + dropped, + ) + credential_source = [ + replace(item.token, source_role="suffix") for item in dropped if item.reason is DropReason.CREDENTIAL + ] + source_suffix = [*([suffix_token] if suffix_token is not None else []), *credential_source] + source_order = None + if packed_surname_first and not prefix_source and not source_suffix: + source_order = ("surname", "given", *("middle" for _token in middle)) + source = self._components( + [*prefix_source, *given], + middle, + surname, + source_suffix, + order=source_order, + ) + return self._person(source_text, source, given, middle, surname, suffix, dropped) + + @staticmethod + def _regular_prefix_source( + original_tokens: list[_Token], + assigned_by_position: dict[int, _Token], + first_survivor: _Token, + dropped: list[_DroppedToken], + ) -> list[_Token]: + """Rebuild raw prefix lineage without duplicating an attached-title remainder.""" + prefix: list[_Token] = [] + for token in original_tokens: + if token.position >= first_survivor.position or token.position in assigned_by_position: + continue + if token.position + len(token.text) <= first_survivor.position: + prefix.append(replace(token, source_role="given")) + continue + prefix.extend( + replace(item.token, source_role="given") + for item in dropped + if item.reason is DropReason.TITLE and item.token.position == token.position + ) + return prefix + + def _normalize_comma_name( # noqa: PLR0913 + self, + source_text: str, + family_tokens: list[_Token], + given_tokens: list[_Token], + suffix: str, + suffix_token: _Token | None, + dropped: list[_DroppedToken], + ) -> PersonNameNormalizationResult: + family_tokens = [replace(token, source_role="surname") for token in family_tokens] + given_tokens = [replace(token, source_role="given") for token in given_tokens] + family_tokens = self._strip_leading_titles(family_tokens, dropped, preserve_ambiguous_credentials=True) + given_tokens = self._strip_leading_titles(given_tokens, dropped, preserve_ambiguous_credentials=True) + family_tokens = self._strip_standalone_periods(family_tokens, dropped) + given_tokens = self._strip_standalone_periods(given_tokens, dropped) + family_tokens = self._strip_boundary_markers(family_tokens, dropped) + given_tokens = self._strip_boundary_markers(given_tokens, dropped) + given_tokens, boundary_suffix, boundary_suffix_token = self._strip_trailing_boundaries( + given_tokens, + dropped, + has_external_name_context=bool(family_tokens), + ) + if boundary_suffix: + if suffix: + return self._invalid("name has multiple suffixes", dropped) + suffix = boundary_suffix + suffix_token = boundary_suffix_token + + family_tokens = self._strip_attached_affiliations(family_tokens, dropped) + given_tokens = self._strip_attached_affiliations(given_tokens, dropped) + invalid_reason = self._invalid_token_reason([*family_tokens, *given_tokens]) + if invalid_reason is not None: + return self._invalid(invalid_reason, dropped) + if not family_tokens or not given_tokens: + return self._invalid("comma form requires family and given tokens", dropped) + if self._looks_like_two_complete_names(family_tokens, given_tokens): + return self._non_person("comma separates two complete names") + + given = [replace(given_tokens[0], source_role="given")] + middle = [replace(token, source_role="middle") for token in given_tokens[1:]] + surname = [replace(token, source_role="surname") for token in family_tokens] + source_suffix = [suffix_token] if suffix_token is not None else [] + source = self._components( + given, + middle, + surname, + source_suffix, + order=tuple(["surname"] * len(surname) + ["given"] + ["middle"] * len(middle) + ["suffix"] * len(source_suffix)), + ) + return self._person(source_text, source, given, middle, surname, suffix, dropped) + + def _person( # noqa: PLR0913 + self, + source_text: str, + source: NameComponents, + given: list[_Token], + middle: list[_Token], + surname: list[_Token], + suffix: str, + dropped: list[_DroppedToken], + ) -> PersonNameNormalizationResult: + normalized_given = self._canonical_tokens(given, "given") + normalized_middle = self._canonical_tokens(middle, "middle") + normalized_surname = self._canonical_tokens(surname, "surname") + normalized_suffix = tuple([suffix] if suffix else []) + normalized = NameComponents( + given_name=" ".join(normalized_given), + middle_name=" ".join(normalized_middle), + surname=" ".join(normalized_surname), + suffix=suffix, + given_tokens=normalized_given, + middle_tokens=normalized_middle, + surname_tokens=normalized_surname, + suffix_tokens=normalized_suffix, + order=tuple( + ["given"] * len(normalized_given) + + ["middle"] * len(normalized_middle) + + ["surname"] * len(normalized_surname) + + ["suffix"] * len(normalized_suffix), + ), + ) + text = " ".join( + component + for component in (normalized.given_name, normalized.middle_name, normalized.surname, normalized.suffix) + if component + ) + if not text: + return self._invalid("no personal-name tokens remain", dropped) + + canonical_name = CanonicalName(source_text=source_text, text=text, source=source, normalized=normalized) + dropped = self._infer_dropped_roles(dropped, [*given, *middle, *surname]) + return PersonNameNormalizationResult( + outcome=PersonNameOutcome.PERSON, + canonical_name=canonical_name, + dropped_tokens=self._public_dropped(dropped), + ) + + @staticmethod + def _normalize_surface(value: str) -> str: + if value.isascii(): + normalized = value.translate(_ASCII_JOINER_TRANSLATION) + else: + normalized = value.translate(_PRE_NFKC_JOINER_TRANSLATION) + normalized = unicodedata.normalize("NFKC", normalized) + normalized = "".join(PersonNameNormalizationService._normalize_joiner(character) for character in normalized) + normalized = unicodedata.normalize("NFC", normalized) + normalized = _WHITESPACE_RE.sub(" ", normalized) + normalized = _LEADING_STRAY_JOINER_RE.sub("", normalized) + normalized = _JOINER_SPACING_RE.sub(r"\1", normalized) + normalized = _DUPLICATE_APOSTROPHE_RE.sub("'", normalized) + return normalized.strip(" \t\r\n,") + + @staticmethod + def _strip_leading_superscript_affiliation(value: str) -> tuple[str, str]: + """Strip fused leading superscript digits while returning their lineage.""" + marker_end = 0 + while marker_end < len(value): + character = value[marker_end] + if "SUPERSCRIPT" not in unicodedata.name(character, "") or not character.isdigit(): + break + marker_end += 1 + if marker_end == 0 or marker_end >= len(value) or not value[marker_end].isalpha(): + return value, "" + return value[marker_end:], value[:marker_end] + + def _strip_trailing_affiliation_surface( + self, + surface: str, + ) -> tuple[str, list[_DroppedToken]]: + """Remove an organization phrase after at least two name tokens.""" + tokens = self._tokens(surface, "suffix", 0) + for index, token in enumerate(tokens): + if index < _TWO_COMPONENTS or self._compact_key(token.text) not in _ORGANIZATION_WORDS: + continue + dropped = [_DroppedToken(item, DropReason.AFFILIATION) for item in tokens[index:]] + return surface[: token.position].rstrip(" ,"), dropped + return surface, [] + + def _collapse_parenthetical_duplicate_surface(self, surface: str) -> str: + """Collapse ``Alan (Alan B.) Cantor``-style duplicate given forms.""" + match = _PARENTHETICAL_DUPLICATE_RE.fullmatch(surface) + if match is None: + return surface + outer_given, parenthetical, remainder = match.groups() + parenthetical_tokens = self._tokens(parenthetical, "", 0) + if not parenthetical_tokens: + return surface + if self._compact_key(parenthetical_tokens[0].text) != self._compact_key(outer_given): + return surface + return f"{parenthetical} {remainder}" + + @staticmethod + def _normalize_joiner(character: str) -> str: + character_name = unicodedata.name(character, "") + is_single_quote = "SINGLE" in character_name and "QUOTATION MARK" in character_name + if character in _APOSTROPHE_LIKE or "APOSTROPHE" in character_name or is_single_quote: + return "'" + if ( + unicodedata.category(character) == "Pd" + or character in _DASH_LIKE_EXTRAS + or "HYPHEN" in character_name + or "MINUS SIGN" in character_name + ): + return "-" + return character + + @staticmethod + def _tokens(value: str, source_role: str, offset: int) -> list[_Token]: + return [_Token(match.group(), source_role, offset + match.start()) for match in _TOKEN_RE.finditer(value.strip())] + + @staticmethod + def _compact_key(token: str) -> str: + if token.isalnum(): + return token.casefold() + return "".join(character.casefold() for character in token if character.isalnum()) + + def _strip_leading_titles( + self, + tokens: list[_Token], + dropped: list[_DroppedToken], + *, + preserve_ambiguous_credentials: bool = False, + ) -> list[_Token]: + remaining = list(tokens) + if self._has_leading_et_al_contamination(remaining): + dropped.extend(_DroppedToken(token, DropReason.CONNECTOR) for token in remaining[:2]) + remaining = remaining[2:] + stripped_title = False + while remaining: + token = remaining[0] + attached_title = self._split_attached_leading_title(token) + if attached_title is not None: + title, remainder = attached_title + dropped.append(_DroppedToken(title, DropReason.TITLE)) + remaining[0] = remainder + stripped_title = True + continue + key = self._compact_key(token.text) + if token.text == "AND": + dropped.append(_DroppedToken(token, DropReason.CONNECTOR)) + remaining.pop(0) + continue + multi_initial = bool(_MULTI_INITIAL_RE.fullmatch(token.text)) + prefixed_academic_title = ( + token.text == "PD" and len(remaining) > 1 and self._compact_key(remaining[1].text) in _TITLE_KEYS + ) + if ( + (key in _TITLE_KEYS and not multi_initial) + or (stripped_title and key in _TITLE_QUALIFIER_KEYS) + or prefixed_academic_title + ): + dropped.append(_DroppedToken(token, DropReason.TITLE)) + remaining.pop(0) + stripped_title = True + continue + ambiguous_name_token = self._is_ambiguous_credential(token.text) and ( + preserve_ambiguous_credentials or len(remaining) == _TWO_COMPONENTS + ) + leading_name_abbreviation = ( + key in _LEADING_NAME_ABBREVIATION_KEYS and token.text.endswith(".") and len(remaining) >= _TWO_COMPONENTS + ) or self._is_exact_ma_given_abbreviation(remaining) + if self._is_credential(token.text) and not ambiguous_name_token and not leading_name_abbreviation: + dropped.append(_DroppedToken(token, DropReason.CREDENTIAL)) + remaining.pop(0) + continue + break + return remaining + + @staticmethod + def _has_leading_et_al_contamination(tokens: list[_Token]) -> bool: + """Match only a leading citation marker followed by a two-token name.""" + return bool( + len(tokens) >= _FOUR_COMPONENTS and tokens[0].text.casefold() == "et" and tokens[1].text.casefold() == "al.", + ) + + def _split_attached_leading_title(self, token: _Token) -> tuple[_Token, _Token] | None: + """Split ``Dr.Name`` only when the attached prefix is a known title.""" + prefix, separator, remainder = token.text.partition(".") + if not separator or not remainder or self._compact_key(prefix) not in _TITLE_KEYS: + return None + if not any(character.isalpha() for character in remainder): + return None + title = replace(token, text=f"{prefix}.") + name = replace(token, text=remainder, position=token.position + len(prefix) + 1) + return title, name + + def _strip_trailing_boundaries( + self, + tokens: list[_Token], + dropped: list[_DroppedToken], + *, + has_external_name_context: bool = False, + ) -> tuple[list[_Token], str, _Token | None]: + remaining = list(tokens) + suffix = "" + suffix_token: _Token | None = None + while len(remaining) >= _FOUR_COMPONENTS: + credential_width = self._trailing_spaced_credential_width(remaining) + if not credential_width: + break + dropped.extend(_DroppedToken(token, DropReason.CREDENTIAL) for token in remaining[-credential_width:]) + del remaining[-credential_width:] + while remaining: + token = remaining[-1] + ambiguous_name_token = self._is_ambiguous_credential(token.text) and ( + (not has_external_name_context and len(remaining) == _TWO_COMPONENTS) + or (has_external_name_context and len(remaining) == 1) + ) + if self._is_credential(token.text) and not ambiguous_name_token: + dropped.append(_DroppedToken(token, DropReason.CREDENTIAL)) + remaining.pop() + continue + if token.text.strip(".,").isdigit(): + dropped.append(_DroppedToken(token, DropReason.AFFILIATION)) + remaining.pop() + continue + candidate = self._canonical_suffix(token.text, explicit=False) + has_complete_name = len(remaining) > _TWO_COMPONENTS or has_external_name_context + is_roman = candidate in _RAW_ROMAN_SUFFIXES + if candidate and not suffix and (not is_roman or has_complete_name): + suffix = candidate + suffix_token = replace(token, source_role="suffix") + remaining.pop() + continue + break + return remaining, suffix, suffix_token + + def _consume_boundary_segment( + self, + tokens: list[_Token], + ) -> tuple[str, _Token | None, list[_DroppedToken]] | None: + suffix = "" + suffix_token: _Token | None = None + dropped: list[_DroppedToken] = [] + if self._is_spaced_credential_group(tokens): + dropped.extend(_DroppedToken(replace(token, source_role="suffix"), DropReason.CREDENTIAL) for token in tokens) + return suffix, suffix_token, dropped + for token in tokens: + if self._is_credential(token.text, explicit_context=True): + dropped.append(_DroppedToken(replace(token, source_role="suffix"), DropReason.CREDENTIAL)) + continue + if token.text.strip(".,").isdigit(): + dropped.append(_DroppedToken(replace(token, source_role="suffix"), DropReason.AFFILIATION)) + continue + candidate = self._canonical_suffix(token.text, explicit=False) + if candidate and not suffix: + suffix = candidate + suffix_token = replace(token, source_role="suffix") + continue + return None + return suffix, suffix_token, dropped + + def _consume_explicit_suffix( # noqa: PLR0911 + self, + tokens: list[_Token], + ) -> tuple[str, _Token | None, list[_DroppedToken]]: + if not tokens: + return "", None, [] + if len(tokens) > 1: + boundary = self._consume_boundary_segment(tokens) + if boundary is not None: + return boundary + return " ".join(self._canonicalize_token(token.text, "suffix") for token in tokens), tokens[0], [] + + token = tokens[0] + if self._is_credential(token.text, explicit_context=True): + return "", None, [_DroppedToken(token, DropReason.CREDENTIAL)] + if token.text.strip(".,").isdigit(): + return "", None, [_DroppedToken(token, DropReason.AFFILIATION)] + canonical = self._canonical_suffix(token.text, explicit=True) + if canonical: + return canonical, token, [] + return self._canonicalize_token(token.text, "suffix"), token, [] + + def _strip_attached_affiliations( + self, + tokens: list[_Token], + dropped: list[_DroppedToken], + ) -> list[_Token]: + cleaned: list[_Token] = [] + for token in tokens: + match = _TRAILING_DIGITS_RE.search(token.text) + if match is None or not any(character.isalpha() for character in token.text[: match.start()]): + cleaned.append(token) + continue + cleaned.append(replace(token, text=token.text[: match.start()])) + digit_token = _Token(match.group(), token.source_role, token.position + match.start()) + dropped.append(_DroppedToken(digit_token, DropReason.AFFILIATION)) + return cleaned + + @staticmethod + def _strip_standalone_periods( + tokens: list[_Token], + dropped: list[_DroppedToken], + ) -> list[_Token]: + """Drop standalone full stops while retaining their source lineage.""" + remaining = [] + for token in tokens: + if token.text == ".": + dropped.append(_DroppedToken(token, DropReason.CONNECTOR)) + else: + remaining.append(token) + return remaining + + @staticmethod + def _strip_boundary_markers( + tokens: list[_Token], + dropped: list[_DroppedToken], + ) -> list[_Token]: + """Drop boundary-only punctuation/symbol tokens such as author footnote daggers.""" + remaining = list(tokens) + while remaining and not any(character.isalnum() for character in remaining[0].text): + dropped.append(_DroppedToken(remaining.pop(0), DropReason.CONNECTOR)) + while remaining and not any(character.isalnum() for character in remaining[-1].text): + dropped.append(_DroppedToken(remaining.pop(), DropReason.CONNECTOR)) + return remaining + + @staticmethod + def _strip_structured_middle_surname_artifact( + middle: list[_Token], + surname: list[_Token], + dropped: list[_DroppedToken], + ) -> list[_Token]: + """Remove an exact duplicated surname plus one lowercase source marker.""" + marker_width = 1 + artifact_width = len(surname) + marker_width + if not surname or len(middle) <= artifact_width: + return list(middle) + marker = middle[-1].text + if len(marker) != marker_width or not marker.isalpha() or not marker.islower(): + return list(middle) + duplicate = middle[-artifact_width:-marker_width] + if tuple(token.text for token in duplicate) != tuple(token.text for token in surname): + return list(middle) + dropped.extend(_DroppedToken(token, DropReason.DUPLICATE) for token in duplicate) + dropped.append(_DroppedToken(middle[-1], DropReason.CONNECTOR)) + return list(middle[:-artifact_width]) + + def _repair_leading_marker_initial( + self, + tokens: list[_Token], + dropped: list[_DroppedToken], + ) -> list[_Token]: + """Split ``tE. L. Surname`` only at its fully constrained leading boundary.""" + if ( + len(tokens) != _THREE_COMPONENTS + or not self._is_initial(tokens[1].text) + or not self._is_full_name_token(tokens[2].text) + ): + return list(tokens) + match = _LOWER_MARKER_INITIAL_RE.fullmatch(tokens[0].text) + if match is None: + return list(tokens) + marker, initial = match.groups() + first = tokens[0] + marker_token = replace(first, text=marker, source_text=None) + dropped.append(_DroppedToken(marker_token, DropReason.CONNECTOR)) + repaired = replace( + first, + text=f"{initial}.", + position=first.position + len(marker), + source_text=first.source_text or first.text, + ) + return [repaired, *tokens[1:]] + + def _join_separated_compound_initials(self, tokens: list[_Token]) -> list[_Token]: + """Join ``H. -J. Surname`` only when both visible pieces are initials.""" + if len(tokens) < _THREE_COMPONENTS or not self._is_initial(tokens[0].text): + return list(tokens) + trailing_initial = _LEADING_HYPHEN_INITIAL_RE.fullmatch(tokens[1].text) + if trailing_initial is None or not self._is_full_name_token(tokens[-1].text): + return list(tokens) + combined = replace(tokens[0], text=f"{tokens[0].text}-{trailing_initial.group(1)}.") + return [combined, *tokens[2:]] + + def _is_exact_ma_given_abbreviation(self, tokens: list[_Token]) -> bool: + """Recognize exact mixed-case ``Ma. Initial Surname`` given-name context.""" + return bool( + len(tokens) == _THREE_COMPONENTS + and tokens[0].text == "Ma." + and self._is_initial(tokens[1].text) + and self._is_full_name_token(tokens[2].text), + ) + + def _is_full_name_token(self, token: str) -> bool: + """Return whether a token is a full name rather than an initial.""" + cleaned = self._clean_name_token(token) + return bool( + len(self._compact_key(cleaned)) > 1 + and not self._is_initial(cleaned) + and any(character.isalpha() for character in cleaned) + and all(self._allowed_name_character(character) for character in cleaned), + ) + + def _strong_particle_width(self, tokens: list[_Token], start: int) -> int: + keys = tuple(self._particle_key(token.text) for token in tokens) + for span in _STRONG_FAMILY_PARTICLE_SPANS: + end = start + len(span) + if keys[start:end] == span and end < len(tokens) and all(token.text.islower() for token in tokens[start:end]): + return len(span) + return 0 + + def _find_strong_particle_span(self, tokens: list[_Token], *, start: int = 1) -> tuple[int, int] | None: + for index in range(start, len(tokens)): + if width := self._strong_particle_width(tokens, index): + return index, width + return None + + def _particle_surname_first_roles( + self, + tokens: list[_Token], + ) -> tuple[list[_Token], list[_Token], list[_Token]] | None: + """Parse ``Carvalho da Silva Roberto José`` behind an exact strong span.""" + if len(tokens) < _FOUR_COMPONENTS or not self._is_full_name_token(tokens[0].text): + return None + particle_width = self._strong_particle_width(tokens, 1) + if not particle_width: + return None + family_end = _TWO_COMPONENTS + particle_width + trailing = tokens[family_end:] + if len(trailing) != _TWO_COMPONENTS or not all(self._is_full_name_token(token.text) for token in trailing): + return None + return ( + [replace(trailing[0], source_role="given")], + [replace(trailing[1], source_role="middle")], + [replace(token, source_role="surname") for token in tokens[:family_end]], + ) + + def _is_packed_surname_first_initials(self, tokens: list[_Token]) -> bool: + return bool( + len(tokens) >= _THREE_COMPONENTS + and self._is_full_name_token(tokens[0].text) + and all(token.text.endswith(".") and self._is_initial(token.text) for token in tokens[1:]), + ) + + def _infer_regular_roles(self, tokens: list[_Token]) -> tuple[list[_Token], list[_Token], list[_Token]]: + if len(tokens) == 1: + return [replace(tokens[0], source_role="given")], [], [] + + if particle_roles := self._particle_surname_first_roles(tokens): + return particle_roles + + if self._is_packed_surname_first_initials(tokens): + return ( + [replace(tokens[1], source_role="given")], + [replace(token, source_role="middle") for token in tokens[2:]], + [replace(tokens[0], source_role="surname")], + ) + + surname_start = len(tokens) - 1 + strong_particle = self._find_strong_particle_span(tokens) + if strong_particle is not None and (strong_particle[0] > _TWO_COMPONENTS or self._is_initial(tokens[0].text)): + surname_start = strong_particle[0] - 1 + elif particle_positions := [ + index for index, token in enumerate(tokens[1:-1], start=1) if self._particle_key(token.text) in _FAMILY_PARTICLES + ]: + surname_start = particle_positions[0] + elif len(tokens) == _FOUR_COMPONENTS and self._is_initial(tokens[1].text) and not self._is_initial(tokens[2].text): + surname_start = 2 + given = [replace(tokens[0], source_role="given")] + middle = [replace(token, source_role="middle") for token in tokens[1:surname_start]] + surname = [replace(token, source_role="surname") for token in tokens[surname_start:]] + return given, middle, surname + + def _structured_particle_surname_first_roles( + self, + given: list[_Token], + middle: list[_Token], + surname: list[_Token], + ) -> tuple[list[_Token], list[_Token], list[_Token]] | None: + """Repair one mechanically shifted surname-first strong-particle layout.""" + if ( + len(given) != 1 + or len(surname) != 1 + or not self._is_full_name_token(given[0].text) + or not self._is_full_name_token(surname[0].text) + ): + return None + particle_width = self._strong_particle_width(middle, 0) if middle else 0 + family_end = particle_width + 1 + if not particle_width or family_end >= len(middle): + return None + trailing = middle[family_end:] + if len(trailing) != 1 or not self._is_full_name_token(trailing[0].text): + return None + return ( + [replace(trailing[0], source_role="given")], + [replace(surname[0], source_role="middle")], + [ + replace(given[0], source_role="surname"), + *(replace(token, source_role="surname") for token in middle[:family_end]), + ], + ) + + def _expand_structured_surname_floor( + self, + given: list[_Token], + middle: list[_Token], + surname: list[_Token], + ) -> tuple[list[_Token], list[_Token]]: + """Move a strong particle span and its preceding anchor into surname.""" + if not given or not middle or not surname: + return middle, surname + combined = [*middle, *surname] + strong_particle = self._find_strong_particle_span(combined) + if strong_particle is None or strong_particle[0] >= len(middle): + return middle, surname + surname_start = strong_particle[0] - 1 + if surname_start < 0 or (surname_start == 0 and not all(self._is_initial(token.text) for token in given)): + return middle, surname + return ( + [replace(token, source_role="middle") for token in middle[:surname_start]], + [replace(token, source_role="surname") for token in [*middle[surname_start:], *surname]], + ) + + def _repair_structured_roles( # noqa: C901, PLR0911 + self, + by_role: dict[str, list[_Token]], + ) -> tuple[list[_Token], list[_Token], list[_Token]]: + """Preserve surviving source roles, repairing only structurally empty boundaries.""" + given = [replace(token, source_role="given") for token in by_role["given"]] + middle = [replace(token, source_role="middle") for token in by_role["middle"]] + surname = [replace(token, source_role="surname") for token in by_role["surname"]] + if shifted := self._structured_particle_surname_first_roles(given, middle, surname): + return shifted + if surname: + if given: + middle, surname = self._expand_structured_surname_floor(given, middle, surname) + return given, middle, surname + if middle: + return [replace(middle[0], source_role="given")], middle[1:], surname + if len(surname) == 1: + fused = _FUSED_INITIAL_SURNAME_RE.fullmatch(surname[0].text) + if fused is not None and all(self._allowed_name_character(character) for character in fused.group(2)): + initial, family = fused.groups() + return ( + [replace(surname[0], text=f"{initial}.", source_role="given")], + [], + [ + replace( + surname[0], + text=family, + source_role="surname", + position=surname[0].position + len(initial) + 1, + ), + ], + ) + if len(surname) >= _TWO_COMPONENTS and self._particle_key(surname[0].text) not in _FAMILY_PARTICLES: + if surname[-1].text.isupper() and not any(self._is_initial(token.text) for token in surname[:-1]): + return ( + [replace(token, source_role="given") for token in surname[:-1]], + [], + [replace(surname[-1], source_role="surname")], + ) + return self._infer_regular_roles(surname) + return [], [], surname + + surviving_groups = [group for group in (given, middle) if group] + if len(surviving_groups) == _TWO_COMPONENTS: + return ( + [replace(token, source_role="given") for token in surviving_groups[0]], + [], + [replace(token, source_role="surname") for token in surviving_groups[1]], + ) + if not surviving_groups: + return [], [], [] + + tokens = surviving_groups[0] + if len(tokens) == 1: + return [replace(tokens[0], source_role="given")], [], [] + return self._infer_regular_roles(tokens) + + def _invalid_token_reason(self, tokens: list[_Token]) -> str | None: + for token in tokens: + cleaned = self._clean_name_token(token.text) + if not cleaned: + return f"empty name token at position {token.position}" + if self._is_parenthesized_name_token(cleaned): + continue + if not any(character.isalpha() for character in cleaned): + return f"name token has no letters: {token.text!r}" + if cleaned[0] in "-'" or cleaned[-1] == "-": + return f"name joiner is not between letters: {token.text!r}" + if any(not self._allowed_name_character(character) for character in cleaned): + return f"unsupported character in name token: {token.text!r}" + return None + + @staticmethod + def _allowed_name_character(character: str) -> bool: + return character.isalpha() or unicodedata.category(character).startswith("M") or character in "-'." + + def _canonical_tokens(self, tokens: list[_Token], role: str) -> tuple[str, ...]: + return tuple(self._canonicalize_token(token.text, role) for token in tokens) + + def _canonicalize_token(self, token: str, role: str) -> str: # noqa: C901 + cleaned = self._clean_name_token(token) + if role in {"middle", "surname"} and cleaned in _LOWERCASE_RELATIONAL_TOKENS: + return cleaned + if _INITIAL_RE.fullmatch(cleaned): + return cleaned[0].upper() + "." + if _MULTI_INITIAL_RE.fullmatch(cleaned) or self._is_uppercase_dotted_abbreviation(cleaned): + return "".join(character.upper() if character.isalpha() else character for character in cleaned) + if ( + role != "surname" + and cleaned.isalpha() + and cleaned.isupper() + and len(cleaned) == _TWO_COMPONENTS + and not self._is_ambiguous_credential(cleaned) + ): + return cleaned + + parts = re.split(r"([-'])", cleaned) + canonical: list[str] = [] + preserve_mixed_case = not (cleaned.islower() or cleaned.isupper()) + capitalize_after_apostrophe = parts[0].casefold() in {"a", "d", "o"} + for index, part in enumerate(parts): + if part in {"-", "'"}: + canonical.append(part) + continue + if not part: + continue + particle = self._particle_key(part) in _FAMILY_PARTICLES + preserve_lower_particle = particle and part.islower() + normalize_surname_particle = role == "surname" and particle and part.isupper() + if (preserve_lower_particle or normalize_surname_particle) and (len(parts) == 1 or index < len(parts) - 1): + canonical.append(part.casefold()) + elif capitalize_after_apostrophe and index >= _TWO_COMPONENTS and parts[index - 1] == "'": + canonical.append(self._name_case(part)) + elif preserve_mixed_case and not self._looks_like_mixed_ocr_case(part): + canonical.append(part) + else: + canonical.append(self._name_case(part)) + return "".join(canonical) + + @staticmethod + def _name_case(part: str) -> str: + if len(part) == 1: + return part.upper() + if part.startswith("Mc") and len(part) > _MC_PREFIX_LENGTH and part[2:].isupper(): + return part[:2] + part[2].upper() + part[3:].lower() + mixed_ocr_case = PersonNameNormalizationService._looks_like_mixed_ocr_case(part) + if not (part.islower() or part.isupper() or mixed_ocr_case): + return part + lowered_tail = part[1:].lower().replace("i\u0307", "i") + cased = part[0].upper() + lowered_tail + if cased.startswith("Mc") and len(cased) > _MC_PREFIX_LENGTH: + cased = cased[:2] + cased[2].upper() + cased[3:] + return cased + + @staticmethod + def _looks_like_mixed_ocr_case(part: str) -> bool: + """Return whether a token has an initial mixed-case prefix plus an all-caps OCR tail.""" + return bool( + (part.startswith("Mc") and len(part) > _MC_PREFIX_LENGTH and part[2:].isupper()) + or (len(part) > _TWO_COMPONENTS and part[0].isupper() and part[1].islower() and part[2:].isupper()), + ) + + @staticmethod + def _clean_name_token(token: str) -> str: + if PersonNameNormalizationService._is_parenthesized_name_token(token): + return token + cleaned = token.strip('"()[]{}<>:;,\u201c\u201d') + if cleaned.endswith(".") and not ( + _INITIAL_RE.fullmatch(cleaned) + or _MULTI_INITIAL_RE.fullmatch(cleaned) + or _ABBREVIATED_TOKEN_RE.fullmatch(cleaned) + or PersonNameNormalizationService._is_dotted_letter_token(cleaned) + or _COMPOUND_INITIAL_RE.fullmatch(cleaned) + or PersonNameNormalizationService._compact_key(cleaned) in _LONG_NAME_ABBREVIATION_KEYS + ): + cleaned = cleaned[:-1] + return cleaned + + @staticmethod + def _is_parenthesized_name_token(token: str) -> bool: + """Return whether a token is one balanced parenthesized personal-name alternative.""" + if len(token) < _MIN_PARENTHESIZED_TOKEN_LENGTH or not token.startswith("(") or not token.endswith(")"): + return False + inner = token[1:-1] + return any(character.isalpha() for character in inner) and all( + PersonNameNormalizationService._allowed_name_character(character) for character in inner + ) + + @staticmethod + def _is_dotted_letter_token(token: str) -> bool: + """Return whether a token contains only letters and at least one internal dot.""" + return "." in token[:-1] and all(character.isalpha() or character == "." for character in token) + + @staticmethod + def _is_uppercase_dotted_abbreviation(token: str) -> bool: + """Preserve short abbreviations whose source already supplies uppercase letters.""" + letters = token[:-1] if token.endswith(".") else "" + return bool( + letters and len(letters) <= _MAX_DOTTED_ABBREVIATION_LETTERS and letters.isalpha() and letters.isupper(), + ) + + @staticmethod + def _particle_key(token: str) -> str: + return token.strip(".").casefold() + + def _canonical_suffix(self, token: str, *, explicit: bool) -> str: + stripped = token.strip(".,") + key = stripped.casefold() + standard = _STANDARD_SUFFIXES.get(key) + if standard is not None: + return standard + roman = stripped.upper() + supported = _EXPLICIT_ROMAN_SUFFIXES if explicit else _RAW_ROMAN_SUFFIXES + if roman in supported and (explicit or stripped == roman or roman in _CASE_INSENSITIVE_ROMAN_SUFFIXES): + return roman + return "" + + def _is_credential(self, token: str, *, explicit_context: bool = False) -> bool: + key = self._compact_key(token) + if key not in _CREDENTIAL_KEYS: + return False + if key not in _AMBIGUOUS_CREDENTIAL_KEYS or explicit_context: + return True + letters = "".join(character for character in token if character.isalpha()) + return "." in token or letters.isupper() or token.strip(".,") == _MIXED_CASE_CREDENTIALS.get(key) + + def _is_ambiguous_credential(self, token: str) -> bool: + return self._compact_key(token) in _AMBIGUOUS_CREDENTIAL_KEYS + + def _is_spaced_credential_group(self, tokens: list[_Token]) -> bool: + if not _TWO_COMPONENTS <= len(tokens) <= _MAX_SPACED_CREDENTIAL_TOKENS: + return False + if not all("." in token.text and self._compact_key(token.text) for token in tokens): + return False + return "".join(self._compact_key(token.text) for token in tokens) in _CREDENTIAL_KEYS + + def _trailing_spaced_credential_width(self, tokens: list[_Token]) -> int: + max_width = min(_MAX_SPACED_CREDENTIAL_TOKENS, len(tokens) - _TWO_COMPONENTS) + for width in range(max_width, _TWO_COMPONENTS - 1, -1): + if self._is_spaced_credential_group(tokens[-width:]): + return width + return 0 + + def _components( + self, + given: list[_Token], + middle: list[_Token], + surname: list[_Token], + suffix: list[_Token], + *, + order: tuple[str, ...] | None = None, + ) -> NameComponents: + if order is None: + order = tuple( + ["given"] * len(given) + ["middle"] * len(middle) + ["surname"] * len(surname) + ["suffix"] * len(suffix), + ) + + def source_text(token: _Token) -> str: + return token.source_text or token.text + + return NameComponents( + given_name=" ".join(source_text(token) for token in given), + middle_name=" ".join(source_text(token) for token in middle), + surname=" ".join(source_text(token) for token in surname), + suffix=" ".join(source_text(token) for token in suffix), + given_tokens=tuple(source_text(token) for token in given), + middle_tokens=tuple(source_text(token) for token in middle), + surname_tokens=tuple(source_text(token) for token in surname), + suffix_tokens=tuple(source_text(token) for token in suffix), + order=order, + ) + + @staticmethod + def _structured_source_order(by_role: dict[str, list[_Token]]) -> tuple[str, ...]: + return tuple(role for role in ("given", "middle", "surname", "suffix") for _token in by_role[role]) + + def _looks_like_two_complete_names(self, left: list[_Token], right: list[_Token]) -> bool: + if len(left) < _TWO_COMPONENTS or len(right) < _TWO_COMPONENTS: + return False + if any(self._particle_key(token.text) in _FAMILY_PARTICLES for token in left[:-1]): + return False + return not any(self._is_initial(token.text) for token in [*left, *right]) + + def _non_person_reason(self, surface: str) -> str | None: + lowered = surface.casefold() + if "@" in surface or "://" in surface: + return "contact or URL input" + if any(separator in surface for separator in (";", "&", "|")): + return "multiple-name separator" + and_match = _WORD_AND_RE.search(surface) + if and_match is not None and and_match.start() > 0: + return "author-list connector" + words = {self._compact_key(token) for token in _TOKEN_RE.findall(lowered)} + if words & _ORGANIZATION_WORDS: + return "organization input" + return None + + @staticmethod + def _is_initial(token: str) -> bool: + cleaned = PersonNameNormalizationService._clean_name_token(token) + return bool( + (len(cleaned) == 1 and cleaned.isalpha()) or _INITIAL_RE.fullmatch(cleaned) or _MULTI_INITIAL_RE.fullmatch(cleaned), + ) + + def _public_dropped(self, dropped: list[_DroppedToken]) -> tuple[DroppedNameToken, ...]: + return tuple( + DroppedNameToken(item.token.text, item.token.source_role, item.reason) + for item in sorted(dropped, key=lambda item: item.token.position) + ) + + @staticmethod + def _infer_dropped_roles(dropped: list[_DroppedToken], assigned: list[_Token]) -> list[_DroppedToken]: + inferred: list[_DroppedToken] = [] + for item in dropped: + if item.token.source_role: + inferred.append(item) + continue + if item.reason in {DropReason.TITLE, DropReason.CONNECTOR}: + role = "given" + elif item.reason is DropReason.CREDENTIAL: + role = "suffix" + else: + prior = [token for token in assigned if token.position <= item.token.position] + role = prior[-1].source_role if prior else "suffix" + inferred.append(replace(item, token=replace(item.token, source_role=role))) + return inferred + + def _invalid( + self, + reason: str, + dropped: list[_DroppedToken] | None = None, + ) -> PersonNameNormalizationResult: + return PersonNameNormalizationResult( + outcome=PersonNameOutcome.INVALID, + reason=reason, + dropped_tokens=self._public_dropped(dropped or []), + ) + + @staticmethod + def _non_person(reason: str) -> PersonNameNormalizationResult: + return PersonNameNormalizationResult(outcome=PersonNameOutcome.NON_PERSON, reason=reason) + + +__all__ = [ + "DropReason", + "DroppedNameToken", + "PersonNameNormalizationResult", + "PersonNameNormalizationService", + "PersonNameOutcome", +] diff --git a/sinonym/text_processing/text_preprocessor.py b/sinonym/text_processing/text_preprocessor.py index 3a5b431..842688a 100644 --- a/sinonym/text_processing/text_preprocessor.py +++ b/sinonym/text_processing/text_preprocessor.py @@ -140,6 +140,8 @@ def compact_all_chinese_input(self, text: str) -> str: """Return compact CJK text when the input contains only CJK plus separators.""" if not text or not text.strip(): return "" + if text.isascii(): + return "" cleaned = self._config.sep_pattern.sub("", text.strip()) cleaned = self._config.whitespace_pattern.sub("", cleaned) @@ -202,6 +204,8 @@ def contains_non_chinese_scripts(self, text: str) -> bool: """ if not text: return False + if text.isascii(): + return False # Vietnamese-specific characters (conservative list to avoid Pinyin overlap) vietnamese_specific = "ĐđăâêôơưĂÂÊÔƠƯ" diff --git a/sinonym/timo/config.yaml b/sinonym/timo/config.yaml index da69c43..c2014e8 100644 --- a/sinonym/timo/config.yaml +++ b/sinonym/timo/config.yaml @@ -20,3 +20,23 @@ model_variants: cuda: False integration_test: sinonym.timo.integration_test.TestRoutingIntegration docker_run_commands: [] + sinonym_v2: + instance: sinonym.timo.interface.Instance + prediction: sinonym.timo.interface.PredictionV2 + predictor: sinonym.timo.interface.PredictorV2 + predictor_config: sinonym.timo.interface.PredictorConfig + artifacts_s3_path: null + python_version: "3.10" + cuda: False + integration_test: sinonym.timo.integration_test.TestIntegrationV2 + docker_run_commands: [] + sinonym_routing_v2: + instance: sinonym.timo.interface.RoutingInstance + prediction: sinonym.timo.interface.RoutedPaperPredictionV2 + predictor: sinonym.timo.interface.RoutingPredictorV2 + predictor_config: sinonym.timo.interface.PredictorConfig + artifacts_s3_path: null + python_version: "3.10" + cuda: False + integration_test: sinonym.timo.integration_test.TestRoutingIntegrationV2 + docker_run_commands: [] diff --git a/sinonym/timo/integration_test.py b/sinonym/timo/integration_test.py index 22991bb..44c7e2b 100644 --- a/sinonym/timo/integration_test.py +++ b/sinonym/timo/integration_test.py @@ -3,12 +3,16 @@ from sinonym.timo.interface import ( Instance, Prediction, + PredictionV2, Predictor, PredictorConfig, + PredictorV2, RoutedPaperPrediction, + RoutedPaperPredictionV2, RoutedPrediction, RoutingInstance, RoutingPredictor, + RoutingPredictorV2, ) @@ -42,6 +46,8 @@ def test_batch_superset_output(self): self.assertIsNotNone(r.confidence) self.assertIsNotNone(r.format_pattern) # shared batch pattern replicated onto each row + assert results[0].format_pattern is not None + assert results[1].format_pattern is not None self.assertEqual( results[0].format_pattern.dominant_format, results[1].format_pattern.dominant_format, @@ -122,3 +128,38 @@ def test_empty_paper_still_emits_one_prediction(self): def test_predict_batch_empty(self): self.assertEqual(self.predictor.predict_batch([]), []) + + +class TestIntegrationV2(TestIntegration): + """Integration contract for the flat TIMO v2 variant.""" + + @classmethod + def setUpClass(cls): + cls.predictor = PredictorV2(config=PredictorConfig(), artifacts_dir=".") + + def test_non_chinese_canonical_name(self): + (result,) = self.predictor.predict_batch([Instance(name="Dr. Steve Marsh PhD")]) + + self.assertIsInstance(result, PredictionV2) + self.assertFalse(result.success) + self.assertIsNotNone(result.canonical_name) + assert result.canonical_name is not None + self.assertEqual(result.canonical_name.text, "Steve Marsh") + + +class TestRoutingIntegrationV2(TestRoutingIntegration): + """Integration contract for the routed TIMO v2 variant.""" + + @classmethod + def setUpClass(cls): + cls.predictor = RoutingPredictorV2(config=PredictorConfig(), artifacts_dir=".") + + def test_non_chinese_canonical_name(self): + (paper,) = self.predictor.predict_batch([RoutingInstance(pp_names=["Steve Blando IV"])]) + + self.assertIsInstance(paper, RoutedPaperPredictionV2) + canonical_name = paper.authors[0].canonical_name + self.assertIsNotNone(canonical_name) + assert canonical_name is not None + self.assertEqual(canonical_name.text, "Steve Blando IV") + self.assertEqual(canonical_name.normalized.suffix, "IV") diff --git a/sinonym/timo/interface.py b/sinonym/timo/interface.py index cee1feb..be62be3 100644 --- a/sinonym/timo/interface.py +++ b/sinonym/timo/interface.py @@ -1,4 +1,5 @@ from enum import Enum +from typing import cast from pydantic import BaseModel, BaseSettings, Field, root_validator @@ -122,6 +123,38 @@ class Prediction(TimoModel): format_pattern: FormatPattern | None = Field(default=None, description="shared batch order pattern (same on every row)") +class CanonicalNameComponents(TimoModel): + """Semantic name components plus their token and display-order lineage.""" + + given_name: str = "" + middle_name: str = "" + surname: str = "" + suffix: str = "" + given_tokens: list[str] = Field(default_factory=list) + middle_tokens: list[str] = Field(default_factory=list) + surname_tokens: list[str] = Field(default_factory=list) + suffix_tokens: list[str] = Field(default_factory=list) + order: list[str] = Field(default_factory=list) + + +class CanonicalNameValue(TimoModel): + """TIMO representation of an all-person canonical name.""" + + source_text: str + text: str + source: CanonicalNameComponents + normalized: CanonicalNameComponents + + +class PredictionV2(Prediction): + """Versioned prediction that adds canonical data without changing v1.""" + + canonical_name: CanonicalNameValue | None = Field( + default=None, + description="canonical representation for person names, including non-Chinese names", + ) + + class Candidate(TimoModel): surname_tokens: list[str] given_tokens: list[str] @@ -182,6 +215,18 @@ class BatchSummary(TimoModel): confidences: list[float] = Field(description="per-name confidence from individual_analyses, aligned with results") +class BatchPredictionV2(BatchPrediction): + """Full batch result with v2 per-name predictions.""" + + results: list[PredictionV2] + + +class BatchSummaryV2(BatchSummary): + """Trimmed batch result with v2 per-name predictions.""" + + results: list[PredictionV2] + + class RoutingDecisionValue(str, Enum): """Serialized pp-vys-abstain router decision.""" @@ -270,6 +315,27 @@ class RoutedPaperPrediction(TimoModel): ) +class RoutedPredictionV2(RoutedPrediction): + """Versioned routed result with canonical data on the answer and candidates.""" + + pp: PredictionV2 + vys: PredictionV2 | None = None + canonical_name: CanonicalNameValue | None = None + + +class PPRoutedPredictionV2(PPRoutedPrediction): + """Versioned PP-only routed result with canonical data.""" + + pp: PredictionV2 + canonical_name: CanonicalNameValue | None = None + + +class RoutedPaperPredictionV2(RoutedPaperPrediction): + """One paper's routed v2 results.""" + + authors: list[RoutedPredictionV2] = Field(default_factory=list) + + class PredictorConfig(BaseSettings): parallel: ParallelMode = "auto" mp_max_workers: int | None = None @@ -619,7 +685,7 @@ def _route_pp_vys_batches( success=bool(chosen is not None and chosen.success), **self._routed_name_fields(parsed), router_prediction=pred, - router_reason=row.get("router_reason", ""), + router_reason=cast("str", row.get("router_reason", "")), input_order_candidate=ioc, pp=self._to_prediction(pp_res, format_pattern=pp_fp.copy(deep=True)), vys=self._to_prediction(vys_res, format_pattern=vys_fp.copy(deep=True)), @@ -650,7 +716,7 @@ def _route_pp_batch(self, pp_batch: BatchParseResult) -> list[PPRoutedPrediction success=bool(parsed is not None), **self._routed_name_fields(parsed), router_prediction=pred, - router_reason=row.get("router_reason", ""), + router_reason=cast("str", row.get("router_reason", "")), pp=self._to_prediction(res, format_pattern=pp_fp.copy(deep=True)), ), ) @@ -704,7 +770,10 @@ class RoutingPredictor(Predictor): papers. Genuine routing errors also propagate. """ - def predict_batch(self, instances: list[RoutingInstance]) -> list[RoutedPaperPrediction]: # type: ignore[override] + def predict_batch( # ty: ignore[invalid-method-override] + self, + instances: list[RoutingInstance], + ) -> list[RoutedPaperPrediction]: predictions: list[RoutedPaperPrediction | None] = [None] * len(instances) batch_inputs: list[list[str]] = [] plans: list[tuple[int, str, int, int | None, int]] = [] @@ -754,3 +823,367 @@ def predict_batch(self, instances: list[RoutingInstance]) -> list[RoutedPaperPre message = "routing prediction plan did not fill every instance slot" raise RuntimeError(message) return [prediction for prediction in predictions if prediction is not None] + + +class PredictorV2(Predictor): + """TIMO v2 predictor surfacing all-person canonical name metadata. + + Chinese recognition fields and every routing decision continue to come from + the same detector and routing functions as v1. Only response conversion is + versioned, so v1 payloads and schemas remain unchanged. + """ + + def predict_batch(self, instances: list[Instance]) -> list[PredictionV2]: # ty: ignore[invalid-method-override] + """Analyze one flat TIMO batch and return v2 rows.""" + return cast("list[PredictionV2]", super().predict_batch(instances)) + + def analyze_name_batch( + self, + names: list[str], + format_threshold: float | None = None, + minimum_batch_size: int | None = None, + ) -> BatchPredictionV2: + """Return full batch analysis with v2 rows.""" + return cast( + "BatchPredictionV2", + super().analyze_name_batch(names, format_threshold, minimum_batch_size), + ) + + def process_name_batch( + self, + names: list[str], + format_threshold: float | None = None, + minimum_batch_size: int | None = None, + ) -> list[PredictionV2]: # ty: ignore[invalid-method-override] + """Process one batch and return v2 rows.""" + return cast( + "list[PredictionV2]", + super().process_name_batch(names, format_threshold, minimum_batch_size), + ) + + def process_name_batch_multiprocess( + self, + names: list[str], + max_workers: int | None = None, + chunk_size: int | None = None, + ) -> list[PredictionV2]: # ty: ignore[invalid-method-override] + """Process one batch through the multiprocess helper and return v2 rows.""" + return cast( + "list[PredictionV2]", + super().process_name_batch_multiprocess(names, max_workers, chunk_size), + ) + + def process_name_batches( # noqa: PLR0913 + self, + batches: list[list[str]], + *, + parallel: ParallelMode | None = None, + max_workers: int | None = None, + chunk_size: int | None = None, + min_parallel_batches: int | None = None, + format_threshold: float | None = None, + minimum_batch_size: int | None = None, + ) -> list[list[PredictionV2]]: # ty: ignore[invalid-method-override] + """Process multiple batches and return v2 rows.""" + return cast( + "list[list[PredictionV2]]", + super().process_name_batches( + batches, + parallel=parallel, + max_workers=max_workers, + chunk_size=chunk_size, + min_parallel_batches=min_parallel_batches, + format_threshold=format_threshold, + minimum_batch_size=minimum_batch_size, + ), + ) + + def score_name_batch( + self, + names: list[str], + format_threshold: float | None = None, + minimum_batch_size: int | None = None, + ) -> BatchSummaryV2: + """Return trimmed batch analysis with v2 rows.""" + return cast( + "BatchSummaryV2", + super().score_name_batch(names, format_threshold, minimum_batch_size), + ) + + def route_pp_vys( + self, + pp_names: list[str], + vys_pool_names: list[str], + ) -> list[RoutedPredictionV2]: # ty: ignore[invalid-method-override] + """Route one paper with venue context and return v2 rows.""" + return cast("list[RoutedPredictionV2]", super().route_pp_vys(pp_names, vys_pool_names)) + + def route_pp(self, names: list[str]) -> list[PPRoutedPredictionV2]: # ty: ignore[invalid-method-override] + """Route one PP-only batch and return v2 rows.""" + return cast("list[PPRoutedPredictionV2]", super().route_pp(names)) + + @staticmethod + def _to_canonical_components(components) -> CanonicalNameComponents: + return CanonicalNameComponents( + given_name=components.given_name, + middle_name=components.middle_name, + surname=components.surname, + suffix=components.suffix, + given_tokens=list(components.given_tokens), + middle_tokens=list(components.middle_tokens), + surname_tokens=list(components.surname_tokens), + suffix_tokens=list(components.suffix_tokens), + order=list(components.order), + ) + + def _to_canonical_name(self, canonical_name) -> CanonicalNameValue | None: + if canonical_name is None: + return None + return CanonicalNameValue( + source_text=canonical_name.source_text, + text=canonical_name.text, + source=self._to_canonical_components(canonical_name.source), + normalized=self._to_canonical_components(canonical_name.normalized), + ) + + @staticmethod + def _canonical_components_from_parsed(parsed, suffix: str = "") -> CanonicalNameComponents: + counts = { + "given": len(parsed.given_tokens), + "middle": len(parsed.middle_tokens), + "surname": len(parsed.surname_tokens), + } + occurrences = {role: parsed.order.count(role) for role in counts} + expanded_order: list[str] = [] + for role in parsed.order: + count = counts.get(role, 0) + if occurrences.get(role) == 1: + expanded_order.extend([role] * count) + elif count: + expanded_order.append(role) + if suffix: + expanded_order.append("suffix") + return CanonicalNameComponents( + given_name=parsed.given_name, + middle_name=parsed.middle_name, + surname=parsed.surname, + suffix=suffix, + given_tokens=list(parsed.given_tokens), + middle_tokens=list(parsed.middle_tokens), + surname_tokens=list(parsed.surname_tokens), + suffix_tokens=[suffix] if suffix else [], + order=expanded_order, + ) + + def _to_routed_canonical_name(self, parse_result, parsed) -> CanonicalNameValue | None: + """Convert canonical data, matching a PP-abstain input-order parse when needed.""" + canonical = self._to_canonical_name(parse_result.canonical_name) + if canonical is None or parsed is None or parsed is parse_result.parsed: + return canonical + + suffix = canonical.normalized.suffix + normalized = self._canonical_components_from_parsed(parsed, suffix=suffix) + text = " ".join( + component + for component in (normalized.given_name, normalized.middle_name, normalized.surname, normalized.suffix) + if component + ) + return CanonicalNameValue( + source_text=canonical.source_text, + text=text, + source=canonical.source, + normalized=normalized, + ) + + def _to_prediction( + self, + parse_result, + *, + confidence: float | None = None, + format_pattern: FormatPattern | None = None, + ) -> PredictionV2: + return PredictionV2( + success=parse_result.success, + error_message=parse_result.error_message, + given_name=parse_result.parsed.given_name if parse_result.parsed else None, + surname=parse_result.parsed.surname if parse_result.parsed else None, + middle_name=(parse_result.parsed.middle_name if parse_result.parsed and parse_result.parsed.middle_name else None), + confidence=confidence, + format_pattern=format_pattern, + canonical_name=self._to_canonical_name(parse_result.canonical_name), + ) + + def _to_batch_prediction(self, batch_result) -> BatchPredictionV2: + return BatchPredictionV2( + names=list(batch_result.names), + results=[self._to_prediction(result) for result in batch_result.results], + format_pattern=self._to_format_pattern(batch_result.format_pattern), + individual_analyses=[self._to_individual_analysis(analysis) for analysis in batch_result.individual_analyses], + improvements=list(batch_result.improvements), + name_order_evidence=[self._to_name_order_evidence(evidence) for evidence in batch_result.name_order_evidence], + ) + + def _to_batch_summary(self, batch_result) -> BatchSummaryV2: + return BatchSummaryV2( + names=list(batch_result.names), + results=[self._to_prediction(result) for result in batch_result.results], + format_pattern=self._to_format_pattern(batch_result.format_pattern), + confidences=[analysis.confidence for analysis in batch_result.individual_analyses], + ) + + def _route_pp_vys_batches( # ty: ignore[invalid-method-override] + self, + pp_batch: BatchParseResult, + pool: BatchParseResult, + n: int, + ) -> list[RoutedPredictionV2]: + """Route analyzed PP/VYS batches and convert their unchanged decisions to v2.""" + vys_batch = BatchParseResult( + names=list(pool.names[:n]), + results=list(pool.results[:n]), + format_pattern=pool.format_pattern, + individual_analyses=list(pool.individual_analyses[:n]), + improvements=[index for index in pool.improvements if index < n], + name_order_evidence=list(pool.name_order_evidence[:n]), + ) + rows = route_pp_vys_abstain_batches(pp_batch, vys_batch) + + pp_format = self._to_format_pattern(pp_batch.format_pattern) + vys_format = self._to_format_pattern(vys_batch.format_pattern) + output: list[RoutedPredictionV2] = [] + for index, row in enumerate(rows): + decision = row["router_prediction"] + input_order_candidate = row.get("input_order_candidate", "unknown") + pp_result = pp_batch.results[index] + vys_result = vys_batch.results[index] + if decision == "pp": + chosen = pp_result + elif decision == "vys": + chosen = vys_result + elif decision == "abstain": + chosen = {"pp": pp_result, "vys": vys_result}.get(input_order_candidate) + if chosen is None: + message = f"abstain with unexpected input_order_candidate={input_order_candidate!r} (expected 'pp'/'vys')" + raise ValueError(message) + elif decision == "not_person": + chosen = None + else: + message = f"pp-vys router returned unexpected router_prediction={decision!r}" + raise ValueError(message) + + parsed = chosen.parsed if (chosen is not None and chosen.success) else None + canonical_result = chosen or pp_result + output.append( + RoutedPredictionV2( + success=bool(chosen is not None and chosen.success), + **self._routed_name_fields(parsed), + router_prediction=decision, + router_reason=cast("str", row.get("router_reason", "")), + input_order_candidate=input_order_candidate, + pp=self._to_prediction(pp_result, format_pattern=pp_format.copy(deep=True)), + vys=self._to_prediction(vys_result, format_pattern=vys_format.copy(deep=True)), + canonical_name=self._to_routed_canonical_name(canonical_result, parsed), + ), + ) + return output + + def _route_pp_batch( # ty: ignore[invalid-method-override] + self, + pp_batch: BatchParseResult, + ) -> list[PPRoutedPredictionV2]: + """Route an analyzed PP-only batch and convert its unchanged decisions to v2.""" + rows = route_pp_abstain_rows(build_pp_abstain_rows(pp_batch, self._detector)) + pp_format = self._to_format_pattern(pp_batch.format_pattern) + + output: list[PPRoutedPredictionV2] = [] + for index, row in enumerate(rows): + decision = row["router_prediction"] + result = pp_batch.results[index] + if decision == "pp": + parsed = result.parsed if result.success else None + elif decision == "abstain": + parsed = pp_abstain_parsed(result, row) + elif decision == "not_person": + parsed = None + else: + message = f"pp-abstain router returned unexpected router_prediction={decision!r}" + raise ValueError(message) + output.append( + PPRoutedPredictionV2( + success=bool(parsed is not None), + **self._routed_name_fields(parsed), + router_prediction=decision, + router_reason=cast("str", row.get("router_reason", "")), + pp=self._to_prediction(result, format_pattern=pp_format.copy(deep=True)), + canonical_name=self._to_routed_canonical_name(result, parsed), + ), + ) + return output + + def route( # ty: ignore[invalid-method-override] + self, + pp_names: list[str], + vys_pool_names: list[str] | None = None, + ) -> list[RoutedPredictionV2]: + """Run the unchanged unified router and retain v2 canonical fields.""" + if vys_pool_names: + return self.route_pp_vys(pp_names, vys_pool_names) + return [RoutedPredictionV2(**result.dict(), input_order_candidate=None, vys=None) for result in self.route_pp(pp_names)] + + +class RoutingPredictorV2(PredictorV2): + """TIMO-served v2 routing variant with one prediction per paper.""" + + def predict_batch( # ty: ignore[invalid-method-override] + self, + instances: list[RoutingInstance], + ) -> list[RoutedPaperPredictionV2]: + predictions: list[RoutedPaperPredictionV2 | None] = [None] * len(instances) + batch_inputs: list[list[str]] = [] + plans: list[tuple[int, str, int, int | None, int]] = [] + + for instance_index, instance in enumerate(instances): + if not instance.pp_names: + predictions[instance_index] = RoutedPaperPredictionV2(authors=[]) + continue + + if instance.vys_pool_names: + self._validate_vys_pool_names(instance.pp_names, instance.vys_pool_names) + pp_batch_index = len(batch_inputs) + batch_inputs.append(instance.pp_names) + vys_batch_index = len(batch_inputs) + batch_inputs.append(instance.vys_pool_names) + plans.append((instance_index, "pp_vys", pp_batch_index, vys_batch_index, len(instance.pp_names))) + else: + pp_batch_index = len(batch_inputs) + batch_inputs.append(instance.pp_names) + plans.append((instance_index, "pp_only", pp_batch_index, None, 0)) + + batch_results = self._detector.analyze_name_batches( + batch_inputs, + parallel=self._config.parallel, + min_parallel_batches=self._config.mp_min_parallel_batches, + max_workers=self._config.mp_max_workers, + chunk_size=self._config.mp_chunk_size, + mp_start_method=self._config.mp_start_method, + ) + + for instance_index, mode, pp_batch_index, vys_batch_index, pp_count in plans: + if mode == "pp_vys": + if vys_batch_index is None: + message = "pp_vys routing plan missing VYS batch index" + raise RuntimeError(message) + authors = self._route_pp_vys_batches( + batch_results[pp_batch_index], + batch_results[vys_batch_index], + pp_count, + ) + else: + pp_only = self._route_pp_batch(batch_results[pp_batch_index]) + authors = [RoutedPredictionV2(**result.dict(), input_order_candidate=None, vys=None) for result in pp_only] + predictions[instance_index] = RoutedPaperPredictionV2(authors=authors) + + if any(prediction is None for prediction in predictions): + message = "routing prediction plan did not fill every instance slot" + raise RuntimeError(message) + return [prediction for prediction in predictions if prediction is not None] diff --git a/tests/test_bug_report_fixes.py b/tests/test_bug_report_fixes.py index 0ec4981..70a0004 100644 --- a/tests/test_bug_report_fixes.py +++ b/tests/test_bug_report_fixes.py @@ -14,7 +14,7 @@ route_pp_vys_abstain_rows, ) from sinonym.services.batch_analysis import LATIN_ONLY_REPRESENTATION, BatchCandidateEntry -from sinonym.timo.interface import Instance, Predictor, PredictorConfig +from sinonym.timo.interface import Instance, Predictor, PredictorConfig, PredictorV2 def _pp_abstain_row(**overrides): @@ -266,13 +266,46 @@ def test_route_pp_preserves_latin_compound_surname_first_parse(): def test_internal_compound_surname_span_reports_real_width(detector): batch = detector.analyze_name_batch(["Wei Zhu Ge Ming"]) + result = batch.results[0] evidence = batch.name_order_evidence[0] - assert batch.results[0].success + assert result.success + assert result.result == "Wei-Ming Zhu Ge" + assert result.parsed is not None + assert result.parsed.surname_tokens == ["Zhu", "Ge"] + assert result.parsed.given_tokens == ["Wei", "Ming"] assert evidence.selected_surname_position == "internal" assert evidence.selected_surname_token_count == 2 +def test_internal_compound_surname_fix_propagates_through_pp_only_routing(): + predictor = Predictor(PredictorConfig(parallel="never"), "") + + routed = predictor.route_pp(["Wei Zhu Ge Ming"])[0] + + assert routed.router_prediction.value == "abstain" + assert routed.router_reason == "weak_zero_batch" + assert (routed.given_name, routed.middle_name, routed.surname) == ("Wei-Ming", None, "Zhu Ge") + assert (routed.pp.given_name, routed.pp.middle_name, routed.pp.surname) == ("Wei-Ming", None, "Zhu Ge") + + +def test_internal_compound_surname_fix_propagates_to_v2_canonical_name(): + predictor = PredictorV2(PredictorConfig(parallel="never"), "") + + routed = predictor.route_pp(["Wei Zhu Ge Ming"])[0] + + assert routed.router_prediction.value == "abstain" + assert routed.router_reason == "weak_zero_batch" + assert (routed.given_name, routed.middle_name, routed.surname) == ("Wei-Ming", None, "Zhu Ge") + assert routed.canonical_name is not None + assert routed.canonical_name.text == "Wei-Ming Zhu Ge" + assert ( + routed.canonical_name.normalized.given_name, + routed.canonical_name.normalized.middle_name, + routed.canonical_name.normalized.surname, + ) == ("Wei-Ming", "", "Zhu Ge") + + @pytest.mark.parametrize( ("raw_name", "expected_result", "expected_surname", "expected_given"), [ diff --git a/tests/test_canonical_name_integration.py b/tests/test_canonical_name_integration.py new file mode 100644 index 0000000..28120f6 --- /dev/null +++ b/tests/test_canonical_name_integration.py @@ -0,0 +1,140 @@ +"""Public canonical-name behavior without changing legacy Chinese recognition.""" + +from __future__ import annotations + + +def test_non_chinese_result_surfaces_canonical_name_without_legacy_success(detector): + result = detector.normalize_name("Dr. Steve Marsh PhD") + + assert not result.success + assert result.result == "" + assert result.parsed is None + assert result.canonical_name is not None + assert result.canonical_name.text == "Steve Marsh" + assert result.canonical_name.normalized.given_name == "Steve" + assert result.canonical_name.normalized.middle_name == "" + assert result.canonical_name.normalized.surname == "Marsh" + assert result.canonical_name.normalized.suffix == "" + + +def test_canonical_name_normalizes_all_joiner_variants(detector): + result = detector.normalize_name("Ms. Ana\u2013Maria O\u2019Neill MS") + + assert not result.success + assert result.canonical_name is not None + assert result.canonical_name.text == "Ana-Maria O'Neill" + assert result.canonical_name.normalized.given_tokens == ("Ana-Maria",) + assert result.canonical_name.normalized.surname_tokens == ("O'Neill",) + + +def test_canonical_name_moves_true_suffix_to_suffix_component(detector): + result = detector.normalize_name("Steve Blando IV") + + assert not result.success + assert result.canonical_name is not None + assert result.canonical_name.text == "Steve Blando IV" + assert result.canonical_name.normalized.given_name == "Steve" + assert result.canonical_name.normalized.surname == "Blando" + assert result.canonical_name.normalized.suffix == "IV" + assert result.canonical_name.normalized.order == ("given", "surname", "suffix") + + +def test_two_token_suffixes_and_credentials_use_full_canonical_pipeline(detector): + cases = ( + ("John Jr", "John Jr.", "Jr."), + ("John Senior", "John Sr.", "Sr."), + ("John Phd", "John", ""), + ("Phd Smith", "Smith", ""), + ("PD Dr", None, None), + ) + + for raw_name, expected_text, expected_suffix in cases: + public = detector.normalize_name(raw_name).canonical_name + generic = detector.normalize_person_name(raw_name) + + assert public == generic + if expected_text is None: + assert public is None + continue + assert public is not None + assert public.text == expected_text + assert public.normalized.suffix == expected_suffix + + +def test_structured_component_normalization_repairs_roles_after_drops(detector): + canonical = detector.normalize_person_name_components( + first_name="dr steve", + middle_name="marsh", + last_name="phd", + ) + + assert canonical is not None + assert canonical.text == "Steve Marsh" + assert canonical.normalized.given_name == "Steve" + assert canonical.normalized.middle_name == "" + assert canonical.normalized.surname == "Marsh" + assert canonical.normalized.suffix == "" + + +def test_explicit_family_first_comma_is_rendered_in_canonical_order(detector): + canonical = detector.normalize_person_name("Smith, John Q. Jr.") + + assert canonical is not None + assert canonical.text == "John Q. Smith Jr." + assert canonical.source.order == ("surname", "given", "middle", "suffix") + assert canonical.normalized.order == ("given", "middle", "surname", "suffix") + + +def test_chinese_result_canonical_name_matches_selected_parse(detector): + result = detector.normalize_name("Wei Zhu Ge Ming") + + assert result.success + assert result.result == "Wei-Ming Zhu Ge" + assert result.parsed is not None + assert result.canonical_name is not None + assert result.canonical_name.text == result.result + assert result.canonical_name.normalized.given_name == result.parsed.given_name + assert result.canonical_name.normalized.surname == result.parsed.surname + assert result.canonical_name.normalized.given_tokens == tuple(result.parsed.given_tokens) + assert result.canonical_name.normalized.surname_tokens == tuple(result.parsed.surname_tokens) + + +def test_chinese_canonical_source_preserves_fused_token_lineage(detector): + result = detector.normalize_name("Wang Weiming") + + assert result.canonical_name is not None + assert result.canonical_name.source.given_tokens == ("Weiming",) + assert result.canonical_name.source.surname_tokens == ("Wang",) + assert result.canonical_name.source.order == ("surname", "given") + assert result.canonical_name.normalized.given_tokens == ("Wei", "Ming") + + +def test_batch_results_surface_canonical_name_after_final_selection(detector): + batch = detector.analyze_name_batch(["Li Wei", "John Smith"]) + + chinese, western = batch.results + assert chinese.parsed is not None + assert chinese.canonical_name is not None + assert chinese.canonical_name.text == chinese.result + assert chinese.canonical_name.normalized.given_name == chinese.parsed.given_name + assert chinese.canonical_name.normalized.surname == chinese.parsed.surname + assert western.canonical_name is not None + assert western.canonical_name.text == "John Smith" + assert not western.success + assert western.parsed is None + + +def test_invalid_and_obvious_non_person_inputs_have_no_canonical_name(detector): + assert detector.normalize_name("").canonical_name is None + assert detector.normalize_name("---").canonical_name is None + assert detector.normalize_name("Veecon Music & Entertainment").canonical_name is None + assert detector.normalize_name("北京大学").canonical_name is None + + +def test_korean_native_name_surfaces_semantic_family_first_canonical_name(detector): + result = detector.normalize_name("김민준") + + assert not result.success + assert result.canonical_name is not None + assert result.canonical_name.text == "민준 김" + assert result.canonical_name.source.order == ("surname", "given") diff --git a/tests/test_canonical_results.py b/tests/test_canonical_results.py new file mode 100644 index 0000000..2479e83 --- /dev/null +++ b/tests/test_canonical_results.py @@ -0,0 +1,86 @@ +"""Tests for canonical-name result metadata.""" + +from dataclasses import FrozenInstanceError + +import pytest + +from sinonym.coretypes import CanonicalName, NameComponents, ParsedName, ParseResult + + +def _canonical_name(text: str = "Steve Marsh Blando IV") -> CanonicalName: + source = NameComponents( + given_name="dr steve", + middle_name="marsh", + surname="blando", + suffix="IV", + given_tokens=("dr", "steve"), + middle_tokens=("marsh",), + surname_tokens=("blando",), + suffix_tokens=("IV",), + order=("given", "given", "middle", "surname", "suffix"), + ) + normalized = NameComponents( + given_name="Steve", + middle_name="Marsh", + surname="Blando", + suffix="IV", + given_tokens=("Steve",), + middle_tokens=("Marsh",), + surname_tokens=("Blando",), + suffix_tokens=("IV",), + order=("given", "middle", "surname", "suffix"), + ) + return CanonicalName(source_text="dr steve marsh blando IV", text=text, source=source, normalized=normalized) + + +def test_canonical_name_and_components_are_deeply_immutable() -> None: + canonical_name = _canonical_name() + text_attribute = "text" + surname_attribute = "surname" + append_method = "append" + + with pytest.raises(FrozenInstanceError): + setattr(canonical_name, text_attribute, "Changed") + with pytest.raises(FrozenInstanceError): + setattr(canonical_name.normalized, surname_attribute, "Changed") + with pytest.raises(AttributeError): + getattr(canonical_name.normalized.surname_tokens, append_method)("Changed") + + +def test_parse_result_positional_constructor_remains_compatible() -> None: + parsed = ParsedName("Zhang", "Wei", ["Zhang"], ["Wei"]) + + result = ParseResult(True, "Wei Zhang", None, None, parsed, parsed) + + assert result.parsed is parsed + assert result.parsed_original_order is parsed + assert result.canonical_name is None + + +def test_map_preserves_canonical_name_on_success_and_callback_failure() -> None: + canonical_name = _canonical_name() + result = ParseResult.success_with_name("Steve Marsh Blando IV", canonical_name=canonical_name) + + mapped = result.map(str.upper) + failed = result.map(lambda _value: 1 / 0) + + assert mapped.result == "STEVE MARSH BLANDO IV" + assert mapped.canonical_name is canonical_name + assert not failed.success + assert failed.canonical_name is canonical_name + + +def test_flat_map_inherits_canonical_name_without_replacing_callback_metadata() -> None: + canonical_name = _canonical_name() + replacement = _canonical_name("Stephen Blando") + result = ParseResult.success_with_name("Steve Marsh Blando IV", canonical_name=canonical_name) + + inherited = result.flat_map(lambda _value: ParseResult.success_with_name("Steve Blando")) + inherited_failure = result.flat_map(lambda _value: ParseResult.failure("not normalized")) + replaced = result.flat_map( + lambda _value: ParseResult.success_with_name("Stephen Blando", canonical_name=replacement), + ) + + assert inherited.canonical_name is canonical_name + assert inherited_failure.canonical_name is canonical_name + assert replaced.canonical_name is replacement diff --git a/tests/test_chinese_classification_evidence_gates.py b/tests/test_chinese_classification_evidence_gates.py new file mode 100644 index 0000000..397e89c --- /dev/null +++ b/tests/test_chinese_classification_evidence_gates.py @@ -0,0 +1,249 @@ +"""Regression tests for narrow cross-cultural Chinese classification evidence.""" + +import pytest + +from sinonym import ChineseNameDetector +from sinonym.services.person_name_normalization import DropReason, PersonNameNormalizationService, PersonNameOutcome + + +@pytest.fixture(scope="module") +def detector() -> ChineseNameDetector: + """Return one initialized detector for the evidence-gate cases.""" + return ChineseNameDetector() + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Jun-Fun Horng", "Jun-Fun Horng"), + ("Shugi Hsien", "Shu-Gi Hsien"), + ("Zhou Df", "Df Zhou"), + ], +) +def test_reviewed_surname_alias_and_compact_initial_evidence_accepts_chinese( + detector: ChineseNameDetector, + raw_name: str, + expected: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert result.success + assert result.result == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Yi-xin Han", "Yi-Xin Han"), + ("Peng-qian Han", "Peng-Qian Han"), + ], +) +def test_broad_korean_shape_is_not_used( + detector: ChineseNameDetector, + raw_name: str, + expected: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert result.success + assert result.result == expected + + +@pytest.mark.parametrize( + "raw_name", + [ + "In-sun Yu", + "Jin Uk Ha", + "Ok Jeung", + ], +) +def test_directional_korean_surname_given_evidence_precedes_chinese_shortcuts( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert not result.success + assert result.error_message == "Korean structural patterns detected" + + +@pytest.mark.parametrize( + "raw_name", + [ + "Aimin Yang", + "Bing Han", + "Boyi Kang", + "Chen-Yu Lee", + "Chia-Yuan Chang", + "Ching-Wen Yang", + "Chien-Yao Wang", + ], +) +def test_directional_korean_gate_preserves_reviewed_chinese_controls( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + assert detector.normalize_name(raw_name).success + + +@pytest.mark.parametrize("raw_name", ["Zhang Thi", "Wang Thi", "Thi Zhang", "Thi Wang"]) +def test_vietnamese_thi_veto_precedes_chinese_surname_evidence( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert not result.success + assert result.error_message == "appears to be Vietnamese name" + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Jungting Yu", "Jung-Ting Yu"), + ("Tsung-Jr Chen", "Tsung-Jr Chen"), + ], +) +def test_contextual_taiwan_romanization_is_admitted_and_preserved( + detector: ChineseNameDetector, + raw_name: str, + expected: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert result.success + assert result.result == expected + assert result.parsed_original_order is not None + assert result.parsed_original_order.order == ["given", "surname"] + + +@pytest.mark.parametrize( + "raw_name", + [ + "Jungting Kim", + "Tsung-Jr Kim", + "Jungting Nguyen", + "Tsung-Jr Tran", + "Robert Chen Jr", + "Jung Kim", + "Wade Wang", + "Sina Huang", + ], +) +def test_contextual_taiwan_gate_rejects_collision_controls( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + assert not detector.normalize_name(raw_name).success + + +@pytest.mark.parametrize("raw_name", ["Jung Yu", "J.-W. Koo"]) +def test_bare_jung_and_initials_are_not_contextual_taiwan_evidence( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + normalized = detector._normalizer.apply(raw_name) # noqa: SLF001 + + assert detector._ethnicity_service.contextual_taiwan_given_parts(normalized.roman_tokens) is None # noqa: SLF001 + + +def test_ordinary_short_non_chinese_name_is_not_treated_as_initial_bundle( + detector: ChineseNameDetector, +) -> None: + assert not detector.normalize_name("Sun Kim").success + + +def test_leading_et_al_contamination_is_removed_before_chinese_classification( + detector: ChineseNameDetector, +) -> None: + result = detector.normalize_name("Et al. Biquan Mo") + + assert result.success + assert result.result == "Bi-Quan Mo" + assert result.parsed is not None + assert (result.parsed.given_name, result.parsed.surname) == ("Bi-Quan", "Mo") + + +def test_leading_et_al_contamination_has_dropped_token_audit() -> None: + normalized = PersonNameNormalizationService().normalize_text("Et al. Biquan Mo") + structured = PersonNameNormalizationService().normalize_components( + first_name="Et", + middle_name="al.", + last_name="Biquan Mo", + ) + + assert normalized.outcome is PersonNameOutcome.PERSON + assert normalized.canonical_name is not None + assert normalized.canonical_name.text == "Biquan Mo" + assert [(item.text, item.source_role, item.reason) for item in normalized.dropped_tokens] == [ + ("Et", "given", DropReason.CONNECTOR), + ("al.", "given", DropReason.CONNECTOR), + ] + assert structured.outcome is PersonNameOutcome.PERSON + assert structured.canonical_name is not None + assert structured.canonical_name.text == "Biquan Mo" + assert [(item.text, item.source_role, item.reason) for item in structured.dropped_tokens] == [ + ("Et", "given", DropReason.CONNECTOR), + ("al.", "middle", DropReason.CONNECTOR), + ] + + +@pytest.mark.parametrize( + "raw_name", + [ + "Etienne Al", + "Et Al Biquan Mo", + "Biquan et al. Mo", + "Biquan Mo et al.", + "Et Albright Mo", + ], +) +def test_et_al_cleanup_requires_exact_leading_citation_prefix( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + normalized = PersonNameNormalizationService().normalize_text(raw_name) + + assert not normalized.dropped_tokens + assert not detector.normalize_name(raw_name).success + + +@pytest.mark.parametrize( + ("raw_name", "expected", "expected_surname"), + [ + ("Peter Ch'en", "Peter Ch'en", "Ch'en"), + ("Ch'en Peter", "Peter Ch'en", "Ch'en"), + ("Peter Ts'ai", "Peter Ts'ai", "Ts'ai"), + ], +) +def test_wade_giles_apostrophe_surname_is_decisive_chinese_evidence( + detector: ChineseNameDetector, + raw_name: str, + expected: str, + expected_surname: str, +) -> None: + result = detector.normalize_name(raw_name) + + assert result.success + assert result.result == expected + assert result.parsed is not None + assert result.parsed.given_name == "Peter" + assert result.parsed.surname == expected_surname + + +@pytest.mark.parametrize( + "raw_name", + [ + "Peter O'Brien", + "D'Angelo Marie", + "Peter Ch'energy", + "Ch'energy Peter", + "Peter Ch'en-Smith", + "O'Ch'en Peter", + ], +) +def test_apostrophe_surname_evidence_requires_exact_wade_giles_token( + detector: ChineseNameDetector, + raw_name: str, +) -> None: + assert not detector.normalize_name(raw_name).success diff --git a/tests/test_east_asian_name_order.py b/tests/test_east_asian_name_order.py new file mode 100644 index 0000000..418aba1 --- /dev/null +++ b/tests/test_east_asian_name_order.py @@ -0,0 +1,123 @@ +"""Regression tests for conservative East Asian family-first routing.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sinonym.services.east_asian_name_order import EastAsianNameOrderService + +if TYPE_CHECKING: + from sinonym import ChineseNameDetector + + +def test_japanese_romanized_routes_only_unambiguous_dictionary_direction( + detector: ChineseNameDetector, +) -> None: + routed = detector.normalize_person_name("Shirakawa Hideki") + ambiguous = detector.normalize_person_name("Motoki Kouzaki") + + assert routed is not None + assert routed.text == "Hideki Shirakawa" + assert routed.normalized.given_name == "Hideki" + assert routed.normalized.surname == "Shirakawa" + assert routed.source.order == ("surname", "given") + assert ambiguous is not None + assert ambiguous.text == "Motoki Kouzaki" + assert ambiguous.source.order == ("given", "surname") + + +def test_structured_japanese_romanized_matches_raw_directional_route( + detector: ChineseNameDetector, +) -> None: + raw = detector.normalize_person_name("Kumagai Jin") + structured = detector.normalize_person_name_components(first_name="Kumagai", last_name="Jin") + + assert raw is not None + assert structured is not None + assert structured.text == raw.text == "Jin Kumagai" + assert structured.normalized == raw.normalized + assert structured.source_text == "Kumagai Jin" + assert structured.source.given_name == "Kumagai" + assert structured.source.surname == "Jin" + assert structured.source.given_tokens == ("Kumagai",) + assert structured.source.surname_tokens == ("Jin",) + assert structured.source.order == ("given", "surname") + + +def test_structured_japanese_romanized_preserves_ambiguous_or_given_first_pairs( + detector: ChineseNameDetector, +) -> None: + for first_name, last_name in (("Motoki", "Kouzaki"), ("Akira", "Kurosawa")): + surface = f"{first_name} {last_name}" + raw = detector.normalize_person_name(surface) + structured = detector.normalize_person_name_components(first_name=first_name, last_name=last_name) + + assert raw is not None + assert structured is not None + assert structured.text == raw.text == surface + assert structured.normalized == raw.normalized + assert structured.source.given_name == first_name + assert structured.source.surname == last_name + assert structured.source.order == ("given", "surname") + + +def test_japanese_native_uses_classifier_then_component_boundary() -> None: + decision = EastAsianNameOrderService().infer( + "中田英寿", + japanese_probability=lambda _name: 1.0, + ) + + assert decision is not None + assert decision.first_name == "英寿" + assert decision.last_name == "中田" + assert decision.source_order == ("surname", "given") + + +def test_japanese_native_preserves_when_classifier_abstains() -> None: + decision = EastAsianNameOrderService().infer( + "中田英寿", + japanese_probability=lambda _name: 0.79, + ) + + assert decision is None + + +def test_korean_routes_strict_shapes_and_preserves_ambiguous_romanization( + detector: ChineseNameDetector, +) -> None: + romanized = detector.normalize_person_name("Kim Min-jun") + ambiguous = detector.normalize_person_name("Kim Yuna") + native = detector.normalize_person_name("김민수") + + assert romanized is not None + assert romanized.text == "Min-jun Kim" + assert romanized.source.order == ("surname", "given") + assert ambiguous is not None + assert ambiguous.text == "Kim Yuna" + assert native is not None + assert native.text == "민수 김" + assert native.source.order == ("surname", "given") + + +def test_vietnamese_requires_unicode_evidence_and_preserves_given_span( + detector: ChineseNameDetector, +) -> None: + unicode_name = detector.normalize_person_name("Nguyễn Văn An") + ascii_name = detector.normalize_person_name("Nguyen Van An") + + assert unicode_name is not None + assert unicode_name.text == "An Văn Nguyễn" + assert unicode_name.normalized.given_name == "An" + assert unicode_name.normalized.middle_name == "Văn" + assert unicode_name.normalized.surname == "Nguyễn" + assert unicode_name.source.order == ("surname", "middle", "given") + assert ascii_name is not None + assert ascii_name.text == "Nguyen Van An" + + +def test_comma_order_remains_authoritative(detector: ChineseNameDetector) -> None: + canonical = detector.normalize_person_name("Kim, Min-jun") + + assert canonical is not None + assert canonical.text == "Min-jun Kim" + assert canonical.source.order == ("surname", "given") diff --git a/tests/test_person_name_normalization.py b/tests/test_person_name_normalization.py new file mode 100644 index 0000000..df3f2db --- /dev/null +++ b/tests/test_person_name_normalization.py @@ -0,0 +1,1307 @@ +"""Focused tests for canonical person-name normalization.""" + +from dataclasses import FrozenInstanceError + +import pytest + +from sinonym.services import person_name_normalization +from sinonym.services.person_name_normalization import ( + DropReason, + PersonNameNormalizationService, + PersonNameOutcome, +) + + +@pytest.fixture +def normalizer() -> PersonNameNormalizationService: + """Return the dependency-free canonical name normalizer.""" + return PersonNameNormalizationService() + + +@pytest.mark.parametrize( + "dash", + ["-", "\u2010", "\u2011", "\u2012", "\u2013", "\u2014", "\u2015", "\u2043", "\u2212", "\ufe58", "\ufe63", "\uff0d"], +) +def test_normalize_text_converts_dash_like_joiners( + normalizer: PersonNameNormalizationService, + dash: str, +) -> None: + result = normalizer.normalize_text(f"anne {dash} marie smith") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Anne-Marie Smith" + assert result.canonical_name.normalized.given_name == "Anne-Marie" + + +@pytest.mark.parametrize( + "apostrophe", + [ + "'", + "\u2018", + "\u2019", + "\u201a", + "\u201b", + "\u2032", + "\u2035", + "\u02bb", + "\u02bc", + "\u02b9", + "\ua78b", + "\ua78c", + "\uff07", + "`", + "\uff40", + "\u00b4", + ], +) +def test_normalize_text_converts_apostrophe_like_joiners( + normalizer: PersonNameNormalizationService, + apostrophe: str, +) -> None: + result = normalizer.normalize_text(f"sean o {apostrophe} connor") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Sean O'Connor" + assert result.canonical_name.normalized.surname == "O'Connor" + + +def test_nfkc_apostrophe_expansion_preserves_its_letter( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("sean o\u0149eill") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Sean O'Neill" + + +def test_ascii_surface_fast_path_skips_unicode_scans(monkeypatch: pytest.MonkeyPatch) -> None: + def unexpected_call(*_args: object, **_kwargs: object) -> str: + raise AssertionError + + class UnexpectedUnicodeData: + normalize = staticmethod(unexpected_call) + + monkeypatch.setattr(person_name_normalization, "unicodedata", UnexpectedUnicodeData) + monkeypatch.setattr(PersonNameNormalizationService, "_normalize_joiner", staticmethod(unexpected_call)) + + assert ( + PersonNameNormalizationService._normalize_surface( # noqa: SLF001 + " sean o ` connor ", + ) + == "sean o'connor" + ) + + +@pytest.mark.parametrize("raw_name", ["John Smith", "JOHN SMITH", "jOhN mCdonald", "A Li", "CS Smith", "Ben Sherwood"]) +def test_simple_two_token_fast_path_matches_full_pipeline( + normalizer: PersonNameNormalizationService, + monkeypatch: pytest.MonkeyPatch, + raw_name: str, +) -> None: + fast_result = normalizer._normalize_simple_two_token_text(raw_name) # noqa: SLF001 + assert fast_result is not None + + monkeypatch.setattr( + PersonNameNormalizationService, + "_normalize_simple_two_token_text", + lambda _self, _raw_name: None, + ) + + assert normalizer.normalize_text(raw_name) == fast_result + + +@pytest.mark.parametrize( + "raw_name", + ["John Jr", "John Phd", "Phd Smith", "PD Dr", "John University", "John AND", "John II"], +) +def test_simple_two_token_fast_path_abstains_on_policy_tokens( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + assert normalizer._normalize_simple_two_token_text(raw_name) is None # noqa: SLF001 + + +def test_normalize_text_strips_stacked_title_and_credentials_at_boundaries( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Dr. Steve Marsh Blando, Ph.D., M.S.") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Steve Marsh Blando" + assert result.canonical_name.normalized.given_name == "Steve" + assert result.canonical_name.normalized.middle_name == "Marsh" + assert result.canonical_name.normalized.surname == "Blando" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("Dr.", "given", DropReason.TITLE), + ("Ph.D.", "suffix", DropReason.CREDENTIAL), + ("M.S.", "suffix", DropReason.CREDENTIAL), + ] + + +@pytest.mark.parametrize( + ("first_name", "middle_name", "last_name", "expected", "expected_first", "expected_last"), + [ + ("Chaplain", "David", "Walden", "David Walden", "David", "Walden"), + ("BSc", "", "Millie Chau", "Millie Chau", "Millie", "Chau"), + ("Frau", "", "Schäfer-Graf", "Schäfer-Graf", "", "Schäfer-Graf"), + ], +) +def test_structured_titles_and_credentials_repair_emptied_given_boundary( # noqa: PLR0913 + normalizer: PersonNameNormalizationService, + first_name: str, + middle_name: str, + last_name: str, + expected: str, + expected_first: str, + expected_last: str, +) -> None: + result = normalizer.normalize_components( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + ) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == expected + assert result.canonical_name.normalized.given_name == expected_first + assert result.canonical_name.normalized.surname == expected_last + + +@pytest.mark.parametrize( + ("last_name", "expected_first", "expected_middle", "expected_last"), + [ + ("Mifta Rezki", "Mifta", "", "Rezki"), + ("Rajashree Vishnoo Naik", "Rajashree", "Vishnoo", "Naik"), + ("Njarasoa Charlette RANDRIAMALALA", "Njarasoa Charlette", "", "Randriamalala"), + ("Sathya S.", "Sathya", "", "S."), + ], +) +def test_last_only_complete_names_repair_structurally_empty_given_boundary( + normalizer: PersonNameNormalizationService, + last_name: str, + expected_first: str, + expected_middle: str, + expected_last: str, +) -> None: + result = normalizer.normalize_components(last_name=last_name) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.normalized.given_name == expected_first + assert result.canonical_name.normalized.middle_name == expected_middle + assert result.canonical_name.normalized.surname == expected_last + + +@pytest.mark.parametrize("marker", ["†", "‡", "*", "§"]) +def test_boundary_symbol_and_fused_initial_surname_are_repaired( + normalizer: PersonNameNormalizationService, + marker: str, +) -> None: + result = normalizer.normalize_components(first_name=marker, last_name="W.Tsujita") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "W. Tsujita" + assert [(token.text, token.reason) for token in result.dropped_tokens] == [(marker, DropReason.CONNECTOR)] + + +def test_last_only_particle_surname_is_not_split( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components(last_name="van der Waals") + + assert result.canonical_name is not None + assert result.canonical_name.normalized.given_name == "" + assert result.canonical_name.normalized.surname == "van der Waals" + + +def test_terminal_transliteration_apostrophe_is_preserved( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components(first_name="P", middle_name="V", last_name="Bigar'") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "P V Bigar'" + + +@pytest.mark.parametrize( + ("first_name", "middle_name", "last_name", "expected"), + [ + ("Edilene", "Natália", "Araújo das Graças", "Edilene Natália Araújo das Graças"), + ("Frederico", "Ferreira de", "Oliveira", "Frederico Ferreira de Oliveira"), + ("ROBERT", "C.", "McKEAN", "Robert C. McKean"), + ("O.", "", "LöWENSTEIN", "O. Löwenstein"), + ], +) +def test_source_particles_and_mixed_ocr_case_are_normalized( + normalizer: PersonNameNormalizationService, + first_name: str, + middle_name: str, + last_name: str, + expected: str, +) -> None: + result = normalizer.normalize_components( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + ) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("H.-J. Pompino", "H.-J. Pompino"), + ("Safiye ŞAHİN", "Safiye Şahin"), + ("Mihajlo (Michael) B Jakovljevic", "Mihajlo (Michael) B Jakovljevic"), + ], +) +def test_compound_initial_unicode_case_and_parenthetical_name_are_preserved( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +def test_normalize_text_extracts_true_suffix(normalizer: PersonNameNormalizationService) -> None: + result = normalizer.normalize_text("Steve Blando IV") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Steve Blando IV" + assert result.canonical_name.normalized.given_name == "Steve" + assert result.canonical_name.normalized.middle_name == "" + assert result.canonical_name.normalized.surname == "Blando" + assert result.canonical_name.normalized.suffix == "IV" + + +def test_comma_suffix_does_not_trigger_family_first_parsing( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Thomas L. Duvall, Jr.") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Thomas L. Duvall Jr." + assert result.canonical_name.normalized.given_name == "Thomas" + assert result.canonical_name.normalized.middle_name == "L." + assert result.canonical_name.normalized.surname == "Duvall" + assert result.canonical_name.normalized.suffix == "Jr." + + +def test_mixed_case_roman_looking_surname_is_not_a_suffix( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Hiroshi Ii") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Hiroshi Ii" + assert result.canonical_name.normalized.surname == "Ii" + assert result.canonical_name.normalized.suffix == "" + + +def test_explicit_comma_parses_family_first_and_preserves_family_particles( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("de la cruz, maría elena") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "María Elena de la Cruz" + assert result.canonical_name.normalized.given_name == "María" + assert result.canonical_name.normalized.middle_name == "Elena" + assert result.canonical_name.normalized.surname == "de la Cruz" + assert result.canonical_name.normalized.order == ("given", "middle", "surname", "surname", "surname") + assert result.canonical_name.source.order == ("surname", "surname", "surname", "given", "middle") + + +def test_trailing_family_particles_are_kept_with_surname( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("ludwig van der waals") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Ludwig van der Waals" + assert result.canonical_name.normalized.middle_name == "" + assert result.canonical_name.normalized.surname == "van der Waals" + + +def test_structured_components_reinfer_roles_after_boundary_tokens_are_dropped( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components( + first_name="dr steve", + middle_name="marsh", + last_name="phd", + ) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Steve Marsh" + assert result.canonical_name.source.given_name == "dr steve" + assert result.canonical_name.source.middle_name == "marsh" + assert result.canonical_name.source.surname == "phd" + assert result.canonical_name.normalized.given_name == "Steve" + assert result.canonical_name.normalized.middle_name == "" + assert result.canonical_name.normalized.surname == "Marsh" + assert result.canonical_name.normalized.suffix == "" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("dr", "given", DropReason.TITLE), + ("phd", "surname", DropReason.CREDENTIAL), + ] + + +def test_particle_only_surname_expansion_cannot_cross_explicit_middle( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components(first_name="An", middle_name="Van", last_name="Nguyen") + + assert result.canonical_name is not None + assert result.canonical_name.normalized.given_name == "An" + assert result.canonical_name.normalized.middle_name == "Van" + assert result.canonical_name.normalized.surname == "Nguyen" + + +@pytest.mark.parametrize( + ("first_name", "middle_name", "last_name"), + [ + ("Byung", "Chan", "Lee"), + ("Ricardo", "Burciaga", "Castañeda"), + ("Doménica", "Alejandra", "Delgado González"), + ], +) +def test_structured_source_roles_are_preserved_without_mechanical_corruption( + normalizer: PersonNameNormalizationService, + first_name: str, + middle_name: str, + last_name: str, +) -> None: + result = normalizer.normalize_components( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + ) + + assert result.canonical_name is not None + assert result.canonical_name.normalized.given_name == first_name + assert result.canonical_name.normalized.middle_name == middle_name + assert result.canonical_name.normalized.surname == last_name + + +def test_ambiguous_degree_key_in_surname_case_is_preserved( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Bao Do") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Bao Do" + assert result.canonical_name.normalized.surname == "Do" + assert result.dropped_tokens == () + + +def test_uppercase_leading_credential_is_dropped( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("MD Jane Smith") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Jane Smith" + assert [(token.text, token.reason) for token in result.dropped_tokens] == [ + ("MD", DropReason.CREDENTIAL), + ] + + +@pytest.mark.parametrize( + ("raw_name", "expected", "expected_given", "expected_surname"), + [ + ("JOHN MA", "John Ma", "John", "Ma"), + ("MA SMITH", "Ma Smith", "Ma", "Smith"), + ("Smith, Ma", "Ma Smith", "Ma", "Smith"), + ("SMITH, MA", "Ma Smith", "Ma", "Smith"), + ], +) +def test_ambiguous_uppercase_credential_token_is_preserved_when_required_as_name( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, + expected_given: str, + expected_surname: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == expected + assert result.canonical_name.normalized.given_name == expected_given + assert result.canonical_name.normalized.surname == expected_surname + assert result.dropped_tokens == () + + +def test_ambiguous_uppercase_credential_is_still_dropped_from_complete_name( + normalizer: PersonNameNormalizationService, +) -> None: + results = [ + normalizer.normalize_text("JOHN SMITH MA"), + normalizer.normalize_text("John Smith, MA"), + ] + + for result in results: + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "John Smith" + assert [(token.text, token.reason) for token in result.dropped_tokens] == [ + ("MA", DropReason.CREDENTIAL), + ] + + +def test_meng_surname_is_preserved_while_meng_degree_is_dropped( + normalizer: PersonNameNormalizationService, +) -> None: + surname = normalizer.normalize_text("Wenhua Meng") + credential = normalizer.normalize_text("John Smith MEng") + + assert surname.canonical_name is not None + assert surname.canonical_name.text == "Wenhua Meng" + assert surname.canonical_name.normalized.surname == "Meng" + assert surname.dropped_tokens == () + assert credential.canonical_name is not None + assert credential.canonical_name.text == "John Smith" + assert [(token.text, token.reason) for token in credential.dropped_tokens] == [ + ("MEng", DropReason.CREDENTIAL), + ] + + +@pytest.mark.parametrize("raw_name", ["Dr Smith", "Ms Smith"]) +def test_short_unambiguous_titles_are_still_dropped( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == "Smith" + assert [(token.text, token.reason) for token in result.dropped_tokens] == [ + (raw_name.split()[0], DropReason.TITLE), + ] + + +def test_structured_explicit_suffix_supports_single_roman_numeral( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components(first_name="john", last_name="smith", suffix="v") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "John Smith V" + assert result.canonical_name.normalized.suffix == "V" + + +@pytest.mark.parametrize( + ("given_name", "roman_surname"), + [("John", "Vi"), ("John", "Iv"), ("John", "Iii"), ("John", "Vii"), ("Malcolm", "X")], +) +def test_two_token_raw_roman_numeral_is_a_surname_but_explicit_is_a_suffix( + normalizer: PersonNameNormalizationService, + given_name: str, + roman_surname: str, +) -> None: + raw = normalizer.normalize_text(f"{given_name} {roman_surname}") + structured = normalizer.normalize_components(first_name=given_name, suffix=roman_surname) + + assert raw.canonical_name is not None + assert raw.canonical_name.text == f"{given_name} {roman_surname}" + assert raw.canonical_name.normalized.surname == roman_surname + assert raw.canonical_name.normalized.suffix == "" + assert structured.canonical_name is not None + assert structured.canonical_name.text == f"{given_name} {roman_surname.upper()}" + assert structured.canonical_name.normalized.surname == "" + assert structured.canonical_name.normalized.suffix == roman_surname.upper() + + +@pytest.mark.parametrize( + ("raw_name", "expected", "expected_suffix"), + [ + ("Steve Blando IV", "Steve Blando IV", "IV"), + ("John Smith Vi", "John Smith VI", "VI"), + ("Smith, John IV", "John Smith IV", "IV"), + ], +) +def test_raw_roman_suffix_requires_complete_name_context( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, + expected_suffix: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + assert result.canonical_name.normalized.suffix == expected_suffix + + +def test_structured_missing_surname_is_reinferred_from_one_split_component( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components(first_name="Dr. Mary Ann O\u2019Neill", last_name="MS") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Mary Ann O'Neill" + assert result.canonical_name.normalized.given_name == "Mary" + assert result.canonical_name.normalized.middle_name == "Ann" + assert result.canonical_name.normalized.surname == "O'Neill" + + +def test_affiliation_digits_are_dropped_without_losing_name_token( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Anna Spießl1, 4") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Anna Spießl" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("1", "surname", DropReason.AFFILIATION), + ("4", "suffix", DropReason.AFFILIATION), + ] + + +@pytest.mark.parametrize( + ("raw_name", "expected_outcome"), + [ + ("", PersonNameOutcome.INVALID), + ("---", PersonNameOutcome.INVALID), + ("John Smith and Jane Doe", PersonNameOutcome.NON_PERSON), + ("John Smith, Jane Doe", PersonNameOutcome.NON_PERSON), + ("Stanford University", PersonNameOutcome.NON_PERSON), + ], +) +def test_normalize_text_returns_typed_non_person_and_invalid_outcomes( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected_outcome: PersonNameOutcome, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.outcome is expected_outcome + assert result.canonical_name is None + assert result.reason + + +def test_result_and_dropped_token_lineage_are_immutable( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Mr. Steve Blando PhD") + + assert result.canonical_name is not None + reason_attribute = "reason" + text_attribute = "text" + with pytest.raises(FrozenInstanceError): + setattr(result, reason_attribute, "changed") + with pytest.raises(FrozenInstanceError): + setattr(result.dropped_tokens[0], text_attribute, "changed") + + +@pytest.mark.parametrize( + "raw_name", + [ + "Dr. Steve Marsh Blando, Ph.D.", + "de la Cruz, María Elena", + "Thomas L. Duvall, Jr.", + "Anne-Marie O'Connor", + ], +) +def test_component_orders_contain_exactly_one_role_per_token( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + for components in (result.canonical_name.source, result.canonical_name.normalized): + token_count = sum( + len(tokens) + for tokens in ( + components.given_tokens, + components.middle_tokens, + components.surname_tokens, + components.suffix_tokens, + ) + ) + assert len(components.order) == token_count + + +def test_structured_non_person_and_invalid_results_have_no_canonical_name( + normalizer: PersonNameNormalizationService, +) -> None: + non_person = normalizer.normalize_components(first_name="Stanford", last_name="University") + invalid = normalizer.normalize_components() + + assert non_person.outcome is PersonNameOutcome.NON_PERSON + assert non_person.canonical_name is None + assert invalid.outcome is PersonNameOutcome.INVALID + assert invalid.canonical_name is None + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Vincent El Ghouzzi", "Vincent El Ghouzzi"), + ("Monica Da Costa", "Monica Da Costa"), + ("P. T. d'Orbán", "P. T. d'Orbán"), + ("de la cruz, maría elena", "María Elena de la Cruz"), + ], +) +def test_family_particles_preserve_credible_source_case( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("E V Usol'tseva", "E V Usol'tseva"), + ("Wa\u2019el Tuqan", "Wa'el Tuqan"), + ("sean o \u2019 connor", "Sean O'Connor"), + ("Michael F. O''Rourke", "Michael F. O'Rourke"), + ("Carol O\u2019sullivan", "Carol O'Sullivan"), + ("Ruth D'arcy Hart", "Ruth D'Arcy Hart"), + ], +) +def test_apostrophe_normalization_preserves_mixed_case_and_collapses_duplicates( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("' J. LEECH", "J. Leech"), + ("Ph. R. Hénon", "Ph. R. Hénon"), + ("Zd. Servit", "Zd. Servit"), + ("A. D\u2018A. BELLAIRS", "A. D'A. Bellairs"), + ], +) +def test_stray_joiners_and_abbreviated_tokens_are_normalized( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +def test_additional_boundary_credentials_and_credential_like_surnames( + normalizer: PersonNameNormalizationService, +) -> None: + mpa = normalizer.normalize_text("Amina Helmi MPA") + spaced_md = normalizer.normalize_text("William A. Horwitz M. D.") + surname = normalizer.normalize_text("E C Mba") + + assert mpa.canonical_name is not None + assert mpa.canonical_name.text == "Amina Helmi" + assert spaced_md.canonical_name is not None + assert spaced_md.canonical_name.text == "William A. Horwitz" + assert surname.canonical_name is not None + assert surname.canonical_name.text == "E C Mba" + + +@pytest.mark.parametrize("credential", ["Ph. D.", "M. Sc."]) +def test_spaced_credentials_are_dropped_in_raw_and_structured_suffix_forms( + normalizer: PersonNameNormalizationService, + credential: str, +) -> None: + raw = normalizer.normalize_text(f"John Smith {credential}") + structured = normalizer.normalize_components(first_name="John", last_name="Smith", suffix=credential) + + for result in (raw, structured): + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "John Smith" + assert all(token.reason is DropReason.CREDENTIAL for token in result.dropped_tokens) + + +@pytest.mark.parametrize( + ("raw_name", "expected_suffix"), + [("McKinley Glover Iv", "IV"), ("N David Yanez Iii", "III"), ("Hiroshi Ii", "")], +) +def test_mixed_case_long_roman_suffixes_are_unambiguous( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected_suffix: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.normalized.suffix == expected_suffix + + +def test_stacked_academic_titles_and_parenthetical_duplicate_are_removed( + normalizer: PersonNameNormalizationService, +) -> None: + titled = normalizer.normalize_text("Univ.-Prof. Dr. med. Prof. honoraire Dr. h.c. C. C. Zouboulis") + parenthetical = normalizer.normalize_text("Alan (Alan B.) Cantor") + + assert titled.canonical_name is not None + assert titled.canonical_name.text == "C. C. Zouboulis" + assert parenthetical.canonical_name is not None + assert parenthetical.canonical_name.text == "Alan B. Cantor" + + +def test_trailing_affiliation_is_removed_after_complete_person_name( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Rachel Webster University of New South Wales") + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Rachel Webster" + + +def test_two_token_particle_like_given_name_is_not_duplicated( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Ben Sherwood") + + assert result.canonical_name is not None + assert result.canonical_name.text == "Ben Sherwood" + assert result.canonical_name.normalized.given_name == "Ben" + assert result.canonical_name.normalized.surname == "Sherwood" + + +def test_uppercase_initial_clusters_remain_uppercase( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("Gordon CS Smith") + + assert result.canonical_name is not None + assert result.canonical_name.text == "Gordon CS Smith" + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("N.p Sunil-Chandra", "N.P Sunil-Chandra"), + ("G.Y Minuk", "G.Y Minuk"), + ("K.D.D.I Kodithuwakku", "K.D.D.I Kodithuwakku"), + ("Alekseeva M.Yu. Alekseeva", "Alekseeva M.Yu. Alekseeva"), + ("P.Sh. Ibragimov", "P.Sh. Ibragimov"), + ("MM. Cunningham", "MM. Cunningham"), + ], +) +def test_period_policy_distinguishes_initial_clusters_from_transliteration_abbreviations( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Dr.Wenjun Zhang", "Wenjun Zhang"), + ("Mrs.E. Sumathi", "E. Sumathi"), + ("Dr.AARCHA S S", "Aarcha S S"), + ("PD Dr. M. Mengel", "M. Mengel"), + ], +) +def test_attached_and_stacked_titles_are_removed_without_dropping_the_name( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + assert all(token.reason is DropReason.TITLE for token in result.dropped_tokens) + expected_first_key = "".join(character.casefold() for character in expected.split()[0] if character.isalnum()) + source_first_count = sum( + "".join(character.casefold() for character in token if character.isalnum()) == expected_first_key + for token in result.canonical_name.source.given_tokens + ) + assert source_first_count == 1 + + +def test_period_context_preserves_md_given_abbreviation_and_ms_initials( + normalizer: PersonNameNormalizationService, +) -> None: + md = normalizer.normalize_text("Md. Abu Bakar Siddiq") + initials = normalizer.normalize_text("M.S. Crouse") + credential = normalizer.normalize_text("Jaume Bosch M.D.") + + assert md.canonical_name is not None + assert md.canonical_name.text == "Md. Abu Bakar Siddiq" + assert initials.canonical_name is not None + assert initials.canonical_name.text == "M.S. Crouse" + assert credential.canonical_name is not None + assert credential.canonical_name.text == "Jaume Bosch" + + +def test_terminal_and_standalone_sentence_periods_are_removed( + normalizer: PersonNameNormalizationService, +) -> None: + terminal = normalizer.normalize_text("Subramanian. A") + standalone = normalizer.normalize_text("A.Kalaikannan .") + + assert terminal.canonical_name is not None + assert terminal.canonical_name.text == "Subramanian A" + assert standalone.canonical_name is not None + assert standalone.canonical_name.text == "A.Kalaikannan" + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("DAVID NG", "David Ng"), + ("JUAN DE LA CRUZ", "Juan de la Cruz"), + ], +) +def test_uppercase_two_letter_surnames_and_particles_use_name_case( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected_middle", "expected_surname"), + [ + ("Juan J Llibre Rodriguez", "J", "Llibre Rodriguez"), + ("Carlos A. Henríquez Q.", "A.", "Henríquez Q."), + ("Landys A. Lopez Quezada", "A.", "Lopez Quezada"), + ], +) +def test_initial_followed_by_full_token_preserves_two_token_family_name( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected_middle: str, + expected_surname: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.normalized.middle_name == expected_middle + assert result.canonical_name.normalized.surname == expected_surname + + +def test_ordinary_three_token_name_keeps_middle_name( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_text("John Michael Smith") + + assert result.canonical_name is not None + assert result.canonical_name.normalized.middle_name == "Michael" + assert result.canonical_name.normalized.surname == "Smith" + + +def test_structured_middle_drops_only_surname_copy_with_lowercase_marker( + normalizer: PersonNameNormalizationService, +) -> None: + result = normalizer.normalize_components( + first_name="Mery", + middle_name="Luz Rojas Aire z", + last_name="Rojas Aire", + ) + + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Mery Luz Rojas Aire" + assert result.canonical_name.source.middle_name == "Luz Rojas Aire z" + assert result.canonical_name.normalized.given_name == "Mery" + assert result.canonical_name.normalized.middle_name == "Luz" + assert result.canonical_name.normalized.surname == "Rojas Aire" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("Rojas", "middle", DropReason.DUPLICATE), + ("Aire", "middle", DropReason.DUPLICATE), + ("z", "middle", DropReason.CONNECTOR), + ] + + +@pytest.mark.parametrize( + ("middle_name", "last_name", "expected_middle"), + [ + ("Smith", "Smith", "Smith"), + ("Luz Rojas Aire", "Rojas Aire", "Luz Rojas Aire"), + ("Luz Rojas Aire Z", "Rojas Aire", "Luz Rojas Aire Z"), + ("Luz Rojas Air z", "Rojas Aire", "Luz Rojas Air Z"), + ], +) +def test_structured_middle_does_not_broadly_deduplicate_surnames( + normalizer: PersonNameNormalizationService, + middle_name: str, + last_name: str, + expected_middle: str, +) -> None: + result = normalizer.normalize_components( + first_name="Mery", + middle_name=middle_name, + last_name=last_name, + ) + + assert result.canonical_name is not None + assert result.canonical_name.normalized.middle_name == expected_middle + assert result.canonical_name.normalized.surname == last_name + + +def test_exact_leading_ma_abbreviation_is_preserved_before_initial_and_surname( + normalizer: PersonNameNormalizationService, +) -> None: + results = [ + normalizer.normalize_text("Ma. E. Zayas"), + normalizer.normalize_components(first_name="Ma.", middle_name="E.", last_name="Zayas"), + ] + + for result in results: + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Ma. E. Zayas" + assert result.canonical_name.normalized.given_name == "Ma." + assert result.canonical_name.normalized.middle_name == "E." + assert result.canonical_name.normalized.surname == "Zayas" + assert result.dropped_tokens == () + + +@pytest.mark.parametrize("raw_name", ["M.A. E. Zayas", "Jane Smith M.A."]) +def test_ma_credential_forms_are_still_dropped( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert "M.A." not in result.canonical_name.text + assert [(token.text, token.reason) for token in result.dropped_tokens] == [ + ("M.A.", DropReason.CREDENTIAL), + ] + + +def test_lowercase_marker_fused_to_leading_initial_is_split_with_lineage( + normalizer: PersonNameNormalizationService, +) -> None: + results = [ + normalizer.normalize_text("tE. L. Winkelman"), + normalizer.normalize_components(first_name="tE.", middle_name="L.", last_name="Winkelman"), + ] + + for result in results: + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "E. L. Winkelman" + assert result.canonical_name.source.given_name == "tE." + assert result.canonical_name.normalized.given_name == "E." + assert result.canonical_name.normalized.middle_name == "L." + assert result.canonical_name.normalized.surname == "Winkelman" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("t", "given", DropReason.CONNECTOR), + ] + + +@pytest.mark.parametrize("raw_name", ["tE. Louis Winkelman", "eBay L. Smith", "JoAnn L. Smith"]) +def test_leading_mixed_case_names_are_not_generically_split( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == raw_name + assert result.dropped_tokens == () + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("Dr Li", "Li"), + ("Doctor Li", "Li"), + ("Pastor John Smith", "John Smith"), + ("Frau Ng", "Ng"), + ("Rabbi Ng", "Ng"), + ("Lord Ng", "Ng"), + ("Pastor Ng", "Ng"), + ], +) +def test_unambiguous_title_contexts_are_still_dropped( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + assert len(result.dropped_tokens) == 1 + assert result.dropped_tokens[0].reason is DropReason.TITLE + + +def test_separated_compound_initials_join_only_at_visible_initial_boundary( + normalizer: PersonNameNormalizationService, +) -> None: + raw = normalizer.normalize_text("H. -J. Schneider") + structured = normalizer.normalize_components(first_name="H.", middle_name="-J.", last_name="Schneider") + + for result in (raw, structured): + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "H.-J. Schneider" + assert result.canonical_name.normalized.given_name == "H.-J." + assert result.canonical_name.normalized.middle_name == "" + assert result.canonical_name.normalized.surname == "Schneider" + + assert structured.canonical_name is not None + assert structured.canonical_name.source.given_name == "H." + assert structured.canonical_name.source.middle_name == "-J." + + +def test_compound_initial_join_does_not_change_normal_initial_order_or_words( + normalizer: PersonNameNormalizationService, +) -> None: + normal = normalizer.normalize_text("H. J. Schneider") + word = normalizer.normalize_components(first_name="H.", middle_name="-Jean", last_name="Schneider") + + assert normal.canonical_name is not None + assert normal.canonical_name.text == "H. J. Schneider" + assert normal.canonical_name.normalized.middle_name == "J." + assert word.outcome is PersonNameOutcome.INVALID + + +def test_leading_superscript_affiliation_marker_is_removed_with_lineage( + normalizer: PersonNameNormalizationService, +) -> None: + raw = normalizer.normalize_text("\u00b9Matias Julyus") + structured = normalizer.normalize_components(first_name="\u00b9Matias", last_name="Julyus") + + for result in (raw, structured): + assert result.outcome is PersonNameOutcome.PERSON + assert result.canonical_name is not None + assert result.canonical_name.text == "Matias Julyus" + assert result.canonical_name.source.given_name == "\u00b9Matias" + assert result.canonical_name.normalized.given_name == "Matias" + assert result.canonical_name.normalized.surname == "Julyus" + assert [(token.text, token.source_role, token.reason) for token in result.dropped_tokens] == [ + ("\u00b9", "given", DropReason.AFFILIATION), + ] + + +@pytest.mark.parametrize("raw_name", ["1Matias Julyus", "Ma\u00b9tias Julyus"]) +def test_ordinary_digits_and_internal_superscripts_are_not_affiliation_prefixes( + normalizer: PersonNameNormalizationService, + raw_name: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.outcome is PersonNameOutcome.INVALID + + +@pytest.mark.parametrize( + ("first_name", "middle_name", "last_name", "expected"), + [ + ("\u00c9va", "d. H.", "Alm\u00e1si", "\u00c9va d. H. Alm\u00e1si"), + ("L.", "M. b.", "Hoskins", "L. M. b. Hoskins"), + ("F.", "E. Soares e", "Silva", "F. E. Soares e Silva"), + ("Yuldosheva", "Zulfizar Sobir", "kizi", "Yuldosheva Zulfizar Sobir kizi"), + ("Aysel", "Mammad", "qizi", "Aysel Mammad qizi"), + ], +) +def test_lowercase_relational_tokens_preserve_source_case_by_role( + normalizer: PersonNameNormalizationService, + first_name: str, + middle_name: str, + last_name: str, + expected: str, +) -> None: + result = normalizer.normalize_components( + first_name=first_name, + middle_name=middle_name, + last_name=last_name, + ) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +@pytest.mark.parametrize( + ("raw_name", "expected"), + [ + ("d. Smith", "D. Smith"), + ("kizi Smith", "Kizi Smith"), + ("McDONALD Smith", "McDonald Smith"), + ("McDonald Smith", "McDonald Smith"), + ("FitzGerald Smith", "FitzGerald Smith"), + ], +) +def test_relational_case_rule_does_not_preserve_given_or_ocr_casing( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +def test_known_alphabetic_name_abbreviation_preserves_meaningful_period( + normalizer: PersonNameNormalizationService, +) -> None: + raw = normalizer.normalize_text("Most. Sumaiya Khatun Kali") + structured = normalizer.normalize_components(first_name="Most.", middle_name="Sumaiya Khatun", last_name="Kali") + + for result in (raw, structured): + assert result.canonical_name is not None + assert result.canonical_name.text == "Most. Sumaiya Khatun Kali" + assert result.canonical_name.normalized.given_name == "Most." + + +@pytest.mark.parametrize(("raw_name", "expected"), [("Mark. Smith", "Mark Smith"), ("Subramanian. A", "Subramanian A")]) +def test_sentence_like_periods_are_not_preserved_as_abbreviations( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.text == expected + + +def test_packed_surname_first_with_two_trailing_initials_is_reordered( + normalizer: PersonNameNormalizationService, +) -> None: + raw = normalizer.normalize_text("Masterov R. A.") + structured = normalizer.normalize_components(last_name="Masterov R. A.") + + for result in (raw, structured): + assert result.canonical_name is not None + assert result.canonical_name.text == "R. A. Masterov" + assert result.canonical_name.normalized.given_name == "R." + assert result.canonical_name.normalized.middle_name == "A." + assert result.canonical_name.normalized.surname == "Masterov" + + assert raw.canonical_name is not None + assert raw.canonical_name.source.order == ("surname", "given", "middle") + + +def test_packed_surname_first_gate_preserves_normal_order_and_source_last_floor( + normalizer: PersonNameNormalizationService, +) -> None: + normal = normalizer.normalize_text("R. A. Masterov") + one_initial = normalizer.normalize_text("Masterov R.") + source_last = normalizer.normalize_components(first_name="John", last_name="Masterov R. A.") + + assert normal.canonical_name is not None + assert normal.canonical_name.text == "R. A. Masterov" + assert normal.canonical_name.normalized.surname == "Masterov" + assert one_initial.canonical_name is not None + assert one_initial.canonical_name.text == "Masterov R." + assert one_initial.canonical_name.normalized.given_name == "Masterov" + assert source_last.canonical_name is not None + assert source_last.canonical_name.normalized.given_name == "John" + assert source_last.canonical_name.normalized.surname == "Masterov R. A." + + +@pytest.mark.parametrize( + ("raw_name", "expected_first", "expected_middle", "expected_last"), + [ + ("M. Calder\u00f3n de la Barca S\u00e1nchez", "M.", "", "Calder\u00f3n de la Barca S\u00e1nchez"), + ("Roberto Jos\u00e9 Carvalho da Silva", "Roberto", "Jos\u00e9", "Carvalho da Silva"), + ("Carvalho da Silva Roberto Jos\u00e9", "Roberto", "Jos\u00e9", "Carvalho da Silva"), + ], +) +def test_strong_particle_spans_expand_compound_surname_floor( + normalizer: PersonNameNormalizationService, + raw_name: str, + expected_first: str, + expected_middle: str, + expected_last: str, +) -> None: + result = normalizer.normalize_text(raw_name) + + assert result.canonical_name is not None + assert result.canonical_name.normalized.given_name == expected_first + assert result.canonical_name.normalized.middle_name == expected_middle + assert result.canonical_name.normalized.surname == expected_last + + +def test_structured_strong_particle_repairs_cover_floor_and_mechanical_shift( + normalizer: PersonNameNormalizationService, +) -> None: + calderon = normalizer.normalize_components( + first_name="M.", + middle_name="Calder\u00f3n de la Barca", + last_name="S\u00e1nchez", + ) + carvalho = normalizer.normalize_components( + first_name="Carvalho", + middle_name="da Silva Roberto", + last_name="Jos\u00e9", + ) + + assert calderon.canonical_name is not None + assert calderon.canonical_name.text == "M. Calder\u00f3n de la Barca S\u00e1nchez" + assert calderon.canonical_name.normalized.middle_name == "" + assert calderon.canonical_name.normalized.surname == "Calder\u00f3n de la Barca S\u00e1nchez" + assert carvalho.canonical_name is not None + assert carvalho.canonical_name.text == "Roberto Jos\u00e9 Carvalho da Silva" + assert carvalho.canonical_name.normalized.given_name == "Roberto" + assert carvalho.canonical_name.normalized.middle_name == "Jos\u00e9" + assert carvalho.canonical_name.normalized.surname == "Carvalho da Silva" + + +def test_particle_floor_does_not_override_weak_context_or_source_last( + normalizer: PersonNameNormalizationService, +) -> None: + weak = normalizer.normalize_components(first_name="John", middle_name="Michael de la", last_name="Smith") + incomplete_shift = normalizer.normalize_components(first_name="Carvalho", middle_name="da Silva", last_name="Jos\u00e9") + title_case_surname = normalizer.normalize_text("H. K. Das Gupta") + + assert weak.canonical_name is not None + assert weak.canonical_name.normalized.given_name == "John" + assert weak.canonical_name.normalized.middle_name == "Michael de la" + assert weak.canonical_name.normalized.surname == "Smith" + assert incomplete_shift.canonical_name is not None + assert incomplete_shift.canonical_name.normalized.given_name == "Carvalho" + assert incomplete_shift.canonical_name.normalized.middle_name == "da Silva" + assert incomplete_shift.canonical_name.normalized.surname == "Jos\u00e9" + assert title_case_surname.canonical_name is not None + assert title_case_surname.canonical_name.normalized.middle_name == "K." + assert title_case_surname.canonical_name.normalized.surname == "Das Gupta" diff --git a/tests/test_regression_proposals.py b/tests/test_regression_proposals.py index 0d1417e..7c65c6a 100644 --- a/tests/test_regression_proposals.py +++ b/tests/test_regression_proposals.py @@ -664,6 +664,24 @@ def test_aligned_bilingual_pairs_use_han_identity(detector): assert roman_first_given_first.parsed_original_order.order == ["given", "surname"] +def test_aligned_bilingual_polyphonic_surname_precedes_roman_ethnicity_rejection(detector): + raw_name = "Zhexu \u54f2\u65ed Shan \u5355" + normalized = detector._normalizer.apply(raw_name) + pairs = detector._normalizer.aligned_bilingual_pairs(normalized) + + assert pairs is not None + assert [pair.han_pinyin for pair in pairs] == [("zhe", "xu"), ("shan",)] + assert detector._normalizer.classify_script_representation(normalized) == "bilingual_aligned" + + result = detector.normalize_name(raw_name) + + assert result.success + assert result.result == "Zhe-Xu Shan" + assert result.parsed.surname == "Shan" + assert result.parsed.given_name == "Zhe-Xu" + assert result.parsed_original_order.order == ["given", "surname"] + + @pytest.mark.parametrize("lu_token", ["Lu", "L\u00fc"]) def test_aligned_bilingual_lu_alias_matches_lv_pinyin(detector, lu_token): result = detector.normalize_name(f"{lu_token} \u5415 Wei \u4f1f") diff --git a/tests/test_timo_v2_interface.py b/tests/test_timo_v2_interface.py new file mode 100644 index 0000000..8504a81 --- /dev/null +++ b/tests/test_timo_v2_interface.py @@ -0,0 +1,194 @@ +"""Versioning and canonical-name coverage for TIMO v2.""" + +import hashlib +import json +from pathlib import Path + +import pytest + +from sinonym.timo.interface import ( + BatchPrediction, + BatchSummary, + Instance, + PPRoutedPrediction, + Prediction, + PredictionV2, + Predictor, + PredictorConfig, + PredictorV2, + RoutedPaperPrediction, + RoutedPaperPredictionV2, + RoutedPrediction, + RoutingInstance, + RoutingPredictorV2, + TimoModel, +) + +V1_SCHEMA_FINGERPRINTS = { + "Prediction": "32ec52f7cae59d443bd8c221db6a7fc8f06dd5658978e54d264f919820807775", + "BatchPrediction": "3f3ccaa1fc9185296f6bb0f68acbe087da4aa900326eedc325e4505dfc3b15c2", + "BatchSummary": "01640c80b212f7f58dc3f5c0d83c876c60864dad88ec77c0c4ac326c0fee0f97", + "RoutedPrediction": "859a0b5c5f57061ee6850cebbc623667ae3a6b6e4e0d5b952f8d4774da7f5f22", + "PPRoutedPrediction": "ed2febe81a511784ece36c52a17f57dbbbe4b57ff64de9c68c205e47a348debf", + "RoutedPaperPrediction": "b45280f7abb5af523f30d129907e4ff09e0c2932074a23de5d4deca73bcf8738", +} + + +@pytest.fixture(scope="module") +def predictor_v1() -> Predictor: + return Predictor(config=PredictorConfig(parallel="never"), artifacts_dir=".") + + +@pytest.fixture(scope="module") +def predictor_v2() -> PredictorV2: + return PredictorV2(config=PredictorConfig(parallel="never"), artifacts_dir=".") + + +def _schema_fingerprint(model: type[TimoModel]) -> str: + payload = json.dumps(model.schema(), sort_keys=True, separators=(",", ":")) + return hashlib.sha256(payload.encode()).hexdigest() + + +def _without_canonical(value): + if isinstance(value, list): + return [_without_canonical(item) for item in value] + if isinstance(value, dict): + return {key: _without_canonical(item) for key, item in value.items() if key != "canonical_name"} + return value + + +def _assert_steve_marsh(prediction: PredictionV2) -> None: + assert not prediction.success + assert prediction.given_name is None + assert prediction.surname is None + assert prediction.canonical_name is not None + assert prediction.canonical_name.text == "Steve Marsh" + assert prediction.canonical_name.normalized.given_name == "Steve" + assert prediction.canonical_name.normalized.middle_name == "" + assert prediction.canonical_name.normalized.surname == "Marsh" + assert prediction.canonical_name.normalized.suffix == "" + + +def test_v1_schema_fingerprints_match_main_before_v2() -> None: + """Adding v2 types must not mutate any existing TIMO response schema.""" + models = (Prediction, BatchPrediction, BatchSummary, RoutedPrediction, PPRoutedPrediction, RoutedPaperPrediction) + + assert {model.__name__: _schema_fingerprint(model) for model in models} == V1_SCHEMA_FINGERPRINTS + + +def test_v1_dict_excludes_canonical_name(predictor_v1: Predictor) -> None: + (prediction,) = predictor_v1.predict_batch([Instance(name="Dr. Steve Marsh PhD")]) + + assert prediction.dict() == { + "success": False, + "error_message": "no Chinese evidence found", + "given_name": None, + "surname": None, + "middle_name": None, + "confidence": 0.0, + "format_pattern": { + "dominant_format": "mixed", + "confidence": 0.0, + "decision_confidence": 0.0, + "surname_first_count": 0, + "given_first_count": 0, + "total_count": 0, + "voting_count": 0, + "vote_margin_count": 0, + "vote_margin": 0.0, + "threshold_met": False, + }, + } + + +def test_v2_flat_prediction_surfaces_non_chinese_components_and_suffix(predictor_v2: PredictorV2) -> None: + steve_marsh, steve_blando = predictor_v2.predict_batch( + [Instance(name="Dr. Steve Marsh PhD"), Instance(name="Steve Blando IV")], + ) + + _assert_steve_marsh(steve_marsh) + assert steve_blando.canonical_name is not None + assert steve_blando.canonical_name.text == "Steve Blando IV" + assert steve_blando.canonical_name.normalized.given_name == "Steve" + assert steve_blando.canonical_name.normalized.surname == "Blando" + assert steve_blando.canonical_name.normalized.suffix == "IV" + assert steve_blando.canonical_name.normalized.suffix_tokens == ["IV"] + assert steve_blando.canonical_name.normalized.order == ["given", "surname", "suffix"] + + +def test_v2_batch_and_process_helpers_retain_canonical_name(predictor_v2: PredictorV2) -> None: + name = "Dr. Steve Marsh PhD" + + _assert_steve_marsh(predictor_v2.process_name_batch([name])[0]) + _assert_steve_marsh(predictor_v2.process_name_batches([[name]], parallel="never")[0][0]) + _assert_steve_marsh(predictor_v2.analyze_name_batch([name]).results[0]) + _assert_steve_marsh(predictor_v2.score_name_batch([name]).results[0]) + + +def test_v2_multiprocess_helper_retains_canonical_name(predictor_v2: PredictorV2) -> None: + (prediction,) = predictor_v2.process_name_batch_multiprocess( + ["Dr. Steve Marsh PhD"], + max_workers=1, + chunk_size=1, + ) + + _assert_steve_marsh(prediction) + + +def test_v2_pp_only_routing_retains_canonical_without_changing_decision( + predictor_v1: Predictor, + predictor_v2: PredictorV2, +) -> None: + names = ["Steve Blando IV", "Yue Lin", "Wei Wang"] + v1 = predictor_v1.route_pp(names) + v2 = predictor_v2.route_pp(names) + + assert [_without_canonical(result.dict()) for result in v2] == [result.dict() for result in v1] + assert v2[0].canonical_name is not None + assert v2[0].canonical_name.normalized.suffix == "IV" + assert v2[0].pp.canonical_name == v2[0].canonical_name + + +def test_v2_pp_vys_routing_retains_canonical_without_changing_decision( + predictor_v1: Predictor, + predictor_v2: PredictorV2, +) -> None: + pp_names = ["Steve Blando IV", "Yue Lin"] + pool = [*pp_names, "Wei Wang", "Jun Zhao", "Hui Li", "Tao Sun"] + v1 = predictor_v1.route_pp_vys(pp_names, pool) + v2 = predictor_v2.route_pp_vys(pp_names, pool) + + assert [_without_canonical(result.dict()) for result in v2] == [result.dict() for result in v1] + assert v2[0].canonical_name is not None + assert v2[0].canonical_name.normalized.suffix == "IV" + assert v2[0].pp.canonical_name is not None + assert v2[0].vys is not None + assert v2[0].vys.canonical_name is not None + + +def test_routing_predictor_v2_is_one_to_one_and_round_trips() -> None: + predictor = RoutingPredictorV2(config=PredictorConfig(parallel="never"), artifacts_dir=".") + instances = [ + RoutingInstance(pp_names=["Steve Blando IV"]), + RoutingInstance(pp_names=["Yue Lin"], vys_pool_names=["Yue Lin", "Wei Wang", "Jun Zhao"]), + RoutingInstance(pp_names=[]), + ] + + results = predictor.predict_batch(instances) + + assert len(results) == len(instances) + assert [len(result.authors) for result in results] == [1, 1, 0] + assert results[0].authors[0].canonical_name is not None + assert results[0].authors[0].canonical_name.normalized.suffix == "IV" + assert [RoutedPaperPredictionV2(**result.dict()) for result in results] == results + + +def test_timo_config_exposes_separate_v2_variants() -> None: + config = Path("sinonym/timo/config.yaml").read_text(encoding="utf-8") + + assert "sinonym_v2:" in config + assert "prediction: sinonym.timo.interface.PredictionV2" in config + assert "predictor: sinonym.timo.interface.PredictorV2" in config + assert "sinonym_routing_v2:" in config + assert "prediction: sinonym.timo.interface.RoutedPaperPredictionV2" in config + assert "predictor: sinonym.timo.interface.RoutingPredictorV2" in config diff --git a/uv.lock b/uv.lock index 918136d..d1545b3 100644 --- a/uv.lock +++ b/uv.lock @@ -690,7 +690,7 @@ wheels = [ [[package]] name = "sinonym" -version = "0.3.0" +version = "0.4.0" source = { editable = "." } dependencies = [ { name = "joblib" },