diff --git a/source/config/featureFlagEnums.py b/source/config/featureFlagEnums.py index 68f6e019aad..99e247d0f28 100644 --- a/source/config/featureFlagEnums.py +++ b/source/config/featureFlagEnums.py @@ -1,7 +1,7 @@ # A part of NonVisual Desktop Access (NVDA) # Copyright (C) 2022-2026 NV Access Limited, Bill Dengler, Rob Meredith, Leonard de Ruijter, Wang Chong -# This file is covered by the GNU General Public License. -# See the file COPYING for more details. +# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. +# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt """ Feature flag value enumerations. @@ -144,18 +144,21 @@ class WordNavigationUnitFlag(DisplayStringEnum): DEFAULT = enum.auto() AUTO = enum.auto() - UNISCRIBE = enum.auto() CHINESE = enum.auto() + ICU = enum.auto() + UNISCRIBE = enum.auto() @property def _displayStringLabels(self) -> dict["WordNavigationUnitFlag", str]: return { # Translators: Label for a method of word segmentation. - self.AUTO: _("Auto"), - # Translators: Label for a method of word segmentation. - self.UNISCRIBE: _("Standard"), + self.AUTO: _("Automatic"), # Translators: Label for a method of word segmentation. self.CHINESE: _("Chinese"), + # Translators: Label for a method of word segmentation. + self.ICU: _("Unicode (ICU)"), + # Translators: Label for a method of word segmentation. + self.UNISCRIBE: _("Legacy (Uniscribe)"), } diff --git a/source/textInfos/offsets.py b/source/textInfos/offsets.py index 91b66b90a02..a5f06589813 100755 --- a/source/textInfos/offsets.py +++ b/source/textInfos/offsets.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2006-2025 NV Access Limited, Babbage B.V., Leonard de Ruijter, Wang Chong +# Copyright (C) 2006-2026 NV Access Limited, Babbage B.V., Leonard de Ruijter, Wang Chong # This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. # For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt @@ -266,6 +266,8 @@ def wordSegFlag(self) -> WordSegFlag | None: return WordSegFlag.AUTO case config.featureFlagEnums.WordNavigationUnitFlag.CHINESE: return WordSegFlag.CHINESE + case config.featureFlagEnums.WordNavigationUnitFlag.ICU: + return WordSegFlag.ICU case _: log.error(f"Unknown word segmentation standard, {self.wordSegConf.calculated()!r}") return None diff --git a/source/textUtils/_wordSeg/wordSegStrategy.py b/source/textUtils/_wordSeg/wordSegStrategy.py index 27e3f0e116b..3b269d3e492 100644 --- a/source/textUtils/_wordSeg/wordSegStrategy.py +++ b/source/textUtils/_wordSeg/wordSegStrategy.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2025 NV Access Limited, Wang Chong +# Copyright (C) 2025-2026 NV Access Limited, Wang Chong, Leonard de Ruijter # This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. # For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt @@ -142,10 +142,13 @@ def getSegmentForOffset(self, offset: int) -> tuple[int, int] | None: """Return (start inclusive, end exclusive) or None. Offsets are str offsets relative to self.text.""" pass - @abstractmethod def segmentedText(self, sep: str = " ", newSepIndex: list[int] | None = None) -> str: - """Segmented result with separators.""" - pass + """Segmented result with separators. + + The default returns the text unchanged; only strategies that insert separators + for braille output (e.g. Chinese) override this. + """ + return self.text def getWordOffsetRange( self, @@ -226,9 +229,6 @@ def _calculateUniscribeOffsets( def getSegmentForOffset(self, offset: int) -> tuple[int, int] | None: return self._calculateUniscribeOffsets(self.text, offset) - def segmentedText(self, sep: str = " ", newSepIndex: list[int] | None = None) -> str: - return self.text - class ChineseWordSegmentationStrategy(WordSegmentationStrategy): _lib: CDLL | None = None @@ -347,3 +347,25 @@ def getSegmentForOffset(self, offset: int) -> tuple[int, int] | None: def __init__(self, text: str, encoding: str | None = None) -> None: super().__init__(text, encoding) self.wordEnds = self._callCppJieba() + + +class IcuWordSegmentationStrategy(WordSegmentationStrategy): + """ICU-based word segmentation (Windows built-in ICU library). + + Word boundaries follow Unicode Standard Annex #29 default rules plus automatic + dictionary-based segmentation selected by the script of the text. + SegmentedText returns the text unchanged (no braille separator insertion). + """ + + def getSegmentForOffset(self, offset: int) -> tuple[int, int] | None: + from textUtils import icu + + if self.encoding == textUtils.WCHAR_ENCODING: + return icu.calculateWordOffsets(self.text, offset) + # Convert the str offset to a UTF-16 offset for ICU, then convert the result back. + offsetConverter = textUtils.WideStringOffsetConverter(self.text) + wideOffset = offsetConverter.strToEncodedOffsets(offset, offset)[0] + result = icu.calculateWordOffsets(self.text, wideOffset) + if result is None: + return None + return offsetConverter.encodedToStrOffsets(*result) diff --git a/source/textUtils/_wordSeg/wordSegmenter.py b/source/textUtils/_wordSeg/wordSegmenter.py index 745af441e2e..2a0c0ba7579 100644 --- a/source/textUtils/_wordSeg/wordSegmenter.py +++ b/source/textUtils/_wordSeg/wordSegmenter.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2025-2026 NV Access Limited, Wang Chong +# Copyright (C) 2025-2026 NV Access Limited, Wang Chong, Leonard de Ruijter # This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. # For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt @@ -10,6 +10,7 @@ from ..segFlag import WordSegFlag from . import wordSegStrategy +from winBindings.icu import ICU_AVAILABLE _GET_SEGMENT_RECOVERABLE_EXCEPTIONS = ( @@ -43,29 +44,34 @@ def __init__( def _chooseStrategy( self, ) -> wordSegStrategy.WordSegmentationStrategy: - """Choose the appropriate segmentation strategy based on the text content.""" - if self.wordSegFlag == WordSegFlag.AUTO: - if ( - wordSegStrategy.ChineseWordSegmentationStrategy._lib - and WordSegmenter._CHINESE_CHARACTER_AND_JAPANESE_KANJI.search( - self.text, - ) + """Choose the segmentation strategy, falling back Chinese -> ICU -> Uniscribe. + + The CHINESE flag always uses the Chinese strategy when cppjieba is loaded; under + AUTO the Chinese strategy is used only for Chinese (non-kana) text. ICU is used + for AUTO and ICU, and as the fallback when cppjieba is unavailable: it follows + UAX#29 plus script-driven dictionary segmentation, handling complex scripts that + Uniscribe breaks poorly. Uniscribe is the final fallback and the only strategy + for the UNISCRIBE flag (it stays pinned where it is strictly required, e.g. + EditTextInfo, to match the Windows edit control / Notepad). + """ + flag = self.wordSegFlag + # Chinese: always for the CHINESE flag, or under AUTO for Chinese (non-kana) text. + if ( + flag in (WordSegFlag.AUTO, WordSegFlag.CHINESE) + and wordSegStrategy.ChineseWordSegmentationStrategy._lib + ): + if flag == WordSegFlag.CHINESE or ( + WordSegmenter._CHINESE_CHARACTER_AND_JAPANESE_KANJI.search(self.text) and not WordSegmenter._KANA.search(self.text) ): return wordSegStrategy.ChineseWordSegmentationStrategy(self.text, self.encoding) - return wordSegStrategy.UniscribeWordSegmentationStrategy(self.text, self.encoding) - match self.wordSegFlag: - case WordSegFlag.UNISCRIBE: - return wordSegStrategy.UniscribeWordSegmentationStrategy(self.text, self.encoding) - case WordSegFlag.CHINESE: - if wordSegStrategy.ChineseWordSegmentationStrategy._lib: - return wordSegStrategy.ChineseWordSegmentationStrategy(self.text, self.encoding) - log.debugWarning( - "Chinese word segmenter is currently unavailable. Falling back to Uniscribe.", - ) - return wordSegStrategy.UniscribeWordSegmentationStrategy(self.text, self.encoding) - case _: - pass + elif flag == WordSegFlag.CHINESE: + log.debugWarning("Chinese word segmenter is unavailable. Falling back to ICU/Uniscribe.") + # ICU for everything except the explicit UNISCRIBE flag. + if flag != WordSegFlag.UNISCRIBE and ICU_AVAILABLE: + return wordSegStrategy.IcuWordSegmentationStrategy(self.text, self.encoding) + elif flag == WordSegFlag.ICU: + log.debugWarning("ICU word segmenter is unavailable. Falling back to Uniscribe.") return wordSegStrategy.UniscribeWordSegmentationStrategy(self.text, self.encoding) def getSegmentForOffset(self, offset: int) -> tuple[int, int] | None: diff --git a/source/textUtils/icu.py b/source/textUtils/icu.py new file mode 100644 index 00000000000..dc8e2344a8c --- /dev/null +++ b/source/textUtils/icu.py @@ -0,0 +1,115 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2026 NV Access Limited, Leonard de Ruijter +# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. +# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt + +"""ICU-based text boundary utilities using the Windows built-in ICU library. + +Requires Windows 10 version 1703 (Creators Update) or later. +""" + +import ctypes +from contextlib import contextmanager + +import winBindings.icu as icu +from logHandler import log + +_ROOT_LOCALE: bytes = b"" +"""ICU root locale. Word and character segmentation are script-driven, not +locale-driven (see calculateWordOffsets), so the root locale is always used. +""" + + +@contextmanager +def _breakIterator(kind: int, locale: bytes, buf: ctypes.Array[ctypes.c_wchar]): + """Context manager that opens an ICU BreakIterator, yields it, then closes it. + + The caller owns the text buffer and must keep it alive for the duration of the + block, satisfying ICU's requirement that the text pointer remains valid while + the iterator is in use. + + :param kind: One of the UBRK members from winBindings.icu. + :param locale: ICU locale byte string (the root locale, _ROOT_LOCALE). + :param buf: NUL-terminated UTF-16 buffer (ctypes.create_unicode_buffer) to analyze. + :raises RuntimeError: If ICU reports an error opening the iterator. + """ + textLength = len(buf) - 1 + status = icu.UErrorCode(0) + bi = icu.ubrk_open(kind, locale, buf, textLength, ctypes.byref(status)) + if icu.U_FAILURE(status.value) or not bi: + raise RuntimeError(f"ubrk_open failed with status {status.value}") + try: + yield bi + finally: + icu.ubrk_close(bi) + + +def calculateWordOffsets( + text: str, + offset: int, +) -> tuple[int, int] | None: + """Calculate the UTF-16 start and end offsets of the word at the given offset. + + Word boundaries follow Unicode Standard Annex #29 default rules plus automatic + dictionary-based segmentation for scripts such as Thai, Lao, Khmer, and CJK + ideographs. ICU selects the dictionary by the script of the characters, not by + the locale, so no language is passed: any locale (including unrecognised codes) + would yield identical word boundaries and ICU never errors on an unknown locale + (it silently falls back to the root locale). The root locale is therefore used + unconditionally. (Locale-sensitive break types such as line and sentence + breaking would need a locale, but those are not used here.) + + Trailing whitespace is included in the preceding word segment, matching the + behaviour of NVDA's Uniscribe implementation (textUtils.cpp). When the offset + falls inside a whitespace run, the returned segment is the preceding word plus + the whitespace. + + Note: ICU coalesces a run of identical whitespace into one segment but splits + mixed whitespace (e.g. space + tab) into separate segments, so a mixed run is + not merged into a single word. This is not worth special-casing: the legacy + Uniscribe/Notepad behaviour for mixed whitespace runs is itself inconsistent. + + :param text: The line text as a Python str. + :param offset: UTF-16 code unit offset within text at which to find the boundary. + :return: (startOffset, endOffset) as UTF-16 code unit indices (endOffset exclusive), + or None if the ICU call failed. + """ + # A c_wchar buffer is UTF-16 code-unit indexed on Windows, so buf[a:b] is exactly + # the segment ICU's offsets refer to (lone surrogates decode as non-space). + buf = ctypes.create_unicode_buffer(text) + textLength = len(buf) - 1 + if offset >= textLength: + return (offset, offset + 1) + + try: + with _breakIterator(icu.UBRK.WORD, _ROOT_LOCALE, buf) as bi: + # Find [start, end) — the ICU segment containing offset. + # ICU offsets are UTF-16 code-unit indexed, so anchor on the boundary following + # offset and take the boundary preceding that. (ubrk_preceding(offset + 1) + # would snap back for multi-unit segments.) + end = icu.ubrk_following(bi, offset) + if end == icu.UBRK_DONE: + end = textLength + start = icu.ubrk_preceding(bi, end) + if start == icu.UBRK_DONE: + start = 0 + + if buf[start:end].isspace(): + # Offset is inside a whitespace run. Attach this run to the + # preceding segment (mirroring the Uniscribe trailing-space rule). + if start > 0: + wordStart = icu.ubrk_preceding(bi, start) + if wordStart == icu.UBRK_DONE: + wordStart = 0 + return (wordStart, end) + else: + # Offset is inside a word/punctuation segment. Extend the end + # through any immediately following whitespace run. + nextEnd = icu.ubrk_following(bi, end) + if nextEnd != icu.UBRK_DONE and buf[end:nextEnd].isspace(): + return (start, nextEnd) + + return (start, end) + except RuntimeError: + log.debugWarning("ICU word break iterator failed", exc_info=True) + return None diff --git a/source/textUtils/segFlag.py b/source/textUtils/segFlag.py index e996c244211..ceb42757fb9 100644 --- a/source/textUtils/segFlag.py +++ b/source/textUtils/segFlag.py @@ -1,5 +1,5 @@ # A part of NonVisual Desktop Access (NVDA) -# Copyright (C) 2025 NV Access Limited, Wang Chong +# Copyright (C) 2025-2026 NV Access Limited, Wang Chong, Leonard de Ruijter # This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. # For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt @@ -9,6 +9,7 @@ _AUTO: int = 1 << 0 _UNISCRIBE: int = 1 << 1 _CHINESE: int = 1 << 2 +_ICU: int = 1 << 3 class CharSegFlag(IntFlag): @@ -26,3 +27,4 @@ class WordSegFlag(IntFlag): AUTO = _AUTO UNISCRIBE = _UNISCRIBE CHINESE = _CHINESE + ICU = _ICU diff --git a/source/winBindings/icu.py b/source/winBindings/icu.py new file mode 100644 index 00000000000..b6338e5a8b7 --- /dev/null +++ b/source/winBindings/icu.py @@ -0,0 +1,108 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2026 NV Access Limited, Leonard de Ruijter +# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. +# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt + +"""ctypes bindings for the Windows built-in ICU library. + +ICU has been built into Windows since Windows 10 version 1703 (Creators Update). +The combined icu.dll is available from Windows 10 version 1903 (May 2019 Update). +Only the C APIs are exposed; no C++ APIs are available due to ABI instability. + +The ``ubrk_*`` function bindings are only defined when :data:`ICU_AVAILABLE` is True. + +.. seealso:: + https://learn.microsoft.com/windows/win32/intl/international-components-for-unicode--icu- +""" + +from ctypes import ( + WINFUNCTYPE, + windll, + c_int32, + c_void_p, + c_char_p, + c_wchar_p, + POINTER, +) +from enum import IntEnum + +# Load the combined icu.dll (Windows 10 1903+) first, then icuuc.dll (Windows 10 1703+). +# ubrk_* functions are part of the "common" library, present in both. +try: + dll = windll.icu +except OSError: + try: + dll = windll.icuuc + except OSError: + dll = None + +ICU_AVAILABLE: bool = dll is not None +"""True if an ICU library was successfully loaded.""" + +UBRK_DONE: int = -1 +"""Returned by iterator functions when there are no more boundaries.""" + +UErrorCode = c_int32 +"""Signed 32-bit integer error code. U_ZERO_ERROR = 0; positive values indicate errors.""" + + +class UBRK(IntEnum): + """The possible types of text boundaries (UBreakIteratorType).""" + + WORD = 1 + """Word breaks.""" + + +def U_FAILURE(code: int) -> bool: + """Return True if the given UErrorCode indicates an error.""" + return code > 0 + + +if ICU_AVAILABLE: + ubrk_open = WINFUNCTYPE(None)(("ubrk_open", dll)) + """Create a new break iterator. + + :param kind: UBreakIteratorType (one of the UBRK members). + :param locale: Null-terminated UTF-8 locale ID, or NULL/empty for the root locale. + :param text: UTF-16 text to analyze. + :param textLength: Number of UTF-16 code units, or -1 for NUL-terminated. + :param status: In/out UErrorCode; pass a pointer to a zero-initialised value. + :return: Opaque UBreakIterator* handle; must be freed with ubrk_close. + """ + ubrk_open.argtypes = ( + c_int32, # kind: UBreakIteratorType + c_char_p, # locale: UTF-8 locale ID or NULL + c_wchar_p, # text: UTF-16 text to analyze + c_int32, # textLength: code units, or -1 for NUL-terminated + POINTER(UErrorCode), # status: in/out error code + ) + ubrk_open.restype = c_void_p + + ubrk_close = WINFUNCTYPE(None)(("ubrk_close", dll)) + """Free a break iterator created by ubrk_open.""" + ubrk_close.argtypes = ( + c_void_p, # bi: UBreakIterator* handle to free + ) + ubrk_close.restype = None + + ubrk_preceding = WINFUNCTYPE(None)(("ubrk_preceding", dll)) + """Return the largest boundary position strictly less than offset. + + Sets the iterator to that position. + """ + ubrk_preceding.argtypes = ( + c_void_p, # bi: UBreakIterator* handle + c_int32, # offset: position to search before + ) + ubrk_preceding.restype = c_int32 + + ubrk_following = WINFUNCTYPE(None)(("ubrk_following", dll)) + """Return the smallest boundary position strictly greater than offset. + + Sets the iterator to that position. Returns UBRK_DONE if past the end. + """ + ubrk_following.argtypes = ( + c_void_p, # bi: UBreakIterator* handle + c_int32, # offset: position to search after + ) + ubrk_following.restype = c_int32 diff --git a/tests/system/robot/chromeTests.py b/tests/system/robot/chromeTests.py index 0a223521dfd..c9c04690bd7 100644 --- a/tests/system/robot/chromeTests.py +++ b/tests/system/robot/chromeTests.py @@ -1184,9 +1184,11 @@ def test_ariaRoleDescription_inline_browseMode(): "Unlabeled graphic Our", ) actualSpeech = _chrome.getSpeechAfterKey("control+rightArrow") + # ICU word segmentation (the AUTO default) follows UAX#29, which treats the + # trailing period as its own word segment, so it is no longer read with "logo". _asserts.strings_match( actualSpeech, - "logo.", + "logo", ) diff --git a/tests/unit/test_textUtils_backendComparison.py b/tests/unit/test_textUtils_backendComparison.py new file mode 100644 index 00000000000..1c92d58b0a1 --- /dev/null +++ b/tests/unit/test_textUtils_backendComparison.py @@ -0,0 +1,222 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2026 NV Access Limited, Leonard de Ruijter +# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. +# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt + +"""Comparison tests between the Uniscribe and ICU word boundary backends. + +These tests document where the two backends agree and where they diverge, +using the same inputs on both sides. Tests that require ICU are skipped +when the ICU library is not present on the system. + +Word-offset comparisons are done by constructing a WordSegmenter with the +appropriate WordSegFlag and calling getSegmentForOffset. +""" + +import unittest + +import textUtils +from winBindings.icu import ICU_AVAILABLE +from textUtils._wordSeg.wordSegmenter import WordSegmenter +from textUtils.segFlag import WordSegFlag + + +skipIfNoICU = unittest.skipUnless(ICU_AVAILABLE, "ICU library not available on this system") + +# Encoding used for all WordSegmenter calls — matches what NVDA uses internally. +_ENCODING = textUtils.WCHAR_ENCODING + + +def _icuWordOffsets(text: str, offset: int) -> tuple[int, int] | None: + """Get word offsets via the ICU backend (UTF-16 offsets).""" + return WordSegmenter(text, _ENCODING, WordSegFlag.ICU).getSegmentForOffset(offset) + + +def _uniscribeWordOffsets(text: str, offset: int) -> tuple[int, int] | None: + """Get word offsets via the Uniscribe backend (UTF-16 offsets).""" + return WordSegmenter(text, _ENCODING, WordSegFlag.UNISCRIBE).getSegmentForOffset(offset) + + +class _WordOffsetsParityTest(unittest.TestCase): + """Base for per-script word offset parity tests. + + Subclasses set TEXT and add test_* methods that assert the exact span via + _assertSameWordOffsets. Has no test methods of its own, so the loader runs nothing. + """ + + TEXT: str + + def _assertSameWordOffsets(self, offset: int) -> tuple[int, int] | None: + """Assert both backends return the same word offsets for self.TEXT at offset. + + :param offset: UTF-16 code unit offset within self.TEXT to query. + :return: The (start, end) offsets, so callers can additionally assert the exact span. + :raises AssertionError: If the ICU and Uniscribe backends disagree. + """ + icu_result = _icuWordOffsets(self.TEXT, offset) + uni_result = _uniscribeWordOffsets(self.TEXT, offset) + self.assertEqual( + icu_result, + uni_result, + f"Backends disagree on word offsets for {self.TEXT!r} at offset {offset}: " + f"ICU={icu_result!r} Uniscribe={uni_result!r}", + ) + return icu_result + + +@skipIfNoICU +class TestWordOffsetsEnglish(_WordOffsetsParityTest): + """Word offset comparison for English text. + + Both backends include trailing whitespace as part of the preceding word. + NVDA's Uniscribe implementation (textUtils.cpp) does this natively; + the ICU implementation mirrors that behaviour explicitly. + """ + + TEXT = "hello world" + + def test_first_word(self): + """Both backends: "hello " — trailing space included.""" + result = self._assertSameWordOffsets(0) + self.assertEqual(result, (0, 6)) + + def test_mid_first_word(self): + result = self._assertSameWordOffsets(2) + self.assertEqual(result, (0, 6)) + + def test_space(self): + """Both backends: querying at the space returns the preceding word+space.""" + result = self._assertSameWordOffsets(5) + self.assertEqual(result, (0, 6)) + + def test_second_word(self): + """Both backends: "world" — no trailing space at end of string.""" + result = self._assertSameWordOffsets(6) + self.assertEqual(result, (6, 11)) + + def test_mid_second_word(self): + result = self._assertSameWordOffsets(8) + self.assertEqual(result, (6, 11)) + + +@skipIfNoICU +class TestWordOffsetsHebrew(_WordOffsetsParityTest): + """Word offset comparison for vocalized (Biblical) Hebrew — שָׁלוֹם עוֹלָם (peace, world). + + The niqqud (combining vowel and shin points) attach to their base letters, so each + word stays a single segment: שָׁלוֹם is 7 UTF-16 code units, עוֹלָם is 6. + """ + + TEXT = "שָׁלוֹם עוֹלָם" + + def test_first_word(self): + """Both backends: "שָׁלוֹם " — trailing space included, offsets (0, 8).""" + result = self._assertSameWordOffsets(0) + self.assertEqual(result, (0, 8)) + + def test_mid_first_word(self): + result = self._assertSameWordOffsets(2) + self.assertEqual(result, (0, 8)) + + def test_space(self): + """Both backends: querying at offset 7 (space) returns the preceding word+space.""" + result = self._assertSameWordOffsets(7) + self.assertEqual(result, (0, 8)) + + def test_second_word(self): + """Both backends: "עוֹלָם" — no trailing space.""" + result = self._assertSameWordOffsets(8) + self.assertEqual(result, (8, 14)) + + +@skipIfNoICU +class TestWordOffsetsComplexScriptDivergence(unittest.TestCase): + """Japanese and Khmer: Uniscribe falls back to character/cluster level; ICU groups words.""" + + def test_japanese_uniscribe_is_character_level(self): + """Japanese "これは日本語です": Uniscribe returns single code points; ICU groups words.""" + text = "これは日本語です" + # Offset 0 ("こ"): ICU groups "これ"; Uniscribe returns just "こ". + icu_result = _icuWordOffsets(text, 0) + uni_result = _uniscribeWordOffsets(text, 0) + self.assertNotEqual(icu_result, uni_result) + # Uniscribe is character-level here (single code point). + self.assertEqual(uni_result[1] - uni_result[0], 1) + # ICU groups more than one code point into a word. + self.assertGreater(icu_result[1] - icu_result[0], 1) + + def test_khmer_uniscribe_breaks_clusters(self): + """Khmer "ខ្ញុំស្រលាញ់": Uniscribe breaks the second word into syllable clusters; ICU keeps it whole.""" + text = "ខ្ញុំស្រលាញ់" + # Offset 5 is the start of the second Khmer word "ស្រលាញ់". + icu_result = _icuWordOffsets(text, 5) + uni_result = _uniscribeWordOffsets(text, 5) + self.assertNotEqual(icu_result, uni_result) + # ICU spans the whole word; Uniscribe returns a shorter cluster. + self.assertGreater(icu_result[1] - icu_result[0], uni_result[1] - uni_result[0]) + + +@skipIfNoICU +class TestWordOffsetsEmojiZwjSequence(unittest.TestCase): + """Multi-person emoji ZWJ sequence with skin-tone modifiers. + + "👩🏻‍👧🏻‍👦🏻" (woman + girl + boy family, each with a light skin-tone modifier, + joined by ZERO WIDTH JOINER) is a single UAX#29 word. ICU treats the whole + sequence as one segment; Uniscribe falls back to grapheme/surrogate-level + boundaries and returns only the leading part. + + The sequence is 14 UTF-16 code units: + 👩 (2) 🏻 (2) ZWJ (1) 👧 (2) 🏻 (2) ZWJ (1) 👦 (2) 🏻 (2). + """ + + TEXT = "👩🏻‍👧🏻‍👦🏻" + # UTF-16 code-unit length of the whole sequence. + LENGTH = len(TEXT.encode("utf-16-le")) // 2 + + def test_length_is_as_expected(self): + """Guard the constant the assertions below rely on.""" + self.assertEqual(self.LENGTH, 14) + + def test_icu_groups_whole_sequence(self): + """ICU returns the entire ZWJ sequence as one word from offset 0.""" + self.assertEqual(_icuWordOffsets(self.TEXT, 0), (0, self.LENGTH)) + + def test_backends_diverge(self): + """Uniscribe does not group the whole sequence; ICU does.""" + icu_result = _icuWordOffsets(self.TEXT, 0) + uni_result = _uniscribeWordOffsets(self.TEXT, 0) + self.assertNotEqual(icu_result, uni_result) + # ICU spans the full sequence; Uniscribe returns a shorter leading run. + self.assertEqual(icu_result, (0, self.LENGTH)) + self.assertLess(uni_result[1] - uni_result[0], self.LENGTH) + + +@skipIfNoICU +class TestWordOffsetsTrailingPunctuationDivergence(unittest.TestCase): + """Trailing punctuation: ICU splits it into its own word; Uniscribe keeps it attached. + + UAX#29 treats the full stop as a separate word segment, so ICU returns "logo" + and then "." as two words. NVDA's Uniscribe implementation keeps the trailing + punctuation attached to the preceding word ("logo."). ICU's behaviour matches + modern Windows edit controls such as the Start menu search field. + """ + + TEXT = "logo." + + def test_icu_splits_word_from_punctuation(self): + """ICU: "logo" (0, 4) then "." (4, 5).""" + self.assertEqual(_icuWordOffsets(self.TEXT, 0), (0, 4)) + self.assertEqual(_icuWordOffsets(self.TEXT, 4), (4, 5)) + + def test_uniscribe_keeps_punctuation_attached(self): + """Uniscribe: "logo." kept whole (0, 5).""" + self.assertEqual(_uniscribeWordOffsets(self.TEXT, 0), (0, 5)) + + def test_backends_diverge(self): + """ICU stops before the full stop; Uniscribe includes it.""" + icu_result = _icuWordOffsets(self.TEXT, 0) + uni_result = _uniscribeWordOffsets(self.TEXT, 0) + self.assertNotEqual(icu_result, uni_result) + # ICU ends the word before the punctuation; Uniscribe runs through it. + self.assertEqual(icu_result, (0, 4)) + self.assertEqual(uni_result, (0, 5)) diff --git a/tests/unit/test_wordSegIcu.py b/tests/unit/test_wordSegIcu.py new file mode 100644 index 00000000000..0d0c30b6651 --- /dev/null +++ b/tests/unit/test_wordSegIcu.py @@ -0,0 +1,71 @@ +# A part of NonVisual Desktop Access (NVDA) +# Copyright (C) 2025-2026 NV Access Limited, Wang Chong, Leonard de Ruijter +# This file may be used under the terms of the GNU General Public License, version 2 or later, as modified by the NVDA license. +# For full terms and any additional permissions, see the NVDA license file: https://github.com/nvaccess/nvda/blob/master/copying.txt + +"""Unit tests for ICU word segmentation strategy.""" + +import unittest +from unittest.mock import patch + +from textUtils._wordSeg import wordSegStrategy + + +class TestIcuStrategy(unittest.TestCase): + def test_icu_strategy_getSegmentForOffset_calls_primitive(self): + text = "hello world" + with patch("textUtils.icu.calculateWordOffsets", return_value=(0, 6)) as mockCalc: + strat = wordSegStrategy.IcuWordSegmentationStrategy(text, None) + result = strat.getSegmentForOffset(2) + mockCalc.assert_called_once_with(text, 2) + self.assertEqual(result, (0, 6)) + + def test_icu_segmentedText_returns_text_unchanged(self): + strat = wordSegStrategy.IcuWordSegmentationStrategy("hello", None) + self.assertEqual(strat.segmentedText(), "hello") + + def test_explicit_icu_flag_selects_icu_when_available(self): + from textUtils._wordSeg import wordSegmenter + from textUtils.segFlag import WordSegFlag + + with patch.object(wordSegmenter, "ICU_AVAILABLE", True): + seg = wordSegmenter.WordSegmenter("hello", None, WordSegFlag.ICU) + self.assertIsInstance(seg.strategy, wordSegStrategy.IcuWordSegmentationStrategy) + + def test_explicit_icu_flag_falls_back_when_unavailable(self): + from textUtils._wordSeg import wordSegmenter + from textUtils.segFlag import WordSegFlag + + with patch.object(wordSegmenter, "ICU_AVAILABLE", False): + seg = wordSegmenter.WordSegmenter("hello", None, WordSegFlag.ICU) + self.assertIsInstance(seg.strategy, wordSegStrategy.UniscribeWordSegmentationStrategy) + + def test_auto_prefers_icu_for_latin_when_available(self): + from textUtils._wordSeg import wordSegmenter + from textUtils.segFlag import WordSegFlag + + with ( + patch.object(wordSegmenter, "ICU_AVAILABLE", True), + patch.object( + wordSegStrategy.ChineseWordSegmentationStrategy, + "_lib", + None, + ), + ): + seg = wordSegmenter.WordSegmenter("hello world", None, WordSegFlag.AUTO) + self.assertIsInstance(seg.strategy, wordSegStrategy.IcuWordSegmentationStrategy) + + def test_auto_falls_back_to_uniscribe_when_icu_unavailable(self): + from textUtils._wordSeg import wordSegmenter + from textUtils.segFlag import WordSegFlag + + with ( + patch.object(wordSegmenter, "ICU_AVAILABLE", False), + patch.object( + wordSegStrategy.ChineseWordSegmentationStrategy, + "_lib", + None, + ), + ): + seg = wordSegmenter.WordSegmenter("hello world", None, WordSegFlag.AUTO) + self.assertIsInstance(seg.strategy, wordSegStrategy.UniscribeWordSegmentationStrategy) diff --git a/user_docs/en/changes.md b/user_docs/en/changes.md index 2cb07f65701..51b40725b86 100644 --- a/user_docs/en/changes.md +++ b/user_docs/en/changes.md @@ -8,7 +8,9 @@ * Add-ons can be removed from the "Updatable add-ons" tab in the Add-on Store. (#15030, @nvdaes) * Chinese text can now be navigated by word using built-in input gestures. - A Word Segmentation Standard setting was added to the "Document Navigation" panel. (#18735, @CrazySteve0605, @Cary-rowen) + * A Word Segmentation Standard setting was added to the "Document Navigation" panel. (#18735, @CrazySteve0605, @Cary-rowen) + * Word segmentation can also use the Windows built-in ICU library for boundary detection, improving navigation for Japanese and emoji. (#20343, @LeonarddeR) + * By default, ICU is preferred over the legacy Windows segmentation wherever available, while Chinese word segmentation takes precedence for Chinese text. * Braille output for Chinese now includes spaces between words. (#18865, @CrazySteve0605, @Cary-rowen) * Added sequential two-flick touch gestures that combine two flicks performed in quick succession into a single gesture, increasing the number of touch gestures that can be bound to scripts. (#19938, @kefaslungu) * Twelve combinations are recognised: opposite-direction pairs (e.g. flick right then flick left) and perpendicular L-shaped pairs (e.g. flick right then flick up). diff --git a/user_docs/en/userGuide.md b/user_docs/en/userGuide.md index 6b52fbcd165..68391227d3a 100644 --- a/user_docs/en/userGuide.md +++ b/user_docs/en/userGuide.md @@ -3567,14 +3567,15 @@ When a Chinese braille output table is in use, NVDA can insert spaces between Ch | . {.hideHeaderRow} |.| |---|---| -| Options | Default (Auto), Auto, Standard, Chinese | -| Default | Auto | +| Options | Default (Automatic), Automatic, Chinese, Unicode (ICU), Legacy (Uniscribe) | +| Default | Automatic | | Option | Behaviour | |---|---| -| Auto | Use Chinese word segmentation for Chinese text when available. For other text, use standard word segmentation. | -| Standard | Use standard Windows word segmentation. | -| Chinese | Use Chinese word segmentation. If Chinese word segmentation is not available, NVDA falls back to standard word segmentation. | +| Automatic | Use Chinese word segmentation for Chinese text when available. Otherwise, prefer Unicode (ICU) word segmentation when available, falling back to the legacy Windows word segmentation. | +| Chinese | Use Chinese word segmentation. If Chinese word segmentation is not available, NVDA uses the fallback path as in `Automatic`. | +| Unicode (ICU) | Use the Windows built-in ICU library for word segmentation available in Windows 10 version 1703 (Creators Update) and later, falling back to legacy. | +| Legacy (Uniscribe) | Use the legacy Windows word segmentation as used in Notepad classic and other legacy Win32 edit controls. | #### Math Settings {#MathSettings}