Add visual highlighting for web math navigation

source/NVDAObjects/IAccessible/ia2Web.py

@@ -8,6 +8,7 @@
 from typing import (
 	Generator,
 	Optional,
+	TYPE_CHECKING,
 	Tuple,
 )
 import re
@@ -35,6 +36,10 @@
 import config
 import NVDAObjects
 
+if TYPE_CHECKING:
+	from locationHelper import RectLTRB
+	from mathPres.mathMlNode import MathMlNodePath, MathMlNodeRectInfo
+
 
 class IA2WebAnnotationTarget(AnnotationTarget):
 	def __init__(self, target: IAccessible):
@@ -331,6 +336,73 @@ class EditorChunk(Ia2Web):
 
 
 class Math(Ia2Web):
+	def _getMathObjChildren(self, obj: NVDAObjects.NVDAObject) -> tuple[NVDAObjects.NVDAObject, ...]:
+		try:
+			return tuple(obj.children)
+		except RuntimeError:
+			log.debugWarning("Error fetching MathML node children", exc_info=True)
+			return ()
+
+	def _getMathObjAttributes(self, obj: NVDAObjects.NVDAObject) -> dict[str, str]:
+		try:
+			return obj.IA2Attributes
+		except AttributeError:
+			return {}
+
+	def _getMathElementChildren(self, obj: NVDAObjects.NVDAObject) -> tuple[NVDAObjects.NVDAObject, ...]:
+		return tuple(
+			child for child in self._getMathObjChildren(obj) if self._getMathObjAttributes(child).get("tag")
+		)
+
+	def _getMathNodeMapRoot(self) -> NVDAObjects.NVDAObject:
+		if self._getMathObjAttributes(self).get("tag") == "math":
+			return self
+		mathChildren = tuple(
+			child
+			for child in self._getMathElementChildren(self)
+			if self._getMathObjAttributes(child).get("tag") == "math"
+		)
+		return mathChildren[0] if len(mathChildren) == 1 else self
+
+	def _getMathNodeRectFromObj(self, obj: NVDAObjects.NVDAObject) -> "RectLTRB | None":
+		try:
+			if obj.hasIrrelevantLocation:
+				return None
+			location = obj.location
+		except Exception:
+			log.debugWarning("Error fetching MathML node location", exc_info=True)
+			return None
+		if not location or not location.width or not location.height:
+			return None
+		return location.toLTRB()
+
+	def getMathNodeInfoByPath(self) -> dict["MathMlNodePath", "MathMlNodeRectInfo"]:
+		"""Map MathML element paths to tag names and screen rectangles for this IA2 math subtree.
+
+		Paths are tuples where each entry indicates an index of a child node to be traversed from the root.
+		"""
+		from mathPres.mathMlNode import MathMlNodeRectInfo
+
+		nodeInfoByPath: dict["MathMlNodePath", "MathMlNodeRectInfo"] = {}
+		stack: list[tuple[NVDAObjects.NVDAObject, "MathMlNodePath"]] = [
+			(self._getMathNodeMapRoot(), ()),
+		]
+		visitedCount = 0
+		while stack:
+			obj, path = stack.pop()
+			visitedCount += 1
+			tag = self._getMathObjAttributes(obj).get("tag")
+			if rect := self._getMathNodeRectFromObj(obj):
+				if tag:
+					nodeInfoByPath[path] = MathMlNodeRectInfo(path=path, tag=tag, rect=rect)
+			children = self._getMathElementChildren(obj)
+			stack.extend((child, path + (index,)) for index, child in reversed(tuple(enumerate(children))))
+		log.debug(
+			f"Math highlight built IA2 path map with {len(nodeInfoByPath)} usable rectangles "
+			f"after visiting {visitedCount} MathML element objects",
+		)
+		return nodeInfoByPath
+
 	def _get_mathMl(self):
 		# Chromium browsers now expose a 'math' IAccessible2 attribute,
 		# which contains all the raw MathML.

source/browseMode.py

@@ -712,7 +712,7 @@ def _activatePosition(self, obj=None):
 			import mathPres
 
 			try:
-				return mathPres.interactWithMathMl(obj.mathMl)
+				return mathPres.interactWithMathMl(obj.mathMl, sourceObj=obj)
 			except (NotImplementedError, LookupError):
 				pass
 			return

source/globalCommands.py

@@ -4811,20 +4811,28 @@ def script_toggleConfigProfileTriggers(self, gesture):
 	def script_interactWithMath(self, gesture):
 		import mathPres
 
-		mathMl = mathPres.getMathMlFromTextInfo(api.getReviewPosition())
+		reviewPosition = api.getReviewPosition()
+		mathMl = mathPres.getMathMlFromTextInfo(reviewPosition)
+		sourceObj = None
 		if not mathMl:
 			obj = api.getNavigatorObject()
 			if obj.role == controlTypes.Role.MATH:
 				try:
 					mathMl = obj.mathMl
+					sourceObj = obj
 				except (NotImplementedError, LookupError):
 					mathMl = None
+		else:
+			try:
+				sourceObj = reviewPosition.NVDAObjectAtStart
+			except (NotImplementedError, LookupError):
+				pass
 		if not mathMl:
 			# Translators: Reported when the user attempts math interaction
 			# with something that isn't math.
 			ui.message(_("Not math"))
 			return
-		mathPres.interactWithMathMl(mathMl)
+		mathPres.interactWithMathMl(mathMl, sourceObj=sourceObj)
 
 	@script(
 		# Translators: Describes a command.

source/mathPres/MathCAT/MathCAT.py

@@ -12,7 +12,7 @@
 	windll,
 )
 from os import path
-from typing import Type
+from typing import TYPE_CHECKING, Type
 
 import braille
 import config
@@ -43,9 +43,18 @@
 import mathPres
 from .localization import getLanguageToUse
 from .navCommands import NAV_COMMANDS
+from .navNodeMapping import (
+	NAV_NODE_ID_PREFIX,
+	prepareMathMlForNavigation,
+	removeSyntheticIdsFromMathMl,
+)
 from .preferences import applyUserPreferences
 from .speech import convertSSMLTextForNVDA
 
+if TYPE_CHECKING:
+	from locationHelper import RectLTRB
+	from NVDAObjects import NVDAObject
+
 # Translators: The name of the category of MathCAT navigation commands in the Input Gestures dialog.
 SCRCAT_MATHCAT_NAV = _("MathCat navigation")
 
@@ -68,13 +77,19 @@ def __init__(
 		self,
 		provider: mathPres.MathPresentationProvider | None = None,
 		mathMl: str | None = None,
+		sourceObj: "NVDAObject | None" = None,
 	):
 		"""Initialize the MathCATInteraction object.
 
 		:param provider: Optional presentation provider.
 		:param mathMl: Optional initial MathML string.
+		:param sourceObj: Optional source object containing the math.
 		"""
-		super(MathCATInteraction, self).__init__(provider=provider, mathMl=mathMl)
+		super(MathCATInteraction, self).__init__(provider=provider, mathMl=mathMl, sourceObj=sourceObj)
+		self.mathMlForNavigation, self._mathNodeRectsById = prepareMathMlForNavigation(
+			mathMl or "<math></math>",
+			sourceObj,
+		)
 		if mathMl is None:
 			self.initMathML = "<math></math>"
 		else:
@@ -86,10 +101,65 @@ def reportFocus(self) -> None:
 		try:
 			text: str = libmathcat.DoNavigateCommand("ZoomIn")
 			speech.speak(convertSSMLTextForNVDA(text))
+			self._updateMathHighlight()
 		except Exception:
 			log.exception()
 			# Translators: this message reports an error in starting navigation of math.
 			ui.message(pgettext("math", "Error in starting navigation of math."))
+			self._clearMathHighlight()
+
+	def _getHighlightRect(self) -> "RectLTRB | None":
+		"""Get the navigation rectangle for a supported web math source object."""
+		sourceObj = self.sourceObj
+		if not sourceObj:
+			return None
+		from NVDAObjects.IAccessible.ia2Web import Math as Ia2WebMath
+
+		if not isinstance(sourceObj, Ia2WebMath):
+			return None
+		try:
+			nodeId = libmathcat.GetNavigationMathMLId()[0]
+		except Exception:
+			log.debugWarning("Error getting MathCAT navigation node id", exc_info=True)
+		else:
+			if nodeId in self._mathNodeRectsById:
+				return self._mathNodeRectsById[nodeId]
+			if nodeId.startswith(NAV_NODE_ID_PREFIX):
+				log.debug(
+					f"Math highlight found synthetic MathML id {nodeId!r}, but it has no mapped IA2 rectangle",
+				)
+			log.debug(f"Math highlight falling back to source rectangle for node id {nodeId!r}")
+		try:
+			if sourceObj.hasIrrelevantLocation:
+				return None
+			location = sourceObj.location
+		except Exception:
+			log.debugWarning("Error getting math source location", exc_info=True)
+			return None
+		return location.toLTRB() if location else None
+
+	def _updateMathHighlight(self) -> None:
+		try:
+			rect = self._getHighlightRect()
+		except Exception:
+			log.debugWarning("Error updating math highlight", exc_info=True)
+			rect = None
+		try:
+			import vision
+
+			if vision.handler:
+				vision.handler.handleMathNavigation(rect)
+		except Exception:
+			log.debugWarning("Error sending math highlight update", exc_info=True)
+
+	def _clearMathHighlight(self) -> None:
+		try:
+			import vision
+
+			if vision.handler:
+				vision.handler.handleMathNavigation(None)
+		except Exception:
+			log.debugWarning("Error clearing math highlight", exc_info=True)
 
 	def getBrailleRegions(
 		self,
@@ -117,10 +187,12 @@ def _doNavigateCommand(self, commandName: str) -> None:
 		try:
 			text = libmathcat.DoNavigateCommand(commandName)
 			speech.speak(convertSSMLTextForNVDA(text))
+			self._updateMathHighlight()
 		except Exception:
 			log.exception()
 			# Translators: this message alerts users to an error in navigating math.
 			ui.message(pgettext("math", "Error in navigating math"))
+			self._clearMathHighlight()
 
 		self._updateBraille()
 
@@ -197,7 +269,7 @@ def script_rawdataToClip(self, gesture: KeyboardInputGesture) -> None:
 					mathml = self.initMathML
 				if copyAs == "speech":
 					# save the old MathML, set the navigation MathML as MathMl, get the speech, then reset the MathML
-					savedMathML: str = self.initMathML
+					savedMathML: str = self.mathMlForNavigation
 					savedTTS: str = libmathcat.GetPreference("TTS")
 					if savedMathML == "":  # shouldn't happen
 						raise Exception("Internal error -- MathML not set for copy")
@@ -220,12 +292,11 @@ def script_rawdataToClip(self, gesture: KeyboardInputGesture) -> None:
 
 	# not a perfect match sequence, but should capture normal MathML
 	_mathTagHasNameSpace: re.Pattern = re.compile("<math .*?xmlns.+?>")
-	_hasAddedId: re.Pattern = re.compile(" id='[^'].+' data-id-added='true'")
-	_hasDataAttr: re.Pattern = re.compile(" data-[^=]+='[^']*'")
+	_hasDataAttr: re.Pattern = re.compile(r""" data-[^=]+=(['"]).*?\1""")
 
 	def _wrapMathMLForClipBoard(self, text: str) -> str:
 		"""Cleanup the MathML a little."""
-		text = re.sub(self._hasAddedId, "", text)
+		text = removeSyntheticIdsFromMathMl(text)
 		mathMLWithNS: str = re.sub(self._hasDataAttr, "", text)
 		if not re.match(self._mathTagHasNameSpace, mathMLWithNS):
 			mathMLWithNS = mathMLWithNS.replace(
@@ -297,6 +368,8 @@ def _setClipboardData(self, format: int, data: str) -> None:
 
 
 class MathCAT(mathPres.MathPresentationProvider):
+	supportsInteractionSourceObj: bool = True
+
 	def __init__(self):
 		"""Initializes MathCAT, loading the rules specified in the rules directory."""
 
@@ -398,11 +471,20 @@ def getBrailleForMathMl(self, mathml: str) -> str:
 			ui.message(pgettext("math", "Error in brailling math."))
 			return ""
 
-	def interactWithMathMl(self, mathml: str) -> None:
+	def interactWithMathMl(self, mathml: str, sourceObj: "NVDAObject | None" = None) -> None:
 		"""Interact with a MathML string, creating a MathCATInteraction object.
 
 		:param mathml: The MathML representing the math to interact with.
+		:param sourceObj: Optional source object containing the math.
 		"""
-		interaction = MathCATInteraction(provider=self, mathMl=mathml)
+		interaction = MathCATInteraction(provider=self, mathMl=mathml, sourceObj=sourceObj)
+		try:
+			libmathcat.SetMathML(interaction.mathMlForNavigation)
+		except Exception:
+			log.exception(f"MathML is {interaction.mathMlForNavigation}")
+			# Translators: this message reports illegal MathML.
+			ui.message(pgettext("math", "Invalid MathML found."))
+			libmathcat.SetMathML("<math></math>")
+			return
 		interaction.setFocus()
 		interaction._updateBraille()