Make SAX parsing consistent on all adapters

lib/moxml/adapter/base.rb

@@ -19,6 +19,15 @@ class Base
       SERIALIZED_ENTITY_MARKER_RE = /(#{ENTITY_NAME_PATTERN});/
       STANDARD_ENTITIES = %w[amp lt gt quot apos].freeze
 
+      NAMED_ENTITY_DECODE_MAP = {
+        "amp" => "&", "lt" => "<", "gt" => ">",
+        "quot" => '"', "apos" => "'"
+      }.freeze
+      private_constant :NAMED_ENTITY_DECODE_MAP
+
+      ENTITY_DECODE_RE = /&(?:(amp|lt|gt|quot|apos)|#(?:(\d+)|[xX]([0-9a-fA-F]+)));/.freeze
+      private_constant :ENTITY_DECODE_RE
+
       class << self
         include XmlUtils
 
@@ -53,6 +62,29 @@ def preprocess_entities(xml)
           end
         end
 
+        # Resolve numeric (&#NN; / &#xNN;) and the five standard named
+        # (&amp; &lt; &gt; &quot; &apos;) XML entity references in one
+        # single pass. The resulting characters are data; no further
+        # decoding is applied. Numeric refs producing invalid UTF-8
+        # (NUL, surrogate halves, > U+10FFFF) are preserved verbatim
+        # to avoid silently emitting malformed bytes.
+        def decode_entities(text)
+          return text unless text.is_a?(String) && text.include?("&")
+
+          text.gsub(ENTITY_DECODE_RE) do
+            if (named = ::Regexp.last_match(1))
+              NAMED_ENTITY_DECODE_MAP[named]
+            else
+              code = ::Regexp.last_match(2) ? ::Regexp.last_match(2).to_i : ::Regexp.last_match(3).to_i(16)
+              if code.zero? || code.between?(0xD800, 0xDFFF) || code > 0x10FFFF
+                ::Regexp.last_match(0)
+              else
+                [code].pack("U")
+              end
+            end
+          end
+        end
+
         # Restore entity markers back to named entity references.
         def restore_entities(text)
           return text unless text.is_a?(String)

lib/moxml/adapter/customized_libxml/cdata.rb

@@ -5,19 +5,10 @@ module Adapter
     module CustomizedLibxml
       # Wrapper for LibXML CDATA section nodes
       class Cdata < Node
-        # Serialize as XML CDATA section
-        # LibXML auto-escapes content, we need to un-escape it
+        # libxml stores CDATA payload verbatim. Only the `]]>` end-marker
+        # needs splitting before re-wrapping.
         def to_xml
-          content = @native.content
-            .gsub("&quot;", '"')
-            .gsub("&apos;", "'")
-            .gsub("&lt;", "<")
-            .gsub("&gt;", ">")
-            .gsub("&amp;", "&")
-
-          # Handle CDATA end marker escaping (]]> becomes ]]]]><![CDATA[>)
-          # Replace all ]]> markers in the content before wrapping
-          escaped_content = content.gsub("]]>", "]]]]><![CDATA[>")
+          escaped_content = @native.content.gsub("]]>", "]]]]><![CDATA[>")
           "<![CDATA[#{escaped_content}]]>"
         end
       end

lib/moxml/adapter/customized_libxml/comment.rb

@@ -5,16 +5,9 @@ module Adapter
     module CustomizedLibxml
       # Wrapper for LibXML comment nodes
       class Comment < Node
-        # Serialize as XML comment
-        # LibXML auto-escapes content, we need to un-escape it
+        # libxml stores comment payload verbatim.
         def to_xml
-          content = @native.content
-            .gsub("&quot;", '"')
-            .gsub("&apos;", "'")
-            .gsub("&lt;", "<")
-            .gsub("&gt;", ">")
-            .gsub("&amp;", "&")
-          "<!--#{content}-->"
+          "<!--#{@native.content}-->"
         end
       end
     end

lib/moxml/adapter/customized_libxml/node.rb

@@ -18,6 +18,14 @@ def initialize(native_node)
           @native = native_node
         end
 
+        # Swap the wrapped native node. Used by the Libxml adapter when
+        # libxml-ruby's content= setter would silently re-escape stored
+        # text; replacing the node with a fresh raw-storage instance is
+        # the only way to preserve verbatim content.
+        def replace_native!(fresh)
+          @native = fresh
+        end
+
         # Compare wrappers based on their native nodes
         def ==(other)
           return false unless other

lib/moxml/adapter/customized_libxml/processing_instruction.rb

@@ -5,20 +5,12 @@ module Adapter
     module CustomizedLibxml
       # Wrapper for LibXML processing instruction nodes
       class ProcessingInstruction < Node
-        # Serialize as XML processing instruction
-        # LibXML auto-escapes content, we need to un-escape it
+        # XML 1.0 §2.6: PI content is verbatim — no entity resolution, no escaping.
         def to_xml
           target = @native.name
           content = @native.content
-
-          # Un-escape LibXML's automatic escaping
           if content && !content.empty?
-            unescaped = content.gsub("&quot;", '"')
-              .gsub("&apos;", "'")
-              .gsub("&lt;", "<")
-              .gsub("&gt;", ">")
-              .gsub("&amp;", "&")
-            "<?#{target} #{unescaped}?>"
+            "<?#{target} #{content}?>"
           else
             "<?#{target}?>"
           end

lib/moxml/adapter/customized_libxml/text.rb

@@ -13,16 +13,9 @@ def text
           @native.content
         end
 
-        # Serialize as XML with proper escaping
-        # LibXML's .content already contains escaped text, but it over-escapes
-        # quotes which don't need escaping in text nodes (only in attributes)
+        # @native.to_s escapes & < > but leaves quotes alone, which text nodes need.
         def to_xml
-          content = @native.content
-          # Skip the gsub allocation entirely when there's nothing to undo —
-          # the common case for parsed text without literal quotes.
-          return content unless content.include?("&quot;")
-
-          content.gsub("&quot;", '"')
+          @native.to_s
         end
       end
     end

lib/moxml/adapter/customized_oga.rb

@@ -3,6 +3,8 @@
 module Moxml
   module Adapter
     module CustomizedOga
+      autoload :EntityDecoderOverride, "moxml/adapter/customized_oga/entity_decoder"
+      autoload :RawValueOverride, "moxml/adapter/customized_oga/raw_value_override"
       autoload :XmlDeclaration, "moxml/adapter/customized_oga/xml_declaration"
       autoload :XmlGenerator, "moxml/adapter/customized_oga/xml_generator"
     end

lib/moxml/adapter/customized_oga/entity_decoder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require "oga"
+
+module Moxml
+  module Adapter
+    module CustomizedOga
+      # Override Oga::EntityDecoder.decode for XML inputs.
+      #
+      # Oga 3.4's stock Oga::XML::Entities.decode runs multiple passes,
+      # turning the well-formed "&" into "&" rather than the
+      # spec-correct "&". XML 1.0 §4.6 forbids recursive resolution
+      # of parsed entities — exactly one decode pass is correct.
+      #
+      # Moxml::Adapter::Base.decode_entities does the single pass. HTML
+      # inputs fall through to Oga's stock HTML::Entities.decode, which
+      # has HTML-specific legacy rules that differ from XML's.
+      #
+      # Prepending on the singleton class means every Oga reader
+      # (Text#text, Attribute#value, etc.) automatically picks up the
+      # fixed decoder, so the adapter no longer needs to walk the parsed
+      # tree rewriting @text/@value ivars behind Oga's back.
+      module EntityDecoderOverride
+        # rubocop:disable Style/OptionalBooleanParameter -- must match Oga's signature
+        def decode(input, html = false)
+          return super if html
+
+          ::Moxml::Adapter::Base.decode_entities(input)
+        end
+        # rubocop:enable Style/OptionalBooleanParameter
+      end
+    end
+  end
+end
+
+Oga::EntityDecoder.singleton_class.prepend(
+  Moxml::Adapter::CustomizedOga::EntityDecoderOverride,
+)

lib/moxml/adapter/customized_oga/raw_value_override.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require "oga"
+
+module Moxml
+  module Adapter
+    module CustomizedOga
+      # Bypass Oga's lazy decoder for user-authored values.
+      #
+      # Oga::XML::Attribute#value and Oga::XML::Text#text decode `@value` /
+      # `@text` on first read via Oga::EntityDecoder. EntityDecoderOverride
+      # makes that decode single-pass for parsed input. But user-authored
+      # values are already data — running them through the decoder again
+      # would re-interpret any literal `&` / `&#NN;` substrings as entity
+      # references. So we route the read through the NativeAttachment
+      # sidecar first; if the adapter stored the verbatim value there, we
+      # return it as-is, otherwise we fall through to Oga's normal read.
+      module RawValueOverride
+        def value
+          cached = ::Moxml::Adapter::Oga.attachments.get(self, :raw_value)
+          return cached unless cached.nil?
+
+          super
+        end
+
+        def value=(new_value)
+          ::Moxml::Adapter::Oga.attachments.set(self, :raw_value, new_value)
+          super
+        end
+
+        def text
+          cached = ::Moxml::Adapter::Oga.attachments.get(self, :raw_text)
+          return cached unless cached.nil?
+
+          super
+        end
+
+        def text=(new_value)
+          ::Moxml::Adapter::Oga.attachments.set(self, :raw_text, new_value)
+          super
+        end
+      end
+    end
+  end
+end
+
+Oga::XML::Attribute.prepend(
+  Moxml::Adapter::CustomizedOga::RawValueOverride,
+)
+Oga::XML::Text.prepend(
+  Moxml::Adapter::CustomizedOga::RawValueOverride,
+)

lib/moxml/adapter/libxml.rb

@@ -91,7 +91,7 @@ def parse(xml, options = {}, _context = nil)
           xml_string = preprocess_entities(xml_string)
 
           # Extract DOCTYPE before parsing
-          doctype_match = xml_string.match(/<!DOCTYPE\s+(\S+)(?:\s+PUBLIC\s+"([^"]+)"\s+"([^"]+)"|  \s+SYSTEM\s+"([^"]+)")?\s*>/i)
+          doctype_match = xml_string.match(/<!DOCTYPE\s+([^\s>]+)(?:\s+PUBLIC\s+"([^"]+)"\s+"([^"]+)"|\s+SYSTEM\s+"([^"]+)")?\s*>/i)
 
           native_doc = begin
             # Handle both string and file inputs
@@ -730,26 +730,52 @@ def inner_text(node)
 
         def set_text_content(node, content)
           native_node = unpatch_node(node)
-          native_node.content = content.to_s if native_node
+          return unless native_node.is_a?(::LibXML::XML::Node)
+
+          # Text wrapper: swap with new_text to preserve verbatim storage.
+          # Element wrapper: let libxml manage the child text node.
+          if native_node.text?
+            replace_native_verbatim(node, ::LibXML::XML::Node.new_text(content.to_s))
+          else
+            native_node.content = content.to_s
+          end
         end
 
-        def cdata_content(node)
-          native_node = unpatch_node(node)
-          content = native_node&.content
-          # LibXML may HTML-escape CDATA content, un-escape it
-          return nil unless content
+        # Replace `old_native` with `fresh`, preserving position. Both
+        # neighbor pointers are set explicitly to match `replace`'s
+        # semantics (see #replace above).
+        def swap_native_in_place(old_native, fresh)
+          parent = old_native.parent
+          return if parent.nil?
 
-          content.gsub("&quot;", '"')
-            .gsub("&apos;", "'")
-            .gsub("&lt;", "<")
-            .gsub("&gt;", ">")
-            .gsub("&amp;", "&")
+          prev_sibling = old_native.prev
+          next_sibling = old_native.next
+          old_native.remove!
+          prev_sibling.next = fresh if prev_sibling
+          next_sibling.prev = fresh if next_sibling
+          parent << fresh unless prev_sibling || next_sibling
+        end
+        private :swap_native_in_place
+
+        # libxml-ruby's node.content= pre-escapes; building a fresh node via
+        # new_<kind> stores content verbatim. Swap in place and re-point the
+        # wrapper.
+        def replace_native_verbatim(node, fresh)
+          native = unpatch_node(node)
+          return unless native.is_a?(::LibXML::XML::Node)
+
+          swap_native_in_place(native, fresh)
+          node.replace_native!(fresh) if node.is_a?(CustomizedLibxml::Node)
+        end
+        private :replace_native_verbatim
+
+        def cdata_content(node)
+          # libxml stores CDATA payload verbatim; no decoding needed.
+          unpatch_node(node)&.content
         end
 
         def set_cdata_content(node, content)
-          native_node = unpatch_node(node)
-          # CDATA content should NOT be escaped
-          native_node.content = content.to_s if native_node
+          replace_native_verbatim(node, ::LibXML::XML::Node.new_cdata(content.to_s))
         end
 
         def comment_content(node)
@@ -758,8 +784,7 @@ def comment_content(node)
         end
 
         def set_comment_content(node, content)
-          native_node = unpatch_node(node)
-          native_node.content = content.to_s if native_node
+          replace_native_verbatim(node, ::LibXML::XML::Node.new_comment(content.to_s))
         end
 
         def processing_instruction_target(node)
@@ -768,22 +793,18 @@ def processing_instruction_target(node)
         end
 
         def processing_instruction_content(node)
-          native_node = unpatch_node(node)
-          content = native_node&.content
-          # LibXML may HTML-escape the content, un-escape it
-          return nil unless content
-
-          content.gsub("&quot;", '"')
-            .gsub("&apos;", "'")
-            .gsub("&lt;", "<")
-            .gsub("&gt;", ">")
-            .gsub("&amp;", "&")
+          # XML 1.0 §2.6: PI content is verbatim — no entity resolution.
+          unpatch_node(node)&.content
         end
 
         def set_processing_instruction_content(node, content)
           native_node = unpatch_node(node)
-          # Store raw content - LibXML will escape it
-          native_node.content = content.to_s if native_node
+          return unless native_node.is_a?(::LibXML::XML::Node)
+
+          replace_native_verbatim(
+            node,
+            ::LibXML::XML::Node.new_pi(native_node.name, content.to_s),
+          )
         end
 
         def create_native_namespace(element, prefix, uri)
@@ -1694,7 +1715,10 @@ def on_end_document
         end
 
         def on_start_element(name, attributes)
-          attr_hash, ns_hash = split_attributes_and_namespaces(attributes)
+          normalized = (attributes || {}).map { |k, v| [k.to_s, v] }
+          attr_hash, ns_hash = split_attributes_and_namespaces(normalized) do |v|
+            Moxml::Adapter::Base.decode_entities(v)
+          end
           @handler.on_start_element(name.to_s, attr_hash, ns_hash)
         end
 

lib/moxml/adapter/nokogiri.rb

@@ -471,7 +471,9 @@ def end_document
         end
 
         def start_element(name, attributes = [])
-          attr_hash, ns_hash = split_attributes_and_namespaces(attributes)
+          attr_hash, ns_hash = split_attributes_and_namespaces(attributes) do |v|
+            Moxml::Adapter::Base.decode_entities(v)
+          end
           @handler.on_start_element(name, attr_hash, ns_hash)
         end