diff --git a/lib/moxml/adapter/base.rb b/lib/moxml/adapter/base.rb index adda348..57a9495 100644 --- a/lib/moxml/adapter/base.rb +++ b/lib/moxml/adapter/base.rb @@ -19,6 +19,15 @@ class Base SERIALIZED_ENTITY_MARKER_RE = /(#{ENTITY_NAME_PATTERN});/ STANDARD_ENTITIES = %w[amp lt gt quot apos].freeze + NAMED_ENTITY_DECODE_MAP = { + "amp" => "&", "lt" => "<", "gt" => ">", + "quot" => '"', "apos" => "'" + }.freeze + private_constant :NAMED_ENTITY_DECODE_MAP + + ENTITY_DECODE_RE = /&(?:(amp|lt|gt|quot|apos)|#(?:(\d+)|[xX]([0-9a-fA-F]+)));/.freeze + private_constant :ENTITY_DECODE_RE + class << self include XmlUtils @@ -53,6 +62,29 @@ def preprocess_entities(xml) end end + # Resolve numeric (&#NN; / &#xNN;) and the five standard named + # (& < > " ') XML entity references in one + # single pass. The resulting characters are data; no further + # decoding is applied. Numeric refs producing invalid UTF-8 + # (NUL, surrogate halves, > U+10FFFF) are preserved verbatim + # to avoid silently emitting malformed bytes. + def decode_entities(text) + return text unless text.is_a?(String) && text.include?("&") + + text.gsub(ENTITY_DECODE_RE) do + if (named = ::Regexp.last_match(1)) + NAMED_ENTITY_DECODE_MAP[named] + else + code = ::Regexp.last_match(2) ? ::Regexp.last_match(2).to_i : ::Regexp.last_match(3).to_i(16) + if code.zero? || code.between?(0xD800, 0xDFFF) || code > 0x10FFFF + ::Regexp.last_match(0) + else + [code].pack("U") + end + end + end + end + # Restore entity markers back to named entity references. def restore_entities(text) return text unless text.is_a?(String) diff --git a/lib/moxml/adapter/customized_libxml/cdata.rb b/lib/moxml/adapter/customized_libxml/cdata.rb index a3082b5..3a20a32 100644 --- a/lib/moxml/adapter/customized_libxml/cdata.rb +++ b/lib/moxml/adapter/customized_libxml/cdata.rb @@ -5,19 +5,10 @@ module Adapter module CustomizedLibxml # Wrapper for LibXML CDATA section nodes class Cdata < Node - # Serialize as XML CDATA section - # LibXML auto-escapes content, we need to un-escape it + # libxml stores CDATA payload verbatim. Only the `]]>` end-marker + # needs splitting before re-wrapping. def to_xml - content = @native.content - .gsub(""", '"') - .gsub("'", "'") - .gsub("<", "<") - .gsub(">", ">") - .gsub("&", "&") - - # Handle CDATA end marker escaping (]]> becomes ]]]]>) - # Replace all ]]> markers in the content before wrapping - escaped_content = content.gsub("]]>", "]]]]>") + escaped_content = @native.content.gsub("]]>", "]]]]>") "" end end diff --git a/lib/moxml/adapter/customized_libxml/comment.rb b/lib/moxml/adapter/customized_libxml/comment.rb index 57e449a..6fe9826 100644 --- a/lib/moxml/adapter/customized_libxml/comment.rb +++ b/lib/moxml/adapter/customized_libxml/comment.rb @@ -5,16 +5,9 @@ module Adapter module CustomizedLibxml # Wrapper for LibXML comment nodes class Comment < Node - # Serialize as XML comment - # LibXML auto-escapes content, we need to un-escape it + # libxml stores comment payload verbatim. def to_xml - content = @native.content - .gsub(""", '"') - .gsub("'", "'") - .gsub("<", "<") - .gsub(">", ">") - .gsub("&", "&") - "" + "" end end end diff --git a/lib/moxml/adapter/customized_libxml/node.rb b/lib/moxml/adapter/customized_libxml/node.rb index 0ec06e1..f2aff7c 100644 --- a/lib/moxml/adapter/customized_libxml/node.rb +++ b/lib/moxml/adapter/customized_libxml/node.rb @@ -18,6 +18,14 @@ def initialize(native_node) @native = native_node end + # Swap the wrapped native node. Used by the Libxml adapter when + # libxml-ruby's content= setter would silently re-escape stored + # text; replacing the node with a fresh raw-storage instance is + # the only way to preserve verbatim content. + def replace_native!(fresh) + @native = fresh + end + # Compare wrappers based on their native nodes def ==(other) return false unless other diff --git a/lib/moxml/adapter/customized_libxml/processing_instruction.rb b/lib/moxml/adapter/customized_libxml/processing_instruction.rb index da8fb30..974d215 100644 --- a/lib/moxml/adapter/customized_libxml/processing_instruction.rb +++ b/lib/moxml/adapter/customized_libxml/processing_instruction.rb @@ -5,20 +5,12 @@ module Adapter module CustomizedLibxml # Wrapper for LibXML processing instruction nodes class ProcessingInstruction < Node - # Serialize as XML processing instruction - # LibXML auto-escapes content, we need to un-escape it + # XML 1.0 §2.6: PI content is verbatim — no entity resolution, no escaping. def to_xml target = @native.name content = @native.content - - # Un-escape LibXML's automatic escaping if content && !content.empty? - unescaped = content.gsub(""", '"') - .gsub("'", "'") - .gsub("<", "<") - .gsub(">", ">") - .gsub("&", "&") - "" + "" else "" end diff --git a/lib/moxml/adapter/customized_libxml/text.rb b/lib/moxml/adapter/customized_libxml/text.rb index eeb810e..86bf283 100644 --- a/lib/moxml/adapter/customized_libxml/text.rb +++ b/lib/moxml/adapter/customized_libxml/text.rb @@ -13,16 +13,9 @@ def text @native.content end - # Serialize as XML with proper escaping - # LibXML's .content already contains escaped text, but it over-escapes - # quotes which don't need escaping in text nodes (only in attributes) + # @native.to_s escapes & < > but leaves quotes alone, which text nodes need. def to_xml - content = @native.content - # Skip the gsub allocation entirely when there's nothing to undo — - # the common case for parsed text without literal quotes. - return content unless content.include?(""") - - content.gsub(""", '"') + @native.to_s end end end diff --git a/lib/moxml/adapter/customized_oga.rb b/lib/moxml/adapter/customized_oga.rb index a7157ce..111fc24 100644 --- a/lib/moxml/adapter/customized_oga.rb +++ b/lib/moxml/adapter/customized_oga.rb @@ -3,6 +3,8 @@ module Moxml module Adapter module CustomizedOga + autoload :EntityDecoderOverride, "moxml/adapter/customized_oga/entity_decoder" + autoload :RawValueOverride, "moxml/adapter/customized_oga/raw_value_override" autoload :XmlDeclaration, "moxml/adapter/customized_oga/xml_declaration" autoload :XmlGenerator, "moxml/adapter/customized_oga/xml_generator" end diff --git a/lib/moxml/adapter/customized_oga/entity_decoder.rb b/lib/moxml/adapter/customized_oga/entity_decoder.rb new file mode 100644 index 0000000..2fbbec3 --- /dev/null +++ b/lib/moxml/adapter/customized_oga/entity_decoder.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require "oga" + +module Moxml + module Adapter + module CustomizedOga + # Override Oga::EntityDecoder.decode for XML inputs. + # + # Oga 3.4's stock Oga::XML::Entities.decode runs multiple passes, + # turning the well-formed "&#38;" into "&" rather than the + # spec-correct "&". XML 1.0 §4.6 forbids recursive resolution + # of parsed entities — exactly one decode pass is correct. + # + # Moxml::Adapter::Base.decode_entities does the single pass. HTML + # inputs fall through to Oga's stock HTML::Entities.decode, which + # has HTML-specific legacy rules that differ from XML's. + # + # Prepending on the singleton class means every Oga reader + # (Text#text, Attribute#value, etc.) automatically picks up the + # fixed decoder, so the adapter no longer needs to walk the parsed + # tree rewriting @text/@value ivars behind Oga's back. + module EntityDecoderOverride + # rubocop:disable Style/OptionalBooleanParameter -- must match Oga's signature + def decode(input, html = false) + return super if html + + ::Moxml::Adapter::Base.decode_entities(input) + end + # rubocop:enable Style/OptionalBooleanParameter + end + end + end +end + +Oga::EntityDecoder.singleton_class.prepend( + Moxml::Adapter::CustomizedOga::EntityDecoderOverride, +) diff --git a/lib/moxml/adapter/customized_oga/raw_value_override.rb b/lib/moxml/adapter/customized_oga/raw_value_override.rb new file mode 100644 index 0000000..95b966b --- /dev/null +++ b/lib/moxml/adapter/customized_oga/raw_value_override.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +require "oga" + +module Moxml + module Adapter + module CustomizedOga + # Bypass Oga's lazy decoder for user-authored values. + # + # Oga::XML::Attribute#value and Oga::XML::Text#text decode `@value` / + # `@text` on first read via Oga::EntityDecoder. EntityDecoderOverride + # makes that decode single-pass for parsed input. But user-authored + # values are already data — running them through the decoder again + # would re-interpret any literal `&` / `&#NN;` substrings as entity + # references. So we route the read through the NativeAttachment + # sidecar first; if the adapter stored the verbatim value there, we + # return it as-is, otherwise we fall through to Oga's normal read. + module RawValueOverride + def value + cached = ::Moxml::Adapter::Oga.attachments.get(self, :raw_value) + return cached unless cached.nil? + + super + end + + def value=(new_value) + ::Moxml::Adapter::Oga.attachments.set(self, :raw_value, new_value) + super + end + + def text + cached = ::Moxml::Adapter::Oga.attachments.get(self, :raw_text) + return cached unless cached.nil? + + super + end + + def text=(new_value) + ::Moxml::Adapter::Oga.attachments.set(self, :raw_text, new_value) + super + end + end + end + end +end + +Oga::XML::Attribute.prepend( + Moxml::Adapter::CustomizedOga::RawValueOverride, +) +Oga::XML::Text.prepend( + Moxml::Adapter::CustomizedOga::RawValueOverride, +) diff --git a/lib/moxml/adapter/libxml.rb b/lib/moxml/adapter/libxml.rb index 806c8bd..05bed5f 100644 --- a/lib/moxml/adapter/libxml.rb +++ b/lib/moxml/adapter/libxml.rb @@ -91,7 +91,7 @@ def parse(xml, options = {}, _context = nil) xml_string = preprocess_entities(xml_string) # Extract DOCTYPE before parsing - doctype_match = xml_string.match(//i) + doctype_match = xml_string.match(/]+)(?:\s+PUBLIC\s+"([^"]+)"\s+"([^"]+)"|\s+SYSTEM\s+"([^"]+)")?\s*>/i) native_doc = begin # Handle both string and file inputs @@ -730,26 +730,52 @@ def inner_text(node) def set_text_content(node, content) native_node = unpatch_node(node) - native_node.content = content.to_s if native_node + return unless native_node.is_a?(::LibXML::XML::Node) + + # Text wrapper: swap with new_text to preserve verbatim storage. + # Element wrapper: let libxml manage the child text node. + if native_node.text? + replace_native_verbatim(node, ::LibXML::XML::Node.new_text(content.to_s)) + else + native_node.content = content.to_s + end end - def cdata_content(node) - native_node = unpatch_node(node) - content = native_node&.content - # LibXML may HTML-escape CDATA content, un-escape it - return nil unless content + # Replace `old_native` with `fresh`, preserving position. Both + # neighbor pointers are set explicitly to match `replace`'s + # semantics (see #replace above). + def swap_native_in_place(old_native, fresh) + parent = old_native.parent + return if parent.nil? - content.gsub(""", '"') - .gsub("'", "'") - .gsub("<", "<") - .gsub(">", ">") - .gsub("&", "&") + prev_sibling = old_native.prev + next_sibling = old_native.next + old_native.remove! + prev_sibling.next = fresh if prev_sibling + next_sibling.prev = fresh if next_sibling + parent << fresh unless prev_sibling || next_sibling + end + private :swap_native_in_place + + # libxml-ruby's node.content= pre-escapes; building a fresh node via + # new_ stores content verbatim. Swap in place and re-point the + # wrapper. + def replace_native_verbatim(node, fresh) + native = unpatch_node(node) + return unless native.is_a?(::LibXML::XML::Node) + + swap_native_in_place(native, fresh) + node.replace_native!(fresh) if node.is_a?(CustomizedLibxml::Node) + end + private :replace_native_verbatim + + def cdata_content(node) + # libxml stores CDATA payload verbatim; no decoding needed. + unpatch_node(node)&.content end def set_cdata_content(node, content) - native_node = unpatch_node(node) - # CDATA content should NOT be escaped - native_node.content = content.to_s if native_node + replace_native_verbatim(node, ::LibXML::XML::Node.new_cdata(content.to_s)) end def comment_content(node) @@ -758,8 +784,7 @@ def comment_content(node) end def set_comment_content(node, content) - native_node = unpatch_node(node) - native_node.content = content.to_s if native_node + replace_native_verbatim(node, ::LibXML::XML::Node.new_comment(content.to_s)) end def processing_instruction_target(node) @@ -768,22 +793,18 @@ def processing_instruction_target(node) end def processing_instruction_content(node) - native_node = unpatch_node(node) - content = native_node&.content - # LibXML may HTML-escape the content, un-escape it - return nil unless content - - content.gsub(""", '"') - .gsub("'", "'") - .gsub("<", "<") - .gsub(">", ">") - .gsub("&", "&") + # XML 1.0 §2.6: PI content is verbatim — no entity resolution. + unpatch_node(node)&.content end def set_processing_instruction_content(node, content) native_node = unpatch_node(node) - # Store raw content - LibXML will escape it - native_node.content = content.to_s if native_node + return unless native_node.is_a?(::LibXML::XML::Node) + + replace_native_verbatim( + node, + ::LibXML::XML::Node.new_pi(native_node.name, content.to_s), + ) end def create_native_namespace(element, prefix, uri) @@ -1694,7 +1715,10 @@ def on_end_document end def on_start_element(name, attributes) - attr_hash, ns_hash = split_attributes_and_namespaces(attributes) + normalized = (attributes || {}).map { |k, v| [k.to_s, v] } + attr_hash, ns_hash = split_attributes_and_namespaces(normalized) do |v| + Moxml::Adapter::Base.decode_entities(v) + end @handler.on_start_element(name.to_s, attr_hash, ns_hash) end diff --git a/lib/moxml/adapter/nokogiri.rb b/lib/moxml/adapter/nokogiri.rb index 3c93295..4d543ab 100644 --- a/lib/moxml/adapter/nokogiri.rb +++ b/lib/moxml/adapter/nokogiri.rb @@ -471,7 +471,9 @@ def end_document end def start_element(name, attributes = []) - attr_hash, ns_hash = split_attributes_and_namespaces(attributes) + attr_hash, ns_hash = split_attributes_and_namespaces(attributes) do |v| + Moxml::Adapter::Base.decode_entities(v) + end @handler.on_start_element(name, attr_hash, ns_hash) end diff --git a/lib/moxml/adapter/oga.rb b/lib/moxml/adapter/oga.rb index cd53c33..00f5898 100644 --- a/lib/moxml/adapter/oga.rb +++ b/lib/moxml/adapter/oga.rb @@ -5,6 +5,16 @@ module Moxml module Adapter class Oga < Base + # Trigger autoloads that prepend override modules onto Oga's + # native classes before any parse / read runs. + # - EntityDecoderOverride: replace Oga 3.4's multi-pass decoder + # with a single-pass decoder on Oga::EntityDecoder. + # - RawValueOverride: route Attribute#value / Text#text through + # the NativeAttachment sidecar so user-authored values bypass + # the lazy decoder and are returned verbatim. + ::Moxml::Adapter::CustomizedOga::EntityDecoderOverride + ::Moxml::Adapter::CustomizedOga::RawValueOverride + class << self def attachments @attachments ||= Moxml::NativeAttachment.new @@ -34,26 +44,29 @@ def parse(xml, options = {}, _context = nil) DocumentBuilder.new(ctx).build(native_doc) end - # SAX parsing implementation for Oga + # SAX parsing implementation for Oga. + # + # Driven off `Oga.parse_xml` plus a DOM walk because Oga 3.4's + # native SAX parser decodes `on_text` content before delivery and + # exposes no hook to override. The prepended EntityDecoderOverride + # ensures `node.text` invoked during the walk produces the correct + # single-pass decode, so no separate entity pass is needed here. + # Trade-off: peak memory scales with document size (Oga 3.4 is the + # final release of an unmaintained gem; the trade is accepted). # # @param xml [String, IO] XML to parse # @param handler [Moxml::SAX::Handler] Moxml SAX handler # @return [void] def sax_parse(xml, handler) - bridge = OgaSAXBridge.new(handler) - xml_string = xml.is_a?(IO) || xml.is_a?(StringIO) ? xml.read : xml.to_s + native_doc = ::Oga.parse_xml(xml_string) - # Manually call start_document (Oga doesn't) - handler.on_start_document - - ::Oga.sax_parse_xml(bridge, xml_string) - - # Manually call end_document (Oga doesn't) - handler.on_end_document + bridge = OgaSAXBridge.new(handler) + bridge.on_start_document + native_doc.children.each { |child| bridge.emit(child) } + bridge.on_end_document rescue StandardError => e - error = Moxml::ParseError.new(e.message) - handler.on_error(error) + handler.on_error(Moxml::ParseError.new(e.message)) end def create_document(_native_doc = nil) @@ -65,7 +78,14 @@ def create_native_element(name, _owner_doc = nil) end def create_native_text(content, _owner_doc = nil) - ::Oga::XML::Text.new(text: preprocess_entities(content)) + processed = preprocess_entities(content) + text = ::Oga::XML::Text.new(text: processed) + # Oga::XML::Text.new stores the value via ivar, bypassing the + # RawValueOverride setter. Set the sidecar explicitly so reads + # return the user-supplied value verbatim rather than running + # it through the lazy decoder. + attachments.set(text, :raw_text, processed) + text end def create_native_entity_reference(name) @@ -257,11 +277,15 @@ def set_attribute(element, name, value) 2) end + processed = preprocess_entities(value.to_s) attr = ::Oga::XML::Attribute.new( name: name.to_s, namespace_name: namespace_name, - value: preprocess_entities(value.to_s), + value: processed, ) + # See create_native_text: Attribute.new bypasses the override + # setter, so populate the sidecar here. + attachments.set(attr, :raw_value, processed) element.add_attribute(attr) end @@ -270,7 +294,8 @@ def get_attribute(element, name) end def get_attribute_value(element, name) - element[name.to_s] + attr = element.attribute(name.to_s) + attr&.value end def remove_attribute(element, name) @@ -528,50 +553,53 @@ def serialize_without_entity_processing(node, options = {}) ::Moxml::Adapter::CustomizedOga::XmlGenerator.new(node).to_xml end end - end - # Bridge between Oga SAX and Moxml SAX - # - # Translates Oga SAX events to Moxml::SAX::Handler events. - # Oga has different event naming and namespace as first param. - # - # @private - class OgaSAXBridge - include Moxml::SAX::NamespaceSplitter - - def initialize(handler) - @handler = handler - end + # Bridge between a parsed Oga DOM and Moxml SAX events. + # + # @private + class OgaSAXBridge + include Moxml::SAX::NamespaceSplitter - # Oga: on_element(namespace, name, attributes) - # namespace may be nil - # attributes is an array of [name, value] pairs - def on_element(namespace, name, attributes) - element_name = namespace ? "#{namespace}:#{name}" : name - attr_hash, ns_hash = split_attributes_and_namespaces(attributes) - @handler.on_start_element(element_name, attr_hash, ns_hash) - end + def initialize(handler) + @handler = handler + end - # Oga: after_element(namespace, name) - def after_element(namespace, name) - element_name = namespace ? "#{namespace}:#{name}" : name - @handler.on_end_element(element_name) - end + def on_start_document + @handler.on_start_document + end - def on_text(text) - @handler.on_characters(text) - end + def on_end_document + @handler.on_end_document + end - def on_cdata(text) - @handler.on_cdata(text) - end + # Walk a parsed Oga node and emit Moxml SAX events. + def emit(node) + case node + when ::Oga::XML::Element + element_name = qualified_name(node.namespace_name, node.name) + pairs = node.attributes.map do |a| + [qualified_name(a.namespace_name, a.name), a.value] + end + attrs, namespaces = split_attributes_and_namespaces(pairs) + @handler.on_start_element(element_name, attrs, namespaces) + node.children.each { |c| emit(c) } + @handler.on_end_element(element_name) + when ::Oga::XML::Text + @handler.on_characters(node.text) + when ::Oga::XML::Cdata + @handler.on_cdata(node.text) + when ::Oga::XML::Comment + @handler.on_comment(node.text) + when ::Oga::XML::ProcessingInstruction + @handler.on_processing_instruction(node.name, node.text || "") + end + end - def on_comment(text) - @handler.on_comment(text) - end + private - def on_processing_instruction(name, text) - @handler.on_processing_instruction(name, text || "") + def qualified_name(namespace, local) + namespace ? "#{namespace}:#{local}" : local + end end end end diff --git a/lib/moxml/adapter/ox.rb b/lib/moxml/adapter/ox.rb index 230ae7c..83cddf9 100644 --- a/lib/moxml/adapter/ox.rb +++ b/lib/moxml/adapter/ox.rb @@ -504,6 +504,7 @@ def text_content(node) case node when String then node.to_s when ::Moxml::Adapter::CustomizedOx::Text then node.value + when ::Ox::CData then node.value.to_s when ::Moxml::Adapter::CustomizedOx::EntityReference then "" else return "" unless node.is_a?(::Ox::Element) || node.is_a?(::Ox::Document) diff --git a/lib/moxml/adapter/rexml.rb b/lib/moxml/adapter/rexml.rb index c926949..1234993 100644 --- a/lib/moxml/adapter/rexml.rb +++ b/lib/moxml/adapter/rexml.rb @@ -2,6 +2,7 @@ require "rexml/document" require "rexml/xpath" +require "rexml/streamlistener" require "set" unless RUBY_ENGINE == "opal" require "stringio" if RUBY_ENGINE == "opal" @@ -60,18 +61,21 @@ def extract_encoding_from_xml(xml) # @param handler [Moxml::SAX::Handler] Moxml SAX handler # @return [void] def sax_parse(xml, handler) - require "rexml/parsers/sax2parser" + require "rexml/parsers/streamparser" require "rexml/source" - require "stringio" - bridge = REXMLSAX2Bridge.new(handler) + bridge = REXMLStreamBridge.new(handler) xml_string = xml.is_a?(IO) || xml.is_a?(StringIO) ? xml.read : xml.to_s - source = ::REXML::IOSource.new(StringIO.new(xml_string)) - parser = ::REXML::Parsers::SAX2Parser.new(source) - parser.listen(bridge) - parser.parse + # StreamParser (rather than SAX2Parser) is used because + # SAX2Parser normalizes plain "&#NN;" and "&#NN;" attribute + # literals to the same raw string, making spec-correct decoding + # impossible after the fact. StreamParser delivers values in + # the same decoded form as REXML's DOM. + handler.on_start_document + ::REXML::Parsers::StreamParser.new(xml_string, bridge).parse + handler.on_end_document rescue ::REXML::ParseException => e error = Moxml::ParseError.new(e.message, line: e.line) handler.on_error(error) @@ -642,29 +646,33 @@ def write_with_formatter(node, output, indent = 2) end end - # Bridge between REXML SAX2 and Moxml SAX + # Bridge between REXML StreamParser and Moxml SAX # - # Translates REXML::SAX2Parser events to Moxml::SAX::Handler events + # StreamParser (rather than SAX2Parser) is used because SAX2Parser + # normalizes plain "&#NN;" and "&#NN;" attribute literals to + # the same raw string, which makes spec-correct decoding impossible + # after the fact. StreamParser delivers attribute values in the + # same decoded form as REXML's DOM. # # @private - class REXMLSAX2Bridge + class REXMLStreamBridge + include ::REXML::StreamListener include Moxml::SAX::NamespaceSplitter def initialize(handler) @handler = handler end - # REXML splits element name into uri/localname/qname - def start_element(_uri, _localname, qname, attributes) + def tag_start(name, attributes) attr_hash, ns_hash = split_attributes_and_namespaces(attributes) - @handler.on_start_element(qname, attr_hash, ns_hash) + @handler.on_start_element(name, attr_hash, ns_hash) end - def end_element(_uri, _localname, qname) - @handler.on_end_element(qname) + def tag_end(name) + @handler.on_end_element(name) end - def characters(text) + def text(text) @handler.on_characters(text) end @@ -676,26 +684,9 @@ def comment(text) @handler.on_comment(text) end - def processing_instruction(target, data) + def instruction(target, data) @handler.on_processing_instruction(target, data || "") end - - def start_document - @handler.on_start_document - end - - def end_document - @handler.on_end_document - end - - # REXML calls these but we don't need to handle them - def xmldecl(version, encoding, standalone) - # XML declaration - we don't need to do anything - end - - def progress(position) - # Progress callback - we don't need to do anything - end end end end diff --git a/lib/moxml/sax/namespace_splitter.rb b/lib/moxml/sax/namespace_splitter.rb index 6fdca97..ca1e889 100644 --- a/lib/moxml/sax/namespace_splitter.rb +++ b/lib/moxml/sax/namespace_splitter.rb @@ -9,6 +9,8 @@ module SAX # attribute. This module provides a single implementation. module NamespaceSplitter # @param attributes [Hash, Array] attributes as a hash or array of pairs + # @yieldparam value [Object] raw attribute/namespace value + # @yieldreturn [Object] transformed value to store # @return [Array(Hash, Hash)] [regular_attrs, namespaces] def split_attributes_and_namespaces(attributes) attrs = {} @@ -16,11 +18,12 @@ def split_attributes_and_namespaces(attributes) each_attribute(attributes) do |name, value| name_s = name.to_s + v = block_given? ? yield(value) : value if name_s == "xmlns" || name_s.start_with?("xmlns:") prefix = name_s == "xmlns" ? nil : name_s.sub("xmlns:", "") - ns[prefix] = value + ns[prefix] = v else - attrs[name_s] = value + attrs[name_s] = v end end diff --git a/spec/moxml/doctype_spec.rb b/spec/moxml/doctype_spec.rb index e9bb363..d523f99 100644 --- a/spec/moxml/doctype_spec.rb +++ b/spec/moxml/doctype_spec.rb @@ -48,7 +48,7 @@ end describe "parsing" do - %i[nokogiri oga rexml ox].each do |adapter_name| + %i[nokogiri oga rexml ox libxml].each do |adapter_name| context "with #{adapter_name} adapter" do let(:ctx) { Moxml.new(adapter_name) } diff --git a/spec/moxml/sax_entity_parity_spec.rb b/spec/moxml/sax_entity_parity_spec.rb new file mode 100644 index 0000000..c942b1e --- /dev/null +++ b/spec/moxml/sax_entity_parity_spec.rb @@ -0,0 +1,358 @@ +# frozen_string_literal: true + +require "spec_helper" + +# Pins cross-adapter parity for entity-decoding in attribute values and +# text content, across SAX, DOM, and build-then-serialize-then-reparse. +# Previously the four adapters diverged: nokogiri/libxml SAX failed to +# resolve "&#NN;"; rexml SAX delivered raw escaped strings; oga DOM +# and SAX double-decoded. + +# Namespace-isolated container for the test data and the SAX capture +# handler. Wrapping these in a module rather than declaring them as +# top-level constants inside `RSpec.describe` keeps them out of the +# global namespace and lets the spec reference them without tripping +# the RSpec/* cops that forbid local-variable use inside examples. +module SaxEntityParityFixtures + class CaptureHandler < Moxml::SAX::ElementHandler + attr_reader :first_attrs, :text + + def initialize + super + @first_attrs = nil + @text = +"" + end + + def on_start_element(_name, attrs = {}, _namespaces = {}) + @first_attrs = attrs.dup if @first_attrs.nil? + end + + def on_characters(chunk) + @text << chunk + end + end + + ADAPTERS = %i[nokogiri ox oga rexml libxml headed_ox].freeze + + # Each row: [input XML, decoded attribute value, expected attribute + # value as it appears in the serialized XML]. The serialized form + # asserts what every adapter must emit byte-for-byte for the attribute. + ATTRIBUTE_CASES = { + "wrapped amp + decimal ref" => ['', "&", "&#38;"], + "wrapped amp + hex ref" => ['', "&", "&#x26;"], + "wrapped amp + non-std ref" => ['', "©", "&copy;"], + "wrapped amp + lt" => ['', "<", "&lt;"], + "wrapped amp + amp" => ['', "&", "&amp;"], + "two amps" => ['', "&&", "&&"], + "plain amp" => ['', "&", "&"], + "plain decimal ref" => ['', "&", "&"], + "plain hex ref" => ['', "&", "&"], + "high codepoint" => ['', "©", "©"], + "no entities" => ['', "plain", "plain"], + }.freeze + + # Each row: [input XML, decoded text, expected text as it appears in + # serialized XML]. + TEXT_CASES = { + "text wrapped amp + decimal" => ["&#38;", "&", "&#38;"], + "text wrapped amp + non-std" => ["&copy;", "©", "&copy;"], + "text plain amp" => ["&", "&", "&"], + }.freeze + + # XML 1.0 §2.6 — entity references inside PI content are NOT resolved. + # Every adapter must surface the literal source text. The libxml + # adapter previously decoded these five entities via a gsub chain in + # processing_instruction_content; pin each one explicitly. + PI_ENTITY_CASES = { + "&" => "data & more", + "<" => "data < more", + ">" => "data > more", + """ => "data " more", + "'" => "data ' more", + }.freeze + + # Per-adapter overrides for the rebuild-path serialized form. Oga's + # set_attribute calls preprocess_entities which marks non-standard + # named entities so they survive serialization as "&name;" rather than + # being escaped to "&name;". This is the entity-preservation + # feature documented in spec/moxml/adapter/entity_restoration_spec.rb + # and spec/moxml/adapter/oga_spec.rb — not a bug. + REBUILD_SERIALIZED_OVERRIDES = { + oga: { + "wrapped amp + non-std ref" => "©", + }, + }.freeze +end + +RSpec.describe "SAX/DOM entity parity" do + SaxEntityParityFixtures::ADAPTERS.each do |adapter| + context "with #{adapter} adapter" do + let(:ctx) { Moxml.new(adapter) } + + SaxEntityParityFixtures::ATTRIBUTE_CASES.each do |label, (xml, expected, expected_serialized)| + it "decodes attribute via SAX: #{label}" do + handler = SaxEntityParityFixtures::CaptureHandler.new + ctx.sax_parse(xml, handler) + expect(handler.first_attrs["x"]).to eq(expected) + end + + it "decodes attribute via DOM: #{label}" do + expect(ctx.parse(xml).root["x"]).to eq(expected) + end + + it "serializes attribute to expected XML form: #{label}" do + # End-to-end: parse → DOM access → set on fresh doc → serialize + # → assert exact serialized form. This pins what the consumer + # sees on the wire, not just what the parser delivers in memory. + parsed_value = ctx.parse(xml).root["x"] + + doc2 = ctx.create_document + el = doc2.create_element("doc") + el["x"] = parsed_value + doc2.add_child(el) + + serialized = doc2.to_xml.sub(/\A<\?[^>]*\?>\s*/, "") + expected_form = SaxEntityParityFixtures::REBUILD_SERIALIZED_OVERRIDES.dig(adapter, label) || expected_serialized + expect(serialized).to include(%(x="#{expected_form}")) + end + + it "round-trips attribute through build+serialize+reparse: #{label}" do + parsed_value = ctx.parse(xml).root["x"] + + doc2 = ctx.create_document + el = doc2.create_element("doc") + el["x"] = parsed_value + doc2.add_child(el) + + serialized = doc2.to_xml.sub(/\A<\?[^>]*\?>/, "") + expect(ctx.parse(serialized).root["x"]).to eq(parsed_value) + end + + it "re-serializes parsed document idempotently: #{label}" do + # Parse → serialize → re-parse → re-serialize → assert the two + # serializations match. This catches any non-idempotent + # mutations the adapter applies during the round-trip. + first = ctx.parse(xml).to_xml.sub(/\A<\?[^>]*\?>\s*/, "") + second = ctx.parse(first).to_xml.sub(/\A<\?[^>]*\?>\s*/, "") + expect(second).to eq(first) + end + end + + SaxEntityParityFixtures::TEXT_CASES.each do |label, (xml, expected, expected_serialized)| + it "decodes text via SAX: #{label}" do + handler = SaxEntityParityFixtures::CaptureHandler.new + ctx.sax_parse(xml, handler) + expect(handler.text).to eq(expected) + end + + it "decodes text via DOM: #{label}" do + expect(ctx.parse(xml).root.text).to eq(expected) + end + + it "serializes text to expected XML form: #{label}" do + serialized = ctx.parse(xml).to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + expect(serialized).to include(">#{expected_serialized}<") + end + end + + it "does not double-escape '&' in a programmatically-built text node" do + # Pins the libxml customized Text#to_xml fix: an authored "&" + # must serialize to "&" exactly once, never "&amp;". + # Regression target for adapter/customized_libxml/text.rb where + # the previous implementation used #content (decoded text) and + # lost the "&" → "&" escape, and earlier variants risked + # double-escaping on round-trip. + doc = ctx.create_document + el = doc.create_element("doc") + el.add_child("a & b") + doc.add_child(el) + + serialized = doc.to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + expect(serialized).to include(">a & b<") + expect(serialized).not_to include("&amp;") + + # And the round-trip must preserve the original text exactly. + reparsed_text = ctx.parse(serialized).root.text + expect(reparsed_text).to eq("a & b") + end + + it "mutates in-tree text node content without double-escaping '&'" do + # Pins the libxml fix for set_text_content on attached text + # nodes: libxml-ruby's #content= setter pre-escapes the input, + # and #to_s escapes again on serialization. The adapter must + # replace the in-tree node with a fresh raw-storage node to + # avoid silent double-escape data corruption. + doc = ctx.create_document + el = doc.create_element("doc") + text = doc.create_text("placeholder") + el.add_child(text) + doc.add_child(el) + + text.content = "a & b" + expect(text.content).to eq("a & b") + + serialized = doc.to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + expect(serialized).to include(">a & b<") + expect(serialized).not_to include("&amp;") + expect(ctx.parse(serialized).root.text).to eq("a & b") + end + + it "mutates in-tree PI content verbatim without escaping" do + # Pins the libxml fix for set_processing_instruction_content + # on attached PI nodes: libxml-ruby's #content= setter pre- + # escapes quotes and ampersands, but XML 1.0 §2.6 specifies + # PI content is verbatim — entity references are not resolved + # and no escaping is required on serialization. + doc = ctx.create_document + el = doc.create_element("doc") + pi = doc.create_processing_instruction("target", "placeholder") + el.add_child(pi) + doc.add_child(el) + + pi.content = 'a " & b' + expect(pi.content.strip).to eq('a " & b') + + serialized = doc.to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + expect(serialized).to include('a " & b?>') + expect(serialized).not_to include(""") + expect(serialized).not_to include("&") + reparsed = ctx.parse(serialized).root.children.find do |c| + c.is_a?(Moxml::ProcessingInstruction) + end + expect(reparsed.content.strip).to eq('a " & b') + end + + it "mutates unparented text/PI content before adoption into the tree" do + # Pins the swap_native_in_place early-return when parent is nil: + # set_text_content / set_processing_instruction_content called + # before the node is added to any element must still replace the + # wrapper's @native with a fresh raw-storage node, so the value + # survives correctly once the node is later attached and serialized. + doc = ctx.create_document + el = doc.create_element("doc") + + text = doc.create_text("placeholder") + text.content = "a & b" + pi = doc.create_processing_instruction("target", "placeholder") + pi.content = 'a " & b' + + el.add_child(text) + el.add_child(pi) + doc.add_child(el) + + serialized = doc.to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + expect(serialized).to include(">a & b<") + expect(serialized).to include('') + expect(serialized).not_to include("&amp;") + end + + # Regressions around & appearing inside contexts that Oga's + # source-level preprocessor must not touch (CDATA, comments, + # DOCTYPE) — verified across adapters to ensure parity holds. + it "preserves literal & inside CDATA" do + doc = ctx.parse("") + expect(doc.root.text).to eq("a & b") + end + + it "preserves literal & inside comment" do + doc = ctx.parse("") + comment = doc.root.children.find { |c| c.is_a?(Moxml::Comment) } + expect(comment.content.strip).to eq("a & b") + end + + it "preserves literal U+FFFC U+FFFC user data in attribute" do + xml = "" + expect(ctx.parse(xml).root["x"]).to eq("\u{FFFC}\u{FFFC} test") + end + + it "preserves literal U+FDD0 U+FDD0 user data in attribute" do + # Single noncharacters near the Oga marker codepoints — must not + # collide with the internal DEFAULT_AMP_MARKER sentinel. + xml = "" + expect(ctx.parse(xml).root["x"]).to eq("\u{FDD0}\u{FDD0} test") + end + + it "preserves the exact Oga DEFAULT_AMP_MARKER sequence in attribute (no amp)" do + # User data matching the default Oga marker codepoints with NO + # & in the input — restore must not run, so the sequence + # survives intact. + xml = "" + expect(ctx.parse(xml).root["x"]).to eq("\u{FDD0}\u{FDD1}\u{FDD2} plain") + end + + it "preserves the exact Oga DEFAULT_AMP_MARKER sequence in attribute (with amp)" do + # User data matching the default Oga marker codepoints alongside + # an & that the preprocessor would otherwise substitute. + # The adapter must pick a marker not present in user data so + # restore doesn't corrupt the original sequence. + xml = "" + expect(ctx.parse(xml).root["x"]).to eq("\u{FDD0}\u{FDD1}\u{FDD2} & foo") + end + + it "preserves the exact Oga DEFAULT_AMP_MARKER sequence in text" do + xml = "\u{FDD0}\u{FDD1}\u{FDD2} & foo" + expect(ctx.parse(xml).root.text).to eq("\u{FDD0}\u{FDD1}\u{FDD2} & foo") + end + + it "handles DOCTYPE with '[' in quoted system ID" do + # System ID containing a literal "[" before the internal subset + # opener — quote-aware DOCTYPE terminator must not pick "]>" + # based on the bracket inside the quoted string. + xml = %() + expect(ctx.parse(xml).root["x"]).to eq("&") + end + + it "handles bare DOCTYPE with '>' in quoted system ID" do + # Bare DOCTYPE (no internal subset) — the terminator is ">". + # A literal ">" inside the quoted SYSTEM id must not be treated + # as the end of the declaration. For the Oga adapter this also + # exercises find_block_terminator's quote-aware ">" scan; if + # the bare-DOCTYPE end was misidentified, the preprocessor + # would rewrite "&" inside the remaining quoted ID region + # to DEFAULT_AMP_MARKER and leak it into the serialized doctype. + xml = %(b&c">) + expect(ctx.parse(xml).root["x"]).to eq("&") + end + + it "handles DOCTYPE with ']>' inside a quoted entity value" do + # Quote-aware DOCTYPE terminator must skip past a literal "]>" + # appearing inside a quoted ExternalID / ENTITY value so the + # internal subset terminator is matched correctly. For the Oga + # adapter this also exercises the source-level DOCTYPE skipper; + # native-DOCTYPE parsers must reach the same result. + xml = '"> ]>' + expect(ctx.parse(xml).root["x"]).to eq("&") + end + + SaxEntityParityFixtures::PI_ENTITY_CASES.each do |entity_label, payload| + it "preserves literal #{entity_label} inside processing instruction" do + doc = ctx.parse("") + pi = doc.root.children.find { |c| c.is_a?(Moxml::ProcessingInstruction) } + expect(pi.content.strip).to eq(payload) + end + end + end + end + + # Cross-adapter equivalence: feed the same XML into every adapter and + # assert all of them emit the same serialized attribute form. Catches + # any regression where one adapter starts to deviate from the rest. + describe "cross-adapter serialization equivalence" do + def serialize_via(adapter, xml) + ctx = Moxml.new(adapter) + ctx.parse(xml).to_xml.sub(/\A<\?[^>]*\?>\s*/, "").strip + end + + SaxEntityParityFixtures::ATTRIBUTE_CASES.each do |label, (xml, _expected, expected_serialized)| + it "every adapter emits the same attribute form: #{label}" do + results = SaxEntityParityFixtures::ADAPTERS.to_h { |a| [a, serialize_via(a, xml)] } + results.each do |a, out| + expect(out).to( + include(%(x="#{expected_serialized}")), + "#{a} produced #{out.inspect}", + ) + end + end + end + end +end diff --git a/spec/moxml/xpath/functions/node_functions_spec.rb b/spec/moxml/xpath/functions/node_functions_spec.rb index fb6c365..d573eed 100644 --- a/spec/moxml/xpath/functions/node_functions_spec.rb +++ b/spec/moxml/xpath/functions/node_functions_spec.rb @@ -57,7 +57,7 @@ result = proc.call(doc) # Depending on adapter, may include ns: prefix - expect(result).to match(/item/) + expect(result).to include("item") end it "returns empty string when no node matched" do