Fix parsed models dropping post-parse changes

.rubocop.yml

@@ -32,6 +32,11 @@ RSpec/EmptyExampleGroup:
   Exclude:
     - 'spec/lutaml/jsonld/transform_spec.rb'
 
+# emits_nothing_extra carries the expectations for these examples.
+RSpec/NoExpectationExample:
+  AllowedPatterns:
+    - '^emits_nothing_extra'
+
 RSpec/NamedSubject:
   inherit_mode:
     merge:

lib/compat/opal/lutaml_model_boot.rb

@@ -371,6 +371,7 @@
 require "lutaml/xml/serialization/model_import_ext"
 require "lutaml/xml/transformation/custom_method_wrapper"
 require "lutaml/xml/transformation/element_builder"
+require "lutaml/xml/transformation/order_reconciler"
 require "lutaml/xml/transformation/ordered_applier"
 require "lutaml/xml/transformation/rule_applier"
 require "lutaml/xml/transformation/rule_compiler"

lib/lutaml/model/attribute.rb

@@ -286,6 +286,32 @@ def cast_value(value, register)
         build_collection(value.map { |v| cast_element(v, register) })
       end
 
+      # Cast the value a derived reader just recomputed.
+      #
+      # A derived attribute has no ivar and no writer. Its reader runs the
+      # source method and casts the result on every call, which makes the
+      # reader a casting entry point of its own — nothing a writer-side guard
+      # can reach. A collection still has to come back as a collection here,
+      # even when the source method returned nothing.
+      #
+      # @param value [Object] whatever the source method returned
+      # @param register [Symbol] register for type resolution
+      # @return [Object] the cast value, or an empty collection
+      def cast_derived(value, register)
+        return cast_element(value, register) unless collection?
+        return build_collection if no_data?(value)
+        return cast_value(value, register) if collection_instance?(value)
+        # A plain Array is a list of source elements even when the attribute
+        # declares its own collection class, which collection_instance? only
+        # recognises by that class. Without this it becomes one element whose
+        # value is the inspected Array.
+        if value.is_a?(::Array)
+          return build_collection(value.map { |v| cast_element(v, register) })
+        end
+
+        build_collection(cast_element(value, register))
+      end
+
       # Apply a value map to transform a value.
       #
       # value_map keys (:nil, :empty, :omitted) each map either to a symbolic
@@ -331,8 +357,32 @@ def required_value_set?(value)
         true
       end
 
+      # True when the value is "nothing arrived" rather than data.
+      #
+      # nil and the uninitialized sentinel both mean the source carried no
+      # value. Neither is something a type can be asked to turn into an
+      # instance.
+      #
+      # @param value [Object]
+      # @return [Boolean]
+      def no_data?(value)
+        value.nil? || Utils.uninitialized?(value)
+      end
+
       def cast_element(value, register)
+        # Resolve first: an undeclared type has to raise UnknownTypeError even
+        # when the value carries nothing, the way it did before the guard
+        # below existed.
         resolved_type = type(register)
+
+        # Casting is for data. Every built-in type already hands nil and the
+        # uninitialized sentinel straight back, so this changes nothing for
+        # them. A type with its own `self.cast` returns a real instance
+        # instead, and that instance becomes a value nobody wrote — a phantom
+        # element in the document, or a one-item collection the source never
+        # contained. `cast` below guards the format entry point the same way.
+        return value if no_data?(value)
+
         return cast_union(value, nil, register) if union?
         return resolved_type.new(value) if value.is_a?(::Hash) && !hash_type?
 
@@ -623,6 +673,14 @@ def model_instance?(value)
       end
 
       def cast(value, format, register, options = {})
+        # Same rule as cast_element, for the format entry point. from_json /
+        # from_yaml / from_toml / from_hash reach a missing key as nil, and a
+        # type with its own `self.cast` turns that nil into an instance the
+        # document never carried. Unlike cast_element this returns before
+        # resolving the type: resolution here depends on the format options
+        # below, and an undeclared type still raises through cast_element.
+        return value if no_data?(value)
+
         # Namespace-aware type resolution: use type_with_namespace if namespace_uri provided
         namespace_uri = options[:namespace_uri]
         resolved_type = if options[:resolved_type]

lib/lutaml/model/serialize.rb

@@ -321,6 +321,65 @@ def resolve_reference_value(ref)
 
         ref.is_a?(Type::Reference) ? ref.object : ref
       end
+
+      # Hand back a collection the caller can push onto.
+      #
+      # An unused collection points at LAZY_EMPTY_COLLECTION, one frozen Array
+      # shared by every instance of every class, so an untouched collection
+      # costs nothing. A reader cannot hand that array out: `model.items << x`
+      # raises FrozenError, and unfreezing it would leak one instance's items
+      # into all the others. So the first read swaps in this instance's own
+      # empty Array and stores it — which is what the sentinel's own comment
+      # has promised all along. A frozen model keeps the sentinel; there is
+      # nothing to store into.
+      #
+      # nil is left alone on purpose. A nil collection is a state of its own
+      # here, distinct from an empty one — `initialize_empty: false` is what
+      # produces it and `render_nil` renders it — so a reader that turned nil
+      # into [] would erase a documented distinction.
+      #
+      # @param attribute_name [Symbol] the collection attribute
+      # @return [Object] the stored collection
+      def materialize_lazy_collection(attribute_name)
+        current = instance_variable_get(:"@#{attribute_name}")
+        return current unless current.equal?(LAZY_EMPTY_COLLECTION)
+        return current if frozen?
+
+        instance_variable_set(:"@#{attribute_name}", [])
+      end
+
+      # Hand back a reference collection the caller can push onto.
+      #
+      # A reference attribute stores Type::Reference objects, and resolving
+      # them built a fresh Array per read — so `book.co_authors << author`
+      # landed on a throwaway and never reached the document. A Reference
+      # stays lazy until its target registers itself, so the reader can only
+      # store the resolved Array once every reference has actually resolved;
+      # until then it has to keep re-resolving and keeps handing back a copy.
+      # Once stored there is nothing left to resolve: resolve_reference_value
+      # passes a model instance straight through, and the generated key reader
+      # runs Attribute#reference_key over it to get the key back.
+      #
+      # Storing means `#{name}_ref` reports resolved objects from then on. It
+      # is the backing store, and after this the objects are what it holds.
+      #
+      # @param attribute_name [Symbol] the reference collection attribute
+      # @return [Object] the stored collection
+      def materialize_reference_collection(attribute_name)
+        refs = instance_variable_get(:"@#{attribute_name}_ref")
+        return resolve_reference_value(refs) unless refs.is_a?(::Array)
+        return refs if refs.none?(Type::Reference)
+
+        resolved = resolve_reference_value(refs)
+        # Only a Reference that resolved to nothing keeps this lazy. A raw nil
+        # the caller put there is data, and waiting on it would never end.
+        dangling = refs.zip(resolved).any? do |ref, value|
+          ref.is_a?(Type::Reference) && value.nil?
+        end
+        return resolved if frozen? || dangling
+
+        instance_variable_set(:"@#{attribute_name}_ref", resolved)
+      end
     end
   end
 end

lib/lutaml/model/serialize/attribute_definition.rb

@@ -27,8 +27,11 @@ def define_attribute_methods(attr, register = nil)
             unless method_defined?(name, false)
               define_method(name) do
                 value = public_send(attr.method_name)
-                # Cast the derived value to the specified type
-                attr.cast_element(value, register_id)
+                # Cast the derived value to the specified type. cast_derived,
+                # not cast_element: this reader is its own casting entry point,
+                # so it needs the same "nothing arrived, nothing to cast" rule
+                # the writers get, and a collection has to come back as one.
+                attr.cast_derived(value, register_id)
               end
             end
           elsif attr.unresolved_type == Lutaml::Model::Type::Reference
@@ -64,14 +67,22 @@ def define_reference_methods(name, register)
           unless method_defined?(:"#{name}_#{key_method_name}", false)
             define_method("#{name}_#{key_method_name}") do
               ref = instance_variable_get(:"@#{name}_ref")
-              resolve_reference_key(ref)
+              # attr.reference_key first: once a collection reader has stored
+              # its resolved objects, the key has to come back off the object.
+              resolve_reference_key(attr.reference_key(ref))
             end
           end
 
           unless method_defined?(name, false)
-            define_method(name) do
-              ref = instance_variable_get(:"@#{name}_ref")
-              resolve_reference_value(ref)
+            if attr.options[:collection]
+              define_method(name) do
+                materialize_reference_collection(name)
+              end
+            else
+              define_method(name) do
+                ref = instance_variable_get(:"@#{name}_ref")
+                resolve_reference_value(ref)
+              end
             end
           end
 
@@ -101,7 +112,7 @@ def define_regular_attribute_methods(name, attr)
           if attr.collection?
             define_method(name) do |*args|
               if args.empty?
-                instance_variable_get(:"@#{name}")
+                materialize_lazy_collection(name)
               else
                 # Builder-style: g.member(item) appends to collection
                 value = args.first

lib/lutaml/model/serialize/builder.rb

@@ -75,11 +75,10 @@ def ordered?
           mapping&.ordered? || false
         end
 
-        # Whether this instance was constructed via a builder block and
-        # therefore records mutations into element_order. Parsed models
-        # and instances constructed without a block do not track; their
-        # element_order (if any) comes from the parser and is treated as
-        # the complete source of truth by the serializer.
+        # Whether this instance was constructed via a builder block and so
+        # records one element_order entry per mutation. A parsed instance
+        # does not: its order comes from the document, where one entry can
+        # stand for several rules that share an element name.
         # @return [Boolean]
         def order_tracking_enabled?
           @__order_tracking__ ? true : false

lib/lutaml/model/serialize/enum_handling.rb

@@ -49,15 +49,18 @@ def add_enum_methods_to_model(klass, enum_name, values,
               value_set_for(enum_name)
               enum_vals = public_send(:"#{enum_name}")
 
+              # `+` and `-` rather than `<<` and `delete`. The reader hands
+              # back the stored Array now, so mutating it here would reach an
+              # Array a caller already holds — and would land even when the
+              # store below raises on a frozen model.
               enum_vals = if !!val
                             if collection
-                              enum_vals << value
+                              enum_vals.include?(value) ? enum_vals : enum_vals + [value]
                             else
                               [value]
                             end
                           elsif collection
-                            enum_vals.delete(value)
-                            enum_vals
+                            enum_vals - [value]
                           else
                             instance_variable_get(:"@#{enum_name}") - [value]
                           end
@@ -78,13 +81,44 @@ def add_enum_methods_to_model(klass, enum_name, values,
         # @param collection [Boolean] Whether the enum is a collection
         def add_enum_getter_if_not_defined(klass, enum_name, collection)
           Utils.add_method_if_not_defined(klass, enum_name) do
-            i = instance_variable_get(:"@#{enum_name}") || []
+            unless collection
+              i = instance_variable_get(:"@#{enum_name}") || []
+              next i.is_a?(::Array) ? i.first : i
+            end
 
-            if !collection && i.is_a?(Array)
-              i.first
-            else
-              i.uniq
+            current = materialize_lazy_collection(enum_name)
+
+            # An enum collection reads as an Array even when nothing was
+            # stored — the old `(ivar || []).uniq` never returned nil — so
+            # unlike a regular collection this one materializes nil too, and
+            # stores what it materialized so a push reaches the model.
+            if current.nil?
+              next [] if frozen?
+
+              next instance_variable_set(:"@#{enum_name}", [])
             end
+
+            # Not an Array, which means a model-defined writer stored
+            # something else. Read it the way the old reader did and leave
+            # what it stored alone — replacing it here would discard data.
+            next current.uniq unless current.is_a?(::Array)
+
+            # A frozen Array cannot be deduped in place, and handing back a
+            # copy would lose the shared sentinel's identity, so only copy
+            # when there is actually something to remove.
+            if current.frozen?
+              deduped = current.uniq
+              next deduped.size == current.size ? current : deduped
+            end
+
+            # Hand back the stored Array, not a copy. This used to be
+            # `i.uniq`, a fresh Array on every read, so `model.roles << "b"`
+            # pushed onto a throwaway and the value never reached the model —
+            # no error, no warning, the item simply never showed up in the
+            # document. It still has to read unique, which is what that `uniq`
+            # guaranteed, so dedupe in place rather than into a copy.
+            current.uniq!
+            current
           end
         end
 
@@ -105,6 +139,11 @@ def add_enum_setter_if_not_defined(klass, enum_name, _values,
             if collection
               curr_value = public_send(:"#{enum_name}")
 
+              # Build a new Array rather than appending into the stored one.
+              # The reader hands back the real Array now, so `curr_value` may
+              # be this model's own collection — or, when the model defines
+              # its own reader, something shared between instances or not an
+              # Array at all. Duplicates are the reader's job either way.
               instance_variable_set(:"@#{enum_name}", curr_value + value)
             else
               instance_variable_set(:"@#{enum_name}", value)

lib/lutaml/model/serialize/initialization.rb

@@ -387,8 +387,10 @@ def define_scalar_register_methods(name)
         def define_collection_register_methods(name)
           define_method(name) do |*args|
             if args.empty?
-              current = instance_variable_get(:"@#{name}")
-              current.equal?(LAZY_EMPTY_COLLECTION) ? [] : current
+              # Store the materialized Array instead of discarding it, so
+              # `model.items << x` on a register-scoped model reaches the model
+              # rather than a throwaway copy.
+              materialize_lazy_collection(name)
             else
               value = args.first
               current = instance_variable_get(:"@#{name}")

lib/lutaml/xml/model_transform.rb

@@ -359,7 +359,10 @@ def validate_document!(doc, options)
       def set_instance_ordering(instance, doc, ordered_option,
 mixed_content_option, xml_mapping = nil,
 instance_is_serialize = nil)
-        instance.element_order = doc.root.order
+        # dup: XmlElement#order hands back a frozen cache shared with the
+        # DOM. The model's copy has to stay mutable so callers can maintain
+        # element_order themselves.
+        instance.element_order = doc.root.order.dup
         if instance_is_serialize && doc.root.is_a?(::Lutaml::Xml::XmlElement)
           instance.attribute_order = doc.root.attribute_order
         end

lib/lutaml/xml/transformation.rb

@@ -14,11 +14,13 @@ module Xml
     # - TransformationSupport::ValueSerializer: Serializes values to XML strings
     # - TransformationSupport::ElementBuilder: Creates XML elements from values
     # - TransformationSupport::OrderedApplier: Applies rules in element order for round-trip
+    # - TransformationSupport::OrderReconciler: Aligns element_order with current values
     # - TransformationSupport::RuleApplier: Dispatches rule application to handlers
     class Transformation < Lutaml::Model::Transformation
       include TransformationSupport::RuleCompiler
       include TransformationSupport::RuleApplier
       include TransformationSupport::OrderedApplier
+      include TransformationSupport::OrderReconciler
 
       # Transform a model instance into XmlElement tree
       #