From c45b29eb189938ba5ad6fc2b360d12b43973e0ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 11:55:08 +0200 Subject: [PATCH 01/58] Clean up dead code in _load_existing_ntuple --- src/uproot/writing/writable.py | 272 ++++++++++++++++++++++++++++- tests/test_rntuple_update.py | 306 +++++++++++++++++++++++++++++++++ 2 files changed, 570 insertions(+), 8 deletions(-) create mode 100644 tests/test_rntuple_update.py diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index b99642cfe..950ea4f65 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -128,7 +128,6 @@ def recreate(file_path: str | Path | IO, **options): "unrecognized options for uproot.create or uproot.recreate: " + ", ".join(repr(x) for x in options) ) - cascading = uproot.writing._cascade.create_empty( sink, compression, @@ -179,7 +178,6 @@ def update(file_path: str | Path | IO, **options): "unrecognized options for uproot.update: " + ", ".join(repr(x) for x in options) ) - cascading = uproot.writing._cascade.update_existing( sink, initial_directory_bytes, @@ -962,7 +960,6 @@ def _get_del_search(self, where, isget): keys=last._cascading.data.key_names, file_path=self.file_path, ) - return step else: @@ -1021,9 +1018,7 @@ def _get(self, name, cycle): if self._file._has_ntuple(key.seek_location): return self._file._get_ntuple(key.seek_location) else: - raise TypeError( - "WritableDirectory cannot view preexisting RNTuple; open the file with uproot.open instead of uproot.recreate or uproot.update" - ) + return self._load_existing_ntuple(key) else: @@ -1054,6 +1049,127 @@ def get_chunk(start, stop): return readonlykey.get() + def _load_existing_ntuple(self, key): + import numpy + + import uproot.writing._cascade as casc + import uproot.writing._cascadentuple as cnt + + name = key.name.string + existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) + existing = existing_file[name] + _ = existing.keys() + full_akform, _ = existing.to_akform() + am = existing._ntuple.all_members + existing_key = existing_file.key(name + ";1") + anchor_location = existing_key.fSeekKey + existing_key.fKeylen + num_entries = existing.num_entries + existing_footer = existing._footer + existing_page_list_envelopes = existing.page_list_envelopes + existing_field_records = existing._header.field_records + existing_file.close() + + header = cnt.NTuple_Header( + None, existing.name, existing._header.ntuple_description, full_akform + ) + header._checksum = existing._header.checksum + footer = cnt.NTuple_Footer(None, header._checksum) + + for cg in existing_footer.cluster_group_records: + locator = cnt.NTuple_Locator( + cg.page_list_link.locator.num_bytes, cg.page_list_link.locator.offset + ) + envlink = cnt.NTuple_EnvLink(cg.page_list_link.env_uncomp_size, locator) + footer.cluster_group_record_frames.append( + cnt.NTuple_ClusterGroupRecord( + cg.min_entry_num, cg.entry_span, cg.num_clusters, envlink + ) + ) + # copy extension field and column records from existing footer + for fr in existing_footer.extension_links.field_records: + new_field = cnt.NTuple_Field_Description( + fr.parent_field_id, + fr.struct_role, + fr.field_name, + fr.type_name, + field_description=fr.field_desc, + ) + footer.extension_field_record_frames.append(new_field) + for cr in existing_footer.extension_links.column_records: + new_col = cnt.NTuple_Column_Description( + cr.type, cr.nbits, cr.field_id, cr.flags, cr.repr_idx + ) + footer.extension_column_record_frames.append(new_col) + for cg in existing_footer.cluster_group_records: + loc = cg.page_list_link.locator + start = loc.offset - 56 + end = loc.offset + loc.num_bytes + self._cascading._freesegments._data.slices = [ + s + for s in self._cascading._freesegments._data.slices + if not (s[0] < end and start < s[1]) + ] + anchor = cnt.NTuple_Anchor( + anchor_location, + am["fVersionEpoch"], + am["fVersionMajor"], + am["fVersionMinor"], + am["fVersionPatch"], + am["fSeekHeader"], + am["fNBytesHeader"], + am["fLenHeader"], + am["fSeekFooter"], + am["fNBytesFooter"], + am["fLenFooter"], + am["fMaxKeySize"], + ) + ntuple_cascading = cnt.NTuple( + self._cascading, + full_akform, + self._cascading._freesegments, + header, + footer, + [], + anchor, + ) + ntuple_cascading._header_key = casc.Key( + am["fSeekHeader"] - 56, + am["fLenHeader"], + am["fNBytesHeader"], + casc.String(None, "RBlob"), + casc.String(None, ""), + casc.String(None, ""), + 1, + 100, + am["fSeekHeader"], + ) + ntuple_cascading._footer_key = casc.Key( + am["fSeekFooter"] - 56, + am["fLenFooter"], + am["fNBytesFooter"], + casc.String(None, "RBlob"), + casc.String(None, ""), + casc.String(None, ""), + 1, + 100, + am["fSeekFooter"], + ) + ntuple_cascading._num_entries = num_entries + full_header = cnt.NTuple_Header( + None, existing.name, existing._header.ntuple_description, full_akform + ) + ntuple_cascading._column_counts = numpy.array( + [num_entries] * len(full_header._column_keys), dtype=int + ) + ntuple_cascading._existing_footer = existing_footer + ntuple_cascading._existing_page_list_envelopes = existing_page_list_envelopes + ntuple_cascading._existing_field_records = existing_field_records + + path = (*self._path, name) + writable_ntuple = WritableNTuple(path, self._file, ntuple_cascading) + self._file._ntuples[anchor_location] = writable_ntuple + return writable_ntuple + def _del(self, name, cycle): key = self._cascading.data.get_key(name, cycle) if key is None: @@ -1609,7 +1725,6 @@ def update(self, pairs=None, **more_pairs): update. """ streamers = [] - if pairs is not None: if hasattr(pairs, "keys"): all_pairs = itertools.chain( @@ -1635,7 +1750,6 @@ def update(self, pairs=None, **more_pairs): directory = directory[item] uproot.writing.identify.add_to_directory(v, name, directory, streamers) - self._file._cascading.streamers.update_streamers(self._file.sink, streamers) @@ -1912,6 +2026,7 @@ def extend(self, data): """ self._cascading.extend(self._file, self._file.sink, data) + def show( self, *, @@ -2184,6 +2299,147 @@ def extend(self, data): """ self._cascading.extend(self._file, self._file.sink, data) + def add_fields(self, new_fields): + """ + Args: + new_fields (dict of str -> numpy dtype): New field names and types. + + Adds new fields to this RNTuple, back-filled with zeros for existing entries. + + For example, + + .. code-block:: python + + with uproot.update("file.root") as f: + f["mytuple"].add_fields({"z": np.int32}) + """ + import numpy + + import uproot.compression + import uproot.writing._cascadentuple as cnt + + compression = self._cascading._freesegments.fileheader.compression + num_entries = self._cascading._num_entries + header = self._cascading._header + footer = self._cascading._footer + existing_footer = self._cascading._existing_footer + existing_page_list_envelopes = self._cascading._existing_page_list_envelopes + existing_field_records = self._cascading._existing_field_records + + next_field_id = len(existing_field_records) + len( + existing_footer.extension_links.field_records + ) + new_pages = {} + + existing_field_names = {fr.field_name for fr in existing_field_records} + for field_name in new_fields: + if field_name in existing_field_names: + raise ValueError(f"Field {field_name!r} already exists in this RNTuple") + + for field_name, field_dtype_raw in new_fields.items(): + field_dtype = numpy.dtype(field_dtype_raw) + ak_primitive = { + numpy.dtype("float32"): "float32", + numpy.dtype("float64"): "float64", + numpy.dtype("int32"): "int32", + numpy.dtype("int64"): "int64", + numpy.dtype("uint32"): "uint32", + numpy.dtype("uint64"): "uint64", + }.get(field_dtype, "int32") + type_name = cnt._ak_primitive_to_typename_dict[ak_primitive] + type_num = cnt._ak_primitive_to_num_dict[ak_primitive] + type_size = uproot.const.rntuple_col_num_to_size_dict[type_num] + + new_field = cnt.NTuple_Field_Description( + next_field_id, + uproot.const.RNTupleFieldRole.LEAF, + field_name, + type_name, + ) + footer.extension_field_record_frames.append(new_field) + new_col = cnt.NTuple_Column_Description( + type_num, type_size, next_field_id, 0, 0 + ) + footer.extension_column_record_frames.append(new_col) + + new_data = numpy.zeros(num_entries, dtype=field_dtype) + raw_data = new_data.view("uint8") + compressed_data = uproot.compression.compress(raw_data, compression) + page_key = self._cascading.add_rblob( + self._file.sink, compressed_data, len(raw_data) + ) + page_locator = cnt.NTuple_Locator( + len(compressed_data), page_key.location + page_key.allocation + ) + new_pages[field_name] = cnt.NTuple_PageDescription( + num_entries, page_locator + ) + next_field_id += 1 + + footer.cluster_group_record_frames = [] + for cg_idx, cg in enumerate(existing_footer.cluster_group_records): + ple = existing_page_list_envelopes[cg_idx] + new_cluster_page_data = [] + for col_pages in ple.pagelinklist[0]: + existing_pages = [ + cnt.NTuple_PageDescription( + p.num_elements, + cnt.NTuple_Locator(p.locator.num_bytes, p.locator.offset), + ) + for p in col_pages.pages + ] + new_cluster_page_data.append( + cnt.NTuple_ColumnPageListDescription( + existing_pages, col_pages.element_offset, compression.code + ) + ) + for field_name in new_fields: + new_cluster_page_data.append( + cnt.NTuple_ColumnPageListDescription( + [new_pages[field_name]], 0, compression.code + ) + ) + cluster_summaries = [ + cnt.NTuple_ClusterSummary(s.num_first_entry, s.num_entries) + for s in ple.cluster_summaries + ] + pagelistenv = cnt.NTuple_PageListEnvelope( + header._checksum, cluster_summaries, [new_cluster_page_data] + ) + pagelistenv_raw = pagelistenv.serialize() + pagelistenv_key = self._cascading.add_rblob( + self._file.sink, pagelistenv_raw, len(pagelistenv_raw) + ) + pagelistenv_locator = cnt.NTuple_Locator( + len(pagelistenv_raw), + pagelistenv_key.location + pagelistenv_key.allocation, + ) + pagelistenv_envlink = cnt.NTuple_EnvLink( + len(pagelistenv_raw), pagelistenv_locator + ) + footer.cluster_group_record_frames.append( + cnt.NTuple_ClusterGroupRecord( + cg.min_entry_num, + cg.entry_span, + cg.num_clusters, + pagelistenv_envlink, + ) + ) + + footer_raw = footer.serialize() + new_footer_key = self._cascading.add_rblob( + self._file.sink, footer_raw, len(footer_raw) + ) + self._cascading._anchor.seek_footer = ( + new_footer_key.location + new_footer_key.allocation + ) + self._cascading._anchor.nbytes_footer = len(footer_raw) + self._cascading._anchor.len_footer = len(footer_raw) + anchor_raw = self._cascading._anchor.serialize() + self._file.sink.write(self._cascading._anchor._location, anchor_raw) + self._cascading._freesegments.write(self._file.sink) + self._file.sink.flush() + def _is_type_specification(obj): to_check = [obj] diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py new file mode 100644 index 000000000..894a85a57 --- /dev/null +++ b/tests/test_rntuple_update.py @@ -0,0 +1,306 @@ +import uproot +import os +import pytest + +ROOT = pytest.importorskip("ROOT") + +import numpy as np + +import awkward as ak +from skhep_testdata import data_path + +def test_extend_existing_ntuple(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = { + "x": np.array([1, 2, 3, 4, 5], dtype=np.float32), + "y": np.array([10, 20, 30, 40, 50], dtype=np.int32), + } + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend( + { + "x": np.array([6, 7, 8], dtype=np.float32), + "y": np.array([60, 70, 80], dtype=np.int32), + } + ) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all( + nt["x"].array() == np.array([1, 2, 3, 4, 5, 6, 7, 8], dtype=np.float32) + ) + assert ak.all( + nt["y"].array() + == np.array([10, 20, 30, 40, 50, 60, 70, 80], dtype=np.int32) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 8 + + +def test_add_field_ntuple(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3, 4, 5], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"z": np.int32}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) + assert ak.all(nt["z"].array() == np.zeros(5, dtype=np.int32)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 + + +def test_add_field_ntuple_duplicate(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="already exists"): + f["mytuple"].add_fields({"x": np.int32}) + + +def test_extend_ntuple_multiple_times(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend({"x": np.array([4, 5, 6], dtype=np.float32)}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend({"x": np.array([7, 8, 9], dtype=np.float32)}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert ak.all( + f["mytuple"]["x"].array() + == np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 9 + + +def test_add_multiple_fields_ntuple(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"y": np.int32, "z": np.float64}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert ak.all( + f["mytuple"]["x"].array() == np.array([1, 2, 3], dtype=np.float32) + ) + assert ak.all(f["mytuple"]["y"].array() == np.zeros(3, dtype=np.int32)) + assert ak.all(f["mytuple"]["z"].array() == np.zeros(3, dtype=np.float64)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 + + +def test_extend_ntuple_wrong_fields(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = { + "x": np.array([1, 2, 3], dtype=np.float32), + "y": np.array([4, 5, 6], dtype=np.int32), + } + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(Exception): + f["mytuple"].extend( + {"x": np.array([7, 8, 9], dtype=np.float32)} + ) # missing y + + +def test_ntuple_dtypes(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = { + "x_f32": np.array([1, 2, 3], dtype=np.float32), + "x_f64": np.array([1, 2, 3], dtype=np.float64), + "x_i32": np.array([1, 2, 3], dtype=np.int32), + "x_i64": np.array([1, 2, 3], dtype=np.int64), + } + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields( + { + "z_f32": np.float32, + "z_f64": np.float64, + "z_i32": np.int32, + "z_i64": np.int64, + } + ) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x_f32"].array() == np.array([1, 2, 3], dtype=np.float32)) + assert ak.all(nt["z_i64"].array() == np.zeros(3, dtype=np.int64)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 + + +def test_ntuple_variable_length(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = { + "x": ak.Array([[1, 2], [3, 4, 5], [6]]), + } + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend({"x": ak.Array([[7, 8, 9], [10]])}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert ak.all( + f["mytuple"]["x"].array() + == ak.Array([[1, 2], [3, 4, 5], [6], [7, 8, 9], [10]]) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 + + +def test_ntuple_mixed_types_extend(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = { + "pt": np.array([10.0, 20.0, 30.0], dtype=np.float32), + "jets": ak.Array([[1.0, 2.0], [3.0], [4.0, 5.0, 6.0]]), + } + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend( + { + "pt": np.array([40.0, 50.0], dtype=np.float32), + "jets": ak.Array([[7.0, 8.0, 9.0], [10.0]]), + } + ) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all( + nt["pt"].array() == np.array([10, 20, 30, 40, 50], dtype=np.float32) + ) + assert ak.all( + nt["jets"].array() == ak.Array([[1, 2], [3], [4, 5, 6], [7, 8, 9], [10]]) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 + + +def test_ntuple_add_field_then_extend(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + # add new field (backfilled with zeros) + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"z": np.int32}) + + # now extend with both fields + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend( + { + "x": np.array([4, 5, 6], dtype=np.float32), + "z": np.array([40, 50, 60], dtype=np.int32), + } + ) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5, 6], dtype=np.float32)) + assert ak.all( + nt["z"].array() == np.array([0, 0, 0, 40, 50, 60], dtype=np.int32) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 6 + + +def test_ntuple_extend_empty(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f.mkrntuple("mytuple", {"x": np.dtype("float32")}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend({"x": np.array([1, 2, 3], dtype=np.float32)}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert nt.num_entries == 3 + assert ak.all(nt["x"].array() == np.array([1, 2, 3], dtype=np.float32)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 + + +def test_ntuple_multiple_in_file(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["tuple1"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + f["tuple2"] = {"y": np.array([4, 5, 6], dtype=np.int32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["tuple1"].extend({"x": np.array([4, 5], dtype=np.float32)}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert ak.all( + f["tuple1"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32) + ) + assert ak.all(f["tuple2"]["y"].array() == np.array([4, 5, 6], dtype=np.int32)) + + reader1 = ROOT.RNTupleReader.Open("tuple1", os.path.join(tmp_path, "test.root")) + reader2 = ROOT.RNTupleReader.Open("tuple2", os.path.join(tmp_path, "test.root")) + assert reader1.GetNEntries() == 5 + assert reader2.GetNEntries() == 3 + + +def test_ntuple_multiple_add_fields_then_extend(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"y": np.int32}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"z": np.float64}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend( + { + "x": np.array([4, 5, 6], dtype=np.float32), + "y": np.array([40, 50, 60], dtype=np.int32), + "z": np.array([400.0, 500.0, 600.0], dtype=np.float64), + } + ) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5, 6], dtype=np.float32)) + assert ak.all( + nt["y"].array() == np.array([0, 0, 0, 40, 50, 60], dtype=np.int32) + ) + assert ak.all( + nt["z"].array() == np.array([0, 0, 0, 400, 500, 600], dtype=np.float64) + ) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 6 + +def test_ntuple_add_field_and_extend_same_session(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"y": np.int32}) + f["mytuple"].extend({ + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([40, 50], dtype=np.int32), + }) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) + assert ak.all(nt["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 \ No newline at end of file From 1729b65559c8fa82ec4de7409197e2131edfc8ee Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 09:58:23 +0000 Subject: [PATCH 02/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 1 - tests/test_rntuple_update.py | 14 +++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 950ea4f65..e658a6830 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2026,7 +2026,6 @@ def extend(self, data): """ self._cascading.extend(self._file, self._file.sink, data) - def show( self, *, diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index 894a85a57..3cd2b9e84 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -9,6 +9,7 @@ import awkward as ak from skhep_testdata import data_path + def test_extend_existing_ntuple(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = { @@ -286,16 +287,19 @@ def test_ntuple_multiple_add_fields_then_extend(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 6 + def test_ntuple_add_field_and_extend_same_session(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} with uproot.update(os.path.join(tmp_path, "test.root")) as f: f["mytuple"].add_fields({"y": np.int32}) - f["mytuple"].extend({ - "x": np.array([4, 5], dtype=np.float32), - "y": np.array([40, 50], dtype=np.int32), - }) + f["mytuple"].extend( + { + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([40, 50], dtype=np.int32), + } + ) with uproot.open(os.path.join(tmp_path, "test.root")) as f: nt = f["mytuple"] @@ -303,4 +307,4 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): assert ak.all(nt["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 \ No newline at end of file + assert reader.GetNEntries() == 5 From 775a8b8c5f06efa289c2672d468c72af54ae05ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:18:10 +0200 Subject: [PATCH 03/58] Implement accept_new_fields=True in WritableNTuple.extend() --- src/uproot/writing/writable.py | 22 +++++++++++++++++++++- tests/test_rntuple_update.py | 27 +++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 950ea4f65..5810c2c72 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2270,7 +2270,7 @@ def num_entries(self) -> int: """ return self._cascading.num_entries - def extend(self, data): + def extend(self, data, accept_new_fields=False): """ Args: data (dict of str \u2192 arrays): More array data to add to the RNTuple. @@ -2297,6 +2297,26 @@ def extend(self, data): **As a word of warning,** be sure that each call to :ref:`uproot.writing.writable.WritableNTuple.extend` includes at least 100 kB per branch/array. (NumPy and Awkward Arrays have an `nbytes `__ property; you want at least ``100000`` per array.) If you ask Uproot to write very small TBaskets, it will spend more time working on TBasket overhead than actually writing data. The absolute worst case is one-entry-per-:ref:`uproot.writing.writable.WritableTree.extend`. See `#428 (comment) `__. """ + if isinstance(data, dict): + existing_keys = set(self._cascading._header._akform.fields) + if hasattr(self._cascading, "_existing_field_records"): + existing_keys.update( + fr.field_name for fr in self._cascading._existing_field_records + ) + new_field_names = {k for k in data.keys() if k not in existing_keys} + if new_field_names and not accept_new_fields: + raise ValueError( + f"Data contains fields not in this RNTuple: {sorted(new_field_names)}. " + f"Call add_fields() first, or pass accept_new_fields=True to add them automatically." + ) + elif new_field_names and accept_new_fields: + # add new fields with zeros for existing entries + self.add_fields({k: numpy.array(list(data[k])).dtype for k in new_field_names}) + # reload from file so ntuple knows about new fields + key = self._file.root_directory._cascading.data.get_key(self._path[-1], 1) + reloaded = self._file.root_directory._load_existing_ntuple(key) + self._cascading = reloaded._cascading + self._cascading.extend(self._file, self._file.sink, data) def add_fields(self, new_fields): diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index 894a85a57..8b91605f2 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -302,5 +302,32 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 + +def test_ntuple_accept_new_fields(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + # should raise without accept_new_fields + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="not in this RNTuple"): + f["mytuple"].extend({ + "x": np.array([4, 5], dtype=np.float32), + "z": np.array([40, 50], dtype=np.int32), + }) + + # with accept_new_fields=True - z backfilled with zeros, then user values + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].extend({ + "x": np.array([4, 5], dtype=np.float32), + "z": np.array([40, 50], dtype=np.int32), + }, accept_new_fields=True) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) + assert ak.all(nt["z"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 \ No newline at end of file From 411703af0c552010cf65e8938ac708176bf6bfa8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 10:20:14 +0000 Subject: [PATCH 04/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 8 ++++++-- tests/test_rntuple_update.py | 24 +++++++++++++++--------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 752e41505..5aaf5fd0b 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2310,9 +2310,13 @@ def extend(self, data, accept_new_fields=False): ) elif new_field_names and accept_new_fields: # add new fields with zeros for existing entries - self.add_fields({k: numpy.array(list(data[k])).dtype for k in new_field_names}) + self.add_fields( + {k: numpy.array(list(data[k])).dtype for k in new_field_names} + ) # reload from file so ntuple knows about new fields - key = self._file.root_directory._cascading.data.get_key(self._path[-1], 1) + key = self._file.root_directory._cascading.data.get_key( + self._path[-1], 1 + ) reloaded = self._file.root_directory._load_existing_ntuple(key) self._cascading = reloaded._cascading diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index f5b36d14f..5619d95d2 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -309,6 +309,7 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + def test_ntuple_accept_new_fields(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} @@ -316,17 +317,22 @@ def test_ntuple_accept_new_fields(tmp_path): # should raise without accept_new_fields with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises(ValueError, match="not in this RNTuple"): - f["mytuple"].extend({ - "x": np.array([4, 5], dtype=np.float32), - "z": np.array([40, 50], dtype=np.int32), - }) + f["mytuple"].extend( + { + "x": np.array([4, 5], dtype=np.float32), + "z": np.array([40, 50], dtype=np.int32), + } + ) # with accept_new_fields=True - z backfilled with zeros, then user values with uproot.update(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"].extend({ - "x": np.array([4, 5], dtype=np.float32), - "z": np.array([40, 50], dtype=np.int32), - }, accept_new_fields=True) + f["mytuple"].extend( + { + "x": np.array([4, 5], dtype=np.float32), + "z": np.array([40, 50], dtype=np.int32), + }, + accept_new_fields=True, + ) with uproot.open(os.path.join(tmp_path, "test.root")) as f: nt = f["mytuple"] @@ -334,4 +340,4 @@ def test_ntuple_accept_new_fields(tmp_path): assert ak.all(nt["z"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 \ No newline at end of file + assert reader.GetNEntries() == 5 From 3c0b47e66e891a6902eb1e10d9ee163a67a27f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:08:35 +0200 Subject: [PATCH 05/58] Add subfield support in add_fields() and test --- src/uproot/writing/writable.py | 17 +++++++++++++++-- tests/test_rntuple_update.py | 22 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 752e41505..c6ae8a2c3 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2369,10 +2369,23 @@ def add_fields(self, new_fields): type_num = cnt._ak_primitive_to_num_dict[ak_primitive] type_size = uproot.const.rntuple_col_num_to_size_dict[type_num] + if "." in field_name: + parent_name, actual_field_name = field_name.rsplit(".", 1) + parent_field_id = None + for i, fr in enumerate(existing_field_records): + if fr.field_name == parent_name: + parent_field_id = i + break + if parent_field_id is None: + raise ValueError(f"Parent field {parent_name!r} not found in RNTuple") + else: + actual_field_name = field_name + parent_field_id = next_field_id + new_field = cnt.NTuple_Field_Description( - next_field_id, + parent_field_id, uproot.const.RNTupleFieldRole.LEAF, - field_name, + actual_field_name, type_name, ) footer.extension_field_record_frames.append(new_field) diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index f5b36d14f..7707ff15f 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -334,4 +334,24 @@ def test_ntuple_accept_new_fields(tmp_path): assert ak.all(nt["z"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 \ No newline at end of file + assert reader.GetNEntries() == 5 + +def test_ntuple_add_subfield(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array([ + {"particle": {"pt": 1.0, "eta": 2.0}}, + {"particle": {"pt": 3.0, "eta": 4.0}}, + ]) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"particle.phi": np.float32}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert "particle.phi" in nt.keys() + assert ak.all(nt["particle.phi"].array() == np.zeros(2, dtype=np.float32)) + assert ak.all(nt["particle"].array().pt == np.array([1.0, 3.0])) + assert ak.all(nt["particle"].array().phi == np.zeros(2, dtype=np.float32)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 2 \ No newline at end of file From b1d7f423ae6b7abc0ce174ee677876a0cd3de14d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:10:03 +0000 Subject: [PATCH 06/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 4 +++- tests/test_rntuple_update.py | 11 +++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 4a2a17fd6..9606d836e 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2381,7 +2381,9 @@ def add_fields(self, new_fields): parent_field_id = i break if parent_field_id is None: - raise ValueError(f"Parent field {parent_name!r} not found in RNTuple") + raise ValueError( + f"Parent field {parent_name!r} not found in RNTuple" + ) else: actual_field_name = field_name parent_field_id = next_field_id diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index ef7957074..02702f0dd 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -342,12 +342,15 @@ def test_ntuple_accept_new_fields(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + def test_ntuple_add_subfield(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"] = ak.Array([ - {"particle": {"pt": 1.0, "eta": 2.0}}, - {"particle": {"pt": 3.0, "eta": 4.0}}, - ]) + f["mytuple"] = ak.Array( + [ + {"particle": {"pt": 1.0, "eta": 2.0}}, + {"particle": {"pt": 3.0, "eta": 4.0}}, + ] + ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: f["mytuple"].add_fields({"particle.phi": np.float32}) From 4ec88a599a22ae99a1f795db4c708f71799fdf6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:13:49 +0200 Subject: [PATCH 07/58] Add nested subfield support and test --- src/uproot/writing/writable.py | 4 +++- tests/test_rntuple_update.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 4a2a17fd6..95811a344 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2374,7 +2374,9 @@ def add_fields(self, new_fields): type_size = uproot.const.rntuple_col_num_to_size_dict[type_num] if "." in field_name: - parent_name, actual_field_name = field_name.rsplit(".", 1) + parts = field_name.split(".") + actual_field_name = parts[-1] # phi + parent_name = parts[-2] # track (immediate parent) parent_field_id = None for i, fr in enumerate(existing_field_records): if fr.field_name == parent_name: diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index ef7957074..7d69ad9f9 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -361,3 +361,21 @@ def test_ntuple_add_subfield(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 + +def test_ntuple_add_nested_subfield(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array([ + {"particle": {"track": {"pt": 1.0}}}, + {"particle": {"track": {"pt": 3.0}}}, + ]) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"particle.track.phi": np.float32}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert "particle.track.phi" in nt.keys() + assert ak.all(nt["particle.track.phi"].array() == np.zeros(2, dtype=np.float32)) + + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 2 \ No newline at end of file From 792529050b64774a8d1175338331491a675200f6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:14:32 +0000 Subject: [PATCH 08/58] style: pre-commit fixes --- tests/test_rntuple_update.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index f659bf8d8..ba963b055 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -365,12 +365,15 @@ def test_ntuple_add_subfield(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 + def test_ntuple_add_nested_subfield(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"] = ak.Array([ - {"particle": {"track": {"pt": 1.0}}}, - {"particle": {"track": {"pt": 3.0}}}, - ]) + f["mytuple"] = ak.Array( + [ + {"particle": {"track": {"pt": 1.0}}}, + {"particle": {"track": {"pt": 3.0}}}, + ] + ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: f["mytuple"].add_fields({"particle.track.phi": np.float32}) @@ -381,4 +384,4 @@ def test_ntuple_add_nested_subfield(tmp_path): assert ak.all(nt["particle.track.phi"].array() == np.zeros(2, dtype=np.float32)) reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 2 \ No newline at end of file + assert reader.GetNEntries() == 2 From bf5fc8db750165779741e45c1f9cfcf6b28cd06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:32:01 +0200 Subject: [PATCH 09/58] Optimize _load_existing_ntuple: count columns directly instead of constructing full header --- src/uproot/writing/writable.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 0dc79583b..8f91b3fe7 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1155,11 +1155,9 @@ def _load_existing_ntuple(self, key): am["fSeekFooter"], ) ntuple_cascading._num_entries = num_entries - full_header = cnt.NTuple_Header( - None, existing.name, existing._header.ntuple_description, full_akform - ) + num_columns = len(existing._header.column_records) + len(existing._footer.extension_links.column_records) ntuple_cascading._column_counts = numpy.array( - [num_entries] * len(full_header._column_keys), dtype=int + [num_entries] * num_columns, dtype=int ) ntuple_cascading._existing_footer = existing_footer ntuple_cascading._existing_page_list_envelopes = existing_page_list_envelopes From bba7c59ea4e9866d5437896883e0a03f43963357 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:34:33 +0000 Subject: [PATCH 10/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 8f91b3fe7..3d5e103da 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1155,7 +1155,9 @@ def _load_existing_ntuple(self, key): am["fSeekFooter"], ) ntuple_cascading._num_entries = num_entries - num_columns = len(existing._header.column_records) + len(existing._footer.extension_links.column_records) + num_columns = len(existing._header.column_records) + len( + existing._footer.extension_links.column_records + ) ntuple_cascading._column_counts = numpy.array( [num_entries] * num_columns, dtype=int ) From a61f7f49c48a1ba0380e9084290a65563e3b468d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:56:25 +0200 Subject: [PATCH 11/58] Add/update docstrings for _load_existing_ntuple, extend, add_fields --- src/uproot/writing/writable.py | 36 +++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 8f91b3fe7..84c01c0ac 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1050,6 +1050,21 @@ def get_chunk(start, stop): return readonlykey.get() def _load_existing_ntuple(self, key): + """ + Loads an existing RNTuple from disk and reconstructs a writable + :doc:`uproot.writing.writable.WritableNTuple` object from it. + + This is called when accessing a preexisting RNTuple via + ``f["name"]`` in update mode. Reads the existing metadata + (anchor, header, footer, page lists) and sets up the in-memory + state needed for subsequent ``extend`` or ``add_fields`` calls. + + Args: + key: The ROOT key object pointing to the RNTuple in the file. + + Returns: + :doc:`uproot.writing.writable.WritableNTuple` + """ import numpy import uproot.writing._cascade as casc @@ -2271,6 +2286,12 @@ def extend(self, data, accept_new_fields=False): """ Args: data (dict of str \u2192 arrays): More array data to add to the RNTuple. + accept_new_fields (bool): If False (default), raises ValueError if + data contains fields not already in the RNTuple, forcing the user + to call :ref:`uproot.writing.writable.WritableNTuple.add_fields` + first. If True, new fields are automatically added back-filled + with zeros for existing entries, then extended with the provided + values for new entries. This method adds data to an existing RNTuple, whether it was created through assignment or :doc:`uproot.writing.writable.WritableDirectory.mkrntuple`. @@ -2285,10 +2306,13 @@ def extend(self, data, accept_new_fields=False): .. code-block:: python - my_directory.mkrntuple("ntuple6", {"branch1": numpy_dtype, "branch2": awkward_type}) + with uproot.update("file.root") as f: + f["mytuple"].extend({"x": np.array([4, 5, 6])}) - my_directory["ntuple6"].extend({"branch1": another_numpy_array, - "branch2": another_awkward_array}) + # automatically add new field and extend + with uproot.update("file.root") as f: + f["mytuple"].extend({"x": np.array([7, 8]), "z": np.array([70, 80])}, + accept_new_fields=True) .. warning:: @@ -2307,11 +2331,9 @@ def extend(self, data, accept_new_fields=False): f"Call add_fields() first, or pass accept_new_fields=True to add them automatically." ) elif new_field_names and accept_new_fields: - # add new fields with zeros for existing entries self.add_fields( {k: numpy.array(list(data[k])).dtype for k in new_field_names} ) - # reload from file so ntuple knows about new fields key = self._file.root_directory._cascading.data.get_key( self._path[-1], 1 ) @@ -2373,8 +2395,8 @@ def add_fields(self, new_fields): if "." in field_name: parts = field_name.split(".") - actual_field_name = parts[-1] # phi - parent_name = parts[-2] # track (immediate parent) + actual_field_name = parts[-1] + parent_name = parts[-2] parent_field_id = None for i, fr in enumerate(existing_field_records): if fr.field_name == parent_name: From 4c0f5cc39a23acbfe80fb75870fe5e2dde950a18 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 12:57:14 +0000 Subject: [PATCH 12/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index c0433cd18..999621969 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2397,8 +2397,8 @@ def add_fields(self, new_fields): if "." in field_name: parts = field_name.split(".") - actual_field_name = parts[-1] - parent_name = parts[-2] + actual_field_name = parts[-1] + parent_name = parts[-2] parent_field_id = None for i, fr in enumerate(existing_field_records): if fr.field_name == parent_name: From 7b1d83a6e69fd933620bf45082ce72339a15b691 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:13:46 +0200 Subject: [PATCH 13/58] Skip ROOT.RNTupleReader checks on older ROOT versions --- tests/test_rntuple_update.py | 70 +++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 28 deletions(-) diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index ba963b055..bbd5b6699 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -35,8 +35,9 @@ def test_extend_existing_ntuple(tmp_path): == np.array([10, 20, 30, 40, 50, 60, 70, 80], dtype=np.int32) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 8 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 8 def test_add_field_ntuple(tmp_path): @@ -51,8 +52,9 @@ def test_add_field_ntuple(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["z"].array() == np.zeros(5, dtype=np.int32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 def test_add_field_ntuple_duplicate(tmp_path): @@ -80,8 +82,9 @@ def test_extend_ntuple_multiple_times(tmp_path): == np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 9 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 9 def test_add_multiple_fields_ntuple(tmp_path): @@ -98,8 +101,9 @@ def test_add_multiple_fields_ntuple(tmp_path): assert ak.all(f["mytuple"]["y"].array() == np.zeros(3, dtype=np.int32)) assert ak.all(f["mytuple"]["z"].array() == np.zeros(3, dtype=np.float64)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 3 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 def test_extend_ntuple_wrong_fields(tmp_path): @@ -140,8 +144,9 @@ def test_ntuple_dtypes(tmp_path): assert ak.all(nt["x_f32"].array() == np.array([1, 2, 3], dtype=np.float32)) assert ak.all(nt["z_i64"].array() == np.zeros(3, dtype=np.int64)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 3 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 def test_ntuple_variable_length(tmp_path): @@ -159,8 +164,9 @@ def test_ntuple_variable_length(tmp_path): == ak.Array([[1, 2], [3, 4, 5], [6], [7, 8, 9], [10]]) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 def test_ntuple_mixed_types_extend(tmp_path): @@ -187,8 +193,9 @@ def test_ntuple_mixed_types_extend(tmp_path): nt["jets"].array() == ak.Array([[1, 2], [3], [4, 5, 6], [7, 8, 9], [10]]) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 def test_ntuple_add_field_then_extend(tmp_path): @@ -215,8 +222,9 @@ def test_ntuple_add_field_then_extend(tmp_path): nt["z"].array() == np.array([0, 0, 0, 40, 50, 60], dtype=np.int32) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 6 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 6 def test_ntuple_extend_empty(tmp_path): @@ -231,8 +239,9 @@ def test_ntuple_extend_empty(tmp_path): assert nt.num_entries == 3 assert ak.all(nt["x"].array() == np.array([1, 2, 3], dtype=np.float32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 3 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 3 def test_ntuple_multiple_in_file(tmp_path): @@ -284,8 +293,9 @@ def test_ntuple_multiple_add_fields_then_extend(tmp_path): nt["z"].array() == np.array([0, 0, 0, 400, 500, 600], dtype=np.float64) ) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 6 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 6 def test_ntuple_add_field_and_extend_same_session(tmp_path): @@ -306,8 +316,9 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 def test_ntuple_accept_new_fields(tmp_path): @@ -339,8 +350,9 @@ def test_ntuple_accept_new_fields(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["z"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 5 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 5 def test_ntuple_add_subfield(tmp_path): @@ -362,8 +374,9 @@ def test_ntuple_add_subfield(tmp_path): assert ak.all(nt["particle"].array().pt == np.array([1.0, 3.0])) assert ak.all(nt["particle"].array().phi == np.zeros(2, dtype=np.float32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 2 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 2 def test_ntuple_add_nested_subfield(tmp_path): @@ -383,5 +396,6 @@ def test_ntuple_add_nested_subfield(tmp_path): assert "particle.track.phi" in nt.keys() assert ak.all(nt["particle.track.phi"].array() == np.zeros(2, dtype=np.float32)) - reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) - assert reader.GetNEntries() == 2 + if hasattr(ROOT, "RNTupleReader"): + reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) + assert reader.GetNEntries() == 2 From ef30e8af49f22f0f40f1f3af759d709624eecbbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Tue, 21 Jul 2026 17:29:49 +0200 Subject: [PATCH 14/58] Fix missing hasattr check for RNTupleReader in test_ntuple_multiple_in_file --- tests/test_rntuple_update.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_rntuple_update.py b/tests/test_rntuple_update.py index bbd5b6699..5bac0dc4c 100644 --- a/tests/test_rntuple_update.py +++ b/tests/test_rntuple_update.py @@ -258,10 +258,11 @@ def test_ntuple_multiple_in_file(tmp_path): ) assert ak.all(f["tuple2"]["y"].array() == np.array([4, 5, 6], dtype=np.int32)) - reader1 = ROOT.RNTupleReader.Open("tuple1", os.path.join(tmp_path, "test.root")) - reader2 = ROOT.RNTupleReader.Open("tuple2", os.path.join(tmp_path, "test.root")) - assert reader1.GetNEntries() == 5 - assert reader2.GetNEntries() == 3 + if hasattr(ROOT, "RNTupleReader"): + reader1 = ROOT.RNTupleReader.Open("tuple1", os.path.join(tmp_path, "test.root")) + reader2 = ROOT.RNTupleReader.Open("tuple2", os.path.join(tmp_path, "test.root")) + assert reader1.GetNEntries() == 5 + assert reader2.GetNEntries() == 3 def test_ntuple_multiple_add_fields_then_extend(tmp_path): From adfe64ce815324352a73f90768b80a962c2abc59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:05:00 +0200 Subject: [PATCH 15/58] Move imports to top of file per review feedback --- src/uproot/writing/writable.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 999621969..7c0767f93 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1065,7 +1065,6 @@ def _load_existing_ntuple(self, key): Returns: :doc:`uproot.writing.writable.WritableNTuple` """ - import numpy import uproot.writing._cascade as casc import uproot.writing._cascadentuple as cnt @@ -2358,9 +2357,7 @@ def add_fields(self, new_fields): with uproot.update("file.root") as f: f["mytuple"].add_fields({"z": np.int32}) """ - import numpy - import uproot.compression import uproot.writing._cascadentuple as cnt compression = self._cascading._freesegments.fileheader.compression From 7d50cbcc1219e7ea1ebc2a8c8dc37d39838ce956 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:15:05 +0200 Subject: [PATCH 16/58] Fix magic number 56 -> computed _rblob_key_size --- src/uproot/writing/writable.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 7c0767f93..d4e5ff476 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1069,6 +1069,8 @@ def _load_existing_ntuple(self, key): import uproot.writing._cascade as casc import uproot.writing._cascadentuple as cnt + _rblob_key_size = uproot.reading._key_format_big.size + 8 + name = key.name.string existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) existing = existing_file[name] @@ -1116,7 +1118,7 @@ def _load_existing_ntuple(self, key): footer.extension_column_record_frames.append(new_col) for cg in existing_footer.cluster_group_records: loc = cg.page_list_link.locator - start = loc.offset - 56 + start = loc.offset - _rblob_key_size end = loc.offset + loc.num_bytes self._cascading._freesegments._data.slices = [ s @@ -1147,7 +1149,7 @@ def _load_existing_ntuple(self, key): anchor, ) ntuple_cascading._header_key = casc.Key( - am["fSeekHeader"] - 56, + am["fSeekHeader"] - _rblob_key_size, am["fLenHeader"], am["fNBytesHeader"], casc.String(None, "RBlob"), @@ -1158,7 +1160,7 @@ def _load_existing_ntuple(self, key): am["fSeekHeader"], ) ntuple_cascading._footer_key = casc.Key( - am["fSeekFooter"] - 56, + am["fSeekFooter"] - _rblob_key_size, am["fLenFooter"], am["fNBytesFooter"], casc.String(None, "RBlob"), From b172b15653b9d219154c6799cfb9a85f4174fea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:23:18 +0200 Subject: [PATCH 17/58] Normalize add_fields input with _type_specification_to_awkward_form --- src/uproot/writing/writable.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index d4e5ff476..08e9206fa 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2381,15 +2381,12 @@ def add_fields(self, new_fields): raise ValueError(f"Field {field_name!r} already exists in this RNTuple") for field_name, field_dtype_raw in new_fields.items(): - field_dtype = numpy.dtype(field_dtype_raw) - ak_primitive = { - numpy.dtype("float32"): "float32", - numpy.dtype("float64"): "float64", - numpy.dtype("int32"): "int32", - numpy.dtype("int64"): "int64", - numpy.dtype("uint32"): "uint32", - numpy.dtype("uint64"): "uint64", - }.get(field_dtype, "int32") + ak_form = _type_specification_to_awkward_form(field_dtype_raw) + if not isinstance(ak_form, awkward.forms.NumpyForm): + raise TypeError( + f"add_fields only supports simple numeric types, got {field_dtype_raw!r}" + ) + ak_primitive = ak_form.primitive type_name = cnt._ak_primitive_to_typename_dict[ak_primitive] type_num = cnt._ak_primitive_to_num_dict[ak_primitive] type_size = uproot.const.rntuple_col_num_to_size_dict[type_num] @@ -2423,7 +2420,7 @@ def add_fields(self, new_fields): ) footer.extension_column_record_frames.append(new_col) - new_data = numpy.zeros(num_entries, dtype=field_dtype) + new_data = numpy.zeros(num_entries, dtype=numpy.dtype(ak_primitive)) raw_data = new_data.view("uint8") compressed_data = uproot.compression.compress(raw_data, compression) page_key = self._cascading.add_rblob( From 63667b03712f0ec7c5e5c68c53e35860d04ef90f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:24:45 +0200 Subject: [PATCH 18/58] Rename test file to follow convention test_1687_rntuple_update.py --- tests/{test_rntuple_update.py => test_1687_rntuple_update.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{test_rntuple_update.py => test_1687_rntuple_update.py} (100%) diff --git a/tests/test_rntuple_update.py b/tests/test_1687_rntuple_update.py similarity index 100% rename from tests/test_rntuple_update.py rename to tests/test_1687_rntuple_update.py From 792969622fc273256b63ecb1f162bf4018af4fa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:30:54 +0200 Subject: [PATCH 19/58] Fix test file: only skip ROOT parts when ROOT not available --- tests/test_1687_rntuple_update.py | 45 ++++++++++++++++--------------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 5bac0dc4c..e77b1cc98 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -1,14 +1,15 @@ -import uproot import os -import pytest - -ROOT = pytest.importorskip("ROOT") - -import numpy as np import awkward as ak -from skhep_testdata import data_path +import numpy as np +import pytest +import uproot +try: + import ROOT + has_root = True +except ImportError: + has_root = False def test_extend_existing_ntuple(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: @@ -35,7 +36,7 @@ def test_extend_existing_ntuple(tmp_path): == np.array([10, 20, 30, 40, 50, 60, 70, 80], dtype=np.int32) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 8 @@ -52,7 +53,7 @@ def test_add_field_ntuple(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["z"].array() == np.zeros(5, dtype=np.int32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 @@ -82,7 +83,7 @@ def test_extend_ntuple_multiple_times(tmp_path): == np.array([1, 2, 3, 4, 5, 6, 7, 8, 9], dtype=np.float32) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 9 @@ -101,7 +102,7 @@ def test_add_multiple_fields_ntuple(tmp_path): assert ak.all(f["mytuple"]["y"].array() == np.zeros(3, dtype=np.int32)) assert ak.all(f["mytuple"]["z"].array() == np.zeros(3, dtype=np.float64)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 3 @@ -144,7 +145,7 @@ def test_ntuple_dtypes(tmp_path): assert ak.all(nt["x_f32"].array() == np.array([1, 2, 3], dtype=np.float32)) assert ak.all(nt["z_i64"].array() == np.zeros(3, dtype=np.int64)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 3 @@ -164,7 +165,7 @@ def test_ntuple_variable_length(tmp_path): == ak.Array([[1, 2], [3, 4, 5], [6], [7, 8, 9], [10]]) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 @@ -193,7 +194,7 @@ def test_ntuple_mixed_types_extend(tmp_path): nt["jets"].array() == ak.Array([[1, 2], [3], [4, 5, 6], [7, 8, 9], [10]]) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 @@ -222,7 +223,7 @@ def test_ntuple_add_field_then_extend(tmp_path): nt["z"].array() == np.array([0, 0, 0, 40, 50, 60], dtype=np.int32) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 6 @@ -239,7 +240,7 @@ def test_ntuple_extend_empty(tmp_path): assert nt.num_entries == 3 assert ak.all(nt["x"].array() == np.array([1, 2, 3], dtype=np.float32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 3 @@ -258,7 +259,7 @@ def test_ntuple_multiple_in_file(tmp_path): ) assert ak.all(f["tuple2"]["y"].array() == np.array([4, 5, 6], dtype=np.int32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader1 = ROOT.RNTupleReader.Open("tuple1", os.path.join(tmp_path, "test.root")) reader2 = ROOT.RNTupleReader.Open("tuple2", os.path.join(tmp_path, "test.root")) assert reader1.GetNEntries() == 5 @@ -294,7 +295,7 @@ def test_ntuple_multiple_add_fields_then_extend(tmp_path): nt["z"].array() == np.array([0, 0, 0, 400, 500, 600], dtype=np.float64) ) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 6 @@ -317,7 +318,7 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 @@ -351,7 +352,7 @@ def test_ntuple_accept_new_fields(tmp_path): assert ak.all(nt["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(nt["z"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 @@ -375,7 +376,7 @@ def test_ntuple_add_subfield(tmp_path): assert ak.all(nt["particle"].array().pt == np.array([1.0, 3.0])) assert ak.all(nt["particle"].array().phi == np.zeros(2, dtype=np.float32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 @@ -397,6 +398,6 @@ def test_ntuple_add_nested_subfield(tmp_path): assert "particle.track.phi" in nt.keys() assert ak.all(nt["particle.track.phi"].array() == np.zeros(2, dtype=np.float32)) - if hasattr(ROOT, "RNTupleReader"): + if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 From eacfbe27dbc52a7bc5ae10142b02c37f2921648e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:37:46 +0200 Subject: [PATCH 20/58] Add validation for subfield addition edge cases --- src/uproot/writing/writable.py | 9 +++++++++ tests/test_1687_rntuple_update.py | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 08e9206fa..d4eb88e17 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2399,6 +2399,15 @@ def add_fields(self, new_fields): for i, fr in enumerate(existing_field_records): if fr.field_name == parent_name: parent_field_id = i + if fr.type_name != "": + raise ValueError( + f"Field {parent_name!r} has type {fr.type_name!r} and cannot be extended. " + f"Only untyped records (empty type_name) can have subfields added." + ) + if fr.struct_role != uproot.const.RNTupleFieldRole.RECORD: + raise ValueError( + f"Field {parent_name!r} is not a record and cannot have subfields added." + ) break if parent_field_id is None: raise ValueError( diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index e77b1cc98..e1a546ff7 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -401,3 +401,29 @@ def test_ntuple_add_nested_subfield(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 + +def test_ntuple_add_subfield_nonexistent_parent(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="not found"): + f["mytuple"].add_fields({"nonexistent.phi": np.float32}) + + +def test_ntuple_add_subfield_typed_parent(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array([{"pt": 1.0}, {"pt": 2.0}]) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises((ValueError, TypeError)): + f["mytuple"].add_fields({"pt.x": np.float32}) + + +def test_ntuple_add_subfield_to_collection(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"jets": ak.Array([[1.0, 2.0], [3.0]])} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises((ValueError, TypeError)): + f["mytuple"].add_fields({"jets.x": np.float32}) \ No newline at end of file From 62d94b35accba1758c9f2a2db1d81e6bd1788a32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:47:45 +0200 Subject: [PATCH 21/58] Add test for num_entries on existing ntuple in update mode --- tests/test_1687_rntuple_update.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index e1a546ff7..a62a128d8 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -426,4 +426,12 @@ def test_ntuple_add_subfield_to_collection(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises((ValueError, TypeError)): - f["mytuple"].add_fields({"jets.x": np.float32}) \ No newline at end of file + f["mytuple"].add_fields({"jets.x": np.float32}) + +def test_ntuple_num_entries(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + assert nt.num_entries == 3 \ No newline at end of file From 4f8b4d9ff1c841c81e70397a98b3e5627ba8efe0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 12:48:24 +0000 Subject: [PATCH 22/58] style: pre-commit fixes --- tests/test_1687_rntuple_update.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index a62a128d8..cd62c4810 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -7,10 +7,12 @@ try: import ROOT + has_root = True except ImportError: has_root = False + def test_extend_existing_ntuple(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = { @@ -402,6 +404,7 @@ def test_ntuple_add_nested_subfield(tmp_path): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 2 + def test_ntuple_add_subfield_nonexistent_parent(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} @@ -428,10 +431,11 @@ def test_ntuple_add_subfield_to_collection(tmp_path): with pytest.raises((ValueError, TypeError)): f["mytuple"].add_fields({"jets.x": np.float32}) + def test_ntuple_num_entries(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} with uproot.update(os.path.join(tmp_path, "test.root")) as f: nt = f["mytuple"] - assert nt.num_entries == 3 \ No newline at end of file + assert nt.num_entries == 3 From f56b52265c1b5d142127552ca741f8707e1c9acd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:19:35 +0200 Subject: [PATCH 23/58] Add TODO comment for uproot.open in _load_existing_ntuple per review --- src/uproot/writing/writable.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index d4eb88e17..d42023daf 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1072,6 +1072,9 @@ def _load_existing_ntuple(self, key): _rblob_key_size = uproot.reading._key_format_big.size + 8 name = key.name.string + # TODO: opening the file again in read mode to access existing metadata is + # a bit awkward since the file is already open in write mode. We should + # look into a better way to do this in the future. existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) existing = existing_file[name] _ = existing.keys() From 35f71eb2a61bff2927cfd7aee436b41052d01770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:25:25 +0200 Subject: [PATCH 24/58] Use _ntuple.field_records for duplicate check and fix next_field_id --- src/uproot/writing/writable.py | 7 +++---- tests/test_1687_rntuple_update.py | 14 +++++++++++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index d42023daf..039166a1c 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1085,7 +1085,7 @@ def _load_existing_ntuple(self, key): num_entries = existing.num_entries existing_footer = existing._footer existing_page_list_envelopes = existing.page_list_envelopes - existing_field_records = existing._header.field_records + existing_field_records = existing._ntuple.field_records existing_file.close() header = cnt.NTuple_Header( @@ -2373,9 +2373,8 @@ def add_fields(self, new_fields): existing_page_list_envelopes = self._cascading._existing_page_list_envelopes existing_field_records = self._cascading._existing_field_records - next_field_id = len(existing_field_records) + len( - existing_footer.extension_links.field_records - ) + next_field_id = len(existing_field_records) + new_pages = {} existing_field_names = {fr.field_name for fr in existing_field_records} diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index a62a128d8..3ed9ce651 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -434,4 +434,16 @@ def test_ntuple_num_entries(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: nt = f["mytuple"] - assert nt.num_entries == 3 \ No newline at end of file + assert nt.num_entries == 3 + +def test_ntuple_add_field_duplicate_after_extension(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"z": np.int32}) + + # try to add z again - should fail even though it's an extension field + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="already exists"): + f["mytuple"].add_fields({"z": np.float32}) \ No newline at end of file From 0b0951764c4822e70451456b397fee52038e8693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:29:48 +0200 Subject: [PATCH 25/58] Use awkward.asarray instead of list() for dtype detection per review --- src/uproot/writing/writable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 039166a1c..ed0c08741 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2338,7 +2338,7 @@ def extend(self, data, accept_new_fields=False): ) elif new_field_names and accept_new_fields: self.add_fields( - {k: numpy.array(list(data[k])).dtype for k in new_field_names} + {k: numpy.asarray(awkward.flatten(awkward.Array(data[k]), axis=None)).dtype for k in new_field_names} ) key = self._file.root_directory._cascading.data.get_key( self._path[-1], 1 From 411b80036d227f05eafb1dcddbc252df24cc6ff7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:39:49 +0200 Subject: [PATCH 26/58] Add tests for subfield addition edge cases per review --- tests/test_1687_rntuple_update.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 3ed9ce651..51ea694d7 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -428,6 +428,27 @@ def test_ntuple_add_subfield_to_collection(tmp_path): with pytest.raises((ValueError, TypeError)): f["mytuple"].add_fields({"jets.x": np.float32}) +def test_ntuple_add_subfield_to_collection_of_records(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array({ + "jets": [[{"pt": 1., "eta": 0.}], [{"pt": 2., "eta": 3.}]] + }) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises((ValueError, TypeError)): + f["mytuple"].add_fields({"jets.phi": np.float32}) + + +def test_ntuple_add_subfield_to_variant(tmp_path): + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array({ + "variant": ak.Array([{"jet": {"pt": 1., "eta": 2.}}, 2]) + }) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises((ValueError, TypeError, AssertionError)): + f["mytuple"].add_fields({"variant.jet.eta": np.float32}) + def test_ntuple_num_entries(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} From 3906cdf920d02c42e9c4e28f400a7d855dbcd3a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 23 Jul 2026 11:51:21 +0200 Subject: [PATCH 27/58] Fix test_ntuple_add_subfield_typed_parent to use uproot-created file --- tests/test_1687_rntuple_update.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 51ea694d7..3ad941f95 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -1,8 +1,10 @@ import os +import shutil import awkward as ak import numpy as np import pytest +import skhep_testdata import uproot try: @@ -412,11 +414,12 @@ def test_ntuple_add_subfield_nonexistent_parent(tmp_path): def test_ntuple_add_subfield_typed_parent(tmp_path): + # fields with C++ typenames (like those written by ROOT) cannot have subfields added with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = ak.Array([{"pt": 1.0}, {"pt": 2.0}]) with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises((ValueError, TypeError)): + with pytest.raises(ValueError): f["mytuple"].add_fields({"pt.x": np.float32}) @@ -428,6 +431,7 @@ def test_ntuple_add_subfield_to_collection(tmp_path): with pytest.raises((ValueError, TypeError)): f["mytuple"].add_fields({"jets.x": np.float32}) + def test_ntuple_add_subfield_to_collection_of_records(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = ak.Array({ @@ -449,6 +453,7 @@ def test_ntuple_add_subfield_to_variant(tmp_path): with pytest.raises((ValueError, TypeError, AssertionError)): f["mytuple"].add_fields({"variant.jet.eta": np.float32}) + def test_ntuple_num_entries(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} @@ -457,6 +462,7 @@ def test_ntuple_num_entries(tmp_path): nt = f["mytuple"] assert nt.num_entries == 3 + def test_ntuple_add_field_duplicate_after_extension(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} From 0e6b059293b36bfdb2d0f4f600ce74de7bf4b267 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 09:53:15 +0000 Subject: [PATCH 28/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 11 ++++++++--- tests/test_1687_rntuple_update.py | 13 +++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index ed0c08741..26d2ed47a 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2338,7 +2338,12 @@ def extend(self, data, accept_new_fields=False): ) elif new_field_names and accept_new_fields: self.add_fields( - {k: numpy.asarray(awkward.flatten(awkward.Array(data[k]), axis=None)).dtype for k in new_field_names} + { + k: numpy.asarray( + awkward.flatten(awkward.Array(data[k]), axis=None) + ).dtype + for k in new_field_names + } ) key = self._file.root_directory._cascading.data.get_key( self._path[-1], 1 @@ -2373,8 +2378,8 @@ def add_fields(self, new_fields): existing_page_list_envelopes = self._cascading._existing_page_list_envelopes existing_field_records = self._cascading._existing_field_records - next_field_id = len(existing_field_records) - + next_field_id = len(existing_field_records) + new_pages = {} existing_field_names = {fr.field_name for fr in existing_field_records} diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 0fa4f5fa1..3e5bf76a7 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -437,9 +437,9 @@ def test_ntuple_add_subfield_to_collection(tmp_path): def test_ntuple_add_subfield_to_collection_of_records(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"] = ak.Array({ - "jets": [[{"pt": 1., "eta": 0.}], [{"pt": 2., "eta": 3.}]] - }) + f["mytuple"] = ak.Array( + {"jets": [[{"pt": 1.0, "eta": 0.0}], [{"pt": 2.0, "eta": 3.0}]]} + ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises((ValueError, TypeError)): @@ -448,9 +448,9 @@ def test_ntuple_add_subfield_to_collection_of_records(tmp_path): def test_ntuple_add_subfield_to_variant(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"] = ak.Array({ - "variant": ak.Array([{"jet": {"pt": 1., "eta": 2.}}, 2]) - }) + f["mytuple"] = ak.Array( + {"variant": ak.Array([{"jet": {"pt": 1.0, "eta": 2.0}}, 2])} + ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises((ValueError, TypeError, AssertionError)): @@ -465,6 +465,7 @@ def test_ntuple_num_entries(tmp_path): nt = f["mytuple"] assert nt.num_entries == 3 + def test_ntuple_add_field_duplicate_after_extension(tmp_path): with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} From 05e2c168c8cfcad9c6edf35197d1eb7b08c7977f Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Thu, 23 Jul 2026 16:12:34 -0400 Subject: [PATCH 29/58] fix: allow nil UUID (version 0) in ROOT file header ROOT 6.38 writes a nil UUID (uuid_version=0, all-zero bytes) in the file header. The assert uuid_version == 1 in FileHeader.deserialize rejected these files, causing uproot.update to fail immediately. The uuid_version field is not used after parsing, so remove the assertion. Assisted-by: claude-code:claude-sonnet-4-6 --- src/uproot/writing/_cascade.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/uproot/writing/_cascade.py b/src/uproot/writing/_cascade.py index 3be830f9c..79f3ce5d1 100644 --- a/src/uproot/writing/_cascade.py +++ b/src/uproot/writing/_cascade.py @@ -2109,7 +2109,7 @@ def deserialize(cls, raw_bytes, location): compression_code, info_location, info_num_bytes, - uuid_version, + _uuid_version, uuid_bytes, ) = uproot.reading._file_header_fields_small.unpack( raw_bytes[: uproot.reading._file_header_fields_small.size] @@ -2128,7 +2128,7 @@ def deserialize(cls, raw_bytes, location): compression_code, info_location, info_num_bytes, - uuid_version, + _uuid_version, uuid_bytes, ) = uproot.reading._file_header_fields_big.unpack(raw_bytes) assert units == 8 @@ -2145,8 +2145,6 @@ def deserialize(cls, raw_bytes, location): assert compression_code >= 0 assert info_location >= 0 assert info_num_bytes >= 0 - assert uuid_version == 1 - out = FileHeader( end, free_location, From bcc47ba080112be4a75d50c74f3a2ffc3b2f6e8c Mon Sep 17 00:00:00 2001 From: Andres Rios Tascon Date: Fri, 24 Jul 2026 09:55:09 -0400 Subject: [PATCH 30/58] Only accept valid uuid versions --- src/uproot/writing/_cascade.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/uproot/writing/_cascade.py b/src/uproot/writing/_cascade.py index 79f3ce5d1..e3ad033c0 100644 --- a/src/uproot/writing/_cascade.py +++ b/src/uproot/writing/_cascade.py @@ -2109,7 +2109,7 @@ def deserialize(cls, raw_bytes, location): compression_code, info_location, info_num_bytes, - _uuid_version, + uuid_version, uuid_bytes, ) = uproot.reading._file_header_fields_small.unpack( raw_bytes[: uproot.reading._file_header_fields_small.size] @@ -2128,7 +2128,7 @@ def deserialize(cls, raw_bytes, location): compression_code, info_location, info_num_bytes, - _uuid_version, + uuid_version, uuid_bytes, ) = uproot.reading._file_header_fields_big.unpack(raw_bytes) assert units == 8 @@ -2145,6 +2145,8 @@ def deserialize(cls, raw_bytes, location): assert compression_code >= 0 assert info_location >= 0 assert info_num_bytes >= 0 + assert uuid_version <= 8 + out = FileHeader( end, free_location, From 4a76ec54f6e62e569fc553a69d8f6d9a78bd5c70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:41:58 +0200 Subject: [PATCH 31/58] Update typed parent test to use real ROOT file and check all typed structs --- tests/test_1687_rntuple_update.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 0fa4f5fa1..dda1dd8c9 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -417,13 +417,22 @@ def test_ntuple_add_subfield_nonexistent_parent(tmp_path): def test_ntuple_add_subfield_typed_parent(tmp_path): - # fields with C++ typenames (like those written by ROOT) cannot have subfields added - with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: - f["mytuple"] = ak.Array([{"pt": 1.0}, {"pt": 2.0}]) + # structs written by ROOT have C++ typenames and cannot have subfields added + src = skhep_testdata.data_path("test_nested_structs_rntuple_v1-0-0-0.root") + shutil.copy(src, os.path.join(tmp_path, "test.root")) with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises(ValueError): - f["mytuple"].add_fields({"pt.x": np.float32}) + # my_struct has typename TopStruct + with pytest.raises(ValueError, match="TopStruct"): + f["ntuple"].add_fields({"my_struct.new_field": np.float32}) + + # sub_struct has typename SubStruct + with pytest.raises(ValueError, match="SubStruct"): + f["ntuple"].add_fields({"sub_struct.new_field": np.float32}) + + # sub_sub_struct has typename SubSubSruct + with pytest.raises(ValueError, match="SubSubSruct"): + f["ntuple"].add_fields({"sub_sub_struct.new_field": np.float32}) def test_ntuple_add_subfield_to_collection(tmp_path): From dd9b475107b4ed408d6f9fe4698540929750b0ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Mon, 27 Jul 2026 11:48:55 +0200 Subject: [PATCH 32/58] Update docstrings to show multiple fields in extend and add_fields examples --- src/uproot/writing/writable.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 26d2ed47a..a98dc68d3 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2313,7 +2313,10 @@ def extend(self, data, accept_new_fields=False): .. code-block:: python with uproot.update("file.root") as f: - f["mytuple"].extend({"x": np.array([4, 5, 6])}) + f["mytuple"].extend({ + "x": np.array([4, 5, 6]), + "y": np.array([40, 50, 60]), + }) # automatically add new field and extend with uproot.update("file.root") as f: @@ -2365,7 +2368,7 @@ def add_fields(self, new_fields): .. code-block:: python with uproot.update("file.root") as f: - f["mytuple"].add_fields({"z": np.int32}) + f["mytuple"].add_fields({"z": np.int32, "w": np.float32}) """ import uproot.writing._cascadentuple as cnt From 3204819066c8472b862edb8246f93d2257415895 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:48:38 +0200 Subject: [PATCH 33/58] Fix blocking issue 1: silent data corruption on ROOT-written RNTuples --- src/uproot/writing/writable.py | 31 +++++++++++++++++++++++++++++++ tests/test_1687_rntuple_update.py | 11 +++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index a98dc68d3..60038c290 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1092,6 +1092,31 @@ def _load_existing_ntuple(self, key): None, existing.name, existing._header.ntuple_description, full_akform ) header._checksum = existing._header.checksum + + # check column records match existing — store mismatch for later error + # (e.g. ROOT writes split-encoded columns, uproot writes unsplit) + existing_col_records = existing.column_records + new_col_records = header._column_records + _column_encoding_error = None + if len(existing_col_records) != len(new_col_records): + _column_encoding_error = ( + f"cannot extend: existing RNTuple has {len(existing_col_records)} columns " + f"but reconstructed header has {len(new_col_records)}; " + f"schema mismatch — this RNTuple may use column encodings uproot cannot write" + ) + else: + for i, (existing_cr, new_cr) in enumerate(zip(existing_col_records, new_col_records)): + if (existing_cr.type != new_cr.type_num or + existing_cr.nbits != new_cr.bits_on_disk or + existing_cr.field_id != new_cr.field_id): + _column_encoding_error = ( + f"cannot extend: column {i} type mismatch — " + f"existing column has type={existing_cr.type}, nbits={existing_cr.nbits} " + f"but uproot would write type={new_cr.type_num}, nbits={new_cr.bits_on_disk}. " + f"This RNTuple uses column encodings (e.g. split encoding) that uproot cannot write." + ) + break + footer = cnt.NTuple_Footer(None, header._checksum) for cg in existing_footer.cluster_group_records: @@ -1186,6 +1211,7 @@ def _load_existing_ntuple(self, key): path = (*self._path, name) writable_ntuple = WritableNTuple(path, self._file, ntuple_cascading) + writable_ntuple._column_encoding_error = _column_encoding_error self._file._ntuples[anchor_location] = writable_ntuple return writable_ntuple @@ -2197,6 +2223,7 @@ def __init__(self, path, file, cascading): self._path = path self._file = file self._cascading = cascading + self._column_encoding_error = None def __repr__(self): return "".format( @@ -2289,6 +2316,8 @@ def num_entries(self) -> int: return self._cascading.num_entries def extend(self, data, accept_new_fields=False): + if self._column_encoding_error is not None: + raise ValueError(self._column_encoding_error) """ Args: data (dict of str \u2192 arrays): More array data to add to the RNTuple. @@ -2442,6 +2471,8 @@ def add_fields(self, new_fields): new_data = numpy.zeros(num_entries, dtype=numpy.dtype(ak_primitive)) raw_data = new_data.view("uint8") compressed_data = uproot.compression.compress(raw_data, compression) + if self._column_encoding_error is not None: + raise ValueError(self._column_encoding_error) page_key = self._cascading.add_rblob( self._file.sink, compressed_data, len(raw_data) ) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index d1ba0317f..349ea0e34 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -486,3 +486,14 @@ def test_ntuple_add_field_duplicate_after_extension(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises(ValueError, match="already exists"): f["mytuple"].add_fields({"z": np.float32}) + + +def test_ntuple_extend_root_written_raises(tmp_path): + # ROOT-written RNTuples use split encoding which uproot cannot write + src = skhep_testdata.data_path("test_int_float_rntuple_v1-0-0-0.root") + shutil.copy(src, os.path.join(tmp_path, "test.root")) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="column encodings"): + f["ntuple"].extend({"one_integers": np.array([100, 200], dtype=np.int32), + "two_floats": np.array([1.5, 2.5], dtype=np.float32)}) From aa0a3d74aa6250f3bb0a0845c2f6108ab063e228 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:49:23 +0000 Subject: [PATCH 34/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 12 ++++++++---- tests/test_1687_rntuple_update.py | 8 ++++++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 60038c290..8c8a1a8c8 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1105,10 +1105,14 @@ def _load_existing_ntuple(self, key): f"schema mismatch — this RNTuple may use column encodings uproot cannot write" ) else: - for i, (existing_cr, new_cr) in enumerate(zip(existing_col_records, new_col_records)): - if (existing_cr.type != new_cr.type_num or - existing_cr.nbits != new_cr.bits_on_disk or - existing_cr.field_id != new_cr.field_id): + for i, (existing_cr, new_cr) in enumerate( + zip(existing_col_records, new_col_records) + ): + if ( + existing_cr.type != new_cr.type_num + or existing_cr.nbits != new_cr.bits_on_disk + or existing_cr.field_id != new_cr.field_id + ): _column_encoding_error = ( f"cannot extend: column {i} type mismatch — " f"existing column has type={existing_cr.type}, nbits={existing_cr.nbits} " diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 349ea0e34..c42083443 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -495,5 +495,9 @@ def test_ntuple_extend_root_written_raises(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises(ValueError, match="column encodings"): - f["ntuple"].extend({"one_integers": np.array([100, 200], dtype=np.int32), - "two_floats": np.array([1.5, 2.5], dtype=np.float32)}) + f["ntuple"].extend( + { + "one_integers": np.array([100, 200], dtype=np.int32), + "two_floats": np.array([1.5, 2.5], dtype=np.float32), + } + ) From 251c4b03c530b22d99256a7d013af96ad629211a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:53:29 +0200 Subject: [PATCH 35/58] Fix blocking issue 2: wrong element_offset for inner columns of jagged fields --- src/uproot/writing/writable.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 60038c290..1c3b7f7a1 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1202,9 +1202,20 @@ def _load_existing_ntuple(self, key): num_columns = len(existing._header.column_records) + len( existing._footer.extension_links.column_records ) - ntuple_cascading._column_counts = numpy.array( - [num_entries] * num_columns, dtype=int - ) + # recover per-column element counts from existing page lists + # for jagged fields, data columns advance by elements not entries + column_counts = [] + for cg_idx, cg in enumerate(existing_footer.cluster_group_records): + ple = existing_page_list_envelopes[cg_idx] + if not column_counts: + column_counts = [0] * num_columns + for col_idx, col_pages in enumerate(ple.pagelinklist[0]): + column_counts[col_idx] += sum( + p.num_elements for p in col_pages.pages + ) + if not column_counts: + column_counts = [num_entries] * num_columns + ntuple_cascading._column_counts = numpy.array(column_counts, dtype=int) ntuple_cascading._existing_footer = existing_footer ntuple_cascading._existing_page_list_envelopes = existing_page_list_envelopes ntuple_cascading._existing_field_records = existing_field_records From 0bf39d44968278dc64a04c79d1ed587cacf5c853 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 09:54:04 +0000 Subject: [PATCH 36/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 5e4247d28..90fe1c3e8 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1214,9 +1214,7 @@ def _load_existing_ntuple(self, key): if not column_counts: column_counts = [0] * num_columns for col_idx, col_pages in enumerate(ple.pagelinklist[0]): - column_counts[col_idx] += sum( - p.num_elements for p in col_pages.pages - ) + column_counts[col_idx] += sum(p.num_elements for p in col_pages.pages) if not column_counts: column_counts = [num_entries] * num_columns ntuple_cascading._column_counts = numpy.array(column_counts, dtype=int) From 74545cdbe7b9625bba4a41fd8785af800030206f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 11:56:26 +0200 Subject: [PATCH 37/58] Fix blocking issue 3: multi-cluster groups in add_fields --- src/uproot/writing/writable.py | 6 ++++++ tests/test_1687_rntuple_update.py | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 5e4247d28..8b9240d6f 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2502,6 +2502,12 @@ def add_fields(self, new_fields): footer.cluster_group_record_frames = [] for cg_idx, cg in enumerate(existing_footer.cluster_group_records): ple = existing_page_list_envelopes[cg_idx] + if len(ple.pagelinklist) > 1: + raise ValueError( + f"add_fields does not yet support RNTuples with multiple clusters per cluster group " + f"(cluster group {cg_idx} has {len(ple.pagelinklist)} clusters). " + f"This is a known limitation that will be fixed in a future version." + ) new_cluster_page_data = [] for col_pages in ple.pagelinklist[0]: existing_pages = [ diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index c42083443..977ffe96b 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -501,3 +501,12 @@ def test_ntuple_extend_root_written_raises(tmp_path): "two_floats": np.array([1.5, 2.5], dtype=np.float32), } ) + + +def test_ntuple_add_fields_multi_cluster_raises(tmp_path): + src = skhep_testdata.data_path("test_multiple_cluster_groups_rntuple_v1-0-0-0.root") + shutil.copy(src, os.path.join(tmp_path, "test.root")) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError): + f["ntuple"].add_fields({"newcol": np.int32}) From a4d6e77b9a48d5f43c8ecd08b23546b953be296d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:13:56 +0200 Subject: [PATCH 38/58] Fix blocking issue 4: stale in-memory state after add_fields --- src/uproot/writing/writable.py | 38 +++++++++++++++++++++++++++++++ tests/test_1687_rntuple_update.py | 34 +++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 8b9240d6f..9437643fa 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2371,6 +2371,15 @@ def extend(self, data, accept_new_fields=False): **As a word of warning,** be sure that each call to :ref:`uproot.writing.writable.WritableNTuple.extend` includes at least 100 kB per branch/array. (NumPy and Awkward Arrays have an `nbytes `__ property; you want at least ``100000`` per array.) If you ask Uproot to write very small TBaskets, it will spend more time working on TBasket overhead than actually writing data. The absolute worst case is one-entry-per-:ref:`uproot.writing.writable.WritableTree.extend`. See `#428 (comment) `__. """ + # if _column_counts has more columns than _column_keys, reload cascading + # this happens after add_fields adds extension columns + if len(self._cascading._column_counts) > len(self._cascading._header._column_keys): + key = self._file.root_directory._cascading.data.get_key( + self._path[-1], 1 + ) + reloaded = self._file.root_directory._load_existing_ntuple(key) + self._cascading = reloaded._cascading + if isinstance(data, dict): existing_keys = set(self._cascading._header._akform.fields) if hasattr(self._cascading, "_existing_field_records"): @@ -2569,6 +2578,35 @@ def add_fields(self, new_fields): self._cascading._freesegments.write(self._file.sink) self._file.sink.flush() + # reload ntuple so subsequent add_fields/extend calls see updated schema + # find the directory that owns this ntuple and reload via _load_existing_ntuple + parent_dir = self._file._cascading.rootdirectory + key = parent_dir.data.get_key(self._path[-1], 1) + # rebuild cascading from file + existing_file = uproot.open(self._file.file_path, minimal_ttree_metadata=False) + try: + existing = existing_file[self._path[-1]] + _ = existing.keys() + self._cascading._existing_footer = existing._footer + self._cascading._existing_page_list_envelopes = existing.page_list_envelopes + self._cascading._existing_field_records = existing._ntuple.field_records + full_akform, _ = existing.to_akform() + self._cascading._header._akform = full_akform + # update column counts from existing page lists + existing_footer_reload = existing._footer + existing_ples = existing.page_list_envelopes + num_columns = len(existing._header.column_records) + len( + existing._footer.extension_links.column_records + ) if existing._footer else len(existing.column_records) + column_counts = [0] * num_columns + for cg_idx, cg in enumerate(existing_footer_reload.cluster_group_records): + ple = existing_ples[cg_idx] + for col_idx, col_pages in enumerate(ple.pagelinklist[0]): + column_counts[col_idx] += sum(p.num_elements for p in col_pages.pages) + self._cascading._column_counts = numpy.array(column_counts, dtype=int) + finally: + existing_file.close() + def _is_type_specification(obj): to_check = [obj] diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 977ffe96b..6b5cb3fa7 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -510,3 +510,37 @@ def test_ntuple_add_fields_multi_cluster_raises(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: with pytest.raises(ValueError): f["ntuple"].add_fields({"newcol": np.int32}) + + +def test_ntuple_add_fields_sequential_same_session(tmp_path): + # test that holding onto a WritableNTuple object and calling add_fields twice works + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array([{"x": 1.0}, {"x": 2.0}, {"x": 3.0}]) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + nt.add_fields({"y": np.int32}) + nt.add_fields({"z": np.float64}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert set(f["mytuple"].keys()) == {"x", "y", "z"} + assert np.all(f["mytuple"]["y"].array() == 0) + assert np.all(f["mytuple"]["z"].array() == 0.0) + + +def test_ntuple_add_fields_then_extend_same_object(tmp_path): + # test add_fields then extend using same nt object (not re-opening f["mytuple"]) + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] + nt.add_fields({"y": np.int32}) + nt.extend({ + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([40, 50], dtype=np.int32), + }) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert ak.all(f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) + assert ak.all(f["mytuple"]["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) From bfea8bbc3043233d0f1361942dc3ecc930af7f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:23:51 +0200 Subject: [PATCH 39/58] Fix blocking issue 5: subfield parent resolved by full path not bare name --- src/uproot/writing/writable.py | 55 +++++++++++++++++++++++-------- tests/test_1687_rntuple_update.py | 16 +++++++++ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 9437643fa..992038dcd 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2457,24 +2457,51 @@ def add_fields(self, new_fields): if "." in field_name: parts = field_name.split(".") actual_field_name = parts[-1] - parent_name = parts[-2] parent_field_id = None - for i, fr in enumerate(existing_field_records): - if fr.field_name == parent_name: - parent_field_id = i - if fr.type_name != "": - raise ValueError( - f"Field {parent_name!r} has type {fr.type_name!r} and cannot be extended. " - f"Only untyped records (empty type_name) can have subfields added." - ) - if fr.struct_role != uproot.const.RNTupleFieldRole.RECORD: + # walk the full dotted path against the parent-id chain + # for single-level paths (e.g. "parent.field"), match by name only + # for multi-level paths (e.g. "p2.track.field"), walk full chain + if len(parts) == 2: + # single-level: match immediate parent by name only (backward compat) + parent_field_id = None + for i, fr in enumerate(existing_field_records): + if fr.field_name == parts[0]: + parent_field_id = i + break + if parent_field_id is None: + raise ValueError( + f"Field {parts[0]!r} not found in this RNTuple" + ) + fr = existing_field_records[parent_field_id] + else: + # multi-level: walk full dotted path using parent-id chain + current_parent_id = None # None means root + for part_idx, part in enumerate(parts[:-1]): + found = None + for i, fr in enumerate(existing_field_records): + is_root_field = (fr.parent_field_id == 0 or fr.parent_field_id == i) + if fr.field_name == part: + if current_parent_id is None and is_root_field: + found = (i, fr) + break + elif current_parent_id is not None and fr.parent_field_id == current_parent_id: + found = (i, fr) + break + if found is None: raise ValueError( - f"Field {parent_name!r} is not a record and cannot have subfields added." + f"Field {'.'.join(parts[:part_idx+1])!r} not found in this RNTuple" ) - break - if parent_field_id is None: + current_parent_id = found[0] + parent_field_id = found[0] + fr = found[1] + if fr.type_name != "": + raise ValueError( + f"Field {'.'.join(parts[:-1])!r} has type {fr.type_name!r} and cannot be extended. " + f"Only untyped records (empty type_name) can have subfields added." + ) + if fr.struct_role != uproot.const.RNTupleFieldRole.RECORD: raise ValueError( - f"Parent field {parent_name!r} not found in RNTuple" + f"Field {'.'.join(parts[:-1])!r} is not a record and cannot have subfields added." ) else: actual_field_name = field_name diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 6b5cb3fa7..79d3022fc 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -544,3 +544,19 @@ def test_ntuple_add_fields_then_extend_same_object(tmp_path): with uproot.open(os.path.join(tmp_path, "test.root")) as f: assert ak.all(f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) assert ak.all(f["mytuple"]["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) + + +def test_ntuple_add_subfield_correct_parent(tmp_path): + # verify p2.track.phi goes to p2.track not p1.track + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = ak.Array( + [{"p1": {"track": {"pt": 1.0}}, "p2": {"track": {"pt": 2.0}}}] * 2 + ) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"].add_fields({"p2.track.phi": np.float32}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + keys = list(f["mytuple"].keys()) + assert "p2.track.phi" in keys + assert "p1.track.phi" not in keys From 59b63132c54cf3caf3d4549f2150216b53a9079d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 10:24:58 +0000 Subject: [PATCH 40/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 30 ++++++++++++++++++++---------- tests/test_1687_rntuple_update.py | 18 ++++++++++++------ 2 files changed, 32 insertions(+), 16 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 1444505e6..3a4fea936 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2371,10 +2371,10 @@ def extend(self, data, accept_new_fields=False): """ # if _column_counts has more columns than _column_keys, reload cascading # this happens after add_fields adds extension columns - if len(self._cascading._column_counts) > len(self._cascading._header._column_keys): - key = self._file.root_directory._cascading.data.get_key( - self._path[-1], 1 - ) + if len(self._cascading._column_counts) > len( + self._cascading._header._column_keys + ): + key = self._file.root_directory._cascading.data.get_key(self._path[-1], 1) reloaded = self._file.root_directory._load_existing_ntuple(key) self._cascading = reloaded._cascading @@ -2477,12 +2477,17 @@ def add_fields(self, new_fields): for part_idx, part in enumerate(parts[:-1]): found = None for i, fr in enumerate(existing_field_records): - is_root_field = (fr.parent_field_id == 0 or fr.parent_field_id == i) + is_root_field = ( + fr.parent_field_id == 0 or fr.parent_field_id == i + ) if fr.field_name == part: if current_parent_id is None and is_root_field: found = (i, fr) break - elif current_parent_id is not None and fr.parent_field_id == current_parent_id: + elif ( + current_parent_id is not None + and fr.parent_field_id == current_parent_id + ): found = (i, fr) break if found is None: @@ -2620,14 +2625,19 @@ def add_fields(self, new_fields): # update column counts from existing page lists existing_footer_reload = existing._footer existing_ples = existing.page_list_envelopes - num_columns = len(existing._header.column_records) + len( - existing._footer.extension_links.column_records - ) if existing._footer else len(existing.column_records) + num_columns = ( + len(existing._header.column_records) + + len(existing._footer.extension_links.column_records) + if existing._footer + else len(existing.column_records) + ) column_counts = [0] * num_columns for cg_idx, cg in enumerate(existing_footer_reload.cluster_group_records): ple = existing_ples[cg_idx] for col_idx, col_pages in enumerate(ple.pagelinklist[0]): - column_counts[col_idx] += sum(p.num_elements for p in col_pages.pages) + column_counts[col_idx] += sum( + p.num_elements for p in col_pages.pages + ) self._cascading._column_counts = numpy.array(column_counts, dtype=int) finally: existing_file.close() diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 79d3022fc..1a81ae256 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -536,14 +536,20 @@ def test_ntuple_add_fields_then_extend_same_object(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: nt = f["mytuple"] nt.add_fields({"y": np.int32}) - nt.extend({ - "x": np.array([4, 5], dtype=np.float32), - "y": np.array([40, 50], dtype=np.int32), - }) + nt.extend( + { + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([40, 50], dtype=np.int32), + } + ) with uproot.open(os.path.join(tmp_path, "test.root")) as f: - assert ak.all(f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) - assert ak.all(f["mytuple"]["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32)) + assert ak.all( + f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32) + ) + assert ak.all( + f["mytuple"]["y"].array() == np.array([0, 0, 0, 40, 50], dtype=np.int32) + ) def test_ntuple_add_subfield_correct_parent(tmp_path): From 6dac39e6322f3bb36d3698dd47793e738fb18dfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:14:54 +0200 Subject: [PATCH 41/58] Fix file-like object regression in _load_existing_ntuple --- src/uproot/writing/writable.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 1444505e6..5fb51712e 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1072,6 +1072,11 @@ def _load_existing_ntuple(self, key): _rblob_key_size = uproot.reading._key_format_big.size + 8 name = key.name.string + if self.file_path is None: + raise TypeError( + "uproot.update() on a file-like object does not support accessing " + "existing RNTuples; use uproot.update() with a file path instead." + ) # TODO: opening the file again in read mode to access existing metadata is # a bit awkward since the file is already open in write mode. We should # look into a better way to do this in the future. From 687f44b0a803cacca31925b797ce3da1414fb973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:16:16 +0200 Subject: [PATCH 42/58] Fix file handle leak and cache key bug in _load_existing_ntuple --- src/uproot/writing/writable.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 5fb51712e..31d6a3b2b 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1081,17 +1081,19 @@ def _load_existing_ntuple(self, key): # a bit awkward since the file is already open in write mode. We should # look into a better way to do this in the future. existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) - existing = existing_file[name] - _ = existing.keys() - full_akform, _ = existing.to_akform() - am = existing._ntuple.all_members - existing_key = existing_file.key(name + ";1") - anchor_location = existing_key.fSeekKey + existing_key.fKeylen - num_entries = existing.num_entries - existing_footer = existing._footer - existing_page_list_envelopes = existing.page_list_envelopes - existing_field_records = existing._ntuple.field_records - existing_file.close() + try: + existing = existing_file[name] + _ = existing.keys() + full_akform, _ = existing.to_akform() + am = existing._ntuple.all_members + existing_key = existing_file.key(name + ";1") + anchor_location = existing_key.fSeekKey + existing_key.fKeylen + num_entries = existing.num_entries + existing_footer = existing._footer + existing_page_list_envelopes = existing.page_list_envelopes + existing_field_records = existing._ntuple.field_records + finally: + existing_file.close() header = cnt.NTuple_Header( None, existing.name, existing._header.ntuple_description, full_akform @@ -1230,7 +1232,7 @@ def _load_existing_ntuple(self, key): path = (*self._path, name) writable_ntuple = WritableNTuple(path, self._file, ntuple_cascading) writable_ntuple._column_encoding_error = _column_encoding_error - self._file._ntuples[anchor_location] = writable_ntuple + self._file._ntuples[key.seek_location] = writable_ntuple return writable_ntuple def _del(self, name, cycle): From 9140b7e0bce38fbf3d6008a1f09a131762dfac23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:19:12 +0200 Subject: [PATCH 43/58] Fix reconstructed Keys missing big=True --- src/uproot/writing/writable.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 31d6a3b2b..320005345 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1197,6 +1197,7 @@ def _load_existing_ntuple(self, key): 1, 100, am["fSeekHeader"], + big=True, ) ntuple_cascading._footer_key = casc.Key( am["fSeekFooter"] - _rblob_key_size, @@ -1208,6 +1209,7 @@ def _load_existing_ntuple(self, key): 1, 100, am["fSeekFooter"], + big=True, ) ntuple_cascading._num_entries = num_entries num_columns = len(existing._header.column_records) + len( From 3289dd43b8f7d03f218a388d722dbbb3c2977909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:21:19 +0200 Subject: [PATCH 44/58] Remove dead freesegments carve-out code --- src/uproot/writing/writable.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 320005345..ddb3466a6 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1155,15 +1155,6 @@ def _load_existing_ntuple(self, key): cr.type, cr.nbits, cr.field_id, cr.flags, cr.repr_idx ) footer.extension_column_record_frames.append(new_col) - for cg in existing_footer.cluster_group_records: - loc = cg.page_list_link.locator - start = loc.offset - _rblob_key_size - end = loc.offset + loc.num_bytes - self._cascading._freesegments._data.slices = [ - s - for s in self._cascading._freesegments._data.slices - if not (s[0] < end and start < s[1]) - ] anchor = cnt.NTuple_Anchor( anchor_location, am["fVersionEpoch"], From 31c43c2faf0b53e0a7fd1fb0c2f84d450fbedd99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:23:33 +0200 Subject: [PATCH 45/58] Fix uuid_version assert and add test for ROOT-written file access --- src/uproot/writing/_cascade.py | 2 +- tests/test_1687_rntuple_update.py | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/uproot/writing/_cascade.py b/src/uproot/writing/_cascade.py index e3ad033c0..ddf2cd009 100644 --- a/src/uproot/writing/_cascade.py +++ b/src/uproot/writing/_cascade.py @@ -2145,7 +2145,7 @@ def deserialize(cls, raw_bytes, location): assert compression_code >= 0 assert info_location >= 0 assert info_num_bytes >= 0 - assert uuid_version <= 8 + assert 0 <= uuid_version <= 8 out = FileHeader( end, diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 79d3022fc..cda391c02 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -560,3 +560,17 @@ def test_ntuple_add_subfield_correct_parent(tmp_path): keys = list(f["mytuple"].keys()) assert "p2.track.phi" in keys assert "p1.track.phi" not in keys + + +def test_ntuple_update_root_written_file_opens(tmp_path): + # ROOT-written files use UUID versions other than 1 — verify uproot.update can open them + # This test uses a ROOT-written file and checks it can be accessed in update mode + src = skhep_testdata.data_path("test_int_float_rntuple_v1-0-0-0.root") + shutil.copy(src, os.path.join(tmp_path, "test.root")) + + # should not raise — previously failed with assert uuid_version == 1 + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + # accessing the ntuple should work (even though we can't extend ROOT-written files) + with pytest.raises(ValueError, match="column encodings"): + f["ntuple"].extend({"one_integers": np.array([1], dtype=np.int32), + "two_floats": np.array([1.0], dtype=np.float32)}) From 16352771b5bd7b4a8cf71dc221008ef4eafd3367 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:33:09 +0200 Subject: [PATCH 46/58] Code quality: add lazy-init comment and improve add_fields docstring --- src/uproot/writing/writable.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index ddb3466a6..6cc54e969 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1083,7 +1083,7 @@ def _load_existing_ntuple(self, key): existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) try: existing = existing_file[name] - _ = existing.keys() + _ = existing.keys() # trigger lazy loading of footer and page lists full_akform, _ = existing.to_akform() am = existing._ntuple.all_members existing_key = existing_file.key(name + ";1") @@ -2411,8 +2411,21 @@ def add_fields(self, new_fields): """ Args: new_fields (dict of str -> numpy dtype): New field names and types. + Only scalar numeric types are accepted (e.g. ``np.int32``, + ``np.float64``). Variable-length or nested types are not supported. Adds new fields to this RNTuple, back-filled with zeros for existing entries. + Only top-level fields or subfields of existing untyped record fields are supported. + + Note: when using ``accept_new_fields=True`` in :meth:`extend`, new fields + must be flat (scalar) types. Jagged or nested new fields will fail the + form compatibility check in ``extend``. + + Raises: + TypeError: if a field type is not a simple scalar numeric type. + ValueError: if a field already exists, if the subfield parent is not + found or is a typed struct (C++ typename), or if the file was + opened without a file path (file-like objects are not supported). For example, @@ -2611,7 +2624,7 @@ def add_fields(self, new_fields): existing_file = uproot.open(self._file.file_path, minimal_ttree_metadata=False) try: existing = existing_file[self._path[-1]] - _ = existing.keys() + _ = existing.keys() # trigger lazy loading of footer and page lists self._cascading._existing_footer = existing._footer self._cascading._existing_page_list_envelopes = existing.page_list_envelopes self._cascading._existing_field_records = existing._ntuple.field_records From 43bda03fdab43a3a8b96b14e8c4fb49043aaf64f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:41:17 +0200 Subject: [PATCH 47/58] Improve test coverage and error handling --- tests/test_1687_rntuple_update.py | 40 +++++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index cda391c02..767261115 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -119,7 +119,7 @@ def test_extend_ntuple_wrong_fields(tmp_path): } with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises(Exception): + with pytest.raises(ValueError): f["mytuple"].extend( {"x": np.array([7, 8, 9], dtype=np.float32)} ) # missing y @@ -440,7 +440,7 @@ def test_ntuple_add_subfield_to_collection(tmp_path): f["mytuple"] = {"jets": ak.Array([[1.0, 2.0], [3.0]])} with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises((ValueError, TypeError)): + with pytest.raises(ValueError, match="type"): f["mytuple"].add_fields({"jets.x": np.float32}) @@ -451,7 +451,7 @@ def test_ntuple_add_subfield_to_collection_of_records(tmp_path): ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises((ValueError, TypeError)): + with pytest.raises(ValueError, match="not a record"): f["mytuple"].add_fields({"jets.phi": np.float32}) @@ -462,7 +462,7 @@ def test_ntuple_add_subfield_to_variant(tmp_path): ) with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises((ValueError, TypeError, AssertionError)): + with pytest.raises((ValueError, AssertionError)): f["mytuple"].add_fields({"variant.jet.eta": np.float32}) @@ -574,3 +574,35 @@ def test_ntuple_update_root_written_file_opens(tmp_path): with pytest.raises(ValueError, match="column encodings"): f["ntuple"].extend({"one_integers": np.array([1], dtype=np.int32), "two_floats": np.array([1.0], dtype=np.float32)}) + + +def test_ntuple_root_written_add_fields_raises(tmp_path): + # ROOT-written files use split encoding which uproot cannot write + # add_fields should raise a clear error not corrupt data + src = skhep_testdata.data_path("ntpl001_staff_rntuple_v1-0-0-0.root") + shutil.copy(src, os.path.join(tmp_path, "test.root")) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + with pytest.raises(ValueError, match="column encodings"): + f["Staff"].add_fields({"new_field": np.float32}) + + +def test_ntuple_hold_object_across_operations(tmp_path): + # hold WritableNTuple object across add_fields and extend + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["mytuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + nt = f["mytuple"] # hold the object + nt.add_fields({"y": np.int32}) + nt.add_fields({"z": np.float64}) + nt.extend({ + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([10, 20], dtype=np.int32), + "z": np.array([1.1, 2.2], dtype=np.float64), + }) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert set(f["mytuple"].keys()) == {"x", "y", "z"} + assert np.all(f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) + assert np.all(f["mytuple"]["y"].array() == np.array([0, 0, 0, 10, 20], dtype=np.int32)) From d9c0bcbd0feb5857d2a26bd14ffae6b4ab22bb48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 29 Jul 2026 14:47:41 +0200 Subject: [PATCH 48/58] Improve ROOT verification in tests to check column values --- tests/test_1687_rntuple_update.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index 767261115..c1852644e 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -43,6 +43,12 @@ def test_extend_existing_ntuple(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 8 + entry = reader.CreateEntry() + vals = [] + for i in range(reader.GetNEntries()): + reader.LoadEntry(i, entry) + vals.append(entry["x"]) + assert vals == pytest.approx([1, 2, 3, 4, 5, 6, 7, 8]) def test_add_field_ntuple(tmp_path): @@ -60,6 +66,12 @@ def test_add_field_ntuple(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + entry = reader.CreateEntry() + vals = [] + for i in range(reader.GetNEntries()): + reader.LoadEntry(i, entry) + vals.append(entry["x"]) + assert vals == pytest.approx([1.0, 2.0, 3.0, 4.0, 5.0]) def test_add_field_ntuple_duplicate(tmp_path): @@ -201,6 +213,12 @@ def test_ntuple_mixed_types_extend(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + entry = reader.CreateEntry() + vals = [] + for i in range(reader.GetNEntries()): + reader.LoadEntry(i, entry) + vals.append(entry["pt"]) + assert vals == pytest.approx([10.0, 20.0, 30.0, 40.0, 50.0]) def test_ntuple_add_field_then_extend(tmp_path): @@ -325,6 +343,12 @@ def test_ntuple_add_field_and_extend_same_session(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + entry = reader.CreateEntry() + vals = [] + for i in range(reader.GetNEntries()): + reader.LoadEntry(i, entry) + vals.append(entry["x"]) + assert vals == pytest.approx([1.0, 2.0, 3.0, 4.0, 5.0]) def test_ntuple_accept_new_fields(tmp_path): @@ -359,6 +383,12 @@ def test_ntuple_accept_new_fields(tmp_path): if has_root and hasattr(ROOT, "RNTupleReader"): reader = ROOT.RNTupleReader.Open("mytuple", os.path.join(tmp_path, "test.root")) assert reader.GetNEntries() == 5 + entry = reader.CreateEntry() + vals = [] + for i in range(reader.GetNEntries()): + reader.LoadEntry(i, entry) + vals.append(entry["x"]) + assert vals == pytest.approx([1.0, 2.0, 3.0, 4.0, 5.0]) def test_ntuple_add_subfield(tmp_path): From 19942147b5f55660a0723bab91ce2de9eec59a6b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 12:48:59 +0000 Subject: [PATCH 49/58] style: pre-commit fixes --- tests/test_1687_rntuple_update.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index a211e143a..7d51939e8 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -608,8 +608,12 @@ def test_ntuple_update_root_written_file_opens(tmp_path): with uproot.update(os.path.join(tmp_path, "test.root")) as f: # accessing the ntuple should work (even though we can't extend ROOT-written files) with pytest.raises(ValueError, match="column encodings"): - f["ntuple"].extend({"one_integers": np.array([1], dtype=np.int32), - "two_floats": np.array([1.0], dtype=np.float32)}) + f["ntuple"].extend( + { + "one_integers": np.array([1], dtype=np.int32), + "two_floats": np.array([1.0], dtype=np.float32), + } + ) def test_ntuple_root_written_add_fields_raises(tmp_path): @@ -632,13 +636,19 @@ def test_ntuple_hold_object_across_operations(tmp_path): nt = f["mytuple"] # hold the object nt.add_fields({"y": np.int32}) nt.add_fields({"z": np.float64}) - nt.extend({ - "x": np.array([4, 5], dtype=np.float32), - "y": np.array([10, 20], dtype=np.int32), - "z": np.array([1.1, 2.2], dtype=np.float64), - }) + nt.extend( + { + "x": np.array([4, 5], dtype=np.float32), + "y": np.array([10, 20], dtype=np.int32), + "z": np.array([1.1, 2.2], dtype=np.float64), + } + ) with uproot.open(os.path.join(tmp_path, "test.root")) as f: assert set(f["mytuple"].keys()) == {"x", "y", "z"} - assert np.all(f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32)) - assert np.all(f["mytuple"]["y"].array() == np.array([0, 0, 0, 10, 20], dtype=np.int32)) + assert np.all( + f["mytuple"]["x"].array() == np.array([1, 2, 3, 4, 5], dtype=np.float32) + ) + assert np.all( + f["mytuple"]["y"].array() == np.array([0, 0, 0, 10, 20], dtype=np.int32) + ) From a8df700be070c281aab9149724bad64b7ad66afe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:10:12 +0200 Subject: [PATCH 50/58] Support multiple cluster groups in add_fields --- src/uproot/writing/writable.py | 61 +++++++++++++++++++------------ tests/test_1687_rntuple_update.py | 21 ++++++++--- 2 files changed, 53 insertions(+), 29 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 4960caa3e..670999082 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2554,38 +2554,51 @@ def add_fields(self, new_fields): footer.cluster_group_record_frames = [] for cg_idx, cg in enumerate(existing_footer.cluster_group_records): ple = existing_page_list_envelopes[cg_idx] - if len(ple.pagelinklist) > 1: - raise ValueError( - f"add_fields does not yet support RNTuples with multiple clusters per cluster group " - f"(cluster group {cg_idx} has {len(ple.pagelinklist)} clusters). " - f"This is a known limitation that will be fixed in a future version." - ) - new_cluster_page_data = [] - for col_pages in ple.pagelinklist[0]: - existing_pages = [ - cnt.NTuple_PageDescription( - p.num_elements, - cnt.NTuple_Locator(p.locator.num_bytes, p.locator.offset), + all_cluster_page_data = [] + for cluster_idx, cluster_col_pages in enumerate(ple.pagelinklist): + new_cluster_page_data = [] + cluster_num_entries = ple.cluster_summaries[cluster_idx].num_entries + for col_pages in cluster_col_pages: + existing_pages = [ + cnt.NTuple_PageDescription( + p.num_elements, + cnt.NTuple_Locator(p.locator.num_bytes, p.locator.offset), + ) + for p in col_pages.pages + ] + new_cluster_page_data.append( + cnt.NTuple_ColumnPageListDescription( + existing_pages, col_pages.element_offset, compression.code + ) ) - for p in col_pages.pages - ] - new_cluster_page_data.append( - cnt.NTuple_ColumnPageListDescription( - existing_pages, col_pages.element_offset, compression.code + # write one new page per new field for this cluster + for field_name, field_dtype_raw in new_fields.items(): + ak_form = _type_specification_to_awkward_form(field_dtype_raw) + ak_primitive = ak_form.primitive + cluster_data = numpy.zeros(cluster_num_entries, dtype=numpy.dtype(ak_primitive)) + raw_cluster = cluster_data.view("uint8") + compressed_cluster = uproot.compression.compress(raw_cluster, compression) + cluster_page_key = self._cascading.add_rblob( + self._file.sink, compressed_cluster, len(raw_cluster) ) - ) - for field_name in new_fields: - new_cluster_page_data.append( - cnt.NTuple_ColumnPageListDescription( - [new_pages[field_name]], 0, compression.code + cluster_page_locator = cnt.NTuple_Locator( + len(compressed_cluster), + cluster_page_key.location + cluster_page_key.allocation, ) - ) + new_cluster_page_data.append( + cnt.NTuple_ColumnPageListDescription( + [cnt.NTuple_PageDescription(cluster_num_entries, cluster_page_locator)], + 0, + compression.code, + ) + ) + all_cluster_page_data.append(new_cluster_page_data) cluster_summaries = [ cnt.NTuple_ClusterSummary(s.num_first_entry, s.num_entries) for s in ple.cluster_summaries ] pagelistenv = cnt.NTuple_PageListEnvelope( - header._checksum, cluster_summaries, [new_cluster_page_data] + header._checksum, cluster_summaries, all_cluster_page_data ) pagelistenv_raw = pagelistenv.serialize() pagelistenv_key = self._cascading.add_rblob( diff --git a/tests/test_1687_rntuple_update.py b/tests/test_1687_rntuple_update.py index a211e143a..56f9674e1 100644 --- a/tests/test_1687_rntuple_update.py +++ b/tests/test_1687_rntuple_update.py @@ -533,13 +533,24 @@ def test_ntuple_extend_root_written_raises(tmp_path): ) -def test_ntuple_add_fields_multi_cluster_raises(tmp_path): - src = skhep_testdata.data_path("test_multiple_cluster_groups_rntuple_v1-0-0-0.root") - shutil.copy(src, os.path.join(tmp_path, "test.root")) +def test_ntuple_add_fields_multi_cluster_groups(tmp_path): + # multiple cluster groups (one cluster each) — should work + with uproot.recreate(os.path.join(tmp_path, "test.root")) as f: + f["ntuple"] = {"x": np.array([1, 2, 3], dtype=np.float32)} with uproot.update(os.path.join(tmp_path, "test.root")) as f: - with pytest.raises(ValueError): - f["ntuple"].add_fields({"newcol": np.int32}) + f["ntuple"].extend({"x": np.array([4, 5, 6], dtype=np.float32)}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["ntuple"].extend({"x": np.array([7, 8, 9], dtype=np.float32)}) + + with uproot.update(os.path.join(tmp_path, "test.root")) as f: + f["ntuple"].add_fields({"y": np.int32}) + + with uproot.open(os.path.join(tmp_path, "test.root")) as f: + assert f["ntuple"].num_entries == 9 + assert np.all(f["ntuple"]["x"].array() == np.arange(1, 10, dtype=np.float32)) + assert np.all(f["ntuple"]["y"].array() == 0) def test_ntuple_add_fields_sequential_same_session(tmp_path): From e828ef152fc2491365920c04213404e2124c51df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 30 Jul 2026 12:21:44 +0200 Subject: [PATCH 51/58] Minimize file reload in add_fields state refresh --- src/uproot/writing/writable.py | 35 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 670999082..853ae60f6 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2634,37 +2634,26 @@ def add_fields(self, new_fields): self._cascading._freesegments.write(self._file.sink) self._file.sink.flush() - # reload ntuple so subsequent add_fields/extend calls see updated schema - # find the directory that owns this ntuple and reload via _load_existing_ntuple - parent_dir = self._file._cascading.rootdirectory - key = parent_dir.data.get_key(self._path[-1], 1) - # rebuild cascading from file + # update in-memory state without full reload + # update field records and column counts directly from what we just wrote + self._cascading._existing_field_records = ( + list(existing_field_records) + list(footer.extension_field_record_frames) + ) + self._cascading._column_counts = numpy.append( + self._cascading._column_counts, + numpy.zeros(len(new_fields), dtype=int) + ) + # reload footer, page list envelopes and akform from file + # (needed for next add_fields call; read-only footer has cluster_group_records + # while writable footer only has cluster_group_record_frames) existing_file = uproot.open(self._file.file_path, minimal_ttree_metadata=False) try: existing = existing_file[self._path[-1]] _ = existing.keys() # trigger lazy loading of footer and page lists self._cascading._existing_footer = existing._footer self._cascading._existing_page_list_envelopes = existing.page_list_envelopes - self._cascading._existing_field_records = existing._ntuple.field_records full_akform, _ = existing.to_akform() self._cascading._header._akform = full_akform - # update column counts from existing page lists - existing_footer_reload = existing._footer - existing_ples = existing.page_list_envelopes - num_columns = ( - len(existing._header.column_records) - + len(existing._footer.extension_links.column_records) - if existing._footer - else len(existing.column_records) - ) - column_counts = [0] * num_columns - for cg_idx, cg in enumerate(existing_footer_reload.cluster_group_records): - ple = existing_ples[cg_idx] - for col_idx, col_pages in enumerate(ple.pagelinklist[0]): - column_counts[col_idx] += sum( - p.num_elements for p in col_pages.pages - ) - self._cascading._column_counts = numpy.array(column_counts, dtype=int) finally: existing_file.close() From 086f2e6682748c04bb32d3054cd1536571249795 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 30 Jul 2026 10:25:36 +0000 Subject: [PATCH 52/58] style: pre-commit fixes --- src/uproot/writing/writable.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 853ae60f6..e2be79b23 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2575,9 +2575,13 @@ def add_fields(self, new_fields): for field_name, field_dtype_raw in new_fields.items(): ak_form = _type_specification_to_awkward_form(field_dtype_raw) ak_primitive = ak_form.primitive - cluster_data = numpy.zeros(cluster_num_entries, dtype=numpy.dtype(ak_primitive)) + cluster_data = numpy.zeros( + cluster_num_entries, dtype=numpy.dtype(ak_primitive) + ) raw_cluster = cluster_data.view("uint8") - compressed_cluster = uproot.compression.compress(raw_cluster, compression) + compressed_cluster = uproot.compression.compress( + raw_cluster, compression + ) cluster_page_key = self._cascading.add_rblob( self._file.sink, compressed_cluster, len(raw_cluster) ) @@ -2587,7 +2591,11 @@ def add_fields(self, new_fields): ) new_cluster_page_data.append( cnt.NTuple_ColumnPageListDescription( - [cnt.NTuple_PageDescription(cluster_num_entries, cluster_page_locator)], + [ + cnt.NTuple_PageDescription( + cluster_num_entries, cluster_page_locator + ) + ], 0, compression.code, ) @@ -2636,12 +2644,11 @@ def add_fields(self, new_fields): # update in-memory state without full reload # update field records and column counts directly from what we just wrote - self._cascading._existing_field_records = ( - list(existing_field_records) + list(footer.extension_field_record_frames) + self._cascading._existing_field_records = list(existing_field_records) + list( + footer.extension_field_record_frames ) self._cascading._column_counts = numpy.append( - self._cascading._column_counts, - numpy.zeros(len(new_fields), dtype=int) + self._cascading._column_counts, numpy.zeros(len(new_fields), dtype=int) ) # reload footer, page list envelopes and akform from file # (needed for next add_fields call; read-only footer has cluster_group_records From ba1e624bacb5ad657e9c42b723b493a7217b4541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Thu, 30 Jul 2026 14:22:50 +0200 Subject: [PATCH 53/58] Fix ruff linting errors --- src/uproot/writing/writable.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 853ae60f6..f012302f9 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1113,7 +1113,7 @@ def _load_existing_ntuple(self, key): ) else: for i, (existing_cr, new_cr) in enumerate( - zip(existing_col_records, new_col_records) + zip(existing_col_records, new_col_records, strict=False) ): if ( existing_cr.type != new_cr.type_num @@ -1209,7 +1209,7 @@ def _load_existing_ntuple(self, key): # recover per-column element counts from existing page lists # for jagged fields, data columns advance by elements not entries column_counts = [] - for cg_idx, cg in enumerate(existing_footer.cluster_group_records): + for cg_idx, _cg in enumerate(existing_footer.cluster_group_records): ple = existing_page_list_envelopes[cg_idx] if not column_counts: column_counts = [0] * num_columns @@ -2572,7 +2572,7 @@ def add_fields(self, new_fields): ) ) # write one new page per new field for this cluster - for field_name, field_dtype_raw in new_fields.items(): + for _field_name, field_dtype_raw in new_fields.items(): ak_form = _type_specification_to_awkward_form(field_dtype_raw) ak_primitive = ak_form.primitive cluster_data = numpy.zeros(cluster_num_entries, dtype=numpy.dtype(ak_primitive)) From beecdf2a4c6e36722f40c3d6fdff34012ffebeae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Mon, 3 Aug 2026 12:28:07 +0200 Subject: [PATCH 54/58] Implement deferred columns for add_fields --- src/uproot/writing/_cascadentuple.py | 12 +++++-- src/uproot/writing/writable.py | 53 ++++------------------------ 2 files changed, 17 insertions(+), 48 deletions(-) diff --git a/src/uproot/writing/_cascadentuple.py b/src/uproot/writing/_cascadentuple.py index ebca2a72f..d92348246 100644 --- a/src/uproot/writing/_cascadentuple.py +++ b/src/uproot/writing/_cascadentuple.py @@ -242,21 +242,29 @@ def serialize(self): # https://github.com/root-project/root/blob/master/tree/ntuple/v7/doc/specifications.md#column-description class NTuple_Column_Description: - def __init__(self, type_num, bits_on_disk, field_id, flags, repr_index): + def __init__(self, type_num, bits_on_disk, field_id, flags, repr_index, first_element_index=0): self.type_num = type_num self.bits_on_disk = bits_on_disk self.field_id = field_id self.flags = flags self.repr_index = repr_index + self.first_element_index = first_element_index def serialize(self): + import uproot.const + flags = self.flags + if self.first_element_index > 0: + flags = flags | uproot.const.RNTupleColumnFlags.DEFERRED header_bytes = _rntuple_column_record_format.pack( self.type_num, self.bits_on_disk, self.field_id, - self.flags, + int(flags), self.repr_index, ) + if self.first_element_index > 0: + import struct + header_bytes += struct.pack(" Date: Mon, 3 Aug 2026 10:31:53 +0000 Subject: [PATCH 55/58] style: pre-commit fixes --- src/uproot/writing/_cascadentuple.py | 6 +++++- src/uproot/writing/writable.py | 15 ++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/src/uproot/writing/_cascadentuple.py b/src/uproot/writing/_cascadentuple.py index d92348246..f28deb1ee 100644 --- a/src/uproot/writing/_cascadentuple.py +++ b/src/uproot/writing/_cascadentuple.py @@ -242,7 +242,9 @@ def serialize(self): # https://github.com/root-project/root/blob/master/tree/ntuple/v7/doc/specifications.md#column-description class NTuple_Column_Description: - def __init__(self, type_num, bits_on_disk, field_id, flags, repr_index, first_element_index=0): + def __init__( + self, type_num, bits_on_disk, field_id, flags, repr_index, first_element_index=0 + ): self.type_num = type_num self.bits_on_disk = bits_on_disk self.field_id = field_id @@ -252,6 +254,7 @@ def __init__(self, type_num, bits_on_disk, field_id, flags, repr_index, first_el def serialize(self): import uproot.const + flags = self.flags if self.first_element_index > 0: flags = flags | uproot.const.RNTupleColumnFlags.DEFERRED @@ -264,6 +267,7 @@ def serialize(self): ) if self.first_element_index > 0: import struct + header_bytes += struct.pack(" Date: Mon, 3 Aug 2026 14:28:46 +0200 Subject: [PATCH 56/58] Fix ruff warnings: remove unused variables new_pages and cluster_num_entries --- src/uproot/writing/writable.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 3279c297c..2296efad0 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2448,8 +2448,6 @@ def add_fields(self, new_fields): next_field_id = len(existing_field_records) - new_pages = {} - existing_field_names = {fr.field_name for fr in existing_field_records} for field_name in new_fields: if field_name in existing_field_names: @@ -2546,7 +2544,6 @@ def add_fields(self, new_fields): all_cluster_page_data = [] for cluster_idx, cluster_col_pages in enumerate(ple.pagelinklist): new_cluster_page_data = [] - cluster_num_entries = ple.cluster_summaries[cluster_idx].num_entries for col_pages in cluster_col_pages: existing_pages = [ cnt.NTuple_PageDescription( From 9095236924197247048625992597670446c08937 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:38:38 +0200 Subject: [PATCH 57/58] Fix ruff: rename unused cluster_idx to _cluster_idx --- src/uproot/writing/writable.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 8f9917a6f..32cabd280 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -2551,7 +2551,7 @@ def add_fields(self, new_fields): for cg_idx, cg in enumerate(existing_footer.cluster_group_records): ple = existing_page_list_envelopes[cg_idx] all_cluster_page_data = [] - for cluster_idx, cluster_col_pages in enumerate(ple.pagelinklist): + for _cluster_idx, cluster_col_pages in enumerate(ple.pagelinklist): new_cluster_page_data = [] for col_pages in cluster_col_pages: existing_pages = [ From 66c622f70c4f9d743160fae90483c8909cc7e76c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jok=C5=ABbas=20Ma=C4=8Diulis?= <183698357+Yokubas@users.noreply.github.com> Date: Wed, 12 Aug 2026 17:14:26 +0200 Subject: [PATCH 58/58] Replace uproot.open with _ReadForUpdate in _load_existing_ntuple and add_fields --- src/uproot/writing/_cascade.py | 1 + src/uproot/writing/writable.py | 115 ++++++++++++++++++++++++--------- 2 files changed, 86 insertions(+), 30 deletions(-) diff --git a/src/uproot/writing/_cascade.py b/src/uproot/writing/_cascade.py index ddf2cd009..cb200b90c 100644 --- a/src/uproot/writing/_cascade.py +++ b/src/uproot/writing/_cascade.py @@ -1059,6 +1059,7 @@ def __init__(self, file_path, uuid, get_chunk=None, tlist_of_streamers=None): self._get_chunk = get_chunk self._tlist_of_streamers = tlist_of_streamers self._custom_classes = None + self.source = self @property def detached(self): diff --git a/src/uproot/writing/writable.py b/src/uproot/writing/writable.py index 32cabd280..e002a8565 100644 --- a/src/uproot/writing/writable.py +++ b/src/uproot/writing/writable.py @@ -1077,23 +1077,46 @@ def _load_existing_ntuple(self, key): "uproot.update() on a file-like object does not support accessing " "existing RNTuples; use uproot.update() with a file path instead." ) - # TODO: opening the file again in read mode to access existing metadata is - # a bit awkward since the file is already open in write mode. We should - # look into a better way to do this in the future. - existing_file = uproot.open(self.file_path, minimal_ttree_metadata=False) - try: - existing = existing_file[name] - _ = existing.keys() # trigger lazy loading of footer and page lists - full_akform, _ = existing.to_akform() - am = existing._ntuple.all_members - existing_key = existing_file.key(name + ";1") - anchor_location = existing_key.fSeekKey + existing_key.fKeylen - num_entries = existing.num_entries - existing_footer = existing._footer - existing_page_list_envelopes = existing.page_list_envelopes - existing_field_records = existing._ntuple.field_records - finally: - existing_file.close() + # use _ReadForUpdate to avoid loading entire file into memory + self._file.sink.flush() + + def _get_chunk_rn(start, stop): + raw_bytes = self._file.sink.read(start, stop - start) + return uproot.source.chunk.Chunk.wrap( + _readforupdate_rn, raw_bytes, start=start + ) + + _readforupdate_rn = uproot.writing._cascade._ReadForUpdate( + self._file.file_path, + self._file.uuid, + _get_chunk_rn, + self._file._cascading.tlist_of_streamers, + ) + _readforupdate_rn.options = dict(uproot.reading.open.defaults) + _readforupdate_rn.options["minimal_ttree_metadata"] = False + + _raw_bytes_rn = self._file.sink.read( + key.seek_location, + key.num_bytes + key.compressed_bytes, + ) + _chunk_rn = uproot.source.chunk.Chunk.wrap( + _readforupdate_rn, _raw_bytes_rn, start=key.seek_location + ) + _cursor_rn = uproot.source.cursor.Cursor( + key.seek_location, origin=key.num_bytes + ) + _readonlykey_rn = uproot.reading.ReadOnlyKey( + _chunk_rn, _cursor_rn, {}, _readforupdate_rn, self, read_strings=True + ) + existing = _readonlykey_rn.get() + _ = existing.keys() + full_akform, _ = existing.to_akform() + am = existing._ntuple.all_members + anchor_location = key.seek_location + key.num_bytes + num_entries = existing.num_entries + existing_footer = existing._footer + existing_page_list_envelopes = existing.page_list_envelopes + existing_field_records = existing._ntuple.field_records header = cnt.NTuple_Header( None, existing.name, existing._header.ntuple_description, full_akform @@ -2617,19 +2640,51 @@ def add_fields(self, new_fields): self._cascading._column_counts = numpy.append( self._cascading._column_counts, numpy.zeros(len(new_fields), dtype=int) ) - # reload footer, page list envelopes and akform from file - # (needed for next add_fields call; read-only footer has cluster_group_records - # while writable footer only has cluster_group_record_frames) - existing_file = uproot.open(self._file.file_path, minimal_ttree_metadata=False) - try: - existing = existing_file[self._path[-1]] - _ = existing.keys() # trigger lazy loading of footer and page lists - self._cascading._existing_footer = existing._footer - self._cascading._existing_page_list_envelopes = existing.page_list_envelopes - full_akform, _ = existing.to_akform() - self._cascading._header._akform = full_akform - finally: - existing_file.close() + # reload footer, page list envelopes and akform using _ReadForUpdate + self._file.sink.flush() + + def _get_chunk_reload(start, stop): + raw_bytes = self._file.sink.read(start, stop - start) + return uproot.source.chunk.Chunk.wrap( + _readforupdate_reload, raw_bytes, start=start + ) + + _readforupdate_reload = uproot.writing._cascade._ReadForUpdate( + self._file.file_path, + self._file.uuid, + _get_chunk_reload, + self._file._cascading.tlist_of_streamers, + ) + _readforupdate_reload.options = dict(uproot.reading.open.defaults) + _readforupdate_reload.options["minimal_ttree_metadata"] = False + + _ntuple_key = self._file._cascading.rootdirectory.data.get_key(self._path[-1]) + _raw_reload = self._file.sink.read( + _ntuple_key.seek_location, + _ntuple_key.num_bytes + _ntuple_key.compressed_bytes, + ) + _chunk_reload = uproot.source.chunk.Chunk.wrap( + _readforupdate_reload, _raw_reload, start=_ntuple_key.seek_location + ) + _cursor_reload = uproot.source.cursor.Cursor( + _ntuple_key.seek_location, origin=_ntuple_key.num_bytes + ) + _key_reload = uproot.reading.ReadOnlyKey( + _chunk_reload, + _cursor_reload, + {}, + _readforupdate_reload, + self, + read_strings=True, + ) + existing_reload = _key_reload.get() + _ = existing_reload.keys() + self._cascading._existing_footer = existing_reload._footer + self._cascading._existing_page_list_envelopes = ( + existing_reload.page_list_envelopes + ) + full_akform_reload, _ = existing_reload.to_akform() + self._cascading._header._akform = full_akform_reload def _is_type_specification(obj):