Support for compressed lists and its derivatives

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Version 0.4.0
+
+- Support for compressed lists.
+
 ## Version 0.3.0
 
 - Update to support the compressed genomicranges list implementation.

src/dolomite_ranges/__init__.py

@@ -20,4 +20,5 @@
 from .save_genomic_ranges  import save_genomic_ranges
 from .read_genomic_ranges import read_genomic_ranges 
 from .save_genomic_ranges_list import save_compressed_genomic_ranges_list
-from .read_genomic_ranges_list import read_genomic_ranges_list
+from .read_genomic_ranges_list import read_genomic_ranges_list
+from .read_atomic_vector_list import read_atomic_vector_list

src/dolomite_ranges/read_atomic_vector_list.py

@@ -0,0 +1,55 @@
+import os
+from typing import Optional
+
+import dolomite_base as dl
+import h5py
+from compressed_lists import Partitioning, splitAsCompressedList
+from dolomite_base.read_object import read_object_registry
+
+read_object_registry["atomic_vector_list"] = "dolomite_ranges.read_atomic_vector_list"
+
+
+def read_atomic_vector_list(path: str, metadata: Optional[dict], **kwargs):
+    """Load a list of atomic vectors from its on-disk representation.
+
+    Args:
+        path:
+            Path to the directory containing the object.
+
+        metadata:
+            Metadata for the object.
+
+        kwargs:
+            Further arguments, ignored.
+
+
+    """
+    return _read_compressed_list(path, metadata, "atomic_vector_list", **kwargs)
+
+
+def _read_compressed_list(path: str, metadata: Optional[dict], name: str, **kwargs):
+    concat_path = os.path.join(path, "concatenated")
+    concat = dl.alt_read_object(concat_path, **kwargs)
+
+    fpath = os.path.join(path, "partitions.h5")
+    with h5py.File(fpath, "r") as fhandle:
+        ghandle = fhandle[name]
+        lengths = dl.load_vector_from_hdf5(ghandle["lengths"], expected_type=int, report_1darray=True)
+
+        names = None
+        if "names" in ghandle:
+            names = dl.load_vector_from_hdf5(ghandle["names"], expected_type=str, report_1darray=True)
+
+        output = splitAsCompressedList(concat, Partitioning.from_lengths(lengths=lengths, names=names))
+
+    _elem_annotation_path = os.path.join(path, "element_annotations")
+    if os.path.exists(_elem_annotation_path):
+        _mcols = dl.alt_read_object(_elem_annotation_path, **kwargs)
+        output = output.set_element_metadata(_mcols)
+
+    _meta_path = os.path.join(path, "other_annotations")
+    if os.path.exists(_meta_path):
+        _meta = dl.alt_read_object(_meta_path, **kwargs)
+        output = output.set_metadata(_meta.as_dict())
+
+    return output

src/dolomite_ranges/read_frame_list.py

@@ -0,0 +1,32 @@
+from typing import Optional
+
+from compressed_lists import CompressedSplitBiocFrameList
+from dolomite_base.read_object import read_object_registry
+
+from .read_atomic_vector_list import _read_compressed_list
+
+read_object_registry["data_frame_list"] = "dolomite_ranges.read_data_frame_list"
+
+
+def read_data_frame_list(path: str, metadata: Optional[dict], **kwargs) -> CompressedSplitBiocFrameList:
+    """Load data frame list.
+
+    This method
+    should generally not be called directly but instead be invoked by
+    :py:meth:`~dolomite_base.read_object.read_object`.
+
+    Args:
+        path:
+            Path to the directory containing the object.
+
+        metadata:
+            Metadata for the object.
+
+        kwargs:
+            Further arguments, ignored.
+
+    Returns:
+        A :py:class:`~compressed_lists.biocframe_list.CompressedSplitBiocFrameList` object.
+    """
+
+    return _read_compressed_list(path, metadata=metadata, name="data_frame_list", **kwargs)

src/dolomite_ranges/read_genomic_ranges_list.py

@@ -1,12 +1,10 @@
-import os
 from typing import Optional
 
-import dolomite_base as dl
-import h5py
-from compressed_lists import Partitioning
 from dolomite_base.read_object import read_object_registry
 from genomicranges import CompressedGenomicRangesList
 
+from .read_atomic_vector_list import _read_compressed_list
+
 read_object_registry["genomic_ranges_list"] = "dolomite_ranges.read_genomic_ranges_list"
 
 
@@ -32,39 +30,4 @@ def read_genomic_ranges_list(path: str, metadata: Optional[dict], **kwargs) -> C
         A :py:class:`~genomicranges.grangeslist.CompressedGenomicRangesList` object.
     """
 
-    with h5py.File(os.path.join(path, "partitions.h5"), "r") as handle:
-        ghandle = handle["genomic_ranges_list"]
-
-        lengths = dl.load_vector_from_hdf5(ghandle["lengths"], expected_type=int, report_1darray=True)
-
-        names = None
-        if "names" in ghandle:
-            names = dl.load_vector_from_hdf5(ghandle["names"], expected_type=str, report_1darray=True)
-
-    _all_granges = dl.alt_read_object(path=os.path.join(path, "concatenated"), **kwargs)
-
-    counter = 0
-    _split_granges = []
-    if lengths.sum() == 0:
-        _split_granges = _all_granges
-    else:
-        for ilen in lengths:
-            _frag = _all_granges[counter : (counter + ilen)]
-            _split_granges.append(_frag)
-            counter += ilen
-
-    grl = CompressedGenomicRangesList(
-        unlist_data=_all_granges, partitioning=Partitioning.from_lengths(lengths=lengths, names=names)
-    )
-
-    _elem_annotation_path = os.path.join(path, "element_annotations")
-    if os.path.exists(_elem_annotation_path):
-        _mcols = dl.alt_read_object(_elem_annotation_path, **kwargs)
-        grl = grl.set_element_metadata(_mcols)
-
-    _meta_path = os.path.join(path, "other_annotations")
-    if os.path.exists(_meta_path):
-        _meta = dl.alt_read_object(_meta_path, **kwargs)
-        grl = grl.set_metadata(_meta.as_dict())
-
-    return grl
+    return _read_compressed_list(path, metadata=metadata, name="genomic_ranges_list", **kwargs)

src/dolomite_ranges/save_atomic_vector_list.py

@@ -0,0 +1,66 @@
+import os
+from typing import Optional
+
+import dolomite_base as dl
+import h5py
+from biocutils import combine_sequences
+from compressed_lists import CompressedList
+
+
+@dl.save_object.register
+@dl.validate_saves
+def save_atomic_vector_list(x: CompressedList, path: str, **kwargs):
+    """Method for saving :py:class:`~CompressedList`
+    objects to their corresponding file representations, see
+    :py:meth:`~dolomite_base.save_object.save_object` for details.
+
+    Args:
+        x:
+            Object to be staged.
+
+        path:
+            Path to a directory in which to save ``x``.
+
+        data_frame_args:
+            Further arguments to pass to the ``save_object`` method for
+            ``mcols``.
+
+        kwargs:
+            Further arguments to be passed to individual methods.
+
+    Returns:
+        `x` is saved to `path`.
+    """
+    return _save_compressed_list(x, path=path, name="atomic_vector_list", **kwargs)
+
+
+def _save_compressed_list(x, path, name, **kwargs):
+    os.mkdir(path)
+
+    _all_ranges = x.get_unlist_data()
+    if isinstance(_all_ranges, list) and len(_all_ranges) > 1:
+        _all_ranges = combine_sequences(*x.get_unlist_data())
+
+    dl.alt_save_object(_all_ranges, path=os.path.join(path, "concatenated"), **kwargs)
+
+    _elem_annotation = x.get_element_metadata()
+    if _elem_annotation is not None:
+        dl.alt_save_object(_elem_annotation, path=os.path.join(path, "element_annotations"), **kwargs)
+
+    _meta = x.get_metadata()
+    if _meta is not None and len(_meta) > 0:
+        dl.alt_save_object(_meta, path=os.path.join(path, "other_annotations"), **kwargs)
+
+    with h5py.File(os.path.join(path, "partitions.h5"), "w") as handle:
+        ghandle = handle.create_group(name)
+
+        dl.write_integer_vector_to_hdf5(ghandle, name="lengths", h5type="u4", x=x.get_element_lengths())
+
+        if x.get_names() is not None:
+            dl.write_string_vector_to_hdf5(ghandle, name="names", x=x.get_names())
+
+    _info = {}
+    _info[name] = {"version": "1.0"}
+    dl.save_object_file(path, name, _info)
+
+    return

src/dolomite_ranges/save_frame_list.py

@@ -0,0 +1,31 @@
+import dolomite_base as dl
+from compressed_lists import CompressedSplitBiocFrameList
+
+from .save_atomic_vector_list import _save_compressed_list
+
+
+@dl.save_object.register
+@dl.validate_saves
+def save_compressed_genomic_ranges_list(x: CompressedSplitBiocFrameList, path: str, **kwargs):
+    """Method for saving :py:class:`~compressed_lists.biocframe_list.CompressedSplitBiocFrameList`
+    objects to their corresponding file representations, see
+    :py:meth:`~dolomite_base.save_object.save_object` for details.
+
+    Args:
+        x:
+            Object to be staged.
+
+        path:
+            Path to a directory in which to save ``x``.
+
+        data_frame_args:
+            Further arguments to pass to the ``save_object`` method for
+            ``mcols``.
+
+        kwargs:
+            Further arguments to be passed to individual methods.
+
+    Returns:
+        `x` is saved to `path`.
+    """
+    return _save_compressed_list(x, path=path, name="genomic_ranges_list", **kwargs)

src/dolomite_ranges/save_genomic_ranges_list.py

@@ -1,11 +1,10 @@
-import os
 from typing import Optional
 
 import dolomite_base as dl
-import h5py
-from biocutils import combine_sequences
 from genomicranges import CompressedGenomicRangesList
 
+from .save_atomic_vector_list import _save_compressed_list
+
 
 @dl.save_object.register
 @dl.validate_saves
@@ -33,38 +32,4 @@ def save_compressed_genomic_ranges_list(
     Returns:
         `x` is saved to `path`.
     """
-    os.mkdir(path)
-
-    if data_frame_args is None:
-        data_frame_args = {}
-
-    _info = {"genomic_ranges_list": {"version": "1.0"}}
-    dl.save_object_file(path, "genomic_ranges_list", _info)
-
-    with h5py.File(os.path.join(path, "partitions.h5"), "w") as handle:
-        ghandle = handle.create_group("genomic_ranges_list")
-
-        dl.write_integer_vector_to_hdf5(ghandle, name="lengths", h5type="u4", x=x.get_element_lengths())
-
-        if x.get_names() is not None:
-            dl.write_string_vector_to_hdf5(ghandle, name="names", x=x.get_names())
-
-    _all_ranges = x.get_unlist_data()
-    if isinstance(_all_ranges, list) and len(_all_ranges) > 1:
-        _all_ranges = combine_sequences(*x.get_unlist_data())
-
-    dl.alt_save_object(_all_ranges, path=os.path.join(path, "concatenated"), **kwargs)
-
-    _elem_annotation = x.get_element_metadata()
-    if _elem_annotation is not None:
-        dl.alt_save_object(
-            _elem_annotation,
-            path=os.path.join(path, "element_annotations"),
-            **data_frame_args,
-        )
-
-    _meta = x.get_metadata()
-    if _meta is not None and len(_meta) > 0:
-        dl.alt_save_object(_meta, path=os.path.join(path, "other_annotations"), **kwargs)
-
-    return
+    return _save_compressed_list(x, path=path, name="genomic_ranges_list", **kwargs)

tests/test_granges_list.py

@@ -30,6 +30,7 @@ def test_genomic_ranges_list():
     save_object(grl, dir)
 
     roundtrip = read_object(dir)
+    assert isinstance(roundtrip, CompressedGenomicRangesList)
     assert roundtrip.get_names() == grl.get_names()
     assert len(roundtrip.get_unlist_data()) == len(grl.get_unlist_data())
     assert np.allclose(roundtrip["a"].start, grl["a"].start)