From eeecea7b2310d193550e1aaa5d393f5a9c70ec4d Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 18 Feb 2026 16:35:32 -0500 Subject: [PATCH 01/12] wip --- src/nd2/_parse/_chunk_decode.py | 2 +- src/nd2/_readers/_modern/modern_reader.py | 36 +++++++++++++++-------- src/nd2/_readers/protocol.py | 16 ++++++++-- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/src/nd2/_parse/_chunk_decode.py b/src/nd2/_parse/_chunk_decode.py index b159a82d..bc16c2d7 100644 --- a/src/nd2/_parse/_chunk_decode.py +++ b/src/nd2/_parse/_chunk_decode.py @@ -93,7 +93,7 @@ def get_version(fh: BinaryIO | StrOrBytesPath) -> tuple[int, int]: with ctx as fh: fh.seek(0) - fname = str(fh.name) + fname = str(getattr(fh, "name", "")) chunk = START_FILE_CHUNK.unpack(fh.read(START_FILE_CHUNK.size)) magic, name_length, data_length, name, data = cast("StartFileChunk", chunk) diff --git a/src/nd2/_readers/_modern/modern_reader.py b/src/nd2/_readers/_modern/modern_reader.py index 7ece2ec8..e603b737 100644 --- a/src/nd2/_readers/_modern/modern_reader.py +++ b/src/nd2/_readers/_modern/modern_reader.py @@ -331,18 +331,21 @@ def read_frame(self, index: int) -> np.ndarray: if self.attributes().compressionType == "lossless": return self._read_compressed_frame(index) - try: - return np.ndarray( - shape=self._actual_frame_shape(), - dtype=self._dtype(), - buffer=self._mmap, - offset=offset, - strides=self._strides, - ) - except TypeError: - # If the chunkmap is wrong, and the mmap isn't long enough - # for the requested offset & size, a TypeError is raised. - return self._missing_frame(index) + if self._mmap is not None: + try: + return np.ndarray( + shape=self._actual_frame_shape(), + dtype=self._dtype(), + buffer=self._mmap, + offset=offset, + strides=self._strides, + ) + except TypeError: + # If the chunkmap is wrong, and the mmap isn't long enough + # for the requested offset & size, a TypeError is raised. + return self._missing_frame(index) + + return self._read_frame_bytes(offset) def _read_compressed_frame(self, index: int) -> np.ndarray: ch = self._load_chunk(f"ImageDataSeq|{index}!".encode()) @@ -353,6 +356,15 @@ def _read_compressed_frame(self, index: int) -> np.ndarray: strides=self._strides, ) + def _read_frame_bytes(self, offset: int) -> np.ndarray: + """Read a frame via seek/read (fallback when mmap is unavailable).""" + shape = self._actual_frame_shape() + dtype = self._dtype() + nbytes = int(np.prod(shape)) * dtype.itemsize + self._fh.seek(offset) + data = self._fh.read(nbytes) + return np.frombuffer(data, dtype=dtype).reshape(shape) + def _missing_frame(self, index: int = 0) -> np.ndarray: # TODO: add other modes for filling missing data return np.zeros(self._raw_frame_shape(), self._dtype()) diff --git a/src/nd2/_readers/protocol.py b/src/nd2/_readers/protocol.py index 5a379f20..cb4ee2bf 100644 --- a/src/nd2/_readers/protocol.py +++ b/src/nd2/_readers/protocol.py @@ -67,7 +67,7 @@ def create( ctx = open(path, "rb") with ctx as fh: - fname = fh.name + fname = getattr(fh, "name", "") fh.seek(0) magic_num = fh.read(4) @@ -85,8 +85,14 @@ def __init__(self, path: FileOrBinaryIO, error_radius: int | None = None) -> Non if hasattr(path, "read"): self._fh: BinaryIO | None = cast("BinaryIO", path) self._was_open = not self._fh.closed - self._path: Path = Path(self._fh.name) - self._mmap = mmap.mmap(self._fh.fileno(), 0, access=mmap.ACCESS_READ) + name = getattr(self._fh, "name", None) + self._path: Path | None = Path(name) if isinstance(name, str) else None + try: + self._mmap = mmap.mmap( + self._fh.fileno(), 0, access=mmap.ACCESS_READ + ) + except Exception: + pass # remote/non-fileno file-likes: mmap not available else: self._was_open = False self._path = Path(path) @@ -101,6 +107,10 @@ def is_legacy(self) -> bool: def open(self) -> None: """Open the file handle.""" if self._fh is None or self._fh.closed: + if self._path is None: + raise RuntimeError( + "Cannot reopen a remote/file-like ND2 source after closing" + ) self._fh = open(self._path, "rb") self._mmap = mmap.mmap(self._fh.fileno(), 0, access=mmap.ACCESS_READ) From 2b877078e62e92ee41b2aec20cf39b6ef4b75586 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Wed, 18 Feb 2026 20:16:38 -0500 Subject: [PATCH 02/12] feat: add support for remote URL handling in ND2File and ND2Reader --- pyproject.toml | 2 ++ src/nd2/_nd2file.py | 35 +++++++++++++++++----- src/nd2/_readers/protocol.py | 32 ++++++++++++++------ src/nd2/_util.py | 58 ++++++++++++++++++++++++++++++++++++ 4 files changed, 111 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a63a1b4e..9ef7f3f0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,7 @@ ome-zarr-tensorstore = [ "pydantic>=2.11.0", "yaozarrs[write-tensorstore]>=0.2.0; python_version >= '3.10'", ] +remote = ["fsspec", "s3fs"] [dependency-groups] test = [ @@ -72,6 +73,7 @@ dev = [ "pytest-xdist>=3.8.0", "rich>=14.2.0", "ruff>=0.14.0", + "s3fs>=2025.10.0", "types-lxml>=2025.8.25", ] docs = [ diff --git a/src/nd2/_nd2file.py b/src/nd2/_nd2file.py index 05032a72..4a72d337 100644 --- a/src/nd2/_nd2file.py +++ b/src/nd2/_nd2file.py @@ -5,6 +5,7 @@ import threading import warnings from itertools import product +from pathlib import Path from types import MappingProxyType from typing import TYPE_CHECKING, Callable, cast, overload @@ -13,7 +14,7 @@ from nd2 import _util from ._readers.protocol import ND2Reader -from ._util import AXIS, is_supported_file +from ._util import AXIS, _is_fsspec_url, is_supported_file try: from functools import cached_property @@ -24,7 +25,6 @@ if TYPE_CHECKING: from collections.abc import Mapping, Sequence, Sized from os import PathLike - from pathlib import Path from typing import Any, Literal, SupportsInt import dask.array @@ -101,12 +101,18 @@ def __init__( *, validate_frames: bool = False, search_window: int = 100, + storage_options: dict | None = None, ) -> None: self._error_radius: int | None = ( search_window * 1000 if validate_frames else None ) - self._rdr = ND2Reader.create(path, self._error_radius) + self._storage_options = storage_options + self._rdr = ND2Reader.create(path, self._error_radius, storage_options) self._path = self._rdr._path + # For URL inputs the reader may not extract the full URL from the handle; + # preserve the original URL string so pickling round-trips correctly. + if isinstance(path, str) and _is_fsspec_url(path): + self._path = path self._lock = threading.RLock() @staticmethod @@ -230,7 +236,11 @@ def __setstate__(self, d: dict[str, Any]) -> None: _was_closed = d.pop("_closed", False) self.__dict__ = d self._lock = threading.RLock() - self._rdr = ND2Reader.create(self._path, self._error_radius) + self._rdr = ND2Reader.create( + self._path, + self._error_radius, + getattr(self, "_storage_options", None), + ) if _was_closed: self._rdr.close() @@ -1349,7 +1359,8 @@ def __repr__(self) -> str: """Return a string representation of the ND2File.""" try: details = " (closed)" if self.closed else f" {self.dtype}: {self.sizes!r}" - extra = f": {self._path.name!r}{details}" + path_name = self._path.name if isinstance(self._path, Path) else self._path + extra = f": {path_name!r}{details}" except Exception: extra = "" return f"" @@ -1450,6 +1461,7 @@ def imread( dask: Literal[False] = ..., xarray: Literal[False] = ..., validate_frames: bool = ..., + storage_options: dict | None = ..., ) -> np.ndarray: ... @@ -1460,6 +1472,7 @@ def imread( dask: bool = ..., xarray: Literal[True], validate_frames: bool = ..., + storage_options: dict | None = ..., ) -> xr.DataArray: ... @@ -1470,6 +1483,7 @@ def imread( dask: Literal[True], xarray: Literal[False] = ..., validate_frames: bool = ..., + storage_options: dict | None = ..., ) -> dask.array.core.Array: ... @@ -1479,13 +1493,15 @@ def imread( dask: bool = False, xarray: bool = False, validate_frames: bool = False, + storage_options: dict | None = None, ) -> np.ndarray | xr.DataArray | dask.array.core.Array: """Open `file`, return requested array type, and close `file`. Parameters ---------- file : Path | str - Filepath (`str`) or `Path` object to ND2 file. + Filepath (`str`) or `Path` object to ND2 file. May also be a remote + URL string (e.g. ``"s3://bucket/key.nd2"``) when fsspec is installed. dask : bool If `True`, returns a (delayed) `dask.array.Array`. This will avoid reading any data from disk until specifically requested by using `.compute()` or @@ -1501,13 +1517,18 @@ def imread( shifted relative to the predicted offset (i.e. in a corrupted file). This comes at a slight performance penalty at file open, but may "rescue" some corrupt files. by default False. + storage_options : dict, optional + Extra kwargs forwarded to fsspec when opening a remote URL, e.g. + ``{"anon": True}`` for public S3 buckets. Returns ------- Union[np.ndarray, dask.array.Array, xarray.DataArray] Array subclass, depending on arguments used. """ - with ND2File(file, validate_frames=validate_frames) as nd2: + with ND2File( + file, validate_frames=validate_frames, storage_options=storage_options + ) as nd2: if xarray: return nd2.to_xarray(delayed=dask) elif dask: diff --git a/src/nd2/_readers/protocol.py b/src/nd2/_readers/protocol.py index cb4ee2bf..a1ccc534 100644 --- a/src/nd2/_readers/protocol.py +++ b/src/nd2/_readers/protocol.py @@ -8,6 +8,7 @@ from typing import TYPE_CHECKING, BinaryIO, cast from nd2._parse._chunk_decode import get_version +from nd2._util import _is_fsspec_url, _open_fsspec_url if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -41,6 +42,7 @@ def create( cls, path: FileOrBinaryIO, error_radius: int | None = None, + storage_options: dict | None = None, ) -> ND2Reader: """Create an ND2Reader for the given path, using the appropriate subclass. @@ -52,6 +54,8 @@ def create( If b"ND2 FILEMAP SIGNATURE NAME 0001!" is not found at expected location and `error_radius` is not None, then an area of +/- `error_radius` bytes will be searched for the signature. + storage_options : dict, optional + Extra kwargs passed to fsspec when opening remote URLs. """ from nd2._readers import LegacyReader, ModernReader @@ -62,6 +66,8 @@ def create( "File handles passed to ND2File must be in binary mode" ) ctx: AbstractContextManager[BinaryIO] = nullcontext(path) + elif _is_fsspec_url(path): + ctx = nullcontext(_open_fsspec_url(str(path), storage_options)) else: path = Path(path).expanduser().absolute() ctx = open(path, "rb") @@ -73,24 +79,30 @@ def create( for subcls in (ModernReader, LegacyReader): if magic_num == subcls.HEADER_MAGIC: - return subcls(path, error_radius=error_radius) + # For URL/file-like cases pass the open handle; for local paths + # pass the Path so the reader can reopen it as needed. + effective_path = ( + fh if _is_fsspec_url(path) or hasattr(path, "read") else path + ) + return subcls(effective_path, error_radius=error_radius) raise OSError( - f"file {fname} not recognized as ND2. First 4 bytes: {magic_num!r}" + f"file {fname!r} not recognized as ND2. First 4 bytes: {magic_num!r}" ) def __init__(self, path: FileOrBinaryIO, error_radius: int | None = None) -> None: self._chunkmap: dict | None = None + self._version: tuple[int, int] | None = None self._mmap: mmap.mmap | None = None if hasattr(path, "read"): self._fh: BinaryIO | None = cast("BinaryIO", path) self._was_open = not self._fh.closed - name = getattr(self._fh, "name", None) - self._path: Path | None = Path(name) if isinstance(name, str) else None + name = getattr(self._fh, "full_name", None) or getattr( + self._fh, "name", None + ) + self._path: str | Path | None = name if isinstance(name, str) else None try: - self._mmap = mmap.mmap( - self._fh.fileno(), 0, access=mmap.ACCESS_READ - ) + self._mmap = mmap.mmap(self._fh.fileno(), 0, access=mmap.ACCESS_READ) except Exception: pass # remote/non-fileno file-likes: mmap not available else: @@ -107,7 +119,7 @@ def is_legacy(self) -> bool: def open(self) -> None: """Open the file handle.""" if self._fh is None or self._fh.closed: - if self._path is None: + if not isinstance(self._path, Path): raise RuntimeError( "Cannot reopen a remote/file-like ND2 source after closing" ) @@ -138,7 +150,9 @@ def __exit__(self, *_: Any) -> None: def version(self) -> tuple[int, int]: """Return the file format version as a tuple of ints.""" - return get_version(self._fh or self._path) + if self._version is None: + self._version = get_version(self._fh or self._path) + return self._version def rois(self) -> list[ROI]: """Return ROIs in the file.""" diff --git a/src/nd2/_util.py b/src/nd2/_util.py index 2a354210..347ca671 100644 --- a/src/nd2/_util.py +++ b/src/nd2/_util.py @@ -30,6 +30,64 @@ def _open_binary(path: StrOrPath) -> BinaryIO: return open(path, "rb") +def _is_fsspec_url(path: object) -> bool: + """True if `path` is a string with a remote URL scheme (e.g. 's3://').""" + if not isinstance(path, str): + return False + idx = path.find("://") + # idx > 1 excludes Windows drive letters like C:/ + return idx > 1 + + +def _open_fsspec_url( + url: str, + storage_options: dict | None = None, +) -> BinaryIO: + """Open a remote nd2 URL with a pre-warmed metadata cache. + + Pre-fetches the first 512 KB and last 5 MB with cat_ranges(), then opens + with cache_type='parts' so all nd2 metadata reads hit the in-memory cache. + Frame data reads fall through to the normal fetcher (strict=False). + """ + try: + import fsspec + except ImportError as e: + raise ImportError( + "fsspec is required to open remote URLs. Install with: pip install fsspec" + ) from e + + sopts = storage_options or {} + fs, fpath = fsspec.url_to_fs(url, **sopts) + size: int = fs.info(fpath)["size"] + + START_BYTES = 512 * 1024 # 512 KB — covers magic + ImageMetadataSeqLV|0! + END_BYTES = 5 * 1024 * 1024 # 5 MB — covers chunkmap + all end metadata + + if size <= START_BYTES + END_BYTES: + raw = fs.cat_file(fpath) + known: dict[tuple[int, int], bytes] = {(0, size): raw} + else: + start_raw, end_raw = fs.cat_ranges( + [fpath, fpath], + starts=[0, size - END_BYTES], + ends=[START_BYTES, size], + ) + known = { + (0, START_BYTES): start_raw, + (size - END_BYTES, size): end_raw, + } + + return cast( + "BinaryIO", + fs.open( + fpath, + "rb", + cache_type="parts", + cache_options={"data": known, "strict": False}, + ), + ) + + def is_supported_file( path: FileOrBinaryIO, open_: Callable[[StrOrPath], BinaryIO] = _open_binary, From e78921cba9f208b1e016a6cadebbb25382c8777e Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 10:33:20 -0500 Subject: [PATCH 03/12] more thorough testing --- pyproject.toml | 4 +- src/nd2/_ome.py | 1 + src/nd2/_readers/_modern/modern_reader.py | 20 ++++-- src/nd2/_util.py | 33 ++++------ tests/test_s3.py | 75 +++++++++++++++++++++++ 5 files changed, 106 insertions(+), 27 deletions(-) create mode 100644 tests/test_s3.py diff --git a/pyproject.toml b/pyproject.toml index 9ef7f3f0..c55fef0f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -51,10 +51,11 @@ remote = ["fsspec", "s3fs"] [dependency-groups] test = [ - "nd2[tiff]", + "nd2[tiff,remote]", "dask[array]>=2023.5.0", "imagecodecs>=2023.1.23", "lxml>=5.3.0", + "moto[s3,server]>=5.1.0", "psutil>=5.9.8", "pytest-codspeed>=4.1.1", "pytest-cov>=4.0.0", @@ -66,6 +67,7 @@ test = [ dev = [ { include-group = "test" }, { include-group = "docs" }, + "boto3-stubs[s3]>=1.42.52", "ipython>=8.0.0", "mypy>=1.18.2", "pdbpp>=0.11.7; sys_platform != 'win32'", diff --git a/src/nd2/_ome.py b/src/nd2/_ome.py index fa0f3459..8c210ba5 100644 --- a/src/nd2/_ome.py +++ b/src/nd2/_ome.py @@ -224,6 +224,7 @@ def nd2_ome_metadata( if include_unstructured: all_meta = m.MapAnnotation( + id="Annotation:0", description="ND2 unstructured metadata, encoded as a JSON string. " "Each key in this MapAnnotation is the name of a metadata chunk found in " "the ND2 file, and the value is the JSON-encoded data for that chunk.", diff --git a/src/nd2/_readers/_modern/modern_reader.py b/src/nd2/_readers/_modern/modern_reader.py index e603b737..c74853bc 100644 --- a/src/nd2/_readers/_modern/modern_reader.py +++ b/src/nd2/_readers/_modern/modern_reader.py @@ -1,6 +1,5 @@ from __future__ import annotations -import os import warnings import zlib from contextlib import suppress @@ -180,9 +179,6 @@ def _cached_global_metadata(self) -> GlobalMetadata: exp_loops=self.experiment(), text_info=self.text_info(), ) - if self._global_metadata["time"]["absoluteJulianDayNumber"] < 1: - julian_day = os.stat(self._path).st_ctime / 86400.0 + 2440587.5 - self._global_metadata["time"]["absoluteJulianDayNumber"] = julian_day return self._global_metadata @@ -358,11 +354,25 @@ def _read_compressed_frame(self, index: int) -> np.ndarray: def _read_frame_bytes(self, offset: int) -> np.ndarray: """Read a frame via seek/read (fallback when mmap is unavailable).""" + if self._fh is None: # pragma: no cover + raise ValueError("Attempt to read from closed nd2 file") + shape = self._actual_frame_shape() dtype = self._dtype() - nbytes = int(np.prod(shape)) * dtype.itemsize + if self._strides is not None: + nbytes = shape[0] * (self.attributes().widthBytes or 0) + else: + nbytes = int(np.prod(shape)) * dtype.itemsize self._fh.seek(offset) data = self._fh.read(nbytes) + if self._strides is not None: + arr = np.ndarray( + shape=shape, + dtype=dtype, + buffer=data, + strides=self._strides, + ) + return arr.copy() return np.frombuffer(data, dtype=dtype).reshape(shape) def _missing_frame(self, index: int = 0) -> np.ndarray: diff --git a/src/nd2/_util.py b/src/nd2/_util.py index 347ca671..70c6808f 100644 --- a/src/nd2/_util.py +++ b/src/nd2/_util.py @@ -45,9 +45,9 @@ def _open_fsspec_url( ) -> BinaryIO: """Open a remote nd2 URL with a pre-warmed metadata cache. - Pre-fetches the first 512 KB and last 5 MB with cat_ranges(), then opens - with cache_type='parts' so all nd2 metadata reads hit the in-memory cache. - Frame data reads fall through to the normal fetcher (strict=False). + For smaller files, pre-fetches the full object and opens with + cache_type='parts'. For larger files, opens directly via filesystem + defaults to avoid non-contiguous cache composition artifacts. """ try: import fsspec @@ -66,26 +66,17 @@ def _open_fsspec_url( if size <= START_BYTES + END_BYTES: raw = fs.cat_file(fpath) known: dict[tuple[int, int], bytes] = {(0, size): raw} - else: - start_raw, end_raw = fs.cat_ranges( - [fpath, fpath], - starts=[0, size - END_BYTES], - ends=[START_BYTES, size], + return cast( + "BinaryIO", + fs.open( + fpath, + "rb", + cache_type="parts", + cache_options={"data": known, "strict": False}, + ), ) - known = { - (0, START_BYTES): start_raw, - (size - END_BYTES, size): end_raw, - } - return cast( - "BinaryIO", - fs.open( - fpath, - "rb", - cache_type="parts", - cache_options={"data": known, "strict": False}, - ), - ) + return cast("BinaryIO", fs.open(fpath, "rb")) def is_supported_file( diff --git a/tests/test_s3.py b/tests/test_s3.py new file mode 100644 index 00000000..3850fe02 --- /dev/null +++ b/tests/test_s3.py @@ -0,0 +1,75 @@ +from collections.abc import Iterable +from pathlib import Path +from typing import TYPE_CHECKING +from uuid import uuid4 + +import numpy as np +import pytest +from nd2 import ND2File, imread + + +@pytest.fixture(scope="session") +def s3_endpoint() -> Iterable[str]: + if TYPE_CHECKING: + from moto.server import ThreadedMotoServer + else: + ThreadedMotoServer = pytest.importorskip("moto.server").ThreadedMotoServer + server = ThreadedMotoServer(port=0) + server.start() + _, port = server.get_host_and_port() + try: + yield f"http://127.0.0.1:{port}" + finally: + server.stop() + + +@pytest.fixture() +def s3_nd2_url(small_nd2s: Path, s3_endpoint: str) -> Iterable[tuple[str, str]]: + """Yield an S3 URL pointing to `single_nd2`, along with the endpoint URL.""" + if TYPE_CHECKING: + import boto3 + else: + boto3 = pytest.importorskip("boto3") + + bucket = f"nd2-test-{uuid4().hex[:12]}" + client = boto3.client("s3", endpoint_url=s3_endpoint, region_name="us-east-1") + client.create_bucket(Bucket=bucket) + client.upload_file(str(small_nd2s), bucket, small_nd2s.name) + yield f"s3://{bucket}/{small_nd2s.name}", s3_endpoint + + +def test_nd2file_reads_from_s3_url( + s3_nd2_url: tuple[str, str], small_nd2s: Path +) -> None: + url, endpoint = s3_nd2_url + storage_options = { + "client_kwargs": {"endpoint_url": endpoint}, + # necessary for moto's S3 implementation + # to avoid checksum validation errors on multipart uploads + "config_kwargs": {"response_checksum_validation": "when_required"}, + } + remote_nd = ND2File(url, storage_options=storage_options) + local_nd = ND2File(small_nd2s) + with remote_nd, local_nd: + assert remote_nd.path == url + assert remote_nd.shape == local_nd.shape + assert remote_nd.metadata == local_nd.metadata + assert remote_nd.attributes == local_nd.attributes + assert remote_nd.text_info == local_nd.text_info + assert remote_nd.experiment == local_nd.experiment + assert remote_nd.frame_metadata(0) == local_nd.frame_metadata(0) + assert remote_nd.events() == local_nd.events() + assert remote_nd.ome_metadata() == local_nd.ome_metadata() + np.testing.assert_array_equal(remote_nd.read_frame(0), local_nd.read_frame(0)) + if local_nd.binary_data is not None: + assert remote_nd.binary_data is not None + for rb, lb in zip( + remote_nd.binary_data, local_nd.binary_data, strict=False + ): + np.testing.assert_array_equal(rb.asarray(), lb.asarray()) + for rr, lr in zip(remote_nd.rois.items(), local_nd.rois.items(), strict=False): + assert rr == lr + + full_local_read = imread(small_nd2s) + full_remote_read = imread(url, storage_options=storage_options) + np.testing.assert_array_equal(full_remote_read, full_local_read) From 505bffffc0028187cd17cdf02ff4a81bf3bbf915 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 12:25:50 -0500 Subject: [PATCH 04/12] type safety --- .pre-commit-config.yaml | 2 + mkdocs.yml | 1 + src/nd2/_nd2file.py | 34 ++++++++--- src/nd2/_parse/_chunk_decode.py | 66 ++++++++++++--------- src/nd2/_readers/_legacy/legacy_reader.py | 11 ++-- src/nd2/_readers/protocol.py | 70 ++++++++++++----------- src/nd2/_util.py | 59 ++++++++++--------- 7 files changed, 143 insertions(+), 100 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 243d2091..4cb002ca 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -32,3 +32,5 @@ repos: # - types-lxml - ome-types - pydantic + - fsspec + diff --git a/mkdocs.yml b/mkdocs.yml index 07c81e72..c8eadc6e 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -64,6 +64,7 @@ plugins: - https://docs.xarray.dev/en/stable/objects.inv - https://docs.dask.org/en/stable/objects.inv - https://ome-types.readthedocs.io/en/latest/objects.inv + - https://filesystem-spec.readthedocs.io/en/latest/objects.inv options: docstring_section_style: list # or "table" diff --git a/src/nd2/_nd2file.py b/src/nd2/_nd2file.py index 4a72d337..5422d640 100644 --- a/src/nd2/_nd2file.py +++ b/src/nd2/_nd2file.py @@ -14,7 +14,7 @@ from nd2 import _util from ._readers.protocol import ND2Reader -from ._util import AXIS, _is_fsspec_url, is_supported_file +from ._util import AXIS, is_fsspec_url, is_supported_file try: from functools import cached_property @@ -83,8 +83,14 @@ class ND2File: Parameters ---------- - path : Path | str - Filename of an nd2 file. + path : Path | str | ReadSeekBinary + Filename, Path, or URL of an nd2 file. May also be any file-like object that + supports binary read and seek, including those returned by `fsspec` filesystems: + + - `closed(self) -> bool: ...` + - `read(self, size: int | None = ..., /) -> bytes: ...` + - `seek(self, offset: int, whence: int = ..., /) -> int: ...` + - `close(self) -> None: ...` validate_frames : bool Whether to verify (and attempt to fix) frames whose positions have been shifted relative to the predicted offset (i.e. in a corrupted file). @@ -93,6 +99,8 @@ class ND2File: search_window : int When validate_frames is true, this is the search window (in KB) that will be used to try to find the actual chunk position. by default 100 KB + storage_options : dict, optional + Extra kwargs passed to [`fsspec.core.url_to_fs`][] when opening remote URLs. """ def __init__( @@ -106,12 +114,12 @@ def __init__( self._error_radius: int | None = ( search_window * 1000 if validate_frames else None ) - self._storage_options = storage_options + self._storage_options: dict | None = storage_options self._rdr = ND2Reader.create(path, self._error_radius, storage_options) - self._path = self._rdr._path + self._path: str | Path | None = self._rdr._path # For URL inputs the reader may not extract the full URL from the handle; # preserve the original URL string so pickling round-trips correctly. - if isinstance(path, str) and _is_fsspec_url(path): + if is_fsspec_url(path): self._path = path self._lock = threading.RLock() @@ -148,7 +156,7 @@ def version(self) -> tuple[int, ...]: @property def path(self) -> str: """Path of the image.""" - return str(self._path) + return "" if self._path is None else str(self._path) @property def is_legacy(self) -> bool: @@ -202,7 +210,9 @@ def __enter__(self) -> ND2File: def __del__(self) -> None: """Delete file handle on garbage collection.""" # if it came in as an open file handle, it's ok to remain open after deletion - if not getattr(self, "closed", True) and not self._rdr._was_open: + if not hasattr(self, "_rdr"): + return + if not self.closed and not self._rdr._was_open: # this stack inspection is a hack to avoid an unnecessary warning/closure. # when using the to_dask() method, calling dask map_blocks will greedily # pickle/unpickle the object. @@ -224,6 +234,10 @@ def __exit__(self, *_: Any) -> None: def __getstate__(self) -> dict[str, Any]: """Return state for pickling.""" + if self._path is None: + raise TypeError( + "Cannot pickle ND2File created from an unnamed file-like object" + ) state = self.__dict__.copy() del state["_rdr"] del state["_lock"] @@ -236,10 +250,12 @@ def __setstate__(self, d: dict[str, Any]) -> None: _was_closed = d.pop("_closed", False) self.__dict__ = d self._lock = threading.RLock() + if self._path is None: + raise TypeError("Cannot restore ND2File without a file path or URL") self._rdr = ND2Reader.create( self._path, self._error_radius, - getattr(self, "_storage_options", None), + cast("dict | None", self.__dict__.get("_storage_options")), ) if _was_closed: self._rdr.close() diff --git a/src/nd2/_parse/_chunk_decode.py b/src/nd2/_parse/_chunk_decode.py index bc16c2d7..c8d836d1 100644 --- a/src/nd2/_parse/_chunk_decode.py +++ b/src/nd2/_parse/_chunk_decode.py @@ -4,20 +4,21 @@ import mmap import struct -from contextlib import contextmanager, nullcontext -from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO, cast +from contextlib import AbstractContextManager, contextmanager, nullcontext +from os import PathLike as OSPathLike +from typing import TYPE_CHECKING, cast import numpy as np if TYPE_CHECKING: from collections.abc import Iterator - from contextlib import AbstractContextManager from os import PathLike from typing import Final from numpy.typing import DTypeLike + from nd2._util import ReadSeekBinary + StrOrBytesPath = str | bytes | PathLike[str] | PathLike[bytes] StartFileChunk = tuple[int, int, int, bytes, bytes] @@ -68,12 +69,12 @@ # uint64_t offset -def get_version(fh: BinaryIO | StrOrBytesPath) -> tuple[int, int]: +def get_version(fh: ReadSeekBinary | StrOrBytesPath) -> tuple[int, int]: """Get the version of the ND2 file or raise an exception. Parameters ---------- - fh : BinaryIO | str | bytes | Path + fh : ReadSeekBinary | str | bytes | Path The file handle or path to the ND2 file. Returns @@ -87,14 +88,14 @@ def get_version(fh: BinaryIO | StrOrBytesPath) -> tuple[int, int]: If the file is not a valid ND2 file or the header chunk is corrupt. """ if hasattr(fh, "read"): - ctx: AbstractContextManager[BinaryIO] = nullcontext(cast("BinaryIO", fh)) + ctx: AbstractContextManager = nullcontext(cast("ReadSeekBinary", fh)) else: ctx = open(fh, "rb") - with ctx as fh: - fh.seek(0) - fname = str(getattr(fh, "name", "")) - chunk = START_FILE_CHUNK.unpack(fh.read(START_FILE_CHUNK.size)) + with ctx as handle: + handle.seek(0) + fname = str(getattr(handle, "name", "")) + chunk = START_FILE_CHUNK.unpack(handle.read(START_FILE_CHUNK.size)) magic, name_length, data_length, name, data = cast("StartFileChunk", chunk) @@ -112,7 +113,7 @@ def get_version(fh: BinaryIO | StrOrBytesPath) -> tuple[int, int]: return (int(chr(data[3])), int(chr(data[5]))) -def get_chunkmap(fh: BinaryIO, error_radius: int | None = None) -> ChunkMap: +def get_chunkmap(fh: ReadSeekBinary, error_radius: int | None = None) -> ChunkMap: """Read the map of the chunks at the end of an ND2 file. A Chunkmap is mapping of chunk names (bytes) to (offset, size) pairs. @@ -128,7 +129,7 @@ def get_chunkmap(fh: BinaryIO, error_radius: int | None = None) -> ChunkMap: Parameters ---------- - fh : BinaryIO + fh : ReadSeekBinary An open nd2 file. File is assumed to be a valid ND2 file. (use `get_version`) error_radius : int, optional If b"ND2 FILEMAP SIGNATURE NAME 0001!" is not found at expected location and @@ -149,7 +150,10 @@ def get_chunkmap(fh: BinaryIO, error_radius: int | None = None) -> ChunkMap: fh.seek(-40, 2) sig, location = SIG_CHUNKMAP_LOC.unpack(fh.read(SIG_CHUNKMAP_LOC.size)) if sig != ND2_CHUNKMAP_SIGNATURE: # pragma: no cover - raise ValueError(f"Invalid ChunkMap signature {sig!r} in file {fh.name!r}") + raise ValueError( + f"Invalid ChunkMap signature {sig!r} in file " + f"{getattr(fh, 'name', '')!r}" + ) # get all of the data in the chunkmap chunkmap_data = _robustly_read_named_chunk( @@ -182,7 +186,7 @@ def get_chunkmap(fh: BinaryIO, error_radius: int | None = None) -> ChunkMap: def read_nd2_chunk( - fh: BinaryIO, start_position: int, expect_name: bytes | None = None + fh: ReadSeekBinary, start_position: int, expect_name: bytes | None = None ) -> bytes: """Read a single chunk in an ND2 file at `start_position`. @@ -197,7 +201,7 @@ def read_nd2_chunk( Parameters ---------- - fh : BinaryIO + fh : ReadSeekBinary An open nd2 file. File is assumed to be a valid ND2 file. (use `get_version`) start_position : int The position in the file to start reading the chunk. @@ -235,7 +239,7 @@ def read_nd2_chunk( def _robustly_read_named_chunk( - fh: BinaryIO, + fh: ReadSeekBinary, start_position: int, expect_name: bytes = ND2_FILEMAP_SIGNATURE, search_radius: int | None = None, @@ -248,7 +252,7 @@ def _robustly_read_named_chunk( Parameters ---------- - fh : BinaryIO + fh : ReadSeekBinary An open nd2 file. File is assumed to be a valid ND2 file. start_position : int The position in the file to start reading the chunk. @@ -261,8 +265,9 @@ def _robustly_read_named_chunk( try: return read_nd2_chunk(fh, start_position, expect_name=expect_name) except ValueError as e: + file_name = getattr(fh, "name", "") err_msg = ( - f"File {fh.name!r} appears to be corrupt. Expected " + f"File {file_name!r} appears to be corrupt. Expected " f"{expect_name!r} at position " f"{start_position} but did not find it." ) @@ -282,7 +287,7 @@ def _robustly_read_named_chunk( raise ValueError(err_msg) from e -def iter_chunks(handle: BinaryIO) -> Iterator[tuple[str, int, int]]: +def iter_chunks(handle: ReadSeekBinary) -> Iterator[tuple[str, int, int]]: file_size = handle.seek(0, 2) handle.seek(0) pos = 0 @@ -304,7 +309,7 @@ def iter_chunks(handle: BinaryIO) -> Iterator[tuple[str, int, int]]: def rescue_nd2( - handle: BinaryIO | str, + handle: ReadSeekBinary | StrOrBytesPath, frame_shape: tuple[int, ...] = (), dtype: DTypeLike = "uint16", max_iters: int | None = None, @@ -321,7 +326,7 @@ def rescue_nd2( Parameters ---------- - handle : BinaryIO | str + handle : ReadSeekBinary | str | bytes | PathLike Filepath string, or binary file handle (For example `handle = open('some.nd2', 'rb')`) frame_shape : Tuple[int, ...], optional @@ -356,7 +361,10 @@ def rescue_nd2( """ dtype = np.dtype(dtype) with ensure_handle(handle) as _fh: - mm = mmap.mmap(_fh.fileno(), 0, access=mmap.ACCESS_READ) + fileno = getattr(_fh, "fileno", None) + if not callable(fileno): + raise TypeError("rescue_nd2 requires a file handle with fileno()") + mm = mmap.mmap(fileno(), 0, access=mmap.ACCESS_READ) offset = 0 iters = 0 @@ -407,11 +415,15 @@ def rescue_nd2( @contextmanager -def ensure_handle(obj: str | BinaryIO) -> Iterator[BinaryIO]: - fh = open(obj, "rb") if isinstance(obj, (str, bytes, Path)) else obj +def ensure_handle(obj: StrOrBytesPath | ReadSeekBinary) -> Iterator[ReadSeekBinary]: + if isinstance(obj, (str, bytes, OSPathLike)): + opened_here = True + fh = cast("ReadSeekBinary", open(obj, "rb")) + else: + opened_here = False + fh = obj try: yield fh finally: - # close it if we were the one to open it - if not hasattr(obj, "fileno"): + if opened_here: fh.close() diff --git a/src/nd2/_readers/_legacy/legacy_reader.py b/src/nd2/_readers/_legacy/legacy_reader.py index f47a19ff..d60d1e89 100644 --- a/src/nd2/_readers/_legacy/legacy_reader.py +++ b/src/nd2/_readers/_legacy/legacy_reader.py @@ -23,9 +23,9 @@ if TYPE_CHECKING: from collections.abc import Mapping - from typing import Any, BinaryIO, TypedDict + from typing import Any, TypedDict - from nd2._util import FileOrBinaryIO + from nd2._util import FileOrBinaryIO, ReadSeekBinary class RawExperimentLoop(TypedDict, total=False): Type: int @@ -365,9 +365,10 @@ def read_frame(self, index: int) -> np.ndarray: try: from imagecodecs import jpeg2k_decode except ModuleNotFoundError as e: # pragma: no cover + file_name = getattr(self._fh, "name", "") raise ModuleNotFoundError( f"{e}\n" - f"Reading legacy format nd2 {self._fh.name!r} requires imagecodecs.\n" + f"Reading legacy format nd2 {file_name!r} requires imagecodecs.\n" "Please install with `pip install imagecodecs`." ) from e @@ -415,7 +416,7 @@ def header(self) -> dict: pos = self.chunkmap[b"jp2h"][0] except (KeyError, IndexError) as e: # pragma: no cover raise KeyError("No valid jp2h header found in file") from e - fh = cast("BinaryIO", self._fh) + fh = cast("ReadSeekBinary", self._fh) fh.seek(pos + I4s.size + 4) # 4 bytes for "label" if fh.read(4) != b"ihdr": raise KeyError("No valid ihdr header found in jp2h header") @@ -438,7 +439,7 @@ def events(self, orient: str, null_value: Any) -> list | Mapping: return [] if orient == "records" else {} -def legacy_nd2_chunkmap(fh: BinaryIO) -> dict[bytes, list[int]]: +def legacy_nd2_chunkmap(fh: ReadSeekBinary) -> dict[bytes, list[int]]: fh.seek(-40, 2) sig, map_start = struct.unpack("<32sQ", fh.read()) if sig != b"LABORATORY IMAGING ND BOX MAP 00": # pragma: no cover diff --git a/src/nd2/_readers/protocol.py b/src/nd2/_readers/protocol.py index a1ccc534..d3b51ec2 100644 --- a/src/nd2/_readers/protocol.py +++ b/src/nd2/_readers/protocol.py @@ -3,22 +3,21 @@ import abc import mmap import warnings -from contextlib import nullcontext +from contextlib import nullcontext, suppress from pathlib import Path -from typing import TYPE_CHECKING, BinaryIO, cast +from typing import TYPE_CHECKING, Any, cast from nd2._parse._chunk_decode import get_version -from nd2._util import _is_fsspec_url, _open_fsspec_url +from nd2._util import is_fsspec_url, is_read_seek_binary, open_fsspec_url if TYPE_CHECKING: from collections.abc import Mapping, Sequence - from contextlib import AbstractContextManager - from typing import Any, Literal + from typing import Literal import numpy as np from nd2._binary import BinaryLayers - from nd2._util import FileOrBinaryIO + from nd2._util import FileOrBinaryIO, ReadSeekBinary from nd2.jobs.types import JobsDict from nd2.structures import ( ROI, @@ -59,18 +58,19 @@ def create( """ from nd2._readers import LegacyReader, ModernReader - if hasattr(path, "read"): - path = cast("BinaryIO", path) - if "b" not in path.mode: + is_url = is_fsspec_url(path) + if is_file_handle := is_read_seek_binary(path): + mode = getattr(path, "mode", "b") + if isinstance(mode, str) and "b" not in mode: raise ValueError( "File handles passed to ND2File must be in binary mode" ) - ctx: AbstractContextManager[BinaryIO] = nullcontext(path) - elif _is_fsspec_url(path): - ctx = nullcontext(_open_fsspec_url(str(path), storage_options)) + ctx = cast("Any", nullcontext(path)) + elif is_url: + ctx = cast("Any", nullcontext(open_fsspec_url(str(path), storage_options))) else: - path = Path(path).expanduser().absolute() - ctx = open(path, "rb") + path = Path(cast("str | Path", path)).expanduser().absolute() + ctx = cast("Any", open(path, "rb")) with ctx as fh: fname = getattr(fh, "name", "") @@ -81,33 +81,31 @@ def create( if magic_num == subcls.HEADER_MAGIC: # For URL/file-like cases pass the open handle; for local paths # pass the Path so the reader can reopen it as needed. - effective_path = ( - fh if _is_fsspec_url(path) or hasattr(path, "read") else path - ) + effective_path = fh if (is_url or is_file_handle) else path return subcls(effective_path, error_radius=error_radius) raise OSError( f"file {fname!r} not recognized as ND2. First 4 bytes: {magic_num!r}" ) - def __init__(self, path: FileOrBinaryIO, error_radius: int | None = None) -> None: + def __init__(self, obj: FileOrBinaryIO, error_radius: int | None = None) -> None: self._chunkmap: dict | None = None self._version: tuple[int, int] | None = None self._mmap: mmap.mmap | None = None - if hasattr(path, "read"): - self._fh: BinaryIO | None = cast("BinaryIO", path) - self._was_open = not self._fh.closed - name = getattr(self._fh, "full_name", None) or getattr( - self._fh, "name", None - ) - self._path: str | Path | None = name if isinstance(name, str) else None - try: - self._mmap = mmap.mmap(self._fh.fileno(), 0, access=mmap.ACCESS_READ) - except Exception: - pass # remote/non-fileno file-likes: mmap not available + self._fh: ReadSeekBinary | None + self._path: str | Path | None + if is_read_seek_binary(obj): + self._fh = obj + self._was_open = not obj.closed + name = getattr(obj, "full_name", None) or getattr(obj, "name", None) + self._path = name if isinstance(name, str) else None + with suppress(Exception): + # remote/non-fileno file-likes: mmap not available + if (fileno := getattr(self._fh, "fileno", None)) and callable(fileno): + self._mmap = mmap.mmap(fileno(), 0, access=mmap.ACCESS_READ) else: self._was_open = False - self._path = Path(path) + self._path = Path(cast("str | Path", obj)) self._fh = None self._error_radius: int | None = error_radius self.open() @@ -124,7 +122,8 @@ def open(self) -> None: "Cannot reopen a remote/file-like ND2 source after closing" ) self._fh = open(self._path, "rb") - self._mmap = mmap.mmap(self._fh.fileno(), 0, access=mmap.ACCESS_READ) + fh = cast("Any", self._fh) + self._mmap = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ) def close(self) -> None: """Close the file handle.""" @@ -151,7 +150,14 @@ def __exit__(self, *_: Any) -> None: def version(self) -> tuple[int, int]: """Return the file format version as a tuple of ints.""" if self._version is None: - self._version = get_version(self._fh or self._path) + if self._fh is not None: + self._version = get_version(self._fh) + elif self._path is not None: + self._version = get_version(self._path) + else: + raise RuntimeError( + "Cannot determine version without an open file handle" + ) return self._version def rois(self) -> list[ROI]: diff --git a/src/nd2/_util.py b/src/nd2/_util.py index 70c6808f..b15a6b7a 100644 --- a/src/nd2/_util.py +++ b/src/nd2/_util.py @@ -5,32 +5,37 @@ from contextlib import suppress from datetime import datetime, timezone from itertools import product -from typing import TYPE_CHECKING, BinaryIO, NamedTuple, cast +from typing import TYPE_CHECKING, NamedTuple, cast if TYPE_CHECKING: from collections.abc import Mapping, Sequence from os import PathLike - from typing import Any, Callable, ClassVar, Final, Union + from typing import Any, ClassVar, Final, Protocol, TypeAlias, Union + + from typing_extensions import TypeGuard from nd2.structures import ExpLoop - StrOrPath = Union[str, PathLike] - FileOrBinaryIO = Union[StrOrPath, BinaryIO] + class ReadSeekBinary(Protocol): + @property + def closed(self) -> bool: ... + def read(self, size: int | None = -1, /) -> bytes: ... + def seek(self, offset: int, whence: int = 0, /) -> int: ... + def close(self) -> None: ... + + StrOrPath: TypeAlias = Union[str, PathLike] + FileOrBinaryIO: TypeAlias = Union[StrOrPath, ReadSeekBinary] - ListOfDicts = list[dict[str, Any]] - DictOfLists = Mapping[str, Sequence[Any]] - DictOfDicts = Mapping[str, dict[int, Any]] + ListOfDicts: TypeAlias = list[dict[str, Any]] + DictOfLists: TypeAlias = Mapping[str, Sequence[Any]] + DictOfDicts: TypeAlias = Mapping[str, dict[int, Any]] NEW_HEADER_MAGIC = b"\xda\xce\xbe\n" OLD_HEADER_MAGIC = b"\x00\x00\x00\x0c" VERSION = re.compile(r"^ND2 FILE SIGNATURE CHUNK NAME01!Ver([\d\.]+)$") -def _open_binary(path: StrOrPath) -> BinaryIO: - return open(path, "rb") - - -def _is_fsspec_url(path: object) -> bool: +def is_fsspec_url(path: Any) -> TypeGuard[str]: """True if `path` is a string with a remote URL scheme (e.g. 's3://').""" if not isinstance(path, str): return False @@ -39,10 +44,7 @@ def _is_fsspec_url(path: object) -> bool: return idx > 1 -def _open_fsspec_url( - url: str, - storage_options: dict | None = None, -) -> BinaryIO: +def open_fsspec_url(url: str, storage_options: dict | None = None) -> ReadSeekBinary: """Open a remote nd2 URL with a pre-warmed metadata cache. For smaller files, pre-fetches the full object and opens with @@ -67,7 +69,7 @@ def _open_fsspec_url( raw = fs.cat_file(fpath) known: dict[tuple[int, int], bytes] = {(0, size): raw} return cast( - "BinaryIO", + "ReadSeekBinary", fs.open( fpath, "rb", @@ -76,33 +78,36 @@ def _open_fsspec_url( ), ) - return cast("BinaryIO", fs.open(fpath, "rb")) + return cast("ReadSeekBinary", fs.open(fpath, "rb")) + + +def is_read_seek_binary(obj: object) -> TypeGuard[ReadSeekBinary]: + return ( + hasattr(obj, "read") + and hasattr(obj, "seek") + and hasattr(obj, "close") + and hasattr(obj, "closed") + ) -def is_supported_file( - path: FileOrBinaryIO, - open_: Callable[[StrOrPath], BinaryIO] = _open_binary, -) -> bool: +def is_supported_file(path: FileOrBinaryIO) -> bool: """Return `True` if `path` can be opened as an nd2 file. Parameters ---------- path : Union[str, bytes, PathLike] A path to query - open_ : Callable[[StrOrBytesPath, str], BinaryIO] - Filesystem opener, by default `builtins.open` Returns ------- bool Whether the can be opened. """ - if hasattr(path, "read"): - path = cast("BinaryIO", path) + if is_read_seek_binary(path): path.seek(0) magic = path.read(4) else: - with open_(path) as fh: + with open(cast("StrOrPath", path), "rb") as fh: magic = fh.read(4) return magic in (NEW_HEADER_MAGIC, OLD_HEADER_MAGIC) From 09cb184141dded2e34031fe288174db88b6f1e8f Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:23:37 -0500 Subject: [PATCH 05/12] feat: enhance S3 URL handling and improve fsspec integration in ND2File --- src/nd2/_nd2file.py | 12 ++---- src/nd2/_readers/protocol.py | 10 +++-- src/nd2/_util.py | 59 ++++++++++++++-------------- tests/test_s3.py | 75 +++++++++++++++++++++++++++++------- 4 files changed, 101 insertions(+), 55 deletions(-) diff --git a/src/nd2/_nd2file.py b/src/nd2/_nd2file.py index 5422d640..4d1b4eea 100644 --- a/src/nd2/_nd2file.py +++ b/src/nd2/_nd2file.py @@ -14,7 +14,7 @@ from nd2 import _util from ._readers.protocol import ND2Reader -from ._util import AXIS, is_fsspec_url, is_supported_file +from ._util import AXIS, is_supported_file try: from functools import cached_property @@ -109,18 +109,14 @@ def __init__( *, validate_frames: bool = False, search_window: int = 100, - storage_options: dict | None = None, + storage_options: dict[str, Any] | None = None, ) -> None: self._error_radius: int | None = ( search_window * 1000 if validate_frames else None ) - self._storage_options: dict | None = storage_options + self._storage_options = storage_options self._rdr = ND2Reader.create(path, self._error_radius, storage_options) self._path: str | Path | None = self._rdr._path - # For URL inputs the reader may not extract the full URL from the handle; - # preserve the original URL string so pickling round-trips correctly. - if is_fsspec_url(path): - self._path = path self._lock = threading.RLock() @staticmethod @@ -250,7 +246,7 @@ def __setstate__(self, d: dict[str, Any]) -> None: _was_closed = d.pop("_closed", False) self.__dict__ = d self._lock = threading.RLock() - if self._path is None: + if self._path is None: # pragma: no cover (unreachable with current public API) raise TypeError("Cannot restore ND2File without a file path or URL") self._rdr = ND2Reader.create( self._path, diff --git a/src/nd2/_readers/protocol.py b/src/nd2/_readers/protocol.py index d3b51ec2..92d567c5 100644 --- a/src/nd2/_readers/protocol.py +++ b/src/nd2/_readers/protocol.py @@ -3,7 +3,7 @@ import abc import mmap import warnings -from contextlib import nullcontext, suppress +from contextlib import AbstractContextManager, nullcontext, suppress from pathlib import Path from typing import TYPE_CHECKING, Any, cast @@ -59,18 +59,20 @@ def create( from nd2._readers import LegacyReader, ModernReader is_url = is_fsspec_url(path) + ctx: AbstractContextManager if is_file_handle := is_read_seek_binary(path): mode = getattr(path, "mode", "b") if isinstance(mode, str) and "b" not in mode: raise ValueError( "File handles passed to ND2File must be in binary mode" ) - ctx = cast("Any", nullcontext(path)) + ctx = nullcontext(path) elif is_url: - ctx = cast("Any", nullcontext(open_fsspec_url(str(path), storage_options))) + fh = open_fsspec_url(str(path), storage_options=storage_options) + ctx = nullcontext(fh) else: path = Path(cast("str | Path", path)).expanduser().absolute() - ctx = cast("Any", open(path, "rb")) + ctx = open(path, "rb") with ctx as fh: fname = getattr(fh, "name", "") diff --git a/src/nd2/_util.py b/src/nd2/_util.py index b15a6b7a..4df784f0 100644 --- a/src/nd2/_util.py +++ b/src/nd2/_util.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io import math import re from contextlib import suppress @@ -39,17 +40,28 @@ def is_fsspec_url(path: Any) -> TypeGuard[str]: """True if `path` is a string with a remote URL scheme (e.g. 's3://').""" if not isinstance(path, str): return False - idx = path.find("://") - # idx > 1 excludes Windows drive letters like C:/ - return idx > 1 - - -def open_fsspec_url(url: str, storage_options: dict | None = None) -> ReadSeekBinary: - """Open a remote nd2 URL with a pre-warmed metadata cache. - - For smaller files, pre-fetches the full object and opens with - cache_type='parts'. For larger files, opens directly via filesystem - defaults to avoid non-contiguous cache composition artifacts. + # RFC 3986 scheme syntax + return bool(re.match(r"^[a-zA-Z][a-zA-Z0-9+\-.]+://", path)) + + +def open_fsspec_url( + url: str, + *, + small_file_limit: int = 32 * 1024 * 1024, # 32 MB + block_size: int = 512 * 1024, # 512 KB + storage_options: dict | None = None, +) -> ReadSeekBinary: + """Open a remote nd2 URL. + + Files up to `small_file_limit` are fetched eagerly into a BytesIO object. + ND2 access is heavily random (magic bytes at start, chunkmap at end, + scattered frame offsets), so keeping small files fully in memory avoids + repeated range requests. + + For larger files a 512 KB `block_size` is used rather than fsspec's 5 MB + default. Benchmarking shows this cuts open + chunkmap latency ~2.5x: + each chunkmap seek fetches 512 KB instead of 5 MB, and scattered metadata + chunks typically fit within a single block at this size. """ try: import fsspec @@ -60,25 +72,12 @@ def open_fsspec_url(url: str, storage_options: dict | None = None) -> ReadSeekBi sopts = storage_options or {} fs, fpath = fsspec.url_to_fs(url, **sopts) - size: int = fs.info(fpath)["size"] - - START_BYTES = 512 * 1024 # 512 KB — covers magic + ImageMetadataSeqLV|0! - END_BYTES = 5 * 1024 * 1024 # 5 MB — covers chunkmap + all end metadata - - if size <= START_BYTES + END_BYTES: - raw = fs.cat_file(fpath) - known: dict[tuple[int, int], bytes] = {(0, size): raw} - return cast( - "ReadSeekBinary", - fs.open( - fpath, - "rb", - cache_type="parts", - cache_options={"data": known, "strict": False}, - ), - ) - - return cast("ReadSeekBinary", fs.open(fpath, "rb")) + size = fs.info(fpath).get("size") + + if size is not None and size <= small_file_limit: + return io.BytesIO(fs.cat_file(fpath)) + + return cast("ReadSeekBinary", fs.open(fpath, "rb", block_size=block_size)) def is_read_seek_binary(obj: object) -> TypeGuard[ReadSeekBinary]: diff --git a/tests/test_s3.py b/tests/test_s3.py index 3850fe02..5ab57b0a 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -1,12 +1,17 @@ -from collections.abc import Iterable -from pathlib import Path +from __future__ import annotations + from typing import TYPE_CHECKING -from uuid import uuid4 import numpy as np import pytest from nd2 import ND2File, imread +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + + from mypy_boto3_s3.service_resource import Bucket + @pytest.fixture(scope="session") def s3_endpoint() -> Iterable[str]: @@ -23,25 +28,36 @@ def s3_endpoint() -> Iterable[str]: server.stop() -@pytest.fixture() -def s3_nd2_url(small_nd2s: Path, s3_endpoint: str) -> Iterable[tuple[str, str]]: - """Yield an S3 URL pointing to `single_nd2`, along with the endpoint URL.""" +@pytest.fixture(scope="session") +def s3_bucket(s3_endpoint: str) -> Bucket: if TYPE_CHECKING: import boto3 else: boto3 = pytest.importorskip("boto3") + s3 = boto3.resource("s3", endpoint_url=s3_endpoint, region_name="us-east-1") + return s3.create_bucket(Bucket="nd2-test") - bucket = f"nd2-test-{uuid4().hex[:12]}" - client = boto3.client("s3", endpoint_url=s3_endpoint, region_name="us-east-1") - client.create_bucket(Bucket=bucket) - client.upload_file(str(small_nd2s), bucket, small_nd2s.name) - yield f"s3://{bucket}/{small_nd2s.name}", s3_endpoint + +@pytest.fixture() +def any_s3_nd2_url(small_nd2s: Path, s3_bucket: Bucket) -> Iterable[tuple[str, str]]: + """Yield an S3 URL pointing to any nd2, along with the endpoint URL.""" + s3_bucket.upload_file(str(small_nd2s), small_nd2s.name) + endpoint = s3_bucket.meta.client.meta.endpoint_url + yield f"s3://{s3_bucket.name}/{small_nd2s.name}", endpoint + + +@pytest.fixture() +def single_s3_nd2_url(single_nd2: Path, s3_bucket: Bucket) -> Iterable[tuple[str, str]]: + """Yield an S3 URL pointing to `single_nd2`, along with the endpoint URL.""" + s3_bucket.upload_file(str(single_nd2), single_nd2.name) + endpoint = s3_bucket.meta.client.meta.endpoint_url + yield f"s3://{s3_bucket.name}/{single_nd2.name}", endpoint def test_nd2file_reads_from_s3_url( - s3_nd2_url: tuple[str, str], small_nd2s: Path + any_s3_nd2_url: tuple[str, str], small_nd2s: Path ) -> None: - url, endpoint = s3_nd2_url + url, endpoint = any_s3_nd2_url storage_options = { "client_kwargs": {"endpoint_url": endpoint}, # necessary for moto's S3 implementation @@ -73,3 +89,36 @@ def test_nd2file_reads_from_s3_url( full_local_read = imread(small_nd2s) full_remote_read = imread(url, storage_options=storage_options) np.testing.assert_array_equal(full_remote_read, full_local_read) + + +def test_nd2file_reads_from_fsspec_obj( + single_s3_nd2_url: tuple[str, str], single_nd2: Path +) -> None: + """Test that the user can construct their own fsspec file-like for ND2File.""" + if TYPE_CHECKING: + import fsspec + else: + fsspec = pytest.importorskip("fsspec") + + url, endpoint = single_s3_nd2_url + storage_options = { + "client_kwargs": {"endpoint_url": endpoint}, + "config_kwargs": {"response_checksum_validation": "when_required"}, + } + fs = fsspec.filesystem("s3", **storage_options) + with fs.open(url, "rb") as fs_fh: + remote_nd = ND2File(fs_fh, storage_options=storage_options) + local_nd = ND2File(single_nd2) + with remote_nd, local_nd: + assert remote_nd.path == url + assert remote_nd.shape == local_nd.shape + assert remote_nd.metadata == local_nd.metadata + assert remote_nd.attributes == local_nd.attributes + assert remote_nd.text_info == local_nd.text_info + assert remote_nd.experiment == local_nd.experiment + assert remote_nd.frame_metadata(0) == local_nd.frame_metadata(0) + assert remote_nd.events() == local_nd.events() + assert remote_nd.ome_metadata() == local_nd.ome_metadata() + np.testing.assert_array_equal( + remote_nd.read_frame(0), local_nd.read_frame(0) + ) From 2f6b87bef94fde846f769ad10b84d10422712e8c Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:32:18 -0500 Subject: [PATCH 06/12] add pins, fix test --- pyproject.toml | 2 +- src/nd2/_nd2file.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c55fef0f..91ed150f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ ome-zarr-tensorstore = [ "pydantic>=2.11.0", "yaozarrs[write-tensorstore]>=0.2.0; python_version >= '3.10'", ] -remote = ["fsspec", "s3fs"] +remote = ["fsspec>=2024.2.0", "s3fs>=2024.2.0"] [dependency-groups] test = [ diff --git a/src/nd2/_nd2file.py b/src/nd2/_nd2file.py index 4d1b4eea..38fa09a9 100644 --- a/src/nd2/_nd2file.py +++ b/src/nd2/_nd2file.py @@ -14,7 +14,7 @@ from nd2 import _util from ._readers.protocol import ND2Reader -from ._util import AXIS, is_supported_file +from ._util import AXIS, is_fsspec_url, is_supported_file try: from functools import cached_property @@ -117,6 +117,10 @@ def __init__( self._storage_options = storage_options self._rdr = ND2Reader.create(path, self._error_radius, storage_options) self._path: str | Path | None = self._rdr._path + # For URL inputs the reader may not extract the full URL from the handle; + # preserve the original URL string so pickling round-trips correctly. + if is_fsspec_url(path): + self._path = path self._lock = threading.RLock() @staticmethod From 4ff2957ae7ca1efae9cc72241c032a3b2ed1e3e3 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:33:33 -0500 Subject: [PATCH 07/12] add note to docstrings --- docs/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/index.md b/docs/index.md index 379a1937..10b3d6f5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -44,6 +44,16 @@ install with support for these files use the `legacy` extra: pip install nd2[legacy] ``` +### With remote nd2 file support + +If you would like to be able to read nd2 files from remote sources (e.g. S3, HTTP), +then fsspec and/or s3fs are required. +To install with support for remote files use the `remote` extra: + +```sh +pip install nd2[remote] +``` + ### Faster XML parsing Much of the metadata in the file stored as XML. If found in the environment, From 02874bee713c2655c39784ed2773fc7321e7b3bb Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:40:41 -0500 Subject: [PATCH 08/12] try fix CI tests --- src/nd2/_nd2file.py | 4 +--- tests/test_s3.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/nd2/_nd2file.py b/src/nd2/_nd2file.py index 38fa09a9..99fb6da8 100644 --- a/src/nd2/_nd2file.py +++ b/src/nd2/_nd2file.py @@ -250,10 +250,8 @@ def __setstate__(self, d: dict[str, Any]) -> None: _was_closed = d.pop("_closed", False) self.__dict__ = d self._lock = threading.RLock() - if self._path is None: # pragma: no cover (unreachable with current public API) - raise TypeError("Cannot restore ND2File without a file path or URL") self._rdr = ND2Reader.create( - self._path, + self._path, # type: ignore self._error_radius, cast("dict | None", self.__dict__.get("_storage_options")), ) diff --git a/tests/test_s3.py b/tests/test_s3.py index 5ab57b0a..d64607ee 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from typing import TYPE_CHECKING import numpy as np @@ -19,6 +20,16 @@ def s3_endpoint() -> Iterable[str]: from moto.server import ThreadedMotoServer else: ThreadedMotoServer = pytest.importorskip("moto.server").ThreadedMotoServer + + # ThreadedMotoServer is a real HTTP server and does not patch boto3's + # credential chain. Provide dummy credentials so boto3 and s3fs don't + # raise NoCredentialsError in environments without ~/.aws/credentials. + os.environ.setdefault("AWS_ACCESS_KEY_ID", "testing") + os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "testing") + os.environ.setdefault("AWS_SECURITY_TOKEN", "testing") + os.environ.setdefault("AWS_SESSION_TOKEN", "testing") + os.environ.setdefault("AWS_DEFAULT_REGION", "us-east-1") + server = ThreadedMotoServer(port=0) server.start() _, port = server.get_host_and_port() From 779b3e3cbb216288750fe8f7b3dcff3062fddb00 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:45:37 -0500 Subject: [PATCH 09/12] fix file close --- src/nd2/_readers/protocol.py | 11 ++++++++++- tests/test_reader.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/nd2/_readers/protocol.py b/src/nd2/_readers/protocol.py index 92d567c5..81025778 100644 --- a/src/nd2/_readers/protocol.py +++ b/src/nd2/_readers/protocol.py @@ -59,6 +59,7 @@ def create( from nd2._readers import LegacyReader, ModernReader is_url = is_fsspec_url(path) + opened_here = False ctx: AbstractContextManager if is_file_handle := is_read_seek_binary(path): mode = getattr(path, "mode", "b") @@ -69,6 +70,7 @@ def create( ctx = nullcontext(path) elif is_url: fh = open_fsspec_url(str(path), storage_options=storage_options) + opened_here = True ctx = nullcontext(fh) else: path = Path(cast("str | Path", path)).expanduser().absolute() @@ -84,7 +86,14 @@ def create( # For URL/file-like cases pass the open handle; for local paths # pass the Path so the reader can reopen it as needed. effective_path = fh if (is_url or is_file_handle) else path - return subcls(effective_path, error_radius=error_radius) + try: + return subcls(effective_path, error_radius=error_radius) + except Exception: + if opened_here: + fh.close() + raise + if opened_here: + fh.close() raise OSError( f"file {fname!r} not recognized as ND2. First 4 bytes: {magic_num!r}" ) diff --git a/tests/test_reader.py b/tests/test_reader.py index f445fde6..dc8f2ebf 100644 --- a/tests/test_reader.py +++ b/tests/test_reader.py @@ -1,3 +1,4 @@ +import io import json import pickle import sys @@ -8,6 +9,7 @@ import pytest from nd2 import ND2File, imread from nd2._parse._chunk_decode import get_version +from nd2._readers import protocol from nd2._util import AXIS, is_supported_file from resource_backed_dask_array import ResourceBackedDaskArray @@ -287,3 +289,18 @@ def test_file_handles(single_nd2: Path) -> None: assert isinstance(f.asarray(), np.ndarray) assert fh.closed assert f.closed + + +def test_url_handle_closed_on_reader_create_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class Handle(io.BytesIO): + pass + + handle = Handle(b"NOT!") + monkeypatch.setattr(protocol, "open_fsspec_url", lambda *args, **kwargs: handle) + + with pytest.raises(OSError, match="not recognized as ND2"): + protocol.ND2Reader.create("s3://bucket/not-nd2") + + assert handle.closed From 7af8adc4bab678bd2428a8e0c37a1ab78761dcf1 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 13:46:26 -0500 Subject: [PATCH 10/12] fix 3.9 --- tests/test_s3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_s3.py b/tests/test_s3.py index d64607ee..3b01aaed 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -94,7 +94,7 @@ def test_nd2file_reads_from_s3_url( remote_nd.binary_data, local_nd.binary_data, strict=False ): np.testing.assert_array_equal(rb.asarray(), lb.asarray()) - for rr, lr in zip(remote_nd.rois.items(), local_nd.rois.items(), strict=False): + for rr, lr in zip(remote_nd.rois.items(), local_nd.rois.items()): assert rr == lr full_local_read = imread(small_nd2s) From 6332361b567eb02feb2deaeb2b73ebb457a30422 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 14:08:55 -0500 Subject: [PATCH 11/12] another 3.9 fix --- tests/test_s3.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/test_s3.py b/tests/test_s3.py index 3b01aaed..14d4b9f7 100644 --- a/tests/test_s3.py +++ b/tests/test_s3.py @@ -90,9 +90,7 @@ def test_nd2file_reads_from_s3_url( np.testing.assert_array_equal(remote_nd.read_frame(0), local_nd.read_frame(0)) if local_nd.binary_data is not None: assert remote_nd.binary_data is not None - for rb, lb in zip( - remote_nd.binary_data, local_nd.binary_data, strict=False - ): + for rb, lb in zip(remote_nd.binary_data, local_nd.binary_data): np.testing.assert_array_equal(rb.asarray(), lb.asarray()) for rr, lr in zip(remote_nd.rois.items(), local_nd.rois.items()): assert rr == lr From 1dd2e468dffba37f8a092fb7a1684dade3384843 Mon Sep 17 00:00:00 2001 From: Talley Lambert Date: Thu, 19 Feb 2026 14:12:23 -0500 Subject: [PATCH 12/12] ignore warning --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 91ed150f..4352afbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -159,6 +159,7 @@ filterwarnings = [ "ignore:::xarray", "ignore:Accessing the 'model_fields' attribute::", "ignore:Failing to pass a value:DeprecationWarning", # xsdata + "ignore:Boto3 will no longer support Python 3.9" ] # https://mypy.readthedocs.io/en/stable/config_file.html