From ba53d8bcc25d537f616f9e2bd6c3db2bfb23edbd Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 12:55:30 +0000 Subject: [PATCH 1/2] fix(ingest): contain data-root URIs after symlink resolution A data-root-relative ingest URI was checked lexically and then re-joined to the data root and resolved later, following symlinks. A directory symlink plus parent segments, or an in-root file symlink, could make the worker open and republish a recording outside the data root. Resolve each local reference once against the real data root, refuse it when the real path leaves the root, and hand that real path to processing, batch sizing, and the media plan filter instead of the original spelling. --- docs/RUNTIME.md | 6 ++++ src/hflow/app.py | 24 +++++++++++--- src/hflow/runtime/_templates.py | 10 +++++- src/hflow/stage_execution.py | 57 ++++++++++++++++++++++----------- src/hflow/stage_planning.py | 21 ++++++++---- src/hflow/uri.py | 20 +++++++++++- 6 files changed, 106 insertions(+), 32 deletions(-) diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 7244b666c..4d57feab0 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -195,6 +195,12 @@ anything is triggered): hflow ingest episodes-in/run_0001.mcap episodes-in/run_0002.mcap ``` +On a local data root, a URI must also land inside the data root after +symlinks are followed: a file or directory symlink that leads outside it is +refused when the episode is processed and counted as that episode's error. To +keep recordings on another disk, mount that disk inside the data root or point +the data root itself at it. + Three flags select what runs and how: - **`--profile`** picks the run profile: which stage sub-DAGs the master diff --git a/src/hflow/app.py b/src/hflow/app.py index 25b7b3828..387f010b1 100644 --- a/src/hflow/app.py +++ b/src/hflow/app.py @@ -101,6 +101,7 @@ stamps_from_provenance, write_canonical_episode, ) +from hflow.uri import resolve_local_data_root_path from hflow.workspace import RUNTIME_BUNDLE_DIRECTORY_NAME, Workspace logger = logging.getLogger(__name__) @@ -122,13 +123,21 @@ class SourceNotFound(FileNotFoundError): def _local_source_candidate_paths( source_reference: Path | str, root_path: Path -) -> tuple[Path, Path]: - """The cwd and data-root interpretations of one local reference.""" +) -> tuple[Path, Path | None]: + """The cwd and data-root interpretations of one local reference. + + The data-root interpretation is ``None`` when resolving it leaves the + real data root: a data-root key never names a file outside the root. + """ source_path = Path(source_reference) if source_path.is_absolute(): resolved_path = source_path.resolve() return resolved_path, resolved_path - return source_path.resolve(), (root_path / source_path).resolve() + try: + root_candidate: Path | None = resolve_local_data_root_path(root_path, source_path) + except ValueError: + root_candidate = None + return source_path.resolve(), root_candidate def _resolve_local_source_reference(source_reference: Path | str, root_path: Path) -> Path: @@ -143,6 +152,13 @@ def _resolve_local_source_reference(source_reference: Path | str, root_path: Pat if cwd_candidate == root_candidate: return cwd_candidate cwd_candidate_exists = cwd_candidate.is_file() + if root_candidate is None: + if cwd_candidate_exists: + return cwd_candidate + raise ValueError( + f"relative source reference {str(source_reference)!r} resolves outside " + f"the data root {root_path.resolve()}" + ) root_candidate_exists = root_candidate.is_file() if cwd_candidate_exists and root_candidate_exists and cwd_candidate != root_candidate: @@ -1679,7 +1695,7 @@ def _resolve_source_identity( cwd_candidate, root_candidate = _local_source_candidate_paths( episode, self.storage_root.path ) - if cwd_candidate.is_file() or root_candidate.is_file(): + if root_candidate is None or cwd_candidate.is_file() or root_candidate.is_file(): return resolved_identity root_identity = _local_source_identity(root_candidate, self.storage_root.path) diff --git a/src/hflow/runtime/_templates.py b/src/hflow/runtime/_templates.py index 790ed805b..54d00955f 100644 --- a/src/hflow/runtime/_templates.py +++ b/src/hflow/runtime/_templates.py @@ -721,9 +721,17 @@ def error_budget_gate(batch_counts): from hflow.format import CAMERA_SCHEMA_NAMES from hflow.reader import open_reader + from hflow.uri import parse_data_root_relative_uri, resolve_local_data_root_path def has_camera(uri): - reader = open_reader(Path($data_root) / str(uri)) + try: + source = resolve_local_data_root_path( + Path($data_root), parse_data_root_relative_uri(str(uri)) + ) + except ValueError: + # Kept, so process_batch refuses it as a counted error. + return True + reader = open_reader(source) try: return any( info.schema_name in CAMERA_SCHEMA_NAMES diff --git a/src/hflow/stage_execution.py b/src/hflow/stage_execution.py index 6f92e9695..55efe1cb2 100644 --- a/src/hflow/stage_execution.py +++ b/src/hflow/stage_execution.py @@ -45,8 +45,12 @@ selected_step_names_for_stage, ) from hflow.steps import IngestMode, Stage -from hflow.storage import is_bucket_url, parse_storage_root -from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri +from hflow.storage import LocalStorageRoot, is_bucket_url, parse_storage_root +from hflow.uri import ( + DataRootRelativeUri, + parse_data_root_relative_uri, + resolve_local_data_root_path, +) if TYPE_CHECKING: from hflow.app import App @@ -136,10 +140,14 @@ def require_application_data_root(application: "App", expected_data_root: str) - def resolve_episode_reference(data_root: str, uri: DataRootRelativeUri) -> "Path | str": - """A conf URI (relative to the data root) as a processable reference.""" + """A conf URI (relative to the data root) as a processable reference. + + A local reference is the resolved real path, refused when it leaves the + real data root, so processing opens exactly the file accepted here. + """ if is_bucket_url(data_root): return data_root.rstrip("/") + "/" + uri - return Path(data_root) / uri + return resolve_local_data_root_path(Path(data_root), uri) def resolve_step_selection( @@ -254,17 +262,20 @@ def plan_stage_batches( ] data_root_storage = parse_storage_root(data_root) - # The conf keeps the URI's own spelling, so `a/../b.mcap` stays the - # identity; only the size lookup normalizes, because a storage key is - # validated for containment and would refuse the literal segments. + + def planned_item_size(uri: DataRootRelativeUri) -> tuple[str, int]: + # The conf keeps the URI's own spelling, so `a/../b.mcap` stays the + # identity. A local root sizes the file processing will open; a bucket + # key is validated for containment and would refuse the literal + # segments, so it is sized normalized. + if isinstance(data_root_storage, LocalStorageRoot): + real_path = resolve_local_data_root_path(data_root_storage.path, uri) + return str(uri), real_path.stat().st_size + return str(uri), data_root_storage.file_size(normpath(str(uri))) + workers = min(STAGE_PLAN_FILE_SIZE_WORKERS, len(validated_uris)) with ThreadPoolExecutor(max_workers=workers) as executor: - item_sizes = dict( - executor.map( - lambda uri: (str(uri), data_root_storage.file_size(normpath(str(uri)))), - validated_uris, - ) - ) + item_sizes = dict(executor.map(planned_item_size, validated_uris)) all_planned: list[PlannedStageBatch] = [] current_delay_s = 0.0 @@ -597,12 +608,20 @@ def _plan_after_sync( failed_sync_uri_set = {str(uri) for uri in failed_sync_uris} data_root = str(application.data_root) - identity_by_uri = { - str(uri): application.source_identity( - resolve_episode_reference(data_root, parse_data_root_relative_uri(str(uri))) - ) - for uri in uris - } + + def source_identity_for(uri: str) -> str: + try: + return application.source_identity( + resolve_episode_reference(data_root, parse_data_root_relative_uri(uri)) + ) + except ValueError: + # Sync already refused this reference and counted it; only its plan + # is needed, and that is NoCanonicalEpisode whatever it is called. + if uri in failed_sync_uri_set: + return uri + raise + + identity_by_uri = {str(uri): source_identity_for(str(uri)) for uri in uris} successful_identities = sorted( {identity for uri, identity in identity_by_uri.items() if uri not in failed_sync_uri_set} ) diff --git a/src/hflow/stage_planning.py b/src/hflow/stage_planning.py index c475f83f2..256bc68b4 100644 --- a/src/hflow/stage_planning.py +++ b/src/hflow/stage_planning.py @@ -46,7 +46,7 @@ registered_step_is_selected, ) from hflow.steps import RAN_STATUSES, SETTLED_STATUSES, Stage -from hflow.uri import parse_data_root_relative_uri +from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri if TYPE_CHECKING: import duckdb @@ -458,20 +458,27 @@ def outstanding_stage_uris( from hflow.stage_execution import resolve_episode_reference validated_uris = [parse_data_root_relative_uri(str(uri)) for uri in uris] - identity_by_uri = { - uri: application.source_identity(resolve_episode_reference(data_root, uri)) - for uri in validated_uris - } + identity_by_uri: dict[DataRootRelativeUri, str | None] = {} + for uri in validated_uris: + try: + identity_by_uri[uri] = application.source_identity( + resolve_episode_reference(data_root, uri) + ) + except ValueError: + # A reference that resolves outside the data root has no catalog + # row either; it is handed on and refused again by processing. + identity_by_uri[uri] = None plans = plan_outstanding_stages( application, - list(identity_by_uri.values()), + [identity for identity in identity_by_uri.values() if identity is not None], (stage,), step_names=step_names, ) stage_by_step_name = application._registered_step_stages() outstanding: list[str] = [] for uri in validated_uris: - plan = plans.get(identity_by_uri[uri]) + identity = identity_by_uri[uri] + plan = None if identity is None else plans.get(identity) if not isinstance(plan, OutstandingStages): outstanding.append(str(uri)) if _stage_steps_by_uri is not None: diff --git a/src/hflow/uri.py b/src/hflow/uri.py index 1d51b05ab..af054612e 100644 --- a/src/hflow/uri.py +++ b/src/hflow/uri.py @@ -1,6 +1,6 @@ """Validation for episode URIs relative to a runtime data root.""" -from pathlib import PureWindowsPath +from pathlib import Path, PurePath, PureWindowsPath from posixpath import normpath from typing import NewType @@ -35,3 +35,21 @@ def parse_data_root_relative_uri(uri: str) -> DataRootRelativeUri: raise ValueError(f"{candidate!r} is not relative to the data root") return DataRootRelativeUri(candidate) + + +def resolve_local_data_root_path(data_root: Path, relative_reference: str | PurePath) -> Path: + """The real file a data-root-relative reference names on a local root. + + ``parse_data_root_relative_uri`` can only judge a spelling. Resolution + follows symlinks, so ``link/../x`` or a file symlink can leave the root + even though the spelling never does. The returned real path is the one + object callers may open: re-joining the original spelling later would + resolve it again. + """ + real_data_root = data_root.resolve() + real_path = (real_data_root / relative_reference).resolve() + if not real_path.is_relative_to(real_data_root): + raise ValueError( + f"{str(relative_reference)!r} resolves outside the data root {real_data_root}" + ) + return real_path From 7bfdbc8d9e84be5b8bc9934ab208d5994f6da7f6 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 20 Nov 2026 13:21:45 +0000 Subject: [PATCH 2/2] fix(ingest): leave batch-plan URI refusals to per-recording processing A URI refused for resolving outside the data root no longer fails the batch plan. It is planned at zero bytes without being stat'ed, so process_stage_batch refuses it again and counts it as that recording's error while its siblings process. --- src/hflow/stage_execution.py | 8 +++++++- tests/test_stage_execution.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/hflow/stage_execution.py b/src/hflow/stage_execution.py index 55efe1cb2..fb5f4ac33 100644 --- a/src/hflow/stage_execution.py +++ b/src/hflow/stage_execution.py @@ -269,7 +269,13 @@ def planned_item_size(uri: DataRootRelativeUri) -> tuple[str, int]: # key is validated for containment and would refuse the literal # segments, so it is sized normalized. if isinstance(data_root_storage, LocalStorageRoot): - real_path = resolve_local_data_root_path(data_root_storage.path, uri) + try: + real_path = resolve_local_data_root_path(data_root_storage.path, uri) + except ValueError: + # Planned at zero bytes, never stat'ed: process_stage_batch + # refuses it again and counts it as that recording's error, + # so one refusal cannot fail the plan for its siblings. + return str(uri), 0 return str(uri), real_path.stat().st_size return str(uri), data_root_storage.file_size(normpath(str(uri))) diff --git a/tests/test_stage_execution.py b/tests/test_stage_execution.py index 5ea7116e5..02d06d3d8 100644 --- a/tests/test_stage_execution.py +++ b/tests/test_stage_execution.py @@ -26,6 +26,7 @@ summarize_error_budget, summarize_quarantine_budget, ) +from hflow.testing import SyntheticEpisodeSpec, synthesize_episode class TestBudgets: @@ -120,6 +121,32 @@ def test_batch_lane_trims_uri_and_sizes_safe_internal_segments(self, tmp_path: P assert batches == [{"items": ["a/../b.mcap"], "start_delay_s": 0.0}] + def test_batch_lane_leaves_a_refused_uri_to_processing_and_runs_its_sibling( + self, tmp_path: Path + ) -> None: + data_root = tmp_path / "data" + (data_root / "episodes-in").mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + spec = SyntheticEpisodeSpec(duration_s=1.0, cameras=()) + synthesize_episode(data_root / "episodes-in" / "sibling.mcap", spec) + synthesize_episode(outside / "elsewhere.mcap", spec) + (data_root / "episodes-in" / "linked.mcap").symlink_to(outside / "elsewhere.mcap") + uris = ["episodes-in/sibling.mcap", "episodes-in/linked.mcap"] + + batches = plan_stage_batches(uris, mode="batch", batch_count=None, data_root=str(data_root)) + + assert sorted(item for batch in batches for item in batch["items"]) == sorted(uris) + app = hflow.App("planning", data_root=data_root, default_checks=()) + batch_counts = [ + asyncio.run(process_stage_batch(app, batch["items"], "sync")) for batch in batches + ] + assert sum(counts["processed"] for counts in batch_counts) == 1 + assert sum(counts["quarantined"] for counts in batch_counts) == 0 + assert sum(counts["errors"] for counts in batch_counts) == 1 + assert list((data_root / "episodes").glob("sibling-*/sibling.canonical.mcap")) + assert not list((data_root / "episodes").glob("*/elsewhere.canonical.mcap")) + def test_empty_uris_plan_nothing(self, tmp_path: Path) -> None: assert plan_stage_batches([], mode="batch", batch_count=None, data_root=str(tmp_path)) == []