diff --git a/docs/RUNTIME.md b/docs/RUNTIME.md index 7244b666c..4d57feab0 100644 --- a/docs/RUNTIME.md +++ b/docs/RUNTIME.md @@ -195,6 +195,12 @@ anything is triggered): hflow ingest episodes-in/run_0001.mcap episodes-in/run_0002.mcap ``` +On a local data root, a URI must also land inside the data root after +symlinks are followed: a file or directory symlink that leads outside it is +refused when the episode is processed and counted as that episode's error. To +keep recordings on another disk, mount that disk inside the data root or point +the data root itself at it. + Three flags select what runs and how: - **`--profile`** picks the run profile: which stage sub-DAGs the master diff --git a/src/hflow/app.py b/src/hflow/app.py index 25b7b3828..387f010b1 100644 --- a/src/hflow/app.py +++ b/src/hflow/app.py @@ -101,6 +101,7 @@ stamps_from_provenance, write_canonical_episode, ) +from hflow.uri import resolve_local_data_root_path from hflow.workspace import RUNTIME_BUNDLE_DIRECTORY_NAME, Workspace logger = logging.getLogger(__name__) @@ -122,13 +123,21 @@ class SourceNotFound(FileNotFoundError): def _local_source_candidate_paths( source_reference: Path | str, root_path: Path -) -> tuple[Path, Path]: - """The cwd and data-root interpretations of one local reference.""" +) -> tuple[Path, Path | None]: + """The cwd and data-root interpretations of one local reference. + + The data-root interpretation is ``None`` when resolving it leaves the + real data root: a data-root key never names a file outside the root. + """ source_path = Path(source_reference) if source_path.is_absolute(): resolved_path = source_path.resolve() return resolved_path, resolved_path - return source_path.resolve(), (root_path / source_path).resolve() + try: + root_candidate: Path | None = resolve_local_data_root_path(root_path, source_path) + except ValueError: + root_candidate = None + return source_path.resolve(), root_candidate def _resolve_local_source_reference(source_reference: Path | str, root_path: Path) -> Path: @@ -143,6 +152,13 @@ def _resolve_local_source_reference(source_reference: Path | str, root_path: Pat if cwd_candidate == root_candidate: return cwd_candidate cwd_candidate_exists = cwd_candidate.is_file() + if root_candidate is None: + if cwd_candidate_exists: + return cwd_candidate + raise ValueError( + f"relative source reference {str(source_reference)!r} resolves outside " + f"the data root {root_path.resolve()}" + ) root_candidate_exists = root_candidate.is_file() if cwd_candidate_exists and root_candidate_exists and cwd_candidate != root_candidate: @@ -1679,7 +1695,7 @@ def _resolve_source_identity( cwd_candidate, root_candidate = _local_source_candidate_paths( episode, self.storage_root.path ) - if cwd_candidate.is_file() or root_candidate.is_file(): + if root_candidate is None or cwd_candidate.is_file() or root_candidate.is_file(): return resolved_identity root_identity = _local_source_identity(root_candidate, self.storage_root.path) diff --git a/src/hflow/runtime/_templates.py b/src/hflow/runtime/_templates.py index 790ed805b..54d00955f 100644 --- a/src/hflow/runtime/_templates.py +++ b/src/hflow/runtime/_templates.py @@ -721,9 +721,17 @@ def error_budget_gate(batch_counts): from hflow.format import CAMERA_SCHEMA_NAMES from hflow.reader import open_reader + from hflow.uri import parse_data_root_relative_uri, resolve_local_data_root_path def has_camera(uri): - reader = open_reader(Path($data_root) / str(uri)) + try: + source = resolve_local_data_root_path( + Path($data_root), parse_data_root_relative_uri(str(uri)) + ) + except ValueError: + # Kept, so process_batch refuses it as a counted error. + return True + reader = open_reader(source) try: return any( info.schema_name in CAMERA_SCHEMA_NAMES diff --git a/src/hflow/stage_execution.py b/src/hflow/stage_execution.py index 6f92e9695..fb5f4ac33 100644 --- a/src/hflow/stage_execution.py +++ b/src/hflow/stage_execution.py @@ -45,8 +45,12 @@ selected_step_names_for_stage, ) from hflow.steps import IngestMode, Stage -from hflow.storage import is_bucket_url, parse_storage_root -from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri +from hflow.storage import LocalStorageRoot, is_bucket_url, parse_storage_root +from hflow.uri import ( + DataRootRelativeUri, + parse_data_root_relative_uri, + resolve_local_data_root_path, +) if TYPE_CHECKING: from hflow.app import App @@ -136,10 +140,14 @@ def require_application_data_root(application: "App", expected_data_root: str) - def resolve_episode_reference(data_root: str, uri: DataRootRelativeUri) -> "Path | str": - """A conf URI (relative to the data root) as a processable reference.""" + """A conf URI (relative to the data root) as a processable reference. + + A local reference is the resolved real path, refused when it leaves the + real data root, so processing opens exactly the file accepted here. + """ if is_bucket_url(data_root): return data_root.rstrip("/") + "/" + uri - return Path(data_root) / uri + return resolve_local_data_root_path(Path(data_root), uri) def resolve_step_selection( @@ -254,17 +262,26 @@ def plan_stage_batches( ] data_root_storage = parse_storage_root(data_root) - # The conf keeps the URI's own spelling, so `a/../b.mcap` stays the - # identity; only the size lookup normalizes, because a storage key is - # validated for containment and would refuse the literal segments. + + def planned_item_size(uri: DataRootRelativeUri) -> tuple[str, int]: + # The conf keeps the URI's own spelling, so `a/../b.mcap` stays the + # identity. A local root sizes the file processing will open; a bucket + # key is validated for containment and would refuse the literal + # segments, so it is sized normalized. + if isinstance(data_root_storage, LocalStorageRoot): + try: + real_path = resolve_local_data_root_path(data_root_storage.path, uri) + except ValueError: + # Planned at zero bytes, never stat'ed: process_stage_batch + # refuses it again and counts it as that recording's error, + # so one refusal cannot fail the plan for its siblings. + return str(uri), 0 + return str(uri), real_path.stat().st_size + return str(uri), data_root_storage.file_size(normpath(str(uri))) + workers = min(STAGE_PLAN_FILE_SIZE_WORKERS, len(validated_uris)) with ThreadPoolExecutor(max_workers=workers) as executor: - item_sizes = dict( - executor.map( - lambda uri: (str(uri), data_root_storage.file_size(normpath(str(uri)))), - validated_uris, - ) - ) + item_sizes = dict(executor.map(planned_item_size, validated_uris)) all_planned: list[PlannedStageBatch] = [] current_delay_s = 0.0 @@ -597,12 +614,20 @@ def _plan_after_sync( failed_sync_uri_set = {str(uri) for uri in failed_sync_uris} data_root = str(application.data_root) - identity_by_uri = { - str(uri): application.source_identity( - resolve_episode_reference(data_root, parse_data_root_relative_uri(str(uri))) - ) - for uri in uris - } + + def source_identity_for(uri: str) -> str: + try: + return application.source_identity( + resolve_episode_reference(data_root, parse_data_root_relative_uri(uri)) + ) + except ValueError: + # Sync already refused this reference and counted it; only its plan + # is needed, and that is NoCanonicalEpisode whatever it is called. + if uri in failed_sync_uri_set: + return uri + raise + + identity_by_uri = {str(uri): source_identity_for(str(uri)) for uri in uris} successful_identities = sorted( {identity for uri, identity in identity_by_uri.items() if uri not in failed_sync_uri_set} ) diff --git a/src/hflow/stage_planning.py b/src/hflow/stage_planning.py index c475f83f2..256bc68b4 100644 --- a/src/hflow/stage_planning.py +++ b/src/hflow/stage_planning.py @@ -46,7 +46,7 @@ registered_step_is_selected, ) from hflow.steps import RAN_STATUSES, SETTLED_STATUSES, Stage -from hflow.uri import parse_data_root_relative_uri +from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri if TYPE_CHECKING: import duckdb @@ -458,20 +458,27 @@ def outstanding_stage_uris( from hflow.stage_execution import resolve_episode_reference validated_uris = [parse_data_root_relative_uri(str(uri)) for uri in uris] - identity_by_uri = { - uri: application.source_identity(resolve_episode_reference(data_root, uri)) - for uri in validated_uris - } + identity_by_uri: dict[DataRootRelativeUri, str | None] = {} + for uri in validated_uris: + try: + identity_by_uri[uri] = application.source_identity( + resolve_episode_reference(data_root, uri) + ) + except ValueError: + # A reference that resolves outside the data root has no catalog + # row either; it is handed on and refused again by processing. + identity_by_uri[uri] = None plans = plan_outstanding_stages( application, - list(identity_by_uri.values()), + [identity for identity in identity_by_uri.values() if identity is not None], (stage,), step_names=step_names, ) stage_by_step_name = application._registered_step_stages() outstanding: list[str] = [] for uri in validated_uris: - plan = plans.get(identity_by_uri[uri]) + identity = identity_by_uri[uri] + plan = None if identity is None else plans.get(identity) if not isinstance(plan, OutstandingStages): outstanding.append(str(uri)) if _stage_steps_by_uri is not None: diff --git a/src/hflow/uri.py b/src/hflow/uri.py index 1d51b05ab..af054612e 100644 --- a/src/hflow/uri.py +++ b/src/hflow/uri.py @@ -1,6 +1,6 @@ """Validation for episode URIs relative to a runtime data root.""" -from pathlib import PureWindowsPath +from pathlib import Path, PurePath, PureWindowsPath from posixpath import normpath from typing import NewType @@ -35,3 +35,21 @@ def parse_data_root_relative_uri(uri: str) -> DataRootRelativeUri: raise ValueError(f"{candidate!r} is not relative to the data root") return DataRootRelativeUri(candidate) + + +def resolve_local_data_root_path(data_root: Path, relative_reference: str | PurePath) -> Path: + """The real file a data-root-relative reference names on a local root. + + ``parse_data_root_relative_uri`` can only judge a spelling. Resolution + follows symlinks, so ``link/../x`` or a file symlink can leave the root + even though the spelling never does. The returned real path is the one + object callers may open: re-joining the original spelling later would + resolve it again. + """ + real_data_root = data_root.resolve() + real_path = (real_data_root / relative_reference).resolve() + if not real_path.is_relative_to(real_data_root): + raise ValueError( + f"{str(relative_reference)!r} resolves outside the data root {real_data_root}" + ) + return real_path diff --git a/tests/test_stage_execution.py b/tests/test_stage_execution.py index 5ea7116e5..02d06d3d8 100644 --- a/tests/test_stage_execution.py +++ b/tests/test_stage_execution.py @@ -26,6 +26,7 @@ summarize_error_budget, summarize_quarantine_budget, ) +from hflow.testing import SyntheticEpisodeSpec, synthesize_episode class TestBudgets: @@ -120,6 +121,32 @@ def test_batch_lane_trims_uri_and_sizes_safe_internal_segments(self, tmp_path: P assert batches == [{"items": ["a/../b.mcap"], "start_delay_s": 0.0}] + def test_batch_lane_leaves_a_refused_uri_to_processing_and_runs_its_sibling( + self, tmp_path: Path + ) -> None: + data_root = tmp_path / "data" + (data_root / "episodes-in").mkdir(parents=True) + outside = tmp_path / "outside" + outside.mkdir() + spec = SyntheticEpisodeSpec(duration_s=1.0, cameras=()) + synthesize_episode(data_root / "episodes-in" / "sibling.mcap", spec) + synthesize_episode(outside / "elsewhere.mcap", spec) + (data_root / "episodes-in" / "linked.mcap").symlink_to(outside / "elsewhere.mcap") + uris = ["episodes-in/sibling.mcap", "episodes-in/linked.mcap"] + + batches = plan_stage_batches(uris, mode="batch", batch_count=None, data_root=str(data_root)) + + assert sorted(item for batch in batches for item in batch["items"]) == sorted(uris) + app = hflow.App("planning", data_root=data_root, default_checks=()) + batch_counts = [ + asyncio.run(process_stage_batch(app, batch["items"], "sync")) for batch in batches + ] + assert sum(counts["processed"] for counts in batch_counts) == 1 + assert sum(counts["quarantined"] for counts in batch_counts) == 0 + assert sum(counts["errors"] for counts in batch_counts) == 1 + assert list((data_root / "episodes").glob("sibling-*/sibling.canonical.mcap")) + assert not list((data_root / "episodes").glob("*/elsewhere.canonical.mcap")) + def test_empty_uris_plan_nothing(self, tmp_path: Path) -> None: assert plan_stage_batches([], mode="batch", batch_count=None, data_root=str(tmp_path)) == []