Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/RUNTIME.md
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,12 @@ anything is triggered):
hflow ingest episodes-in/run_0001.mcap episodes-in/run_0002.mcap
```

On a local data root, a URI must also land inside the data root after
symlinks are followed: a file or directory symlink that leads outside it is
refused when the episode is processed and counted as that episode's error. To
keep recordings on another disk, mount that disk inside the data root or point
the data root itself at it.

Three flags select what runs and how:

- **`--profile`** picks the run profile: which stage sub-DAGs the master
Expand Down
24 changes: 20 additions & 4 deletions src/hflow/app.py
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,7 @@
stamps_from_provenance,
write_canonical_episode,
)
from hflow.uri import resolve_local_data_root_path
from hflow.workspace import RUNTIME_BUNDLE_DIRECTORY_NAME, Workspace

logger = logging.getLogger(__name__)
Expand All @@ -122,13 +123,21 @@ class SourceNotFound(FileNotFoundError):

def _local_source_candidate_paths(
source_reference: Path | str, root_path: Path
) -> tuple[Path, Path]:
"""The cwd and data-root interpretations of one local reference."""
) -> tuple[Path, Path | None]:
"""The cwd and data-root interpretations of one local reference.

The data-root interpretation is ``None`` when resolving it leaves the
real data root: a data-root key never names a file outside the root.
"""
source_path = Path(source_reference)
if source_path.is_absolute():
resolved_path = source_path.resolve()
return resolved_path, resolved_path
return source_path.resolve(), (root_path / source_path).resolve()
try:
root_candidate: Path | None = resolve_local_data_root_path(root_path, source_path)
except ValueError:
root_candidate = None
return source_path.resolve(), root_candidate


def _resolve_local_source_reference(source_reference: Path | str, root_path: Path) -> Path:
Expand All @@ -143,6 +152,13 @@ def _resolve_local_source_reference(source_reference: Path | str, root_path: Pat
if cwd_candidate == root_candidate:
return cwd_candidate
cwd_candidate_exists = cwd_candidate.is_file()
if root_candidate is None:
if cwd_candidate_exists:
return cwd_candidate
raise ValueError(
f"relative source reference {str(source_reference)!r} resolves outside "
f"the data root {root_path.resolve()}"
)
root_candidate_exists = root_candidate.is_file()

if cwd_candidate_exists and root_candidate_exists and cwd_candidate != root_candidate:
Expand Down Expand Up @@ -1679,7 +1695,7 @@ def _resolve_source_identity(
cwd_candidate, root_candidate = _local_source_candidate_paths(
episode, self.storage_root.path
)
if cwd_candidate.is_file() or root_candidate.is_file():
if root_candidate is None or cwd_candidate.is_file() or root_candidate.is_file():
return resolved_identity

root_identity = _local_source_identity(root_candidate, self.storage_root.path)
Expand Down
10 changes: 9 additions & 1 deletion src/hflow/runtime/_templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -721,9 +721,17 @@ def error_budget_gate(batch_counts):

from hflow.format import CAMERA_SCHEMA_NAMES
from hflow.reader import open_reader
from hflow.uri import parse_data_root_relative_uri, resolve_local_data_root_path

def has_camera(uri):
reader = open_reader(Path($data_root) / str(uri))
try:
source = resolve_local_data_root_path(
Path($data_root), parse_data_root_relative_uri(str(uri))
)
except ValueError:
# Kept, so process_batch refuses it as a counted error.
return True
reader = open_reader(source)
try:
return any(
info.schema_name in CAMERA_SCHEMA_NAMES
Expand Down
63 changes: 44 additions & 19 deletions src/hflow/stage_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,12 @@
selected_step_names_for_stage,
)
from hflow.steps import IngestMode, Stage
from hflow.storage import is_bucket_url, parse_storage_root
from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri
from hflow.storage import LocalStorageRoot, is_bucket_url, parse_storage_root
from hflow.uri import (
DataRootRelativeUri,
parse_data_root_relative_uri,
resolve_local_data_root_path,
)

if TYPE_CHECKING:
from hflow.app import App
Expand Down Expand Up @@ -136,10 +140,14 @@ def require_application_data_root(application: "App", expected_data_root: str) -


def resolve_episode_reference(data_root: str, uri: DataRootRelativeUri) -> "Path | str":
"""A conf URI (relative to the data root) as a processable reference."""
"""A conf URI (relative to the data root) as a processable reference.

A local reference is the resolved real path, refused when it leaves the
real data root, so processing opens exactly the file accepted here.
"""
if is_bucket_url(data_root):
return data_root.rstrip("/") + "/" + uri
return Path(data_root) / uri
return resolve_local_data_root_path(Path(data_root), uri)


def resolve_step_selection(
Expand Down Expand Up @@ -254,17 +262,26 @@ def plan_stage_batches(
]

data_root_storage = parse_storage_root(data_root)
# The conf keeps the URI's own spelling, so `a/../b.mcap` stays the
# identity; only the size lookup normalizes, because a storage key is
# validated for containment and would refuse the literal segments.

def planned_item_size(uri: DataRootRelativeUri) -> tuple[str, int]:
# The conf keeps the URI's own spelling, so `a/../b.mcap` stays the
# identity. A local root sizes the file processing will open; a bucket
# key is validated for containment and would refuse the literal
# segments, so it is sized normalized.
if isinstance(data_root_storage, LocalStorageRoot):
try:
real_path = resolve_local_data_root_path(data_root_storage.path, uri)
except ValueError:
# Planned at zero bytes, never stat'ed: process_stage_batch
# refuses it again and counts it as that recording's error,
# so one refusal cannot fail the plan for its siblings.
return str(uri), 0
return str(uri), real_path.stat().st_size
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
return str(uri), data_root_storage.file_size(normpath(str(uri)))

workers = min(STAGE_PLAN_FILE_SIZE_WORKERS, len(validated_uris))
with ThreadPoolExecutor(max_workers=workers) as executor:
item_sizes = dict(
executor.map(
lambda uri: (str(uri), data_root_storage.file_size(normpath(str(uri)))),
validated_uris,
)
)
item_sizes = dict(executor.map(planned_item_size, validated_uris))
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved

all_planned: list[PlannedStageBatch] = []
current_delay_s = 0.0
Expand Down Expand Up @@ -597,12 +614,20 @@ def _plan_after_sync(

failed_sync_uri_set = {str(uri) for uri in failed_sync_uris}
data_root = str(application.data_root)
identity_by_uri = {
str(uri): application.source_identity(
resolve_episode_reference(data_root, parse_data_root_relative_uri(str(uri)))
)
for uri in uris
}

def source_identity_for(uri: str) -> str:
try:
return application.source_identity(
resolve_episode_reference(data_root, parse_data_root_relative_uri(uri))
)
except ValueError:
# Sync already refused this reference and counted it; only its plan
# is needed, and that is NoCanonicalEpisode whatever it is called.
if uri in failed_sync_uri_set:
return uri
raise

identity_by_uri = {str(uri): source_identity_for(str(uri)) for uri in uris}
successful_identities = sorted(
{identity for uri, identity in identity_by_uri.items() if uri not in failed_sync_uri_set}
)
Expand Down
21 changes: 14 additions & 7 deletions src/hflow/stage_planning.py
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
registered_step_is_selected,
)
from hflow.steps import RAN_STATUSES, SETTLED_STATUSES, Stage
from hflow.uri import parse_data_root_relative_uri
from hflow.uri import DataRootRelativeUri, parse_data_root_relative_uri

if TYPE_CHECKING:
import duckdb
Expand Down Expand Up @@ -458,20 +458,27 @@ def outstanding_stage_uris(
from hflow.stage_execution import resolve_episode_reference

validated_uris = [parse_data_root_relative_uri(str(uri)) for uri in uris]
identity_by_uri = {
uri: application.source_identity(resolve_episode_reference(data_root, uri))
for uri in validated_uris
}
identity_by_uri: dict[DataRootRelativeUri, str | None] = {}
for uri in validated_uris:
try:
identity_by_uri[uri] = application.source_identity(
resolve_episode_reference(data_root, uri)
)
except ValueError:
# A reference that resolves outside the data root has no catalog
# row either; it is handed on and refused again by processing.
identity_by_uri[uri] = None
plans = plan_outstanding_stages(
application,
list(identity_by_uri.values()),
[identity for identity in identity_by_uri.values() if identity is not None],
(stage,),
step_names=step_names,
)
stage_by_step_name = application._registered_step_stages()
outstanding: list[str] = []
for uri in validated_uris:
plan = plans.get(identity_by_uri[uri])
identity = identity_by_uri[uri]
plan = None if identity is None else plans.get(identity)
if not isinstance(plan, OutstandingStages):
outstanding.append(str(uri))
if _stage_steps_by_uri is not None:
Expand Down
20 changes: 19 additions & 1 deletion src/hflow/uri.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
"""Validation for episode URIs relative to a runtime data root."""

from pathlib import PureWindowsPath
from pathlib import Path, PurePath, PureWindowsPath
from posixpath import normpath
from typing import NewType

Expand Down Expand Up @@ -35,3 +35,21 @@ def parse_data_root_relative_uri(uri: str) -> DataRootRelativeUri:
raise ValueError(f"{candidate!r} is not relative to the data root")

return DataRootRelativeUri(candidate)


def resolve_local_data_root_path(data_root: Path, relative_reference: str | PurePath) -> Path:
"""The real file a data-root-relative reference names on a local root.

``parse_data_root_relative_uri`` can only judge a spelling. Resolution
follows symlinks, so ``link/../x`` or a file symlink can leave the root
even though the spelling never does. The returned real path is the one
object callers may open: re-joining the original spelling later would
resolve it again.
"""
real_data_root = data_root.resolve()
real_path = (real_data_root / relative_reference).resolve()
if not real_path.is_relative_to(real_data_root):
raise ValueError(
f"{str(relative_reference)!r} resolves outside the data root {real_data_root}"
)
return real_path
27 changes: 27 additions & 0 deletions tests/test_stage_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
summarize_error_budget,
summarize_quarantine_budget,
)
from hflow.testing import SyntheticEpisodeSpec, synthesize_episode


class TestBudgets:
Expand Down Expand Up @@ -120,6 +121,32 @@ def test_batch_lane_trims_uri_and_sizes_safe_internal_segments(self, tmp_path: P

assert batches == [{"items": ["a/../b.mcap"], "start_delay_s": 0.0}]

def test_batch_lane_leaves_a_refused_uri_to_processing_and_runs_its_sibling(
self, tmp_path: Path
) -> None:
data_root = tmp_path / "data"
(data_root / "episodes-in").mkdir(parents=True)
outside = tmp_path / "outside"
outside.mkdir()
spec = SyntheticEpisodeSpec(duration_s=1.0, cameras=())
synthesize_episode(data_root / "episodes-in" / "sibling.mcap", spec)
synthesize_episode(outside / "elsewhere.mcap", spec)
(data_root / "episodes-in" / "linked.mcap").symlink_to(outside / "elsewhere.mcap")
uris = ["episodes-in/sibling.mcap", "episodes-in/linked.mcap"]

batches = plan_stage_batches(uris, mode="batch", batch_count=None, data_root=str(data_root))

assert sorted(item for batch in batches for item in batch["items"]) == sorted(uris)
app = hflow.App("planning", data_root=data_root, default_checks=())
batch_counts = [
asyncio.run(process_stage_batch(app, batch["items"], "sync")) for batch in batches
]
assert sum(counts["processed"] for counts in batch_counts) == 1
assert sum(counts["quarantined"] for counts in batch_counts) == 0
assert sum(counts["errors"] for counts in batch_counts) == 1
assert list((data_root / "episodes").glob("sibling-*/sibling.canonical.mcap"))
assert not list((data_root / "episodes").glob("*/elsewhere.canonical.mcap"))

def test_empty_uris_plan_nothing(self, tmp_path: Path) -> None:
assert plan_stage_batches([], mode="batch", batch_count=None, data_root=str(tmp_path)) == []

Expand Down