Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/RUNTIME.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,9 @@ hflow ingest --profile metadata_backfill \
--step camera_integrity --step hand_activity episodes-in/*.mcap
```

The profile must enable every selected step's owning stage. `sync` still runs
The profile must enable every selected step's owning stage. In-process, a name
no step is registered under, or one whose stage the profile disables, is
refused with exit code 2 before any episode is read. `sync` still runs
when the profile includes it; unselected steps write no catalog rows and
remain available to a later run.

Expand Down
7 changes: 6 additions & 1 deletion src/hflow/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,7 +610,7 @@ def _ingest_in_process(
turns off the per-episode planning that would otherwise skip the ones the
catalog already records as current.
"""
from hflow.stage_execution import run_stages_directly
from hflow.stage_execution import resolve_step_selection, run_stages_directly
from hflow.stage_planning import StageSelection
from hflow.steps import stages_for_profile

Expand All @@ -621,6 +621,11 @@ def _ingest_in_process(
return 2
stages = stages_for_profile(profile)
selection = StageSelection.EVERY_STAGE if all_stages else StageSelection.OUTSTANDING
try:
resolve_step_selection(app, step_names, stages)
except ValueError as error:
print(f"ingest: {error}", file=sys.stderr)
return 2
print(
f"ingest: no runtime addressed; processing {len(uris)} episode(s) "
f"in this process against {app.data_root}",
Expand Down
24 changes: 15 additions & 9 deletions src/hflow/stage_execution.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,18 @@ def resolve_episode_reference(data_root: str, uri: DataRootRelativeUri) -> "Path
return Path(data_root) / uri


def resolve_step_selection(
application: "App",
step_names: Iterable[str] | None,
enabled_stages: Iterable[Stage],
) -> RegisteredStepSelection:
"""Parse a run's step selection against the enabled stages, before any I/O.

Raises ``ValueError`` for an unknown step or one whose stage is disabled.
"""
return application._resolve_registered_step_selection(step_names, enabled_stages)


def resolve_step_names_for_stage(
application: "App",
step_names: Iterable[str] | None,
Expand All @@ -161,9 +173,7 @@ def resolve_step_names_for_stage(
f"stage {stage.value!r} is not enabled for this run; enabled stages: "
f"{sorted(enabled.value for enabled in enabled_stages)}"
)
registered_step_selection = application._resolve_registered_step_selection(
step_names, enabled_stages
)
registered_step_selection = resolve_step_selection(application, step_names, enabled_stages)
return selected_step_names_for_stage(
registered_step_selection,
stage,
Expand Down Expand Up @@ -325,9 +335,7 @@ async def process_stage_batch(
elif step_names is None:
registered_step_selection = ALL_REGISTERED_STEPS
else:
registered_step_selection = application._resolve_registered_step_selection(
step_names, {stage}
)
registered_step_selection = resolve_step_selection(application, step_names, {stage})
data_root = str(application.data_root)
counts: StageBatchCounts = {"processed": 0, "quarantined": 0, "errors": 0}
# The labels and media gates read the episode's cataloged quarantine
Expand Down Expand Up @@ -463,9 +471,7 @@ async def run_stages_directly(
"""
enabled_stages = set(stages)
ordered_stages = [stage for stage in Stage if stage in enabled_stages]
registered_step_selection = application._resolve_registered_step_selection(
step_names, enabled_stages
)
registered_step_selection = resolve_step_selection(application, step_names, enabled_stages)
stage_by_step_name = application._registered_step_stages()

def selected_names_for_stage(stage: Stage) -> frozenset[str] | None:
Expand Down
37 changes: 37 additions & 0 deletions tests/test_ingest_in_process.py
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,43 @@ def test_an_absolute_uri_is_refused_before_anything_runs(
assert "not relative to the data root" in capsys.readouterr().err


@pytest.mark.parametrize(
("step_arguments", "expected_message"),
[
(
["--step", "not_registered"],
"ingest: unknown step names ['not_registered']; registered steps: ",
),
(
["--profile", "relabel", "--step", "episode_duration"],
"ingest: selected steps belong to stages that are not enabled: "
"['episode_duration (meta)']; enabled stages: ['labels']",
),
],
ids=["unknown-step-name", "step-in-a-stage-the-profile-disables"],
)
def test_a_step_selection_the_run_cannot_honour_is_refused_before_anything_runs(
project: Path,
monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str],
step_arguments: list[str],
expected_message: str,
) -> None:
"""A mistyped --step is bad input (exit 2), not a crash or a failed run."""
monkeypatch.delenv("HFLOW_DATA_ROOT", raising=False)
monkeypatch.delenv("HFLOW_AIRFLOW_URL", raising=False)
monkeypatch.chdir(project)

assert cli_main(["ingest", "episodes-in/episode_0001.mcap", *step_arguments]) == 2

printed = capsys.readouterr()
assert expected_message in printed.err
assert "Traceback" not in printed.err
assert "processing 1 episode(s)" not in printed.err
assert printed.out == ""
assert not (project / "data" / "catalog").exists()


def test_stages_run_in_graph_order_whatever_order_they_are_asked_for(project: Path) -> None:
"""Order is the stage graph's, not the caller's.

Expand Down
Loading