diff --git a/examples/README.md b/examples/README.md index cc784b7d..ab3a9916 100644 --- a/examples/README.md +++ b/examples/README.md @@ -474,17 +474,23 @@ byte-for-byte (each source (chunk, file) keeping its identity in the destination path) and slices the selected data rows from the source chunk parquets, so cameras, Parquet column types, array shapes, and frame timing match the source. Data files retain the source Arrow schema, including float32 -widths and field and schema metadata. Episode indexes are renumbered sequentially -in selection order. The exporter drives the public `hflow import lerobot` entry point to +widths and field and schema metadata; each written data file's schema is checked +against its source before publication. The feature registry retains the source +declaration (dtype, shape, names) for every written non-video feature, including +integer indexes and boolean flags, and omits declarations for unwritten columns. +Episode indexes are renumbered sequentially in selection order. The exporter +drives the public `hflow import lerobot` entry point to materialize the source archive (meta, data chunks, video chunks) and copies the video chunks byte-for-byte. A dataset-card draft (`README.md`) and `export-provenance.json` record the source repository, source commit, exporter version, selection SQL/manifest, and selected source episode indexes. -The v2 exporter preserves source Parquet schemas; regenerate v3 exports made -with v1 to restore source dtypes and array shapes. This does not change HFlow's -canonical episode format or transform behavior version. +The v3 exporter completes the feature registry and preserves the source Parquet +schemas introduced in exporter v2. Regenerate exports made with v1 to restore +source dtypes and array shapes, and those made with v1 or v2 to restore missing +integer and boolean feature declarations. This does not change HFlow's canonical +episode format or transform behavior version. From a curation manifest: @@ -507,7 +513,7 @@ uv run python examples/lerobot/export.py \ Failures abort before any output is published: mixed source repositories or revisions, non-immutable revisions, missing provenance, missing source episodes, duplicate selections, and a staged dataset that fails structural -or (when `lerobot` is installed) official-API validation never replace a +validation never replace a previously valid destination. Code: [`lerobot/export.py`](./lerobot/export.py) diff --git a/examples/lerobot/export.py b/examples/lerobot/export.py index 8be961df..780ff22a 100644 --- a/examples/lerobot/export.py +++ b/examples/lerobot/export.py @@ -37,6 +37,7 @@ import tempfile from collections.abc import Sequence from dataclasses import dataclass +from itertools import zip_longest from pathlib import Path import duckdb @@ -45,7 +46,7 @@ import hflow -EXPORT_VERSION = "lerobot-export-v2" +EXPORT_VERSION = "lerobot-export-v3" _COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$") @@ -369,6 +370,28 @@ def _episode_rows(table: pa.Table) -> list[dict]: return out +def _check_written_data_schema(path: Path, source_schema: pa.Schema) -> None: + """Check the written footer without reading any frame data.""" + written_schema = pq.read_schema(path) + if written_schema.equals(source_schema, check_metadata=True): + return + difference = "schema metadata" + for source_field, written_field in zip_longest(source_schema, written_schema): + if source_field is None: + difference = f"unexpected field {written_field.name!r}" + elif written_field is None: + difference = f"missing field {source_field.name!r}" + elif source_field.equals(written_field, check_metadata=True): + continue + else: + difference = ( + f"field {source_field.name!r} (type, nullability, or metadata): " + f"expected {source_field}, found {written_field}" + ) + break + raise ValueError(f"exported data file {path}: schema differs from source at {difference}") + + def _write_v3_repository( *, corpus: dict, @@ -579,6 +602,7 @@ def _fetch_video(cam: str, vw: dict) -> Path: total_frames = 0 video_paths: list[Path] = [] copied_videos: set[tuple[str, int, int]] = set() + written_data_columns: set[str] = set() for ( new_idx, @@ -600,10 +624,10 @@ def _fetch_video(cam: str, vw: dict) -> Path: data_frames.append(d) frame_rows.append(d) total_frames += length - pq.write_table( - pa.Table.from_pylist(frame_rows, schema=source_data_schema), - data_dir / f"file-{new_idx:03d}.parquet", - ) + data_path = data_dir / f"file-{new_idx:03d}.parquet" + pq.write_table(pa.Table.from_pylist(frame_rows, schema=source_data_schema), data_path) + _check_written_data_schema(data_path, source_data_schema) + written_data_columns.update(source_data_schema.names) # keep the source (chunk, file) identity in the destination path ep_video_refs: dict[str, dict] = {} @@ -655,10 +679,10 @@ def _fetch_video(cam: str, vw: dict) -> Path: cam: corpus["info"]["features"].get(cam, {"dtype": "video", "shape": [480, 640, 3]}) for cam in camera_keys } - numeric_features = { + data_features = { k: v for k, v in corpus["info"].get("features", {}).items() - if isinstance(v, dict) and v.get("dtype") == "float32" + if isinstance(v, dict) and v.get("dtype") != "video" and k in written_data_columns } out_info = { "code": "LeRobotDataset/v3", @@ -670,7 +694,7 @@ def _fetch_video(cam: str, vw: dict) -> Path: "splits": {"train": [f"episode_{i:06d}" for i in range(len(selections))]}, "data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet", "video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4", - "features": {**numeric_features, **video_features}, + "features": {**data_features, **video_features}, "version": 1, } (destination / "meta" / "info.json").write_text(json.dumps(out_info, indent=2)) diff --git a/tests/test_lerobot_export.py b/tests/test_lerobot_export.py index 817bbf85..cedda609 100644 --- a/tests/test_lerobot_export.py +++ b/tests/test_lerobot_export.py @@ -228,6 +228,119 @@ def _provenance_json(**overrides: object) -> str: return json.dumps({**json.loads(_provenance_meta(0)), **overrides}) +def test_export_registry_matches_written_features(fake_corpus: dict, tmp_path: Path) -> None: + """All written non-video features retain their exact source declarations.""" + import pyarrow as pa + import pyarrow.parquet as pq + + source_path = Path(fake_corpus["cache_dir"]) / "data/chunk-000/file-000.parquet" + source = pq.read_table(source_path) + for name in ("index", "episode_index", "frame_index"): + source = source.set_column( + source.schema.get_field_index(name), name, source[name].cast(pa.int64()) + ) + source = source.append_column("task_index", pa.array([0] * source.num_rows, pa.int64())) + source = source.append_column( + "next.done", pa.array([i % 2 == 0 for i in range(source.num_rows)], pa.bool_()) + ) + pq.write_table(source, source_path) + features = fake_corpus["info"]["features"] + for name in ("index", "episode_index", "frame_index", "task_index", "next.done"): + features[name] = { + "dtype": "bool" if name == "next.done" else "int64", + "shape": [1], + "names": [name], + } + features["unwritten.float"] = {"dtype": "float32", "shape": [1], "names": None} + features["unwritten.integer"] = {"dtype": "int64", "shape": [1], "names": None} + (tmp_path / "meta/info.json").write_text(json.dumps(fake_corpus["info"])) + manifest = _fake_manifest( + tmp_path, + [{"metadata_json": _provenance_meta(0)}, {"metadata_json": _provenance_meta(2)}], + ) + destination = tmp_path / "out" + + export.export(destination, manifest=manifest, camera_keys=CAMS[:1]) + + registry = json.loads((destination / "meta/info.json").read_text())["features"] + assert {key for key, value in registry.items() if value["dtype"] == "video"} == {CAMS[0]} + data_features = {key: value for key, value in registry.items() if value["dtype"] != "video"} + for exported_file in sorted((destination / "data").rglob("*.parquet")): + exported = pq.read_table(exported_file) + assert data_features == {name: features[name] for name in exported.column_names} + assert set(registry) == set(exported.column_names) | {CAMS[0]} + for name in ("index", "episode_index", "frame_index", "task_index"): + assert exported.schema.field(name).type == pa.int64() + assert exported.schema.field("next.done").type == pa.bool_() + + +@pytest.mark.parametrize( + ("mismatch", "differing_field"), + [ + ("type", "index"), + ("nullability", "index"), + ("field_metadata", "index"), + ("schema_metadata", "schema metadata"), + ("missing_field", "action"), + ("extra_field", "extra"), + ("field_order", "index"), + ], +) +def test_export_refuses_written_schema_mismatch( + fake_corpus: dict, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + mismatch: str, + differing_field: str, +) -> None: + """A changed schema in the second data file aborts publication with a useful error.""" + import pyarrow as pa + import pyarrow.parquet as pq + + manifest = _fake_manifest( + tmp_path, + [{"metadata_json": _provenance_meta(0)}, {"metadata_json": _provenance_meta(2)}], + ) + write_table = pq.write_table + + def write_changed_schema(table: pa.Table, where: Path) -> None: + if where.parent.name == "chunk-000" and where.name == "file-001.parquet": + first_field = table.schema.field(0) + match mismatch: + case "type": + table = table.set_column( + 0, first_field.name, table.column(0).cast(pa.float64()) + ) + case "nullability": + table = table.cast(table.schema.set(0, first_field.with_nullable(False))) + case "field_metadata": + table = table.cast( + table.schema.set(0, first_field.with_metadata({b"changed": b"true"})) + ) + case "schema_metadata": + table = table.replace_schema_metadata({b"changed": b"true"}) + case "missing_field": + table = table.drop_columns(["action"]) + case "extra_field": + table = table.append_column("extra", pa.array([0] * table.num_rows)) + case "field_order": + table = table.select(list(reversed(table.column_names))) + write_table(table, where) + + monkeypatch.setattr(pq, "write_table", write_changed_schema) + destination = tmp_path / "out" + destination.mkdir() + (destination / "sentinel.txt").write_text("previous export") + with pytest.raises(ValueError) as error: + export.export(destination, manifest=manifest, camera_keys=CAMS) + message = str(error.value) + assert "data/chunk-000/file-001.parquet" in message + assert "schema differs from source" in message + assert differing_field in message + assert (destination / "sentinel.txt").read_text() == "previous export" + assert sorted(path.name for path in destination.iterdir()) == ["sentinel.txt"] + + def test_export_preserves_source_data_schema(fake_corpus: dict, tmp_path: Path) -> None: """Renumbered rows retain source float widths, array shapes, and metadata.""" import pyarrow as pa