Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 12 additions & 6 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -474,17 +474,23 @@ byte-for-byte (each source (chunk, file) keeping its identity in the
destination path) and slices the selected data rows from the source chunk
parquets, so cameras, Parquet column types, array shapes, and frame timing
match the source. Data files retain the source Arrow schema, including float32
widths and field and schema metadata. Episode indexes are renumbered sequentially
in selection order. The exporter drives the public `hflow import lerobot` entry point to
widths and field and schema metadata; each written data file's schema is checked
against its source before publication. The feature registry retains the source
declaration (dtype, shape, names) for every written non-video feature, including
integer indexes and boolean flags, and omits declarations for unwritten columns.
Episode indexes are renumbered sequentially in selection order. The exporter
drives the public `hflow import lerobot` entry point to
materialize the source archive (meta, data chunks, video chunks) and copies
the video chunks byte-for-byte. A dataset-card draft (`README.md`) and
`export-provenance.json` record the source repository, source commit,
exporter version, selection SQL/manifest, and selected source episode
indexes.

The v2 exporter preserves source Parquet schemas; regenerate v3 exports made
with v1 to restore source dtypes and array shapes. This does not change HFlow's
canonical episode format or transform behavior version.
The v3 exporter completes the feature registry and preserves the source Parquet
schemas introduced in exporter v2. Regenerate exports made with v1 to restore
source dtypes and array shapes, and those made with v1 or v2 to restore missing
integer and boolean feature declarations. This does not change HFlow's canonical
episode format or transform behavior version.

From a curation manifest:

Expand All @@ -507,7 +513,7 @@ uv run python examples/lerobot/export.py \
Failures abort before any output is published: mixed source repositories
or revisions, non-immutable revisions, missing provenance, missing source
episodes, duplicate selections, and a staged dataset that fails structural
or (when `lerobot` is installed) official-API validation never replace a
validation never replace a
previously valid destination.

Code: [`lerobot/export.py`](./lerobot/export.py)
Expand Down
40 changes: 32 additions & 8 deletions examples/lerobot/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
import tempfile
from collections.abc import Sequence
from dataclasses import dataclass
from itertools import zip_longest
from pathlib import Path

import duckdb
Expand All @@ -45,7 +46,7 @@

import hflow

EXPORT_VERSION = "lerobot-export-v2"
EXPORT_VERSION = "lerobot-export-v3"

_COMMIT_SHA_RE = re.compile(r"^[0-9a-f]{40}$")

Expand Down Expand Up @@ -369,6 +370,28 @@ def _episode_rows(table: pa.Table) -> list[dict]:
return out


def _check_written_data_schema(path: Path, source_schema: pa.Schema) -> None:
"""Check the written footer without reading any frame data."""
written_schema = pq.read_schema(path)
if written_schema.equals(source_schema, check_metadata=True):
return
difference = "schema metadata"
for source_field, written_field in zip_longest(source_schema, written_schema):
if source_field is None:
difference = f"unexpected field {written_field.name!r}"
elif written_field is None:
difference = f"missing field {source_field.name!r}"
elif source_field.equals(written_field, check_metadata=True):
continue
else:
difference = (
f"field {source_field.name!r} (type, nullability, or metadata): "
f"expected {source_field}, found {written_field}"
)
break
raise ValueError(f"exported data file {path}: schema differs from source at {difference}")


def _write_v3_repository(
*,
corpus: dict,
Expand Down Expand Up @@ -579,6 +602,7 @@ def _fetch_video(cam: str, vw: dict) -> Path:
total_frames = 0
video_paths: list[Path] = []
copied_videos: set[tuple[str, int, int]] = set()
written_data_columns: set[str] = set()

for (
new_idx,
Expand All @@ -600,10 +624,10 @@ def _fetch_video(cam: str, vw: dict) -> Path:
data_frames.append(d)
frame_rows.append(d)
total_frames += length
pq.write_table(
pa.Table.from_pylist(frame_rows, schema=source_data_schema),
data_dir / f"file-{new_idx:03d}.parquet",
)
data_path = data_dir / f"file-{new_idx:03d}.parquet"
pq.write_table(pa.Table.from_pylist(frame_rows, schema=source_data_schema), data_path)
_check_written_data_schema(data_path, source_data_schema)
written_data_columns.update(source_data_schema.names)

# keep the source (chunk, file) identity in the destination path
ep_video_refs: dict[str, dict] = {}
Expand Down Expand Up @@ -655,10 +679,10 @@ def _fetch_video(cam: str, vw: dict) -> Path:
cam: corpus["info"]["features"].get(cam, {"dtype": "video", "shape": [480, 640, 3]})
for cam in camera_keys
}
numeric_features = {
data_features = {
k: v
for k, v in corpus["info"].get("features", {}).items()
if isinstance(v, dict) and v.get("dtype") == "float32"
if isinstance(v, dict) and v.get("dtype") != "video" and k in written_data_columns
}
out_info = {
"code": "LeRobotDataset/v3",
Expand All @@ -670,7 +694,7 @@ def _fetch_video(cam: str, vw: dict) -> Path:
"splits": {"train": [f"episode_{i:06d}" for i in range(len(selections))]},
"data_path": "data/chunk-{chunk_index:03d}/file-{file_index:03d}.parquet",
"video_path": "videos/{video_key}/chunk-{chunk_index:03d}/file-{file_index:03d}.mp4",
"features": {**numeric_features, **video_features},
"features": {**data_features, **video_features},
"version": 1,
}
(destination / "meta" / "info.json").write_text(json.dumps(out_info, indent=2))
Expand Down
113 changes: 113 additions & 0 deletions tests/test_lerobot_export.py
Original file line number Diff line number Diff line change
Expand Up @@ -228,6 +228,119 @@ def _provenance_json(**overrides: object) -> str:
return json.dumps({**json.loads(_provenance_meta(0)), **overrides})


def test_export_registry_matches_written_features(fake_corpus: dict, tmp_path: Path) -> None:
"""All written non-video features retain their exact source declarations."""
import pyarrow as pa
import pyarrow.parquet as pq

source_path = Path(fake_corpus["cache_dir"]) / "data/chunk-000/file-000.parquet"
source = pq.read_table(source_path)
for name in ("index", "episode_index", "frame_index"):
source = source.set_column(
source.schema.get_field_index(name), name, source[name].cast(pa.int64())
)
source = source.append_column("task_index", pa.array([0] * source.num_rows, pa.int64()))
source = source.append_column(
"next.done", pa.array([i % 2 == 0 for i in range(source.num_rows)], pa.bool_())
)
pq.write_table(source, source_path)
features = fake_corpus["info"]["features"]
for name in ("index", "episode_index", "frame_index", "task_index", "next.done"):
features[name] = {
"dtype": "bool" if name == "next.done" else "int64",
"shape": [1],
"names": [name],
}
features["unwritten.float"] = {"dtype": "float32", "shape": [1], "names": None}
features["unwritten.integer"] = {"dtype": "int64", "shape": [1], "names": None}
(tmp_path / "meta/info.json").write_text(json.dumps(fake_corpus["info"]))
manifest = _fake_manifest(
tmp_path,
[{"metadata_json": _provenance_meta(0)}, {"metadata_json": _provenance_meta(2)}],
)
destination = tmp_path / "out"

export.export(destination, manifest=manifest, camera_keys=CAMS[:1])

registry = json.loads((destination / "meta/info.json").read_text())["features"]
assert {key for key, value in registry.items() if value["dtype"] == "video"} == {CAMS[0]}
data_features = {key: value for key, value in registry.items() if value["dtype"] != "video"}
for exported_file in sorted((destination / "data").rglob("*.parquet")):
exported = pq.read_table(exported_file)
assert data_features == {name: features[name] for name in exported.column_names}
assert set(registry) == set(exported.column_names) | {CAMS[0]}
for name in ("index", "episode_index", "frame_index", "task_index"):
assert exported.schema.field(name).type == pa.int64()
assert exported.schema.field("next.done").type == pa.bool_()


@pytest.mark.parametrize(
("mismatch", "differing_field"),
[
("type", "index"),
("nullability", "index"),
("field_metadata", "index"),
("schema_metadata", "schema metadata"),
("missing_field", "action"),
("extra_field", "extra"),
("field_order", "index"),
],
)
def test_export_refuses_written_schema_mismatch(
fake_corpus: dict,
tmp_path: Path,
monkeypatch: pytest.MonkeyPatch,
mismatch: str,
differing_field: str,
) -> None:
"""A changed schema in the second data file aborts publication with a useful error."""
import pyarrow as pa
import pyarrow.parquet as pq

manifest = _fake_manifest(
tmp_path,
[{"metadata_json": _provenance_meta(0)}, {"metadata_json": _provenance_meta(2)}],
)
write_table = pq.write_table

def write_changed_schema(table: pa.Table, where: Path) -> None:
if where.parent.name == "chunk-000" and where.name == "file-001.parquet":
first_field = table.schema.field(0)
match mismatch:
case "type":
table = table.set_column(
0, first_field.name, table.column(0).cast(pa.float64())
)
case "nullability":
table = table.cast(table.schema.set(0, first_field.with_nullable(False)))
case "field_metadata":
table = table.cast(
table.schema.set(0, first_field.with_metadata({b"changed": b"true"}))
)
case "schema_metadata":
table = table.replace_schema_metadata({b"changed": b"true"})
case "missing_field":
table = table.drop_columns(["action"])
case "extra_field":
table = table.append_column("extra", pa.array([0] * table.num_rows))
case "field_order":
table = table.select(list(reversed(table.column_names)))
write_table(table, where)

monkeypatch.setattr(pq, "write_table", write_changed_schema)
destination = tmp_path / "out"
destination.mkdir()
(destination / "sentinel.txt").write_text("previous export")
with pytest.raises(ValueError) as error:
export.export(destination, manifest=manifest, camera_keys=CAMS)
message = str(error.value)
assert "data/chunk-000/file-001.parquet" in message
assert "schema differs from source" in message
assert differing_field in message
assert (destination / "sentinel.txt").read_text() == "previous export"
assert sorted(path.name for path in destination.iterdir()) == ["sentinel.txt"]


def test_export_preserves_source_data_schema(fake_corpus: dict, tmp_path: Path) -> None:
"""Renumbered rows retain source float widths, array shapes, and metadata."""
import pyarrow as pa
Expand Down
Loading