Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion docs/ARCHITECTURE.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,9 @@ On-disk identifiers (metadata record names, channel naming) are neutral and form

The `hebbian-mcap` dependency owns generic grouping and MCAP serialization. HFlow
owns group selection, derived targets, episode schemas, transformations and provenance.
The shared package also owns H.264 inspection and optional camera-to-MP4 export;
The shared package also owns `CompressedVideo` encoding, reading and payload
replacement for protobuf and ROS 2 CDR channels, H.264 inspection and optional
camera-to-MP4 export;
`hflow.mcap_video` adapts HFlow media limits to that exporter. Codec dependencies
remain behind the `video` extra.
We maintain both repositories; coordinated changes to `mcap-tools` are welcome.
Expand Down
2 changes: 1 addition & 1 deletion docs/FORMAT.md
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ The single overriding rule: **a canonical episode is spec-conforming MCAP.** Eve

- MCAP magic, `Header`, data section, `DataEnd`, summary section, `Footer`, closing magic, per the [MCAP spec](https://mcap.dev/spec).
- `Header.profile` is the empty string `""` (the file mixes protobuf video channels with pass-through channels of arbitrary encoding, so no single profile applies).
- `Header.library` is informational only (currently `hebbian-mcap/0.1.0`). Its exact value is not part of the format contract, and no reader may key behavior off it (see [Identifier rules](#identifier-rules)).
- `Header.library` is informational only (currently `hebbian-mcap/0.1.1`). Its exact value is not part of the format contract, and no reader may key behavior off it (see [Identifier rules](#identifier-rules)).
- Chunks are compressed with **zstd** by default (`"none"` is permitted). Each `Chunk` record carries `uncompressed_crc`; the `Footer` carries a summary CRC.
- The summary section repeats all `Schema` and `Channel` records and contains `Statistics`, all `ChunkIndex` records, `AttachmentIndex`/`MetadataIndex` records, and `SummaryOffset` records. A canonical episode always has a complete summary; unindexed files are not canonical.
- Each topic names exactly one channel. MCAP allows several channels to share a topic; a canonical episode does not, because topic-keyed reads would be ambiguous.
Expand Down
6 changes: 3 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,11 @@ keywords = [
requires-python = ">=3.11"
dependencies = [
"duckdb>=2.0.0",
"foxglove-schemas-protobuf>=0.4.0",
"httpx>=0.28.1,<1",
"httpx2>=2.12.0",
"huggingface-hub>=1.31.0,<2",
"mcap>=1.4.0",
"hebbian-mcap==0.1.0",
"hebbian-mcap==0.1.1",
"mcap-protobuf-support>=0.5.4",
"mcap-ros2-support>=0.5.7",
"numpy>=2.3.5",
Expand Down Expand Up @@ -57,7 +56,7 @@ mediapipe = ["mediapipe>=1.0.1"]
# dependency is real, and confined to the one check that needs it. Headless
# because a data pipeline runs where there is no display.
motion = ["opencv-python-headless>=5.0.0.93"]
video = ["av>=18.1.0", "hebbian-mcap[video]==0.1.0"]
video = ["av>=18.1.0", "hebbian-mcap[video]==0.1.1"]
native-build = ["Cython>=3.3.0", "setuptools>=84.0.0"]
openai = ["openai>=3.3.1"]

Expand All @@ -72,6 +71,7 @@ hflow = "hflow.cli:main"
[dependency-groups]
dev = [
"av>=18.1.0", # the video extra, exercised by timestamp-preserving camera export tests
"foxglove-schemas-protobuf>=0.4.0", # tests build foxglove messages to feed readers
"Cython>=3.3.0", # the native-build extra, present so its real build path is tested
"hflow-server", # the workspace API package, present in dev so the suite tests it
"obstore>=0.11.1", # the bucket extra, present in dev so the suite tests it
Expand Down
8 changes: 7 additions & 1 deletion src/hflow/behavior.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,4 +108,10 @@
# hebbian-mcap. Message semantics and group isolation are preserved, but the
# library identifier, schema/channel placement, and compressed bytes can change.
# Regenerate derived outputs rather than mixing both writers under one identity.
TRANSFORM_BEHAVIOR_VERSION: str = "12"
#
# "13": video messages are encoded and rewritten by hebbian-mcap 0.1.1. The
# CompressedVideo schema record now carries protobuf ``json_name`` entries, the
# library identifier names the new release, and a pass-through protobuf message
# that gains an access unit delimiter keeps its original field order. Decoded
# message content is unchanged.
TRANSFORM_BEHAVIOR_VERSION: str = "13"
41 changes: 20 additions & 21 deletions src/hflow/doctor.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,13 @@
"""

import math
from collections.abc import Callable, Iterator
from collections.abc import Iterator
from dataclasses import dataclass, field
from enum import StrEnum
from pathlib import Path
from typing import Any

from hebbian_mcap import CompressedVideoCodec
from mcap.reader import make_reader
from mcap.records import Channel, Schema

Expand Down Expand Up @@ -190,7 +191,7 @@ def _check_video_payload(


class VideoEncodingUnsupported(ValueError):
"""No decoder factory in the supported set handles this message encoding.
"""The channel's message encoding or schema is outside the supported set.

A subclass of ``ValueError`` because every existing caller treats
resolution failure as a ``ValueError``; the distinct type lets the video
Expand All @@ -200,19 +201,17 @@ class VideoEncodingUnsupported(ValueError):
"""


def _resolve_video_decoder(topic: str, channel: Channel, schema: Schema) -> Callable[[bytes], Any]:
def _resolve_video_codec(topic: str, channel: Channel, schema: Schema) -> CompressedVideoCodec:
"""Resolve either supported encoded-video payload representation."""
from mcap_protobuf.decoder import DecoderFactory as ProtobufDecoderFactory
from mcap_ros2.decoder import DecoderFactory as Ros2DecoderFactory

for factory in (Ros2DecoderFactory(), ProtobufDecoderFactory()):
decoder = factory.decoder_for(channel.message_encoding, schema)
if decoder is not None:
return decoder
raise VideoEncodingUnsupported(
f"video topic {topic!r} has message encoding {channel.message_encoding!r} "
"that no available decoder handles"
)
try:
return CompressedVideoCodec(
channel.message_encoding, schema.encoding, schema.name, schema.data
)
except ValueError as error:
raise VideoEncodingUnsupported(
f"video topic {topic!r} has message encoding {channel.message_encoding!r} "
f"that no available decoder handles ({error})"
) from error


def _unreadable_message(stream: Any, error: Exception) -> str:
Expand Down Expand Up @@ -503,7 +502,7 @@ def iter_all_messages() -> Iterator[tuple[int, int, bytes]]:
video_log_times: dict[int, list[int]] = {}
video_keyframes: dict[int, list[bool | None]] = {}
try:
video_decoders: dict[int, Callable[[bytes], Any]] = {}
video_codecs: dict[int, CompressedVideoCodec] = {}

for channel_id, log_time, payload in iter_all_messages():
previous = last_log_time_by_channel.get(channel_id)
Expand All @@ -523,12 +522,12 @@ def iter_all_messages() -> Iterator[tuple[int, int, bytes]]:
continue
message_index = video_message_counts.get(channel_id, 0)
video_message_counts[channel_id] = message_index + 1
decoder = video_decoders.get(channel_id)
if decoder is None:
codec = video_codecs.get(channel_id)
if codec is None:
channel = summary.channels[channel_id]
schema = summary.schemas[channel.schema_id]
try:
decoder = _resolve_video_decoder(channel.topic, channel, schema)
codec = _resolve_video_codec(channel.topic, channel, schema)
except VideoEncodingUnsupported as error:
video_check_skipped_topics.add(topics_by_channel_id[channel_id])
collector.add(
Expand All @@ -538,14 +537,14 @@ def iter_all_messages() -> Iterator[tuple[int, int, bytes]]:
"cannot run on this topic (the file bytes are not implicated)",
)
continue
video_decoders[channel_id] = decoder
decoded = decoder(payload)
video_codecs[channel_id] = codec
decoded = codec.decode(payload)
is_keyframe = _check_video_payload(
collector,
topics_by_channel_id[channel_id],
message_index,
decoded.format,
bytes(decoded.data),
decoded.data,
is_first_message=message_index == 0,
)
video_log_times.setdefault(channel_id, []).append(log_time)
Expand Down
16 changes: 5 additions & 11 deletions src/hflow/importers/lerobot.py
Original file line number Diff line number Diff line change
Expand Up @@ -1276,10 +1276,8 @@ def _feature_rows(feature_name: str) -> list | None:
landing_relative_key = _landing_relative_key(episode_index)
output_file_name = Path(landing_relative_key).name

from foxglove_schemas_protobuf.CompressedVideo_pb2 import CompressedVideo
from mcap_protobuf.schema import build_file_descriptor_set
from hebbian_mcap import COMPRESSED_VIDEO_SCHEMA, encode_compressed_video

video_schema_data = build_file_descriptor_set(CompressedVideo).SerializeToString()
state_schema_name = "lerobot_msgs/msg/State"
action_schema_name = "lerobot_msgs/msg/Action"
state_schema_text = f"float32[{numeric_schemas['observation.state'].dim}] position"
Expand All @@ -1296,7 +1294,7 @@ def _feature_rows(feature_name: str) -> list | None:
mcap_writer.start(profile="", library="hflow LeRobot source adapter")

video_schema_id = mcap_writer.register_schema(
"foxglove.CompressedVideo", "protobuf", video_schema_data
"foxglove.CompressedVideo", "protobuf", COMPRESSED_VIDEO_SCHEMA
)
state_schema_id = mcap_writer.register_schema(
state_schema_name, "ros2msg", state_schema_data
Expand Down Expand Up @@ -1328,16 +1326,12 @@ def _feature_rows(feature_name: str) -> list | None:
)
for camera_key in camera_keys:
access_units, _presentation_timestamps = video_data_by_camera[camera_key]
video_message = CompressedVideo()
video_message.timestamp.seconds = log_time_ns // NANOSECONDS_PER_SECOND
video_message.timestamp.nanos = log_time_ns % NANOSECONDS_PER_SECOND
video_message.frame_id = camera_key
video_message.data = access_units[frame_index]
video_message.format = "h264"
mcap_writer.add_message(
channel_id=video_channel_ids[camera_key],
log_time=log_time_ns,
data=video_message.SerializeToString(),
data=encode_compressed_video(
log_time_ns, camera_key, access_units[frame_index], "h264"
),
publish_time=log_time_ns,
sequence=frame_index,
)
Expand Down
16 changes: 5 additions & 11 deletions src/hflow/importers/video.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,8 @@
from fractions import Fraction
from pathlib import Path

from foxglove_schemas_protobuf.CompressedVideo_pb2 import CompressedVideo
from hebbian_mcap import COMPRESSED_VIDEO_SCHEMA, encode_compressed_video
from mcap.writer import Writer
from mcap_protobuf.schema import build_file_descriptor_set

from hflow._field_guards import (
require_finite_float,
Expand Down Expand Up @@ -374,7 +373,7 @@ def _import_inspected_video(
schema_id = writer.register_schema(
name=CANONICAL_VIDEO_SCHEMA_NAME,
encoding="protobuf",
data=build_file_descriptor_set(CompressedVideo).SerializeToString(),
data=COMPRESSED_VIDEO_SCHEMA,
)
channel_id = writer.register_channel(
topic=f"/{config.camera_name}/compressed",
Expand Down Expand Up @@ -408,19 +407,14 @@ def _import_inspected_video(
)
for frame_index, access_unit in enumerate(access_units):
timestamp_ns = _sample_timestamp_ns(config, frame_index)
message = CompressedVideo()
message.timestamp.seconds, message.timestamp.nanos = divmod(
timestamp_ns, NANOSECONDS_PER_SECOND
)
message.frame_id = config.camera_name
message.format = "h264"
message.data = access_unit.data
writer.add_message(
channel_id=channel_id,
log_time=timestamp_ns,
publish_time=timestamp_ns,
sequence=frame_index,
data=message.SerializeToString(),
data=encode_compressed_video(
timestamp_ns, config.camera_name, access_unit.data, "h264"
),
)
writer.finish()
# A hard link is an atomic create-if-absent on the same filesystem;
Expand Down
Loading
Loading