From e2ae827f1b87c4235e2e8adc3bc219227a0bbceb Mon Sep 17 00:00:00 2001 From: Kingston Date: Sun, 15 Nov 2026 03:16:40 -0800 Subject: [PATCH] refactor(testing): encode synthetic joint states with hebbian-mcap 0.1.2 The synthetic episode generator hand-rolled its JointState CDR encoder. The shared encoder now takes velocities and efforts and produces the same bytes, so the generator uses it. The one test that builds deliberately malformed joint messages uses the schema-driven ROS 2 encoder instead. TRANSFORM_BEHAVIOR_VERSION stays at 13, which has not shipped in a release; its note now names the 0.1.2 library identifier. --- docs/FORMAT.md | 2 +- pyproject.toml | 4 ++-- src/hflow/behavior.py | 2 +- src/hflow/testing.py | 35 ++++++------------------------- tests/test_checks_nan_failopen.py | 25 +++++++++++++++------- uv.lock | 18 ++++++++-------- 6 files changed, 36 insertions(+), 50 deletions(-) diff --git a/docs/FORMAT.md b/docs/FORMAT.md index c9adf7f1..e4e648c2 100644 --- a/docs/FORMAT.md +++ b/docs/FORMAT.md @@ -19,7 +19,7 @@ The single overriding rule: **a canonical episode is spec-conforming MCAP.** Eve - MCAP magic, `Header`, data section, `DataEnd`, summary section, `Footer`, closing magic, per the [MCAP spec](https://mcap.dev/spec). - `Header.profile` is the empty string `""` (the file mixes protobuf video channels with pass-through channels of arbitrary encoding, so no single profile applies). -- `Header.library` is informational only (currently `hebbian-mcap/0.1.1`). Its exact value is not part of the format contract, and no reader may key behavior off it (see [Identifier rules](#identifier-rules)). +- `Header.library` is informational only (currently `hebbian-mcap/0.1.2`). Its exact value is not part of the format contract, and no reader may key behavior off it (see [Identifier rules](#identifier-rules)). - Chunks are compressed with **zstd** by default (`"none"` is permitted). Each `Chunk` record carries `uncompressed_crc`; the `Footer` carries a summary CRC. - The summary section repeats all `Schema` and `Channel` records and contains `Statistics`, all `ChunkIndex` records, `AttachmentIndex`/`MetadataIndex` records, and `SummaryOffset` records. A canonical episode always has a complete summary; unindexed files are not canonical. - Each topic names exactly one channel. MCAP allows several channels to share a topic; a canonical episode does not, because topic-keyed reads would be ambiguous. diff --git a/pyproject.toml b/pyproject.toml index 848f6efd..38ae244f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,7 +26,7 @@ dependencies = [ "httpx2>=2.12.0", "huggingface-hub>=1.31.0,<2", "mcap>=1.4.0", - "hebbian-mcap==0.1.1", + "hebbian-mcap==0.1.2", "mcap-protobuf-support>=0.5.4", "mcap-ros2-support>=0.5.7", "numpy>=2.3.5", @@ -56,7 +56,7 @@ mediapipe = ["mediapipe>=1.0.1"] # dependency is real, and confined to the one check that needs it. Headless # because a data pipeline runs where there is no display. motion = ["opencv-python-headless>=5.0.0.93"] -video = ["av>=18.1.0", "hebbian-mcap[video]==0.1.1"] +video = ["av>=18.1.0", "hebbian-mcap[video]==0.1.2"] native-build = ["Cython>=3.3.0", "setuptools>=84.0.0"] openai = ["openai>=3.3.1"] diff --git a/src/hflow/behavior.py b/src/hflow/behavior.py index dc0dc874..8f8b81c7 100644 --- a/src/hflow/behavior.py +++ b/src/hflow/behavior.py @@ -109,7 +109,7 @@ # library identifier, schema/channel placement, and compressed bytes can change. # Regenerate derived outputs rather than mixing both writers under one identity. # -# "13": video messages are encoded and rewritten by hebbian-mcap 0.1.1. The +# "13": video messages are encoded and rewritten by hebbian-mcap (0.1.2). The # CompressedVideo schema record now carries protobuf ``json_name`` entries, the # library identifier names the new release, and a pass-through protobuf message # that gains an access unit delimiter keeps its original field order. Decoded diff --git a/src/hflow/testing.py b/src/hflow/testing.py index 465e7edd..bc491bfb 100644 --- a/src/hflow/testing.py +++ b/src/hflow/testing.py @@ -44,6 +44,7 @@ from dataclasses import dataclass from pathlib import Path +from hebbian_mcap import encode_joint_state from mcap.writer import Writer from hflow._field_guards import require_float, require_int @@ -180,10 +181,6 @@ def write_i32(self, value: int) -> None: self.align(4) self._body += struct.pack(" None: - self.align(8) - self._body += struct.pack(" None: encoded_value = value.encode("utf-8") # The uint32 length prefix includes the trailing NUL terminator. @@ -203,26 +200,6 @@ def _write_header(encoder: _CdrEncoder, stamp_ns: int, frame_id: str) -> None: encoder.write_string(frame_id) -def _encode_joint_state( - stamp_ns: int, - frame_id: str, - joint_names: list[str], - positions: list[float], - velocities: list[float], - efforts: list[float], -) -> bytes: - encoder = _CdrEncoder() - _write_header(encoder, stamp_ns, frame_id) - encoder.write_u32(len(joint_names)) - for joint_name in joint_names: - encoder.write_string(joint_name) - for float_sequence in (positions, velocities, efforts): - encoder.write_u32(len(float_sequence)) - for element in float_sequence: - encoder.write_f64(element) - return encoder.payload() - - def _encode_compressed_image(stamp_ns: int, frame_id: str, jpeg_bytes: bytes) -> bytes: encoder = _CdrEncoder() _write_header(encoder, stamp_ns, frame_id) @@ -690,11 +667,11 @@ def _build_joint_messages(spec: SyntheticEpisodeSpec) -> list[tuple[int, bytes]] JOINT_SINE_AMPLITUDE_RAD * 2.0 * math.pi * frequency_hz * math.cos(angle_rad) ) log_time_ns = _message_log_time_ns(spec, message_index, spec.joint_hz) - payload = _encode_joint_state( - stamp_ns=log_time_ns, - frame_id="base", - joint_names=joint_names, - positions=positions, + payload = encode_joint_state( + log_time_ns, + "base", + joint_names, + positions, velocities=velocities, efforts=[0.0] * spec.joint_count, ) diff --git a/tests/test_checks_nan_failopen.py b/tests/test_checks_nan_failopen.py index 66fa29cd..2f9a26bd 100644 --- a/tests/test_checks_nan_failopen.py +++ b/tests/test_checks_nan_failopen.py @@ -20,6 +20,7 @@ import pytest from mcap.writer import Writer +from mcap_ros2.writer import serialize_dynamic from hflow import Episode from hflow.checks import ( @@ -34,7 +35,6 @@ from hflow.testing import ( _JOINT_STATE_SCHEMA_TEXT, JOINT_STATE_SCHEMA_NAME, - _encode_joint_state, ) TOPIC = "/joint_states" @@ -48,6 +48,9 @@ def _joint_states_episode( *, omit_positions: bool = False, ) -> Episode: + encode_joint_state = serialize_dynamic(JOINT_STATE_SCHEMA_NAME, _JOINT_STATE_SCHEMA_TEXT)[ + JOINT_STATE_SCHEMA_NAME + ] with path.open("wb") as stream: writer = Writer(stream) writer.start(profile="ros2", library="test-546") @@ -63,13 +66,19 @@ def _joint_states_episode( METADATA_RECORD_EPISODE, {"task": "nan-failopen", "embodiment": "probe"} ) for sequence, (stamp_ns, position) in enumerate(zip(stamps_ns, positions, strict=True)): - payload = _encode_joint_state( - stamp_ns, - "probe", - ["j0"], - [] if omit_positions else [position], - [], - [], + # A message naming a joint without a position is malformed on purpose, + # so it is built with the schema-driven encoder, which does not check it. + payload = encode_joint_state( + { + "header": { + "stamp": {"sec": stamp_ns // NS_PER_S, "nanosec": stamp_ns % NS_PER_S}, + "frame_id": "probe", + }, + "name": ["j0"], + "position": [] if omit_positions else [position], + "velocity": [], + "effort": [], + } ) writer.add_message( channel_id=channel_id, diff --git a/uv.lock b/uv.lock index f1e85f0f..3a0b5dda 100644 --- a/uv.lock +++ b/uv.lock @@ -1222,15 +1222,15 @@ wheels = [ [[package]] name = "hebbian-mcap" -version = "0.1.1" +version = "0.1.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/36/00/93440d933a7b0f6238cfbfc15ecaa63687bb93a735de1f5c68961a779b19/hebbian_mcap-0.1.1.tar.gz", hash = "sha256:89c41bfaedea4dc78f5c21015538b5ae19ee6ed764443b2fd3b5bcc636976b17", size = 45064, upload-time = "2026-11-15T10:28:14.103Z" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/e5/d61bfc133be570c2f91869b20650148edc87c3debd0f4fd3acfcf9c2c050/hebbian_mcap-0.1.2.tar.gz", hash = "sha256:9fc55a9b450cf7c40efb657e194350c08317f2511491f578ffa36834cc8dcd42", size = 45298, upload-time = "2026-11-15T11:05:18.119Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/a3/6d9e987cc4c6f6b5c61ac1beed8df46db31c03b1b935591244dbd80ee22b/hebbian_mcap-0.1.1-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:9ab2dd5c1ff0b665968ffa08dbc122a49778a5d303e2bc40c12a1953432e737f", size = 634284, upload-time = "2026-11-15T10:28:04.531Z" }, - { url = "https://files.pythonhosted.org/packages/48/c8/212378fe1cd2fb9030a50032e0f3a43131965b20cd84d615187776d7b812/hebbian_mcap-0.1.1-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:cc319552d6514b5210a14ce3099cb07b6804e7bf5e20d99fef88664439bca213", size = 578239, upload-time = "2026-11-15T10:28:07.776Z" }, - { url = "https://files.pythonhosted.org/packages/21/63/1bb0a63aac128c9eb627cd255ad2a9dac9100efb3ed8450aec5d35344634/hebbian_mcap-0.1.1-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:8c4289507648799032a84804a4b13c47ea5c5ff606cb78aa4ef3a2664a5df342", size = 709197, upload-time = "2026-11-15T10:28:09.51Z" }, - { url = "https://files.pythonhosted.org/packages/18/cc/0ea8816b7c8c97f661d5af6016cb4f9878dc31014fe6129ab1cc70c97332/hebbian_mcap-0.1.1-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b596562a8be241de0ab4e404f9859e43640ade2ff270558fad7ebef78540d1f0", size = 708167, upload-time = "2026-11-15T10:28:11.573Z" }, - { url = "https://files.pythonhosted.org/packages/64/17/ec82211116102e6a1fd35daa07ca01f16c004dee3976d0a1d6d947fe22f1/hebbian_mcap-0.1.1-cp311-abi3-win_amd64.whl", hash = "sha256:a7704c336cd77ebcc51c3b30ef4dc7b97e9b0c767a60b56d951f39c980833fe7", size = 507248, upload-time = "2026-11-15T10:28:12.98Z" }, + { url = "https://files.pythonhosted.org/packages/16/57/5b8bfaef8e1c903549b5c193985c6edbc6fca89a5bdc0f8434f04f096736/hebbian_mcap-0.1.2-cp311-abi3-macosx_10_12_x86_64.whl", hash = "sha256:3bb88c08b456295235b8d217a3870abaf93e4757ce52e3771bf93b2a93307159", size = 634854, upload-time = "2026-11-15T11:05:12.453Z" }, + { url = "https://files.pythonhosted.org/packages/3b/8e/1b0d85d8052b9f2c1d8d7182856e7c172645afda29b123860a15d9ac5178/hebbian_mcap-0.1.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:ecdb539053bb6b87680f9ec8557066a42329d8b2d0ecfc4e0f89413382466e24", size = 578988, upload-time = "2026-11-15T11:05:13.85Z" }, + { url = "https://files.pythonhosted.org/packages/ab/6d/cda444ecd4a3e8c69b95e2262722640d3a4c4fe4529d18125da4e19f72f7/hebbian_mcap-0.1.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:eab5c0036c720050749293ff480176f4e40c5af9241534f6e2caa295656095c7", size = 709811, upload-time = "2026-11-15T11:05:14.893Z" }, + { url = "https://files.pythonhosted.org/packages/32/d7/56e4b2e17852d7ad54cc9503848e43fafe2c2cc03f17e3f4ac78f2d277e0/hebbian_mcap-0.1.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7cecd93764aa28e10d8dcd321223827bdeafec93415c3e3ef09d1d1d59296c31", size = 708756, upload-time = "2026-11-15T11:05:15.85Z" }, + { url = "https://files.pythonhosted.org/packages/2a/6d/3c1484a700eb50ee344999c1dbbbf6c0cd080c2ce201e5d402363707fef2/hebbian_mcap-0.1.2-cp311-abi3-win_amd64.whl", hash = "sha256:91fb86c1992e4c399731657d20d86659ef52793b0f4bcf70fe705aebbbce0864", size = 507980, upload-time = "2026-11-15T11:05:17.172Z" }, ] [package.optional-dependencies] @@ -1339,8 +1339,8 @@ requires-dist = [ { name = "av", marker = "extra == 'video'", specifier = ">=18.1.0" }, { name = "cython", marker = "extra == 'native-build'", specifier = ">=3.3.0" }, { name = "duckdb", specifier = ">=2.0.0" }, - { name = "hebbian-mcap", specifier = "==0.1.1" }, - { name = "hebbian-mcap", extras = ["video"], marker = "extra == 'video'", specifier = "==0.1.1" }, + { name = "hebbian-mcap", specifier = "==0.1.2" }, + { name = "hebbian-mcap", extras = ["video"], marker = "extra == 'video'", specifier = "==0.1.2" }, { name = "httpx", specifier = ">=0.28.1,<1" }, { name = "httpx2", specifier = ">=2.12.0" }, { name = "huggingface-hub", specifier = ">=1.31.0,<2" },