|
9 | 9 | from hflow.batching import PlannedBatch, plan_batches, plan_batches_from_files |
10 | 10 |
|
11 | 11 |
|
12 | | -def test_fixed_count_batches_are_near_equal_bytes() -> None: |
| 12 | +@pytest.mark.parametrize( |
| 13 | + ("sizes", "item_cap"), |
| 14 | + [ |
| 15 | + pytest.param([900, 850, 500, 400, 300, 250, 200, 150, 100, 50], None, id="positive"), |
| 16 | + pytest.param([0] * 7, None, id="all-zero"), |
| 17 | + pytest.param([0] * 4, 3, id="all-zero-with-cap"), |
| 18 | + pytest.param([900, 0, 0, 0], None, id="mixed-zero"), |
| 19 | + pytest.param([0, 0], None, id="fewer-items-than-batches"), |
| 20 | + ], |
| 21 | +) |
| 22 | +def test_fixed_count_batches_are_near_equal_bytes(sizes: list[int], item_cap: int | None) -> None: |
13 | 23 | # A naive round-robin unbalances these widely varying sizes; least-loaded |
14 | | - # keeps the spread within one item. |
15 | | - item_sizes = { |
16 | | - f"ep{index:02d}": size |
17 | | - for index, size in enumerate([900, 850, 500, 400, 300, 250, 200, 150, 100, 50]) |
18 | | - } |
19 | | - batches = plan_batches(item_sizes, batch_count=3) |
20 | | - assert len(batches) == 3 |
| 24 | + # keeps the spread within one item. Zero bytes must not erase requested batches. |
| 25 | + item_sizes = {f"ep{index:02d}": size for index, size in enumerate(sizes)} |
| 26 | + batches = plan_batches(item_sizes, batch_count=3, maximum_items_per_batch=item_cap) |
| 27 | + assert len(batches) == min(3, len(item_sizes)) |
21 | 28 | totals = [batch.total_bytes for batch in batches] |
22 | 29 | assert sum(totals) == sum(item_sizes.values()) |
23 | 30 | assert max(totals) - min(totals) <= max(item_sizes.values()) |
24 | | - assert {item for batch in batches for item in batch.items} == set(item_sizes) |
| 31 | + assigned = [item for batch in batches for item in batch.items] |
| 32 | + assert sorted(assigned) == sorted(item_sizes) |
| 33 | + assert all(batch.items for batch in batches) |
| 34 | + if item_cap is not None: |
| 35 | + assert all(len(batch.items) <= item_cap for batch in batches) |
| 36 | + assert batches == plan_batches( |
| 37 | + dict(reversed(list(item_sizes.items()))), |
| 38 | + batch_count=3, |
| 39 | + maximum_items_per_batch=item_cap, |
| 40 | + ) |
25 | 41 |
|
26 | 42 |
|
27 | 43 | def test_plan_is_deterministic() -> None: |
@@ -156,13 +172,15 @@ def test_plan_from_files_forwards_stagger_validation(tmp_path: Path) -> None: |
156 | 172 | plan_batches_from_files([item], batch_count=1, stagger_interval_s=float("nan")) |
157 | 173 |
|
158 | 174 |
|
159 | | -def test_plan_from_real_files(tmp_path: Path) -> None: |
| 175 | +@pytest.mark.parametrize("sizes", [(100, 10_000), (0, 0)], ids=["positive", "zero-byte"]) |
| 176 | +def test_plan_from_real_files(tmp_path: Path, sizes: tuple[int, int]) -> None: |
160 | 177 | small = tmp_path / "small.mcap" |
161 | | - small.write_bytes(b"x" * 100) |
| 178 | + small.write_bytes(b"x" * sizes[0]) |
162 | 179 | large = tmp_path / "large.mcap" |
163 | | - large.write_bytes(b"x" * 10_000) |
| 180 | + large.write_bytes(b"x" * sizes[1]) |
164 | 181 | batches = plan_batches_from_files([small, large], batch_count=2) |
165 | | - assert batches[0].total_bytes == 10_000 |
| 182 | + assert len(batches) == 2 |
| 183 | + assert batches[0].total_bytes == sizes[1] |
166 | 184 | assert batches[0].items == (str(large),) |
167 | 185 |
|
168 | 186 |
|
|
0 commit comments