From 0099829aa72bf600ca8e987fb5a7ca054584dc0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Royeth?= Date: Wed, 19 Aug 2026 20:38:31 -0400 Subject: [PATCH 1/2] perf(mcbyte): skip mask-only work without mask evidence --- CHANGELOG.md | 1 + src/trackers/core/mcbyte/mask_association.py | 59 +++++++----- tests/core/test_mcbyte_mask_association.py | 99 ++++++++++++++++---- 3 files changed, 116 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bcd13079..375b42f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), ### 🌱 Changed +- **McByte skips mask-only association work when mask evidence is unavailable** — clear-match locking and reduced assignment are preserved, while ambiguity and isolated-candidate matrices are no longer built for the default mask-disabled path. - **McByte CMC now defaults to `cmc_downscale=6`** — this aggregate-performance default halves median CMC latency versus factor `2` on the complete 45-clip, 1280x720 SportsMOT validation split and passes the dataset-level mean/median quality criterion. The benchmark used ground-truth detections with masks disabled; 9/45 clips regressed under the previous strict per-clip gate. Pass `cmc_downscale=2` to preserve the previous conservative behavior. Generic `CMCConfig` and `BoTSORTTracker` remain at `2`. - **Mask stack moved from `trackers.core.mcbyte.masks` to `trackers.core.masks`** — SAM mask generation, Cutie propagation, and `MaskManager` reference no tracker and are not McByte-specific, so they now live beside the trackers rather than inside one. Import from `trackers.core.masks` instead ([#543](https://github.com/roboflow/trackers/pull/543)). diff --git a/src/trackers/core/mcbyte/mask_association.py b/src/trackers/core/mcbyte/mask_association.py index 286466a3..5246dc0f 100644 --- a/src/trackers/core/mcbyte/mask_association.py +++ b/src/trackers/core/mcbyte/mask_association.py @@ -231,7 +231,9 @@ def _apply_mask_similarity_boosts( remaining_detection_indices: list[int], tracklet_ids: list[int], detection_boxes: np.ndarray, - mask_output: MaskOutput | None, + masks: np.ndarray, + tracklet_mask_dict: dict[int, int], + mask_avg_prob_dict: dict[int, float], minimum_mask_average_confidence: float, minimum_mask_coverage: float, minimum_mask_fill_ratio: float, @@ -251,7 +253,7 @@ def _apply_mask_similarity_boosts( For every candidate pair, the function: 1. resolves the stable tracklet ID associated with the original row; - 2. finds the corresponding local mask index in ``mask_output``; + 2. finds the corresponding local mask index in ``tracklet_mask_dict``; 3. verifies that the mask index and average mask confidence are valid; 4. computes mask coverage and mask fill ratio for the original detection box; @@ -282,11 +284,10 @@ def _apply_mask_similarity_boosts( detection_boxes: Detection boxes in ``xyxy`` format, ordered according to the columns of the original full association matrix. Expected shape is ``(num_detections, 4)``. - mask_output: Current propagated mask output. Its - ``tracklet_mask_dict`` maps stable tracklet IDs to local mask-array - indices, while ``mask_avg_prob_dict`` stores average mask confidence - keyed by stable tracklet ID. If the output, masks, or confidence - mapping is unavailable, no scores are modified. + masks: Current propagated masks with shape ``(N, H, W)``. + tracklet_mask_dict: Mapping from stable tracklet IDs to local mask-array + indices. + mask_avg_prob_dict: Average mask confidence keyed by stable tracklet ID. minimum_mask_average_confidence: Minimum average propagated-mask confidence required before mask evidence may be used. minimum_mask_coverage: Minimum fraction of the complete visible mask @@ -297,9 +298,6 @@ def _apply_mask_similarity_boosts( Returns: None. ``conditioned_similarity`` is updated in place. """ - if mask_output is None or mask_output.masks is None or mask_output.mask_avg_prob_dict is None: - return - # Local indices in the reduced matrix. candidate_rows, candidate_columns = np.where(candidate_matrix) @@ -307,12 +305,12 @@ def _apply_mask_similarity_boosts( mask_areas: dict[int, int] = {} for local_track_index in np.unique(candidate_rows): tracklet_id = tracklet_ids[remaining_track_indices[local_track_index]] - mask_index = mask_output.tracklet_mask_dict.get(tracklet_id) - if mask_index is None or not 0 <= mask_index < mask_output.masks.shape[0]: + mask_index = tracklet_mask_dict.get(tracklet_id) + if mask_index is None or not 0 <= mask_index < masks.shape[0]: continue if mask_index not in mask_areas: - mask_areas[mask_index] = int(mask_output.masks[mask_index].astype(bool, copy=False).sum()) + mask_areas[mask_index] = int(masks[mask_index].astype(bool, copy=False).sum()) for local_track_index, local_detection_index in zip( candidate_rows, @@ -323,19 +321,19 @@ def _apply_mask_similarity_boosts( # Resolve stable tracklet ID tracklet_id = tracklet_ids[original_track_index] - mask_index = mask_output.tracklet_mask_dict.get(tracklet_id) + mask_index = tracklet_mask_dict.get(tracklet_id) if mask_index is None: continue - if not 0 <= mask_index < mask_output.masks.shape[0]: + if not 0 <= mask_index < masks.shape[0]: continue - average_confidence = mask_output.mask_avg_prob_dict.get(tracklet_id) + average_confidence = mask_avg_prob_dict.get(tracklet_id) if average_confidence is None or average_confidence < minimum_mask_average_confidence: continue metrics = _get_mask_metrics_with_visible_area( - mask_bool=mask_output.masks[mask_index].astype(bool, copy=False), + mask_bool=masks[mask_index].astype(bool, copy=False), detection_xyxy=detection_boxes[original_detection_index], visible_mask_area=mask_areas[mask_index], ) @@ -477,6 +475,22 @@ def condition_similarity_with_masks( ) ].copy() + conditioned_association = MaskConditionedAssociation( + conditioned_similarity=reduced_similarity, + locked_matches=locked_matches, + remaining_track_indices=remaining_track_indices, + remaining_detection_indices=remaining_detection_indices, + ) + + if ( + mask_output is None + or mask_output.masks is None + or mask_output.masks.shape[0] == 0 + or not mask_output.tracklet_mask_dict + or not mask_output.mask_avg_prob_dict + ): + return conditioned_association + # Ambiguity is a property of the original association situation before any # modifications, hence computed from base_similarity. ambiguous_candidates = _get_ambiguous_candidate_matrix( @@ -513,15 +527,12 @@ def condition_similarity_with_masks( remaining_detection_indices=remaining_detection_indices, tracklet_ids=tracklet_ids, detection_boxes=detection_boxes, - mask_output=mask_output, + masks=mask_output.masks, + tracklet_mask_dict=mask_output.tracklet_mask_dict, + mask_avg_prob_dict=mask_output.mask_avg_prob_dict, minimum_mask_average_confidence=minimum_mask_average_confidence, minimum_mask_coverage=minimum_mask_coverage, minimum_mask_fill_ratio=minimum_mask_fill_ratio, ) - return MaskConditionedAssociation( - conditioned_similarity=reduced_similarity, - locked_matches=locked_matches, - remaining_track_indices=remaining_track_indices, - remaining_detection_indices=remaining_detection_indices, - ) + return conditioned_association diff --git a/tests/core/test_mcbyte_mask_association.py b/tests/core/test_mcbyte_mask_association.py index e01502ab..d05bedea 100644 --- a/tests/core/test_mcbyte_mask_association.py +++ b/tests/core/test_mcbyte_mask_association.py @@ -6,6 +6,8 @@ from __future__ import annotations +from unittest.mock import patch + import numpy as np import pytest @@ -341,28 +343,87 @@ def test_mask_bonus_is_not_clamped_to_one() -> None: assert np.isclose(result.conditioned_similarity[0, 0], 1.8) -def test_missing_mask_output_keeps_ambiguous_scores_unchanged() -> None: - similarity = np.array([[0.7, 0.6]], dtype=np.float32) - - result = condition_similarity_with_masks( - similarity=similarity, - raw_iou_similarity=similarity, - tracklet_ids=[10], - detection_boxes=np.array( - [ - [0, 0, 5, 5], - [5, 5, 10, 10], - ], - dtype=np.float32, +@pytest.mark.parametrize( + "mask_output", + [ + pytest.param(None, id="missing-output"), + pytest.param( + MaskOutput(masks=None, tracklet_mask_dict={}, mask_avg_prob_dict={}), + id="missing-masks", ), - mask_output=None, - minimum_similarity=0.5, + pytest.param( + MaskOutput( + masks=np.ones((1, 10, 10), dtype=bool), + tracklet_mask_dict={10: 0}, + mask_avg_prob_dict=None, + ), + id="missing-confidence-map", + ), + pytest.param( + MaskOutput( + masks=np.zeros((0, 10, 10), dtype=bool), + tracklet_mask_dict={10: 0}, + mask_avg_prob_dict={10: 0.9}, + ), + id="zero-masks", + ), + pytest.param( + MaskOutput( + masks=np.ones((1, 10, 10), dtype=bool), + tracklet_mask_dict={}, + mask_avg_prob_dict={10: 0.9}, + ), + id="empty-tracklet-map", + ), + pytest.param( + MaskOutput( + masks=np.ones((1, 10, 10), dtype=bool), + tracklet_mask_dict={10: 0}, + mask_avg_prob_dict={}, + ), + id="empty-confidence-map", + ), + ], +) +def test_missing_mask_evidence_skips_candidate_matrix_work(mask_output: MaskOutput | None) -> None: + similarity = np.array( + [ + [0.9, 0.1, 0.0], + [0.1, 0.7, 0.6], + [0.0, 0.6, 0.7], + ], + dtype=np.float32, ) + original = similarity.copy() - np.testing.assert_array_equal( - result.conditioned_similarity, - similarity, - ) + with ( + patch("trackers.core.mcbyte.mask_association._get_ambiguous_candidate_matrix") as ambiguous_candidates, + patch("trackers.core.mcbyte.mask_association._get_isolated_candidate_matrix") as isolated_candidates, + ): + result = condition_similarity_with_masks( + similarity=similarity, + raw_iou_similarity=similarity, + tracklet_ids=[10, 20, 30], + detection_boxes=np.array( + [ + [0, 0, 5, 5], + [5, 5, 10, 10], + [10, 10, 15, 15], + ], + dtype=np.float32, + ), + mask_output=mask_output, + minimum_similarity=0.5, + enable_isolated_mask_matching=True, + ) + + ambiguous_candidates.assert_not_called() + isolated_candidates.assert_not_called() + assert result.locked_matches == [(0, 0)] + assert result.remaining_track_indices == [1, 2] + assert result.remaining_detection_indices == [1, 2] + np.testing.assert_array_equal(result.conditioned_similarity, original[1:, 1:]) + np.testing.assert_array_equal(similarity, original) def test_missing_tracklet_mask_keeps_scores_unchanged() -> None: From 55bc82fff2bcd168363bbbf8cce5a77c8e3487a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Royeth?= Date: Mon, 24 Aug 2026 14:59:49 -0400 Subject: [PATCH 2/2] refactor(mcbyte): clarify mask-association helper inputs and result construction Addresses review feedback on the mask-only skip. _apply_mask_similarity_boosts takes MaskOutput again instead of the three unpacked fields, and narrows the two optional ones (masks, mask_avg_prob_dict) inside the helper. tracklet_mask_dict is not optional, so it never needed unpacking. condition_similarity_with_masks no longer builds MaskConditionedAssociation before the boosts run. The no-evidence path builds and returns its own result, and the mask path builds it after the boosts have been applied, so the in-place mutation happens on a plain local array rather than through an already constructed frozen dataclass. Output is unchanged: both paths compared bit-identical against the previous revision over 5000 randomised cases, with the caller matrix untouched. --- src/trackers/core/mcbyte/mask_association.py | 58 +++++++++++--------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/src/trackers/core/mcbyte/mask_association.py b/src/trackers/core/mcbyte/mask_association.py index 5246dc0f..3708a037 100644 --- a/src/trackers/core/mcbyte/mask_association.py +++ b/src/trackers/core/mcbyte/mask_association.py @@ -231,9 +231,7 @@ def _apply_mask_similarity_boosts( remaining_detection_indices: list[int], tracklet_ids: list[int], detection_boxes: np.ndarray, - masks: np.ndarray, - tracklet_mask_dict: dict[int, int], - mask_avg_prob_dict: dict[int, float], + mask_output: MaskOutput, minimum_mask_average_confidence: float, minimum_mask_coverage: float, minimum_mask_fill_ratio: float, @@ -253,7 +251,7 @@ def _apply_mask_similarity_boosts( For every candidate pair, the function: 1. resolves the stable tracklet ID associated with the original row; - 2. finds the corresponding local mask index in ``tracklet_mask_dict``; + 2. finds the corresponding local mask index in ``mask_output``; 3. verifies that the mask index and average mask confidence are valid; 4. computes mask coverage and mask fill ratio for the original detection box; @@ -284,10 +282,11 @@ def _apply_mask_similarity_boosts( detection_boxes: Detection boxes in ``xyxy`` format, ordered according to the columns of the original full association matrix. Expected shape is ``(num_detections, 4)``. - masks: Current propagated masks with shape ``(N, H, W)``. - tracklet_mask_dict: Mapping from stable tracklet IDs to local mask-array - indices. - mask_avg_prob_dict: Average mask confidence keyed by stable tracklet ID. + mask_output: Current propagated mask output. Its + ``tracklet_mask_dict`` maps stable tracklet IDs to local mask-array + indices, while ``mask_avg_prob_dict`` stores average mask confidence + keyed by stable tracklet ID. Callers must have already established + that masks and confidences are present. minimum_mask_average_confidence: Minimum average propagated-mask confidence required before mask evidence may be used. minimum_mask_coverage: Minimum fraction of the complete visible mask @@ -298,6 +297,9 @@ def _apply_mask_similarity_boosts( Returns: None. ``conditioned_similarity`` is updated in place. """ + if mask_output.masks is None or mask_output.mask_avg_prob_dict is None: + return + # Local indices in the reduced matrix. candidate_rows, candidate_columns = np.where(candidate_matrix) @@ -305,12 +307,12 @@ def _apply_mask_similarity_boosts( mask_areas: dict[int, int] = {} for local_track_index in np.unique(candidate_rows): tracklet_id = tracklet_ids[remaining_track_indices[local_track_index]] - mask_index = tracklet_mask_dict.get(tracklet_id) - if mask_index is None or not 0 <= mask_index < masks.shape[0]: + mask_index = mask_output.tracklet_mask_dict.get(tracklet_id) + if mask_index is None or not 0 <= mask_index < mask_output.masks.shape[0]: continue if mask_index not in mask_areas: - mask_areas[mask_index] = int(masks[mask_index].astype(bool, copy=False).sum()) + mask_areas[mask_index] = int(mask_output.masks[mask_index].astype(bool, copy=False).sum()) for local_track_index, local_detection_index in zip( candidate_rows, @@ -321,19 +323,19 @@ def _apply_mask_similarity_boosts( # Resolve stable tracklet ID tracklet_id = tracklet_ids[original_track_index] - mask_index = tracklet_mask_dict.get(tracklet_id) + mask_index = mask_output.tracklet_mask_dict.get(tracklet_id) if mask_index is None: continue - if not 0 <= mask_index < masks.shape[0]: + if not 0 <= mask_index < mask_output.masks.shape[0]: continue - average_confidence = mask_avg_prob_dict.get(tracklet_id) + average_confidence = mask_output.mask_avg_prob_dict.get(tracklet_id) if average_confidence is None or average_confidence < minimum_mask_average_confidence: continue metrics = _get_mask_metrics_with_visible_area( - mask_bool=masks[mask_index].astype(bool, copy=False), + mask_bool=mask_output.masks[mask_index].astype(bool, copy=False), detection_xyxy=detection_boxes[original_detection_index], visible_mask_area=mask_areas[mask_index], ) @@ -475,13 +477,6 @@ def condition_similarity_with_masks( ) ].copy() - conditioned_association = MaskConditionedAssociation( - conditioned_similarity=reduced_similarity, - locked_matches=locked_matches, - remaining_track_indices=remaining_track_indices, - remaining_detection_indices=remaining_detection_indices, - ) - if ( mask_output is None or mask_output.masks is None @@ -489,7 +484,13 @@ def condition_similarity_with_masks( or not mask_output.tracklet_mask_dict or not mask_output.mask_avg_prob_dict ): - return conditioned_association + # No tracklet can receive mask evidence, so the reduced problem is final. + return MaskConditionedAssociation( + conditioned_similarity=reduced_similarity, + locked_matches=locked_matches, + remaining_track_indices=remaining_track_indices, + remaining_detection_indices=remaining_detection_indices, + ) # Ambiguity is a property of the original association situation before any # modifications, hence computed from base_similarity. @@ -527,12 +528,15 @@ def condition_similarity_with_masks( remaining_detection_indices=remaining_detection_indices, tracklet_ids=tracklet_ids, detection_boxes=detection_boxes, - masks=mask_output.masks, - tracklet_mask_dict=mask_output.tracklet_mask_dict, - mask_avg_prob_dict=mask_output.mask_avg_prob_dict, + mask_output=mask_output, minimum_mask_average_confidence=minimum_mask_average_confidence, minimum_mask_coverage=minimum_mask_coverage, minimum_mask_fill_ratio=minimum_mask_fill_ratio, ) - return conditioned_association + return MaskConditionedAssociation( + conditioned_similarity=reduced_similarity, + locked_matches=locked_matches, + remaining_track_indices=remaining_track_indices, + remaining_detection_indices=remaining_detection_indices, + )