Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions src/trackers/eval/evaluate.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,27 @@ def _discover_sequences(
return sorted(p.parent.parent.name for p in gt_dir.glob("*/gt/gt.txt") if not p.name.startswith("."))


def _ground_truth_path(
gt_dir: Path,
seq_name: str,
data_format: Literal["flat", "mot"],
) -> Path:
"""Get the ground truth file path for a sequence.

Args:
gt_dir: Ground truth directory (direct parent of sequences).
seq_name: Sequence name.
data_format: Directory format.

Returns:
Path to the sequence's ground truth file, which is not guaranteed to exist.
"""
if data_format == "flat":
return gt_dir / f"{seq_name}.txt"
# MOT format: gt_dir/{seq}/gt/gt.txt
return gt_dir / seq_name / "gt" / "gt.txt"


def _get_paths(
gt_dir: Path,
tracker_dir: Path,
Expand All @@ -355,16 +376,8 @@ def _get_paths(
Returns:
Tuple of (gt_path, tracker_path).
"""
if data_format == "flat":
gt_path = gt_dir / f"{seq_name}.txt"
else:
# MOT format: gt_dir/{seq}/gt/gt.txt
gt_path = gt_dir / seq_name / "gt" / "gt.txt"

# Tracker files are always flat: tracker_dir/{seq}.txt
tracker_path = tracker_dir / f"{seq_name}.txt"

return gt_path, tracker_path
return _ground_truth_path(gt_dir, seq_name, data_format), tracker_dir / f"{seq_name}.txt"


def _parse_seqmap(seqmap_path: str | Path) -> list[str]:
Expand Down
18 changes: 11 additions & 7 deletions src/trackers/tune/tuner.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
import supervision as sv

from trackers.core.base import BaseTracker
from trackers.eval.evaluate import evaluate_mot_sequences
from trackers.eval.evaluate import _detect_format, _ground_truth_path, evaluate_mot_sequences
from trackers.eval.results import BenchmarkResult
from trackers.io.frames import load_mot_frame_image
from trackers.io.mot import _mot_frame_to_detections, _MOTOutput, load_mot_file
Expand Down Expand Up @@ -201,7 +201,8 @@ def _validate_sequence_files(self) -> None:
"""Validate that every selected sequence has required MOT files.

This performs eager filesystem validation so configuration errors are reported during tuner initialization
rather than later during trial execution.
rather than later during trial execution. Ground truth is accepted in either the flat `{seq}.txt` layout or the
MOT `{seq}/gt/gt.txt` layout, matching what `evaluate_mot_sequences` detects.
"""
missing_detection_files = [
str(self._detections_dir / f"{seq_name}.txt")
Expand All @@ -213,11 +214,14 @@ def _validate_sequence_files(self) -> None:
"Missing detection files for selected sequences: " + ", ".join(missing_detection_files)
)

missing_gt_files = [
str(self._gt_dir / f"{seq_name}.txt")
for seq_name in self._sequences
if not (self._gt_dir / f"{seq_name}.txt").is_file()
]
# Ground truth may use either layout that `evaluate_mot_sequences` accepts, so validation
# resolves paths the same way rather than assuming the flat one.
gt_format = _detect_format(self._gt_dir)
missing_gt_files = []
for seq_name in self._sequences:
gt_path = _ground_truth_path(self._gt_dir, seq_name, gt_format)
if not gt_path.is_file():
missing_gt_files.append(str(gt_path))
if missing_gt_files:
raise FileNotFoundError("Missing ground-truth files for selected sequences: " + ", ".join(missing_gt_files))

Expand Down
43 changes: 43 additions & 0 deletions tests/tune/test_tuner.py
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,49 @@ def test_objective_normalized_to_uppercase(self, tmp_path: Path) -> None:
assert tuner._objective_metric == "MOTA"


class TestTunerGroundTruthLayout:
"""Ground-truth validation must accept every layout evaluation accepts.

`Tuner` hands `gt_dir` straight to `evaluate_mot_sequences`, which auto-detects the flat `{seq}.txt` layout and the
MOT `{seq}/gt/gt.txt` layout that downloaded datasets ship in. Eager validation has to agree, or it rejects trees
that would evaluate fine.
"""

def test_accepts_mot_layout_ground_truth(self, tmp_path: Path) -> None:
"""A MOT-layout ground-truth tree initializes instead of being reported as missing."""
det_dir = tmp_path / "det"
det_dir.mkdir()
(det_dir / "seq1.txt").write_text(_MOT_LINE)
gt_dir = tmp_path / "gt"
(gt_dir / "seq1" / "gt").mkdir(parents=True)
(gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE)

tuner = Tuner("bytetrack", gt_dir, det_dir)

assert tuner._sequences == ["seq1"]

def test_reports_missing_ground_truth_at_mot_layout_path(self, tmp_path: Path) -> None:
"""A sequence missing from a MOT-layout tree is reported at its MOT path."""
det_dir = tmp_path / "det"
det_dir.mkdir()
(det_dir / "seq1.txt").write_text(_MOT_LINE)
(det_dir / "seq2.txt").write_text(_MOT_LINE)
gt_dir = tmp_path / "gt"
(gt_dir / "seq1" / "gt").mkdir(parents=True)
(gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE)

with pytest.raises(FileNotFoundError, match=r"seq2.*gt.*gt\.txt"):
Tuner("bytetrack", gt_dir, det_dir)

def test_still_reports_missing_flat_ground_truth(self, tmp_path: Path) -> None:
"""The flat layout keeps reporting missing files at the flat path."""
gt_dir, det_dir = _setup_dirs(tmp_path)
(det_dir / "seq2.txt").write_text(_MOT_LINE)

with pytest.raises(FileNotFoundError, match=r"seq2\.txt"):
Tuner("bytetrack", gt_dir, det_dir)


class TestTunerSeed:
def test_create_optuna_study_uses_seeded_sampler(self) -> None:
with patch.object(optuna, "create_study", wraps=optuna.create_study) as mock_create:
Expand Down