diff --git a/src/trackers/eval/evaluate.py b/src/trackers/eval/evaluate.py index 0c6fdb29..c45f6487 100644 --- a/src/trackers/eval/evaluate.py +++ b/src/trackers/eval/evaluate.py @@ -338,6 +338,27 @@ def _discover_sequences( return sorted(p.parent.parent.name for p in gt_dir.glob("*/gt/gt.txt") if not p.name.startswith(".")) +def _ground_truth_path( + gt_dir: Path, + seq_name: str, + data_format: Literal["flat", "mot"], +) -> Path: + """Get the ground truth file path for a sequence. + + Args: + gt_dir: Ground truth directory (direct parent of sequences). + seq_name: Sequence name. + data_format: Directory format. + + Returns: + Path to the sequence's ground truth file, which is not guaranteed to exist. + """ + if data_format == "flat": + return gt_dir / f"{seq_name}.txt" + # MOT format: gt_dir/{seq}/gt/gt.txt + return gt_dir / seq_name / "gt" / "gt.txt" + + def _get_paths( gt_dir: Path, tracker_dir: Path, @@ -355,16 +376,8 @@ def _get_paths( Returns: Tuple of (gt_path, tracker_path). """ - if data_format == "flat": - gt_path = gt_dir / f"{seq_name}.txt" - else: - # MOT format: gt_dir/{seq}/gt/gt.txt - gt_path = gt_dir / seq_name / "gt" / "gt.txt" - # Tracker files are always flat: tracker_dir/{seq}.txt - tracker_path = tracker_dir / f"{seq_name}.txt" - - return gt_path, tracker_path + return _ground_truth_path(gt_dir, seq_name, data_format), tracker_dir / f"{seq_name}.txt" def _parse_seqmap(seqmap_path: str | Path) -> list[str]: diff --git a/src/trackers/tune/tuner.py b/src/trackers/tune/tuner.py index c5c705ec..bf8165d7 100644 --- a/src/trackers/tune/tuner.py +++ b/src/trackers/tune/tuner.py @@ -17,7 +17,7 @@ import supervision as sv from trackers.core.base import BaseTracker -from trackers.eval.evaluate import evaluate_mot_sequences +from trackers.eval.evaluate import _detect_format, _ground_truth_path, evaluate_mot_sequences from trackers.eval.results import BenchmarkResult from trackers.io.frames import load_mot_frame_image from trackers.io.mot import _mot_frame_to_detections, _MOTOutput, load_mot_file @@ -201,7 +201,8 @@ def _validate_sequence_files(self) -> None: """Validate that every selected sequence has required MOT files. This performs eager filesystem validation so configuration errors are reported during tuner initialization - rather than later during trial execution. + rather than later during trial execution. Ground truth is accepted in either the flat `{seq}.txt` layout or the + MOT `{seq}/gt/gt.txt` layout, matching what `evaluate_mot_sequences` detects. """ missing_detection_files = [ str(self._detections_dir / f"{seq_name}.txt") @@ -213,11 +214,14 @@ def _validate_sequence_files(self) -> None: "Missing detection files for selected sequences: " + ", ".join(missing_detection_files) ) - missing_gt_files = [ - str(self._gt_dir / f"{seq_name}.txt") - for seq_name in self._sequences - if not (self._gt_dir / f"{seq_name}.txt").is_file() - ] + # Ground truth may use either layout that `evaluate_mot_sequences` accepts, so validation + # resolves paths the same way rather than assuming the flat one. + gt_format = _detect_format(self._gt_dir) + missing_gt_files = [] + for seq_name in self._sequences: + gt_path = _ground_truth_path(self._gt_dir, seq_name, gt_format) + if not gt_path.is_file(): + missing_gt_files.append(str(gt_path)) if missing_gt_files: raise FileNotFoundError("Missing ground-truth files for selected sequences: " + ", ".join(missing_gt_files)) diff --git a/tests/tune/test_tuner.py b/tests/tune/test_tuner.py index 4deaeb3a..5508735e 100644 --- a/tests/tune/test_tuner.py +++ b/tests/tune/test_tuner.py @@ -257,6 +257,49 @@ def test_objective_normalized_to_uppercase(self, tmp_path: Path) -> None: assert tuner._objective_metric == "MOTA" +class TestTunerGroundTruthLayout: + """Ground-truth validation must accept every layout evaluation accepts. + + `Tuner` hands `gt_dir` straight to `evaluate_mot_sequences`, which auto-detects the flat `{seq}.txt` layout and the + MOT `{seq}/gt/gt.txt` layout that downloaded datasets ship in. Eager validation has to agree, or it rejects trees + that would evaluate fine. + """ + + def test_accepts_mot_layout_ground_truth(self, tmp_path: Path) -> None: + """A MOT-layout ground-truth tree initializes instead of being reported as missing.""" + det_dir = tmp_path / "det" + det_dir.mkdir() + (det_dir / "seq1.txt").write_text(_MOT_LINE) + gt_dir = tmp_path / "gt" + (gt_dir / "seq1" / "gt").mkdir(parents=True) + (gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE) + + tuner = Tuner("bytetrack", gt_dir, det_dir) + + assert tuner._sequences == ["seq1"] + + def test_reports_missing_ground_truth_at_mot_layout_path(self, tmp_path: Path) -> None: + """A sequence missing from a MOT-layout tree is reported at its MOT path.""" + det_dir = tmp_path / "det" + det_dir.mkdir() + (det_dir / "seq1.txt").write_text(_MOT_LINE) + (det_dir / "seq2.txt").write_text(_MOT_LINE) + gt_dir = tmp_path / "gt" + (gt_dir / "seq1" / "gt").mkdir(parents=True) + (gt_dir / "seq1" / "gt" / "gt.txt").write_text(_MOT_LINE) + + with pytest.raises(FileNotFoundError, match=r"seq2.*gt.*gt\.txt"): + Tuner("bytetrack", gt_dir, det_dir) + + def test_still_reports_missing_flat_ground_truth(self, tmp_path: Path) -> None: + """The flat layout keeps reporting missing files at the flat path.""" + gt_dir, det_dir = _setup_dirs(tmp_path) + (det_dir / "seq2.txt").write_text(_MOT_LINE) + + with pytest.raises(FileNotFoundError, match=r"seq2\.txt"): + Tuner("bytetrack", gt_dir, det_dir) + + class TestTunerSeed: def test_create_optuna_study_uses_seeded_sampler(self) -> None: with patch.object(optuna, "create_study", wraps=optuna.create_study) as mock_create: