diff --git a/src/trackers/eval/evaluate.py b/src/trackers/eval/evaluate.py index 0c6fdb29..bfd81885 100644 --- a/src/trackers/eval/evaluate.py +++ b/src/trackers/eval/evaluate.py @@ -58,7 +58,9 @@ def evaluate_mot_sequence( Raises: FileNotFoundError: If `gt_path` or `tracker_path` does not exist. - ValueError: If an unsupported metric family is requested. + ValueError: If an unsupported metric family is requested, or if the + ground-truth file contains no scored rows once ignored and + non-pedestrian rows are filtered out. Examples: >>> from trackers.eval import evaluate_mot_sequence # doctest: +SKIP @@ -95,6 +97,14 @@ def evaluate_mot_sequence( # Prepare sequence (compute IoU, remap IDs) seq_data = _prepare_mot_sequence(gt_data, tracker_data) + if gt_data and seq_data.num_gt_dets == 0: + raise ValueError( + f"Ground truth file has no scored ground-truth rows: {gt_path}. Only pedestrian-class (1) rows " + "marked for consideration are scored, mirroring TrackEval, and every row was dropped by that " + "filter. Note that tracker files written by this library record class -1, so tracker output " + "cannot be reused as ground truth." + ) + # Compute metrics clear_metrics: CLEARMetrics | None = None hota_metrics: HOTAMetrics | None = None diff --git a/src/trackers/io/mot.py b/src/trackers/io/mot.py index bb53ea35..0546d8b5 100644 --- a/src/trackers/io/mot.py +++ b/src/trackers/io/mot.py @@ -156,7 +156,8 @@ def load_mot_file(path: str | Path) -> dict[int, _MOTFrameData]: Raises: FileNotFoundError: If the file does not exist. - ValueError: If the file is empty or has invalid format. + ValueError: If the file is empty, has invalid format, or contains a + frame number that is not a whole number greater than zero. Examples: >>> from trackers import load_mot_file # doctest: +SKIP @@ -207,10 +208,18 @@ def load_mot_file(path: str | Path) -> dict[int, _MOTFrameData]: ) try: - frame = int(float(row[0])) + frame_number = float(row[0]) except ValueError as e: raise ValueError(f"Invalid frame number in {path}: {row[0]}") from e + # Consumers walk frames with `range(1, num_frames + 1)`, so anything outside that + # domain would be loaded here and then silently never evaluated. + if not frame_number.is_integer(): + raise ValueError(f"Frame numbers must be whole numbers in {path}, got {row[0]} in row: {row}") + frame = int(frame_number) + if frame < 1: + raise ValueError(f"MOT frame numbers are 1-based, got {frame} in {path} in row: {row}") + if frame not in frame_data: frame_data[frame] = [] frame_data[frame].append(row) diff --git a/tests/eval/test_evaluate.py b/tests/eval/test_evaluate.py index 59628133..57cc7ddb 100644 --- a/tests/eval/test_evaluate.py +++ b/tests/eval/test_evaluate.py @@ -101,3 +101,43 @@ def test_json_hota_only(self, sample_mot_files: tuple[Path, Path]) -> None: json_str = result.json() assert "HOTA" in json_str assert "DetA" in json_str + + +class TestEvaluateMOTSequenceUnscorableGroundTruth: + """Ground truth that filters down to nothing is reported instead of scored as zeros. + + Only pedestrian-class (1) rows marked for consideration are scored, mirroring TrackEval. Ground truth whose rows are + all dropped by that filter cannot produce a meaningful score, and returning zeros looks like a tracker that found + nothing rather than a ground-truth file the evaluator cannot use. + """ + + def test_tracker_output_used_as_ground_truth_raises(self, tmp_path: Path) -> None: + """Files written by `_MOTOutput` carry class -1, so they cannot be reused as ground truth.""" + gt_path = tmp_path / "gt.txt" + tracker_path = tmp_path / "tracker.txt" + gt_path.write_text("1,1,100,200,50,60,0.9000,-1,-1,-1\n2,1,105,205,50,60,0.9000,-1,-1,-1\n") + tracker_path.write_text("1,10,102,202,50,60,0.9,1\n2,10,107,207,50,60,0.9,1\n") + + with pytest.raises(ValueError, match="no scored ground-truth"): + evaluate_mot_sequence(gt_path=gt_path, tracker_path=tracker_path) + + def test_all_ignored_ground_truth_raises(self, tmp_path: Path) -> None: + """Ground truth where every row is marked ignored (confidence 0) is also unscorable.""" + gt_path = tmp_path / "gt.txt" + tracker_path = tmp_path / "tracker.txt" + gt_path.write_text("1,1,100,200,50,60,0,1\n2,1,105,205,50,60,0,1\n") + tracker_path.write_text("1,10,102,202,50,60,0.9,1\n") + + with pytest.raises(ValueError, match="no scored ground-truth"): + evaluate_mot_sequence(gt_path=gt_path, tracker_path=tracker_path) + + def test_partially_filtered_ground_truth_is_scored(self, tmp_path: Path) -> None: + """A file keeping at least one scored row evaluates normally.""" + gt_path = tmp_path / "gt.txt" + tracker_path = tmp_path / "tracker.txt" + gt_path.write_text("1,1,100,200,50,60,1,1\n1,2,150,250,40,50,1,8\n") + tracker_path.write_text("1,10,102,202,50,60,0.9,1\n") + + result = evaluate_mot_sequence(gt_path=gt_path, tracker_path=tracker_path) + + assert result.CLEAR is not None diff --git a/tests/io/test_mot.py b/tests/io/test_mot.py index 788c1090..7eb11fab 100644 --- a/tests/io/test_mot.py +++ b/tests/io/test_mot.py @@ -6,10 +6,12 @@ from __future__ import annotations +from pathlib import Path + import numpy as np import pytest -from trackers.io.mot import _MOTFrameData, _prepare_mot_sequence +from trackers.io.mot import _MOTFrameData, _prepare_mot_sequence, load_mot_file def _frame( @@ -27,6 +29,55 @@ def _frame( ) +def _write_mot_file(path: Path, *rows: str) -> Path: + """Write raw MOT rows to `path` and return it.""" + path.write_text("\n".join(rows) + "\n") + return path + + +class TestLoadMotFileFrameValidation: + """Frame numbers must be 1-based whole numbers. + + Every consumer walks frames with `range(1, num_frames + 1)`, so a row on frame 0 or a negative frame is loaded and + then never evaluated. Truncating a fractional frame silently moves a detection to a different frame. Both are + rejected rather than accepted and quietly dropped. + """ + + @pytest.mark.parametrize( + "frame", + [ + pytest.param("0", id="zero"), + pytest.param("-3", id="negative"), + ], + ) + def test_rejects_nonpositive_frame(self, tmp_path: Path, frame: str) -> None: + """A frame below 1 is rejected instead of being loaded and never evaluated.""" + path = _write_mot_file(tmp_path / "gt.txt", f"{frame},1,10,10,20,20,1,1") + + with pytest.raises(ValueError, match="1-based"): + load_mot_file(path) + + def test_rejects_fractional_frame(self, tmp_path: Path) -> None: + """A fractional frame is rejected instead of being truncated onto another frame.""" + path = _write_mot_file(tmp_path / "gt.txt", "1.7,1,10,10,20,20,1,1") + + with pytest.raises(ValueError, match="whole number"): + load_mot_file(path) + + def test_accepts_integral_float_frame(self, tmp_path: Path) -> None: + """A frame written as a float with no fractional part is still valid.""" + path = _write_mot_file(tmp_path / "gt.txt", "1.0,1,10,10,20,20,1,1") + + assert list(load_mot_file(path)) == [1] + + def test_reports_offending_row(self, tmp_path: Path) -> None: + """The error names the file and the offending row so the bad line can be found.""" + path = _write_mot_file(tmp_path / "gt.txt", "1,1,10,10,20,20,1,1", "0,2,30,30,20,20,1,1") + + with pytest.raises(ValueError, match=r"gt\.txt"): + load_mot_file(path) + + class TestMotDistractorPreprocessing: """GT preprocessing must follow TrackEval's class-based distractor handling.