Add benchmark Makefile for eval and Codabench submission.

.github/workflows/ci-integrations.yml

@@ -27,7 +27,7 @@ jobs:
           activate-environment: true
 
       - name: 🚀 Install Packages
-        run: uv sync --frozen --group dev
+        run: uv sync --frozen --group dev --extra reid
 
 
       - name: 🧪 Run Integration Tests

README.md

@@ -22,6 +22,7 @@ Keeping track of objects across video frames is one of those problems that sound
 - **Benchmarked across four datasets.** MOT17, SportsMOT, SoccerNet, and DanceTrack — at default parameters and after hyperparameter tuning, so you know what to expect before you deploy.
 - **Tunable out of the box.** Built-in Optuna-based hyperparameter search via `trackers tune` so you can optimize for your specific scene and detector.
 - **Camera motion compensation.** BoT-SORT handles moving cameras natively, keeping track IDs stable even when the whole frame shifts.
+- **Optional appearance ReID.** BoT-SORT can fuse visual embeddings with motion for harder association scenes: install `trackers[reid]` (pulls in the standalone [`reid`](https://github.com/roboflow/re-ID) package) and pass a `reid.ReIDModel` as `reid_model`.
 
 ## Install
 

benchmark/.gitignore

@@ -0,0 +1,4 @@
+data/
+benchmark_prep/
+benchmark_outputs/
+__pycache__/

benchmark/Makefile

@@ -0,0 +1,286 @@
+# MOT benchmark workflow — Makefile orchestrates `trackers tune`, `trackers eval`,
+# small helper scripts under scripts/, and `codabench_submit.py`.
+#
+#   cd benchmark
+#   make help
+#   make data-check
+#   make benchmark-default TRACKER=sort CODABENCH_TOKEN=...
+#   make benchmark-tuned   TRACKER=sort CODABENCH_TOKEN=...
+#   make benchmark BENCHMARK_CONFIG=all TRACKER=sort CODABENCH_TOKEN=...
+#
+# See README.md for data setup.
+
+SHELL     := /bin/bash
+ROOT      := $(CURDIR)
+REPO_ROOT := $(abspath $(ROOT)/..)
+PYTHON    ?= python
+UV        ?= uv
+
+DATA_ROOT  ?= $(ROOT)/data
+PREP_DIR   ?= $(ROOT)/benchmark_prep
+OUTPUT_DIR ?= $(ROOT)/benchmark_outputs
+
+TRACKER   ?= sort
+DATASET   ?= mot17
+CONFIG    ?= default
+BENCHMARK_CONFIG ?= default
+# Multi-tracker comparison (docs/trackers/comparison.md layout). Set to `all` or list ids.
+# Tracker list is defined once in scripts/datasets.py (COMPARISON_TRACKERS).
+TRACKERS  ?=
+DATASETS_PY := $(PYTHON) scripts/datasets.py
+COMPARISON_TRACKERS ?= $(shell $(DATASETS_PY) --field comparison_trackers)
+COMPARISON_TRACKERS_CSV ?= $(shell $(DATASETS_PY) --field comparison_trackers_csv)
+comma := ,
+empty :=
+space := $(empty) $(empty)
+ifeq ($(TRACKERS),all)
+  BENCHMARK_TRACKERS := $(COMPARISON_TRACKERS)
+else ifneq ($(strip $(TRACKERS)),)
+  BENCHMARK_TRACKERS := $(TRACKERS)
+else ifeq ($(words $(DATASETS)),1)
+  # One dataset → all trackers + comparison table (docs/trackers/comparison.md layout).
+  BENCHMARK_TRACKERS := $(COMPARISON_TRACKERS)
+else
+  BENCHMARK_TRACKERS := $(TRACKER)
+endif
+N_TRIALS  ?= 10
+OBJECTIVE ?= HOTA
+THRESHOLD ?= 0.5
+SEED      ?=
+FIXED_PARAMS ?=
+# Optional BoT-SORT appearance: set REID_ENCODER=fastreid_mot17_sbs50 to enable.
+REID_ENCODER ?=
+APPEARANCE_THRESHOLD ?= 0.2
+
+CODABENCH_URL           ?= https://www.codabench.org
+CODABENCH_TOKEN         ?=
+CODABENCH_WAIT_TIMEOUT  ?= 3600
+CODABENCH_POLL_INTERVAL ?= 10
+
+# BoT-SORT requires CMC on by default for the published numbers.
+ifeq ($(TRACKER),botsort)
+  ifeq ($(strip $(FIXED_PARAMS)),)
+    # No spaces — Make word-splits $(FIXED_PARAMS) when expanding recipe lines.
+    FIXED_PARAMS := {"enable_cmc":true}
+  endif
+endif
+
+CODABENCH_DATASETS := mot17 sportsmot dancetrack
+LOCAL_DATASETS     := soccernet
+ALL_DATASETS       := $(CODABENCH_DATASETS) $(LOCAL_DATASETS)
+DATASETS           ?= $(ALL_DATASETS)
+
+# Per-dataset tune splits. Score split is "test" for everyone (Codabench test for
+# codabench datasets, public test GT for soccernet). Path lookups for image dirs
+# delegate to scripts/datasets.py to stay DRY.
+mot17_TUNE_SPLIT      := val
+sportsmot_TUNE_SPLIT  := val
+dancetrack_TUNE_SPLIT := val
+soccernet_TUNE_SPLIT  := train
+SCORE_SPLIT := test
+
+# Codabench (competition_id, phase_id) per dataset.
+mot17_CB      := 10049 16382
+sportsmot_CB  := 13077 21402
+dancetrack_CB := 14885 24635
+
+LAYOUT := $(DATASETS_PY) --data-root "$(DATA_ROOT)"
+
+.PHONY: help setup data-check prep prep-all tune track-default track-tuned _track-and-score \
+        benchmark benchmark-default benchmark-tuned benchmark-comparison-default benchmark-comparison-tuned \
+        upload collect collect-comparison poll clean
+
+help:
+	@echo "MOT benchmark workflow — run from \`cd benchmark\`"
+	@echo ""
+	@echo "Targets:"
+	@echo "  setup         Install \`trackers[tune,reid]\` from $(REPO_ROOT)"
+	@echo "  data-check    Print present/missing assets under $(DATA_ROOT)"
+	@echo "  prep          Prep one dataset (DATASET=...) into $(PREP_DIR)"
+	@echo "  prep-all      Prep every dataset"
+	@echo "  tune          Tune one (TRACKER=, DATASET=)"
+	@echo "  track-default     Track on default params, then score (TRACKER=, DATASET=)"
+	@echo "  track-tuned       Track on best_params.json, then score (TRACKER=, DATASET=)"
+	@echo "  upload            Upload an existing submission.zip (TRACKER=, DATASET=, CONFIG=default|tuned)"
+	@echo "  benchmark         Full pipeline (TRACKER=, BENCHMARK_CONFIG=default|tuned|all, DATASETS=...)"
+	@echo "  benchmark-default Same as \`make benchmark BENCHMARK_CONFIG=default\`"
+	@echo "  benchmark-tuned   Same as \`make benchmark BENCHMARK_CONFIG=tuned\`"
+	@echo "  benchmark-comparison-default  All $(COMPARISON_TRACKERS) on DATASET= (default params)"
+	@echo "  benchmark-comparison-tuned    All $(COMPARISON_TRACKERS) on DATASET= (tune + tuned)"
+	@echo "  collect           Rebuild tables.md for one tracker (TRACKER=)"
+	@echo "  collect-comparison  Tracker comparison table (DATASET=dancetrack)"
+	@echo "  poll              Poll an existing Codabench submission (SUBMISSION_ID=, DATASET=, CONFIG=)"
+	@echo "  clean         Remove $(PREP_DIR) and $(OUTPUT_DIR)"
+	@echo ""
+	@echo "BoT-SORT ReID: pass REID_ENCODER=fastreid_mot17_sbs50 APPEARANCE_THRESHOLD=0.2"
+	@echo "Codabench upload requires CODABENCH_TOKEN. See README for data setup."
+
+setup:
+	$(UV) pip install -e "$(REPO_ROOT)[tune,reid]"
+
+data-check:
+	@$(PYTHON) scripts/data_check.py --data-root "$(DATA_ROOT)"
+
+prep:
+	$(PYTHON) scripts/prep_data.py --dataset $(DATASET) --split all \
+	  --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)"
+
+prep-all:
+	$(PYTHON) scripts/prep_data.py --dataset all --split all \
+	  --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)"
+
+# Tune via the `trackers tune` CLI. Pass --images-dir only for BoT-SORT CMC (ByteTrack/SORT/OC-SORT are det-only).
+tune:
+	@if [ ! -d "$(PREP_DIR)/$(DATASET)/$($(DATASET)_TUNE_SPLIT)" ]; then \
+	  echo "Run: make prep DATASET=$(DATASET)"; exit 1; \
+	fi
+	@mkdir -p "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)"
+	$(eval TUNE_SPLIT := $($(DATASET)_TUNE_SPLIT))
+	$(eval TUNE_IMAGES := $(shell $(LAYOUT) --dataset $(DATASET) --split $(TUNE_SPLIT) --field images_dir))
+	trackers tune \
+	  --tracker $(TRACKER) \
+	  --gt-dir "$(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT)/gt" \
+	  --detections-dir "$(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT)/dets" \
+	  --metrics CLEAR HOTA Identity --objective $(OBJECTIVE) --threshold $(THRESHOLD) \
+	  --n-trials $(N_TRIALS) \
+	  $(if $(SEED),--seed $(SEED),) \
+	  $(if $(FIXED_PARAMS),--fixed-params '$(FIXED_PARAMS)',) \
+	  $(if $(and $(filter botsort,$(TRACKER)),$(TUNE_IMAGES)),--images-dir "$(TUNE_IMAGES)",) \
+	  --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json"
+
+track-default:
+	@$(MAKE) _track-and-score TRACKER=$(TRACKER) DATASET=$(DATASET) CONFIG=default
+
+track-tuned:
+	@test -f "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json" || \
+	  { echo "missing best_params.json — run \`make tune TRACKER=$(TRACKER) DATASET=$(DATASET)\`"; exit 1; }
+	@$(MAKE) _track-and-score TRACKER=$(TRACKER) DATASET=$(DATASET) CONFIG=tuned
+
+# 1) track on the score split (test) → predictions
+# 2) score: `trackers eval` for soccernet, `mot_format.py` + `codabench_submit.py` for the rest.
+_track-and-score:
+	@if [ ! -d "$(PREP_DIR)/$(DATASET)/$(SCORE_SPLIT)" ]; then \
+	  echo "Run: make prep DATASET=$(DATASET)"; exit 1; \
+	fi
+	$(PYTHON) scripts/track_split.py \
+	  --tracker $(TRACKER) --dataset $(DATASET) --split $(SCORE_SPLIT) \
+	  --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)" \
+	  --output-dir "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)" \
+	  $(if $(filter tuned,$(CONFIG)),--params "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json",) \
+	  $(if $(REID_ENCODER),--reid "$(REID_ENCODER)",) \
+	  $(if $(and $(REID_ENCODER),$(APPEARANCE_THRESHOLD)),--appearance-threshold $(APPEARANCE_THRESHOLD),)
+	@if [ "$(DATASET)" = "soccernet" ]; then \
+	  trackers eval \
+	    --gt-dir "$(PREP_DIR)/soccernet/test/gt" \
+	    --tracker-dir "$(OUTPUT_DIR)/$(TRACKER)/soccernet/$(CONFIG)/pred" \
+	    --metrics CLEAR HOTA Identity --threshold $(THRESHOLD) \
+	    --output "$(OUTPUT_DIR)/$(TRACKER)/soccernet/$(CONFIG)/eval.json"; \
+	else \
+	  [ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set — required for $(DATASET)."; exit 1; }; \
+	  $(PYTHON) scripts/mot_format.py --dataset $(DATASET) \
+	    --pred-dir "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/pred" \
+	    --out-zip "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip"; \
+	  $(PYTHON) scripts/codabench_submit.py \
+	    "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" \
+	    --competition-id $(word 1,$($(DATASET)_CB)) \
+	    --phase $(word 2,$($(DATASET)_CB)) \
+	    --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \
+	    --description "$(TRACKER) $(CONFIG) — benchmark Makefile" \
+	    --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \
+	    --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json"; \
+	fi
+
+# Re-upload a zip without re-tracking (e.g. after Codabench daily limit resets).
+upload:
+	@test -f "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" || \
+	  { echo "missing $(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip"; exit 1; }
+	@[ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set."; exit 1; }
+	@test "$(DATASET)" != "soccernet" || { echo "soccernet is scored locally — use track-default/track-tuned"; exit 1; }
+	$(PYTHON) scripts/codabench_submit.py \
+	  "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" \
+	  --competition-id $(word 1,$($(DATASET)_CB)) \
+	  --phase $(word 2,$($(DATASET)_CB)) \
+	  --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \
+	  --description "$(TRACKER) $(CONFIG) — benchmark Makefile" \
+	  --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \
+	  --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json"
+
+# Poll Codabench for scores without re-uploading (e.g. after a transient 502 during benchmark).
+poll:
+	@test -n "$(SUBMISSION_ID)" || { echo "Set SUBMISSION_ID= (from Codabench upload log)"; exit 1; }
+	@test "$(DATASET)" != "soccernet" || { echo "soccernet is scored locally"; exit 1; }
+	@[ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set."; exit 1; }
+	$(PYTHON) scripts/codabench_submit.py --submission-id $(SUBMISSION_ID) \
+	  --competition-id $(word 1,$($(DATASET)_CB)) \
+	  --phase $(word 2,$($(DATASET)_CB)) \
+	  --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \
+	  --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \
+	  --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json"
+
+collect:
+	$(PYTHON) scripts/collect.py --tracker $(TRACKER) --output-dir "$(OUTPUT_DIR)" \
+	  --datasets "$(subst $(space),$(comma),$(DATASETS))"
+
+collect-comparison:
+	@if [ "$(origin DATASET)" != "command line" ]; then \
+	  echo "Set DATASET=dancetrack"; exit 1; \
+	fi
+	$(PYTHON) scripts/collect.py --compare-dataset $(DATASET) --output-dir "$(OUTPUT_DIR)" \
+	  --trackers "$(if $(filter all,$(TRACKERS)),$(COMPARISON_TRACKERS_CSV),$(if $(strip $(TRACKERS)),$(subst $(space),$(comma),$(TRACKERS)),$(COMPARISON_TRACKERS_CSV)))"
+
+# Full pipeline. BENCHMARK_CONFIG=default|tuned|all (default: default only — one Codabench pass).
+# Set TRACKERS=all (or a space-separated list) to benchmark every tracker and write comparison tables.
+benchmark:
+	@case "$(BENCHMARK_CONFIG)" in default|tuned|all) ;; \
+	  *) echo "Set BENCHMARK_CONFIG=default, tuned, or all (or use benchmark-default / benchmark-tuned)"; exit 1;; \
+	esac
+	@if [ -n "$(strip $(filter $(CODABENCH_DATASETS),$(DATASETS)))" ] && [ -z "$(CODABENCH_TOKEN)" ]; then \
+	  echo "ERROR: CODABENCH_TOKEN must be set (required for: $(filter $(CODABENCH_DATASETS),$(DATASETS)))."; exit 1; \
+	fi
+	@for d in $(DATASETS); do \
+	  echo "===== prep [$$d] ====="; \
+	  $(MAKE) prep DATASET=$$d || exit 1; \
+	done
+	@for t in $(BENCHMARK_TRACKERS); do \
+	  for d in $(DATASETS); do \
+	    if [ "$(BENCHMARK_CONFIG)" = "default" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \
+	      echo ""; echo "===== [$$t/$$d] default ====="; \
+	      $(MAKE) track-default TRACKER=$$t DATASET=$$d || exit 1; \
+	    fi; \
+	    if [ "$(BENCHMARK_CONFIG)" = "tuned" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \
+	      echo ""; echo "===== [$$t/$$d] tune ====="; \
+	      extras=""; \
+	      if [ "$$t" = "botsort" ]; then extras='FIXED_PARAMS={"enable_cmc":true}'; fi; \
+	      $(MAKE) tune TRACKER=$$t DATASET=$$d N_TRIALS=$(N_TRIALS) $$extras || exit 1; \
+	      echo "===== [$$t/$$d] tuned ====="; \
+	      $(MAKE) track-tuned TRACKER=$$t DATASET=$$d || exit 1; \
+	    fi; \
+	  done; \
+	done
+	@if [ "$(words $(BENCHMARK_TRACKERS))" -gt 1 ]; then \
+	  for d in $(DATASETS); do \
+	    echo ""; echo "===== collect comparison [$$d] ====="; \
+	    $(PYTHON) scripts/collect.py --compare-dataset $$d --output-dir "$(OUTPUT_DIR)" \
+	      --trackers "$(subst $(space),$(comma),$(BENCHMARK_TRACKERS))" || exit 1; \
+	  done; \
+	else \
+	  echo ""; echo "===== collect [$(TRACKER)] ====="; \
+	  $(MAKE) collect TRACKER=$(TRACKER) DATASETS="$(DATASETS)"; \
+	fi
+
+benchmark-comparison-default:
+	@test -n "$(DATASET)" || { echo "Set DATASET= (e.g. dancetrack)"; exit 1; }
+	@$(MAKE) benchmark BENCHMARK_CONFIG=default DATASETS="$(DATASET)" TRACKERS=all
+
+benchmark-comparison-tuned:
+	@test -n "$(DATASET)" || { echo "Set DATASET= (e.g. dancetrack)"; exit 1; }
+	@$(MAKE) benchmark BENCHMARK_CONFIG=tuned DATASETS="$(DATASET)" TRACKERS=all
+
+benchmark-default:
+	@$(MAKE) benchmark BENCHMARK_CONFIG=default
+
+benchmark-tuned:
+	@$(MAKE) benchmark BENCHMARK_CONFIG=tuned
+
+clean:
+	rm -rf "$(PREP_DIR)" "$(OUTPUT_DIR)"