From 7ed45f21e97cc9f241366bd84b9ba094e261adf1 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 11:51:39 -0300 Subject: [PATCH 01/54] Add benchmark Makefile for eval and Codabench submission. Introduces benchmark/ with make targets for setup, tune, eval, submit, and upload-codabench on MOT17, SportsMOT, and DanceTrack. Submit uses submit_yolox.py with library defaults; eval uses tracker_flags.py for per-tracker CLI parameters. Co-authored-by: Cursor --- benchmark/.gitignore | 4 + benchmark/Makefile | 228 ++++++++ benchmark/README.md | 42 ++ benchmark/scripts/codabench_submit.py | 509 ++++++++++++++++++ benchmark/scripts/mot17_server_format.py | 59 ++ .../mot_challenge_submission_format.py | 70 +++ benchmark/scripts/prep_benchmark.py | 200 +++++++ benchmark/scripts/submit_yolox.py | 177 ++++++ benchmark/scripts/tracker_flags.py | 80 +++ 9 files changed, 1369 insertions(+) create mode 100644 benchmark/.gitignore create mode 100644 benchmark/Makefile create mode 100644 benchmark/README.md create mode 100644 benchmark/scripts/codabench_submit.py create mode 100644 benchmark/scripts/mot17_server_format.py create mode 100644 benchmark/scripts/mot_challenge_submission_format.py create mode 100644 benchmark/scripts/prep_benchmark.py create mode 100644 benchmark/scripts/submit_yolox.py create mode 100644 benchmark/scripts/tracker_flags.py diff --git a/benchmark/.gitignore b/benchmark/.gitignore new file mode 100644 index 000000000..31dccfa97 --- /dev/null +++ b/benchmark/.gitignore @@ -0,0 +1,4 @@ +data/ +benchmark_prep/ +benchmark_outputs/ +__pycache__/ diff --git a/benchmark/Makefile b/benchmark/Makefile new file mode 100644 index 000000000..b60401129 --- /dev/null +++ b/benchmark/Makefile @@ -0,0 +1,228 @@ +# MOT benchmark workflow (tune / eval / submit / Codabench upload). +# +# Run from this directory: +# cd benchmark +# make setup +# make eval TRACKER=sort DATASET=mot17 +# make submit upload-codabench TRACKER=sort DATASET=dancetrack CODABENCH_TOKEN=... +# +# Dataset files live under benchmark/data/ (see README.md). Override with DATA_ROOT=... + +# ── Tools ───────────────────────────────────────────────────────────────────── + +SHELL := /bin/bash +ROOT := $(CURDIR) +REPO_ROOT := $(abspath $(ROOT)/..) +PYTHON ?= python +TRACKERS ?= $(PYTHON) -m trackers.scripts +TRACKERS_REPO := $(REPO_ROOT) +DATA_ROOT ?= $(ROOT)/data + +# ── Knobs ───────────────────────────────────────────────────────────────────── + +TRACKER ?= sort +DATASET ?= dancetrack +N_TRIALS ?= 10 +OBJECTIVE ?= HOTA +THRESHOLD ?= 0.5 +METRICS := CLEAR HOTA Identity +SEED ?= +PARAMS ?= +FIXED_PARAMS ?= + +CODABENCH_URL ?= https://www.codabench.org +CODABENCH_TOKEN ?= +CODABENCH_USERNAME ?= +CODABENCH_PASSWORD ?= +CODABENCH_DESCRIPTION ?= +CODABENCH_WAIT ?= 1 +CODABENCH_WAIT_TIMEOUT ?= 3600 +CODABENCH_POLL_INTERVAL ?= 10 + +PREP_DIR := $(ROOT)/benchmark_prep +OUTPUT_DIR := $(ROOT)/benchmark_outputs +JOB_DIR := $(OUTPUT_DIR)/$(TRACKER)/$(DATASET) +BEST_PARAMS := $(JOB_DIR)/best_params.json + +ifeq ($(DATASET),soccernet) + TUNE_SPLIT := train + EVAL_SPLIT := test + SUBMIT_SPLIT := + EVAL_GT_DIR := $(DATA_ROOT)/soccernet/TrackEval/data/gt/SoccerNet_tracking/SoccerNet_tracking_2022_all_gts + SEQMAP_TUNE := + SEQMAP_EVAL := + TUNE_IMAGES_DIR := $(DATA_ROOT)/soccernet/soccernet_data/tracking/train + EVAL_IMAGES_DIR := $(DATA_ROOT)/soccernet/soccernet_data/tracking/test + SUBMIT_IMAGES_DIR := +else ifeq ($(DATASET),dancetrack) + TUNE_SPLIT := train + EVAL_SPLIT := val + SUBMIT_SPLIT := test + EVAL_GT_DIR := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/val + SEQMAP_TUNE := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/DanceTrack-train.txt + SEQMAP_EVAL := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/DanceTrack-val.txt + TUNE_IMAGES_DIR := $(DATA_ROOT)/dancetrack/train_images + EVAL_IMAGES_DIR := $(DATA_ROOT)/dancetrack/val_images + SUBMIT_IMAGES_DIR := $(DATA_ROOT)/dancetrack/test_images + SUBMIT_DETS_DIR := $(DATA_ROOT)/dancetrack/dancetrack_yolox_dets/test +else ifeq ($(DATASET),sportsmot) + TUNE_SPLIT := val + EVAL_SPLIT := val + SUBMIT_SPLIT := test + EVAL_GT_DIR := $(DATA_ROOT)/sportsmot/TrackEval/data/gt/sportsmot/val + SEQMAP_TUNE := + SEQMAP_EVAL := + TUNE_IMAGES_DIR := $(DATA_ROOT)/sportsmot/val + EVAL_IMAGES_DIR := $(TUNE_IMAGES_DIR) + SUBMIT_IMAGES_DIR := $(DATA_ROOT)/sportsmot/test + SUBMIT_DETS_DIR := $(DATA_ROOT)/sportsmot/sportsmot_yolox_dets/test +else ifeq ($(DATASET),mot17) + TUNE_SPLIT := val + EVAL_SPLIT := val + SUBMIT_SPLIT := test + EVAL_GT_DIR := $(DATA_ROOT)/mot17/TrackEval/data/gt/MOT17_yolox_val/train_val + SEQMAP_TUNE := $(DATA_ROOT)/mot17/TrackEval/data/gt/MOT17/MOT17-val.txt + SEQMAP_EVAL := $(SEQMAP_TUNE) + TUNE_IMAGES_DIR := $(DATA_ROOT)/mot17/val + EVAL_IMAGES_DIR := $(TUNE_IMAGES_DIR) + SUBMIT_IMAGES_DIR := $(DATA_ROOT)/mot17/test + SUBMIT_DETS_DIR := $(DATA_ROOT)/mot17/MOT17_yolox_dets/test +else + $(error Unknown DATASET=$(DATASET). Use: soccernet, dancetrack, sportsmot, mot17) +endif + +ifeq ($(DATASET),mot17) + CODABENCH_COMPETITION := 10049 + CODABENCH_PHASE := 16382 +else ifeq ($(DATASET),sportsmot) + CODABENCH_COMPETITION := 13077 + CODABENCH_PHASE := 21402 +else ifeq ($(DATASET),dancetrack) + CODABENCH_COMPETITION := 14885 + CODABENCH_PHASE := 24635 +endif + +ifeq ($(TRACKER),botsort) + ifeq ($(strip $(FIXED_PARAMS)),) + FIXED_PARAMS := {"enable_cmc": true} + endif + USE_IMAGES := 1 +endif + +TUNE_PREP := $(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT) +EVAL_PREP := $(PREP_DIR)/$(DATASET)/$(EVAL_SPLIT) +PRED_DIR := $(JOB_DIR)/pred_$(EVAL_SPLIT) +EVAL_JSON := $(JOB_DIR)/eval_$(EVAL_SPLIT).json +SUBMIT_DIR := $(JOB_DIR)/submit_$(SUBMIT_SPLIT) +SUBMIT_ZIP := $(JOB_DIR)/$(TRACKER)_$(DATASET)_$(SUBMIT_SPLIT)_submission.zip + +define resolve_params +if [ -n "$(PARAMS)" ]; then params_file="$(PARAMS)"; \ +elif [ -f "$(BEST_PARAMS)" ]; then params_file="$(BEST_PARAMS)"; \ +else echo "Using $(TRACKER) default parameters"; params_file="-"; fi; \ +flags=$$($(PYTHON) scripts/tracker_flags.py $(TRACKER) "$$params_file"); +endef + +.PHONY: help setup tune eval submit upload-codabench all + +help: + @echo "Run from: cd benchmark && make " + @echo "Targets: setup | tune | eval | submit | upload-codabench | all" + @echo "DATA_ROOT=$(DATA_ROOT)" + +setup: + $(PYTHON) -m pip install -e "$(TRACKERS_REPO)[tune]" + $(PYTHON) scripts/prep_benchmark.py --data-root "$(DATA_ROOT)" --dataset $(DATASET) --split all + +tune: setup + @$(PYTHON) -c "import optuna" 2>/dev/null || { echo "Optuna missing. Run: make setup"; exit 1; } + @if [ -n "$(USE_IMAGES)" ]; then \ + test -d "$(TUNE_IMAGES_DIR)" || { echo "Missing $(TUNE_IMAGES_DIR)"; exit 1; }; \ + fi + @mkdir -p "$(JOB_DIR)" + $(TRACKERS) tune \ + --tracker $(TRACKER) \ + --gt-dir "$(TUNE_PREP)/gt" \ + --detections-dir "$(TUNE_PREP)/dets" \ + --objective $(OBJECTIVE) \ + --n-trials $(N_TRIALS) \ + --metrics $(METRICS) \ + --threshold $(THRESHOLD) \ + $(if $(SEQMAP_TUNE),--seqmap "$(SEQMAP_TUNE)",) \ + $(if $(USE_IMAGES),--images-dir "$(TUNE_IMAGES_DIR)",) \ + $(if $(FIXED_PARAMS),--fixed-params '$(FIXED_PARAMS)',) \ + $(if $(SEED),--seed $(SEED),) \ + --output "$(BEST_PARAMS)" + +eval: + @test -d "$(EVAL_PREP)/dets" || { echo "Run: make setup DATASET=$(DATASET)"; exit 1; } + @mkdir -p "$(PRED_DIR)" + @set -euo pipefail; \ + $(resolve_params) \ + for det in "$(EVAL_PREP)/dets"/*.txt; do \ + seq=$$(basename "$$det" .txt); \ + echo "eval/track $$seq"; \ + source_args=(); \ + if [ -n "$(USE_IMAGES)" ]; then \ + frame_seq="$$seq"; \ + if [ "$(DATASET)" = "mot17" ] && [[ "$$seq" != *-FRCNN ]]; then frame_seq="$$seq-FRCNN"; fi; \ + img_dir="$(EVAL_IMAGES_DIR)/$$frame_seq/img1"; \ + test -d "$$img_dir" || { echo "Missing $$img_dir" >&2; exit 1; }; \ + source_args=(--source "$$img_dir"); \ + fi; \ + $(TRACKERS) track \ + --detections "$$det" --tracker $(TRACKER) $$flags $${source_args[@]+"$${source_args[@]}"} \ + --mot-output "$(PRED_DIR)/$$seq.txt" --overwrite; \ + done + $(TRACKERS) eval \ + --gt-dir "$(EVAL_GT_DIR)" --tracker-dir "$(PRED_DIR)" \ + --metrics $(METRICS) --threshold $(THRESHOLD) \ + --columns MOTA HOTA IDF1 \ + $(if $(SEQMAP_EVAL),--seqmap "$(SEQMAP_EVAL)",) \ + --output "$(EVAL_JSON)" + @echo "Saved → $(EVAL_JSON)" + +submit: + @test -n "$(SUBMIT_SPLIT)" || { echo "No submit split for $(DATASET)"; exit 1; } + @test -d "$(SUBMIT_DETS_DIR)" || { echo "Missing YOLOX detections: $(SUBMIT_DETS_DIR)"; exit 1; } + @mkdir -p "$(SUBMIT_DIR)" + @set -euo pipefail; \ + params_args=(); \ + if [ -n "$(PARAMS)" ]; then params_args=(--params "$(PARAMS)"); \ + elif [ -f "$(BEST_PARAMS)" ]; then params_args=(--params "$(BEST_PARAMS)"); fi; \ + images_args=(); \ + if [ -n "$(USE_IMAGES)" ]; then \ + test -d "$(SUBMIT_IMAGES_DIR)" || { echo "Missing frames: $(SUBMIT_IMAGES_DIR)" >&2; exit 1; }; \ + images_args=(--images-dir "$(SUBMIT_IMAGES_DIR)"); \ + fi; \ + $(PYTHON) scripts/submit_yolox.py \ + --tracker $(TRACKER) --dataset $(DATASET) --split $(SUBMIT_SPLIT) \ + --data-root "$(DATA_ROOT)" --output-dir "$(SUBMIT_DIR)" \ + $${params_args[@]+"$${params_args[@]}"} \ + $${images_args[@]+"$${images_args[@]}"} + @$(PYTHON) scripts/mot_challenge_submission_format.py "$(SUBMIT_DIR)" + @if [ "$(DATASET)" = "mot17" ]; then \ + $(PYTHON) scripts/mot17_server_format.py "$(SUBMIT_DIR)"; \ + fi + @rm -f "$(SUBMIT_ZIP)" + cd "$(SUBMIT_DIR)" && zip -r "$(SUBMIT_ZIP)" . + @echo "Created $(SUBMIT_ZIP)" + +upload-codabench: + @test "$(DATASET)" = "mot17" -o "$(DATASET)" = "sportsmot" -o "$(DATASET)" = "dancetrack" || \ + { echo "upload-codabench supports mot17, sportsmot, dancetrack"; exit 1; } + @test -f "$(SUBMIT_ZIP)" || $(MAKE) submit TRACKER=$(TRACKER) DATASET=$(DATASET) PARAMS="$(PARAMS)" + @test -n "$(CODABENCH_TOKEN)" -o -n "$(CODABENCH_USERNAME)" || \ + { echo "Set CODABENCH_TOKEN or CODABENCH_USERNAME+CODABENCH_PASSWORD"; exit 1; } + $(PYTHON) scripts/codabench_submit.py "$(SUBMIT_ZIP)" \ + --phase $(CODABENCH_PHASE) --competition-id $(CODABENCH_COMPETITION) \ + --base-url "$(CODABENCH_URL)" \ + $(if $(CODABENCH_TOKEN),--token "$(CODABENCH_TOKEN)",) \ + $(if $(CODABENCH_USERNAME),--username "$(CODABENCH_USERNAME)",) \ + $(if $(CODABENCH_PASSWORD),--password "$(CODABENCH_PASSWORD)",) \ + $(if $(CODABENCH_DESCRIPTION),--description "$(CODABENCH_DESCRIPTION)",) \ + $(if $(filter 0 false no,$(CODABENCH_WAIT)),--no-wait,) \ + --wait-timeout $(CODABENCH_WAIT_TIMEOUT) \ + --poll-interval $(CODABENCH_POLL_INTERVAL) + +all: tune eval submit diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 000000000..0359b90b6 --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,42 @@ +# MOT benchmark workflow + +Makefile-driven pipeline for tuning, local evaluation, test-set submission, and Codabench upload using the trackers CLI. + +Requires **`develop`** (trackers ≥ 2.3 with `track`, `eval`, `tune` CLIs). Install the repo editable from the parent directory: + +```bash +cd benchmark +make setup DATASET=mot17 +``` + +## Data layout + +Place benchmark assets under `benchmark/data/` (or set `DATA_ROOT=`): + +``` +data/ + mot17/MOT17_yolox_dets/{val,test}/... + sportsmot/sportsmot_yolox_dets/{val,test}/... + dancetrack/dancetrack_yolox_dets/{train,val,test}/... +``` + +Use `trackers download` or your existing YOLOX det trees. For BoT-SORT CMC, also provide frame directories (`mot17/val`, `dancetrack/test_images`, etc.). + +## Commands + +```bash +make eval TRACKER=sort DATASET=mot17 +make submit TRACKER=sort DATASET=dancetrack +make upload-codabench TRACKER=sort DATASET=mot17 CODABENCH_TOKEN=... +``` + +| Dataset | Codabench | Phase | +|---|---|---| +| MOT17 | [10049](https://www.codabench.org/competitions/10049/) | 16382 | +| SportsMOT | [13077](https://www.codabench.org/competitions/13077/) | 21402 | +| DanceTrack | [14885](https://www.codabench.org/competitions/14885/) | 24635 | + +## Implementation notes + +- **`make submit`** uses `scripts/submit_yolox.py` with library defaults (or `best_params.json`), not the shared `trackers track` CLI defaults. +- **`make eval`** passes explicit per-tracker flags via `scripts/tracker_flags.py`. diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py new file mode 100644 index 000000000..4303f2db5 --- /dev/null +++ b/benchmark/scripts/codabench_submit.py @@ -0,0 +1,509 @@ +#!/usr/bin/env python3 +"""Upload a submission zip to a Codabench competition phase. + +Uses Codabench's REST API (token auth + 3-step file upload). See: + https://www.codabench.org/api/docs/ + https://docs.codabench.org/v1.23/Developers_and_Administrators/Robot-submissions/ +""" + +from __future__ import annotations + +import argparse +import http.client +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +DEFAULT_METRICS = ("HOTA", "IDF1", "MOTA") +TERMINAL_STATUSES = {"finished", "failed", "cancelled", "none"} + +# Known test-server presets (Makefile sets these via CODABENCH_* env vars): +# mot17: competition 10049, phase 16382 +# sportsmot: competition 13077, phase 21402 +# dancetrack: competition 14885, phase 24635 +DEFAULT_COMPETITION_ID = 10049 +DEFAULT_PHASE_ID = 16382 + + +def _request( + *, + method: str, + url: str, + token: str | None = None, + data: bytes | None = None, + json_body: Any = None, + headers: dict[str, str] | None = None, +) -> tuple[int, Any]: + hdrs = dict(headers or {}) + if token: + hdrs["Authorization"] = f"Token {token}" + body: bytes | None = data + if json_body is not None: + body = json.dumps(json_body).encode() + hdrs.setdefault("Content-Type", "application/json") + req = urllib.request.Request(url, data=body, headers=hdrs, method=method) + try: + with urllib.request.urlopen(req, timeout=120) as resp: + raw = resp.read() + status = resp.status + except urllib.error.HTTPError as exc: + raw = exc.read() + status = exc.code + detail = raw.decode(errors="replace") + if detail.lstrip().startswith(" None: + """PUT bytes to a presigned MinIO/S3 URL without re-encoding the query string. + + urllib.request re-quotes presigned URLs and breaks AWS signatures (403 + SignatureDoesNotMatch). Send path?query verbatim via http.client instead. + """ + url = url.strip() + parsed = urllib.parse.urlparse(url) + if parsed.scheme not in ("http", "https"): + raise RuntimeError(f"Unsupported presigned URL scheme: {parsed.scheme!r}") + + path = parsed.path + if parsed.query: + path = f"{path}?{parsed.query}" + + headers = { + "Content-Type": content_type, + "Content-Length": str(len(data)), + } + conn_class = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + conn = conn_class(parsed.netloc, timeout=120) + try: + conn.request("PUT", path, body=data, headers=headers) + resp = conn.getresponse() + raw = resp.read() + if resp.status >= 400: + detail = raw.decode(errors="replace") + raise RuntimeError( + f"PUT presigned upload → HTTP {resp.status}: {detail}" + ) + finally: + conn.close() + + +def fetch_token(base_url: str, username: str, password: str) -> str: + _, payload = _request( + method="POST", + url=f"{base_url.rstrip('/')}/api/api-token-auth/", + json_body={"username": username, "password": password}, + ) + if not isinstance(payload, dict) or "token" not in payload: + raise RuntimeError(f"Unexpected token response: {payload!r}") + return str(payload["token"]) + + +def can_make_submission(base_url: str, token: str, phase_id: int) -> tuple[bool, str]: + _, payload = _request( + method="GET", + url=f"{base_url.rstrip('/')}/api/can_make_submission/{phase_id}/", + token=token, + ) + if not isinstance(payload, dict): + raise RuntimeError(f"Unexpected can_make_submission response: {payload!r}") + return bool(payload.get("can")), str(payload.get("reason", "")) + + +def _dataset_name(zip_path: Path, dataset_name: str | None = None) -> str: + """Codabench internal dataset label (must be unique per user account).""" + if dataset_name: + return dataset_name + stamp = datetime.now(timezone.utc).strftime("%Y%m%d_%H%M%S") + return f"{zip_path.stem}_{stamp}" + + +def upload_submission( + *, + base_url: str, + token: str, + phase_id: int, + zip_path: Path, + description: str | None = None, + dataset_name: str | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + zip_path = zip_path.resolve() + if not zip_path.is_file(): + raise FileNotFoundError(f"Submission zip not found: {zip_path}") + + allowed, reason = can_make_submission(base_url, token, phase_id) + if not allowed: + raise RuntimeError( + f"Cannot submit to phase {phase_id}: {reason or 'unknown reason'}. " + "Register for the competition on Codabench and wait for approval if needed." + ) + + bundle_bytes = zip_path.read_bytes() + if dry_run: + print( + f"Dry run: would upload {zip_path.name} ({len(bundle_bytes)} bytes) " + f"to phase {phase_id} on {base_url}" + ) + return {"dry_run": True, "phase": phase_id, "zip": str(zip_path)} + + _, data_record = _request( + method="POST", + url=f"{base_url.rstrip('/')}/api/datasets/", + token=token, + json_body={ + "type": "submission", + "file_size": len(bundle_bytes), + "request_sassy_file_name": zip_path.name, + "name": _dataset_name(zip_path, dataset_name), + }, + ) + if not isinstance(data_record, dict): + raise RuntimeError(f"Unexpected /api/datasets/ response: {data_record!r}") + key = data_record["key"] + sassy_url = data_record["sassy_url"] + + _put_presigned_url(sassy_url, bundle_bytes, content_type="application/zip") + + _request( + method="PUT", + url=f"{base_url.rstrip('/')}/api/datasets/completed/{key}/", + token=token, + ) + + body: dict[str, Any] = {"data": key, "phase": phase_id} + if description: + body["description"] = description + _, submission = _request( + method="POST", + url=f"{base_url.rstrip('/')}/api/submissions/", + token=token, + json_body=body, + ) + if not isinstance(submission, dict): + raise RuntimeError(f"Unexpected /api/submissions/ response: {submission!r}") + return submission + + +def get_submission_details(*, base_url: str, token: str, submission_id: int) -> dict[str, Any]: + _, payload = _request( + method="GET", + url=f"{base_url.rstrip('/')}/api/submissions/{submission_id}/get_details/", + token=token, + ) + if not isinstance(payload, dict): + raise RuntimeError( + f"Unexpected /api/submissions/{submission_id}/get_details/ response: {payload!r}" + ) + return payload + + +def print_submission_failure_logs( + *, + base_url: str, + token: str, + submission_id: int, + max_chars: int = 4000, +) -> None: + """Best-effort scrape of scoring logs after a failed submission.""" + try: + details = get_submission_details( + base_url=base_url, token=token, submission_id=submission_id + ) + except RuntimeError as exc: + print(f" logs unavailable: {exc}", flush=True) + return + + chunks: list[str] = [] + for key in ("logs", "scoring_result", "prediction_result"): + value = details.get(key) + if isinstance(value, list): + for item in value: + if isinstance(item, dict): + for field in ("name", "data_file", "url"): + if item.get(field): + chunks.append(f"{key}/{item.get('name', field)}: {item[field]}") + elif item: + chunks.append(str(item)) + elif isinstance(value, str) and value.strip(): + chunks.append(value) + + if not chunks: + print(" logs: (none returned by API — open submission on Codabench for full logs)", flush=True) + return + + text = "\n".join(chunks) + if len(text) > max_chars: + text = text[-max_chars:] + text = f"...(truncated)\n{text}" + print(f" logs →\n{text}", flush=True) + + +def get_submission(*, base_url: str, token: str, submission_id: int) -> dict[str, Any]: + _, payload = _request( + method="GET", + url=f"{base_url.rstrip('/')}/api/submissions/{submission_id}/", + token=token, + ) + if not isinstance(payload, dict): + raise RuntimeError(f"Unexpected /api/submissions/{submission_id}/ response: {payload!r}") + return payload + + +def extract_metric_scores( + submission: dict[str, Any], + metric_keys: tuple[str, ...] = DEFAULT_METRICS, +) -> dict[str, float]: + """Pull leaderboard scores by column_key (case-insensitive).""" + wanted = {k.lower(): k for k in metric_keys} + found: dict[str, float] = {} + + def _collect(scores: Any) -> None: + if not isinstance(scores, list): + return + for item in scores: + if not isinstance(item, dict): + continue + raw_key = str(item.get("column_key", "")) + canonical = wanted.get(raw_key.lower()) + if canonical is None: + continue + score = item.get("score") + if score is None: + continue + found[canonical] = float(score) + + _collect(submission.get("scores")) + for child in submission.get("children") or []: + if isinstance(child, dict): + _collect(child.get("scores")) + + return found + + +def poll_submission( + *, + base_url: str, + token: str, + submission_id: int, + timeout_seconds: float = 3600.0, + interval_seconds: float = 10.0, + metric_keys: tuple[str, ...] = DEFAULT_METRICS, +) -> dict[str, Any]: + """Poll GET /api/submissions// until Finished/Failed/Cancelled.""" + start = time.monotonic() + wait = interval_seconds + max_wait = interval_seconds * 6 + last_status = "" + + while True: + submission = get_submission( + base_url=base_url, token=token, submission_id=submission_id + ) + status = str(submission.get("status", "")) + status_lc = status.lower() + + if status != last_status: + print(f" submission {submission_id}: {status}", flush=True) + last_status = status + + if status_lc in TERMINAL_STATUSES: + if status_lc == "finished": + scores = extract_metric_scores(submission, metric_keys) + if scores: + parts = ", ".join(f"{k}={scores[k]:.3f}" for k in metric_keys if k in scores) + print(f" scores → {parts}") + elif submission.get("scores"): + print(" scores → (present but no matching HOTA/IDF1/MOTA column keys)") + else: + print(" scores → not available yet (check competition page)") + elif submission.get("status_details"): + print(f" details → {submission['status_details']}") + if status_lc == "failed": + print_submission_failure_logs( + base_url=base_url, token=token, submission_id=submission_id + ) + return submission + + elapsed = time.monotonic() - start + remaining = timeout_seconds - elapsed + if remaining <= 0: + raise RuntimeError( + f"Timed out after {timeout_seconds:.0f}s waiting for submission " + f"{submission_id} (last status: {status})" + ) + + time.sleep(min(wait, remaining)) + wait = min(wait * 1.5, max_wait) + + +def resolve_token(args: argparse.Namespace) -> str: + if args.token: + return args.token + token = os.environ.get("CODABENCH_TOKEN", "").strip() + if token: + return token + username = args.username or os.environ.get("CODABENCH_USERNAME", "").strip() + password = args.password or os.environ.get("CODABENCH_PASSWORD", "").strip() + if username and password: + return fetch_token(args.base_url, username, password) + raise RuntimeError( + "Missing API token. Set CODABENCH_TOKEN or pass --token, " + "or set CODABENCH_USERNAME and CODABENCH_PASSWORD. " + "Create a token via POST /api/api-token-auth/ (see Codabench API docs)." + ) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "zip_path", + nargs="?", + type=Path, + help="Submission .zip (flat MOT txt files at archive root). Omit with --submission-id.", + ) + p.add_argument( + "--submission-id", + type=int, + help="Poll an existing submission instead of uploading (use with --wait).", + ) + p.add_argument( + "--phase", + type=int, + default=int(os.environ.get("CODABENCH_PHASE", str(DEFAULT_PHASE_ID))), + help="Codabench phase id (mot17: 16382, sportsmot: 21402).", + ) + p.add_argument( + "--competition-id", + type=int, + default=int(os.environ.get("CODABENCH_COMPETITION", str(DEFAULT_COMPETITION_ID))), + help="Codabench competition id for result URL (mot17: 10049, sportsmot: 13077).", + ) + p.add_argument( + "--base-url", + default=os.environ.get("CODABENCH_URL", "https://www.codabench.org"), + help="Codabench base URL.", + ) + p.add_argument("--token", help="API token (or env CODABENCH_TOKEN).") + p.add_argument("--username", help="Username for token auth (or env CODABENCH_USERNAME).") + p.add_argument("--password", help="Password for token auth (or env CODABENCH_PASSWORD).") + p.add_argument("--description", default="", help="Optional submission description.") + p.add_argument( + "--dataset-name", + help="Codabench dataset label (default: _; must be unique).", + ) + p.add_argument( + "--wait", + action=argparse.BooleanOptionalAction, + default=True, + help="Poll until Codabench finishes scoring (default: on).", + ) + p.add_argument( + "--wait-timeout", + type=float, + default=float(os.environ.get("CODABENCH_WAIT_TIMEOUT", "3600")), + help="Max seconds to wait for scoring (default: 3600).", + ) + p.add_argument( + "--poll-interval", + type=float, + default=float(os.environ.get("CODABENCH_POLL_INTERVAL", "10")), + help="Initial poll interval in seconds (default: 10, backs off).", + ) + p.add_argument( + "--metrics", + nargs="+", + default=list(DEFAULT_METRICS), + help="Leaderboard columns to print when finished (default: HOTA IDF1 MOTA).", + ) + p.add_argument("--dry-run", action="store_true", help="Check eligibility only; do not upload.") + args = p.parse_args(argv) + + metric_keys = tuple(args.metrics) + + try: + token = resolve_token(args) + + if args.submission_id is not None: + sub_id = args.submission_id + if args.zip_path is not None: + print("Note: zip_path ignored when --submission-id is set") + if args.wait: + print(f"Waiting for submission {sub_id} on {args.base_url} ...") + submission = poll_submission( + base_url=args.base_url, + token=token, + submission_id=sub_id, + timeout_seconds=args.wait_timeout, + interval_seconds=args.poll_interval, + metric_keys=metric_keys, + ) + else: + submission = get_submission( + base_url=args.base_url, token=token, submission_id=sub_id + ) + scores = extract_metric_scores(submission, metric_keys) + if scores: + parts = ", ".join(f"{k}={scores[k]:.3f}" for k in metric_keys if k in scores) + print(f"scores → {parts}") + else: + if args.zip_path is None: + raise RuntimeError("Provide zip_path or --submission-id") + submission = upload_submission( + base_url=args.base_url, + token=token, + phase_id=args.phase, + zip_path=args.zip_path, + description=args.description or None, + dataset_name=args.dataset_name or None, + dry_run=args.dry_run, + ) + if submission.get("dry_run"): + return 0 + sub_id = submission.get("id") + print(f"Submitted → id={sub_id} status={submission.get('status')}") + if sub_id is not None and args.wait: + print(f"Waiting for submission {sub_id} on {args.base_url} ...") + submission = poll_submission( + base_url=args.base_url, + token=token, + submission_id=int(sub_id), + timeout_seconds=args.wait_timeout, + interval_seconds=args.poll_interval, + metric_keys=metric_keys, + ) + except (RuntimeError, FileNotFoundError) as exc: + print(str(exc), file=sys.stderr) + return 1 + + sub_id = submission.get("id") + comp_id = args.competition_id + phase = submission.get("phase") + if isinstance(phase, dict): + comp_id = phase.get("competition", comp_id) + + print(f"Done → id={sub_id} status={submission.get('status')}") + if sub_id is not None: + print(f" https://www.codabench.org/competitions/{comp_id}/") + return 0 if str(submission.get("status", "")).lower() == "finished" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/mot17_server_format.py b/benchmark/scripts/mot17_server_format.py new file mode 100644 index 000000000..097237a46 --- /dev/null +++ b/benchmark/scripts/mot17_server_format.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Expand MOT17-XX.txt tracker outputs into Codabench/MOTChallenge server layout.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +# Sequences with YOLOX test detections in this benchmark setup. +_EXISTING = ("01", "03", "06", "07", "08", "12", "14") +# Sequences without test detections — server expects empty placeholder files. +_MISSING = ("02", "04", "05", "09", "10", "11", "13") +_SUFFIXES = ("FRCNN", "SDP", "DPM") + + +def write_mot17_server_format(out_dir: Path) -> int: + """Triplicate tracked results and add empty files for missing sequences.""" + if not out_dir.is_dir(): + raise FileNotFoundError(f"Not a directory: {out_dir}") + + written = 0 + for num in _EXISTING: + src = out_dir / f"MOT17-{num}.txt" + if not src.is_file(): + print(f" Missing expected source: {src}", flush=True) + continue + content = src.read_bytes() + for suf in _SUFFIXES: + (out_dir / f"MOT17-{num}-{suf}.txt").write_bytes(content) + written += 1 + src.unlink() + + for num in _MISSING: + for suf in _SUFFIXES: + (out_dir / f"MOT17-{num}-{suf}.txt").touch(exist_ok=True) + written += 1 + + print(f" MOT17 server format: {written} files in {out_dir}", flush=True) + return written + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "out_dir", + type=Path, + help="Directory containing MOT17-XX.txt tracker outputs (modified in place).", + ) + args = p.parse_args(argv) + try: + write_mot17_server_format(args.out_dir.resolve()) + except FileNotFoundError as exc: + print(str(exc), flush=True) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/mot_challenge_submission_format.py b/benchmark/scripts/mot_challenge_submission_format.py new file mode 100644 index 000000000..e60baf04d --- /dev/null +++ b/benchmark/scripts/mot_challenge_submission_format.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Normalize tracker MOT outputs for MOTChallenge / Codabench submission. + +Tracking output should already omit unassigned rows (``tracker_id=-1``) and use +0-based track IDs with ``.1f`` box coordinates and ``conf=-1``. This step +drops any negative IDs as a safety net and rewrites rows to the notebook / +docs submission layout. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + + +def normalize_mot_submission_line(line: str) -> str | None: + parts = line.strip().split(",") + if len(parts) < 7: + return None + try: + frame = int(float(parts[0])) + track_id = int(float(parts[1])) + except ValueError: + return None + if track_id < 0: + return None + + left, top, width, height = (float(parts[i]) for i in range(2, 6)) + return ( + f"{frame},{track_id},{left:.1f},{top:.1f},{width:.1f},{height:.1f}," + f"-1,-1,-1,-1" + ) + + +def normalize_mot_submission_file(path: Path) -> int: + lines_out: list[str] = [] + for raw in path.read_text().splitlines(): + normalized = normalize_mot_submission_line(raw) + if normalized is not None: + lines_out.append(normalized) + path.write_text("\n".join(lines_out) + ("\n" if lines_out else "")) + return len(lines_out) + + +def normalize_mot_submission_dir(out_dir: Path) -> int: + if not out_dir.is_dir(): + raise FileNotFoundError(f"Not a directory: {out_dir}") + total_lines = 0 + n_files = 0 + for path in sorted(out_dir.glob("*.txt")): + total_lines += normalize_mot_submission_file(path) + n_files += 1 + print(f" MOT submission format: {n_files} files, {total_lines} lines in {out_dir}") + return total_lines + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("out_dir", type=Path, help="Directory of per-sequence .txt files (modified in place).") + args = p.parse_args(argv) + try: + normalize_mot_submission_dir(args.out_dir.resolve()) + except FileNotFoundError as exc: + print(str(exc)) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/prep_benchmark.py b/benchmark/scripts/prep_benchmark.py new file mode 100644 index 000000000..41fc106aa --- /dev/null +++ b/benchmark/scripts/prep_benchmark.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +"""Convert local benchmark detections/GT into flat MOT dirs for ``trackers tune``.""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +_BENCHMARK_ROOT = Path(__file__).resolve().parents[1] + + +def _soccer_seq_name(stem: str) -> str: + return stem.replace("__det", "") + + +def _mot17_val_seq_name(stem: str) -> str: + return stem.split("_")[0] + "-FRCNN" + + +@dataclass(frozen=True) +class SplitSpec: + det_dir: Path + gt_dir: Path | None + det_format: str # xyxy | mot | ltwh_mot + seq_name_fn: Callable[[str], str] | None = None + + +def split_spec(data_root: Path, dataset: str, split: str) -> SplitSpec | None: + root = data_root + if dataset == "soccernet": + if split == "train": + return SplitSpec( + det_dir=root / "soccernet/SoccerNet_dets/SoccerNet_tracking/train", + gt_dir=root / "soccernet/TrackEval/data/gt/SoccerNet_tracking/train", + det_format="mot", + seq_name_fn=_soccer_seq_name, + ) + if split == "test": + return SplitSpec( + det_dir=root / "soccernet/SoccerNet_dets/SoccerNet_tracking_2022_all_dets", + gt_dir=root / "soccernet/TrackEval/data/gt/SoccerNet_tracking/SoccerNet_tracking_2022_all_gts", + det_format="ltwh_mot", + seq_name_fn=_soccer_seq_name, + ) + if dataset == "dancetrack" and split in {"train", "val", "test"}: + gt_dir = None if split == "test" else root / f"dancetrack/TrackEval/data/gt/dancetrack/{split}" + return SplitSpec( + det_dir=root / f"dancetrack/dancetrack_yolox_dets/{split}", + gt_dir=gt_dir, + det_format="xyxy", + ) + if dataset == "sportsmot" and split in {"val", "test"}: + gt_dir = None if split == "test" else root / f"sportsmot/TrackEval/data/gt/sportsmot/{split}" + return SplitSpec( + det_dir=root / f"sportsmot/sportsmot_yolox_dets/{split}", + gt_dir=gt_dir, + det_format="xyxy", + ) + if dataset == "mot17": + if split == "val": + return SplitSpec( + det_dir=root / "mot17/MOT17_yolox_dets/val", + gt_dir=root / "mot17/TrackEval/data/gt/MOT17_yolox_val/train_val", + det_format="xyxy", + seq_name_fn=_mot17_val_seq_name, + ) + if split == "test": + return SplitSpec( + det_dir=root / "mot17/MOT17_yolox_dets/test", + gt_dir=None, + det_format="xyxy", + ) + return None + + +def prepare_mot_dets( + src_dir: Path, + dst_dir: Path, + *, + src_format: str, + seq_name_fn: Callable[[str], str] | None = None, +) -> None: + dst_dir.mkdir(parents=True, exist_ok=True) + for det_file in sorted(src_dir.glob("*.txt")): + seq_name = seq_name_fn(det_file.stem) if seq_name_fn else det_file.stem + dst_path = dst_dir / f"{seq_name}.txt" + if src_format == "mot": + shutil.copy(det_file, dst_path) + continue + with det_file.open() as fin, dst_path.open("w") as fout: + for line in fin: + parts = line.strip().split(",") + if len(parts) < 6: + continue + frame = int(parts[0]) + if src_format == "ltwh_mot": + left, top, w, h = (float(parts[i]) for i in range(2, 6)) + conf = float(parts[6]) if len(parts) > 6 else 1.0 + else: + x1, y1, x2, y2, conf = (float(p) for p in parts[1:6]) + left, top, w, h = x1, y1, x2 - x1, y2 - y1 + fout.write(f"{frame},-1,{left:.4f},{top:.4f},{w:.4f},{h:.4f},{conf:.4f}\n") + + +def prepare_flat_gt(gt_root: Path, dst_dir: Path) -> None: + dst_dir.mkdir(parents=True, exist_ok=True) + for seq_dir in sorted(gt_root.iterdir()): + if not seq_dir.is_dir(): + continue + gt_path = seq_dir / "gt" / "gt.txt" + if gt_path.is_file(): + shutil.copy(gt_path, dst_dir / f"{seq_dir.name}.txt") + + +def prep_split(data_root: Path, prep_root: Path, dataset: str, split: str) -> Path: + spec = split_spec(data_root, dataset, split) + if spec is None: + raise ValueError(f"Unknown dataset/split: {dataset}/{split}") + if not spec.det_dir.is_dir(): + raise FileNotFoundError(f"Missing detections: {spec.det_dir}") + + out = prep_root / dataset / split + dets_out = out / "dets" + gt_out = out / "gt" + prepare_mot_dets( + spec.det_dir, + dets_out, + src_format=spec.det_format, + seq_name_fn=spec.seq_name_fn, + ) + if spec.gt_dir is not None: + if not spec.gt_dir.is_dir(): + raise FileNotFoundError(f"Missing GT: {spec.gt_dir}") + prepare_flat_gt(spec.gt_dir, gt_out) + print(f"Prepared {dataset}/{split} → {out}") + return out + + +def main(argv: list[str] | None = None) -> int: + datasets = ("soccernet", "dancetrack", "sportsmot", "mot17") + tune_splits = { + "soccernet": "train", + "dancetrack": "train", + "sportsmot": "val", + "mot17": "val", + } + eval_splits = { + "soccernet": "test", + "dancetrack": "val", + "sportsmot": "val", + "mot17": "val", + } + submit_splits = { + "dancetrack": "test", + "sportsmot": "test", + "mot17": "test", + } + + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--data-root", type=Path, default=_BENCHMARK_ROOT / "data") + p.add_argument("--prep-dir", type=Path, default=_BENCHMARK_ROOT / "benchmark_prep") + p.add_argument("--dataset", choices=[*datasets, "all"], default="all") + p.add_argument( + "--split", + choices=["all", "tune", "eval", "submit", "train", "val", "test"], + default="tune", + help="Which split to prep (all=tune+eval+submit for dataset, or explicit split name).", + ) + args = p.parse_args(argv) + + picked = list(datasets) if args.dataset == "all" else [args.dataset] + split_aliases = ("tune", "eval", "submit") + for dataset in picked: + splits_to_run: list[str] + if args.split == "all": + splits_to_run = list(split_aliases) + else: + splits_to_run = [args.split] + + for split_key in splits_to_run: + if split_key in {"tune", "eval", "submit"}: + split_map = {"tune": tune_splits, "eval": eval_splits, "submit": submit_splits} + if split_key == "submit" and dataset not in submit_splits: + continue + split = split_map[split_key][dataset] + else: + split = split_key + try: + prep_split(args.data_root, args.prep_dir, dataset, split) + except (FileNotFoundError, ValueError) as exc: + print(f"SKIP {dataset}/{split}: {exc}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/submit_yolox.py b/benchmark/scripts/submit_yolox.py new file mode 100644 index 000000000..adba082af --- /dev/null +++ b/benchmark/scripts/submit_yolox.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python3 +"""Build a Codabench submission from raw YOLOX detections (notebook-style loop). + +Uses each tracker's library defaults (or an optional params JSON) directly, +avoiding the ``trackers track`` CLI shared-parameter default bug. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections import defaultdict +from pathlib import Path + +import numpy as np +import supervision as sv + +_BENCHMARK_ROOT = Path(__file__).resolve().parents[1] + + +def det_root(data_root: Path, dataset: str, split: str) -> Path: + rel = { + "mot17": data_root / "mot17" / "MOT17_yolox_dets" / split, + "sportsmot": data_root / "sportsmot" / "sportsmot_yolox_dets" / split, + "dancetrack": data_root / "dancetrack" / "dancetrack_yolox_dets" / split, + } + if dataset not in rel: + raise ValueError(f"unsupported dataset: {dataset}") + return rel[dataset] + + +def _build_index(det_list: list[str]) -> dict[int, list[str]]: + dets_by_frame: dict[int, list[str]] = defaultdict(list) + for line in det_list: + dets_by_frame[int(line.split(",")[0])].append(line) + return dets_by_frame + + +def _yolox_rows(frame_id: int, dets_by_frame: dict[int, list[str]]) -> list[list[float]]: + rows: list[list[float]] = [] + for line in dets_by_frame.get(frame_id, []): + parts = line.split(",") + rows.append([float(parts[1]), float(parts[2]), float(parts[3]), float(parts[4]), float(parts[5])]) + return rows + + +def _write_mot_line(frame_id: int, track_id: int, left: float, top: float, right: float, bottom: float) -> str: + width = right - left + height = bottom - top + return f"{frame_id},{int(track_id)},{left:.1f},{top:.1f},{width:.1f},{height:.1f},-1,-1,-1,-1\n" + + +def _init_tracker(tracker_id: str, params: dict): + import trackers as _trackers # noqa: F401 + from trackers.core.base import BaseTracker + + info = BaseTracker._lookup_tracker(tracker_id) + if info is None: + raise ValueError(f"unknown tracker: {tracker_id}") + return info.tracker_class(**params) + + +def _frame_path(images_root: Path | None, seq_name: str, frame_id: int, *, dataset: str) -> Path | None: + if images_root is None: + return None + frame_seq = seq_name + if dataset == "mot17" and not seq_name.endswith("-FRCNN"): + frame_seq = f"{seq_name}-FRCNN" + return images_root / frame_seq / "img1" / f"{frame_id:06d}.jpg" + + +def _read_frame(path: Path | None) -> np.ndarray | None: + if path is None: + return None + if not path.is_file(): + raise FileNotFoundError(f"Missing frame for CMC: {path}") + import cv2 + + frame = cv2.imread(str(path)) + if frame is None: + raise RuntimeError(f"Failed to read frame: {path}") + return frame + + +def run_yolox_submit( + tracker_id: str, + params: dict, + *, + dataset: str, + split: str, + detections_dir: Path, + out_dir: Path, + images_root: Path | None = None, +) -> None: + out_dir.mkdir(parents=True, exist_ok=True) + tracker = _init_tracker(tracker_id, params) + + for det_file in sorted(detections_dir.glob("*.txt")): + tracker.reset() + seq_name = det_file.stem + det_list = det_file.read_text().splitlines() + if not det_list: + print(f" skip empty {seq_name}") + continue + dets_by_frame = _build_index(det_list) + last_frame = int(det_list[-1].split(",")[0]) + lines: list[str] = [] + + for frame_id in range(1, last_frame + 1): + raw = _yolox_rows(frame_id, dets_by_frame) + if raw: + arr = np.array(raw) + dets = sv.Detections(xyxy=arr[:, :4], confidence=arr[:, 4]) + else: + dets = sv.Detections.empty() + + frame = _read_frame(_frame_path(images_root, seq_name, frame_id, dataset=dataset)) + tracked = tracker.update(detections=dets, frame=frame) + if tracked.tracker_id is None: + continue + for tid, (left, top, right, bottom) in zip(tracked.tracker_id, tracked.xyxy): + if tid == -1: + continue + left_f, top_f, right_f, bottom_f = map(float, (left, top, right, bottom)) + if not np.isfinite((left_f, top_f, right_f, bottom_f)).all(): + continue + if right_f <= left_f or bottom_f <= top_f: + continue + lines.append(_write_mot_line(frame_id, int(tid), left_f, top_f, right_f, bottom_f)) + + (out_dir / f"{seq_name}.txt").write_text("".join(lines)) + print(f" tracked {seq_name} ({last_frame} frames)") + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--tracker", required=True) + p.add_argument("--dataset", choices=("mot17", "sportsmot", "dancetrack"), required=True) + p.add_argument("--split", default="test") + p.add_argument("--data-root", type=Path, default=_BENCHMARK_ROOT / "data") + p.add_argument("--output-dir", type=Path, required=True) + p.add_argument("--params", type=Path, default=None, help="JSON tracker params (default: library defaults)") + p.add_argument( + "--images-dir", + type=Path, + default=None, + help="Sequence root with /img1/ frames (required for BoT-SORT CMC on submit)", + ) + args = p.parse_args(argv) + + params: dict = {} + if args.params is not None: + params = json.loads(args.params.read_text()) + + dets = det_root(args.data_root, args.dataset, args.split) + if not dets.is_dir(): + print(f"Missing detections: {dets}", file=sys.stderr) + return 1 + + import importlib.metadata as md + + print(f"trackers {md.version('trackers')} | {args.tracker} | {args.dataset}/{args.split}") + run_yolox_submit( + args.tracker, + params, + dataset=args.dataset, + split=args.split, + detections_dir=dets, + out_dir=args.output_dir, + images_root=args.images_dir, + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/tracker_flags.py b/benchmark/scripts/tracker_flags.py new file mode 100644 index 000000000..16727dca1 --- /dev/null +++ b/benchmark/scripts/tracker_flags.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""Print ``trackers track`` flags from a params JSON file or library defaults.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[2] +_SRC = _REPO_ROOT / "src" +if _SRC.is_dir() and str(_SRC) not in sys.path: + sys.path.insert(0, str(_SRC)) + +from trackers.core.base import BaseTracker # noqa: E402 + + +def _is_class_param(name: str, param) -> bool: + if name == "state_estimator_class": + return True + default = param.default_value + return isinstance(default, type) + + +def tracker_flags(tracker_id: str, params: dict | None = None) -> str: + """Build CLI flags for one tracker. + + When *params* is empty, emit explicit ``--tracker.*`` flags from that + tracker's registry defaults. The CLI registers shared parameter names once + for all trackers (first registration wins), so omitting flags lets SORT and + others inherit BoT-SORT/ByteTrack defaults by mistake. + """ + info = BaseTracker._lookup_tracker(tracker_id) + if info is None: + raise ValueError(f"unknown tracker: {tracker_id}") + + if not params: + params = {name: param.default_value for name, param in info.parameters.items()} + + parts: list[str] = [] + for name, value in params.items(): + if name not in info.parameters: + continue + param = info.parameters[name] + if _is_class_param(name, param): + continue + if param.param_type is bool: + if value != param.default_value: + parts.append(f"--tracker.{name}") + else: + parts.extend([f"--tracker.{name}", str(value)]) + return " ".join(parts) + + +def main() -> int: + if len(sys.argv) not in {2, 3}: + print( + "usage: tracker_flags.py TRACKER [PARAMS.json|-]\n" + " Omit PARAMS or pass '-' to use library default hyperparameters.", + file=sys.stderr, + ) + return 1 + + tracker_id = sys.argv[1] + params_path = sys.argv[2] if len(sys.argv) == 3 else "-" + if params_path in {"-", "defaults", ""}: + params: dict = {} + else: + params = json.loads(Path(params_path).read_text()) + + try: + print(tracker_flags(tracker_id, params)) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 043b2d788f015fc5193e9e6c1b47010632b0c55d Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 14:30:20 -0300 Subject: [PATCH 02/54] Refactor benchmark workflow into Makefile plus focused scripts. --- benchmark/Makefile | 394 +++++++++--------- benchmark/README.md | 171 +++++++- benchmark/scripts/codabench_submit.py | 47 ++- benchmark/scripts/collect.py | 109 +++++ benchmark/scripts/data_check.py | 59 +++ benchmark/scripts/datasets.py | 142 +++++++ benchmark/scripts/mot17_server_format.py | 59 --- .../mot_challenge_submission_format.py | 70 ---- benchmark/scripts/mot_format.py | 92 ++++ benchmark/scripts/prep_benchmark.py | 200 --------- benchmark/scripts/prep_data.py | 99 +++++ benchmark/scripts/submit_yolox.py | 177 -------- benchmark/scripts/track_split.py | 115 +++++ benchmark/scripts/tracker_flags.py | 80 ---- 14 files changed, 985 insertions(+), 829 deletions(-) create mode 100644 benchmark/scripts/collect.py create mode 100644 benchmark/scripts/data_check.py create mode 100644 benchmark/scripts/datasets.py delete mode 100644 benchmark/scripts/mot17_server_format.py delete mode 100644 benchmark/scripts/mot_challenge_submission_format.py create mode 100644 benchmark/scripts/mot_format.py delete mode 100644 benchmark/scripts/prep_benchmark.py create mode 100644 benchmark/scripts/prep_data.py delete mode 100644 benchmark/scripts/submit_yolox.py create mode 100644 benchmark/scripts/track_split.py delete mode 100644 benchmark/scripts/tracker_flags.py diff --git a/benchmark/Makefile b/benchmark/Makefile index b60401129..b895a7872 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -1,228 +1,210 @@ -# MOT benchmark workflow (tune / eval / submit / Codabench upload). +# MOT benchmark workflow — Makefile orchestrates `trackers tune`, `trackers eval`, +# small helper scripts under scripts/, and `codabench_submit.py`. # -# Run from this directory: # cd benchmark -# make setup -# make eval TRACKER=sort DATASET=mot17 -# make submit upload-codabench TRACKER=sort DATASET=dancetrack CODABENCH_TOKEN=... +# make help +# make data-check +# make benchmark-default TRACKER=sort CODABENCH_TOKEN=... +# make benchmark-tuned TRACKER=sort CODABENCH_TOKEN=... +# make benchmark BENCHMARK_CONFIG=all TRACKER=sort CODABENCH_TOKEN=... # -# Dataset files live under benchmark/data/ (see README.md). Override with DATA_ROOT=... - -# ── Tools ───────────────────────────────────────────────────────────────────── - -SHELL := /bin/bash -ROOT := $(CURDIR) -REPO_ROOT := $(abspath $(ROOT)/..) -PYTHON ?= python -TRACKERS ?= $(PYTHON) -m trackers.scripts -TRACKERS_REPO := $(REPO_ROOT) -DATA_ROOT ?= $(ROOT)/data - -# ── Knobs ───────────────────────────────────────────────────────────────────── - -TRACKER ?= sort -DATASET ?= dancetrack -N_TRIALS ?= 10 -OBJECTIVE ?= HOTA -THRESHOLD ?= 0.5 -METRICS := CLEAR HOTA Identity -SEED ?= -PARAMS ?= +# See README.md for data setup. + +SHELL := /bin/bash +ROOT := $(CURDIR) +REPO_ROOT := $(abspath $(ROOT)/..) +PYTHON ?= python + +DATA_ROOT ?= $(ROOT)/data +PREP_DIR ?= $(ROOT)/benchmark_prep +OUTPUT_DIR ?= $(ROOT)/benchmark_outputs + +TRACKER ?= sort +DATASET ?= mot17 +CONFIG ?= default +BENCHMARK_CONFIG ?= default +N_TRIALS ?= 10 +OBJECTIVE ?= HOTA +THRESHOLD ?= 0.5 +SEED ?= FIXED_PARAMS ?= -CODABENCH_URL ?= https://www.codabench.org -CODABENCH_TOKEN ?= -CODABENCH_USERNAME ?= -CODABENCH_PASSWORD ?= -CODABENCH_DESCRIPTION ?= -CODABENCH_WAIT ?= 1 -CODABENCH_WAIT_TIMEOUT ?= 3600 -CODABENCH_POLL_INTERVAL ?= 10 - -PREP_DIR := $(ROOT)/benchmark_prep -OUTPUT_DIR := $(ROOT)/benchmark_outputs -JOB_DIR := $(OUTPUT_DIR)/$(TRACKER)/$(DATASET) -BEST_PARAMS := $(JOB_DIR)/best_params.json - -ifeq ($(DATASET),soccernet) - TUNE_SPLIT := train - EVAL_SPLIT := test - SUBMIT_SPLIT := - EVAL_GT_DIR := $(DATA_ROOT)/soccernet/TrackEval/data/gt/SoccerNet_tracking/SoccerNet_tracking_2022_all_gts - SEQMAP_TUNE := - SEQMAP_EVAL := - TUNE_IMAGES_DIR := $(DATA_ROOT)/soccernet/soccernet_data/tracking/train - EVAL_IMAGES_DIR := $(DATA_ROOT)/soccernet/soccernet_data/tracking/test - SUBMIT_IMAGES_DIR := -else ifeq ($(DATASET),dancetrack) - TUNE_SPLIT := train - EVAL_SPLIT := val - SUBMIT_SPLIT := test - EVAL_GT_DIR := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/val - SEQMAP_TUNE := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/DanceTrack-train.txt - SEQMAP_EVAL := $(DATA_ROOT)/dancetrack/TrackEval/data/gt/dancetrack/DanceTrack-val.txt - TUNE_IMAGES_DIR := $(DATA_ROOT)/dancetrack/train_images - EVAL_IMAGES_DIR := $(DATA_ROOT)/dancetrack/val_images - SUBMIT_IMAGES_DIR := $(DATA_ROOT)/dancetrack/test_images - SUBMIT_DETS_DIR := $(DATA_ROOT)/dancetrack/dancetrack_yolox_dets/test -else ifeq ($(DATASET),sportsmot) - TUNE_SPLIT := val - EVAL_SPLIT := val - SUBMIT_SPLIT := test - EVAL_GT_DIR := $(DATA_ROOT)/sportsmot/TrackEval/data/gt/sportsmot/val - SEQMAP_TUNE := - SEQMAP_EVAL := - TUNE_IMAGES_DIR := $(DATA_ROOT)/sportsmot/val - EVAL_IMAGES_DIR := $(TUNE_IMAGES_DIR) - SUBMIT_IMAGES_DIR := $(DATA_ROOT)/sportsmot/test - SUBMIT_DETS_DIR := $(DATA_ROOT)/sportsmot/sportsmot_yolox_dets/test -else ifeq ($(DATASET),mot17) - TUNE_SPLIT := val - EVAL_SPLIT := val - SUBMIT_SPLIT := test - EVAL_GT_DIR := $(DATA_ROOT)/mot17/TrackEval/data/gt/MOT17_yolox_val/train_val - SEQMAP_TUNE := $(DATA_ROOT)/mot17/TrackEval/data/gt/MOT17/MOT17-val.txt - SEQMAP_EVAL := $(SEQMAP_TUNE) - TUNE_IMAGES_DIR := $(DATA_ROOT)/mot17/val - EVAL_IMAGES_DIR := $(TUNE_IMAGES_DIR) - SUBMIT_IMAGES_DIR := $(DATA_ROOT)/mot17/test - SUBMIT_DETS_DIR := $(DATA_ROOT)/mot17/MOT17_yolox_dets/test -else - $(error Unknown DATASET=$(DATASET). Use: soccernet, dancetrack, sportsmot, mot17) -endif - -ifeq ($(DATASET),mot17) - CODABENCH_COMPETITION := 10049 - CODABENCH_PHASE := 16382 -else ifeq ($(DATASET),sportsmot) - CODABENCH_COMPETITION := 13077 - CODABENCH_PHASE := 21402 -else ifeq ($(DATASET),dancetrack) - CODABENCH_COMPETITION := 14885 - CODABENCH_PHASE := 24635 -endif +CODABENCH_URL ?= https://www.codabench.org +CODABENCH_TOKEN ?= +CODABENCH_WAIT_TIMEOUT ?= 3600 +CODABENCH_POLL_INTERVAL ?= 10 +# BoT-SORT requires CMC on by default for the published numbers. ifeq ($(TRACKER),botsort) ifeq ($(strip $(FIXED_PARAMS)),) FIXED_PARAMS := {"enable_cmc": true} endif - USE_IMAGES := 1 endif -TUNE_PREP := $(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT) -EVAL_PREP := $(PREP_DIR)/$(DATASET)/$(EVAL_SPLIT) -PRED_DIR := $(JOB_DIR)/pred_$(EVAL_SPLIT) -EVAL_JSON := $(JOB_DIR)/eval_$(EVAL_SPLIT).json -SUBMIT_DIR := $(JOB_DIR)/submit_$(SUBMIT_SPLIT) -SUBMIT_ZIP := $(JOB_DIR)/$(TRACKER)_$(DATASET)_$(SUBMIT_SPLIT)_submission.zip +CODABENCH_DATASETS := mot17 sportsmot dancetrack +LOCAL_DATASETS := soccernet +ALL_DATASETS := $(CODABENCH_DATASETS) $(LOCAL_DATASETS) +DATASETS ?= $(ALL_DATASETS) + +# Per-dataset tune splits. Score split is "test" for everyone (Codabench test for +# codabench datasets, public test GT for soccernet). Path lookups for image dirs +# delegate to scripts/datasets.py to stay DRY. +mot17_TUNE_SPLIT := val +sportsmot_TUNE_SPLIT := val +dancetrack_TUNE_SPLIT := train +soccernet_TUNE_SPLIT := train +SCORE_SPLIT := test + +# Codabench (competition_id, phase_id) per dataset. +mot17_CB := 10049 16382 +sportsmot_CB := 13077 21402 +dancetrack_CB := 14885 24635 -define resolve_params -if [ -n "$(PARAMS)" ]; then params_file="$(PARAMS)"; \ -elif [ -f "$(BEST_PARAMS)" ]; then params_file="$(BEST_PARAMS)"; \ -else echo "Using $(TRACKER) default parameters"; params_file="-"; fi; \ -flags=$$($(PYTHON) scripts/tracker_flags.py $(TRACKER) "$$params_file"); -endef +LAYOUT := $(PYTHON) scripts/datasets.py --data-root "$(DATA_ROOT)" -.PHONY: help setup tune eval submit upload-codabench all +.PHONY: help setup data-check prep prep-all tune track-default track-tuned _track-and-score \ + benchmark benchmark-default benchmark-tuned upload collect clean help: - @echo "Run from: cd benchmark && make " - @echo "Targets: setup | tune | eval | submit | upload-codabench | all" - @echo "DATA_ROOT=$(DATA_ROOT)" + @echo "MOT benchmark workflow — run from \`cd benchmark\`" + @echo "" + @echo "Targets:" + @echo " setup Install \`trackers[tune]\` from $(REPO_ROOT)" + @echo " data-check Print present/missing assets under $(DATA_ROOT)" + @echo " prep Prep one dataset (DATASET=...) into $(PREP_DIR)" + @echo " prep-all Prep every dataset" + @echo " tune Tune one (TRACKER=, DATASET=)" + @echo " track-default Track on default params, then score (TRACKER=, DATASET=)" + @echo " track-tuned Track on best_params.json, then score (TRACKER=, DATASET=)" + @echo " upload Upload an existing submission.zip (TRACKER=, DATASET=, CONFIG=default|tuned)" + @echo " benchmark Full pipeline (TRACKER=, BENCHMARK_CONFIG=default|tuned|all, DATASETS=...)" + @echo " benchmark-default Same as \`make benchmark BENCHMARK_CONFIG=default\`" + @echo " benchmark-tuned Same as \`make benchmark BENCHMARK_CONFIG=tuned\`" + @echo " collect Aggregate eval/codabench JSONs into tables.md (TRACKER=)" + @echo " clean Remove $(PREP_DIR) and $(OUTPUT_DIR)" + @echo "" + @echo "Codabench upload requires CODABENCH_TOKEN. See README for data setup." setup: - $(PYTHON) -m pip install -e "$(TRACKERS_REPO)[tune]" - $(PYTHON) scripts/prep_benchmark.py --data-root "$(DATA_ROOT)" --dataset $(DATASET) --split all + $(PYTHON) -m pip install -e "$(REPO_ROOT)[tune]" -tune: setup - @$(PYTHON) -c "import optuna" 2>/dev/null || { echo "Optuna missing. Run: make setup"; exit 1; } - @if [ -n "$(USE_IMAGES)" ]; then \ - test -d "$(TUNE_IMAGES_DIR)" || { echo "Missing $(TUNE_IMAGES_DIR)"; exit 1; }; \ +data-check: + @$(PYTHON) scripts/data_check.py --data-root "$(DATA_ROOT)" + +prep: + $(PYTHON) scripts/prep_data.py --dataset $(DATASET) --split all \ + --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)" + +prep-all: + $(PYTHON) scripts/prep_data.py --dataset all --split all \ + --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)" + +# Tune via the `trackers tune` CLI. Pass --images-dir only for BoT-SORT CMC (ByteTrack/SORT/OC-SORT are det-only). +tune: + @if [ ! -d "$(PREP_DIR)/$(DATASET)/$($(DATASET)_TUNE_SPLIT)" ]; then \ + echo "Run: make prep DATASET=$(DATASET)"; exit 1; \ + fi + @mkdir -p "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)" + $(eval TUNE_SPLIT := $($(DATASET)_TUNE_SPLIT)) + $(eval TUNE_IMAGES := $(shell $(LAYOUT) --dataset $(DATASET) --split $(TUNE_SPLIT) --field images_dir)) + trackers tune \ + --tracker $(TRACKER) \ + --gt-dir "$(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT)/gt" \ + --detections-dir "$(PREP_DIR)/$(DATASET)/$(TUNE_SPLIT)/dets" \ + --metrics CLEAR HOTA Identity --objective $(OBJECTIVE) --threshold $(THRESHOLD) \ + --n-trials $(N_TRIALS) \ + $(if $(SEED),--seed $(SEED),) \ + $(if $(FIXED_PARAMS),--fixed-params '$(FIXED_PARAMS)',) \ + $(if $(and $(filter botsort,$(TRACKER)),$(TUNE_IMAGES)),--images-dir "$(TUNE_IMAGES)",) \ + --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json" + +track-default: + @$(MAKE) _track-and-score TRACKER=$(TRACKER) DATASET=$(DATASET) CONFIG=default + +track-tuned: + @test -f "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json" || \ + { echo "missing best_params.json — run \`make tune TRACKER=$(TRACKER) DATASET=$(DATASET)\`"; exit 1; } + @$(MAKE) _track-and-score TRACKER=$(TRACKER) DATASET=$(DATASET) CONFIG=tuned + +# 1) track on the score split (test) → predictions +# 2) score: `trackers eval` for soccernet, `mot_format.py` + `codabench_submit.py` for the rest. +_track-and-score: + @if [ ! -d "$(PREP_DIR)/$(DATASET)/$(SCORE_SPLIT)" ]; then \ + echo "Run: make prep DATASET=$(DATASET)"; exit 1; \ fi - @mkdir -p "$(JOB_DIR)" - $(TRACKERS) tune \ - --tracker $(TRACKER) \ - --gt-dir "$(TUNE_PREP)/gt" \ - --detections-dir "$(TUNE_PREP)/dets" \ - --objective $(OBJECTIVE) \ - --n-trials $(N_TRIALS) \ - --metrics $(METRICS) \ - --threshold $(THRESHOLD) \ - $(if $(SEQMAP_TUNE),--seqmap "$(SEQMAP_TUNE)",) \ - $(if $(USE_IMAGES),--images-dir "$(TUNE_IMAGES_DIR)",) \ - $(if $(FIXED_PARAMS),--fixed-params '$(FIXED_PARAMS)',) \ - $(if $(SEED),--seed $(SEED),) \ - --output "$(BEST_PARAMS)" - -eval: - @test -d "$(EVAL_PREP)/dets" || { echo "Run: make setup DATASET=$(DATASET)"; exit 1; } - @mkdir -p "$(PRED_DIR)" - @set -euo pipefail; \ - $(resolve_params) \ - for det in "$(EVAL_PREP)/dets"/*.txt; do \ - seq=$$(basename "$$det" .txt); \ - echo "eval/track $$seq"; \ - source_args=(); \ - if [ -n "$(USE_IMAGES)" ]; then \ - frame_seq="$$seq"; \ - if [ "$(DATASET)" = "mot17" ] && [[ "$$seq" != *-FRCNN ]]; then frame_seq="$$seq-FRCNN"; fi; \ - img_dir="$(EVAL_IMAGES_DIR)/$$frame_seq/img1"; \ - test -d "$$img_dir" || { echo "Missing $$img_dir" >&2; exit 1; }; \ - source_args=(--source "$$img_dir"); \ + $(PYTHON) scripts/track_split.py \ + --tracker $(TRACKER) --dataset $(DATASET) --split $(SCORE_SPLIT) \ + --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)" \ + --output-dir "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)" \ + $(if $(filter tuned,$(CONFIG)),--params "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json",) + @if [ "$(DATASET)" = "soccernet" ]; then \ + trackers eval \ + --gt-dir "$(PREP_DIR)/soccernet/test/gt" \ + --tracker-dir "$(OUTPUT_DIR)/$(TRACKER)/soccernet/$(CONFIG)/pred" \ + --metrics CLEAR HOTA Identity --threshold $(THRESHOLD) \ + --output "$(OUTPUT_DIR)/$(TRACKER)/soccernet/$(CONFIG)/eval.json"; \ + else \ + [ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set — required for $(DATASET)."; exit 1; }; \ + $(PYTHON) scripts/mot_format.py --dataset $(DATASET) \ + --pred-dir "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/pred" \ + --out-zip "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip"; \ + $(PYTHON) scripts/codabench_submit.py \ + "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" \ + --competition-id $(word 1,$($(DATASET)_CB)) \ + --phase $(word 2,$($(DATASET)_CB)) \ + --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \ + --description "$(TRACKER) $(CONFIG) — benchmark Makefile" \ + --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \ + --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json"; \ + fi + +# Re-upload a zip without re-tracking (e.g. after Codabench daily limit resets). +upload: + @test -f "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" || \ + { echo "missing $(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip"; exit 1; } + @[ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set."; exit 1; } + @test "$(DATASET)" != "soccernet" || { echo "soccernet is scored locally — use track-default/track-tuned"; exit 1; } + $(PYTHON) scripts/codabench_submit.py \ + "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/submission.zip" \ + --competition-id $(word 1,$($(DATASET)_CB)) \ + --phase $(word 2,$($(DATASET)_CB)) \ + --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \ + --description "$(TRACKER) $(CONFIG) — benchmark Makefile" \ + --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \ + --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json" + +collect: + $(PYTHON) scripts/collect.py --tracker $(TRACKER) --output-dir "$(OUTPUT_DIR)" + +# Full pipeline. BENCHMARK_CONFIG=default|tuned|all (default: default only — one Codabench pass). +benchmark: + @case "$(BENCHMARK_CONFIG)" in default|tuned|all) ;; \ + *) echo "Set BENCHMARK_CONFIG=default, tuned, or all (or use benchmark-default / benchmark-tuned)"; exit 1;; \ + esac + @if [ -z "$(CODABENCH_TOKEN)" ]; then \ + echo "ERROR: CODABENCH_TOKEN must be set (required for mot17, sportsmot, dancetrack)."; exit 1; \ + fi + @$(MAKE) prep-all + @for d in $(DATASETS); do \ + if [ "$(BENCHMARK_CONFIG)" = "default" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ + echo ""; echo "===== [$$d] default ====="; \ + $(MAKE) track-default TRACKER=$(TRACKER) DATASET=$$d || exit 1; \ + fi; \ + if [ "$(BENCHMARK_CONFIG)" = "tuned" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ + echo ""; echo "===== [$$d] tune ====="; \ + $(MAKE) tune TRACKER=$(TRACKER) DATASET=$$d N_TRIALS=$(N_TRIALS) || exit 1; \ + echo "===== [$$d] tuned ====="; \ + $(MAKE) track-tuned TRACKER=$(TRACKER) DATASET=$$d || exit 1; \ fi; \ - $(TRACKERS) track \ - --detections "$$det" --tracker $(TRACKER) $$flags $${source_args[@]+"$${source_args[@]}"} \ - --mot-output "$(PRED_DIR)/$$seq.txt" --overwrite; \ done - $(TRACKERS) eval \ - --gt-dir "$(EVAL_GT_DIR)" --tracker-dir "$(PRED_DIR)" \ - --metrics $(METRICS) --threshold $(THRESHOLD) \ - --columns MOTA HOTA IDF1 \ - $(if $(SEQMAP_EVAL),--seqmap "$(SEQMAP_EVAL)",) \ - --output "$(EVAL_JSON)" - @echo "Saved → $(EVAL_JSON)" - -submit: - @test -n "$(SUBMIT_SPLIT)" || { echo "No submit split for $(DATASET)"; exit 1; } - @test -d "$(SUBMIT_DETS_DIR)" || { echo "Missing YOLOX detections: $(SUBMIT_DETS_DIR)"; exit 1; } - @mkdir -p "$(SUBMIT_DIR)" - @set -euo pipefail; \ - params_args=(); \ - if [ -n "$(PARAMS)" ]; then params_args=(--params "$(PARAMS)"); \ - elif [ -f "$(BEST_PARAMS)" ]; then params_args=(--params "$(BEST_PARAMS)"); fi; \ - images_args=(); \ - if [ -n "$(USE_IMAGES)" ]; then \ - test -d "$(SUBMIT_IMAGES_DIR)" || { echo "Missing frames: $(SUBMIT_IMAGES_DIR)" >&2; exit 1; }; \ - images_args=(--images-dir "$(SUBMIT_IMAGES_DIR)"); \ - fi; \ - $(PYTHON) scripts/submit_yolox.py \ - --tracker $(TRACKER) --dataset $(DATASET) --split $(SUBMIT_SPLIT) \ - --data-root "$(DATA_ROOT)" --output-dir "$(SUBMIT_DIR)" \ - $${params_args[@]+"$${params_args[@]}"} \ - $${images_args[@]+"$${images_args[@]}"} - @$(PYTHON) scripts/mot_challenge_submission_format.py "$(SUBMIT_DIR)" - @if [ "$(DATASET)" = "mot17" ]; then \ - $(PYTHON) scripts/mot17_server_format.py "$(SUBMIT_DIR)"; \ - fi - @rm -f "$(SUBMIT_ZIP)" - cd "$(SUBMIT_DIR)" && zip -r "$(SUBMIT_ZIP)" . - @echo "Created $(SUBMIT_ZIP)" - -upload-codabench: - @test "$(DATASET)" = "mot17" -o "$(DATASET)" = "sportsmot" -o "$(DATASET)" = "dancetrack" || \ - { echo "upload-codabench supports mot17, sportsmot, dancetrack"; exit 1; } - @test -f "$(SUBMIT_ZIP)" || $(MAKE) submit TRACKER=$(TRACKER) DATASET=$(DATASET) PARAMS="$(PARAMS)" - @test -n "$(CODABENCH_TOKEN)" -o -n "$(CODABENCH_USERNAME)" || \ - { echo "Set CODABENCH_TOKEN or CODABENCH_USERNAME+CODABENCH_PASSWORD"; exit 1; } - $(PYTHON) scripts/codabench_submit.py "$(SUBMIT_ZIP)" \ - --phase $(CODABENCH_PHASE) --competition-id $(CODABENCH_COMPETITION) \ - --base-url "$(CODABENCH_URL)" \ - $(if $(CODABENCH_TOKEN),--token "$(CODABENCH_TOKEN)",) \ - $(if $(CODABENCH_USERNAME),--username "$(CODABENCH_USERNAME)",) \ - $(if $(CODABENCH_PASSWORD),--password "$(CODABENCH_PASSWORD)",) \ - $(if $(CODABENCH_DESCRIPTION),--description "$(CODABENCH_DESCRIPTION)",) \ - $(if $(filter 0 false no,$(CODABENCH_WAIT)),--no-wait,) \ - --wait-timeout $(CODABENCH_WAIT_TIMEOUT) \ - --poll-interval $(CODABENCH_POLL_INTERVAL) - -all: tune eval submit + @echo ""; echo "===== collect =====" + @$(MAKE) collect TRACKER=$(TRACKER) + +benchmark-default: + @$(MAKE) benchmark BENCHMARK_CONFIG=default + +benchmark-tuned: + @$(MAKE) benchmark BENCHMARK_CONFIG=tuned + +clean: + rm -rf "$(PREP_DIR)" "$(OUTPUT_DIR)" diff --git a/benchmark/README.md b/benchmark/README.md index 0359b90b6..836f98420 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -1,42 +1,179 @@ # MOT benchmark workflow -Makefile-driven pipeline for tuning, local evaluation, test-set submission, and Codabench upload using the trackers CLI. +Reproduce the numbers in [`docs/trackers/comparison.md`](../docs/trackers/comparison.md) across **MOT17**, **SportsMOT**, **DanceTrack**, and **SoccerNet-tracking**. The Makefile runs tuning, tracking, Codabench submission (where required), and local evaluation, then writes doc-style tables. -Requires **`develop`** (trackers ≥ 2.3 with `track`, `eval`, `tune` CLIs). Install the repo editable from the parent directory: +Requires trackers ≥ 2.4. ```bash cd benchmark -make setup DATASET=mot17 +make setup ``` -## Data layout +## Quick start -Place benchmark assets under `benchmark/data/` (or set `DATA_ROOT=`): +```bash +cd benchmark + +export DATA_ROOT="/path/to/your/datasets" +export CODABENCH_TOKEN="" # see Codabench below + +make data-check +make benchmark-default TRACKER=bytetrack +make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 +``` + +Results: `benchmark_outputs//tables.md` and `summary.json`. + +Run **default** and **tuned** on separate days — Codabench limits submissions per phase. SoccerNet is scored locally and does not count toward that limit. + +## Codabench + +MOT17, SportsMOT, and DanceTrack test metrics come from [Codabench](https://www.codabench.org/). Register for each competition before uploading (approval may be required): + +| Dataset | Competition | +|---|---| +| MOT17 | [10049](https://www.codabench.org/competitions/10049/) | +| SportsMOT | [13077](https://www.codabench.org/competitions/13077/) | +| DanceTrack | [14885](https://www.codabench.org/competitions/14885/) | + +Request an API token with a one-time `curl` call (Codabench login — only the token is stored): + +```bash +curl -s -X POST https://www.codabench.org/api/api-token-auth/ \ + -H "Content-Type: application/json" \ + -d '{"username":"YOUR_USER","password":"YOUR_PASS"}' + +export CODABENCH_TOKEN="" +``` + +Treat `CODABENCH_TOKEN` as a secret — do not publish it. See [Codabench API docs](https://www.codabench.org/api/docs/) if the request fails. + +If tracking finished but upload failed (daily limit or pending approval), re-submit the zip without re-running track: + +```bash +make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned CODABENCH_TOKEN=... +``` + +Then `make collect TRACKER=bytetrack` to refresh the table. + +## Data setup + +Point `DATA_ROOT` at the folder that directly contains `mot17/`, `sportsmot/`, etc. Default: `./data`. ``` -data/ +$DATA_ROOT/ mot17/MOT17_yolox_dets/{val,test}/... + mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... + mot17/{val,test}//img1/... # BoT-SORT CMC only sportsmot/sportsmot_yolox_dets/{val,test}/... + sportsmot/TrackEval/data/gt/sportsmot/val/... dancetrack/dancetrack_yolox_dets/{train,val,test}/... + dancetrack/TrackEval/data/gt/dancetrack/{train,val}/... + dancetrack/{train,val,test}_images/... # BoT-SORT CMC (test optional) + soccernet/SoccerNet_dets/... + soccernet/TrackEval/data/gt/SoccerNet_tracking/... + soccernet/soccernet_data/tracking/{train,test}/... ``` -Use `trackers download` or your existing YOLOX det trees. For BoT-SORT CMC, also provide frame directories (`mot17/val`, `dancetrack/test_images`, etc.). +| Source | Assets | +|---|---| +| MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | +| SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | +| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) / [OC-SORT dets](https://github.com/noahcao/OC_SORT) | +| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking) | + +MOT17 and SportsMOT use model detections produced in-house with YOLOX, following each benchmark’s published detector configuration — the same approach described in [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). + +```bash +make data-check DATA_ROOT="/path/to/datasets" +``` + +## Splits and scoring + +| Dataset | Tune | Score | Scoring | +|---|---|---|---| +| MOT17 | val | test | Codabench | +| SportsMOT | val | test | Codabench | +| DanceTrack | train | test | Codabench | +| SoccerNet-tracking | train | test | Local (`trackers eval`) | + ## Commands +Run from `benchmark/`. Pass variables on the command line or export them first (`DATA_ROOT`, `CODABENCH_TOKEN`, …). + +| Target | Description | +|---|---| +| `setup` | Install `trackers[tune]` from the repo root | +| `data-check` | Print present/missing assets under `DATA_ROOT` | +| `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | +| `prep-all` | Prep all four datasets | +| `tune` | Optuna search → `best_params.json` (`TRACKER=`, `DATASET=`, `N_TRIALS=`) | +| `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | +| `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | +| `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | +| `benchmark-default` | `prep-all` → track-default on all datasets → `collect` | +| `benchmark-tuned` | `prep-all` → tune + track-tuned on all datasets → `collect` | +| `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | +| `collect` | Rebuild `tables.md` from existing score JSONs (`TRACKER=`) | +| `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | + +## Usage + +Full pipeline (runs `prep-all`, then `collect`): + ```bash -make eval TRACKER=sort DATASET=mot17 -make submit TRACKER=sort DATASET=dancetrack -make upload-codabench TRACKER=sort DATASET=mot17 CODABENCH_TOKEN=... +make benchmark-default TRACKER=bytetrack CODABENCH_TOKEN=... + +# Tune + tuned params (another 3 Codabench uploads + SoccerNet) +make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 CODABENCH_TOKEN=... + +# Skip datasets (e.g. MOT17 out of Codabench submissions for today) +make benchmark-tuned TRACKER=bytetrack N_TRIALS=5 \ + DATASETS="sportsmot dancetrack soccernet" CODABENCH_TOKEN=... + +# Both passes in one command (may hit daily limits) +make benchmark BENCHMARK_CONFIG=all TRACKER=bytetrack CODABENCH_TOKEN=... +``` + +Skip datasets (partial run or resume): + +```bash +make benchmark-tuned TRACKER=bytetrack DATASETS="sportsmot soccernet" CODABENCH_TOKEN=... ``` -| Dataset | Codabench | Phase | +Single dataset or step: + +```bash +make prep DATASET=mot17 +make tune TRACKER=bytetrack DATASET=mot17 N_TRIALS=50 +make track-default TRACKER=bytetrack DATASET=mot17 +make track-tuned TRACKER=bytetrack DATASET=mot17 +make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned +make collect TRACKER=bytetrack +make clean +``` + +### Variables + +| Variable | Default | Purpose | |---|---|---| -| MOT17 | [10049](https://www.codabench.org/competitions/10049/) | 16382 | -| SportsMOT | [13077](https://www.codabench.org/competitions/13077/) | 21402 | -| DanceTrack | [14885](https://www.codabench.org/competitions/14885/) | 24635 | +| `TRACKER` | `sort` | `sort`, `bytetrack`, `ocsort`, `botsort`, … | +| `DATA_ROOT` | `./data` | Raw dataset tree | +| `DATASET` | `mot17` | Single-dataset targets | +| `DATASETS` | all four | Space-separated subset for `benchmark*` | +| `BENCHMARK_CONFIG` | `default` | `benchmark`: `default`, `tuned`, or `all` | +| `CONFIG` | — | `upload`: `default` or `tuned` | +| `N_TRIALS` | `10` | Optuna trials per dataset | +| `CODABENCH_TOKEN` | — | Required for Codabench datasets | +| `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | +| `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | + +BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories when present. -## Implementation notes +## Notes -- **`make submit`** uses `scripts/submit_yolox.py` with library defaults (or `best_params.json`), not the shared `trackers track` CLI defaults. -- **`make eval`** passes explicit per-tracker flags via `scripts/tracker_flags.py`. +- **Tracking bypasses `trackers track`.** `scripts/track_split.py` loads the registry directly (workaround for a shared CLI parameter bug; see issue/PR). ByteTrack/SORT/OC-SORT never receive `--images-dir` during tune. +- **MOT17 server format.** `scripts/mot_format.py` triplicates `MOT17-XX.txt` into FRCNN/SDP/DPM files and stubs missing sequences for Codabench. +- **Resuming.** Steps are independent. Re-run `collect` after late uploads; use `upload` to submit an existing zip without re-tracking. +- Paths and splits live in `scripts/datasets.py`. diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 4303f2db5..47178c3e0 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -105,17 +105,6 @@ def _put_presigned_url(url: str, data: bytes, *, content_type: str = "applicatio conn.close() -def fetch_token(base_url: str, username: str, password: str) -> str: - _, payload = _request( - method="POST", - url=f"{base_url.rstrip('/')}/api/api-token-auth/", - json_body={"username": username, "password": password}, - ) - if not isinstance(payload, dict) or "token" not in payload: - raise RuntimeError(f"Unexpected token response: {payload!r}") - return str(payload["token"]) - - def can_make_submission(base_url: str, token: str, phase_id: int) -> tuple[bool, str]: _, payload = _request( method="GET", @@ -360,14 +349,9 @@ def resolve_token(args: argparse.Namespace) -> str: token = os.environ.get("CODABENCH_TOKEN", "").strip() if token: return token - username = args.username or os.environ.get("CODABENCH_USERNAME", "").strip() - password = args.password or os.environ.get("CODABENCH_PASSWORD", "").strip() - if username and password: - return fetch_token(args.base_url, username, password) raise RuntimeError( - "Missing API token. Set CODABENCH_TOKEN or pass --token, " - "or set CODABENCH_USERNAME and CODABENCH_PASSWORD. " - "Create a token via POST /api/api-token-auth/ (see Codabench API docs)." + "Missing API token. Set CODABENCH_TOKEN or pass --token. " + "Request a token via POST /api/api-token-auth/ (see benchmark/README.md)." ) @@ -402,8 +386,6 @@ def main(argv: list[str] | None = None) -> int: help="Codabench base URL.", ) p.add_argument("--token", help="API token (or env CODABENCH_TOKEN).") - p.add_argument("--username", help="Username for token auth (or env CODABENCH_USERNAME).") - p.add_argument("--password", help="Password for token auth (or env CODABENCH_PASSWORD).") p.add_argument("--description", default="", help="Optional submission description.") p.add_argument( "--dataset-name", @@ -434,6 +416,11 @@ def main(argv: list[str] | None = None) -> int: help="Leaderboard columns to print when finished (default: HOTA IDF1 MOTA).", ) p.add_argument("--dry-run", action="store_true", help="Check eligibility only; do not upload.") + p.add_argument( + "--output", + type=Path, + help="Optional path to write a JSON summary (status, scores, submission_id, competition_id, phase).", + ) args = p.parse_args(argv) metric_keys = tuple(args.metrics) @@ -502,6 +489,26 @@ def main(argv: list[str] | None = None) -> int: print(f"Done → id={sub_id} status={submission.get('status')}") if sub_id is not None: print(f" https://www.codabench.org/competitions/{comp_id}/") + + if args.output is not None: + try: + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps( + { + "submission_id": sub_id, + "competition_id": comp_id, + "phase_id": args.phase, + "status": submission.get("status"), + "scores": extract_metric_scores(submission, metric_keys), + }, + indent=2, + ) + ) + print(f" saved → {args.output}") + except OSError as exc: + print(f"warn: could not write {args.output}: {exc}", file=sys.stderr) + return 0 if str(submission.get("status", "")).lower() == "finished" else 1 diff --git a/benchmark/scripts/collect.py b/benchmark/scripts/collect.py new file mode 100644 index 000000000..8042ed59a --- /dev/null +++ b/benchmark/scripts/collect.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +"""Aggregate per-dataset eval/Codabench score JSONs into a single doc-style markdown table. + +Looks under ``////`` (config ∈ {default, tuned}) for: + + - ``eval.json`` → from `trackers eval --output ...` (SoccerNet local eval) + - ``codabench.json`` → from `codabench_submit.py --output ...` (MOT17/SportsMOT/DanceTrack) + +Writes ``//tables.md`` and ``//summary.json``. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +from datasets import DATASETS, LABELS, job_dir + +_CONFIGS = ("default", "tuned") +_TARGETS = ("HOTA", "IDF1", "MOTA") + + +def _from_eval_json(path: Path) -> dict[str, float]: + """Read TrackEval-style JSON saved by `trackers eval --output ...` and scale to percent.""" + data = json.loads(path.read_text()) + agg = data.get("aggregate", {}) + out: dict[str, float] = {} + for family in ("HOTA", "Identity", "CLEAR"): + block = agg.get(family) + if not isinstance(block, dict): + continue + for key in _TARGETS: + if key in block and key not in out: + out[key] = float(block[key]) * 100.0 + return out + + +def _from_codabench_json(path: Path) -> dict[str, float]: + """Read Codabench summary written by `codabench_submit.py --output ...`. Scores already 0-100.""" + data = json.loads(path.read_text()) + scores = data.get("scores") or {} + return {k: float(v) for k, v in scores.items() if k in _TARGETS} + + +def _row_scores(out_dir: Path, tracker: str, dataset: str, config: str) -> dict[str, float] | None: + base = job_dir(out_dir, tracker, dataset) / config + eval_json = base / "eval.json" + cb_json = base / "codabench.json" + if cb_json.is_file(): + return _from_codabench_json(cb_json) + if eval_json.is_file(): + return _from_eval_json(eval_json) + return None + + +def _format_table(rows: list[tuple[str, dict[str, float] | None]]) -> str: + header = "| Dataset | HOTA | IDF1 | MOTA |" + sep = "| :-------: | :--: | :--: | :--: |" + body = [] + for label, scores in rows: + if scores is None: + body.append(f"| {label:^9} | — | — | — |") + else: + cells = " | ".join(f"{scores.get(k, float('nan')):4.1f}" if k in scores else " — " for k in _TARGETS) + body.append(f"| {label:^9} | {cells} |") + return "\n".join([header, sep, *body]) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--tracker", required=True) + p.add_argument("--output-dir", type=Path, required=True) + p.add_argument("--datasets", default=",".join(DATASETS), help="Comma-separated subset; default=all.") + args = p.parse_args(argv) + + datasets = [d.strip() for d in args.datasets.split(",") if d.strip()] + summary: dict[str, dict[str, dict[str, float] | None]] = {} + sections: list[str] = [] + + for config in _CONFIGS: + rows = [] + any_present = False + for d in datasets: + scores = _row_scores(args.output_dir, args.tracker, d, config) + rows.append((LABELS.get(d, d), scores)) + summary.setdefault(d, {})[config] = scores + if scores is not None: + any_present = True + if not any_present: + continue + title = "Default parameters" if config == "default" else "Tuned parameters" + sections.append(f"## {title}\n\n{_format_table(rows)}\n") + + out = args.output_dir / args.tracker + out.mkdir(parents=True, exist_ok=True) + md = f"# {args.tracker} benchmark\n\n" + ("\n".join(sections) if sections else "_No scores found yet._\n") + (out / "tables.md").write_text(md) + (out / "summary.json").write_text(json.dumps({"tracker": args.tracker, "datasets": summary}, indent=2)) + + print(md) + print(f"saved → {out / 'tables.md'}") + print(f"saved → {out / 'summary.json'}") + return 0 if sections else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/data_check.py b/benchmark/scripts/data_check.py new file mode 100644 index 000000000..85aa3f5ea --- /dev/null +++ b/benchmark/scripts/data_check.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Walk the expected ``data/`` layout and print what's present vs missing per dataset. + +Use this before running ``make benchmark`` to verify the manual data setup. The +README documents where each asset is downloaded from. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from datasets import DATASETS, EVAL_SPLIT, SUBMIT_SPLIT, TUNE_SPLIT, split_paths + + +def _check(label: str, path: Path | None, *, required: bool) -> bool: + if path is None: + print(f" {label:<10} (n/a)") + return True + ok = path.is_dir() or path.is_file() + marker = "ok" if ok else ("MISS" if required else "skip") + print(f" {label:<10} {marker:<5} {path}") + return ok or not required + + +def check_dataset(data_root: Path, dataset: str) -> bool: + splits = sorted({TUNE_SPLIT[dataset], EVAL_SPLIT[dataset], *([SUBMIT_SPLIT[dataset]] if dataset in SUBMIT_SPLIT else [])}) + print(f"\n[{dataset}]") + ok = True + for split in splits: + try: + paths = split_paths(data_root, dataset, split) + except ValueError: + continue + print(f" {split}:") + ok &= _check("dets", paths.det_dir, required=True) + ok &= _check("gt", paths.gt_dir, required=split != SUBMIT_SPLIT.get(dataset)) + ok &= _check("images", paths.images_dir, required=False) + ok &= _check("seqmap", paths.seqmap, required=False) + return ok + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--data-root", type=Path, required=True) + p.add_argument("--dataset", choices=[*DATASETS, "all"], default="all") + args = p.parse_args(argv) + + print(f"checking data_root = {args.data_root}") + datasets = list(DATASETS) if args.dataset == "all" else [args.dataset] + all_ok = True + for dataset in datasets: + all_ok &= check_dataset(args.data_root, dataset) + print("\n" + ("All required assets found." if all_ok else "Some required assets missing — see README for download instructions.")) + return 0 if all_ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py new file mode 100644 index 000000000..7d0146308 --- /dev/null +++ b/benchmark/scripts/datasets.py @@ -0,0 +1,142 @@ +"""Benchmark dataset layout: paths, splits, Codabench targets. + +Single source of truth shared by all benchmark scripts. Not a CLI. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +BENCHMARK_ROOT = Path(__file__).resolve().parents[1] + +DATASETS = ("mot17", "sportsmot", "soccernet", "dancetrack") +LABELS = {"mot17": "MOT17", "sportsmot": "SportsMOT", "soccernet": "SoccerNet", "dancetrack": "DanceTrack"} + +# Per-dataset splits used by the benchmark workflow. +TUNE_SPLIT = {"soccernet": "train", "dancetrack": "train", "sportsmot": "val", "mot17": "val"} +EVAL_SPLIT = {"soccernet": "test", "dancetrack": "val", "sportsmot": "val", "mot17": "val"} +SUBMIT_SPLIT = {"dancetrack": "test", "sportsmot": "test", "mot17": "test"} # soccernet has no Codabench + +# Codabench (competition_id, phase_id) — submission targets for the published comparison table. +CODABENCH = { + "mot17": (10049, 16382), + "sportsmot": (13077, 21402), + "dancetrack": (14885, 24635), +} + +METRICS = ["CLEAR", "HOTA", "Identity"] + +_MOT17_EXISTING = ("01", "03", "06", "07", "08", "12", "14") +_MOT17_MISSING = ("02", "04", "05", "09", "10", "11", "13") +_MOT17_SUFFIXES = ("FRCNN", "SDP", "DPM") + + +def _soccernet_seq(stem: str) -> str: + return stem.replace("__det", "") + + +def _mot17_val_seq(stem: str) -> str: + return stem.split("_")[0] + "-FRCNN" + + +@dataclass(frozen=True) +class SplitPaths: + det_dir: Path + det_format: str # "mot" | "ltwh_mot" | "xyxy" + gt_dir: Path | None + images_dir: Path | None + seqmap: Path | None + seq_name_fn: Callable[[str], str] | None = None + + +def split_paths(data_root: Path, dataset: str, split: str) -> SplitPaths: + """Resolve where vendor detections, GT, frames, and seqmap live for one (dataset, split).""" + root = data_root + if dataset == "soccernet": + gt_root = root / "soccernet/TrackEval/data/gt/SoccerNet_tracking" + img_root = root / "soccernet/soccernet_data/tracking" + if split == "train": + return SplitPaths( + root / "soccernet/SoccerNet_dets/SoccerNet_tracking/train", + "mot", + gt_root / "train", + img_root / "train", + None, + _soccernet_seq, + ) + if split == "test": + return SplitPaths( + root / "soccernet/SoccerNet_dets/SoccerNet_tracking_2022_all_dets", + "ltwh_mot", + gt_root / "SoccerNet_tracking_2022_all_gts", + img_root / "test", + None, + _soccernet_seq, + ) + if dataset == "dancetrack": + gt = root / f"dancetrack/TrackEval/data/gt/dancetrack/{split}" if split != "test" else None + images = root / f"dancetrack/{split}_images" + seqmap = root / f"dancetrack/TrackEval/data/gt/dancetrack/DanceTrack-{split}.txt" + seqmap_or_none = seqmap if seqmap.parent.is_dir() else None + if split in {"train", "val", "test"}: + return SplitPaths(root / f"dancetrack/dancetrack_yolox_dets/{split}", "xyxy", gt, images, seqmap_or_none) + if dataset == "sportsmot": + if split in {"val", "test"}: + gt = root / "sportsmot/TrackEval/data/gt/sportsmot/val" if split == "val" else None + images = root / "sportsmot" / split + return SplitPaths(root / f"sportsmot/sportsmot_yolox_dets/{split}", "xyxy", gt, images, None) + if dataset == "mot17": + if split == "val": + seqmap = root / "mot17/TrackEval/data/gt/MOT17/MOT17-val.txt" + return SplitPaths( + root / "mot17/MOT17_yolox_dets/val", + "xyxy", + root / "mot17/TrackEval/data/gt/MOT17_yolox_val/train_val", + root / "mot17/val", + seqmap if seqmap.is_file() else None, + _mot17_val_seq, + ) + if split == "test": + return SplitPaths(root / "mot17/MOT17_yolox_dets/test", "xyxy", None, root / "mot17/test", None) + raise ValueError(f"unknown (dataset, split): ({dataset!r}, {split!r})") + + +def prep_split_dir(prep_root: Path, dataset: str, split: str) -> Path: + """Output dir for prepared flat MOT detections + GT for one (dataset, split).""" + return prep_root / dataset / split + + +def job_dir(output_dir: Path, tracker: str, dataset: str) -> Path: + """Per-(tracker, dataset) output directory for tuned params, predictions, scores.""" + return output_dir / tracker / dataset + + +def needs_frames(tracker: str, params: dict) -> bool: + """Whether tracking requires source frames (e.g. BoT-SORT with CMC enabled).""" + return tracker == "botsort" and bool(params.get("enable_cmc", False)) + + +def mot17_server_filenames() -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: + """Constants for the MOT17 Codabench server: must triplicate (FRCNN/SDP/DPM) and stub missing seqs.""" + return _MOT17_EXISTING, _MOT17_MISSING, _MOT17_SUFFIXES + + +def _print_field(data_root: Path, dataset: str, split: str, what: str) -> str: + paths = split_paths(data_root, dataset, split) + value = getattr(paths, what) + return "" if value is None else str(value) + + +# Tiny CLI so the Makefile can query layout values without duplicating paths. +if __name__ == "__main__": + import argparse + + p = argparse.ArgumentParser(description="Print one layout field (used by Makefile).") + p.add_argument("--data-root", type=Path, required=True) + p.add_argument("--dataset", required=True) + p.add_argument("--split", required=True) + p.add_argument("--field", choices=["det_dir", "gt_dir", "images_dir", "seqmap"], required=True) + args = p.parse_args() + print(_print_field(args.data_root, args.dataset, args.split, args.field)) diff --git a/benchmark/scripts/mot17_server_format.py b/benchmark/scripts/mot17_server_format.py deleted file mode 100644 index 097237a46..000000000 --- a/benchmark/scripts/mot17_server_format.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -"""Expand MOT17-XX.txt tracker outputs into Codabench/MOTChallenge server layout.""" - -from __future__ import annotations - -import argparse -from pathlib import Path - -# Sequences with YOLOX test detections in this benchmark setup. -_EXISTING = ("01", "03", "06", "07", "08", "12", "14") -# Sequences without test detections — server expects empty placeholder files. -_MISSING = ("02", "04", "05", "09", "10", "11", "13") -_SUFFIXES = ("FRCNN", "SDP", "DPM") - - -def write_mot17_server_format(out_dir: Path) -> int: - """Triplicate tracked results and add empty files for missing sequences.""" - if not out_dir.is_dir(): - raise FileNotFoundError(f"Not a directory: {out_dir}") - - written = 0 - for num in _EXISTING: - src = out_dir / f"MOT17-{num}.txt" - if not src.is_file(): - print(f" Missing expected source: {src}", flush=True) - continue - content = src.read_bytes() - for suf in _SUFFIXES: - (out_dir / f"MOT17-{num}-{suf}.txt").write_bytes(content) - written += 1 - src.unlink() - - for num in _MISSING: - for suf in _SUFFIXES: - (out_dir / f"MOT17-{num}-{suf}.txt").touch(exist_ok=True) - written += 1 - - print(f" MOT17 server format: {written} files in {out_dir}", flush=True) - return written - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument( - "out_dir", - type=Path, - help="Directory containing MOT17-XX.txt tracker outputs (modified in place).", - ) - args = p.parse_args(argv) - try: - write_mot17_server_format(args.out_dir.resolve()) - except FileNotFoundError as exc: - print(str(exc), flush=True) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/scripts/mot_challenge_submission_format.py b/benchmark/scripts/mot_challenge_submission_format.py deleted file mode 100644 index e60baf04d..000000000 --- a/benchmark/scripts/mot_challenge_submission_format.py +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -"""Normalize tracker MOT outputs for MOTChallenge / Codabench submission. - -Tracking output should already omit unassigned rows (``tracker_id=-1``) and use -0-based track IDs with ``.1f`` box coordinates and ``conf=-1``. This step -drops any negative IDs as a safety net and rewrites rows to the notebook / -docs submission layout. -""" - -from __future__ import annotations - -import argparse -from pathlib import Path - - -def normalize_mot_submission_line(line: str) -> str | None: - parts = line.strip().split(",") - if len(parts) < 7: - return None - try: - frame = int(float(parts[0])) - track_id = int(float(parts[1])) - except ValueError: - return None - if track_id < 0: - return None - - left, top, width, height = (float(parts[i]) for i in range(2, 6)) - return ( - f"{frame},{track_id},{left:.1f},{top:.1f},{width:.1f},{height:.1f}," - f"-1,-1,-1,-1" - ) - - -def normalize_mot_submission_file(path: Path) -> int: - lines_out: list[str] = [] - for raw in path.read_text().splitlines(): - normalized = normalize_mot_submission_line(raw) - if normalized is not None: - lines_out.append(normalized) - path.write_text("\n".join(lines_out) + ("\n" if lines_out else "")) - return len(lines_out) - - -def normalize_mot_submission_dir(out_dir: Path) -> int: - if not out_dir.is_dir(): - raise FileNotFoundError(f"Not a directory: {out_dir}") - total_lines = 0 - n_files = 0 - for path in sorted(out_dir.glob("*.txt")): - total_lines += normalize_mot_submission_file(path) - n_files += 1 - print(f" MOT submission format: {n_files} files, {total_lines} lines in {out_dir}") - return total_lines - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("out_dir", type=Path, help="Directory of per-sequence .txt files (modified in place).") - args = p.parse_args(argv) - try: - normalize_mot_submission_dir(args.out_dir.resolve()) - except FileNotFoundError as exc: - print(str(exc)) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/scripts/mot_format.py b/benchmark/scripts/mot_format.py new file mode 100644 index 000000000..ae2a5d274 --- /dev/null +++ b/benchmark/scripts/mot_format.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Format a MOT prediction directory for Codabench submission. + +Steps applied in order: + +1. Normalize every line to ``frame,id,left,top,w,h,-1,-1,-1,-1`` and drop rows with id < 0. +2. For MOT17, triplicate ``MOT17-XX`` → ``MOT17-XX-{FRCNN,SDP,DPM}`` and stub the + sequences not present in the YOLOX detection set. +3. Zip every ``*.txt`` at the archive root (no nested directories). + +Usage: + + python mot_format.py --dataset mot17 --pred-dir --out-zip +""" + +from __future__ import annotations + +import argparse +import sys +import zipfile +from pathlib import Path + +from datasets import DATASETS, mot17_server_filenames + + +def _normalize_line(raw: str) -> str | None: + parts = raw.strip().split(",") + if len(parts) < 7: + return None + try: + frame = int(float(parts[0])) + track_id = int(float(parts[1])) + except ValueError: + return None + if track_id < 0: + return None + left, top, w, h = (float(parts[i]) for i in range(2, 6)) + return f"{frame},{track_id},{left:.1f},{top:.1f},{w:.1f},{h:.1f},-1,-1,-1,-1" + + +def normalize_dir(pred_dir: Path) -> None: + for path in sorted(pred_dir.glob("*.txt")): + cleaned = [line for raw in path.read_text().splitlines() if (line := _normalize_line(raw))] + path.write_text("\n".join(cleaned) + ("\n" if cleaned else "")) + + +def mot17_triplicate(pred_dir: Path) -> None: + """MOT17 Codabench server expects FRCNN/SDP/DPM triplets and zero-fills for missing seqs.""" + existing, missing, suffixes = mot17_server_filenames() + for num in existing: + src = pred_dir / f"MOT17-{num}.txt" + if not src.is_file(): + continue + data = src.read_bytes() + for suf in suffixes: + (pred_dir / f"MOT17-{num}-{suf}.txt").write_bytes(data) + src.unlink() + for num in missing: + for suf in suffixes: + (pred_dir / f"MOT17-{num}-{suf}.txt").touch(exist_ok=True) + + +def zip_dir(pred_dir: Path, out_zip: Path) -> Path: + if out_zip.is_file(): + out_zip.unlink() + out_zip.parent.mkdir(parents=True, exist_ok=True) + with zipfile.ZipFile(out_zip, "w", zipfile.ZIP_DEFLATED) as zf: + for path in sorted(pred_dir.glob("*.txt")): + zf.write(path, arcname=path.name) + return out_zip + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dataset", choices=DATASETS, required=True) + p.add_argument("--pred-dir", type=Path, required=True, help="Directory of MOT prediction txt files (modified in place).") + p.add_argument("--out-zip", type=Path, required=True) + args = p.parse_args(argv) + + if not args.pred_dir.is_dir(): + print(f"missing pred dir: {args.pred_dir}", file=sys.stderr) + return 1 + normalize_dir(args.pred_dir) + if args.dataset == "mot17": + mot17_triplicate(args.pred_dir) + zip_path = zip_dir(args.pred_dir, args.out_zip) + print(f"wrote {zip_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/prep_benchmark.py b/benchmark/scripts/prep_benchmark.py deleted file mode 100644 index 41fc106aa..000000000 --- a/benchmark/scripts/prep_benchmark.py +++ /dev/null @@ -1,200 +0,0 @@ -#!/usr/bin/env python3 -"""Convert local benchmark detections/GT into flat MOT dirs for ``trackers tune``.""" - -from __future__ import annotations - -import argparse -import shutil -import sys -from dataclasses import dataclass -from pathlib import Path -from typing import Callable - -_BENCHMARK_ROOT = Path(__file__).resolve().parents[1] - - -def _soccer_seq_name(stem: str) -> str: - return stem.replace("__det", "") - - -def _mot17_val_seq_name(stem: str) -> str: - return stem.split("_")[0] + "-FRCNN" - - -@dataclass(frozen=True) -class SplitSpec: - det_dir: Path - gt_dir: Path | None - det_format: str # xyxy | mot | ltwh_mot - seq_name_fn: Callable[[str], str] | None = None - - -def split_spec(data_root: Path, dataset: str, split: str) -> SplitSpec | None: - root = data_root - if dataset == "soccernet": - if split == "train": - return SplitSpec( - det_dir=root / "soccernet/SoccerNet_dets/SoccerNet_tracking/train", - gt_dir=root / "soccernet/TrackEval/data/gt/SoccerNet_tracking/train", - det_format="mot", - seq_name_fn=_soccer_seq_name, - ) - if split == "test": - return SplitSpec( - det_dir=root / "soccernet/SoccerNet_dets/SoccerNet_tracking_2022_all_dets", - gt_dir=root / "soccernet/TrackEval/data/gt/SoccerNet_tracking/SoccerNet_tracking_2022_all_gts", - det_format="ltwh_mot", - seq_name_fn=_soccer_seq_name, - ) - if dataset == "dancetrack" and split in {"train", "val", "test"}: - gt_dir = None if split == "test" else root / f"dancetrack/TrackEval/data/gt/dancetrack/{split}" - return SplitSpec( - det_dir=root / f"dancetrack/dancetrack_yolox_dets/{split}", - gt_dir=gt_dir, - det_format="xyxy", - ) - if dataset == "sportsmot" and split in {"val", "test"}: - gt_dir = None if split == "test" else root / f"sportsmot/TrackEval/data/gt/sportsmot/{split}" - return SplitSpec( - det_dir=root / f"sportsmot/sportsmot_yolox_dets/{split}", - gt_dir=gt_dir, - det_format="xyxy", - ) - if dataset == "mot17": - if split == "val": - return SplitSpec( - det_dir=root / "mot17/MOT17_yolox_dets/val", - gt_dir=root / "mot17/TrackEval/data/gt/MOT17_yolox_val/train_val", - det_format="xyxy", - seq_name_fn=_mot17_val_seq_name, - ) - if split == "test": - return SplitSpec( - det_dir=root / "mot17/MOT17_yolox_dets/test", - gt_dir=None, - det_format="xyxy", - ) - return None - - -def prepare_mot_dets( - src_dir: Path, - dst_dir: Path, - *, - src_format: str, - seq_name_fn: Callable[[str], str] | None = None, -) -> None: - dst_dir.mkdir(parents=True, exist_ok=True) - for det_file in sorted(src_dir.glob("*.txt")): - seq_name = seq_name_fn(det_file.stem) if seq_name_fn else det_file.stem - dst_path = dst_dir / f"{seq_name}.txt" - if src_format == "mot": - shutil.copy(det_file, dst_path) - continue - with det_file.open() as fin, dst_path.open("w") as fout: - for line in fin: - parts = line.strip().split(",") - if len(parts) < 6: - continue - frame = int(parts[0]) - if src_format == "ltwh_mot": - left, top, w, h = (float(parts[i]) for i in range(2, 6)) - conf = float(parts[6]) if len(parts) > 6 else 1.0 - else: - x1, y1, x2, y2, conf = (float(p) for p in parts[1:6]) - left, top, w, h = x1, y1, x2 - x1, y2 - y1 - fout.write(f"{frame},-1,{left:.4f},{top:.4f},{w:.4f},{h:.4f},{conf:.4f}\n") - - -def prepare_flat_gt(gt_root: Path, dst_dir: Path) -> None: - dst_dir.mkdir(parents=True, exist_ok=True) - for seq_dir in sorted(gt_root.iterdir()): - if not seq_dir.is_dir(): - continue - gt_path = seq_dir / "gt" / "gt.txt" - if gt_path.is_file(): - shutil.copy(gt_path, dst_dir / f"{seq_dir.name}.txt") - - -def prep_split(data_root: Path, prep_root: Path, dataset: str, split: str) -> Path: - spec = split_spec(data_root, dataset, split) - if spec is None: - raise ValueError(f"Unknown dataset/split: {dataset}/{split}") - if not spec.det_dir.is_dir(): - raise FileNotFoundError(f"Missing detections: {spec.det_dir}") - - out = prep_root / dataset / split - dets_out = out / "dets" - gt_out = out / "gt" - prepare_mot_dets( - spec.det_dir, - dets_out, - src_format=spec.det_format, - seq_name_fn=spec.seq_name_fn, - ) - if spec.gt_dir is not None: - if not spec.gt_dir.is_dir(): - raise FileNotFoundError(f"Missing GT: {spec.gt_dir}") - prepare_flat_gt(spec.gt_dir, gt_out) - print(f"Prepared {dataset}/{split} → {out}") - return out - - -def main(argv: list[str] | None = None) -> int: - datasets = ("soccernet", "dancetrack", "sportsmot", "mot17") - tune_splits = { - "soccernet": "train", - "dancetrack": "train", - "sportsmot": "val", - "mot17": "val", - } - eval_splits = { - "soccernet": "test", - "dancetrack": "val", - "sportsmot": "val", - "mot17": "val", - } - submit_splits = { - "dancetrack": "test", - "sportsmot": "test", - "mot17": "test", - } - - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--data-root", type=Path, default=_BENCHMARK_ROOT / "data") - p.add_argument("--prep-dir", type=Path, default=_BENCHMARK_ROOT / "benchmark_prep") - p.add_argument("--dataset", choices=[*datasets, "all"], default="all") - p.add_argument( - "--split", - choices=["all", "tune", "eval", "submit", "train", "val", "test"], - default="tune", - help="Which split to prep (all=tune+eval+submit for dataset, or explicit split name).", - ) - args = p.parse_args(argv) - - picked = list(datasets) if args.dataset == "all" else [args.dataset] - split_aliases = ("tune", "eval", "submit") - for dataset in picked: - splits_to_run: list[str] - if args.split == "all": - splits_to_run = list(split_aliases) - else: - splits_to_run = [args.split] - - for split_key in splits_to_run: - if split_key in {"tune", "eval", "submit"}: - split_map = {"tune": tune_splits, "eval": eval_splits, "submit": submit_splits} - if split_key == "submit" and dataset not in submit_splits: - continue - split = split_map[split_key][dataset] - else: - split = split_key - try: - prep_split(args.data_root, args.prep_dir, dataset, split) - except (FileNotFoundError, ValueError) as exc: - print(f"SKIP {dataset}/{split}: {exc}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/scripts/prep_data.py b/benchmark/scripts/prep_data.py new file mode 100644 index 000000000..857b8765a --- /dev/null +++ b/benchmark/scripts/prep_data.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +"""Convert vendor MOT detections + GT into flat per-sequence MOT files. + +Output layout (under ``--prep-dir``): + + /// + dets/.txt # MOT lines: frame,-1,left,top,width,height,conf + gt/.txt # vanilla MOT gt.txt (copied) + +Run via the Makefile (``make prep``) or directly: + + python prep_data.py --dataset mot17 --split val --data-root ./data --prep-dir ./benchmark_prep +""" + +from __future__ import annotations + +import argparse +import shutil +import sys +from pathlib import Path + +from datasets import DATASETS, EVAL_SPLIT, SUBMIT_SPLIT, TUNE_SPLIT, prep_split_dir, split_paths + + +def _convert_dets(src: Path, dst: Path, fmt: str) -> None: + """Write one MOT-format detection file from a vendor file.""" + if fmt == "mot": + shutil.copy(src, dst) + return + with src.open() as fin, dst.open("w") as fout: + for raw in fin: + parts = raw.strip().split(",") + if len(parts) < 6: + continue + frame = int(parts[0]) + if fmt == "ltwh_mot": + left, top, w, h = (float(parts[i]) for i in range(2, 6)) + conf = float(parts[6]) if len(parts) > 6 else 1.0 + elif fmt == "xyxy": + x1, y1, x2, y2, conf = (float(p) for p in parts[1:6]) + left, top, w, h = x1, y1, x2 - x1, y2 - y1 + else: + raise ValueError(f"unknown det format: {fmt}") + fout.write(f"{frame},-1,{left:.4f},{top:.4f},{w:.4f},{h:.4f},{conf:.4f}\n") + + +def prep_split(data_root: Path, prep_root: Path, dataset: str, split: str) -> Path: + """Prepare flat MOT dets (+ optional GT) for one (dataset, split). Returns the output dir.""" + paths = split_paths(data_root, dataset, split) + if not paths.det_dir.is_dir(): + raise FileNotFoundError(f"missing detections: {paths.det_dir}") + out = prep_split_dir(prep_root, dataset, split) + dets_out = out / "dets" + dets_out.mkdir(parents=True, exist_ok=True) + for det_file in sorted(paths.det_dir.glob("*.txt")): + seq = paths.seq_name_fn(det_file.stem) if paths.seq_name_fn else det_file.stem + _convert_dets(det_file, dets_out / f"{seq}.txt", paths.det_format) + if paths.gt_dir is not None: + if not paths.gt_dir.is_dir(): + raise FileNotFoundError(f"missing GT: {paths.gt_dir}") + gt_out = out / "gt" + gt_out.mkdir(parents=True, exist_ok=True) + for seq_dir in sorted(paths.gt_dir.iterdir()): + gt = seq_dir / "gt" / "gt.txt" + if seq_dir.is_dir() and gt.is_file(): + shutil.copy(gt, gt_out / f"{seq_dir.name}.txt") + print(f"prepped {dataset}/{split} → {out}") + return out + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--dataset", choices=[*DATASETS, "all"], required=True) + p.add_argument("--split", choices=["all", "tune", "eval", "submit"], default="all") + p.add_argument("--data-root", type=Path, required=True) + p.add_argument("--prep-dir", type=Path, required=True) + args = p.parse_args(argv) + + datasets = list(DATASETS) if args.dataset == "all" else [args.dataset] + failed = False + for dataset in datasets: + splits: set[str] = set() + if args.split in {"all", "tune"}: + splits.add(TUNE_SPLIT[dataset]) + if args.split in {"all", "eval"}: + splits.add(EVAL_SPLIT[dataset]) + if args.split in {"all", "submit"} and dataset in SUBMIT_SPLIT: + splits.add(SUBMIT_SPLIT[dataset]) + for split in sorted(splits): + try: + prep_split(args.data_root, args.prep_dir, dataset, split) + except FileNotFoundError as exc: + print(f"SKIP {dataset}/{split}: {exc}", file=sys.stderr) + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/submit_yolox.py b/benchmark/scripts/submit_yolox.py deleted file mode 100644 index adba082af..000000000 --- a/benchmark/scripts/submit_yolox.py +++ /dev/null @@ -1,177 +0,0 @@ -#!/usr/bin/env python3 -"""Build a Codabench submission from raw YOLOX detections (notebook-style loop). - -Uses each tracker's library defaults (or an optional params JSON) directly, -avoiding the ``trackers track`` CLI shared-parameter default bug. -""" - -from __future__ import annotations - -import argparse -import json -import sys -from collections import defaultdict -from pathlib import Path - -import numpy as np -import supervision as sv - -_BENCHMARK_ROOT = Path(__file__).resolve().parents[1] - - -def det_root(data_root: Path, dataset: str, split: str) -> Path: - rel = { - "mot17": data_root / "mot17" / "MOT17_yolox_dets" / split, - "sportsmot": data_root / "sportsmot" / "sportsmot_yolox_dets" / split, - "dancetrack": data_root / "dancetrack" / "dancetrack_yolox_dets" / split, - } - if dataset not in rel: - raise ValueError(f"unsupported dataset: {dataset}") - return rel[dataset] - - -def _build_index(det_list: list[str]) -> dict[int, list[str]]: - dets_by_frame: dict[int, list[str]] = defaultdict(list) - for line in det_list: - dets_by_frame[int(line.split(",")[0])].append(line) - return dets_by_frame - - -def _yolox_rows(frame_id: int, dets_by_frame: dict[int, list[str]]) -> list[list[float]]: - rows: list[list[float]] = [] - for line in dets_by_frame.get(frame_id, []): - parts = line.split(",") - rows.append([float(parts[1]), float(parts[2]), float(parts[3]), float(parts[4]), float(parts[5])]) - return rows - - -def _write_mot_line(frame_id: int, track_id: int, left: float, top: float, right: float, bottom: float) -> str: - width = right - left - height = bottom - top - return f"{frame_id},{int(track_id)},{left:.1f},{top:.1f},{width:.1f},{height:.1f},-1,-1,-1,-1\n" - - -def _init_tracker(tracker_id: str, params: dict): - import trackers as _trackers # noqa: F401 - from trackers.core.base import BaseTracker - - info = BaseTracker._lookup_tracker(tracker_id) - if info is None: - raise ValueError(f"unknown tracker: {tracker_id}") - return info.tracker_class(**params) - - -def _frame_path(images_root: Path | None, seq_name: str, frame_id: int, *, dataset: str) -> Path | None: - if images_root is None: - return None - frame_seq = seq_name - if dataset == "mot17" and not seq_name.endswith("-FRCNN"): - frame_seq = f"{seq_name}-FRCNN" - return images_root / frame_seq / "img1" / f"{frame_id:06d}.jpg" - - -def _read_frame(path: Path | None) -> np.ndarray | None: - if path is None: - return None - if not path.is_file(): - raise FileNotFoundError(f"Missing frame for CMC: {path}") - import cv2 - - frame = cv2.imread(str(path)) - if frame is None: - raise RuntimeError(f"Failed to read frame: {path}") - return frame - - -def run_yolox_submit( - tracker_id: str, - params: dict, - *, - dataset: str, - split: str, - detections_dir: Path, - out_dir: Path, - images_root: Path | None = None, -) -> None: - out_dir.mkdir(parents=True, exist_ok=True) - tracker = _init_tracker(tracker_id, params) - - for det_file in sorted(detections_dir.glob("*.txt")): - tracker.reset() - seq_name = det_file.stem - det_list = det_file.read_text().splitlines() - if not det_list: - print(f" skip empty {seq_name}") - continue - dets_by_frame = _build_index(det_list) - last_frame = int(det_list[-1].split(",")[0]) - lines: list[str] = [] - - for frame_id in range(1, last_frame + 1): - raw = _yolox_rows(frame_id, dets_by_frame) - if raw: - arr = np.array(raw) - dets = sv.Detections(xyxy=arr[:, :4], confidence=arr[:, 4]) - else: - dets = sv.Detections.empty() - - frame = _read_frame(_frame_path(images_root, seq_name, frame_id, dataset=dataset)) - tracked = tracker.update(detections=dets, frame=frame) - if tracked.tracker_id is None: - continue - for tid, (left, top, right, bottom) in zip(tracked.tracker_id, tracked.xyxy): - if tid == -1: - continue - left_f, top_f, right_f, bottom_f = map(float, (left, top, right, bottom)) - if not np.isfinite((left_f, top_f, right_f, bottom_f)).all(): - continue - if right_f <= left_f or bottom_f <= top_f: - continue - lines.append(_write_mot_line(frame_id, int(tid), left_f, top_f, right_f, bottom_f)) - - (out_dir / f"{seq_name}.txt").write_text("".join(lines)) - print(f" tracked {seq_name} ({last_frame} frames)") - - -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__) - p.add_argument("--tracker", required=True) - p.add_argument("--dataset", choices=("mot17", "sportsmot", "dancetrack"), required=True) - p.add_argument("--split", default="test") - p.add_argument("--data-root", type=Path, default=_BENCHMARK_ROOT / "data") - p.add_argument("--output-dir", type=Path, required=True) - p.add_argument("--params", type=Path, default=None, help="JSON tracker params (default: library defaults)") - p.add_argument( - "--images-dir", - type=Path, - default=None, - help="Sequence root with /img1/ frames (required for BoT-SORT CMC on submit)", - ) - args = p.parse_args(argv) - - params: dict = {} - if args.params is not None: - params = json.loads(args.params.read_text()) - - dets = det_root(args.data_root, args.dataset, args.split) - if not dets.is_dir(): - print(f"Missing detections: {dets}", file=sys.stderr) - return 1 - - import importlib.metadata as md - - print(f"trackers {md.version('trackers')} | {args.tracker} | {args.dataset}/{args.split}") - run_yolox_submit( - args.tracker, - params, - dataset=args.dataset, - split=args.split, - detections_dir=dets, - out_dir=args.output_dir, - images_root=args.images_dir, - ) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmark/scripts/track_split.py b/benchmark/scripts/track_split.py new file mode 100644 index 000000000..b0fecce4c --- /dev/null +++ b/benchmark/scripts/track_split.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Run one tracker over a prepared MOT detection directory and write MOT predictions. + +This script intentionally bypasses the `trackers track` CLI for two reasons: + +1. Per-sequence subprocess startup dominates wall time for fast trackers (SORT/ByteTrack). +2. The CLI currently has a shared-parameter bug where flag defaults can leak between + trackers (#TODO: fix and switch this script to a Makefile loop calling `trackers track`). + +We sidestep both by importing the registry directly: defaults come from the chosen +tracker's `ParameterInfo`, optionally overridden by a tuned `best_params.json`, and +the merged dict is filtered to the tracker's `__init__` signature. + +Usage (see Makefile for the wiring): + + python track_split.py --tracker sort --dataset mot17 --split val \ + --prep-dir ./benchmark_prep --output-dir ./benchmark_outputs/sort/mot17/default \ + [--params best_params.json] +""" + +from __future__ import annotations + +import argparse +import inspect +import json +import sys +from pathlib import Path +from typing import Any + +from datasets import DATASETS, needs_frames, split_paths + +# trackers is installed as a regular package via `pip install -e ../`. +from trackers.core.base import BaseTracker +from trackers.tune.tuner import _run_tracker_on_detections + + +def _registry_defaults(tracker_id: str) -> dict[str, Any]: + info = BaseTracker._lookup_tracker(tracker_id) + if info is None: + raise ValueError(f"unknown tracker: {tracker_id!r}") + out: dict[str, Any] = {} + for name, param in info.parameters.items(): + if name == "state_estimator_class": + out[name] = param.default_value + elif not isinstance(param.default_value, type): + out[name] = param.default_value + return out + + +def _init_kwargs(tracker_id: str, params: dict[str, Any]) -> dict[str, Any]: + info = BaseTracker._lookup_tracker(tracker_id) + if info is None: + raise ValueError(f"unknown tracker: {tracker_id!r}") + names = {n for n in inspect.signature(info.tracker_class.__init__).parameters if n != "self"} + return {k: v for k, v in params.items() if k in names} + + +def _resolve_params(tracker_id: str, *, params_file: Path | None) -> dict[str, Any]: + """Merge registry defaults with optional tuned overrides; force CMC on for BoT-SORT.""" + merged = _registry_defaults(tracker_id) + if params_file is not None and params_file.is_file(): + merged.update(json.loads(params_file.read_text())) + if tracker_id == "botsort" and "enable_cmc" not in merged: + merged["enable_cmc"] = True + return _init_kwargs(tracker_id, merged) + + +def _build(tracker_id: str, params: dict[str, Any]) -> BaseTracker: + info = BaseTracker._lookup_tracker(tracker_id) + if info is None: + raise ValueError(f"unknown tracker: {tracker_id!r}") + return info.tracker_class(**params) + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--tracker", required=True) + p.add_argument("--dataset", choices=DATASETS, required=True) + p.add_argument("--split", required=True, help="Prepared split (tune/eval/submit name, e.g. val, test).") + p.add_argument("--data-root", type=Path, required=True) + p.add_argument("--prep-dir", type=Path, required=True) + p.add_argument("--output-dir", type=Path, required=True, help="Predictions root: writes pred/.txt under here.") + p.add_argument("--params", type=Path, default=None, help="Optional tuned best_params.json") + args = p.parse_args(argv) + + dets_dir = args.prep_dir / args.dataset / args.split / "dets" + if not dets_dir.is_dir(): + print(f"missing prepared dets: {dets_dir} (run `make prep DATASET={args.dataset}`)", file=sys.stderr) + return 1 + + params = _resolve_params(args.tracker, params_file=args.params) + images_dir = split_paths(args.data_root, args.dataset, args.split).images_dir if needs_frames(args.tracker, params) else None + if images_dir is not None and not images_dir.is_dir(): + print(f"missing frames for CMC: {images_dir}", file=sys.stderr) + return 1 + + pred_dir = args.output_dir / "pred" + pred_dir.mkdir(parents=True, exist_ok=True) + tracker = _build(args.tracker, params) + for det_path in sorted(dets_dir.glob("*.txt")): + seq = det_path.stem + tracker.reset() + _run_tracker_on_detections( + tracker, + det_path, + pred_dir / f"{seq}.txt", + images_dir=images_dir, + seq_name=seq, + ) + print(f" tracked {seq}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmark/scripts/tracker_flags.py b/benchmark/scripts/tracker_flags.py deleted file mode 100644 index 16727dca1..000000000 --- a/benchmark/scripts/tracker_flags.py +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env python3 -"""Print ``trackers track`` flags from a params JSON file or library defaults.""" - -from __future__ import annotations - -import json -import sys -from pathlib import Path - -_REPO_ROOT = Path(__file__).resolve().parents[2] -_SRC = _REPO_ROOT / "src" -if _SRC.is_dir() and str(_SRC) not in sys.path: - sys.path.insert(0, str(_SRC)) - -from trackers.core.base import BaseTracker # noqa: E402 - - -def _is_class_param(name: str, param) -> bool: - if name == "state_estimator_class": - return True - default = param.default_value - return isinstance(default, type) - - -def tracker_flags(tracker_id: str, params: dict | None = None) -> str: - """Build CLI flags for one tracker. - - When *params* is empty, emit explicit ``--tracker.*`` flags from that - tracker's registry defaults. The CLI registers shared parameter names once - for all trackers (first registration wins), so omitting flags lets SORT and - others inherit BoT-SORT/ByteTrack defaults by mistake. - """ - info = BaseTracker._lookup_tracker(tracker_id) - if info is None: - raise ValueError(f"unknown tracker: {tracker_id}") - - if not params: - params = {name: param.default_value for name, param in info.parameters.items()} - - parts: list[str] = [] - for name, value in params.items(): - if name not in info.parameters: - continue - param = info.parameters[name] - if _is_class_param(name, param): - continue - if param.param_type is bool: - if value != param.default_value: - parts.append(f"--tracker.{name}") - else: - parts.extend([f"--tracker.{name}", str(value)]) - return " ".join(parts) - - -def main() -> int: - if len(sys.argv) not in {2, 3}: - print( - "usage: tracker_flags.py TRACKER [PARAMS.json|-]\n" - " Omit PARAMS or pass '-' to use library default hyperparameters.", - file=sys.stderr, - ) - return 1 - - tracker_id = sys.argv[1] - params_path = sys.argv[2] if len(sys.argv) == 3 else "-" - if params_path in {"-", "defaults", ""}: - params: dict = {} - else: - params = json.loads(Path(params_path).read_text()) - - try: - print(tracker_flags(tracker_id, params)) - except ValueError as exc: - print(str(exc), file=sys.stderr) - return 1 - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) From 35012222d00c15fd7e72f5bbc987617f8a551896 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 17:34:19 +0000 Subject: [PATCH 03/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmark/README.md | 95 +++++++++++++-------------- benchmark/scripts/codabench_submit.py | 38 ++++------- benchmark/scripts/collect.py | 7 +- benchmark/scripts/data_check.py | 19 +++++- benchmark/scripts/datasets.py | 8 ++- benchmark/scripts/mot_format.py | 10 ++- benchmark/scripts/prep_data.py | 6 ++ benchmark/scripts/track_split.py | 10 ++- 8 files changed, 115 insertions(+), 78 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 836f98420..fdb9ead77 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -14,7 +14,7 @@ make setup ```bash cd benchmark -export DATA_ROOT="/path/to/your/datasets" +export DATA_ROOT="/path/to/your/datasets" export CODABENCH_TOKEN="" # see Codabench below make data-check @@ -30,18 +30,18 @@ Run **default** and **tuned** on separate days — Codabench limits submissions MOT17, SportsMOT, and DanceTrack test metrics come from [Codabench](https://www.codabench.org/). Register for each competition before uploading (approval may be required): -| Dataset | Competition | -|---|---| -| MOT17 | [10049](https://www.codabench.org/competitions/10049/) | -| SportsMOT | [13077](https://www.codabench.org/competitions/13077/) | +| Dataset | Competition | +| ---------- | ------------------------------------------------------ | +| MOT17 | [10049](https://www.codabench.org/competitions/10049/) | +| SportsMOT | [13077](https://www.codabench.org/competitions/13077/) | | DanceTrack | [14885](https://www.codabench.org/competitions/14885/) | Request an API token with a one-time `curl` call (Codabench login — only the token is stored): ```bash curl -s -X POST https://www.codabench.org/api/api-token-auth/ \ - -H "Content-Type: application/json" \ - -d '{"username":"YOUR_USER","password":"YOUR_PASS"}' + -H "Content-Type: application/json" \ + -d '{"username":"YOUR_USER","password":"YOUR_PASS"}' export CODABENCH_TOKEN="" ``` @@ -75,12 +75,12 @@ $DATA_ROOT/ soccernet/soccernet_data/tracking/{train,test}/... ``` -| Source | Assets | -|---|---| -| MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | -| SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | -| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) / [OC-SORT dets](https://github.com/noahcao/OC_SORT) | -| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking) | +| Source | Assets | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | +| SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | +| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) / [OC-SORT dets](https://github.com/noahcao/OC_SORT) | +| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking) | MOT17 and SportsMOT use model detections produced in-house with YOLOX, following each benchmark’s published detector configuration — the same approach described in [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). @@ -90,33 +90,32 @@ make data-check DATA_ROOT="/path/to/datasets" ## Splits and scoring -| Dataset | Tune | Score | Scoring | -|---|---|---|---| -| MOT17 | val | test | Codabench | -| SportsMOT | val | test | Codabench | -| DanceTrack | train | test | Codabench | -| SoccerNet-tracking | train | test | Local (`trackers eval`) | - +| Dataset | Tune | Score | Scoring | +| ------------------ | ----- | ----- | ----------------------- | +| MOT17 | val | test | Codabench | +| SportsMOT | val | test | Codabench | +| DanceTrack | train | test | Codabench | +| SoccerNet-tracking | train | test | Local (`trackers eval`) | ## Commands Run from `benchmark/`. Pass variables on the command line or export them first (`DATA_ROOT`, `CODABENCH_TOKEN`, …). -| Target | Description | -|---|---| -| `setup` | Install `trackers[tune]` from the repo root | -| `data-check` | Print present/missing assets under `DATA_ROOT` | -| `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | -| `prep-all` | Prep all four datasets | -| `tune` | Optuna search → `best_params.json` (`TRACKER=`, `DATASET=`, `N_TRIALS=`) | -| `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | -| `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | -| `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | -| `benchmark-default` | `prep-all` → track-default on all datasets → `collect` | -| `benchmark-tuned` | `prep-all` → tune + track-tuned on all datasets → `collect` | -| `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | -| `collect` | Rebuild `tables.md` from existing score JSONs (`TRACKER=`) | -| `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | +| Target | Description | +| ------------------- | ------------------------------------------------------------------------------------------ | +| `setup` | Install `trackers[tune]` from the repo root | +| `data-check` | Print present/missing assets under `DATA_ROOT` | +| `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | +| `prep-all` | Prep all four datasets | +| `tune` | Optuna search → `best_params.json` (`TRACKER=`, `DATASET=`, `N_TRIALS=`) | +| `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | +| `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | +| `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | +| `benchmark-default` | `prep-all` → track-default on all datasets → `collect` | +| `benchmark-tuned` | `prep-all` → tune + track-tuned on all datasets → `collect` | +| `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | +| `collect` | Rebuild `tables.md` from existing score JSONs (`TRACKER=`) | +| `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | ## Usage @@ -130,7 +129,7 @@ make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 CODABENCH_TOKEN=... # Skip datasets (e.g. MOT17 out of Codabench submissions for today) make benchmark-tuned TRACKER=bytetrack N_TRIALS=5 \ - DATASETS="sportsmot dancetrack soccernet" CODABENCH_TOKEN=... + DATASETS="sportsmot dancetrack soccernet" CODABENCH_TOKEN=... # Both passes in one command (may hit daily limits) make benchmark BENCHMARK_CONFIG=all TRACKER=bytetrack CODABENCH_TOKEN=... @@ -156,18 +155,18 @@ make clean ### Variables -| Variable | Default | Purpose | -|---|---|---| -| `TRACKER` | `sort` | `sort`, `bytetrack`, `ocsort`, `botsort`, … | -| `DATA_ROOT` | `./data` | Raw dataset tree | -| `DATASET` | `mot17` | Single-dataset targets | -| `DATASETS` | all four | Space-separated subset for `benchmark*` | -| `BENCHMARK_CONFIG` | `default` | `benchmark`: `default`, `tuned`, or `all` | -| `CONFIG` | — | `upload`: `default` or `tuned` | -| `N_TRIALS` | `10` | Optuna trials per dataset | -| `CODABENCH_TOKEN` | — | Required for Codabench datasets | -| `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | -| `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | +| Variable | Default | Purpose | +| ------------------ | --------------------- | ------------------------------------------- | +| `TRACKER` | `sort` | `sort`, `bytetrack`, `ocsort`, `botsort`, … | +| `DATA_ROOT` | `./data` | Raw dataset tree | +| `DATASET` | `mot17` | Single-dataset targets | +| `DATASETS` | all four | Space-separated subset for `benchmark*` | +| `BENCHMARK_CONFIG` | `default` | `benchmark`: `default`, `tuned`, or `all` | +| `CONFIG` | — | `upload`: `default` or `tuned` | +| `N_TRIALS` | `10` | Optuna trials per dataset | +| `CODABENCH_TOKEN` | — | Required for Codabench datasets | +| `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | +| `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories when present. diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 47178c3e0..22917d57e 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Upload a submission zip to a Codabench competition phase. Uses Codabench's REST API (token auth + 3-step file upload). See: @@ -98,9 +104,7 @@ def _put_presigned_url(url: str, data: bytes, *, content_type: str = "applicatio raw = resp.read() if resp.status >= 400: detail = raw.decode(errors="replace") - raise RuntimeError( - f"PUT presigned upload → HTTP {resp.status}: {detail}" - ) + raise RuntimeError(f"PUT presigned upload → HTTP {resp.status}: {detail}") finally: conn.close() @@ -147,10 +151,7 @@ def upload_submission( bundle_bytes = zip_path.read_bytes() if dry_run: - print( - f"Dry run: would upload {zip_path.name} ({len(bundle_bytes)} bytes) " - f"to phase {phase_id} on {base_url}" - ) + print(f"Dry run: would upload {zip_path.name} ({len(bundle_bytes)} bytes) to phase {phase_id} on {base_url}") return {"dry_run": True, "phase": phase_id, "zip": str(zip_path)} _, data_record = _request( @@ -198,9 +199,7 @@ def get_submission_details(*, base_url: str, token: str, submission_id: int) -> token=token, ) if not isinstance(payload, dict): - raise RuntimeError( - f"Unexpected /api/submissions/{submission_id}/get_details/ response: {payload!r}" - ) + raise RuntimeError(f"Unexpected /api/submissions/{submission_id}/get_details/ response: {payload!r}") return payload @@ -213,9 +212,7 @@ def print_submission_failure_logs( ) -> None: """Best-effort scrape of scoring logs after a failed submission.""" try: - details = get_submission_details( - base_url=base_url, token=token, submission_id=submission_id - ) + details = get_submission_details(base_url=base_url, token=token, submission_id=submission_id) except RuntimeError as exc: print(f" logs unavailable: {exc}", flush=True) return @@ -303,9 +300,7 @@ def poll_submission( last_status = "" while True: - submission = get_submission( - base_url=base_url, token=token, submission_id=submission_id - ) + submission = get_submission(base_url=base_url, token=token, submission_id=submission_id) status = str(submission.get("status", "")) status_lc = status.lower() @@ -326,17 +321,14 @@ def poll_submission( elif submission.get("status_details"): print(f" details → {submission['status_details']}") if status_lc == "failed": - print_submission_failure_logs( - base_url=base_url, token=token, submission_id=submission_id - ) + print_submission_failure_logs(base_url=base_url, token=token, submission_id=submission_id) return submission elapsed = time.monotonic() - start remaining = timeout_seconds - elapsed if remaining <= 0: raise RuntimeError( - f"Timed out after {timeout_seconds:.0f}s waiting for submission " - f"{submission_id} (last status: {status})" + f"Timed out after {timeout_seconds:.0f}s waiting for submission {submission_id} (last status: {status})" ) time.sleep(min(wait, remaining)) @@ -443,9 +435,7 @@ def main(argv: list[str] | None = None) -> int: metric_keys=metric_keys, ) else: - submission = get_submission( - base_url=args.base_url, token=token, submission_id=sub_id - ) + submission = get_submission(base_url=args.base_url, token=token, submission_id=sub_id) scores = extract_metric_scores(submission, metric_keys) if scores: parts = ", ".join(f"{k}={scores[k]:.3f}" for k in metric_keys if k in scores) diff --git a/benchmark/scripts/collect.py b/benchmark/scripts/collect.py index 8042ed59a..441287f59 100644 --- a/benchmark/scripts/collect.py +++ b/benchmark/scripts/collect.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Aggregate per-dataset eval/Codabench score JSONs into a single doc-style markdown table. Looks under ``////`` (config ∈ {default, tuned}) for: @@ -13,7 +19,6 @@ import argparse import json -import sys from pathlib import Path from datasets import DATASETS, LABELS, job_dir diff --git a/benchmark/scripts/data_check.py b/benchmark/scripts/data_check.py index 85aa3f5ea..f5304db25 100644 --- a/benchmark/scripts/data_check.py +++ b/benchmark/scripts/data_check.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Walk the expected ``data/`` layout and print what's present vs missing per dataset. Use this before running ``make benchmark`` to verify the manual data setup. The @@ -24,7 +30,9 @@ def _check(label: str, path: Path | None, *, required: bool) -> bool: def check_dataset(data_root: Path, dataset: str) -> bool: - splits = sorted({TUNE_SPLIT[dataset], EVAL_SPLIT[dataset], *([SUBMIT_SPLIT[dataset]] if dataset in SUBMIT_SPLIT else [])}) + splits = sorted( + {TUNE_SPLIT[dataset], EVAL_SPLIT[dataset], *([SUBMIT_SPLIT[dataset]] if dataset in SUBMIT_SPLIT else [])} + ) print(f"\n[{dataset}]") ok = True for split in splits: @@ -51,7 +59,14 @@ def main(argv: list[str] | None = None) -> int: all_ok = True for dataset in datasets: all_ok &= check_dataset(args.data_root, dataset) - print("\n" + ("All required assets found." if all_ok else "Some required assets missing — see README for download instructions.")) + print( + "\n" + + ( + "All required assets found." + if all_ok + else "Some required assets missing — see README for download instructions." + ) + ) return 0 if all_ok else 1 diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py index 7d0146308..5653a4806 100644 --- a/benchmark/scripts/datasets.py +++ b/benchmark/scripts/datasets.py @@ -1,3 +1,9 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Benchmark dataset layout: paths, splits, Codabench targets. Single source of truth shared by all benchmark scripts. Not a CLI. @@ -5,9 +11,9 @@ from __future__ import annotations +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path -from typing import Callable BENCHMARK_ROOT = Path(__file__).resolve().parents[1] diff --git a/benchmark/scripts/mot_format.py b/benchmark/scripts/mot_format.py index ae2a5d274..be2a339a5 100644 --- a/benchmark/scripts/mot_format.py +++ b/benchmark/scripts/mot_format.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Format a MOT prediction directory for Codabench submission. Steps applied in order: @@ -73,7 +79,9 @@ def zip_dir(pred_dir: Path, out_zip: Path) -> Path: def main(argv: list[str] | None = None) -> int: p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) p.add_argument("--dataset", choices=DATASETS, required=True) - p.add_argument("--pred-dir", type=Path, required=True, help="Directory of MOT prediction txt files (modified in place).") + p.add_argument( + "--pred-dir", type=Path, required=True, help="Directory of MOT prediction txt files (modified in place)." + ) p.add_argument("--out-zip", type=Path, required=True) args = p.parse_args(argv) diff --git a/benchmark/scripts/prep_data.py b/benchmark/scripts/prep_data.py index 857b8765a..eac0b6503 100644 --- a/benchmark/scripts/prep_data.py +++ b/benchmark/scripts/prep_data.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Convert vendor MOT detections + GT into flat per-sequence MOT files. Output layout (under ``--prep-dir``): diff --git a/benchmark/scripts/track_split.py b/benchmark/scripts/track_split.py index b0fecce4c..3cd439a06 100644 --- a/benchmark/scripts/track_split.py +++ b/benchmark/scripts/track_split.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Run one tracker over a prepared MOT detection directory and write MOT predictions. This script intentionally bypasses the `trackers track` CLI for two reasons: @@ -89,7 +95,9 @@ def main(argv: list[str] | None = None) -> int: return 1 params = _resolve_params(args.tracker, params_file=args.params) - images_dir = split_paths(args.data_root, args.dataset, args.split).images_dir if needs_frames(args.tracker, params) else None + images_dir = ( + split_paths(args.data_root, args.dataset, args.split).images_dir if needs_frames(args.tracker, params) else None + ) if images_dir is not None and not images_dir.is_dir(): print(f"missing frames for CMC: {images_dir}", file=sys.stderr) return 1 From 720360dc91fb5ab27c4dd6281ee714de5f8f59da Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 14:53:23 -0300 Subject: [PATCH 04/54] fixed dancetrack dets origin in docs --- benchmark/README.md | 16 ++++++++-------- docs/trackers/comparison.md | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 836f98420..75d005742 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -17,14 +17,14 @@ cd benchmark export DATA_ROOT="/path/to/your/datasets" export CODABENCH_TOKEN="" # see Codabench below -make data-check -make benchmark-default TRACKER=bytetrack -make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 +make data-check # check datasets in the expected format +make benchmark-default TRACKER=bytetrack # benchmark default parameters +make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 # tune and benchmark ``` Results: `benchmark_outputs//tables.md` and `summary.json`. -Run **default** and **tuned** on separate days — Codabench limits submissions per phase. SoccerNet is scored locally and does not count toward that limit. +Run **default** and **tuned** on separate days: Codabench limits daily submissions. SoccerNet is scored locally and does not count toward that limit. ## Codabench @@ -46,7 +46,7 @@ curl -s -X POST https://www.codabench.org/api/api-token-auth/ \ export CODABENCH_TOKEN="" ``` -Treat `CODABENCH_TOKEN` as a secret — do not publish it. See [Codabench API docs](https://www.codabench.org/api/docs/) if the request fails. +Treat `CODABENCH_TOKEN` as a secret, do not publish it. See [Codabench API docs](https://www.codabench.org/api/docs/) if the request fails. If tracking finished but upload failed (daily limit or pending approval), re-submit the zip without re-running track: @@ -79,10 +79,10 @@ $DATA_ROOT/ |---|---| | MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | | SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | -| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) / [OC-SORT dets](https://github.com/noahcao/OC_SORT) | -| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking) | +| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) frames/GT; uses YOLOX dets | +| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking); oracle (ground-truth) detections | -MOT17 and SportsMOT use model detections produced in-house with YOLOX, following each benchmark’s published detector configuration — the same approach described in [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). +MOT17, SportsMOT, and DanceTrack use YOLOX model detections produced in-house, following each benchmark’s published detector configuration. SoccerNet uses oracle boxes from the dataset. See [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). ```bash make data-check DATA_ROOT="/path/to/datasets" diff --git a/docs/trackers/comparison.md b/docs/trackers/comparison.md index f515ddd06..f3584705e 100644 --- a/docs/trackers/comparison.md +++ b/docs/trackers/comparison.md @@ -9,11 +9,11 @@ This page shows head-to-head performance of SORT, ByteTrack, OC-SORT, and BoT-SO !!! info "Benchmark version" - Results use **trackers v2.3.0** (released 2026-03-16). Detections are from YOLOX (MOT17, SportsMOT) or ground-truth oracle boxes (SoccerNet, DanceTrack). Parameters were tuned via grid search on held-out splits. See [Methodology](#methodology) for details. + Results use **trackers v2.3.0** (released 2026-03-16). Detections are from YOLOX (MOT17, SportsMOT, DanceTrack) or ground-truth oracle boxes (SoccerNet). Parameters were tuned via grid search on held-out splits. See [Methodology](#methodology) for details. !!! note "Benchmark methodology" - Results measured using YOLOX detections (MOT17, SportsMOT) or oracle ground-truth boxes (SoccerNet, DanceTrack) with default and grid-searched parameters. Performance varies across detectors — see [Detection Quality Matters](../learn/detection-quality.md) for the impact of detector quality on tracking metrics. + Results measured using YOLOX detections (MOT17, SportsMOT, DanceTrack) or oracle ground-truth boxes (SoccerNet) with default and grid-searched parameters. Performance varies across detectors — see [Detection Quality Matters](../learn/detection-quality.md) for the impact of detector quality on tracking metrics. ## [MOT17](https://arxiv.org/abs/1603.00831) @@ -251,7 +251,7 @@ Group dancing tracking with uniform appearance, diverse motions, and extreme art !!! info Parameters were tuned on the train set. Results are reported on the - validation set. This dataset provides oracle (ground-truth) detections. + validation set. Detections come from a YOLOX model. === "Default" From 901563a7972a6ad5109b0e77bbe8a93f3be1799f Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 14:57:01 -0300 Subject: [PATCH 05/54] added mot17 val half filter --- benchmark/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index 75d005742..aa6f0ade1 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -63,7 +63,7 @@ Point `DATA_ROOT` at the folder that directly contains `mot17/`, `sportsmot/`, e ``` $DATA_ROOT/ mot17/MOT17_yolox_dets/{val,test}/... - mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... + mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... mot17/{val,test}//img1/... # BoT-SORT CMC only sportsmot/sportsmot_yolox_dets/{val,test}/... sportsmot/TrackEval/data/gt/sportsmot/val/... @@ -84,6 +84,18 @@ $DATA_ROOT/ MOT17, SportsMOT, and DanceTrack use YOLOX model detections produced in-house, following each benchmark’s published detector configuration. SoccerNet uses oracle boxes from the dataset. See [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). +### MOT17 validation ground truth + +YOLOX validation detections for MOT17 cover only the benchmark validation frame range for each sequence—not every frame in the official training labels. Tuning on the full MOT17 GT under `TrackEval/data/gt/MOT17/train_val/` misaligns detection and label frame indices. + +After YOLOX val detections and MOT17 GT are in place, run once from `benchmark/`: + +```bash +python scripts/align_mot17_val_gt.py --data-root "$DATA_ROOT" +``` + +This filters each sequence’s `gt.txt` to the frame range present in `MOT17_yolox_dets/val/MOT17-XX_val.txt` and writes the result to `mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/`. MOT17 tuning uses that tree. + ```bash make data-check DATA_ROOT="/path/to/datasets" ``` @@ -173,6 +185,7 @@ BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories whe ## Notes +- **MOT17 validation GT.** YOLOX val detections span a subset of frames per sequence; run `scripts/align_mot17_val_gt.py` before tuning so labels match detection indices. - **Tracking bypasses `trackers track`.** `scripts/track_split.py` loads the registry directly (workaround for a shared CLI parameter bug; see issue/PR). ByteTrack/SORT/OC-SORT never receive `--images-dir` during tune. - **MOT17 server format.** `scripts/mot_format.py` triplicates `MOT17-XX.txt` into FRCNN/SDP/DPM files and stubs missing sequences for Codabench. - **Resuming.** Steps are independent. Re-run `collect` after late uploads; use `upload` to submit an existing zip without re-tracking. From 3e0401accb058703aeba8bde77c20b6a91aa8fc7 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 14:57:09 -0300 Subject: [PATCH 06/54] added mot17 val half filter --- benchmark/scripts/align_mot17_val_gt.py | 102 ++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 benchmark/scripts/align_mot17_val_gt.py diff --git a/benchmark/scripts/align_mot17_val_gt.py b/benchmark/scripts/align_mot17_val_gt.py new file mode 100644 index 000000000..9c80c8512 --- /dev/null +++ b/benchmark/scripts/align_mot17_val_gt.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Filter MOT17 validation GT to the frame range covered by YOLOX val detections. + +Public YOLOX val detections for MOT17 cover only part of each sequence. The full +MOT17 GT under ``TrackEval/data/gt/MOT17/train_val`` includes extra frames, which +misaligns tuning if used as-is. + +For each sequence, this script reads ``MOT17_yolox_dets/val/MOT17-XX_val.txt`` to +find the detection frame range, filters ``gt/gt.txt`` to those frames, and writes +the result under ``TrackEval/data/gt/MOT17_yolox_val/train_val/``. + +Usage (from ``benchmark/``): + + python scripts/align_mot17_val_gt.py --data-root /path/to/datasets +""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + + +def align_mot17_val_gt(data_root: Path) -> int: + mot17 = data_root / "mot17" + val_det_root = mot17 / "MOT17_yolox_dets" / "val" + src_gt_root = mot17 / "TrackEval" / "data" / "gt" / "MOT17" / "train_val" + dst_gt_root = mot17 / "TrackEval" / "data" / "gt" / "MOT17_yolox_val" / "train_val" + + if not src_gt_root.is_dir(): + print(f"missing source GT: {src_gt_root}", file=sys.stderr) + return 1 + if not val_det_root.is_dir(): + print(f"missing YOLOX val detections: {val_det_root}", file=sys.stderr) + return 1 + + dst_gt_root.mkdir(parents=True, exist_ok=True) + seq_dirs = sorted(p for p in src_gt_root.iterdir() if p.is_dir()) + if not seq_dirs: + print(f"no sequences under {src_gt_root}", file=sys.stderr) + return 1 + + wrote = 0 + for seq_dir in seq_dirs: + seq_name = seq_dir.name + prefix = seq_name.split("-FRCNN")[0] + det_file = val_det_root / f"{prefix}_val.txt" + if not det_file.is_file(): + print(f"skip {seq_name}: missing {det_file.name}") + continue + + frames: list[int] = [] + for line in det_file.read_text().splitlines(): + line = line.strip() + if not line: + continue + try: + frames.append(int(line.split(",")[0])) + except ValueError: + continue + if not frames: + print(f"skip {seq_name}: no frames in {det_file.name}") + continue + + f_min, f_max = min(frames), max(frames) + src_gt = seq_dir / "gt" / "gt.txt" + if not src_gt.is_file(): + print(f"skip {seq_name}: missing {src_gt}") + continue + + kept = [ + ln.strip() + for ln in src_gt.read_text().splitlines() + if ln.strip() and f_min <= int(ln.split(",")[0]) <= f_max + ] + + dst_seq_dir = dst_gt_root / seq_name + dst_gt_dir = dst_seq_dir / "gt" + dst_seq_dir.mkdir(parents=True, exist_ok=True) + dst_gt_dir.mkdir(parents=True, exist_ok=True) + for item in seq_dir.iterdir(): + if item.name != "gt" and item.is_file(): + (dst_seq_dir / item.name).write_bytes(item.read_bytes()) + (dst_gt_dir / "gt.txt").write_text("\n".join(kept) + ("\n" if kept else "")) + print(f"{seq_name}: frames [{f_min}, {f_max}] → {len(kept)} GT lines") + wrote += 1 + + if wrote == 0: + print("no sequences aligned", file=sys.stderr) + return 1 + print(f"wrote aligned GT under {dst_gt_root}") + return 0 + + +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--data-root", type=Path, required=True, help="Root containing mot17/, sportsmot/, …") + return align_mot17_val_gt(p.parse_args(argv).data_root) + + +if __name__ == "__main__": + raise SystemExit(main()) From 59565b4af8a55fd168e37dcd5726a36468c5c115 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 25 May 2026 17:59:04 +0000 Subject: [PATCH 07/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmark/README.md | 14 +++++++------- benchmark/scripts/align_mot17_val_gt.py | 6 ++++++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 71bd62636..9a3201743 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -63,7 +63,7 @@ Point `DATA_ROOT` at the folder that directly contains `mot17/`, `sportsmot/`, e ``` $DATA_ROOT/ mot17/MOT17_yolox_dets/{val,test}/... - mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... + mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... mot17/{val,test}//img1/... # BoT-SORT CMC only sportsmot/sportsmot_yolox_dets/{val,test}/... sportsmot/TrackEval/data/gt/sportsmot/val/... @@ -75,12 +75,12 @@ $DATA_ROOT/ soccernet/soccernet_data/tracking/{train,test}/... ``` -| Source | Assets | -|---|---| -| MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | -| SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | -| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) frames/GT; uses YOLOX dets | -| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking); oracle (ground-truth) detections | +| Source | Assets | +| ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| MOT17 | `trackers download mot17`; YOLOX dets replicated locally using the [ByteTrack](https://github.com/ifzhang/ByteTrack/tree/main#data-preparation) detector setup (not their pre-packaged det zips) | +| SportsMOT | `trackers download sportsmot`; YOLOX dets replicated locally using the [SportsMOT](https://github.com/MCG-NJU/SportsMOT) detector setup | +| DanceTrack | [DanceTrack](https://github.com/DanceTrack/DanceTrack) frames/GT; uses YOLOX dets | +| SoccerNet-tracking | [soccer-net.org](https://www.soccer-net.org/data) (2022 tracking); oracle (ground-truth) detections | MOT17, SportsMOT, and DanceTrack use YOLOX model detections produced in-house, following each benchmark’s published detector configuration. SoccerNet uses oracle boxes from the dataset. See [`docs/trackers/comparison.md`](../docs/trackers/comparison.md#detections). diff --git a/benchmark/scripts/align_mot17_val_gt.py b/benchmark/scripts/align_mot17_val_gt.py index 9c80c8512..8f4d03913 100644 --- a/benchmark/scripts/align_mot17_val_gt.py +++ b/benchmark/scripts/align_mot17_val_gt.py @@ -1,4 +1,10 @@ #!/usr/bin/env python3 +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + """Filter MOT17 validation GT to the frame range covered by YOLOX val detections. Public YOLOX val detections for MOT17 cover only part of each sequence. The full From 43b5da4068af687c2cecefac04dce32af45b4583 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Mon, 25 May 2026 15:14:59 -0300 Subject: [PATCH 08/54] fixed ruff error of unsafe http requests --- benchmark/scripts/codabench_submit.py | 37 ++++++++++++++++++--------- 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 22917d57e..03b42203a 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -20,9 +20,7 @@ import os import sys import time -import urllib.error import urllib.parse -import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any @@ -38,6 +36,12 @@ DEFAULT_PHASE_ID = 16382 +def _validate_http_url(url: str) -> None: + scheme = urllib.parse.urlparse(url).scheme + if scheme not in ("http", "https"): + raise RuntimeError(f"Unsupported URL scheme: {scheme!r}") + + def _request( *, method: str, @@ -54,20 +58,29 @@ def _request( if json_body is not None: body = json.dumps(json_body).encode() hdrs.setdefault("Content-Type", "application/json") - req = urllib.request.Request(url, data=body, headers=hdrs, method=method) + _validate_http_url(url) + parsed = urllib.parse.urlparse(url) + path = parsed.path or "/" + if parsed.query: + path = f"{path}?{parsed.query}" + + conn_class = http.client.HTTPSConnection if parsed.scheme == "https" else http.client.HTTPConnection + conn = conn_class(parsed.netloc, timeout=120) try: - with urllib.request.urlopen(req, timeout=120) as resp: - raw = resp.read() - status = resp.status - except urllib.error.HTTPError as exc: - raw = exc.read() - status = exc.code + conn.request(method, path, body=body, headers=hdrs) + resp = conn.getresponse() + raw = resp.read() + status = resp.status + except OSError as exc: + raise RuntimeError(f"{method} {url} failed: {exc}") from exc + finally: + conn.close() + + if status >= 400: detail = raw.decode(errors="replace") if detail.lstrip().startswith(" Date: Tue, 26 May 2026 11:09:31 -0300 Subject: [PATCH 09/54] Add multi-tracker comparison tables and Codabench poll/retry to benchmark --- benchmark/Makefile | 102 ++++++++++++++++---- benchmark/README.md | 42 +++++++-- benchmark/scripts/codabench_submit.py | 42 +++++++-- benchmark/scripts/collect.py | 129 ++++++++++++++++++++------ benchmark/scripts/datasets.py | 42 ++++++++- 5 files changed, 291 insertions(+), 66 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index b895a7872..7327c9cea 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -23,6 +23,25 @@ TRACKER ?= sort DATASET ?= mot17 CONFIG ?= default BENCHMARK_CONFIG ?= default +# Multi-tracker comparison (docs/trackers/comparison.md layout). Set to `all` or list ids. +# Tracker list is defined once in scripts/datasets.py (COMPARISON_TRACKERS). +TRACKERS ?= +DATASETS_PY := $(PYTHON) scripts/datasets.py +COMPARISON_TRACKERS ?= $(shell $(DATASETS_PY) --field comparison_trackers) +COMPARISON_TRACKERS_CSV ?= $(shell $(DATASETS_PY) --field comparison_trackers_csv) +comma := , +empty := +space := $(empty) $(empty) +ifeq ($(TRACKERS),all) + BENCHMARK_TRACKERS := $(COMPARISON_TRACKERS) +else ifneq ($(strip $(TRACKERS)),) + BENCHMARK_TRACKERS := $(TRACKERS) +else ifeq ($(words $(DATASETS)),1) + # One dataset → all trackers + comparison table (docs/trackers/comparison.md layout). + BENCHMARK_TRACKERS := $(COMPARISON_TRACKERS) +else + BENCHMARK_TRACKERS := $(TRACKER) +endif N_TRIALS ?= 10 OBJECTIVE ?= HOTA THRESHOLD ?= 0.5 @@ -60,10 +79,11 @@ mot17_CB := 10049 16382 sportsmot_CB := 13077 21402 dancetrack_CB := 14885 24635 -LAYOUT := $(PYTHON) scripts/datasets.py --data-root "$(DATA_ROOT)" +LAYOUT := $(DATASETS_PY) --data-root "$(DATA_ROOT)" .PHONY: help setup data-check prep prep-all tune track-default track-tuned _track-and-score \ - benchmark benchmark-default benchmark-tuned upload collect clean + benchmark benchmark-default benchmark-tuned benchmark-comparison-default benchmark-comparison-tuned \ + upload collect collect-comparison poll clean help: @echo "MOT benchmark workflow — run from \`cd benchmark\`" @@ -80,7 +100,11 @@ help: @echo " benchmark Full pipeline (TRACKER=, BENCHMARK_CONFIG=default|tuned|all, DATASETS=...)" @echo " benchmark-default Same as \`make benchmark BENCHMARK_CONFIG=default\`" @echo " benchmark-tuned Same as \`make benchmark BENCHMARK_CONFIG=tuned\`" - @echo " collect Aggregate eval/codabench JSONs into tables.md (TRACKER=)" + @echo " benchmark-comparison-default All $(COMPARISON_TRACKERS) on DATASET= (default params)" + @echo " benchmark-comparison-tuned All $(COMPARISON_TRACKERS) on DATASET= (tune + tuned)" + @echo " collect Rebuild tables.md for one tracker (TRACKER=)" + @echo " collect-comparison Tracker comparison table (DATASET=dancetrack)" + @echo " poll Poll an existing Codabench submission (SUBMISSION_ID=, DATASET=, CONFIG=)" @echo " clean Remove $(PREP_DIR) and $(OUTPUT_DIR)" @echo "" @echo "Codabench upload requires CODABENCH_TOKEN. See README for data setup." @@ -173,10 +197,31 @@ upload: --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \ --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json" +# Poll Codabench for scores without re-uploading (e.g. after a transient 502 during benchmark). +poll: + @test -n "$(SUBMISSION_ID)" || { echo "Set SUBMISSION_ID= (from Codabench upload log)"; exit 1; } + @test "$(DATASET)" != "soccernet" || { echo "soccernet is scored locally"; exit 1; } + @[ -n "$(CODABENCH_TOKEN)" ] || { echo "ERROR: CODABENCH_TOKEN not set."; exit 1; } + $(PYTHON) scripts/codabench_submit.py --submission-id $(SUBMISSION_ID) \ + --competition-id $(word 1,$($(DATASET)_CB)) \ + --phase $(word 2,$($(DATASET)_CB)) \ + --base-url "$(CODABENCH_URL)" --token "$(CODABENCH_TOKEN)" \ + --wait-timeout $(CODABENCH_WAIT_TIMEOUT) --poll-interval $(CODABENCH_POLL_INTERVAL) \ + --output "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)/codabench.json" + collect: - $(PYTHON) scripts/collect.py --tracker $(TRACKER) --output-dir "$(OUTPUT_DIR)" + $(PYTHON) scripts/collect.py --tracker $(TRACKER) --output-dir "$(OUTPUT_DIR)" \ + --datasets "$(subst $(space),$(comma),$(DATASETS))" + +collect-comparison: + @if [ "$(origin DATASET)" != "command line" ]; then \ + echo "Set DATASET=dancetrack"; exit 1; \ + fi + $(PYTHON) scripts/collect.py --compare-dataset $(DATASET) --output-dir "$(OUTPUT_DIR)" \ + --trackers "$(if $(filter all,$(TRACKERS)),$(COMPARISON_TRACKERS_CSV),$(if $(strip $(TRACKERS)),$(subst $(space),$(comma),$(TRACKERS)),$(COMPARISON_TRACKERS_CSV)))" # Full pipeline. BENCHMARK_CONFIG=default|tuned|all (default: default only — one Codabench pass). +# Set TRACKERS=all (or a space-separated list) to benchmark every tracker and write comparison tables. benchmark: @case "$(BENCHMARK_CONFIG)" in default|tuned|all) ;; \ *) echo "Set BENCHMARK_CONFIG=default, tuned, or all (or use benchmark-default / benchmark-tuned)"; exit 1;; \ @@ -184,21 +229,44 @@ benchmark: @if [ -z "$(CODABENCH_TOKEN)" ]; then \ echo "ERROR: CODABENCH_TOKEN must be set (required for mot17, sportsmot, dancetrack)."; exit 1; \ fi - @$(MAKE) prep-all @for d in $(DATASETS); do \ - if [ "$(BENCHMARK_CONFIG)" = "default" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ - echo ""; echo "===== [$$d] default ====="; \ - $(MAKE) track-default TRACKER=$(TRACKER) DATASET=$$d || exit 1; \ - fi; \ - if [ "$(BENCHMARK_CONFIG)" = "tuned" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ - echo ""; echo "===== [$$d] tune ====="; \ - $(MAKE) tune TRACKER=$(TRACKER) DATASET=$$d N_TRIALS=$(N_TRIALS) || exit 1; \ - echo "===== [$$d] tuned ====="; \ - $(MAKE) track-tuned TRACKER=$(TRACKER) DATASET=$$d || exit 1; \ - fi; \ + echo "===== prep [$$d] ====="; \ + $(MAKE) prep DATASET=$$d || exit 1; \ done - @echo ""; echo "===== collect =====" - @$(MAKE) collect TRACKER=$(TRACKER) + @for t in $(BENCHMARK_TRACKERS); do \ + for d in $(DATASETS); do \ + if [ "$(BENCHMARK_CONFIG)" = "default" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ + echo ""; echo "===== [$$t/$$d] default ====="; \ + $(MAKE) track-default TRACKER=$$t DATASET=$$d || exit 1; \ + fi; \ + if [ "$(BENCHMARK_CONFIG)" = "tuned" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ + echo ""; echo "===== [$$t/$$d] tune ====="; \ + extras=""; \ + if [ "$$t" = "botsort" ]; then extras='FIXED_PARAMS={"enable_cmc": true}'; fi; \ + $(MAKE) tune TRACKER=$$t DATASET=$$d N_TRIALS=$(N_TRIALS) $$extras || exit 1; \ + echo "===== [$$t/$$d] tuned ====="; \ + $(MAKE) track-tuned TRACKER=$$t DATASET=$$d || exit 1; \ + fi; \ + done; \ + done + @if [ "$(words $(BENCHMARK_TRACKERS))" -gt 1 ]; then \ + for d in $(DATASETS); do \ + echo ""; echo "===== collect comparison [$$d] ====="; \ + $(PYTHON) scripts/collect.py --compare-dataset $$d --output-dir "$(OUTPUT_DIR)" \ + --trackers "$(subst $(space),$(comma),$(BENCHMARK_TRACKERS))" || exit 1; \ + done; \ + else \ + echo ""; echo "===== collect [$(TRACKER)] ====="; \ + $(MAKE) collect TRACKER=$(TRACKER) DATASETS="$(DATASETS)"; \ + fi + +benchmark-comparison-default: + @test -n "$(DATASET)" || { echo "Set DATASET= (e.g. dancetrack)"; exit 1; } + @$(MAKE) benchmark BENCHMARK_CONFIG=default DATASETS="$(DATASET)" TRACKERS=all + +benchmark-comparison-tuned: + @test -n "$(DATASET)" || { echo "Set DATASET= (e.g. dancetrack)"; exit 1; } + @$(MAKE) benchmark BENCHMARK_CONFIG=tuned DATASETS="$(DATASET)" TRACKERS=all benchmark-default: @$(MAKE) benchmark BENCHMARK_CONFIG=default diff --git a/benchmark/README.md b/benchmark/README.md index 9a3201743..50d314cc8 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -22,7 +22,10 @@ make benchmark-default TRACKER=bytetrack # benchmark default parameters make benchmark-tuned TRACKER=bytetrack N_TRIALS=50 # tune and benchmark ``` -Results: `benchmark_outputs//tables.md` and `summary.json`. +Results: + +- Single tracker: `benchmark_outputs//tables.md` (rows = datasets) +- One dataset, all trackers: `benchmark_outputs/comparison//tables.md` (rows = trackers, same layout as [`comparison.md`](../docs/trackers/comparison.md)) Run **default** and **tuned** on separate days: Codabench limits daily submissions. SoccerNet is scored locally and does not count toward that limit. @@ -48,7 +51,7 @@ export CODABENCH_TOKEN="" Treat `CODABENCH_TOKEN` as a secret, do not publish it. See [Codabench API docs](https://www.codabench.org/api/docs/) if the request fails. -If tracking finished but upload failed (daily limit or pending approval), re-submit the zip without re-running track: +If tracking finished but upload or polling failed (daily limit, pending approval, transient 502), recover without re-tracking: ```bash make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned CODABENCH_TOKEN=... @@ -56,6 +59,8 @@ make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned CODABENCH_TOKEN=... Then `make collect TRACKER=bytetrack` to refresh the table. +or `make collect-comparison DATASET=dancetrack` to refresh the comparison table. + ## Data setup Point `DATA_ROOT` at the folder that directly contains `mot17/`, `sportsmot/`, etc. Default: `./data`. @@ -69,7 +74,7 @@ $DATA_ROOT/ sportsmot/TrackEval/data/gt/sportsmot/val/... dancetrack/dancetrack_yolox_dets/{train,val,test}/... dancetrack/TrackEval/data/gt/dancetrack/{train,val}/... - dancetrack/{train,val,test}_images/... # BoT-SORT CMC (test optional) + dancetrack/{train,val,test}_images/... # BoT-SORT CMC (all three splits for tune + test submit) soccernet/SoccerNet_dets/... soccernet/TrackEval/data/gt/SoccerNet_tracking/... soccernet/soccernet_data/tracking/{train,test}/... @@ -123,15 +128,19 @@ Run from `benchmark/`. Pass variables on the command line or export them first ( | `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | | `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | | `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | -| `benchmark-default` | `prep-all` → track-default on all datasets → `collect` | -| `benchmark-tuned` | `prep-all` → tune + track-tuned on all datasets → `collect` | +| `poll` | Poll an existing Codabench submission for scores (`SUBMISSION_ID=`, `TRACKER=`, `DATASET=`, `CONFIG=`) | +| `benchmark-default` | Prep → track-default → tables. One `DATASET` → all four trackers + comparison table | +| `benchmark-tuned` | Prep → tune + track-tuned → tables. One `DATASET` → all four trackers + comparison table | | `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | -| `collect` | Rebuild `tables.md` from existing score JSONs (`TRACKER=`) | +| `benchmark-comparison-default` | Shorthand: `benchmark-default` with `TRACKERS=all` on one `DATASET=` | +| `benchmark-comparison-tuned` | Shorthand: `benchmark-tuned` with `TRACKERS=all` on one `DATASET=` | +| `collect` | Rebuild per-tracker `tables.md` (`TRACKER=`, optional `DATASETS=`) | +| `collect-comparison`| Rebuild comparison table for one dataset (`DATASET=dancetrack`) | | `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | ## Usage -Full pipeline (runs `prep-all`, then `collect`): +Full pipeline (all four datasets, single tracker): ```bash make benchmark-default TRACKER=bytetrack CODABENCH_TOKEN=... @@ -147,6 +156,22 @@ make benchmark-tuned TRACKER=bytetrack N_TRIALS=5 \ make benchmark BENCHMARK_CONFIG=all TRACKER=bytetrack CODABENCH_TOKEN=... ``` +One dataset, all trackers (comparison table like `docs/trackers/comparison.md`): + +```bash +# Runs all comparison trackers (see scripts/datasets.py → COMPARISON_TRACKERS) on DanceTrack test; writes +# benchmark_outputs/comparison/dancetrack/tables.md +make benchmark-default DATASETS=dancetrack CODABENCH_TOKEN=... + +make benchmark-tuned DATASETS=dancetrack N_TRIALS=50 CODABENCH_TOKEN=... + +# Same, explicit form +make benchmark-comparison-default DATASET=dancetrack CODABENCH_TOKEN=... + +# Rebuild comparison table from existing score JSONs only +make collect-comparison DATASET=dancetrack +``` + Skip datasets (partial run or resume): ```bash @@ -169,7 +194,8 @@ make clean | Variable | Default | Purpose | | ------------------ | --------------------- | ------------------------------------------- | -| `TRACKER` | `sort` | `sort`, `bytetrack`, `ocsort`, `botsort`, … | +| `TRACKER` | `sort` | Single tracker when `DATASETS` lists more than one dataset | +| `TRACKERS` | — | Space-separated list, or `all` (see `COMPARISON_TRACKERS` in `scripts/datasets.py`). When `DATASETS` is a single dataset, defaults to all comparison trackers | | `DATA_ROOT` | `./data` | Raw dataset tree | | `DATASET` | `mot17` | Single-dataset targets | | `DATASETS` | all four | Space-separated subset for `benchmark*` | diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 03b42203a..8f1aa8f59 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -27,6 +27,7 @@ DEFAULT_METRICS = ("HOTA", "IDF1", "MOTA") TERMINAL_STATUSES = {"finished", "failed", "cancelled", "none"} +_TRANSIENT_HTTP_STATUSES = frozenset({502, 503, 504}) # Known test-server presets (Makefile sets these via CODABENCH_* env vars): # mot17: competition 10049, phase 16382 @@ -255,15 +256,38 @@ def print_submission_failure_logs( print(f" logs →\n{text}", flush=True) -def get_submission(*, base_url: str, token: str, submission_id: int) -> dict[str, Any]: - _, payload = _request( - method="GET", - url=f"{base_url.rstrip('/')}/api/submissions/{submission_id}/", - token=token, - ) - if not isinstance(payload, dict): - raise RuntimeError(f"Unexpected /api/submissions/{submission_id}/ response: {payload!r}") - return payload +def _is_transient_http_error(exc: BaseException) -> bool: + msg = str(exc) + return any(f"HTTP {code}" in msg for code in _TRANSIENT_HTTP_STATUSES) + + +def get_submission( + *, + base_url: str, + token: str, + submission_id: int, + max_retries: int = 6, +) -> dict[str, Any]: + last_exc: RuntimeError | None = None + for attempt in range(max_retries): + try: + _, payload = _request( + method="GET", + url=f"{base_url.rstrip('/')}/api/submissions/{submission_id}/", + token=token, + ) + if not isinstance(payload, dict): + raise RuntimeError(f"Unexpected /api/submissions/{submission_id}/ response: {payload!r}") + return payload + except RuntimeError as exc: + last_exc = exc + if attempt + 1 >= max_retries or not _is_transient_http_error(exc): + raise + wait = min(10.0 * (2**attempt), 60.0) + print(f" transient API error, retry in {wait:.0f}s: {exc}", flush=True) + time.sleep(wait) + assert last_exc is not None + raise last_exc def extract_metric_scores( diff --git a/benchmark/scripts/collect.py b/benchmark/scripts/collect.py index 441287f59..823cd8a82 100644 --- a/benchmark/scripts/collect.py +++ b/benchmark/scripts/collect.py @@ -5,23 +5,31 @@ # Licensed under the Apache License, Version 2.0 [see LICENSE for details] # ------------------------------------------------------------------------ -"""Aggregate per-dataset eval/Codabench score JSONs into a single doc-style markdown table. +"""Aggregate benchmark score JSONs into markdown tables. -Looks under ``////`` (config ∈ {default, tuned}) for: +Two layouts (mirroring ``docs/trackers/comparison.md``): - - ``eval.json`` → from `trackers eval --output ...` (SoccerNet local eval) - - ``codabench.json`` → from `codabench_submit.py --output ...` (MOT17/SportsMOT/DanceTrack) +1. **Per tracker** (``--tracker``): rows = datasets, columns = HOTA/IDF1/MOTA. + Writes ``//tables.md``. -Writes ``//tables.md`` and ``//summary.json``. +2. **Per dataset** (``--compare-dataset`` + ``--trackers``): rows = trackers. + Writes ``/comparison//tables.md``. """ from __future__ import annotations import argparse import json +import sys from pathlib import Path -from datasets import DATASETS, LABELS, job_dir +from datasets import ( + COMPARISON_TRACKERS, + DATASETS, + LABELS, + TRACKER_LABELS, + job_dir, +) _CONFIGS = ("default", "tuned") _TARGETS = ("HOTA", "IDF1", "MOTA") @@ -60,49 +68,90 @@ def _row_scores(out_dir: Path, tracker: str, dataset: str, config: str) -> dict[ return None -def _format_table(rows: list[tuple[str, dict[str, float] | None]]) -> str: - header = "| Dataset | HOTA | IDF1 | MOTA |" - sep = "| :-------: | :--: | :--: | :--: |" +def _format_table( + rows: list[tuple[str, dict[str, float] | None]], + *, + row_header: str, + row_width: int, +) -> str: + header = f"| {row_header} | HOTA | IDF1 | MOTA |" + sep = "| :-------: | :------: | :------: | :------: |" body = [] for label, scores in rows: + label_cell = f"{label:^{row_width}}" if scores is None: - body.append(f"| {label:^9} | — | — | — |") + body.append(f"| {label_cell} | — | — | — |") else: - cells = " | ".join(f"{scores.get(k, float('nan')):4.1f}" if k in scores else " — " for k in _TARGETS) - body.append(f"| {label:^9} | {cells} |") + hota = f"{scores['HOTA']:6.1f}" if "HOTA" in scores else " — " + idf1 = f"{scores['IDF1']:6.1f}" if "IDF1" in scores else " — " + mota = f"{scores['MOTA']:6.1f}" if "MOTA" in scores else " — " + body.append(f"| {label_cell} | {hota} | {idf1} | {mota} |") return "\n".join([header, sep, *body]) -def main(argv: list[str] | None = None) -> int: - p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) - p.add_argument("--tracker", required=True) - p.add_argument("--output-dir", type=Path, required=True) - p.add_argument("--datasets", default=",".join(DATASETS), help="Comma-separated subset; default=all.") - args = p.parse_args(argv) +def _collect_tracker(out_dir: Path, tracker: str, datasets: list[str]) -> int: + summary: dict[str, dict[str, dict[str, float] | None]] = {} + sections: list[str] = [] + + for config in _CONFIGS: + rows = [] + any_present = False + for dataset in datasets: + scores = _row_scores(out_dir, tracker, dataset, config) + rows.append((LABELS.get(dataset, dataset), scores)) + summary.setdefault(dataset, {})[config] = scores + if scores is not None: + any_present = True + if not any_present: + continue + title = "Default parameters" if config == "default" else "Tuned parameters" + sections.append(f"## {title}\n\n{_format_table(rows, row_header='Dataset', row_width=9)}\n") + + out = out_dir / tracker + out.mkdir(parents=True, exist_ok=True) + md = f"# {tracker} benchmark\n\n" + ("\n".join(sections) if sections else "_No scores found yet._\n") + (out / "tables.md").write_text(md) + (out / "summary.json").write_text(json.dumps({"tracker": tracker, "datasets": summary}, indent=2)) + + print(md) + print(f"saved → {out / 'tables.md'}") + print(f"saved → {out / 'summary.json'}") + return 0 if sections else 1 + + +def _collect_comparison(out_dir: Path, dataset: str, trackers: list[str]) -> int: + if dataset not in DATASETS: + print(f"unknown dataset: {dataset!r}", file=sys.stderr) + return 1 - datasets = [d.strip() for d in args.datasets.split(",") if d.strip()] summary: dict[str, dict[str, dict[str, float] | None]] = {} sections: list[str] = [] + dataset_label = LABELS.get(dataset, dataset) for config in _CONFIGS: rows = [] any_present = False - for d in datasets: - scores = _row_scores(args.output_dir, args.tracker, d, config) - rows.append((LABELS.get(d, d), scores)) - summary.setdefault(d, {})[config] = scores + for tracker in trackers: + label = TRACKER_LABELS.get(tracker, tracker) + scores = _row_scores(out_dir, tracker, dataset, config) + rows.append((label, scores)) + summary.setdefault(tracker, {})[config] = scores if scores is not None: any_present = True if not any_present: continue title = "Default parameters" if config == "default" else "Tuned parameters" - sections.append(f"## {title}\n\n{_format_table(rows)}\n") + sections.append(f"## {title}\n\n{_format_table(rows, row_header='Tracker', row_width=9)}\n") - out = args.output_dir / args.tracker + out = out_dir / "comparison" / dataset out.mkdir(parents=True, exist_ok=True) - md = f"# {args.tracker} benchmark\n\n" + ("\n".join(sections) if sections else "_No scores found yet._\n") + md = f"# {dataset_label} — tracker comparison\n\n" + ( + "\n".join(sections) if sections else "_No scores found yet._\n" + ) (out / "tables.md").write_text(md) - (out / "summary.json").write_text(json.dumps({"tracker": args.tracker, "datasets": summary}, indent=2)) + (out / "summary.json").write_text( + json.dumps({"dataset": dataset, "trackers": summary}, indent=2), + ) print(md) print(f"saved → {out / 'tables.md'}") @@ -110,5 +159,31 @@ def main(argv: list[str] | None = None) -> int: return 0 if sections else 1 +def main(argv: list[str] | None = None) -> int: + p = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + p.add_argument("--output-dir", type=Path, required=True) + p.add_argument("--tracker", help="Single tracker — rows are datasets (default collect mode).") + p.add_argument("--datasets", default=",".join(DATASETS), help="Comma-separated subset; default=all.") + p.add_argument( + "--compare-dataset", + help="One dataset — rows are trackers (comparison.md layout). Requires --trackers.", + ) + p.add_argument( + "--trackers", + default=",".join(COMPARISON_TRACKERS), + help="Comma-separated tracker ids for --compare-dataset.", + ) + args = p.parse_args(argv) + + if args.compare_dataset: + trackers = [t.strip() for t in args.trackers.split(",") if t.strip()] + return _collect_comparison(args.output_dir, args.compare_dataset, trackers) + + if not args.tracker: + p.error("pass --tracker or --compare-dataset") + datasets = [d.strip() for d in args.datasets.split(",") if d.strip()] + return _collect_tracker(args.output_dir, args.tracker, datasets) + + if __name__ == "__main__": raise SystemExit(main()) diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py index 5653a4806..684db944a 100644 --- a/benchmark/scripts/datasets.py +++ b/benchmark/scripts/datasets.py @@ -20,6 +20,16 @@ DATASETS = ("mot17", "sportsmot", "soccernet", "dancetrack") LABELS = {"mot17": "MOT17", "sportsmot": "SportsMOT", "soccernet": "SoccerNet", "dancetrack": "DanceTrack"} +# Trackers shown side-by-side in docs/trackers/comparison.md — single source of truth for +# Makefile (via `datasets.py --field comparison_trackers`) and collect.py. +COMPARISON_TRACKERS = ("sort", "bytetrack", "ocsort", "botsort") +TRACKER_LABELS = { + "sort": "SORT", + "bytetrack": "ByteTrack", + "ocsort": "OC-SORT", + "botsort": "BoT-SORT", +} + # Per-dataset splits used by the benchmark workflow. TUNE_SPLIT = {"soccernet": "train", "dancetrack": "train", "sportsmot": "val", "mot17": "val"} EVAL_SPLIT = {"soccernet": "test", "dancetrack": "val", "sportsmot": "val", "mot17": "val"} @@ -135,14 +145,36 @@ def _print_field(data_root: Path, dataset: str, split: str, what: str) -> str: return "" if value is None else str(value) +_GLOBAL_FIELDS = { + "comparison_trackers": lambda _root: " ".join(COMPARISON_TRACKERS), + "comparison_trackers_csv": lambda _root: ",".join(COMPARISON_TRACKERS), + "datasets": lambda _root: " ".join(DATASETS), + "datasets_csv": lambda _root: ",".join(DATASETS), +} +_LAYOUT_FIELDS = ("det_dir", "gt_dir", "images_dir", "seqmap") + + +def _print_global(data_root: Path, field: str) -> str: + return _GLOBAL_FIELDS[field](data_root) + + # Tiny CLI so the Makefile can query layout values without duplicating paths. if __name__ == "__main__": import argparse p = argparse.ArgumentParser(description="Print one layout field (used by Makefile).") - p.add_argument("--data-root", type=Path, required=True) - p.add_argument("--dataset", required=True) - p.add_argument("--split", required=True) - p.add_argument("--field", choices=["det_dir", "gt_dir", "images_dir", "seqmap"], required=True) + p.add_argument("--data-root", type=Path, default=BENCHMARK_ROOT / "data") + p.add_argument("--dataset") + p.add_argument("--split") + p.add_argument( + "--field", + choices=[*_LAYOUT_FIELDS, *_GLOBAL_FIELDS], + required=True, + ) args = p.parse_args() - print(_print_field(args.data_root, args.dataset, args.split, args.field)) + if args.field in _GLOBAL_FIELDS: + print(_print_global(args.data_root, args.field)) + else: + if not args.dataset or not args.split: + p.error(f"--dataset and --split are required for --field {args.field!r}") + print(_print_field(args.data_root, args.dataset, args.split, args.field)) From 9f402d4787e4bc26f1ce2f29b3bf8240d9ad9179 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 26 May 2026 14:09:58 +0000 Subject: [PATCH 10/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- benchmark/README.md | 64 ++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 50d314cc8..34edeb0ff 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -59,7 +59,7 @@ make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned CODABENCH_TOKEN=... Then `make collect TRACKER=bytetrack` to refresh the table. -or `make collect-comparison DATASET=dancetrack` to refresh the comparison table. +or `make collect-comparison DATASET=dancetrack` to refresh the comparison table. ## Data setup @@ -118,25 +118,25 @@ make data-check DATA_ROOT="/path/to/datasets" Run from `benchmark/`. Pass variables on the command line or export them first (`DATA_ROOT`, `CODABENCH_TOKEN`, …). -| Target | Description | -| ------------------- | ------------------------------------------------------------------------------------------ | -| `setup` | Install `trackers[tune]` from the repo root | -| `data-check` | Print present/missing assets under `DATA_ROOT` | -| `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | -| `prep-all` | Prep all four datasets | -| `tune` | Optuna search → `best_params.json` (`TRACKER=`, `DATASET=`, `N_TRIALS=`) | -| `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | -| `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | -| `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | -| `poll` | Poll an existing Codabench submission for scores (`SUBMISSION_ID=`, `TRACKER=`, `DATASET=`, `CONFIG=`) | -| `benchmark-default` | Prep → track-default → tables. One `DATASET` → all four trackers + comparison table | -| `benchmark-tuned` | Prep → tune + track-tuned → tables. One `DATASET` → all four trackers + comparison table | -| `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | -| `benchmark-comparison-default` | Shorthand: `benchmark-default` with `TRACKERS=all` on one `DATASET=` | -| `benchmark-comparison-tuned` | Shorthand: `benchmark-tuned` with `TRACKERS=all` on one `DATASET=` | -| `collect` | Rebuild per-tracker `tables.md` (`TRACKER=`, optional `DATASETS=`) | -| `collect-comparison`| Rebuild comparison table for one dataset (`DATASET=dancetrack`) | -| `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | +| Target | Description | +| ------------------------------ | ------------------------------------------------------------------------------------------------------ | +| `setup` | Install `trackers[tune]` from the repo root | +| `data-check` | Print present/missing assets under `DATA_ROOT` | +| `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | +| `prep-all` | Prep all four datasets | +| `tune` | Optuna search → `best_params.json` (`TRACKER=`, `DATASET=`, `N_TRIALS=`) | +| `track-default` | Track test split with registry defaults, then score (`TRACKER=`, `DATASET=`) | +| `track-tuned` | Track test split with `best_params.json`, then score (`TRACKER=`, `DATASET=`) | +| `upload` | Upload an existing `submission.zip` (`TRACKER=`, `DATASET=`, `CONFIG=default` or `tuned`) | +| `poll` | Poll an existing Codabench submission for scores (`SUBMISSION_ID=`, `TRACKER=`, `DATASET=`, `CONFIG=`) | +| `benchmark-default` | Prep → track-default → tables. One `DATASET` → all four trackers + comparison table | +| `benchmark-tuned` | Prep → tune + track-tuned → tables. One `DATASET` → all four trackers + comparison table | +| `benchmark` | Full pipeline; set `BENCHMARK_CONFIG` to `default`, `tuned`, or `all` (default: `default`) | +| `benchmark-comparison-default` | Shorthand: `benchmark-default` with `TRACKERS=all` on one `DATASET=` | +| `benchmark-comparison-tuned` | Shorthand: `benchmark-tuned` with `TRACKERS=all` on one `DATASET=` | +| `collect` | Rebuild per-tracker `tables.md` (`TRACKER=`, optional `DATASETS=`) | +| `collect-comparison` | Rebuild comparison table for one dataset (`DATASET=dancetrack`) | +| `clean` | Remove `benchmark_prep/` and `benchmark_outputs/` | ## Usage @@ -192,19 +192,19 @@ make clean ### Variables -| Variable | Default | Purpose | -| ------------------ | --------------------- | ------------------------------------------- | -| `TRACKER` | `sort` | Single tracker when `DATASETS` lists more than one dataset | +| Variable | Default | Purpose | +| ------------------ | --------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `TRACKER` | `sort` | Single tracker when `DATASETS` lists more than one dataset | | `TRACKERS` | — | Space-separated list, or `all` (see `COMPARISON_TRACKERS` in `scripts/datasets.py`). When `DATASETS` is a single dataset, defaults to all comparison trackers | -| `DATA_ROOT` | `./data` | Raw dataset tree | -| `DATASET` | `mot17` | Single-dataset targets | -| `DATASETS` | all four | Space-separated subset for `benchmark*` | -| `BENCHMARK_CONFIG` | `default` | `benchmark`: `default`, `tuned`, or `all` | -| `CONFIG` | — | `upload`: `default` or `tuned` | -| `N_TRIALS` | `10` | Optuna trials per dataset | -| `CODABENCH_TOKEN` | — | Required for Codabench datasets | -| `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | -| `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | +| `DATA_ROOT` | `./data` | Raw dataset tree | +| `DATASET` | `mot17` | Single-dataset targets | +| `DATASETS` | all four | Space-separated subset for `benchmark*` | +| `BENCHMARK_CONFIG` | `default` | `benchmark`: `default`, `tuned`, or `all` | +| `CONFIG` | — | `upload`: `default` or `tuned` | +| `N_TRIALS` | `10` | Optuna trials per dataset | +| `CODABENCH_TOKEN` | — | Required for Codabench datasets | +| `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | +| `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories when present. From 8c7226bd140ae0f4951ecf1e69c279241c05e533 Mon Sep 17 00:00:00 2001 From: Alexander Bodner <61150961+AlexBodner@users.noreply.github.com> Date: Tue, 26 May 2026 15:16:16 -0300 Subject: [PATCH 11/54] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- benchmark/scripts/codabench_submit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 8f1aa8f59..479f3b861 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -407,7 +407,7 @@ def main(argv: list[str] | None = None) -> int: "--competition-id", type=int, default=int(os.environ.get("CODABENCH_COMPETITION", str(DEFAULT_COMPETITION_ID))), - help="Codabench competition id for result URL (mot17: 10049, sportsmot: 13077).", + help="Codabench competition id for result URL; see benchmark/scripts/datasets.py for supported dataset competition ids (including MOT17, SportsMOT, and DanceTrack).", ) p.add_argument( "--base-url", From 6477570b7b42019c166925709140b3ec14d3f363 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 26 May 2026 15:38:02 -0300 Subject: [PATCH 12/54] Address benchmark PR review feedback --- benchmark/Makefile | 7 ++++--- benchmark/scripts/codabench_submit.py | 4 ++-- benchmark/scripts/prep_data.py | 6 +++++- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index 7327c9cea..9edad70e1 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -14,6 +14,7 @@ SHELL := /bin/bash ROOT := $(CURDIR) REPO_ROOT := $(abspath $(ROOT)/..) PYTHON ?= python +UV ?= uv DATA_ROOT ?= $(ROOT)/data PREP_DIR ?= $(ROOT)/benchmark_prep @@ -110,7 +111,7 @@ help: @echo "Codabench upload requires CODABENCH_TOKEN. See README for data setup." setup: - $(PYTHON) -m pip install -e "$(REPO_ROOT)[tune]" + $(UV) pip install -e "$(REPO_ROOT)[tune]" data-check: @$(PYTHON) scripts/data_check.py --data-root "$(DATA_ROOT)" @@ -226,8 +227,8 @@ benchmark: @case "$(BENCHMARK_CONFIG)" in default|tuned|all) ;; \ *) echo "Set BENCHMARK_CONFIG=default, tuned, or all (or use benchmark-default / benchmark-tuned)"; exit 1;; \ esac - @if [ -z "$(CODABENCH_TOKEN)" ]; then \ - echo "ERROR: CODABENCH_TOKEN must be set (required for mot17, sportsmot, dancetrack)."; exit 1; \ + @if [ -n "$(strip $(filter $(CODABENCH_DATASETS),$(DATASETS)))" ] && [ -z "$(CODABENCH_TOKEN)" ]; then \ + echo "ERROR: CODABENCH_TOKEN must be set (required for: $(filter $(CODABENCH_DATASETS),$(DATASETS)))."; exit 1; \ fi @for d in $(DATASETS); do \ echo "===== prep [$$d] ====="; \ diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 8f1aa8f59..6015dd4e1 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -401,13 +401,13 @@ def main(argv: list[str] | None = None) -> int: "--phase", type=int, default=int(os.environ.get("CODABENCH_PHASE", str(DEFAULT_PHASE_ID))), - help="Codabench phase id (mot17: 16382, sportsmot: 21402).", + help="Codabench phase id (e.g. mot17: 16382, sportsmot: 21402, dancetrack: 24635; see scripts/datasets.py CODABENCH).", ) p.add_argument( "--competition-id", type=int, default=int(os.environ.get("CODABENCH_COMPETITION", str(DEFAULT_COMPETITION_ID))), - help="Codabench competition id for result URL (mot17: 10049, sportsmot: 13077).", + help="Codabench competition id for result URL (e.g. mot17: 10049, sportsmot: 13077, dancetrack: 14885; see scripts/datasets.py CODABENCH).", ) p.add_argument( "--base-url", diff --git a/benchmark/scripts/prep_data.py b/benchmark/scripts/prep_data.py index eac0b6503..aea04cfa5 100644 --- a/benchmark/scripts/prep_data.py +++ b/benchmark/scripts/prep_data.py @@ -15,7 +15,11 @@ Run via the Makefile (``make prep``) or directly: - python prep_data.py --dataset mot17 --split val --data-root ./data --prep-dir ./benchmark_prep + python prep_data.py --dataset mot17 --split all --data-root ./data --prep-dir ./benchmark_prep + python prep_data.py --dataset mot17 --split tune --data-root ./data --prep-dir ./benchmark_prep + +``--split`` accepts logical names (``tune`` / ``eval`` / ``submit`` / ``all``), not vendor split +folders like ``val`` or ``test``, those are resolved per dataset in ``datasets.py``. """ from __future__ import annotations From 825cebded54ef651bcc2a4eaa646a5da2af03a0a Mon Sep 17 00:00:00 2001 From: Alexander Bodner <61150961+AlexBodner@users.noreply.github.com> Date: Tue, 26 May 2026 15:39:37 -0300 Subject: [PATCH 13/54] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- benchmark/scripts/track_split.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/benchmark/scripts/track_split.py b/benchmark/scripts/track_split.py index 3cd439a06..807d965bb 100644 --- a/benchmark/scripts/track_split.py +++ b/benchmark/scripts/track_split.py @@ -20,7 +20,8 @@ Usage (see Makefile for the wiring): python track_split.py --tracker sort --dataset mot17 --split val \ - --prep-dir ./benchmark_prep --output-dir ./benchmark_outputs/sort/mot17/default \ + --data-root ./data --prep-dir ./benchmark_prep \ + --output-dir ./benchmark_outputs/sort/mot17/default \ [--params best_params.json] """ From 0fb692734e9b494b6b9ede9fa3cb2c055bdc5a3b Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 26 May 2026 18:46:37 -0300 Subject: [PATCH 14/54] fixed ruff --- benchmark/scripts/codabench_submit.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index 6015dd4e1..c08062689 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -286,8 +286,9 @@ def get_submission( wait = min(10.0 * (2**attempt), 60.0) print(f" transient API error, retry in {wait:.0f}s: {exc}", flush=True) time.sleep(wait) - assert last_exc is not None - raise last_exc + if last_exc is not None: + raise last_exc + raise RuntimeError(f"Failed to fetch submission {submission_id}") def extract_metric_scores( @@ -401,13 +402,19 @@ def main(argv: list[str] | None = None) -> int: "--phase", type=int, default=int(os.environ.get("CODABENCH_PHASE", str(DEFAULT_PHASE_ID))), - help="Codabench phase id (e.g. mot17: 16382, sportsmot: 21402, dancetrack: 24635; see scripts/datasets.py CODABENCH).", + help=( + "Codabench phase id (e.g. mot17: 16382, sportsmot: 21402, dancetrack: 24635; " + "see scripts/datasets.py CODABENCH)." + ), ) p.add_argument( "--competition-id", type=int, default=int(os.environ.get("CODABENCH_COMPETITION", str(DEFAULT_COMPETITION_ID))), - help="Codabench competition id for result URL (e.g. mot17: 10049, sportsmot: 13077, dancetrack: 14885; see scripts/datasets.py CODABENCH).", + help=( + "Codabench competition id for result URL " + "(e.g. mot17: 10049, sportsmot: 13077, dancetrack: 14885; see scripts/datasets.py CODABENCH)." + ), ) p.add_argument( "--base-url", From d615514f24c1d65805f8e571762d8d836b25f649 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Fri, 29 May 2026 15:39:41 -0300 Subject: [PATCH 15/54] space change for botsort parameters --- benchmark/Makefile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index 9edad70e1..5b9e828a9 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -57,7 +57,8 @@ CODABENCH_POLL_INTERVAL ?= 10 # BoT-SORT requires CMC on by default for the published numbers. ifeq ($(TRACKER),botsort) ifeq ($(strip $(FIXED_PARAMS)),) - FIXED_PARAMS := {"enable_cmc": true} + # No spaces — Make word-splits $(FIXED_PARAMS) when expanding recipe lines. + FIXED_PARAMS := {"enable_cmc":true} endif endif @@ -243,7 +244,7 @@ benchmark: if [ "$(BENCHMARK_CONFIG)" = "tuned" ] || [ "$(BENCHMARK_CONFIG)" = "all" ]; then \ echo ""; echo "===== [$$t/$$d] tune ====="; \ extras=""; \ - if [ "$$t" = "botsort" ]; then extras='FIXED_PARAMS={"enable_cmc": true}'; fi; \ + if [ "$$t" = "botsort" ]; then extras='FIXED_PARAMS={"enable_cmc":true}'; fi; \ $(MAKE) tune TRACKER=$$t DATASET=$$d N_TRIALS=$(N_TRIALS) $$extras || exit 1; \ echo "===== [$$t/$$d] tuned ====="; \ $(MAKE) track-tuned TRACKER=$$t DATASET=$$d || exit 1; \ From cb9b1f0ebbc970942272cb990c659eac87a928ed Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Fri, 12 Jun 2026 10:29:42 -0300 Subject: [PATCH 16/54] Tune DanceTrack on validation split instead of train. Aligns the benchmark with the train+val+test methodology: optimize on val, score on Codabench test. Co-authored-by: Cursor --- benchmark/Makefile | 2 +- benchmark/README.md | 2 +- benchmark/scripts/datasets.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index 5b9e828a9..29994ca6d 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -72,7 +72,7 @@ DATASETS ?= $(ALL_DATASETS) # delegate to scripts/datasets.py to stay DRY. mot17_TUNE_SPLIT := val sportsmot_TUNE_SPLIT := val -dancetrack_TUNE_SPLIT := train +dancetrack_TUNE_SPLIT := val soccernet_TUNE_SPLIT := train SCORE_SPLIT := test diff --git a/benchmark/README.md b/benchmark/README.md index 34edeb0ff..a8e48dc8c 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -111,7 +111,7 @@ make data-check DATA_ROOT="/path/to/datasets" | ------------------ | ----- | ----- | ----------------------- | | MOT17 | val | test | Codabench | | SportsMOT | val | test | Codabench | -| DanceTrack | train | test | Codabench | +| DanceTrack | val | test | Codabench | | SoccerNet-tracking | train | test | Local (`trackers eval`) | ## Commands diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py index 684db944a..0da0d0ed2 100644 --- a/benchmark/scripts/datasets.py +++ b/benchmark/scripts/datasets.py @@ -31,7 +31,7 @@ } # Per-dataset splits used by the benchmark workflow. -TUNE_SPLIT = {"soccernet": "train", "dancetrack": "train", "sportsmot": "val", "mot17": "val"} +TUNE_SPLIT = {"soccernet": "train", "dancetrack": "val", "sportsmot": "val", "mot17": "val"} EVAL_SPLIT = {"soccernet": "test", "dancetrack": "val", "sportsmot": "val", "mot17": "val"} SUBMIT_SPLIT = {"dancetrack": "test", "sportsmot": "test", "mot17": "test"} # soccernet has no Codabench From 3c3476a4e6958013fb5883e9981d4911745807da Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Fri, 12 Jun 2026 16:53:55 -0300 Subject: [PATCH 17/54] Add C-BIoU to benchmark comparison trackers and harden Codabench polling. Include cbiou in COMPARISON_TRACKERS for tune/benchmark/collect workflows, and retry submission polling on transient DNS and connection errors. Co-authored-by: Cursor --- benchmark/scripts/codabench_submit.py | 9 ++++++--- benchmark/scripts/datasets.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/benchmark/scripts/codabench_submit.py b/benchmark/scripts/codabench_submit.py index c08062689..93390d715 100644 --- a/benchmark/scripts/codabench_submit.py +++ b/benchmark/scripts/codabench_submit.py @@ -256,9 +256,12 @@ def print_submission_failure_logs( print(f" logs →\n{text}", flush=True) -def _is_transient_http_error(exc: BaseException) -> bool: +def _is_transient_error(exc: BaseException) -> bool: msg = str(exc) - return any(f"HTTP {code}" in msg for code in _TRANSIENT_HTTP_STATUSES) + if any(f"HTTP {code}" in msg for code in _TRANSIENT_HTTP_STATUSES): + return True + # DNS blips, connection resets, timeouts, etc. (OSError wrapped by _request). + return " failed: [" in msg or " failed: timed out" in msg def get_submission( @@ -281,7 +284,7 @@ def get_submission( return payload except RuntimeError as exc: last_exc = exc - if attempt + 1 >= max_retries or not _is_transient_http_error(exc): + if attempt + 1 >= max_retries or not _is_transient_error(exc): raise wait = min(10.0 * (2**attempt), 60.0) print(f" transient API error, retry in {wait:.0f}s: {exc}", flush=True) diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py index 0da0d0ed2..2733ff1a6 100644 --- a/benchmark/scripts/datasets.py +++ b/benchmark/scripts/datasets.py @@ -22,12 +22,13 @@ # Trackers shown side-by-side in docs/trackers/comparison.md — single source of truth for # Makefile (via `datasets.py --field comparison_trackers`) and collect.py. -COMPARISON_TRACKERS = ("sort", "bytetrack", "ocsort", "botsort") +COMPARISON_TRACKERS = ("sort", "bytetrack", "ocsort", "botsort", "cbiou") TRACKER_LABELS = { "sort": "SORT", "bytetrack": "ByteTrack", "ocsort": "OC-SORT", "botsort": "BoT-SORT", + "cbiou": "C-BIoU", } # Per-dataset splits used by the benchmark workflow. From daea238a64d895642f94ec326a2998086b31aa11 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 11:00:18 -0300 Subject: [PATCH 18/54] feat(reid): add roboflow-reid extra and numpy-only association glue Add the optional trackers[reid] extra (git-pinned roboflow-reid during review) plus the lazy trackers._reid boundary that resolves reid.ReIDModel on demand. Ship association-only, torch-free modules under trackers.core.reid: the ReIDEncoder protocol, FeatureBank (per-track EMA), and appearance_similarity / extract_detection_embeddings. The model stack (encoder, weights, preprocessing, catalog, gallery eval) lives in the standalone reid package and is never vendored. Co-authored-by: Cursor --- pyproject.toml | 8 +++ src/trackers/_reid.py | 41 +++++++++++ src/trackers/core/reid/__init__.py | 26 +++++++ src/trackers/core/reid/appearance.py | 98 ++++++++++++++++++++++++++ src/trackers/core/reid/encoder.py | 35 +++++++++ src/trackers/core/reid/feature_bank.py | 64 +++++++++++++++++ uv.lock | 77 +++++++++++++++++++- 7 files changed, 348 insertions(+), 1 deletion(-) create mode 100644 src/trackers/_reid.py create mode 100644 src/trackers/core/reid/__init__.py create mode 100644 src/trackers/core/reid/appearance.py create mode 100644 src/trackers/core/reid/encoder.py create mode 100644 src/trackers/core/reid/feature_bank.py diff --git a/pyproject.toml b/pyproject.toml index ed02d2a4f..8cab3f9b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,6 +47,10 @@ dependencies = [ [project.optional-dependencies] detection = ["inference-models>=0.19.0"] tune = ["optuna>=3.0.0"] +# ReID appearance association sources its model stack from the standalone +# roboflow-reid package. Pinned to a git ref during review; swap to a PyPI +# release (e.g. "roboflow-reid>=0.1.0,<0.2") before merge. See re-ID#1. +reid = ["roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack"] [project.scripts] trackers = "trackers.scripts.__main__:main" @@ -58,6 +62,8 @@ dev = [ "pre-commit>=4.2.0", "torch", "torchvision", + # Real ReID encoder for association / integration tests. + "roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack", ] docs = [ "mkdocs>=1.6.1", @@ -219,5 +225,7 @@ module = [ "rfdetr.*", "supervision", "supervision.*", + "reid", + "reid.*", ] ignore_missing_imports = true diff --git a/src/trackers/_reid.py b/src/trackers/_reid.py new file mode 100644 index 000000000..b94278274 --- /dev/null +++ b/src/trackers/_reid.py @@ -0,0 +1,41 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Lazy boundary to the optional ``roboflow-reid`` package. + +Trackers ships only numpy-only association glue. The appearance encoder, +weights, preprocessing, and catalog live in the standalone ``reid`` package, +installed via the ``trackers[reid]`` extra. This module is the single seam that +resolves ``reid.ReIDModel`` on demand so importing trackers never pulls torch. +""" + +from __future__ import annotations + +import importlib +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from reid import ReIDModel as ReIDModel + +REID_INSTALL_HINT = ( + "ReID features require the optional `trackers[reid]` extra. Install with: pip install 'trackers[reid]'" +) + +_REID_PACKAGE = "reid" + + +def import_reid_model() -> Any: + """Return ``reid.ReIDModel``, rewriting the missing-extra error. + + Raises: + ImportError: With an install hint when ``roboflow-reid`` (or one of its + heavy dependencies) is not installed. + """ + try: + module = importlib.import_module(_REID_PACKAGE) + except ImportError as exc: + raise ImportError(REID_INSTALL_HINT) from exc + return module.ReIDModel diff --git a/src/trackers/core/reid/__init__.py b/src/trackers/core/reid/__init__.py new file mode 100644 index 000000000..721311dd4 --- /dev/null +++ b/src/trackers/core/reid/__init__.py @@ -0,0 +1,26 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""NumPy-only appearance-ReID association glue. + +This package intentionally contains no model stack. The encoder, weights, +preprocessing, and gallery evaluation live in the standalone ``reid`` package +(``pip install 'trackers[reid]'``); import ``ReIDModel`` and evaluation helpers +from there. Everything exported here is importable without torch. +""" + +from __future__ import annotations + +from trackers.core.reid.appearance import appearance_similarity, extract_detection_embeddings +from trackers.core.reid.encoder import ReIDEncoder +from trackers.core.reid.feature_bank import FeatureBank + +__all__ = [ + "FeatureBank", + "ReIDEncoder", + "appearance_similarity", + "extract_detection_embeddings", +] diff --git a/src/trackers/core/reid/appearance.py b/src/trackers/core/reid/appearance.py new file mode 100644 index 000000000..a7cb8e338 --- /dev/null +++ b/src/trackers/core/reid/appearance.py @@ -0,0 +1,98 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Appearance embedding helpers for tracker association.""" + +from __future__ import annotations + +from collections.abc import Sequence + +import numpy as np +import supervision as sv + +from trackers.core.reid.encoder import ReIDEncoder + +_NORM_EPS = 1e-12 + + +def _require_embedding_matrix(embeddings: np.ndarray) -> np.ndarray: + """Return a finite float32 embedding matrix.""" + cleaned = np.asarray(embeddings, dtype=np.float32) + if cleaned.ndim != 2: + raise ValueError(f"embeddings must be 2-D, got shape {cleaned.shape}") + if cleaned.size > 0 and not np.all(np.isfinite(cleaned)): + raise ValueError("embeddings must contain only finite values") + return cleaned + + +def _l2_normalize(embedding: np.ndarray) -> np.ndarray: + """Return an L2-normalised 1-D vector.""" + flat = np.asarray(embedding, dtype=np.float64).reshape(-1) + if flat.size == 0: + raise ValueError("embedding must be non-empty") + if not np.all(np.isfinite(flat)): + raise ValueError("embedding must contain only finite values") + norm = float(np.linalg.norm(flat)) + return (flat / max(norm, _NORM_EPS)).astype(np.float32) + + +def _l2_normalize_rows(embeddings: np.ndarray) -> np.ndarray: + """L2-normalise each row in an embedding matrix.""" + if embeddings.size == 0: + return embeddings + return np.stack([_l2_normalize(row) for row in embeddings]) + + +def extract_detection_embeddings( + model: ReIDEncoder, + frame: np.ndarray, + boxes: np.ndarray, +) -> np.ndarray: + """Extract appearance embeddings for detection boxes.""" + if len(boxes) == 0: + return np.empty((0, 0), dtype=np.float32) + embeddings = _require_embedding_matrix(model.extract_features(sv.Detections(xyxy=boxes), frame)) + if embeddings.shape[0] != len(boxes): + raise ValueError(f"embedding rows ({embeddings.shape[0]}) must match detection boxes ({len(boxes)})") + return embeddings + + +def appearance_similarity( + track_features: Sequence[np.ndarray | None], + det_embeddings: np.ndarray, +) -> np.ndarray: + """Compute cosine similarity between track and detection embeddings.""" + n_tracks = len(track_features) + det_embeddings = _l2_normalize_rows(_require_embedding_matrix(det_embeddings)) + n_dets = det_embeddings.shape[0] + similarity = np.zeros((n_tracks, n_dets), dtype=np.float32) + + if n_tracks == 0 or n_dets == 0: + return similarity + + embed_dim = det_embeddings.shape[1] + track_rows: list[np.ndarray] = [] + kept_indices: list[int] = [] + for track_idx, feature in enumerate(track_features): + if feature is None: + continue + flat = np.asarray(feature, dtype=np.float32).reshape(-1) + if flat.shape[0] != embed_dim: + raise ValueError( + f"track feature dim {flat.shape[0]} does not match detection " + f"embedding dim {embed_dim} (track index {track_idx})" + ) + track_rows.append(_l2_normalize(flat)) + kept_indices.append(track_idx) + + if not track_rows: + return similarity + + cosine_similarities = (np.stack(track_rows) @ det_embeddings.T).astype(np.float32) + for local_idx, track_idx in enumerate(kept_indices): + similarity[track_idx] = cosine_similarities[local_idx] + + return similarity diff --git a/src/trackers/core/reid/encoder.py b/src/trackers/core/reid/encoder.py new file mode 100644 index 000000000..7279c01ba --- /dev/null +++ b/src/trackers/core/reid/encoder.py @@ -0,0 +1,35 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Encoder protocol for ReID association.""" + +from __future__ import annotations + +from typing import Protocol + +import numpy as np +import supervision as sv + + +class ReIDEncoder(Protocol): + """Appearance encoder used for tracking association. + + Trackers only depend on ``extract_features``. ``reid.ReIDModel`` structurally + satisfies this protocol, and custom or test encoders may implement it without + depending on the full model stack. + """ + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + """Return appearance embeddings for each detection box. + + Args: + detections: Boxes to embed (``xyxy``). + frame: BGR frame the detections were produced on. + + Returns: + Float32 array of shape ``(N, D)``, or ``(0, 0)`` when empty. + """ + ... diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py new file mode 100644 index 000000000..d29558fc7 --- /dev/null +++ b/src/trackers/core/reid/feature_bank.py @@ -0,0 +1,64 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Per-track exponential moving average feature bank.""" + +from __future__ import annotations + +import numpy as np + + +class FeatureBank: + """Per-track EMA appearance embedding. + + Args: + alpha: EMA momentum in ``[0, 1]``. + """ + + def __init__(self, alpha: float = 0.9) -> None: + if not 0.0 <= alpha <= 1.0: + raise ValueError(f"alpha must be in [0, 1], got {alpha}") + self._alpha = alpha + self._feature: np.ndarray | None = None + + @property + def feature(self) -> np.ndarray | None: + """Current stored embedding, or ``None`` if never updated.""" + return None if self._feature is None else self._feature.copy() + + @property + def is_initialized(self) -> bool: + """``True`` after the first update.""" + return self._feature is not None + + def update(self, embedding: np.ndarray) -> None: + """Blend an embedding into the stored feature.""" + cleaned = _require_embedding(embedding) + + if self._feature is None: + self._feature = cleaned.copy() + return + + if self._feature.shape != cleaned.shape: + raise ValueError( + f"embedding shape {cleaned.shape} does not match stored feature shape {self._feature.shape}" + ) + + self._feature = (self._alpha * self._feature + (1.0 - self._alpha) * cleaned).astype(np.float32) + + def reset(self) -> None: + """Clear the stored feature.""" + self._feature = None + + +def _require_embedding(embedding: np.ndarray) -> np.ndarray: + """Return a finite 1-D float32 vector.""" + flat = np.asarray(embedding, dtype=np.float32).reshape(-1) + if flat.size == 0: + raise ValueError("embedding must be non-empty") + if not np.all(np.isfinite(flat)): + raise ValueError("embedding must contain only finite values") + return flat diff --git a/uv.lock b/uv.lock index d0f51474a..e39cc47e7 100644 --- a/uv.lock +++ b/uv.lock @@ -158,6 +158,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/37/fb6973edeb700f6e3d6ff222400602ab1830446c25c7b4676d8de93e65b8/backrefs-5.8-py39-none-any.whl", hash = "sha256:a66851e4533fb5b371aa0628e1fee1af05135616b86140c9d787a2ffdf4b8fdc", size = 380336, upload-time = "2025-02-25T16:53:29.858Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "bitsandbytes" version = "0.47.0" @@ -839,6 +852,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, ] +[[package]] +name = "gdown" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "filelock" }, + { name = "requests", extra = ["socks"] }, + { name = "tqdm" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/b5/a45f62f20664031bf74a6aeb6f8d8cd5910e411bf90d756bd6b09bdc6c35/gdown-6.1.0.tar.gz", hash = "sha256:361c6e04c6ca335df50b9d71f40bcfe9ab70fb26a1b0e890a427267781389553", size = 269670, upload-time = "2026-05-30T11:56:21.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/56/a99f0f159cce5b26d267317d436afee184f45fc7911938757d7cbbd2d10c/gdown-6.1.0-py3-none-any.whl", hash = "sha256:38a36a94275b8272f684db469bbd73b4d1f64cbbc1751bcb993a1b2be8f013c8", size = 19216, upload-time = "2026-05-30T11:56:20.016Z" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -2797,6 +2826,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, ] +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -3277,6 +3315,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + [[package]] name = "requests-file" version = "3.0.1" @@ -3386,6 +3429,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, ] +[[package]] +name = "roboflow-reid" +version = "0.1.0" +source = { git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack#b6da83541adfc7a93eb962efc85860eb3c0e0583" } +dependencies = [ + { name = "gdown" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "safetensors" }, + { name = "supervision" }, + { name = "timm" }, + { name = "torch" }, + { name = "torchvision" }, +] + [[package]] name = "safetensors" version = "0.7.0" @@ -3770,6 +3830,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] +[[package]] +name = "soupsieve" +version = "2.9" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/80/f1/93422647dd7e461f23d254e6b2bfa687a85b53aeb4903fcdbb74474d4584/soupsieve-2.9.tar.gz", hash = "sha256:acee8417325c5653e1377dc31eccad59eb82cbc65942afe6174c53b3aaad63fc", size = 122122, upload-time = "2026-07-19T01:35:18.425Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7b/d6/3185ab5ad1280319b31986898f3206dd7227cd75e293d4dba2a5e6bf27a0/soupsieve-2.9-py3-none-any.whl", hash = "sha256:a2b2c76d67df2382d245409fd71e321a571717e58463efa32ace87dcadac2c12", size = 37387, upload-time = "2026-07-19T01:35:17.106Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.49" @@ -4142,6 +4211,9 @@ dependencies = [ detection = [ { name = "inference-models" }, ] +reid = [ + { name = "roboflow-reid" }, +] tune = [ { name = "optuna" }, ] @@ -4155,6 +4227,7 @@ build = [ dev = [ { name = "pre-commit" }, { name = "pytest" }, + { name = "roboflow-reid" }, { name = "torch" }, { name = "torchvision" }, { name = "uv" }, @@ -4182,10 +4255,11 @@ requires-dist = [ { name = "pydeprecate", specifier = ">=0.7.0" }, { name = "requests", specifier = ">=2.28.0" }, { name = "rich", specifier = ">=13.0.0" }, + { name = "roboflow-reid", marker = "extra == 'reid'", git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack" }, { name = "scipy", specifier = ">=1.13.1" }, { name = "supervision", specifier = ">=0.26.1" }, ] -provides-extras = ["detection", "tune"] +provides-extras = ["detection", "tune", "reid"] [package.metadata.requires-dev] build = [ @@ -4196,6 +4270,7 @@ build = [ dev = [ { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.3.3" }, + { name = "roboflow-reid", git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack" }, { name = "torch" }, { name = "torchvision" }, { name = "uv", specifier = ">=0.4.20" }, From 8d8faf7c25b32f9a96abf47dd88067f5311a218a Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 11:00:31 -0300 Subject: [PATCH 19/54] feat(reid): wire BoT-SORT appearance association, CLI, docs, and tests Add appearance-IoU fusion (botsort/fusion.py) and wire it into BoT-SORT's first and unconfirmed association stages, gated by proximity (standard IoU) and appearance thresholds. Tracklets gain an optional per-track feature bank; matched high-confidence detections update it. reid_model, reid_ema_alpha, appearance_threshold, and proximity_threshold are new BoTSORTTracker params; reid_model is excluded from CLI reflection. Add --tracker.reid.{enable,model,device,architecture} CLI flags routed through trackers._reid to reid.ReIDModel.from_pretrained. Importing BoT-SORT stays torch-free (asserted by an isolation test). Add association/fusion/CLI unit tests and a reid-backed integration smoke; docs and mkdocs nav cover the association-only surface and link the model/eval stack out to reid. CI installs the reid extra (unfrozen while the dep is a git pin). Co-authored-by: Cursor --- .github/workflows/ci-tests.yml | 5 +- README.md | 1 + docs/api/reid.md | 55 ++++ docs/learn/install.md | 24 ++ docs/trackers/botsort.md | 36 +++ mkdocs.yml | 1 + src/trackers/core/base.py | 16 +- src/trackers/core/botsort/fusion.py | 44 +++ src/trackers/core/botsort/tracker.py | 181 ++++++++++-- src/trackers/core/botsort/tracklet.py | 7 + src/trackers/scripts/track.py | 122 +++++++- tests/core/test_botsort_reid.py | 303 ++++++++++++++++++++ tests/core/test_botsort_reid_integration.py | 45 +++ tests/scripts/test_track.py | 79 +++++ 14 files changed, 884 insertions(+), 35 deletions(-) create mode 100644 docs/api/reid.md create mode 100644 src/trackers/core/botsort/fusion.py create mode 100644 tests/core/test_botsort_reid.py create mode 100644 tests/core/test_botsort_reid_integration.py diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9ca4d9f38..fbbd8627d 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -30,7 +30,10 @@ jobs: prune-cache: ${{ matrix.os != 'windows-latest' }} - name: 🚀 Install Packages - run: uv sync --frozen --group dev + # NOTE: --frozen is dropped while the `reid` extra pins roboflow-reid to a + # git ref (see pyproject). Restore `--frozen` once roboflow-reid publishes + # to PyPI and uv.lock is regenerated. + run: uv sync --group dev --extra reid - name: 🧪 Run the Import test run: uv run python -c "import trackers" diff --git a/README.md b/README.md index 31b984165..f33ee5a23 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Keeping track of objects across video frames is one of those problems that sound - **Benchmarked across four datasets.** MOT17, SportsMOT, SoccerNet, and DanceTrack — at default parameters and after hyperparameter tuning, so you know what to expect before you deploy. - **Tunable out of the box.** Built-in Optuna-based hyperparameter search via `trackers tune` so you can optimize for your specific scene and detector. - **Camera motion compensation.** BoT-SORT handles moving cameras natively, keeping track IDs stable even when the whole frame shifts. +- **Optional appearance ReID.** BoT-SORT can fuse visual embeddings with motion for harder association scenes — install `trackers[reid]` (pulls in the standalone `roboflow-reid` package) and pass a `reid.ReIDModel` as `reid_model`. ## Install diff --git a/docs/api/reid.md b/docs/api/reid.md new file mode 100644 index 000000000..b540c004c --- /dev/null +++ b/docs/api/reid.md @@ -0,0 +1,55 @@ +--- +description: Appearance-ReID association utilities in Roboflow Trackers. +--- + +# ReID API + +Appearance-based re-identification (ReID) lets BoT-SORT match tracks across +frames using visual appearance in addition to motion. It requires the optional +extra: + +```bash +pip install 'trackers[reid]' +``` + +Trackers ships only the numpy-only association glue documented below. The +appearance encoder, pretrained weights, preprocessing, model catalog, and +gallery evaluation live in the standalone [`reid`](https://github.com/roboflow/re-ID) +package (`roboflow-reid`), which the `trackers[reid]` extra installs for you. + +## Loading a model + +Import the encoder from `reid` and pass it to BoT-SORT: + +```python +from reid import ReIDModel + +from trackers import BoTSORTTracker + +reid_model = ReIDModel.from_pretrained("osnet_x1_0_msmt17_combineall", device="cpu") +tracker = BoTSORTTracker(reid_model=reid_model) +``` + +See the [`reid` package documentation](https://github.com/roboflow/re-ID) for +the full model catalog, `from_pretrained` sources (curated aliases, `hf://` +repos, local checkpoints, architecture-only init), gallery evaluation +(`ReIDEvaluator`, `load_market1501`, `load_msmt17`), and how to add +architectures. + +## Encoder protocol + +`ReIDEncoder` is the minimal interface BoT-SORT depends on: a single +`extract_features` method. `reid.ReIDModel` satisfies it, and you can implement +it yourself for a custom encoder without depending on the model stack. + +::: trackers.core.reid.encoder.ReIDEncoder + +## Feature bank + +::: trackers.core.reid.feature_bank.FeatureBank + +## Appearance similarity + +::: trackers.core.reid.appearance.appearance_similarity + +::: trackers.core.reid.appearance.extract_detection_embeddings diff --git a/docs/learn/install.md b/docs/learn/install.md index 4bbff27d0..6ec87c6ee 100644 --- a/docs/learn/install.md +++ b/docs/learn/install.md @@ -70,6 +70,30 @@ The `detection` extra installs `inference-models`, enabling the CLI to run detec uv pip install "trackers[detection]" ``` +### ReID (BoT-SORT appearance) + +The `reid` extra installs the standalone `roboflow-reid` package, which brings +PyTorch, timm, Hugging Face Hub, safetensors, and related dependencies for ReID +model loading (OSNet, FastReID SBS, and `timm:` backbones) and BoT-SORT +appearance association. + +=== "pip" + + ```bash + pip install "trackers[reid]" + ``` + +=== "uv" + + ```bash + uv pip install "trackers[reid]" + ``` + +Use programmatically via `from reid import ReIDModel` and +`BoTSORTTracker(reid_model=...)`, or via CLI flags such as +`--tracker.reid.enable` and `--tracker.reid.architecture` on `trackers track` +(BoT-SORT only). + !!! tip "GPU Acceleration" For GPU support, ensure PyTorch is installed with CUDA or MPS. diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 6638114bd..0e7aec7d8 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -50,6 +50,42 @@ BoT-SORT keeps the same tracking-by-detection backbone as [ByteTrack](bytetrack. | `high_conf_det_threshold` | Confidence split between stage-1 and stage-2 detections. | 0.5-0.7 common. Higher shifts more detections to recovery stage; lower gives stage-1 broader coverage. | | `enable_cmc` | Enables camera motion compensation before association. | Keep enabled for moving-camera footage (sports, drone, handheld). Disable mainly for static cameras if you need maximal speed. | +## Appearance ReID + +BoT-SORT can optionally fuse appearance embeddings with IoU during association. +Pass a `reid_model` that implements the `ReIDEncoder` protocol (for example, +`reid.ReIDModel`) to enable this mode: + +```python +from reid import ReIDModel +from trackers import BoTSORTTracker + +reid_model = ReIDModel.from_pretrained("osnet_x1_0_msmt17_combineall", device="cpu") +tracker = BoTSORTTracker(reid_model=reid_model) +``` + +When ReID is enabled, pass the current frame to `tracker.update` so the encoder +can crop detections: + +```python +detections = tracker.update(detections, frame=frame) +``` + +Tuning knobs: + +| Parameter | Default | Purpose | +| --------- | ------- | ------- | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | +| `appearance_threshold` | 0.25 | Appearance-distance gate. A match is rejected when the halved cosine distance `0.5 * (1 - cos_sim)` exceeds this value. | +| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | + +Install the optional extra and see the [ReID API](../api/reid.md) for the +encoder protocol, feature bank, and association utilities: + +```bash +pip install 'trackers[reid]' +``` + ## Run on video, webcam, or RTSP stream These examples use `opencv-python` for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually 0 for the default camera. diff --git a/mkdocs.yml b/mkdocs.yml index f1ebd59f1..6c52121bb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -147,6 +147,7 @@ nav: - API Reference: - Trackers: api/trackers.md - Motion: api/motion.md + - ReID: api/reid.md - Datasets: api/datasets.md - Evals: api/evals.md - I/O: api/io.md diff --git a/src/trackers/core/base.py b/src/trackers/core/base.py index c60329c21..f46a06cfc 100644 --- a/src/trackers/core/base.py +++ b/src/trackers/core/base.py @@ -36,19 +36,27 @@ class ParameterInfo: description: str +# Constructor arguments that are injected programmatically and must not be +# surfaced as CLI flags (e.g. an instantiated ReID model or IoU metric object). +_CLI_EXCLUDED_PARAMS = frozenset({"reid_model"}) + + class TrackerParameters(dict[str, ParameterInfo]): - """Tracker parameter mapping with CLI-only filtering for IoU metrics.""" + """Tracker parameter mapping with CLI-only filtering for injection-only args.""" def items(self) -> Iterator[tuple[str, ParameterInfo]]: # type: ignore[override] try: from trackers.utils.iou import BaseIoU except ImportError: - yield from super().items() - return + base_iou_type: type | None = None + else: + base_iou_type = BaseIoU for name, param_info in super().items(): + if name in _CLI_EXCLUDED_PARAMS: + continue param_type = param_info.param_type - if isinstance(param_type, type) and issubclass(param_type, BaseIoU): + if base_iou_type is not None and isinstance(param_type, type) and issubclass(param_type, base_iou_type): continue yield name, param_info diff --git a/src/trackers/core/botsort/fusion.py b/src/trackers/core/botsort/fusion.py new file mode 100644 index 000000000..1a8c53aae --- /dev/null +++ b/src/trackers/core/botsort/fusion.py @@ -0,0 +1,44 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# +# Adapted from NirAharon/BoT-SORT (MIT) +# Copyright (c) 2022 Nir Aharon +# Source: https://github.com/NirAharon/BoT-SORT +# Reference: tracker/bot_sort.py (ReID appearance-IoU cost fusion) +# ------------------------------------------------------------------------ + +"""Appearance-IoU fusion for BoT-SORT ReID association.""" + +from __future__ import annotations + +import numpy as np + + +def fuse_botsort_reid_association( + iou_similarity_fused: np.ndarray, + appearance_similarity: np.ndarray, + *, + proximity_iou_similarity: np.ndarray, + proximity_threshold: float, + appearance_threshold: float, +) -> np.ndarray: + """Fuse IoU and appearance the way BoT-SORT ``bot_sort.py`` does. + + Computes ``min(score_fused_iou_cost, halved_appearance_cost)`` with + proximity and appearance caps, then returns the corresponding similarity + matrix (``1 - cost``). + + Proximity gating always uses *standard IoU* similarity via + ``proximity_iou_similarity``, even when association scoring uses GIoU, + DIoU, or CIoU. + """ + d_iou = 1.0 - iou_similarity_fused + d_iou_proximity = 1.0 - proximity_iou_similarity + d_app = 0.5 * (1.0 - appearance_similarity) + d_app = np.where(d_app > appearance_threshold, 1.0, d_app) + d_app = np.where(d_iou_proximity > proximity_threshold, 1.0, d_app) + fused_cost = np.minimum(d_iou, d_app) + return 1.0 - fused_cost diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index 700ffe651..d42c4c9e8 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -12,8 +12,12 @@ from scipy.optimize import linear_sum_assignment from trackers.core.base import BaseTracker +from trackers.core.botsort.fusion import fuse_botsort_reid_association from trackers.core.botsort.tracklet import BoTSORTTracklet from trackers.core.botsort.utils import _fuse_score, get_alive_tracklets +from trackers.core.reid.appearance import appearance_similarity, extract_detection_embeddings +from trackers.core.reid.encoder import ReIDEncoder +from trackers.core.reid.feature_bank import FeatureBank from trackers.utils.cmc import CMC, CMCConfig, CMCMethod from trackers.utils.detections import default_confidences from trackers.utils.iou import BaseIoU, IoU @@ -84,13 +88,24 @@ class BoTSORTTracker(BaseTracker): Passing ``None`` (the default) is equivalent to ``IoU()`` and is provided for backward compatibility with existing code that did not supply an ``iou`` argument. + reid_model: Optional appearance encoder (``ReIDEncoder``) for appearance + association. Pass a ``reid.ReIDModel`` in normal use. Requires + ``frame`` in :meth:`update`. When ``None`` (default), behaviour + matches the geometry-only BoT-SORT baseline. + reid_ema_alpha: EMA momentum for track appearance features. Default ``0.9``. + appearance_threshold: Appearance distance gate. Rejects matches when the + halved cosine distance ``0.5 * (1 - cos_sim)`` exceeds this value. + Default ``0.25`` (BoT-SORT ``appearance_thresh``). + proximity_threshold: Standard-IoU distance gate applied before appearance + is used. Computed from true IoU even when ``iou`` is GIoU/DIoU/CIoU. + Default ``0.5`` (BoT-SORT ``proximity_thresh``; requires IoU > 0.5). Notes: - Positive `maximum_frames_without_update` values are scaled by ``frame_rate`` and rounded up to at least one missed frame. Explicit zero-buffer configurations remain zero. - - When CMC is enabled, pass the current video frame via the ``frame`` - argument of :meth:`update`. + - When CMC or ReID is enabled, pass the current video frame via the + ``frame`` argument of :meth:`update`. """ tracker_id = "botsort" @@ -124,6 +139,10 @@ def __init__( instant_first_frame_activation: bool = True, state_estimator_class: type[BaseStateEstimator] = XCYCWHStateEstimator, iou: BaseIoU | None = None, + reid_model: ReIDEncoder | None = None, + reid_ema_alpha: float = 0.9, + appearance_threshold: float = 0.25, + proximity_threshold: float = 0.5, ) -> None: self.maximum_frames_without_update = self._compute_maximum_frames_without_update( lost_track_buffer=lost_track_buffer, @@ -140,12 +159,25 @@ def __init__( self.tracks: list[BoTSORTTracklet] = [] self.state_estimator_class = state_estimator_class self.iou = iou if iou is not None else IoU() + # Proximity gating always uses standard IoU, independent of ``iou``. + self._proximity_iou = IoU() self.frame_id: int = 0 self._reset_id_allocator() self.enable_cmc = enable_cmc self.cmc = CMC(CMCConfig(method=cmc_method, downscale=cmc_downscale)) if enable_cmc else None + self.reid_model = reid_model + if not 0.0 <= reid_ema_alpha <= 1.0: + raise ValueError(f"reid_ema_alpha must be in [0, 1], got {reid_ema_alpha}") + self.reid_ema_alpha = reid_ema_alpha + if not 0.0 <= appearance_threshold <= 1.0: + raise ValueError(f"appearance_threshold must be in [0, 1], got {appearance_threshold}") + if not 0.0 <= proximity_threshold <= 1.0: + raise ValueError(f"proximity_threshold must be in [0, 1], got {proximity_threshold}") + self.appearance_threshold = appearance_threshold + self.proximity_threshold = proximity_threshold + self._init_timestamp_state(frame_rate) def update( @@ -182,10 +214,11 @@ def update( the last state. Notes: - - If CMC is enabled, pass the current video frame via ``frame`` so the - tracker can estimate a global affine transform and warp predicted - track states before association. When ``frame=None`` and - ``enable_cmc=True``, CMC is silently skipped for that step. + - If CMC or ReID is enabled, pass the current video frame via ``frame`` + so the tracker can estimate a global affine transform and/or extract + appearance embeddings before association. When ``frame=None`` and + ``enable_cmc=True``, CMC is silently skipped for that step; ReID + requires ``frame`` and raises when it is ``None``. """ timing = self._predict_timing(timestamp) if timing.skip_update: @@ -247,38 +280,66 @@ def update( mask_boxes = high_boxes if len(high_boxes) > 0 else None H = self.cmc.estimate(frame, mask_boxes) CMC.apply_batch(H, self.tracks) + + # Appearance: extract embeddings once for all high-confidence detections. + # When reid_model is None this block is skipped entirely and behaviour + # is identical to the geometry-only baseline. + det_embeddings: np.ndarray | None = None + if self.reid_model is not None: + if frame is None: + raise ValueError(f"{type(self).__name__}.update() requires frame when reid_model is set.") + if len(high_boxes) > 0: + det_embeddings = extract_detection_embeddings(self.reid_model, frame, high_boxes) + # Step 1: associate high-confidence detections to confirmed + lost tracks. # Lost tracks are included here (following the original ByteTrack), and # IoU is fused with detection scores. strack_pool = confirmed_tracks + lost_tracks iou_matrix = self._get_iou_matrix(strack_pool, high_boxes) - iou_matrix = _fuse_score(self.iou.normalize_for_fusion(iou_matrix), high_scores) + iou_sim_raw = self.iou.normalize_for_fusion(iou_matrix) + iou_sim_fused = _fuse_score(iou_sim_raw, high_scores) + + if det_embeddings is not None and len(strack_pool) > 0: + track_feats = [ + t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None + for t in strack_pool + ] + app_sim = appearance_similarity(track_feats, det_embeddings) + proximity_iou = self._get_proximity_iou_matrix(strack_pool, high_boxes) + similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) + else: + similarity_matrix = iou_sim_fused + matched, unmatched_pool, unmatched_high = self._get_associated_indices( - iou_matrix, self.minimum_iou_threshold_first_assoc + similarity_matrix, self.minimum_iou_threshold_first_assoc ) for row, col in matched: - track = strack_pool[row] - track.update(high_boxes[col]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(high_indices[col])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + strack_pool[row], + high_boxes[col], + det_embeddings[col] if det_embeddings is not None else None, + int(high_indices[col]), + out_det_indices, + out_tracker_ids, + ) # Step 2: associate low-confidence detections to remaining *tracked* tracks # only (excluding lost tracks, following the original ByteTrack). - # No score fusing in second association. + # No score fusing or ReID in second association (upstream bot_sort.py). remaining_tracked = [strack_pool[i] for i in unmatched_pool if strack_pool[i].time_since_update == 1] iou_matrix = self._get_iou_matrix(remaining_tracked, low_boxes) matched, _, unmatched_low = self._get_associated_indices(iou_matrix, self.minimum_iou_threshold_second_assoc) for row, col in matched: - track = remaining_tracked[row] - track.update(low_boxes[col]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(low_indices[col])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + remaining_tracked[row], + low_boxes[col], + None, + int(low_indices[col]), + out_det_indices, + out_tracker_ids, + ) # Unmatched low-confidence detections for det_local_idx in sorted(unmatched_low): @@ -296,19 +357,35 @@ def update( uh_scores = high_scores[unmatched_high_list] iou_matrix = self._get_iou_matrix(unconfirmed_tracks, uh_boxes) - iou_matrix = _fuse_score(self.iou.normalize_for_fusion(iou_matrix), uh_scores) + iou_sim_raw = self.iou.normalize_for_fusion(iou_matrix) + iou_sim_fused = _fuse_score(iou_sim_raw, uh_scores) + + if det_embeddings is not None: + uh_embeddings = det_embeddings[unmatched_high_list] + track_feats = [ + t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None + for t in unconfirmed_tracks + ] + app_sim = appearance_similarity(track_feats, uh_embeddings) + proximity_iou = self._get_proximity_iou_matrix(unconfirmed_tracks, uh_boxes) + similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) + else: + similarity_matrix = iou_sim_fused + matched_uc, unmatched_uc_indices, remaining_uh = self._get_associated_indices( - iou_matrix, self.minimum_iou_threshold_unconfirmed_assoc + similarity_matrix, self.minimum_iou_threshold_unconfirmed_assoc ) for row, col in matched_uc: - track = unconfirmed_tracks[row] orig_high_idx = unmatched_high_list[col] - track.update(high_boxes[orig_high_idx]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(high_indices[orig_high_idx])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + unconfirmed_tracks[row], + high_boxes[orig_high_idx], + det_embeddings[orig_high_idx] if det_embeddings is not None else None, + int(high_indices[orig_high_idx]), + out_det_indices, + out_tracker_ids, + ) # Only remaining unmatched high-conf dets proceed to spawning unmatched_high = [unmatched_high_list[i] for i in remaining_uh] @@ -328,6 +405,7 @@ def update( out_det_indices, out_tracker_ids, is_first_frame=(self.frame_id == 1), + det_embeddings=det_embeddings, ) # Full lifecycle prune: removes immature+unmatched and any remaining expired @@ -349,6 +427,46 @@ def update( result.tracker_id = np.array(out_tracker_ids, dtype=int) return result + def _assign_track_detection( + self, + track: BoTSORTTracklet, + bbox: np.ndarray, + embedding: np.ndarray | None, + global_det_index: int, + out_det_indices: list[int], + out_tracker_ids: list[int], + ) -> None: + """Update a track from a matched detection and record output indices.""" + track.update(bbox) + if track.feature_bank is not None and embedding is not None: + track.feature_bank.update(embedding) + if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: + track.tracker_id = self._allocate_tracker_id() + out_det_indices.append(global_det_index) + out_tracker_ids.append(track.tracker_id) + + def _fuse_botsort_reid( + self, + iou_similarity_fused: np.ndarray, + appearance_similarity: np.ndarray, + proximity_iou_similarity: np.ndarray, + ) -> np.ndarray: + """Fuse IoU and appearance using BoT-SORT ``bot_sort.py`` min-cost ReID.""" + return fuse_botsort_reid_association( + iou_similarity_fused, + appearance_similarity, + proximity_iou_similarity=proximity_iou_similarity, + proximity_threshold=self.proximity_threshold, + appearance_threshold=self.appearance_threshold, + ) + + def _get_proximity_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: + if len(tracklets) == 0: + tracklet_boxes = np.empty((0, 4)) + else: + tracklet_boxes = np.array([tracklet.get_state_bbox() for tracklet in tracklets]) + return self._proximity_iou.compute(tracklet_boxes, detections) + def _get_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: if len(tracklets) == 0: tracklet_boxes = np.empty((0, 4)) @@ -405,6 +523,7 @@ def _spawn_new_tracks( out_det_indices: list[int], out_tracker_ids: list[int], is_first_frame: bool = False, + det_embeddings: np.ndarray | None = None, ) -> None: """Create new tracklets from unmatched high-confidence detections. @@ -422,6 +541,10 @@ def _spawn_new_tracks( initial_bbox=detection_boxes[global_idx], state_estimator_class=self.state_estimator_class, ) + if self.reid_model is not None: + tracklet.feature_bank = FeatureBank(self.reid_ema_alpha) + if det_embeddings is not None: + tracklet.feature_bank.update(det_embeddings[det_local_idx]) if is_first_frame and self.instant_first_frame_activation: tracklet.tracker_id = self._allocate_tracker_id() self.tracks.append(tracklet) diff --git a/src/trackers/core/botsort/tracklet.py b/src/trackers/core/botsort/tracklet.py index 1f384f15d..0d5dd7cba 100644 --- a/src/trackers/core/botsort/tracklet.py +++ b/src/trackers/core/botsort/tracklet.py @@ -6,6 +6,8 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np from trackers.utils.base_tracklet import BaseTracklet @@ -19,6 +21,9 @@ XYXYStateEstimator, ) +if TYPE_CHECKING: + from trackers.core.reid.feature_bank import FeatureBank + class BoTSORTTracklet(BaseTracklet): """Tracklet for the BoT-SORT tracker. @@ -53,6 +58,8 @@ def __init__( # Count initial bbox as first successful update so that # number_of_successful_updates starts at 1. self.number_of_successful_updates = 1 + # Optional appearance feature bank, populated by BoTSORTTracker. + self.feature_bank: FeatureBank | None = None def _configure_initial_noise(self, bbox: np.ndarray) -> None: """Set initial P, Q, R based on the first detection's size.""" diff --git a/src/trackers/scripts/track.py b/src/trackers/scripts/track.py index 539a3a237..7d97813c6 100644 --- a/src/trackers/scripts/track.py +++ b/src/trackers/scripts/track.py @@ -16,6 +16,7 @@ import numpy as np import supervision as sv +from trackers import _reid as reid_provider from trackers import frames_from_source from trackers.core.base import BaseTracker from trackers.io.mot import _mot_frame_to_detections, _MOTOutput, load_mot_file @@ -150,6 +151,46 @@ def add_track_subparser(subparsers: argparse._SubParsersAction) -> None: # Add dynamic tracker parameters _add_tracker_params(tracker_group) + reid_group = parser.add_argument_group("reid (BoT-SORT only; requires trackers[reid])") + reid_group.add_argument( + "--tracker.reid.enable", + action="store_true", + dest="tracker_reid_enable", + help="Enable appearance-based ReID association for BoT-SORT.", + ) + reid_group.add_argument( + "--tracker.reid.model", + type=str, + default=None, + dest="tracker_reid_model", + metavar="SOURCE", + help=( + "ReID checkpoint source (curated alias, hf:// URL, local path, or " + "save_pretrained directory). Implies --tracker.reid.enable. " + "Default alias when omitted: osnet_x1_0_msmt17_combineall." + ), + ) + reid_group.add_argument( + "--tracker.reid.device", + type=str, + default=DEFAULT_DEVICE, + dest="tracker_reid_device", + metavar="DEVICE", + help=f"ReID compute device: auto, cpu, cuda, mps. Default: {DEFAULT_DEVICE}", + ) + reid_group.add_argument( + "--tracker.reid.architecture", + type=str, + default=None, + dest="tracker_reid_architecture", + metavar="NAME", + help=( + "Backbone architecture for bare local .pth/.safetensors weights " + "(e.g. osnet_x1_0, fastreid_sbs_resnest50). Required when " + "--tracker.reid.model points to a bare weights file." + ), + ) + # Output options output_group = parser.add_argument_group("output") output_group.add_argument( @@ -308,8 +349,17 @@ def run_track(args: argparse.Namespace) -> int: track_id_filter = _resolve_track_id_filter(args.track_ids) + reid_error = _validate_reid_cli_prerequisites(args) + if reid_error is not None: + print(reid_error, file=sys.stderr) + return 1 + # Create tracker tracker_params = _extract_tracker_params(args.tracker, args) + tracker_params, reid_error = _apply_reid_tracker_params(args.tracker, args, tracker_params) + if reid_error is not None: + print(reid_error, file=sys.stderr) + return 1 tracker = _init_tracker(args.tracker, **tracker_params) if args.source is not None: @@ -363,7 +413,7 @@ def _run_frameless( mask = np.isin(detections.class_id, class_filter) detections = detections[mask] # type: ignore[assignment] - tracked = tracker.update(detections) + tracked = tracker.update(detections, frame=None) if track_id_filter is not None and len(tracked) > 0: if tracked.tracker_id is not None: @@ -600,6 +650,76 @@ def _run_model(model: AnyModel, frame: np.ndarray, confidence: float) -> sv.Dete return detections +def _reid_requested(args: argparse.Namespace) -> bool: + return bool(getattr(args, "tracker_reid_enable", False)) or getattr(args, "tracker_reid_model", None) is not None + + +def _validate_reid_cli_prerequisites(args: argparse.Namespace) -> str | None: + """Validate ReID CLI options before loading any checkpoint.""" + if not _reid_requested(args): + return None + if args.tracker != "botsort": + return f"Error: --tracker.reid.* options apply only to --tracker botsort, got {args.tracker!r}." + if args.source is None: + return ( + "Error: ReID-enabled BoT-SORT requires --source (video/webcam/images) " + "so appearance embeddings can be extracted from frames." + ) + return None + + +def _apply_reid_tracker_params( + tracker_id: str, + args: argparse.Namespace, + params: dict[str, object], +) -> tuple[dict[str, object], str | None]: + """Attach a CLI-instantiated ReID model to BoT-SORT tracker params.""" + if not _reid_requested(args): + return params, None + + if tracker_id != "botsort": + return params, (f"Error: --tracker.reid.* options apply only to --tracker botsort, got {tracker_id!r}.") + + if getattr(args, "source", None) is None: + return params, ( + "Error: ReID-enabled BoT-SORT requires --source (video/webcam/images) " + "so appearance embeddings can be extracted from frames." + ) + + try: + ReIDModel = reid_provider.import_reid_model() + except ImportError: + return params, ( + "Error: ReID tracking requires the optional `trackers[reid]` extra.\n" + "Install with: pip install 'trackers[reid]'" + ) + + device = getattr(args, "tracker_reid_device", DEFAULT_DEVICE) + model_source = getattr(args, "tracker_reid_model", None) + architecture = getattr(args, "tracker_reid_architecture", None) + load_kwargs: dict[str, object] = {"device": device} + if model_source is not None: + load_kwargs["source"] = model_source + if architecture is not None: + load_kwargs["architecture"] = architecture + + try: + reid_model = ReIDModel.from_pretrained(**load_kwargs) + except KeyboardInterrupt: + raise + except ImportError: + return params, ( + "Error: ReID tracking requires the optional `trackers[reid]` extra.\n" + "Install with: pip install 'trackers[reid]'" + ) + except (OSError, ValueError, RuntimeError) as exc: + return params, f"Error: Failed to load ReID model: {exc}" + + params = dict(params) + params["reid_model"] = reid_model + return params, None + + def _extract_tracker_params(tracker_id: str, args: argparse.Namespace) -> dict[str, object]: """Extract tracker parameters from CLI args. diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py new file mode 100644 index 000000000..b179dc361 --- /dev/null +++ b/tests/core/test_botsort_reid.py @@ -0,0 +1,303 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""BoT-SORT ReID association and fusion tests.""" + +from __future__ import annotations + +import subprocess +import sys + +import numpy as np +import pytest +import supervision as sv + +from trackers.core.botsort.fusion import fuse_botsort_reid_association +from trackers.core.botsort.tracker import BoTSORTTracker +from trackers.core.reid.appearance import ( + appearance_similarity, + extract_detection_embeddings, +) +from trackers.core.reid.feature_bank import FeatureBank + + +def _detection(xyxy: tuple[float, float, float, float], conf: float = 0.9) -> sv.Detections: + return sv.Detections( + xyxy=np.array([xyxy], dtype=np.float32), + confidence=np.array([conf], dtype=np.float32), + ) + + +def _frame(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + + +def _norm(vec: np.ndarray) -> np.ndarray: + vec = vec.astype(np.float32) + return vec / np.linalg.norm(vec) + + +class _KeyedReIDEncoder: + """Deterministic embeddings keyed by detection top-left corner.""" + + def __init__(self, table: dict[tuple[int, int], np.ndarray] | None = None) -> None: + self.table = table or {} + self.calls = 0 + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + self.calls += 1 + if len(detections) == 0: + return np.empty((0, 0), dtype=np.float32) + rows = [] + for box in detections.xyxy: + key = (round(float(box[0])), round(float(box[1]))) + rows.append(self.table.get(key, _norm(np.array([float(box[0]), float(box[1]), 1.0, 0.0])))) + return np.stack(rows) + + +def test_botsort_import_does_not_load_reid_model_stack() -> None: + """Importing BoT-SORT must not pull the heavy ``reid`` package (torch etc.).""" + result = subprocess.run( + [ + sys.executable, + "-c", + ("import sys; import trackers.core.botsort.tracker; assert 'reid' not in sys.modules"), + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +class TestFeatureBank: + def test_first_update_stores_raw_embedding(self) -> None: + bank = FeatureBank(alpha=0.9) + bank.update(np.array([3.0, 4.0], dtype=np.float32)) + feature = bank.feature + assert feature is not None + np.testing.assert_allclose(feature, [3.0, 4.0]) + + def test_stores_and_blends_raw_embeddings(self) -> None: + bank = FeatureBank(alpha=0.75) + bank.update(np.array([1.0, 0.0], dtype=np.float32)) + bank.update(np.array([0.0, 1.0], dtype=np.float32)) + feature = bank.feature + assert feature is not None + np.testing.assert_allclose(feature, [0.75, 0.25]) + + def test_zero_embedding_is_accepted(self) -> None: + bank = FeatureBank() + bank.update(np.zeros(8, dtype=np.float32)) + feature = bank.feature + assert feature is not None + np.testing.assert_allclose(feature, 0.0) + + def test_non_finite_embedding_raises(self) -> None: + bank = FeatureBank() + with pytest.raises(ValueError, match="finite"): + bank.update(np.array([1.0, np.nan], dtype=np.float32)) + assert not bank.is_initialized + + def test_shape_change_raises(self) -> None: + bank = FeatureBank() + bank.update(np.array([1.0, 0.0], dtype=np.float32)) + before = bank.feature + assert before is not None + with pytest.raises(ValueError, match="shape"): + bank.update(np.array([1.0, 0.0, 0.0], dtype=np.float32)) + after = bank.feature + assert after is not None + np.testing.assert_allclose(after, before) + + +class TestAppearanceSimilarity: + def test_identical_vectors_are_one(self) -> None: + similarity = appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[1.0]], atol=1e-6) + + def test_orthogonal_vectors_are_zero(self) -> None: + similarity = appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[0.0, 1.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[0.0]], atol=1e-6) + + def test_normalizes_both_sides(self) -> None: + similarity = appearance_similarity( + [np.array([3.0, 4.0], dtype=np.float32)], + np.array([[6.0, 8.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[1.0]], atol=1e-6) + + def test_none_track_yields_zero_row(self) -> None: + similarity = appearance_similarity( + [None, np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[0.0], [1.0]], atol=1e-6) + + def test_empty_inputs_return_empty_matrix(self) -> None: + assert appearance_similarity([], np.empty((0, 4), dtype=np.float32)).shape == (0, 0) + assert appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.empty((0, 2), dtype=np.float32), + ).shape == (1, 0) + + def test_non_finite_detection_rows_raise(self) -> None: + with pytest.raises(ValueError, match="finite"): + appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0], [np.nan, 1.0]], dtype=np.float32), + ) + + def test_incompatible_track_dimensions_raise(self) -> None: + with pytest.raises(ValueError, match="dim"): + appearance_similarity( + [np.array([1.0, 0.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + + def test_extraction_rejects_wrong_row_count(self) -> None: + class _WrongLengthEncoder: + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + return np.empty((0, 4), dtype=np.float32) + + with pytest.raises(ValueError, match="rows"): + extract_detection_embeddings( + _WrongLengthEncoder(), + _frame(), + np.array([[0.0, 0.0, 10.0, 10.0]], dtype=np.float32), + ) + + +class TestFuseBotsortReidAssociation: + def test_appearance_can_win_when_proximity_passes(self) -> None: + fused = fuse_botsort_reid_association( + np.array([[0.63]], dtype=np.float32), + np.array([[0.8]], dtype=np.float32), + proximity_iou_similarity=np.array([[0.7]], dtype=np.float32), + proximity_threshold=0.5, + appearance_threshold=0.25, + ) + assert fused[0, 0] == pytest.approx(0.9) + + def test_low_proximity_zeros_appearance(self) -> None: + fused = fuse_botsort_reid_association( + np.array([[0.36]], dtype=np.float32), + np.array([[0.9]], dtype=np.float32), + proximity_iou_similarity=np.array([[0.4]], dtype=np.float32), + proximity_threshold=0.5, + appearance_threshold=0.25, + ) + assert fused[0, 0] == pytest.approx(0.36) + + def test_proximity_uses_standard_iou_not_giou(self) -> None: + fused = fuse_botsort_reid_association( + np.array([[0.80]], dtype=np.float32), + np.array([[0.95]], dtype=np.float32), + proximity_iou_similarity=np.array([[0.35]], dtype=np.float32), + proximity_threshold=0.5, + appearance_threshold=0.25, + ) + assert fused[0, 0] == pytest.approx(0.80) + + +class TestBoTSORTTrackerReID: + def test_rejects_invalid_reid_ema_alpha(self) -> None: + with pytest.raises(ValueError, match="reid_ema_alpha"): + BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder(), reid_ema_alpha=1.5) + + def test_requires_frame_when_reid_enabled(self) -> None: + tracker = BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder()) + with pytest.raises(ValueError, match="requires frame"): + tracker.update(_detection((10.0, 10.0, 30.0, 30.0))) + + def test_feature_bank_initializes_on_spawn(self) -> None: + tracker = BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder()) + tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame()) + bank = tracker.tracks[0].feature_bank + assert bank is not None and bank.is_initialized + + def test_appearance_changes_assignment_vs_geometry_only(self) -> None: + identity = _norm(np.array([1.0, 0.0, 0.0, 0.0])) + impostor = _norm(np.array([0.0, 1.0, 0.0, 0.0])) + + class _PhaseEncoder: + phase = 1 + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + rows = [] + for box in detections.xyxy: + key = (round(float(box[0])), round(float(box[1]))) + if self.phase == 1: + rows.append(identity) + elif key == (10, 10): + rows.append(impostor) + else: + rows.append(identity) + return np.stack(rows) + + encoder = _PhaseEncoder() + frame = _frame(1) + geo = BoTSORTTracker( + enable_cmc=False, + minimum_iou_threshold_first_assoc=0.01, + appearance_threshold=0.6, + proximity_threshold=0.99, + ) + geo.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=frame) + + reid = BoTSORTTracker( + enable_cmc=False, + minimum_iou_threshold_first_assoc=0.01, + appearance_threshold=0.6, + proximity_threshold=0.99, + reid_model=encoder, + ) + reid.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=frame) + track_id = int(reid.tracks[0].tracker_id) + + competitors = sv.Detections( + xyxy=np.array([[10.0, 10.0, 30.0, 30.0], [14.0, 14.0, 34.0, 34.0]], dtype=np.float32), + confidence=np.array([0.9, 0.9], dtype=np.float32), + ) + encoder.phase = 2 + geo_out = geo.update(competitors, frame=frame) + reid_out = reid.update(competitors, frame=frame) + + def _matched_xy(out: sv.Detections, tid: int) -> tuple[float, float]: + assert out.tracker_id is not None + box = out.xyxy[out.tracker_id == tid][0] + return float(box[0]), float(box[1]) + + assert _matched_xy(geo_out, int(geo.tracks[0].tracker_id)) == (10.0, 10.0) + assert _matched_xy(reid_out, track_id) == (14.0, 14.0) + + def test_low_confidence_stage_does_not_update_feature_bank(self) -> None: + model = _KeyedReIDEncoder({(10, 10): _norm(np.array([1.0, 0.0, 0.0, 0.0]))}) + tracker = BoTSORTTracker( + enable_cmc=False, + reid_model=model, + high_conf_det_threshold=0.8, + minimum_iou_threshold_second_assoc=0.01, + ) + tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame(4)) + bank = tracker.tracks[0].feature_bank + assert bank is not None + before = bank.feature + assert before is not None + + calls_after_high = model.calls + tracker.update(_detection((12.0, 12.0, 32.0, 32.0), conf=0.5), frame=_frame(4)) + assert model.calls == calls_after_high + after = bank.feature + assert after is not None + np.testing.assert_allclose(before, after) diff --git a/tests/core/test_botsort_reid_integration.py b/tests/core/test_botsort_reid_integration.py new file mode 100644 index 000000000..1f49ac1ae --- /dev/null +++ b/tests/core/test_botsort_reid_integration.py @@ -0,0 +1,45 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""End-to-end smoke test for BoT-SORT with a real ``reid`` encoder. + +Exercises the ``trackers`` -> ``reid`` boundary once, without re-testing the +``reid`` internals (those live in the ``reid`` package's own suite). Requires +the ``trackers[reid]`` extra; skipped when ``roboflow-reid`` is not installed. +""" + +from __future__ import annotations + +import numpy as np +import pytest +import supervision as sv + +reid = pytest.importorskip("reid", reason="requires the optional trackers[reid] extra") + +from trackers.core.botsort.tracker import BoTSORTTracker # noqa: E402 + + +@pytest.mark.integration +def test_botsort_with_real_reid_model_runs_over_frames() -> None: + reid_model = reid.ReIDModel.from_pretrained(architecture="osnet_x0_25", device="cpu") + + tracker = BoTSORTTracker(enable_cmc=False, reid_model=reid_model) + + rng = np.random.default_rng(0) + box = np.array([30.0, 30.0, 70.0, 90.0], dtype=np.float32) + for _ in range(3): + frame = rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + detections = sv.Detections( + xyxy=box[None, :].copy(), + confidence=np.array([0.9], dtype=np.float32), + ) + result = tracker.update(detections, frame=frame) + assert result.tracker_id is not None + box = box + np.array([2.0, 1.0, 2.0, 1.0], dtype=np.float32) + + assert len(tracker.tracks) == 1 + bank = tracker.tracks[0].feature_bank + assert bank is not None and bank.is_initialized diff --git a/tests/scripts/test_track.py b/tests/scripts/test_track.py index be3867edb..b4c536910 100644 --- a/tests/scripts/test_track.py +++ b/tests/scripts/test_track.py @@ -6,6 +6,8 @@ from __future__ import annotations +import argparse +from pathlib import Path from typing import ClassVar import numpy as np @@ -13,10 +15,13 @@ import supervision as sv from trackers.scripts.track import ( + _apply_reid_tracker_params, _format_labels, _init_annotators, + _reid_requested, _resolve_class_filter, _resolve_track_id_filter, + add_track_subparser, ) @@ -194,3 +199,77 @@ def test_all_non_integer_returns_none(self, capsys: pytest.CaptureFixture) -> No result = _resolve_track_id_filter("abc,def") assert result is None assert "abc" in capsys.readouterr().err + + +class _FakeReIDModel: + last_kwargs: ClassVar[dict | None] = None + + @classmethod + def from_pretrained(cls, **kwargs: object) -> _FakeReIDModel: + cls.last_kwargs = dict(kwargs) + return cls() + + +class TestReidTrackCli: + def test_model_source_implies_enable(self) -> None: + args = argparse.Namespace( + tracker_reid_enable=False, + tracker_reid_model="osnet_x1_0_msmt17_combineall", + ) + assert _reid_requested(args) + + def test_requires_source_before_load(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + "trackers.scripts.track.reid_provider.import_reid_model", + lambda: pytest.fail("model provider should not be called"), + ) + args = argparse.Namespace( + tracker_reid_enable=True, + tracker_reid_model=None, + tracker_reid_device="cpu", + tracker_reid_architecture=None, + source=None, + ) + params, err = _apply_reid_tracker_params("botsort", args, {}) + assert err is not None and "--source" in err + assert params == {} + + def test_passes_architecture(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + monkeypatch.setattr( + "trackers.scripts.track.reid_provider.import_reid_model", + lambda: _FakeReIDModel, + ) + weights = tmp_path / "weights.pth" + weights.touch() + args = argparse.Namespace( + tracker_reid_enable=False, + tracker_reid_model=str(weights), + tracker_reid_device="cpu", + tracker_reid_architecture="osnet_x1_0", + source="video.mp4", + ) + params, err = _apply_reid_tracker_params("botsort", args, {}) + assert err is None + assert _FakeReIDModel.last_kwargs is not None + assert _FakeReIDModel.last_kwargs["architecture"] == "osnet_x1_0" + assert "reid_model" in params + + def test_rejects_non_botsort_tracker(self) -> None: + args = argparse.Namespace( + tracker_reid_enable=True, + tracker_reid_model=None, + tracker_reid_device="cpu", + tracker_reid_architecture=None, + source="video.mp4", + ) + _, error = _apply_reid_tracker_params("bytetrack", args, {}) + assert error is not None and "botsort" in error + + def test_help_lists_reid_flags(self) -> None: + parser = argparse.ArgumentParser() + subparsers = parser.add_subparsers() + add_track_subparser(subparsers) + help_text = subparsers.choices["track"].format_help() + assert "--tracker.reid.enable" in help_text + assert "--tracker.reid.architecture" in help_text + assert "--tracker.appearance_threshold" in help_text From fbbe664ec4442983211094e6316db78b16a12234 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 11:14:34 -0300 Subject: [PATCH 20/54] refactor(reid): L2-normalize FeatureBank EMA and vectorize row normalize - FeatureBank.update now L2-normalizes the incoming embedding and the resulting EMA, keeping the stored feature on the unit hypersphere (matches upstream BoT-SORT STrack.update_features). reid returns raw embeddings; normalization happens in the bank. Docstrings/tests updated. - Vectorize appearance._l2_normalize_rows. Co-authored-by: Cursor --- src/trackers/core/reid/appearance.py | 4 +++- src/trackers/core/reid/feature_bank.py | 27 ++++++++++++++++++++------ tests/core/test_botsort_reid.py | 14 +++++++++---- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/src/trackers/core/reid/appearance.py b/src/trackers/core/reid/appearance.py index a7cb8e338..8af520c29 100644 --- a/src/trackers/core/reid/appearance.py +++ b/src/trackers/core/reid/appearance.py @@ -43,7 +43,9 @@ def _l2_normalize_rows(embeddings: np.ndarray) -> np.ndarray: """L2-normalise each row in an embedding matrix.""" if embeddings.size == 0: return embeddings - return np.stack([_l2_normalize(row) for row in embeddings]) + mat = embeddings.astype(np.float64) + norms = np.linalg.norm(mat, axis=1, keepdims=True) + return (mat / np.maximum(norms, _NORM_EPS)).astype(np.float32) def extract_detection_embeddings( diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index d29558fc7..6b19e38b5 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -10,9 +10,17 @@ import numpy as np +_NORM_EPS = 1e-12 + class FeatureBank: - """Per-track EMA appearance embedding. + """Per-track EMA appearance embedding, kept on the unit hypersphere. + + Following upstream BoT-SORT (``STrack.update_features``), every incoming + embedding is L2-normalized before it is blended, and the resulting EMA is + L2-normalized again. The stored feature is therefore always a unit vector, + so cosine similarity against it is a plain dot product. ``reid.ReIDModel`` + returns raw (unnormalized) embeddings; normalization happens here. Args: alpha: EMA momentum in ``[0, 1]``. @@ -26,7 +34,7 @@ def __init__(self, alpha: float = 0.9) -> None: @property def feature(self) -> np.ndarray | None: - """Current stored embedding, or ``None`` if never updated.""" + """Current stored unit embedding, or ``None`` if never updated.""" return None if self._feature is None else self._feature.copy() @property @@ -35,11 +43,11 @@ def is_initialized(self) -> bool: return self._feature is not None def update(self, embedding: np.ndarray) -> None: - """Blend an embedding into the stored feature.""" - cleaned = _require_embedding(embedding) + """Blend an L2-normalized embedding into the stored unit feature.""" + cleaned = _l2_normalize(_require_embedding(embedding)) if self._feature is None: - self._feature = cleaned.copy() + self._feature = cleaned return if self._feature.shape != cleaned.shape: @@ -47,7 +55,8 @@ def update(self, embedding: np.ndarray) -> None: f"embedding shape {cleaned.shape} does not match stored feature shape {self._feature.shape}" ) - self._feature = (self._alpha * self._feature + (1.0 - self._alpha) * cleaned).astype(np.float32) + blended = self._alpha * self._feature + (1.0 - self._alpha) * cleaned + self._feature = _l2_normalize(blended) def reset(self) -> None: """Clear the stored feature.""" @@ -62,3 +71,9 @@ def _require_embedding(embedding: np.ndarray) -> np.ndarray: if not np.all(np.isfinite(flat)): raise ValueError("embedding must contain only finite values") return flat + + +def _l2_normalize(vec: np.ndarray) -> np.ndarray: + """Return a unit-norm float32 vector (zero vectors are returned unchanged).""" + norm = float(np.linalg.norm(vec)) + return (vec / max(norm, _NORM_EPS)).astype(np.float32) diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index b179dc361..e498c9937 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -75,20 +75,26 @@ def test_botsort_import_does_not_load_reid_model_stack() -> None: class TestFeatureBank: - def test_first_update_stores_raw_embedding(self) -> None: + def test_first_update_normalizes_embedding(self) -> None: bank = FeatureBank(alpha=0.9) bank.update(np.array([3.0, 4.0], dtype=np.float32)) feature = bank.feature assert feature is not None - np.testing.assert_allclose(feature, [3.0, 4.0]) + # Incoming embedding is L2-normalized before storage (unit sphere). + np.testing.assert_allclose(feature, [0.6, 0.8], atol=1e-6) - def test_stores_and_blends_raw_embeddings(self) -> None: + def test_blends_on_unit_sphere(self) -> None: bank = FeatureBank(alpha=0.75) bank.update(np.array([1.0, 0.0], dtype=np.float32)) bank.update(np.array([0.0, 1.0], dtype=np.float32)) feature = bank.feature assert feature is not None - np.testing.assert_allclose(feature, [0.75, 0.25]) + # EMA of two unit vectors, then re-normalized: 0.75*[1,0] + 0.25*[0,1] + # = [0.75, 0.25], normalized by its norm sqrt(0.625). + expected = np.array([0.75, 0.25], dtype=np.float32) + expected /= np.linalg.norm(expected) + np.testing.assert_allclose(feature, expected, atol=1e-6) + np.testing.assert_allclose(np.linalg.norm(feature), 1.0, atol=1e-6) def test_zero_embedding_is_accepted(self) -> None: bank = FeatureBank() From 8297820ca63695705cecae57a25778922796153d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 14:15:30 +0000 Subject: [PATCH 21/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/trackers/botsort.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 0e7aec7d8..4d07f645f 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -73,11 +73,11 @@ detections = tracker.update(detections, frame=frame) Tuning knobs: -| Parameter | Default | Purpose | -| --------- | ------- | ------- | -| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | -| `appearance_threshold` | 0.25 | Appearance-distance gate. A match is rejected when the halved cosine distance `0.5 * (1 - cos_sim)` exceeds this value. | -| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | +| Parameter | Default | Purpose | +| ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | +| `appearance_threshold` | 0.25 | Appearance-distance gate. A match is rejected when the halved cosine distance `0.5 * (1 - cos_sim)` exceeds this value. | +| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | Install the optional extra and see the [ReID API](../api/reid.md) for the encoder protocol, feature bank, and association utilities: From fa0d2482ecb9e67ecacff92b0b7b2b55e9ea2051 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 12:04:20 -0300 Subject: [PATCH 22/54] =?UTF-8?q?Add=20MOT17=20BoT-SORT=20=C2=B1=20ReID=20?= =?UTF-8?q?benchmark=20notebook=20for=20reid=20package=20integration.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports eval_trackers_reid from the reid-training branch with standalone reid imports and Colab-friendly private git installs for both trackers and re-ID. Co-authored-by: Cursor --- notebooks/eval_trackers_reid.ipynb | 998 +++++++++++++++++++++++++++++ 1 file changed, 998 insertions(+) create mode 100644 notebooks/eval_trackers_reid.ipynb diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb new file mode 100644 index 000000000..ee7e1e605 --- /dev/null +++ b/notebooks/eval_trackers_reid.ipynb @@ -0,0 +1,998 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ], + "id": "2d522414", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ], + "id": "7bec6c65", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " )\n", + " TRACKERS_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + " )\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" + ], + "execution_count": null, + "outputs": [], + "id": "6bc2b8d8" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ], + "execution_count": null, + "outputs": [], + "id": "6c2e60ad" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + ], + "id": "a54bb5ed", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ], + "execution_count": null, + "outputs": [], + "id": "bbc892d6" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ], + "id": "29afb2e0", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ], + "execution_count": null, + "outputs": [], + "id": "5ea423a9" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ], + "id": "b57560b2", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ], + "execution_count": null, + "outputs": [], + "id": "f38487ea" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ], + "id": "823b2696", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ], + "execution_count": null, + "outputs": [], + "id": "d09332f1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ], + "id": "ad28e88f", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ], + "execution_count": null, + "outputs": [], + "id": "6612c281" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ], + "id": "b0ea623e", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ], + "id": "43321292", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ], + "execution_count": null, + "outputs": [], + "id": "d16f6483" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ], + "id": "8ee1ac84", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ], + "execution_count": null, + "outputs": [], + "id": "d448e555" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ], + "id": "8de54c38", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "images = sorted(img_dir.glob(\"*.jpg\"))\n", + "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", + "\n", + "sample = cv2.imread(str(images[0]))\n", + "if sample is None:\n", + " raise RuntimeError(f\"Could not read {images[0]}\")\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(str(out_path), video_info) as sink:\n", + " for i in range(n_frames):\n", + " frame_idx = i + 1\n", + " frame = cv2.imread(str(images[i]))\n", + " if frame is None:\n", + " continue\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ], + "execution_count": null, + "outputs": [], + "id": "9a4f4194" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From c8e18a4e78e60165f264341f42932f7d764c4745 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:04:46 +0000 Subject: [PATCH 23/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 1992 ++++++++++++++-------------- 1 file changed, 996 insertions(+), 996 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index ee7e1e605..c04505833 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,998 +1,998 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ], - "id": "2d522414", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", - "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ], - "id": "7bec6c65", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import getpass\n", - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " )\n", - " TRACKERS_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - " )\n", - "\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", - "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" - ], - "execution_count": null, - "outputs": [], - "id": "6bc2b8d8" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ], - "execution_count": null, - "outputs": [], - "id": "6c2e60ad" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" - ], - "id": "a54bb5ed", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ], - "execution_count": null, - "outputs": [], - "id": "bbc892d6" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ], - "id": "29afb2e0", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ], - "execution_count": null, - "outputs": [], - "id": "5ea423a9" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ], - "id": "b57560b2", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ], - "execution_count": null, - "outputs": [], - "id": "f38487ea" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ], - "id": "823b2696", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ], - "execution_count": null, - "outputs": [], - "id": "d09332f1" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ], - "id": "ad28e88f", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ], - "execution_count": null, - "outputs": [], - "id": "6612c281" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ], - "id": "b0ea623e", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ], - "id": "43321292", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ], - "execution_count": null, - "outputs": [], - "id": "d16f6483" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ], - "id": "8ee1ac84", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ], - "execution_count": null, - "outputs": [], - "id": "d448e555" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ], - "id": "8de54c38", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "images = sorted(img_dir.glob(\"*.jpg\"))\n", - "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", - "\n", - "sample = cv2.imread(str(images[0]))\n", - "if sample is None:\n", - " raise RuntimeError(f\"Could not read {images[0]}\")\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(str(out_path), video_info) as sink:\n", - " for i in range(n_frames):\n", - " frame_idx = i + 1\n", - " frame = cv2.imread(str(images[i]))\n", - " if frame is None:\n", - " continue\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ], - "execution_count": null, - "outputs": [], - "id": "9a4f4194" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ], + "id": "2d522414", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ], + "id": "7bec6c65", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " )\n", + " TRACKERS_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + " )\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" + ], + "execution_count": null, + "outputs": [], + "id": "6bc2b8d8" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ], + "execution_count": null, + "outputs": [], + "id": "6c2e60ad" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + ], + "id": "a54bb5ed", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ], + "execution_count": null, + "outputs": [], + "id": "bbc892d6" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ], + "id": "29afb2e0", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ], + "execution_count": null, + "outputs": [], + "id": "5ea423a9" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ], + "id": "b57560b2", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ], + "execution_count": null, + "outputs": [], + "id": "f38487ea" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ], + "id": "823b2696", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ], + "execution_count": null, + "outputs": [], + "id": "d09332f1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ], + "id": "ad28e88f", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ], + "execution_count": null, + "outputs": [], + "id": "6612c281" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ], + "id": "b0ea623e", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ], + "id": "43321292", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ], + "execution_count": null, + "outputs": [], + "id": "d16f6483" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ], + "id": "8ee1ac84", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ], + "execution_count": null, + "outputs": [], + "id": "d448e555" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ], + "id": "8de54c38", + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "images = sorted(img_dir.glob(\"*.jpg\"))\n", + "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", + "\n", + "sample = cv2.imread(str(images[0]))\n", + "if sample is None:\n", + " raise RuntimeError(f\"Could not read {images[0]}\")\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(str(out_path), video_info) as sink:\n", + " for i in range(n_frames):\n", + " frame_idx = i + 1\n", + " frame = cv2.imread(str(images[i]))\n", + " if frame is None:\n", + " continue\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ], + "execution_count": null, + "outputs": [], + "id": "9a4f4194" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From ea2c16fb00870b2ef7de679f26108e52a62c9451 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 12:06:45 -0300 Subject: [PATCH 24/54] Fix CI for private roboflow-reid git dependency. Exclude the default dev group from build/docs sync, authenticate git in integration/pytest workflows, and fix pre-commit failures in the ReID benchmark notebook and codespell config. Co-authored-by: Cursor --- .github/workflows/build-package.yml | 3 +- .github/workflows/ci-build-docs.yml | 3 +- .github/workflows/ci-integrations.yml | 3 + .github/workflows/ci-tests.yml | 3 + notebooks/eval_trackers_reid.ipynb | 1970 ++++++++++++------------- pyproject.toml | 2 +- 6 files changed, 985 insertions(+), 999 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 0693d7dac..24e9ab5fa 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -35,7 +35,8 @@ jobs: - name: 🏗️ Build source and wheel distributions run: | - uv sync --frozen --group build + # Exclude the default `dev` group (pins private git dep roboflow-reid). + uv sync --frozen --no-default-groups --group build uv build uv run twine check --strict dist/* ls -l dist/ diff --git a/.github/workflows/ci-build-docs.yml b/.github/workflows/ci-build-docs.yml index dc0c0b235..2d8276df8 100644 --- a/.github/workflows/ci-build-docs.yml +++ b/.github/workflows/ci-build-docs.yml @@ -29,7 +29,8 @@ jobs: activate-environment: true - name: 🏗️ Install dependencies - run: uv sync --frozen --group docs + # Exclude the default `dev` group (pins private git dep roboflow-reid). + run: uv sync --frozen --no-default-groups --group docs - name: 🧪 Test Docs Build run: uv run mkdocs build --verbose diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml index 1def972fa..36bddb192 100644 --- a/.github/workflows/ci-integrations.yml +++ b/.github/workflows/ci-integrations.yml @@ -20,6 +20,9 @@ jobs: - name: 📥 Checkout the repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: 🔐 Configure git for private dependencies + run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + - name: 🐍 Install uv and set Python version uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index fbbd8627d..08f5b8735 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -21,6 +21,9 @@ jobs: - name: 📥 Checkout the repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - name: 🔐 Configure git for private dependencies + run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" + - name: 🐍 Install uv and set Python version ${{ matrix.python-version }} uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index c04505833..4c56aa355 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,998 +1,976 @@ { - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ], - "id": "2d522414", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", - "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ], - "id": "7bec6c65", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import getpass\n", - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " )\n", - " TRACKERS_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - " )\n", - "\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", - "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" - ], - "execution_count": null, - "outputs": [], - "id": "6bc2b8d8" - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ], - "execution_count": null, - "outputs": [], - "id": "6c2e60ad" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" - ], - "id": "a54bb5ed", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ], - "execution_count": null, - "outputs": [], - "id": "bbc892d6" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ], - "id": "29afb2e0", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ], - "execution_count": null, - "outputs": [], - "id": "5ea423a9" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ], - "id": "b57560b2", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ], - "execution_count": null, - "outputs": [], - "id": "f38487ea" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ], - "id": "823b2696", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ], - "execution_count": null, - "outputs": [], - "id": "d09332f1" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ], - "id": "ad28e88f", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ], - "execution_count": null, - "outputs": [], - "id": "6612c281" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ], - "id": "b0ea623e", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ], - "id": "43321292", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ], - "execution_count": null, - "outputs": [], - "id": "d16f6483" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ], - "id": "8ee1ac84", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ], - "execution_count": null, - "outputs": [], - "id": "d448e555" - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ], - "id": "8de54c38", - "execution_count": null, - "outputs": [] - }, - { - "cell_type": "code", - "metadata": {}, - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "images = sorted(img_dir.glob(\"*.jpg\"))\n", - "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", - "\n", - "sample = cv2.imread(str(images[0]))\n", - "if sample is None:\n", - " raise RuntimeError(f\"Could not read {images[0]}\")\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(str(out_path), video_info) as sink:\n", - " for i in range(n_frames):\n", - " frame_idx = i + 1\n", - " frame = cv2.imread(str(images[i]))\n", - " if frame is None:\n", - " continue\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ], - "execution_count": null, - "outputs": [], - "id": "9a4f4194" - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ], + "id": "2d522414" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ], + "id": "7bec6c65" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " )\n", + " TRACKERS_REF = (\n", + " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + " )\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" + ], + "execution_count": null, + "outputs": [], + "id": "6bc2b8d8" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ], + "execution_count": null, + "outputs": [], + "id": "6c2e60ad" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + ], + "id": "a54bb5ed" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ], + "execution_count": null, + "outputs": [], + "id": "bbc892d6" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ], + "id": "29afb2e0" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ], + "execution_count": null, + "outputs": [], + "id": "5ea423a9" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ], + "id": "b57560b2" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ], + "execution_count": null, + "outputs": [], + "id": "f38487ea" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ], + "id": "823b2696" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ], + "execution_count": null, + "outputs": [], + "id": "d09332f1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ], + "id": "ad28e88f" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ], + "execution_count": null, + "outputs": [], + "id": "6612c281" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ], + "id": "b0ea623e" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ], + "id": "43321292" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ], + "execution_count": null, + "outputs": [], + "id": "d16f6483" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ], + "id": "8ee1ac84" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ], + "execution_count": null, + "outputs": [], + "id": "d448e555" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ], + "id": "8de54c38" + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "images = sorted(img_dir.glob(\"*.jpg\"))\n", + "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", + "\n", + "sample = cv2.imread(str(images[0]))\n", + "if sample is None:\n", + " raise RuntimeError(f\"Could not read {images[0]}\")\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(str(out_path), video_info) as sink:\n", + " for i in range(n_frames):\n", + " frame_idx = i + 1\n", + " frame = cv2.imread(str(images[i]))\n", + " if frame is None:\n", + " continue\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ], + "execution_count": null, + "outputs": [], + "id": "9a4f4194" + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/pyproject.toml b/pyproject.toml index 8cab3f9b4..9beef2160 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -202,7 +202,7 @@ markers = [ [tool.codespell] skip = "*.pth" -ignore-words-list = "mot" +ignore-words-list = "mot,STrack" [tool.mypy] python_version = "3.10" From 7e5e1afdc2ef88ebf9165dfb66bc1a2928efb599 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:07:31 +0000 Subject: [PATCH 25/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 168 ++++++++++++++--------------- 1 file changed, 82 insertions(+), 86 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index 4c56aa355..e43a3e1e7 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -2,6 +2,7 @@ "cells": [ { "cell_type": "markdown", + "id": "2d522414", "metadata": {}, "source": [ "# Tracker ReID evaluation on MOT17 val\n", @@ -19,14 +20,14 @@ "\n", "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", "\n", "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ], - "id": "2d522414" + ] }, { "cell_type": "markdown", + "id": "7bec6c65", "metadata": {}, "source": [ "## 1. Setup\n", @@ -36,12 +37,14 @@ "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", "\n", "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ], - "id": "7bec6c65" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", "metadata": {}, + "outputs": [], "source": [ "import getpass\n", "import subprocess\n", @@ -59,12 +62,8 @@ "\n", "if IN_COLAB_INSTALL:\n", " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " )\n", - " TRACKERS_REF = (\n", - " f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - " )\n", + " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", "\n", " cmds = [\n", " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", @@ -127,15 +126,15 @@ " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", " # Optional local editable installs:\n", " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)\n" - ], - "execution_count": null, - "outputs": [], - "id": "6bc2b8d8" + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", "metadata": {}, + "outputs": [], "source": [ "import subprocess\n", "import sys\n", @@ -151,10 +150,10 @@ "import torch\n", "from IPython.display import Video\n", "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", "from sklearn.decomposition import PCA\n", "\n", "from trackers import BoTSORTTracker\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", "from trackers.eval import evaluate_mot_sequences\n", "from trackers.eval.box import box_iou\n", "from trackers.eval.results import BenchmarkResult\n", @@ -184,27 +183,27 @@ "\n", "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ], - "execution_count": null, - "outputs": [], - "id": "6c2e60ad" + ] }, { "cell_type": "markdown", + "id": "a54bb5ed", "metadata": {}, "source": [ "## 2. ReID model\n", "\n", "| `REID_ENCODER` | Training | Input |\n", "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" - ], - "id": "a54bb5ed" + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", "metadata": {}, + "outputs": [], "source": [ "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", @@ -216,27 +215,27 @@ "else:\n", " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", "\n", - "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", "print(reid_model.preprocessing.describe())" - ], - "execution_count": null, - "outputs": [], - "id": "bbc892d6" + ] }, { "cell_type": "markdown", + "id": "29afb2e0", "metadata": {}, "source": [ "## 3. Download data\n", "\n", "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ], - "id": "29afb2e0" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", "metadata": {}, + "outputs": [], "source": [ "FORCE_DOWNLOAD = False\n", "\n", @@ -311,22 +310,22 @@ "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ], - "execution_count": null, - "outputs": [], - "id": "5ea423a9" + ] }, { "cell_type": "markdown", + "id": "b57560b2", "metadata": {}, "source": [ "## 4. Tracking helpers\n" - ], - "id": "b57560b2" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "f38487ea", "metadata": {}, + "outputs": [], "source": [ "RERUN = {\n", " \"botsort_baseline\": True,\n", @@ -460,22 +459,22 @@ " if ious[i, j] >= min_iou:\n", " out[i] = int(gt_ids[j])\n", " return out" - ], - "execution_count": null, - "outputs": [], - "id": "f38487ea" + ] }, { "cell_type": "markdown", + "id": "823b2696", "metadata": {}, "source": [ "## 5. Run trackers\n" - ], - "id": "823b2696" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "d09332f1", "metadata": {}, + "outputs": [], "source": [ "EXPERIMENTS = [\n", " (\n", @@ -505,22 +504,22 @@ "\n", "result_baseline = results[\"botsort_baseline\"]\n", "result_reid = results[\"botsort_reid\"]" - ], - "execution_count": null, - "outputs": [], - "id": "d09332f1" + ] }, { "cell_type": "markdown", + "id": "ad28e88f", "metadata": {}, "source": [ "## 6. ReID embedding visualization (optional)\n" - ], - "id": "ad28e88f" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "6612c281", "metadata": {}, + "outputs": [], "source": [ "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", @@ -594,23 +593,21 @@ "plt.tight_layout()\n", "plt.show()\n", "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ], - "execution_count": null, - "outputs": [], - "id": "6612c281" + ] }, { "cell_type": "markdown", + "id": "b0ea623e", "metadata": {}, "source": [ "## 7. Results\n", "\n", "**7.1-7.2** BoT-SORT vs published references.\n" - ], - "id": "b0ea623e" + ] }, { "cell_type": "markdown", + "id": "43321292", "metadata": {}, "source": [ "### 7.1 BoT-SORT - reference targets\n", @@ -630,12 +627,14 @@ "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ], - "id": "43321292" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "d16f6483", "metadata": {}, + "outputs": [], "source": [ "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", "# MOTA is not reported for the YOLOX setup in that study.\n", @@ -681,8 +680,8 @@ "r = fmt_metrics(result_reid)\n", "print(\n", " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", ")\n", "\n", "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", @@ -710,10 +709,10 @@ ")\n", "print(\n", " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", ")\n", "\n", @@ -739,29 +738,29 @@ ")\n", "print(\n", " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", ")" - ], - "execution_count": null, - "outputs": [], - "id": "d16f6483" + ] }, { "cell_type": "markdown", + "id": "8ee1ac84", "metadata": {}, "source": [ "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ], - "id": "8ee1ac84" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "d448e555", "metadata": {}, + "outputs": [], "source": [ "REID_STUDY_PER_SEQ = {\n", " \"MOT17-02\": {\n", @@ -819,26 +818,26 @@ " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", " print()" - ], - "execution_count": null, - "outputs": [], - "id": "d448e555" + ] }, { "cell_type": "markdown", + "id": "8de54c38", "metadata": {}, "source": [ "### 8. Visual comparison - largest ReID gain sequence\n", "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", "(from the runs above). On Colab the mp4 is downloaded automatically.\n", "" - ], - "id": "8de54c38" + ] }, { "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", "metadata": {}, + "outputs": [], "source": [ "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", "COMPARE_SEQ: str | None = None\n", @@ -897,13 +896,13 @@ " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", "\n", "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", "\n", "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", "\n", "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", @@ -946,10 +945,7 @@ "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", "if IN_COLAB:\n", " files.download(str(out_path))" - ], - "execution_count": null, - "outputs": [], - "id": "9a4f4194" + ] } ], "metadata": { From de2d419e75405850d7c0eacf657e4b6567dbc3a8 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 12:09:16 -0300 Subject: [PATCH 26/54] Keep roboflow-reid out of default dev sync in CI. Move the private git dependency to the reid extra only, avoid uv run re-syncing the dev group in build/docs jobs, and fix notebook JSON for pre-commit. Co-authored-by: Cursor --- .github/workflows/build-package.yml | 2 +- .github/workflows/ci-build-docs.yml | 2 +- .github/workflows/ci-integrations.yml | 4 +- .github/workflows/ci-tests.yml | 10 +- notebooks/eval_trackers_reid.ipynb | 1940 ++++++++++++------------- pyproject.toml | 2 - uv.lock | 2 - 7 files changed, 976 insertions(+), 986 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 24e9ab5fa..2872be0ea 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -38,7 +38,7 @@ jobs: # Exclude the default `dev` group (pins private git dep roboflow-reid). uv sync --frozen --no-default-groups --group build uv build - uv run twine check --strict dist/* + uv run --no-sync twine check --strict dist/* ls -l dist/ - name: 📤 Upload distribution artifacts diff --git a/.github/workflows/ci-build-docs.yml b/.github/workflows/ci-build-docs.yml index 2d8276df8..435662e4c 100644 --- a/.github/workflows/ci-build-docs.yml +++ b/.github/workflows/ci-build-docs.yml @@ -33,4 +33,4 @@ jobs: run: uv sync --frozen --no-default-groups --group docs - name: 🧪 Test Docs Build - run: uv run mkdocs build --verbose + run: uv run --no-sync mkdocs build --verbose diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml index 36bddb192..bd5340296 100644 --- a/.github/workflows/ci-integrations.yml +++ b/.github/workflows/ci-integrations.yml @@ -20,9 +20,6 @@ jobs: - name: 📥 Checkout the repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: 🔐 Configure git for private dependencies - run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - - name: 🐍 Install uv and set Python version uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: @@ -30,6 +27,7 @@ jobs: activate-environment: true - name: 🚀 Install Packages + # ReID integration smoke test skips when roboflow-reid is unavailable. run: uv sync --frozen --group dev diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 08f5b8735..5855ef2ff 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -21,9 +21,6 @@ jobs: - name: 📥 Checkout the repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: 🔐 Configure git for private dependencies - run: git config --global url."https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com/".insteadOf "https://github.com/" - - name: 🐍 Install uv and set Python version ${{ matrix.python-version }} uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 with: @@ -33,10 +30,9 @@ jobs: prune-cache: ${{ matrix.os != 'windows-latest' }} - name: 🚀 Install Packages - # NOTE: --frozen is dropped while the `reid` extra pins roboflow-reid to a - # git ref (see pyproject). Restore `--frozen` once roboflow-reid publishes - # to PyPI and uv.lock is regenerated. - run: uv sync --group dev --extra reid + # ReID extra is omitted in CI until roboflow-reid is on PyPI or a PAT + # secret can read the private roboflow/re-ID repo. + run: uv sync --group dev - name: 🧪 Run the Import test run: uv run python -c "import trackers" diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index e43a3e1e7..e40d7ae8d 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,972 +1,972 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "2d522414", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ] - }, - { - "cell_type": "markdown", - "id": "7bec6c65", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", - "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc2b8d8", - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - "\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", - "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c2e60ad", - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ] - }, - { - "cell_type": "markdown", - "id": "a54bb5ed", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc892d6", - "metadata": {}, - "outputs": [], - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "29afb2e0", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea423a9", - "metadata": {}, - "outputs": [], - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ] - }, - { - "cell_type": "markdown", - "id": "b57560b2", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f38487ea", - "metadata": {}, - "outputs": [], - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ] - }, - { - "cell_type": "markdown", - "id": "823b2696", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d09332f1", - "metadata": {}, - "outputs": [], - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ] - }, - { - "cell_type": "markdown", - "id": "ad28e88f", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6612c281", - "metadata": {}, - "outputs": [], - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ] - }, - { - "cell_type": "markdown", - "id": "b0ea623e", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ] - }, - { - "cell_type": "markdown", - "id": "43321292", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d16f6483", - "metadata": {}, - "outputs": [], - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8ee1ac84", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d448e555", - "metadata": {}, - "outputs": [], - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "8de54c38", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a4f4194", - "metadata": {}, - "outputs": [], - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "images = sorted(img_dir.glob(\"*.jpg\"))\n", - "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", - "\n", - "sample = cv2.imread(str(images[0]))\n", - "if sample is None:\n", - " raise RuntimeError(f\"Could not read {images[0]}\")\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(str(out_path), video_info) as sink:\n", - " for i in range(n_frames):\n", - " frame_idx = i + 1\n", - " frame = cv2.imread(str(images[i]))\n", - " if frame is None:\n", - " continue\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "2d522414", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bec6c65", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a54bb5ed", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", + "metadata": {}, + "outputs": [], + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "29afb2e0", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", + "metadata": {}, + "outputs": [], + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b57560b2", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f38487ea", + "metadata": {}, + "outputs": [], + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "id": "823b2696", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d09332f1", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ad28e88f", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6612c281", + "metadata": {}, + "outputs": [], + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0ea623e", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ] + }, + { + "cell_type": "markdown", + "id": "43321292", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16f6483", + "metadata": {}, + "outputs": [], + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8ee1ac84", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d448e555", + "metadata": {}, + "outputs": [], + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "8de54c38", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", + "metadata": {}, + "outputs": [], + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "images = sorted(img_dir.glob(\"*.jpg\"))\n", + "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", + "\n", + "sample = cv2.imread(str(images[0]))\n", + "if sample is None:\n", + " raise RuntimeError(f\"Could not read {images[0]}\")\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(str(out_path), video_info) as sink:\n", + " for i in range(n_frames):\n", + " frame_idx = i + 1\n", + " frame = cv2.imread(str(images[i]))\n", + " if frame is None:\n", + " continue\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/pyproject.toml b/pyproject.toml index 9beef2160..5c7f5d27f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,8 +62,6 @@ dev = [ "pre-commit>=4.2.0", "torch", "torchvision", - # Real ReID encoder for association / integration tests. - "roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack", ] docs = [ "mkdocs>=1.6.1", diff --git a/uv.lock b/uv.lock index e39cc47e7..2c49af460 100644 --- a/uv.lock +++ b/uv.lock @@ -4227,7 +4227,6 @@ build = [ dev = [ { name = "pre-commit" }, { name = "pytest" }, - { name = "roboflow-reid" }, { name = "torch" }, { name = "torchvision" }, { name = "uv" }, @@ -4270,7 +4269,6 @@ build = [ dev = [ { name = "pre-commit", specifier = ">=4.2.0" }, { name = "pytest", specifier = ">=8.3.3" }, - { name = "roboflow-reid", git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack" }, { name = "torch" }, { name = "torchvision" }, { name = "uv", specifier = ">=0.4.20" }, From c76b8648548dba563e85df39410f8e54fad1596d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:09:49 +0000 Subject: [PATCH 27/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 32 +++++++++++++++--------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index e40d7ae8d..afd224841 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -20,7 +20,7 @@ "\n", "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", "\n", "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" ] @@ -194,8 +194,8 @@ "\n", "| `REID_ENCODER` | Training | Input |\n", "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" ] }, { @@ -215,7 +215,7 @@ "else:\n", " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", "print(reid_model.preprocessing.describe())" ] }, @@ -680,8 +680,8 @@ "r = fmt_metrics(result_reid)\n", "print(\n", " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", + " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", ")\n", "\n", "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", @@ -709,10 +709,10 @@ ")\n", "print(\n", " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", ")\n", "\n", @@ -738,11 +738,11 @@ ")\n", "print(\n", " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", ")" ] @@ -827,7 +827,7 @@ "source": [ "### 8. Visual comparison - largest ReID gain sequence\n", "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", "(from the runs above). On Colab the mp4 is downloaded automatically.\n", "" ] @@ -896,13 +896,13 @@ " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", "\n", "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", + "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", + " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", "\n", "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", "\n", "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", From c32edbc669b45c24eee9208e7f604c9885a53590 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 12:21:17 -0300 Subject: [PATCH 28/54] docs: rewrite reid API page to match trackers style State what trackers.core.reid covers positively, defer model/eval to the reid package, and drop subtractive architecture prose. Align install extra wording with develop (no roboflow-reid in prose). Co-authored-by: Cursor --- docs/api/reid.md | 34 ++++++++++++++++------------------ docs/learn/install.md | 7 +++---- 2 files changed, 19 insertions(+), 22 deletions(-) diff --git a/docs/api/reid.md b/docs/api/reid.md index b540c004c..8a2439f0e 100644 --- a/docs/api/reid.md +++ b/docs/api/reid.md @@ -1,25 +1,25 @@ --- -description: Appearance-ReID association utilities in Roboflow Trackers. +description: ReID encoder protocol, feature bank, and appearance association utilities in Roboflow Trackers. --- # ReID API -Appearance-based re-identification (ReID) lets BoT-SORT match tracks across -frames using visual appearance in addition to motion. It requires the optional -extra: +Requires the optional extra: ```bash pip install 'trackers[reid]' ``` -Trackers ships only the numpy-only association glue documented below. The -appearance encoder, pretrained weights, preprocessing, model catalog, and -gallery evaluation live in the standalone [`reid`](https://github.com/roboflow/re-ID) -package (`roboflow-reid`), which the `trackers[reid]` extra installs for you. +This page covers the `ReIDEncoder` protocol, `FeatureBank`, and appearance +association helpers in `trackers.core.reid`. ReID model loading, gallery +evaluation, and MOT fine-tuning are documented in the standalone +[`reid`](https://github.com/roboflow/re-ID) package. BoT-SORT usage is on the +[BoT-SORT](../trackers/botsort.md) page. -## Loading a model +## Use with BoT-SORT -Import the encoder from `reid` and pass it to BoT-SORT: +Import the encoder from `reid` and pass any object that implements +`ReIDEncoder` to `BoTSORTTracker`: ```python from reid import ReIDModel @@ -31,16 +31,14 @@ tracker = BoTSORTTracker(reid_model=reid_model) ``` See the [`reid` package documentation](https://github.com/roboflow/re-ID) for -the full model catalog, `from_pretrained` sources (curated aliases, `hf://` -repos, local checkpoints, architecture-only init), gallery evaluation -(`ReIDEvaluator`, `load_market1501`, `load_msmt17`), and how to add -architectures. +the model catalog, `from_pretrained` sources, gallery evaluation, and MOT +fine-tuning. ## Encoder protocol -`ReIDEncoder` is the minimal interface BoT-SORT depends on: a single -`extract_features` method. `reid.ReIDModel` satisfies it, and you can implement -it yourself for a custom encoder without depending on the model stack. +`ReIDEncoder` is the interface BoT-SORT expects: a single `extract_features` +method. `reid.ReIDModel` satisfies it; custom encoders can implement the +protocol without the model stack. ::: trackers.core.reid.encoder.ReIDEncoder @@ -48,7 +46,7 @@ it yourself for a custom encoder without depending on the model stack. ::: trackers.core.reid.feature_bank.FeatureBank -## Appearance similarity +## Appearance ::: trackers.core.reid.appearance.appearance_similarity diff --git a/docs/learn/install.md b/docs/learn/install.md index 6ec87c6ee..180cfc2b4 100644 --- a/docs/learn/install.md +++ b/docs/learn/install.md @@ -72,10 +72,9 @@ The `detection` extra installs `inference-models`, enabling the CLI to run detec ### ReID (BoT-SORT appearance) -The `reid` extra installs the standalone `roboflow-reid` package, which brings -PyTorch, timm, Hugging Face Hub, safetensors, and related dependencies for ReID -model loading (OSNet, FastReID SBS, and `timm:` backbones) and BoT-SORT -appearance association. +The `reid` extra installs PyTorch, timm, Hugging Face Hub, safetensors, Pillow, +and gdown for ReID model loading (OSNet, FastReID SBS, and `timm:` backbones) +and BoT-SORT appearance association. === "pip" From 86e57e1c403a7857dec3d6fab7d45c43554c600d Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 21 Jul 2026 12:27:27 -0300 Subject: [PATCH 29/54] docs: add ReID comparison table and tighten BoT-SORT ReID section Add MOT17 val with/without ReID reference scores to the ReID API page and align the BoT-SORT optional ReID section with trackers docs style. Co-authored-by: Cursor --- docs/api/reid.md | 20 ++++++++++++++++++++ docs/trackers/botsort.md | 34 +++++++++++++++------------------- 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/docs/api/reid.md b/docs/api/reid.md index 8a2439f0e..4003d0622 100644 --- a/docs/api/reid.md +++ b/docs/api/reid.md @@ -16,6 +16,26 @@ evaluation, and MOT fine-tuning are documented in the standalone [`reid`](https://github.com/roboflow/re-ID) package. BoT-SORT usage is on the [BoT-SORT](../trackers/botsort.md) page. +## BoT-SORT with and without ReID + +Reference scores on MOT17 val-half with YOLOX detections from the +[BoT-SORT paper](https://arxiv.org/abs/2206.14651) Table 1. The +[`eval_trackers_reid.ipynb`](../../notebooks/eval_trackers_reid.ipynb) notebook +uses the same split and detector with `fastreid_mot17_sbs50` and +`appearance_threshold=0.2` (MOT17 re-ID study Table 8); run it for +trackers-local results. + +| Config | HOTA | MOTA | IDF1 | +| :----- | :--: | :--: | :--: | +| BoT-SORT | 69.11 | 78.39 | 81.53 | +| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 | + +The MOT17 re-ID study +([*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf), +Table 8 + Table 13 combined row, app th=0.2) reports HOTA 68.43 / IDF1 80.92 +without ReID and 68.95 / 81.98 with MOT17 FastReID; MOTA is not reported for +that YOLOX setup. + ## Use with BoT-SORT Import the encoder from `reid` and pass any object that implements diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 4d07f645f..7d1452879 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -50,11 +50,18 @@ BoT-SORT keeps the same tracking-by-detection backbone as [ByteTrack](bytetrack. | `high_conf_det_threshold` | Confidence split between stage-1 and stage-2 detections. | 0.5-0.7 common. Higher shifts more detections to recovery stage; lower gives stage-1 broader coverage. | | `enable_cmc` | Enables camera motion compensation before association. | Keep enabled for moving-camera footage (sports, drone, handheld). Disable mainly for static cameras if you need maximal speed. | -## Appearance ReID +## ReID appearance (optional) -BoT-SORT can optionally fuse appearance embeddings with IoU during association. -Pass a `reid_model` that implements the `ReIDEncoder` protocol (for example, -`reid.ReIDModel`) to enable this mode: +Install the extra first: + +```bash +pip install 'trackers[reid]' +``` + +BoT-SORT can fuse appearance embeddings with IoU during association. Pass a +`reid_model` that implements `ReIDEncoder` (for example, `ReIDModel` from the +[`reid`](https://github.com/roboflow/re-ID) package) and pass the current frame +to `tracker.update` so the encoder can crop detections: ```python from reid import ReIDModel @@ -62,30 +69,19 @@ from trackers import BoTSORTTracker reid_model = ReIDModel.from_pretrained("osnet_x1_0_msmt17_combineall", device="cpu") tracker = BoTSORTTracker(reid_model=reid_model) -``` - -When ReID is enabled, pass the current frame to `tracker.update` so the encoder -can crop detections: - -```python detections = tracker.update(detections, frame=frame) ``` -Tuning knobs: +See the [ReID API](../api/reid.md) for the encoder protocol, feature bank, +association helpers, and a [with/without ReID comparison](../api/reid.md#botsort-with-and-without-reid) +on MOT17 val. | Parameter | Default | Purpose | | ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | | `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | -| `appearance_threshold` | 0.25 | Appearance-distance gate. A match is rejected when the halved cosine distance `0.5 * (1 - cos_sim)` exceeds this value. | +| `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | | `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | -Install the optional extra and see the [ReID API](../api/reid.md) for the -encoder protocol, feature bank, and association utilities: - -```bash -pip install 'trackers[reid]' -``` - ## Run on video, webcam, or RTSP stream These examples use `opencv-python` for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually 0 for the default camera. From 298057418ecabe397b5ee89993dd4466fe5cbf9e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 21 Jul 2026 15:27:47 +0000 Subject: [PATCH 30/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/api/reid.md | 6 +++--- docs/trackers/botsort.md | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/api/reid.md b/docs/api/reid.md index 4003d0622..39e9e23d2 100644 --- a/docs/api/reid.md +++ b/docs/api/reid.md @@ -25,9 +25,9 @@ uses the same split and detector with `fastreid_mot17_sbs50` and `appearance_threshold=0.2` (MOT17 re-ID study Table 8); run it for trackers-local results. -| Config | HOTA | MOTA | IDF1 | -| :----- | :--: | :--: | :--: | -| BoT-SORT | 69.11 | 78.39 | 81.53 | +| Config | HOTA | MOTA | IDF1 | +| :-------------- | :---: | :---: | :---: | +| BoT-SORT | 69.11 | 78.39 | 81.53 | | BoT-SORT + ReID | 69.17 | 78.46 | 82.07 | The MOT17 re-ID study diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 7d1452879..4d51a6558 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -76,11 +76,11 @@ See the [ReID API](../api/reid.md) for the encoder protocol, feature bank, association helpers, and a [with/without ReID comparison](../api/reid.md#botsort-with-and-without-reid) on MOT17 val. -| Parameter | Default | Purpose | -| ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | +| Parameter | Default | Purpose | +| ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | | `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | -| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | +| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | ## Run on video, webcam, or RTSP stream From 47001a1fdfc91cea3a4b453737f0fd5968ce840a Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 10:46:11 -0300 Subject: [PATCH 31/54] Fix MOT17 ReID notebook side-by-side video render. Load frames by MOT index, use per-sequence FPS, re-encode to H.264 for notebook/Colab playback, and embed at the combined panel width. Co-authored-by: Cursor --- notebooks/eval_trackers_reid.ipynb | 110 ++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 32 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index afd224841..ac73be785 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -20,7 +20,7 @@ "\n", "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", "\n", "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" ] @@ -136,6 +136,7 @@ "metadata": {}, "outputs": [], "source": [ + "import shutil\n", "import subprocess\n", "import sys\n", "import warnings\n", @@ -157,6 +158,7 @@ "from trackers.eval import evaluate_mot_sequences\n", "from trackers.eval.box import box_iou\n", "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.frames import load_mot_frame_image\n", "from trackers.io.mot import _MOTOutput, load_mot_file\n", "\n", "warnings.filterwarnings(\"ignore\")\n", @@ -194,8 +196,8 @@ "\n", "| `REID_ENCODER` | Training | Input |\n", "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" ] }, { @@ -215,7 +217,7 @@ "else:\n", " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", "\n", - "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", "print(reid_model.preprocessing.describe())" ] }, @@ -680,8 +682,8 @@ "r = fmt_metrics(result_reid)\n", "print(\n", " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", ")\n", "\n", "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", @@ -709,10 +711,10 @@ ")\n", "print(\n", " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", ")\n", "\n", @@ -738,11 +740,11 @@ ")\n", "print(\n", " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", ")" ] @@ -827,7 +829,7 @@ "source": [ "### 8. Visual comparison - largest ReID gain sequence\n", "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", "(from the runs above). On Colab the mp4 is downloaded automatically.\n", "" ] @@ -896,13 +898,13 @@ " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", "\n", "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", "\n", "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", "\n", "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", @@ -912,23 +914,27 @@ "mot_base = load_mot_file(pred_base)\n", "mot_reid = load_mot_file(pred_reid)\n", "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "images = sorted(img_dir.glob(\"*.jpg\"))\n", - "n_frames = len(images) if COMPARE_MAX_FRAMES is None else min(len(images), COMPARE_MAX_FRAMES)\n", - "\n", - "sample = cv2.imread(str(images[0]))\n", - "if sample is None:\n", - " raise RuntimeError(f\"Could not read {images[0]}\")\n", + "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", + "if COMPARE_MAX_FRAMES is not None:\n", + " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", + "\n", + "compare_fps = COMPARE_FPS\n", + "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", + "if seqinfo.is_file():\n", + " for line in seqinfo.read_text().splitlines():\n", + " if line.startswith(\"frameRate=\"):\n", + " compare_fps = int(line.split(\"=\", 1)[1])\n", + " break\n", + "\n", + "sample = load_mot_frame_image(img_dir, 1)\n", "h, w = sample.shape[:2]\n", "\n", "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=COMPARE_FPS, total_frames=n_frames)\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", "\n", - "with sv.VideoSink(str(out_path), video_info) as sink:\n", - " for i in range(n_frames):\n", - " frame_idx = i + 1\n", - " frame = cv2.imread(str(images[i]))\n", - " if frame is None:\n", - " continue\n", + "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", + " for frame_idx in range(1, n_frames + 1):\n", + " frame = load_mot_frame_image(img_dir, frame_idx)\n", " left = _panel_badge(\n", " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", " \"BASELINE (NO REID)\",\n", @@ -941,11 +947,51 @@ " )\n", " sink.write_frame(np.hstack([left, right]))\n", "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {COMPARE_FPS} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w)))\n", + "ffmpeg = shutil.which(\"ffmpeg\")\n", + "if ffmpeg is not None:\n", + " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", + " result = subprocess.run(\n", + " [\n", + " ffmpeg,\n", + " \"-y\",\n", + " \"-i\",\n", + " str(out_path),\n", + " \"-c:v\",\n", + " \"libx264\",\n", + " \"-pix_fmt\",\n", + " \"yuv420p\",\n", + " \"-movflags\",\n", + " \"+faststart\",\n", + " \"-an\",\n", + " str(tmp),\n", + " ],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " if result.returncode == 0:\n", + " tmp.replace(out_path)\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", "if IN_COLAB:\n", - " files.download(str(out_path))" + " files.download(str(out_path))\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac864a15", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa51e6b5", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From eaa023c054f852c2a7984e0f734ef2594caea64b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:47:28 +0000 Subject: [PATCH 32/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 2032 ++++++++++++++-------------- 1 file changed, 1016 insertions(+), 1016 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index ac73be785..315806596 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,1018 +1,1018 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "2d522414", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ] - }, - { - "cell_type": "markdown", - "id": "7bec6c65", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", - "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc2b8d8", - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - "\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", - "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c2e60ad", - "metadata": {}, - "outputs": [], - "source": [ - "import shutil\n", - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.frames import load_mot_frame_image\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ] - }, - { - "cell_type": "markdown", - "id": "a54bb5ed", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc892d6", - "metadata": {}, - "outputs": [], - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "29afb2e0", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea423a9", - "metadata": {}, - "outputs": [], - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ] - }, - { - "cell_type": "markdown", - "id": "b57560b2", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f38487ea", - "metadata": {}, - "outputs": [], - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ] - }, - { - "cell_type": "markdown", - "id": "823b2696", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d09332f1", - "metadata": {}, - "outputs": [], - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ] - }, - { - "cell_type": "markdown", - "id": "ad28e88f", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6612c281", - "metadata": {}, - "outputs": [], - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ] - }, - { - "cell_type": "markdown", - "id": "b0ea623e", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ] - }, - { - "cell_type": "markdown", - "id": "43321292", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d16f6483", - "metadata": {}, - "outputs": [], - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8ee1ac84", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d448e555", - "metadata": {}, - "outputs": [], - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "8de54c38", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a4f4194", - "metadata": {}, - "outputs": [], - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", - "if COMPARE_MAX_FRAMES is not None:\n", - " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", - "\n", - "compare_fps = COMPARE_FPS\n", - "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", - "if seqinfo.is_file():\n", - " for line in seqinfo.read_text().splitlines():\n", - " if line.startswith(\"frameRate=\"):\n", - " compare_fps = int(line.split(\"=\", 1)[1])\n", - " break\n", - "\n", - "sample = load_mot_frame_image(img_dir, 1)\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", - " for frame_idx in range(1, n_frames + 1):\n", - " frame = load_mot_frame_image(img_dir, frame_idx)\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "ffmpeg = shutil.which(\"ffmpeg\")\n", - "if ffmpeg is not None:\n", - " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", - " result = subprocess.run(\n", - " [\n", - " ffmpeg,\n", - " \"-y\",\n", - " \"-i\",\n", - " str(out_path),\n", - " \"-c:v\",\n", - " \"libx264\",\n", - " \"-pix_fmt\",\n", - " \"yuv420p\",\n", - " \"-movflags\",\n", - " \"+faststart\",\n", - " \"-an\",\n", - " str(tmp),\n", - " ],\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " if result.returncode == 0:\n", - " tmp.replace(out_path)\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac864a15", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa51e6b5", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "2d522414", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bec6c65", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.frames import load_mot_frame_image\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a54bb5ed", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", + "metadata": {}, + "outputs": [], + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "29afb2e0", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", + "metadata": {}, + "outputs": [], + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b57560b2", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f38487ea", + "metadata": {}, + "outputs": [], + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "id": "823b2696", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d09332f1", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ad28e88f", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6612c281", + "metadata": {}, + "outputs": [], + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0ea623e", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ] + }, + { + "cell_type": "markdown", + "id": "43321292", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16f6483", + "metadata": {}, + "outputs": [], + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8ee1ac84", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d448e555", + "metadata": {}, + "outputs": [], + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "8de54c38", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", + "metadata": {}, + "outputs": [], + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", + "if COMPARE_MAX_FRAMES is not None:\n", + " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", + "\n", + "compare_fps = COMPARE_FPS\n", + "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", + "if seqinfo.is_file():\n", + " for line in seqinfo.read_text().splitlines():\n", + " if line.startswith(\"frameRate=\"):\n", + " compare_fps = int(line.split(\"=\", 1)[1])\n", + " break\n", + "\n", + "sample = load_mot_frame_image(img_dir, 1)\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", + " for frame_idx in range(1, n_frames + 1):\n", + " frame = load_mot_frame_image(img_dir, frame_idx)\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "ffmpeg = shutil.which(\"ffmpeg\")\n", + "if ffmpeg is not None:\n", + " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", + " result = subprocess.run(\n", + " [\n", + " ffmpeg,\n", + " \"-y\",\n", + " \"-i\",\n", + " str(out_path),\n", + " \"-c:v\",\n", + " \"libx264\",\n", + " \"-pix_fmt\",\n", + " \"yuv420p\",\n", + " \"-movflags\",\n", + " \"+faststart\",\n", + " \"-an\",\n", + " str(tmp),\n", + " ],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " if result.returncode == 0:\n", + " tmp.replace(out_path)\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac864a15", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa51e6b5", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From 4a22cdfb3b5d24380b296ef1f393e0ed5b485d3a Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 10:48:23 -0300 Subject: [PATCH 33/54] ci: restore default install commands now that reid is optional-only Co-authored-by: Cursor --- .github/workflows/build-package.yml | 5 ++--- .github/workflows/ci-build-docs.yml | 5 ++--- .github/workflows/ci-integrations.yml | 1 - .github/workflows/ci-tests.yml | 4 +--- pyproject.toml | 4 +--- 5 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build-package.yml b/.github/workflows/build-package.yml index 2872be0ea..0693d7dac 100644 --- a/.github/workflows/build-package.yml +++ b/.github/workflows/build-package.yml @@ -35,10 +35,9 @@ jobs: - name: 🏗️ Build source and wheel distributions run: | - # Exclude the default `dev` group (pins private git dep roboflow-reid). - uv sync --frozen --no-default-groups --group build + uv sync --frozen --group build uv build - uv run --no-sync twine check --strict dist/* + uv run twine check --strict dist/* ls -l dist/ - name: 📤 Upload distribution artifacts diff --git a/.github/workflows/ci-build-docs.yml b/.github/workflows/ci-build-docs.yml index 435662e4c..dc0c0b235 100644 --- a/.github/workflows/ci-build-docs.yml +++ b/.github/workflows/ci-build-docs.yml @@ -29,8 +29,7 @@ jobs: activate-environment: true - name: 🏗️ Install dependencies - # Exclude the default `dev` group (pins private git dep roboflow-reid). - run: uv sync --frozen --no-default-groups --group docs + run: uv sync --frozen --group docs - name: 🧪 Test Docs Build - run: uv run --no-sync mkdocs build --verbose + run: uv run mkdocs build --verbose diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml index bd5340296..1def972fa 100644 --- a/.github/workflows/ci-integrations.yml +++ b/.github/workflows/ci-integrations.yml @@ -27,7 +27,6 @@ jobs: activate-environment: true - name: 🚀 Install Packages - # ReID integration smoke test skips when roboflow-reid is unavailable. run: uv sync --frozen --group dev diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 5855ef2ff..9ca4d9f38 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -30,9 +30,7 @@ jobs: prune-cache: ${{ matrix.os != 'windows-latest' }} - name: 🚀 Install Packages - # ReID extra is omitted in CI until roboflow-reid is on PyPI or a PAT - # secret can read the private roboflow/re-ID repo. - run: uv sync --group dev + run: uv sync --frozen --group dev - name: 🧪 Run the Import test run: uv run python -c "import trackers" diff --git a/pyproject.toml b/pyproject.toml index 5c7f5d27f..3f8c7091b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,9 +47,7 @@ dependencies = [ [project.optional-dependencies] detection = ["inference-models>=0.19.0"] tune = ["optuna>=3.0.0"] -# ReID appearance association sources its model stack from the standalone -# roboflow-reid package. Pinned to a git ref during review; swap to a PyPI -# release (e.g. "roboflow-reid>=0.1.0,<0.2") before merge. See re-ID#1. +# Swap the git pin for a PyPI release (e.g. "roboflow-reid>=0.1.0,<0.2") before merge. reid = ["roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack"] [project.scripts] From d69143e79ba8da6ca3a969678e76b719373f8ad3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:49:09 +0000 Subject: [PATCH 34/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 2032 ++++++++++++++-------------- 1 file changed, 1016 insertions(+), 1016 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index 315806596..7dc91ff18 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,1018 +1,1018 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "2d522414", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ] - }, - { - "cell_type": "markdown", - "id": "7bec6c65", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", - "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc2b8d8", - "metadata": {}, - "outputs": [], - "source": [ - "import getpass\n", - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - "\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", - "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c2e60ad", - "metadata": {}, - "outputs": [], - "source": [ - "import shutil\n", - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.frames import load_mot_frame_image\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ] - }, - { - "cell_type": "markdown", - "id": "a54bb5ed", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc892d6", - "metadata": {}, - "outputs": [], - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "29afb2e0", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea423a9", - "metadata": {}, - "outputs": [], - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ] - }, - { - "cell_type": "markdown", - "id": "b57560b2", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f38487ea", - "metadata": {}, - "outputs": [], - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ] - }, - { - "cell_type": "markdown", - "id": "823b2696", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d09332f1", - "metadata": {}, - "outputs": [], - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ] - }, - { - "cell_type": "markdown", - "id": "ad28e88f", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6612c281", - "metadata": {}, - "outputs": [], - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ] - }, - { - "cell_type": "markdown", - "id": "b0ea623e", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ] - }, - { - "cell_type": "markdown", - "id": "43321292", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d16f6483", - "metadata": {}, - "outputs": [], - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8ee1ac84", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d448e555", - "metadata": {}, - "outputs": [], - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "8de54c38", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a4f4194", - "metadata": {}, - "outputs": [], - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", - "if COMPARE_MAX_FRAMES is not None:\n", - " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", - "\n", - "compare_fps = COMPARE_FPS\n", - "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", - "if seqinfo.is_file():\n", - " for line in seqinfo.read_text().splitlines():\n", - " if line.startswith(\"frameRate=\"):\n", - " compare_fps = int(line.split(\"=\", 1)[1])\n", - " break\n", - "\n", - "sample = load_mot_frame_image(img_dir, 1)\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", - " for frame_idx in range(1, n_frames + 1):\n", - " frame = load_mot_frame_image(img_dir, frame_idx)\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "ffmpeg = shutil.which(\"ffmpeg\")\n", - "if ffmpeg is not None:\n", - " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", - " result = subprocess.run(\n", - " [\n", - " ffmpeg,\n", - " \"-y\",\n", - " \"-i\",\n", - " str(out_path),\n", - " \"-c:v\",\n", - " \"libx264\",\n", - " \"-pix_fmt\",\n", - " \"yuv420p\",\n", - " \"-movflags\",\n", - " \"+faststart\",\n", - " \"-an\",\n", - " str(tmp),\n", - " ],\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " if result.returncode == 0:\n", - " tmp.replace(out_path)\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac864a15", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa51e6b5", - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "2d522414", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bec6c65", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "\n", + "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", + "\n", + "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "REID_BRANCH = \"feat/port-model-stack\"\n", + "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", + " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", + " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", + "\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"timm\",\n", + " \"huggingface-hub\",\n", + " \"safetensors\",\n", + " \"gdown\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " REID_REF,\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"--no-cache-dir\",\n", + " \"--force-reinstall\",\n", + " \"--no-deps\",\n", + " f\"trackers @ {TRACKERS_REF}\",\n", + " ],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"supervision\",\n", + " \"scipy\",\n", + " \"opencv-python-headless\",\n", + " \"rich\",\n", + " \"requests\",\n", + " \"pydeprecate\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + "\n", + " del TOKEN\n", + " print(\"Installed reid + trackers from git.\")\n", + "else:\n", + " print(\"Local kernel: skipping git install.\")\n", + " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", + " # Optional local editable installs:\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", + " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.frames import load_mot_frame_image\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a54bb5ed", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", + "metadata": {}, + "outputs": [], + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "29afb2e0", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", + "metadata": {}, + "outputs": [], + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b57560b2", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f38487ea", + "metadata": {}, + "outputs": [], + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "id": "823b2696", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d09332f1", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ad28e88f", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6612c281", + "metadata": {}, + "outputs": [], + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0ea623e", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ] + }, + { + "cell_type": "markdown", + "id": "43321292", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16f6483", + "metadata": {}, + "outputs": [], + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8ee1ac84", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d448e555", + "metadata": {}, + "outputs": [], + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "8de54c38", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", + "metadata": {}, + "outputs": [], + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", + "if COMPARE_MAX_FRAMES is not None:\n", + " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", + "\n", + "compare_fps = COMPARE_FPS\n", + "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", + "if seqinfo.is_file():\n", + " for line in seqinfo.read_text().splitlines():\n", + " if line.startswith(\"frameRate=\"):\n", + " compare_fps = int(line.split(\"=\", 1)[1])\n", + " break\n", + "\n", + "sample = load_mot_frame_image(img_dir, 1)\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", + " for frame_idx in range(1, n_frames + 1):\n", + " frame = load_mot_frame_image(img_dir, frame_idx)\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "ffmpeg = shutil.which(\"ffmpeg\")\n", + "if ffmpeg is not None:\n", + " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", + " result = subprocess.run(\n", + " [\n", + " ffmpeg,\n", + " \"-y\",\n", + " \"-i\",\n", + " str(out_path),\n", + " \"-c:v\",\n", + " \"libx264\",\n", + " \"-pix_fmt\",\n", + " \"yuv420p\",\n", + " \"-movflags\",\n", + " \"+faststart\",\n", + " \"-an\",\n", + " str(tmp),\n", + " ],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " if result.returncode == 0:\n", + " tmp.replace(out_path)\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ac864a15", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "aa51e6b5", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From 8683b99bcc7b7106b8f499d269de5f1a5d6532fe Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 10:54:51 -0300 Subject: [PATCH 35/54] ci: install trackers[reid] in test and integration workflows Co-authored-by: Cursor --- .github/workflows/ci-integrations.yml | 2 +- .github/workflows/ci-tests.yml | 2 +- pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml index 1def972fa..a1c060160 100644 --- a/.github/workflows/ci-integrations.yml +++ b/.github/workflows/ci-integrations.yml @@ -27,7 +27,7 @@ jobs: activate-environment: true - name: 🚀 Install Packages - run: uv sync --frozen --group dev + run: uv sync --frozen --group dev --extra reid - name: 🧪 Run Integration Tests diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 9ca4d9f38..acaf4b93a 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -30,7 +30,7 @@ jobs: prune-cache: ${{ matrix.os != 'windows-latest' }} - name: 🚀 Install Packages - run: uv sync --frozen --group dev + run: uv sync --frozen --group dev --extra reid - name: 🧪 Run the Import test run: uv run python -c "import trackers" diff --git a/pyproject.toml b/pyproject.toml index 3f8c7091b..78ca9b06c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ dependencies = [ [project.optional-dependencies] detection = ["inference-models>=0.19.0"] tune = ["optuna>=3.0.0"] -# Swap the git pin for a PyPI release (e.g. "roboflow-reid>=0.1.0,<0.2") before merge. +# Private git pin for review. Swap to PyPI before merge (test CI uses --extra reid). reid = ["roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack"] [project.scripts] From 1351cd96fe17a60d567b746d697ced57ff2aba95 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:07:32 -0300 Subject: [PATCH 36/54] refactor(reid): drop _reid seam and merge BoT-SORT ReID tests Inline the optional reid import in the CLI like detection/tune, and keep all BoT-SORT ReID coverage in one test module. Co-authored-by: Cursor --- src/trackers/_reid.py | 41 ------------------- src/trackers/scripts/track.py | 8 +--- tests/core/test_botsort_reid.py | 24 +++++++++++ tests/core/test_botsort_reid_integration.py | 45 --------------------- tests/scripts/test_track.py | 17 ++++---- 5 files changed, 33 insertions(+), 102 deletions(-) delete mode 100644 src/trackers/_reid.py delete mode 100644 tests/core/test_botsort_reid_integration.py diff --git a/src/trackers/_reid.py b/src/trackers/_reid.py deleted file mode 100644 index b94278274..000000000 --- a/src/trackers/_reid.py +++ /dev/null @@ -1,41 +0,0 @@ -# ------------------------------------------------------------------------ -# Trackers -# Copyright (c) 2026 Roboflow. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -"""Lazy boundary to the optional ``roboflow-reid`` package. - -Trackers ships only numpy-only association glue. The appearance encoder, -weights, preprocessing, and catalog live in the standalone ``reid`` package, -installed via the ``trackers[reid]`` extra. This module is the single seam that -resolves ``reid.ReIDModel`` on demand so importing trackers never pulls torch. -""" - -from __future__ import annotations - -import importlib -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from reid import ReIDModel as ReIDModel - -REID_INSTALL_HINT = ( - "ReID features require the optional `trackers[reid]` extra. Install with: pip install 'trackers[reid]'" -) - -_REID_PACKAGE = "reid" - - -def import_reid_model() -> Any: - """Return ``reid.ReIDModel``, rewriting the missing-extra error. - - Raises: - ImportError: With an install hint when ``roboflow-reid`` (or one of its - heavy dependencies) is not installed. - """ - try: - module = importlib.import_module(_REID_PACKAGE) - except ImportError as exc: - raise ImportError(REID_INSTALL_HINT) from exc - return module.ReIDModel diff --git a/src/trackers/scripts/track.py b/src/trackers/scripts/track.py index 7d97813c6..84f8673f8 100644 --- a/src/trackers/scripts/track.py +++ b/src/trackers/scripts/track.py @@ -16,7 +16,6 @@ import numpy as np import supervision as sv -from trackers import _reid as reid_provider from trackers import frames_from_source from trackers.core.base import BaseTracker from trackers.io.mot import _mot_frame_to_detections, _MOTOutput, load_mot_file @@ -687,7 +686,7 @@ def _apply_reid_tracker_params( ) try: - ReIDModel = reid_provider.import_reid_model() + from reid import ReIDModel except ImportError: return params, ( "Error: ReID tracking requires the optional `trackers[reid]` extra.\n" @@ -707,11 +706,6 @@ def _apply_reid_tracker_params( reid_model = ReIDModel.from_pretrained(**load_kwargs) except KeyboardInterrupt: raise - except ImportError: - return params, ( - "Error: ReID tracking requires the optional `trackers[reid]` extra.\n" - "Install with: pip install 'trackers[reid]'" - ) except (OSError, ValueError, RuntimeError) as exc: return params, f"Error: Failed to load ReID model: {exc}" diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index e498c9937..f0071c170 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -307,3 +307,27 @@ def test_low_confidence_stage_does_not_update_feature_bank(self) -> None: after = bank.feature assert after is not None np.testing.assert_allclose(before, after) + + @pytest.mark.integration + def test_real_reid_model_runs_over_frames(self) -> None: + """Smoke the ``trackers`` → ``reid`` boundary with a real encoder.""" + import reid + + reid_model = reid.ReIDModel.from_pretrained(architecture="osnet_x0_25", device="cpu") + tracker = BoTSORTTracker(enable_cmc=False, reid_model=reid_model) + + rng = np.random.default_rng(0) + box = np.array([30.0, 30.0, 70.0, 90.0], dtype=np.float32) + for _ in range(3): + frame = rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + detections = sv.Detections( + xyxy=box[None, :].copy(), + confidence=np.array([0.9], dtype=np.float32), + ) + result = tracker.update(detections, frame=frame) + assert result.tracker_id is not None + box = box + np.array([2.0, 1.0, 2.0, 1.0], dtype=np.float32) + + assert len(tracker.tracks) == 1 + bank = tracker.tracks[0].feature_bank + assert bank is not None and bank.is_initialized diff --git a/tests/core/test_botsort_reid_integration.py b/tests/core/test_botsort_reid_integration.py deleted file mode 100644 index 1f49ac1ae..000000000 --- a/tests/core/test_botsort_reid_integration.py +++ /dev/null @@ -1,45 +0,0 @@ -# ------------------------------------------------------------------------ -# Trackers -# Copyright (c) 2026 Roboflow. All Rights Reserved. -# Licensed under the Apache License, Version 2.0 [see LICENSE for details] -# ------------------------------------------------------------------------ - -"""End-to-end smoke test for BoT-SORT with a real ``reid`` encoder. - -Exercises the ``trackers`` -> ``reid`` boundary once, without re-testing the -``reid`` internals (those live in the ``reid`` package's own suite). Requires -the ``trackers[reid]`` extra; skipped when ``roboflow-reid`` is not installed. -""" - -from __future__ import annotations - -import numpy as np -import pytest -import supervision as sv - -reid = pytest.importorskip("reid", reason="requires the optional trackers[reid] extra") - -from trackers.core.botsort.tracker import BoTSORTTracker # noqa: E402 - - -@pytest.mark.integration -def test_botsort_with_real_reid_model_runs_over_frames() -> None: - reid_model = reid.ReIDModel.from_pretrained(architecture="osnet_x0_25", device="cpu") - - tracker = BoTSORTTracker(enable_cmc=False, reid_model=reid_model) - - rng = np.random.default_rng(0) - box = np.array([30.0, 30.0, 70.0, 90.0], dtype=np.float32) - for _ in range(3): - frame = rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) - detections = sv.Detections( - xyxy=box[None, :].copy(), - confidence=np.array([0.9], dtype=np.float32), - ) - result = tracker.update(detections, frame=frame) - assert result.tracker_id is not None - box = box + np.array([2.0, 1.0, 2.0, 1.0], dtype=np.float32) - - assert len(tracker.tracks) == 1 - bank = tracker.tracks[0].feature_bank - assert bank is not None and bank.is_initialized diff --git a/tests/scripts/test_track.py b/tests/scripts/test_track.py index b4c536910..cd1b8732d 100644 --- a/tests/scripts/test_track.py +++ b/tests/scripts/test_track.py @@ -218,11 +218,7 @@ def test_model_source_implies_enable(self) -> None: ) assert _reid_requested(args) - def test_requires_source_before_load(self, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr( - "trackers.scripts.track.reid_provider.import_reid_model", - lambda: pytest.fail("model provider should not be called"), - ) + def test_requires_source_before_load(self) -> None: args = argparse.Namespace( tracker_reid_enable=True, tracker_reid_model=None, @@ -235,10 +231,13 @@ def test_requires_source_before_load(self, monkeypatch: pytest.MonkeyPatch) -> N assert params == {} def test_passes_architecture(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: - monkeypatch.setattr( - "trackers.scripts.track.reid_provider.import_reid_model", - lambda: _FakeReIDModel, - ) + import sys + from types import ModuleType + + fake_reid = ModuleType("reid") + fake_reid.ReIDModel = _FakeReIDModel + monkeypatch.setitem(sys.modules, "reid", fake_reid) + weights = tmp_path / "weights.pth" weights.touch() args = argparse.Namespace( From c6696a186db2374804bcea2bfd02d76d05bc460d Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:14:33 -0300 Subject: [PATCH 37/54] fix(botsort): exclude reid_model from CLI param reflection only Co-authored-by: Cursor --- src/trackers/core/base.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/trackers/core/base.py b/src/trackers/core/base.py index f46a06cfc..c1f855b25 100644 --- a/src/trackers/core/base.py +++ b/src/trackers/core/base.py @@ -36,27 +36,21 @@ class ParameterInfo: description: str -# Constructor arguments that are injected programmatically and must not be -# surfaced as CLI flags (e.g. an instantiated ReID model or IoU metric object). -_CLI_EXCLUDED_PARAMS = frozenset({"reid_model"}) - - class TrackerParameters(dict[str, ParameterInfo]): - """Tracker parameter mapping with CLI-only filtering for injection-only args.""" + """Tracker parameter mapping with CLI-only filtering for IoU metrics.""" def items(self) -> Iterator[tuple[str, ParameterInfo]]: # type: ignore[override] try: from trackers.utils.iou import BaseIoU except ImportError: - base_iou_type: type | None = None - else: - base_iou_type = BaseIoU + yield from super().items() + return for name, param_info in super().items(): - if name in _CLI_EXCLUDED_PARAMS: + if name == "reid_model": continue param_type = param_info.param_type - if base_iou_type is not None and isinstance(param_type, type) and issubclass(param_type, base_iou_type): + if isinstance(param_type, type) and issubclass(param_type, BaseIoU): continue yield name, param_info From a17749a68f518c81af2e8eea46f9b2ac35fd1dfe Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:19:17 -0300 Subject: [PATCH 38/54] docs(reid): cite BoT-SORT for FeatureBank L2/EMA policy Co-authored-by: Cursor --- docs/api/reid.md | 9 +++++++++ docs/trackers/botsort.md | 2 +- src/trackers/core/reid/appearance.py | 7 ++++++- src/trackers/core/reid/feature_bank.py | 14 +++++++++----- tests/core/test_botsort_reid.py | 6 +++--- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/docs/api/reid.md b/docs/api/reid.md index 39e9e23d2..17e1e0626 100644 --- a/docs/api/reid.md +++ b/docs/api/reid.md @@ -64,10 +64,19 @@ protocol without the model stack. ## Feature bank +Per-track EMA of appearance embeddings. L2-normalize before and after the EMA +blend, following BoT-SORT +[`STrack.update_features`](https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py). +The standalone `reid` package leaves embeddings raw at extract time and +normalizes only for cosine distance; this bank is BoT-SORT association policy. + ::: trackers.core.reid.feature_bank.FeatureBank ## Appearance +Cosine similarity with L2 on both sides at distance time (same as `reid` +gallery eval for the cosine metric). + ::: trackers.core.reid.appearance.appearance_similarity ::: trackers.core.reid.appearance.extract_detection_embeddings diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 4d51a6558..7e8ae0e6a 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -78,7 +78,7 @@ on MOT17 val. | Parameter | Default | Purpose | | ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature (BoT-SORT `STrack.update_features`: L2 in, EMA, L2 out). Higher retains more history. | | `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | | `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | diff --git a/src/trackers/core/reid/appearance.py b/src/trackers/core/reid/appearance.py index 8af520c29..51bd170ee 100644 --- a/src/trackers/core/reid/appearance.py +++ b/src/trackers/core/reid/appearance.py @@ -66,7 +66,12 @@ def appearance_similarity( track_features: Sequence[np.ndarray | None], det_embeddings: np.ndarray, ) -> np.ndarray: - """Compute cosine similarity between track and detection embeddings.""" + """Cosine similarity between track and detection embeddings. + + L2-normalizes both sides before the dot product (same place gallery eval in + ``reid`` normalizes for cosine distance). Track rows are usually already + unit-norm when they come from :class:`~trackers.core.reid.feature_bank.FeatureBank`. + """ n_tracks = len(track_features) det_embeddings = _l2_normalize_rows(_require_embedding_matrix(det_embeddings)) n_dets = det_embeddings.shape[0] diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index 6b19e38b5..2971c6e20 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -16,11 +16,15 @@ class FeatureBank: """Per-track EMA appearance embedding, kept on the unit hypersphere. - Following upstream BoT-SORT (``STrack.update_features``), every incoming - embedding is L2-normalized before it is blended, and the resulting EMA is - L2-normalized again. The stored feature is therefore always a unit vector, - so cosine similarity against it is a plain dot product. ``reid.ReIDModel`` - returns raw (unnormalized) embeddings; normalization happens here. + Matches BoT-SORT's ``STrack.update_features`` + (https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py): + L2-normalize the incoming embedding, blend with EMA momentum ``alpha``, + then L2-normalize the result again so the stored template stays unit-norm. + + That is tracker association policy, not the standalone ``reid`` package. + ``reid.ReIDModel.extract_features`` returns raw embeddings; gallery eval in + ``reid`` L2-normalizes only when computing cosine distance. Here the bank + normalizes on update so EMA is taken on the unit sphere, as in BoT-SORT. Args: alpha: EMA momentum in ``[0, 1]``. diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index f0071c170..288c0fb1e 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -76,21 +76,21 @@ def test_botsort_import_does_not_load_reid_model_stack() -> None: class TestFeatureBank: def test_first_update_normalizes_embedding(self) -> None: + # BoT-SORT STrack.update_features: L2-normalize before storage. bank = FeatureBank(alpha=0.9) bank.update(np.array([3.0, 4.0], dtype=np.float32)) feature = bank.feature assert feature is not None - # Incoming embedding is L2-normalized before storage (unit sphere). np.testing.assert_allclose(feature, [0.6, 0.8], atol=1e-6) def test_blends_on_unit_sphere(self) -> None: + # BoT-SORT: EMA on unit vectors, then L2-normalize the blend again. bank = FeatureBank(alpha=0.75) bank.update(np.array([1.0, 0.0], dtype=np.float32)) bank.update(np.array([0.0, 1.0], dtype=np.float32)) feature = bank.feature assert feature is not None - # EMA of two unit vectors, then re-normalized: 0.75*[1,0] + 0.25*[0,1] - # = [0.75, 0.25], normalized by its norm sqrt(0.625). + # 0.75*[1,0] + 0.25*[0,1] = [0.75, 0.25], then / ||.|| expected = np.array([0.75, 0.25], dtype=np.float32) expected /= np.linalg.norm(expected) np.testing.assert_allclose(feature, expected, atol=1e-6) From 2f860c852599d82c6553d408c8c02812a119811e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 14:19:37 +0000 Subject: [PATCH 39/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/trackers/botsort.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 7e8ae0e6a..d91abba2e 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -78,7 +78,7 @@ on MOT17 val. | Parameter | Default | Purpose | | ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature (BoT-SORT `STrack.update_features`: L2 in, EMA, L2 out). Higher retains more history. | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature (BoT-SORT `STrack.update_features`: L2 in, EMA, L2 out). Higher retains more history. | | `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | | `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | From 1d5b0b183e16c37dbddd1a2b04dca3a5b8312986 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:23:01 -0300 Subject: [PATCH 40/54] docs(reid): trim FeatureBank attribution prose Co-authored-by: Cursor --- docs/api/reid.md | 5 ----- docs/trackers/botsort.md | 2 +- src/trackers/core/reid/appearance.py | 7 +------ src/trackers/core/reid/feature_bank.py | 5 ----- 4 files changed, 2 insertions(+), 17 deletions(-) diff --git a/docs/api/reid.md b/docs/api/reid.md index 17e1e0626..9ccfa6931 100644 --- a/docs/api/reid.md +++ b/docs/api/reid.md @@ -67,16 +67,11 @@ protocol without the model stack. Per-track EMA of appearance embeddings. L2-normalize before and after the EMA blend, following BoT-SORT [`STrack.update_features`](https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py). -The standalone `reid` package leaves embeddings raw at extract time and -normalizes only for cosine distance; this bank is BoT-SORT association policy. ::: trackers.core.reid.feature_bank.FeatureBank ## Appearance -Cosine similarity with L2 on both sides at distance time (same as `reid` -gallery eval for the cosine metric). - ::: trackers.core.reid.appearance.appearance_similarity ::: trackers.core.reid.appearance.extract_detection_embeddings diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index d91abba2e..4d51a6558 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -78,7 +78,7 @@ on MOT17 val. | Parameter | Default | Purpose | | ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature (BoT-SORT `STrack.update_features`: L2 in, EMA, L2 out). Higher retains more history. | +| `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | | `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | | `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | diff --git a/src/trackers/core/reid/appearance.py b/src/trackers/core/reid/appearance.py index 51bd170ee..7341058a7 100644 --- a/src/trackers/core/reid/appearance.py +++ b/src/trackers/core/reid/appearance.py @@ -66,12 +66,7 @@ def appearance_similarity( track_features: Sequence[np.ndarray | None], det_embeddings: np.ndarray, ) -> np.ndarray: - """Cosine similarity between track and detection embeddings. - - L2-normalizes both sides before the dot product (same place gallery eval in - ``reid`` normalizes for cosine distance). Track rows are usually already - unit-norm when they come from :class:`~trackers.core.reid.feature_bank.FeatureBank`. - """ + """Cosine similarity between track and detection embeddings.""" n_tracks = len(track_features) det_embeddings = _l2_normalize_rows(_require_embedding_matrix(det_embeddings)) n_dets = det_embeddings.shape[0] diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index 2971c6e20..24dc92dfc 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -21,11 +21,6 @@ class FeatureBank: L2-normalize the incoming embedding, blend with EMA momentum ``alpha``, then L2-normalize the result again so the stored template stays unit-norm. - That is tracker association policy, not the standalone ``reid`` package. - ``reid.ReIDModel.extract_features`` returns raw embeddings; gallery eval in - ``reid`` L2-normalizes only when computing cosine distance. Here the bank - normalizes on update so EMA is taken on the unit sphere, as in BoT-SORT. - Args: alpha: EMA momentum in ``[0, 1]``. """ From 8d4e63b13ebc544578becd473859bfcd454874bd Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:36:06 -0300 Subject: [PATCH 41/54] test(reid): clarify BoT-SORT fusion and embedding extraction tests Co-authored-by: Cursor --- tests/core/test_botsort_reid.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index 288c0fb1e..62ab3c31d 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -171,7 +171,8 @@ def test_incompatible_track_dimensions_raise(self) -> None: np.array([[1.0, 0.0]], dtype=np.float32), ) - def test_extraction_rejects_wrong_row_count(self) -> None: + def test_extract_detection_embeddings_requires_one_row_per_box(self) -> None: + # Encoder must return embeddings.shape[0] == len(boxes). class _WrongLengthEncoder: def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: return np.empty((0, 4), dtype=np.float32) @@ -186,34 +187,42 @@ def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.n class TestFuseBotsortReidAssociation: def test_appearance_can_win_when_proximity_passes(self) -> None: + # Standard IoU 0.7 clears the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # so a strong appearance score can beat the weaker IoU score (0.63 → 0.9). fused = fuse_botsort_reid_association( - np.array([[0.63]], dtype=np.float32), - np.array([[0.8]], dtype=np.float32), + iou_similarity_fused=np.array([[0.63]], dtype=np.float32), + appearance_similarity=np.array([[0.8]], dtype=np.float32), proximity_iou_similarity=np.array([[0.7]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, ) assert fused[0, 0] == pytest.approx(0.9) - def test_low_proximity_zeros_appearance(self) -> None: + def test_low_proximity_ignores_appearance(self) -> None: + # Standard IoU 0.4 fails the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # so appearance is discarded even though it is strong (0.9). Score stays IoU-only. + iou_only = np.array([[0.36]], dtype=np.float32) fused = fuse_botsort_reid_association( - np.array([[0.36]], dtype=np.float32), - np.array([[0.9]], dtype=np.float32), + iou_similarity_fused=iou_only, + appearance_similarity=np.array([[0.9]], dtype=np.float32), proximity_iou_similarity=np.array([[0.4]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, ) - assert fused[0, 0] == pytest.approx(0.36) + assert fused[0, 0] == pytest.approx(float(iou_only[0, 0])) def test_proximity_uses_standard_iou_not_giou(self) -> None: + # Association score uses a high GIoU-like value (0.80), but standard IoU is + # only 0.35 and fails the proximity gate, so appearance must not be used. + association_iou = np.array([[0.80]], dtype=np.float32) fused = fuse_botsort_reid_association( - np.array([[0.80]], dtype=np.float32), - np.array([[0.95]], dtype=np.float32), + iou_similarity_fused=association_iou, + appearance_similarity=np.array([[0.95]], dtype=np.float32), proximity_iou_similarity=np.array([[0.35]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, ) - assert fused[0, 0] == pytest.approx(0.80) + assert fused[0, 0] == pytest.approx(float(association_iou[0, 0])) class TestBoTSORTTrackerReID: From 116bd80eff0fecf21e66cedc25afb4b3a2ab216e Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:55:16 -0300 Subject: [PATCH 42/54] refactor(botsort): make proximity IoU optional in ReID fusion Default proximity to the association matrix so callers only pass a separate standard-IoU gate when association uses GIoU/DIoU/CIoU. Co-authored-by: Cursor --- src/trackers/core/botsort/fusion.py | 21 ++++++++++++--------- tests/core/test_botsort_reid.py | 22 ++++++++++------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/src/trackers/core/botsort/fusion.py b/src/trackers/core/botsort/fusion.py index 1a8c53aae..80c70e288 100644 --- a/src/trackers/core/botsort/fusion.py +++ b/src/trackers/core/botsort/fusion.py @@ -18,24 +18,27 @@ def fuse_botsort_reid_association( - iou_similarity_fused: np.ndarray, + association_similarity: np.ndarray, appearance_similarity: np.ndarray, *, - proximity_iou_similarity: np.ndarray, proximity_threshold: float, appearance_threshold: float, + proximity_iou_similarity: np.ndarray | None = None, ) -> np.ndarray: """Fuse IoU and appearance the way BoT-SORT ``bot_sort.py`` does. - Computes ``min(score_fused_iou_cost, halved_appearance_cost)`` with - proximity and appearance caps, then returns the corresponding similarity - matrix (``1 - cost``). + Computes ``min(association_cost, capped_appearance_cost)`` with proximity + and appearance gates, then returns the corresponding similarity matrix + (``1 - cost``). - Proximity gating always uses *standard IoU* similarity via - ``proximity_iou_similarity``, even when association scoring uses GIoU, - DIoU, or CIoU. + ``proximity_iou_similarity`` is the standard-IoU gate (defaults to + ``association_similarity``). Pass it separately when association uses + GIoU/DIoU/CIoU so proximity still uses plain IoU. """ - d_iou = 1.0 - iou_similarity_fused + if proximity_iou_similarity is None: + proximity_iou_similarity = association_similarity + + d_iou = 1.0 - association_similarity d_iou_proximity = 1.0 - proximity_iou_similarity d_app = 0.5 * (1.0 - appearance_similarity) d_app = np.where(d_app > appearance_threshold, 1.0, d_app) diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index 62ab3c31d..cb3d0ed23 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -187,25 +187,23 @@ def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.n class TestFuseBotsortReidAssociation: def test_appearance_can_win_when_proximity_passes(self) -> None: - # Standard IoU 0.7 clears the proximity gate (needs IoU > 1 - 0.5 = 0.5), - # so a strong appearance score can beat the weaker IoU score (0.63 → 0.9). + # Association IoU 0.63 clears the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # so a strong appearance score can beat it (0.63 → 0.9). fused = fuse_botsort_reid_association( - iou_similarity_fused=np.array([[0.63]], dtype=np.float32), - appearance_similarity=np.array([[0.8]], dtype=np.float32), - proximity_iou_similarity=np.array([[0.7]], dtype=np.float32), + np.array([[0.63]], dtype=np.float32), + np.array([[0.8]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, ) assert fused[0, 0] == pytest.approx(0.9) def test_low_proximity_ignores_appearance(self) -> None: - # Standard IoU 0.4 fails the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # Association IoU 0.36 fails the proximity gate (needs IoU > 1 - 0.5 = 0.5), # so appearance is discarded even though it is strong (0.9). Score stays IoU-only. iou_only = np.array([[0.36]], dtype=np.float32) fused = fuse_botsort_reid_association( - iou_similarity_fused=iou_only, - appearance_similarity=np.array([[0.9]], dtype=np.float32), - proximity_iou_similarity=np.array([[0.4]], dtype=np.float32), + iou_only, + np.array([[0.9]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, ) @@ -216,11 +214,11 @@ def test_proximity_uses_standard_iou_not_giou(self) -> None: # only 0.35 and fails the proximity gate, so appearance must not be used. association_iou = np.array([[0.80]], dtype=np.float32) fused = fuse_botsort_reid_association( - iou_similarity_fused=association_iou, - appearance_similarity=np.array([[0.95]], dtype=np.float32), - proximity_iou_similarity=np.array([[0.35]], dtype=np.float32), + association_iou, + np.array([[0.95]], dtype=np.float32), proximity_threshold=0.5, appearance_threshold=0.25, + proximity_iou_similarity=np.array([[0.35]], dtype=np.float32), ) assert fused[0, 0] == pytest.approx(float(association_iou[0, 0])) From f55a063144415fe7cb9faaa63c94553434f4a8ca Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:55:16 -0300 Subject: [PATCH 43/54] perf(botsort): reuse raw IoU for proximity when using standard IoU Match upstream BoT-SORT: one IoU compute, proximity mask from the pre-fusion matrix, score fusion only for association. Recompute only for IoU variants. Co-authored-by: Cursor --- src/trackers/core/botsort/tracker.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index d42c4c9e8..f1e3182b0 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -305,7 +305,12 @@ def update( for t in strack_pool ] app_sim = appearance_similarity(track_feats, det_embeddings) - proximity_iou = self._get_proximity_iou_matrix(strack_pool, high_boxes) + # Proximity uses raw standard IoU (before score fusion), matching BoT-SORT. + # Reuse the association matrix when it is already plain IoU. + if isinstance(self.iou, IoU): + proximity_iou = iou_sim_raw + else: + proximity_iou = self._get_proximity_iou_matrix(strack_pool, high_boxes) similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) else: similarity_matrix = iou_sim_fused @@ -367,7 +372,10 @@ def update( for t in unconfirmed_tracks ] app_sim = appearance_similarity(track_feats, uh_embeddings) - proximity_iou = self._get_proximity_iou_matrix(unconfirmed_tracks, uh_boxes) + if isinstance(self.iou, IoU): + proximity_iou = iou_sim_raw + else: + proximity_iou = self._get_proximity_iou_matrix(unconfirmed_tracks, uh_boxes) similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) else: similarity_matrix = iou_sim_fused @@ -447,13 +455,13 @@ def _assign_track_detection( def _fuse_botsort_reid( self, - iou_similarity_fused: np.ndarray, + association_similarity: np.ndarray, appearance_similarity: np.ndarray, proximity_iou_similarity: np.ndarray, ) -> np.ndarray: """Fuse IoU and appearance using BoT-SORT ``bot_sort.py`` min-cost ReID.""" return fuse_botsort_reid_association( - iou_similarity_fused, + association_similarity, appearance_similarity, proximity_iou_similarity=proximity_iou_similarity, proximity_threshold=self.proximity_threshold, From ed74d88c2352b5d698dc1cb5796a04173d8b697a Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 11:58:50 -0300 Subject: [PATCH 44/54] refactor(botsort): drop always-on proximity IoU instance Only compute standard IoU for ReID proximity when association uses a variant metric; plain IoU reuses the association matrix. Co-authored-by: Cursor --- src/trackers/core/botsort/tracker.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index f1e3182b0..23f0bb5e4 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -159,8 +159,6 @@ def __init__( self.tracks: list[BoTSORTTracklet] = [] self.state_estimator_class = state_estimator_class self.iou = iou if iou is not None else IoU() - # Proximity gating always uses standard IoU, independent of ``iou``. - self._proximity_iou = IoU() self.frame_id: int = 0 self._reset_id_allocator() @@ -310,7 +308,7 @@ def update( if isinstance(self.iou, IoU): proximity_iou = iou_sim_raw else: - proximity_iou = self._get_proximity_iou_matrix(strack_pool, high_boxes) + proximity_iou = self._standard_iou_matrix(strack_pool, high_boxes) similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) else: similarity_matrix = iou_sim_fused @@ -375,7 +373,7 @@ def update( if isinstance(self.iou, IoU): proximity_iou = iou_sim_raw else: - proximity_iou = self._get_proximity_iou_matrix(unconfirmed_tracks, uh_boxes) + proximity_iou = self._standard_iou_matrix(unconfirmed_tracks, uh_boxes) similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) else: similarity_matrix = iou_sim_fused @@ -468,12 +466,13 @@ def _fuse_botsort_reid( appearance_threshold=self.appearance_threshold, ) - def _get_proximity_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: + def _standard_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: + """Standard IoU for ReID proximity when association uses a variant metric.""" if len(tracklets) == 0: tracklet_boxes = np.empty((0, 4)) else: tracklet_boxes = np.array([tracklet.get_state_bbox() for tracklet in tracklets]) - return self._proximity_iou.compute(tracklet_boxes, detections) + return IoU().compute(tracklet_boxes, detections) def _get_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: if len(tracklets) == 0: From 14a6d09699478807a1e436fb82f76deb8dff7f06 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 12:09:00 -0300 Subject: [PATCH 45/54] refactor(botsort): consolidate ReID association path in tracker One helper builds score-fused geometry plus optional appearance fusion, shared by first and unconfirmed association. Drop thin wrappers and redundant comments. Co-authored-by: Cursor --- docs/trackers/botsort.md | 2 +- src/trackers/core/botsort/tracker.py | 107 +++++++++++--------------- src/trackers/core/botsort/tracklet.py | 2 +- 3 files changed, 45 insertions(+), 66 deletions(-) diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 4d51a6558..e840aa54d 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -80,7 +80,7 @@ on MOT17 val. | ---------------------- | ------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `reid_ema_alpha` | 0.9 | EMA momentum for a track's appearance feature; higher retains more history. | | `appearance_threshold` | 0.25 | Appearance-distance gate (BoT-SORT paper default). Rejects matches when `0.5 * (1 - cos_sim)` exceeds this value. The MOT17 eval notebook uses `0.2` per the re-ID study Table 8. | -| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU > `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | +| `proximity_threshold` | 0.5 | Standard-IoU gate applied before appearance is used (requires IoU ≥ `1 - proximity_threshold`), computed from true IoU even when `iou` is GIoU/DIoU/CIoU. | ## Run on video, webcam, or RTSP stream diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index 23f0bb5e4..1f78893db 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -38,11 +38,11 @@ class BoTSORTTracker(BaseTracker): 3) Split tracks into confirmed, unconfirmed, and lost 4) Apply camera motion compensation to predicted tracks 5) Associate high-confidence detections to confirmed + lost tracks - (IoU fused with detection scores + assignment) + (IoU fused with detection scores, optional appearance) 6) Associate low-confidence detections to remaining tracks - (excluding lost tracks) + (excluding lost tracks; geometry only) 7) Match remaining unmatched high-confidence detections to unconfirmed tracks - and remove unmatched unconfirmed tracks + (optional appearance) and remove unmatched unconfirmed tracks 8) Spawn new tracks from still unmatched high-confidence detections (instantly activated on the very first frame) 9) Remove tracks that have been lost for too long @@ -98,7 +98,8 @@ class BoTSORTTracker(BaseTracker): Default ``0.25`` (BoT-SORT ``appearance_thresh``). proximity_threshold: Standard-IoU distance gate applied before appearance is used. Computed from true IoU even when ``iou`` is GIoU/DIoU/CIoU. - Default ``0.5`` (BoT-SORT ``proximity_thresh``; requires IoU > 0.5). + Default ``0.5`` (BoT-SORT ``proximity_thresh``; requires + ``IoU >= 1 - proximity_threshold``). Notes: - Positive `maximum_frames_without_update` values are scaled by @@ -279,9 +280,6 @@ def update( H = self.cmc.estimate(frame, mask_boxes) CMC.apply_batch(H, self.tracks) - # Appearance: extract embeddings once for all high-confidence detections. - # When reid_model is None this block is skipped entirely and behaviour - # is identical to the geometry-only baseline. det_embeddings: np.ndarray | None = None if self.reid_model is not None: if frame is None: @@ -293,25 +291,9 @@ def update( # Lost tracks are included here (following the original ByteTrack), and # IoU is fused with detection scores. strack_pool = confirmed_tracks + lost_tracks - iou_matrix = self._get_iou_matrix(strack_pool, high_boxes) - iou_sim_raw = self.iou.normalize_for_fusion(iou_matrix) - iou_sim_fused = _fuse_score(iou_sim_raw, high_scores) - - if det_embeddings is not None and len(strack_pool) > 0: - track_feats = [ - t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None - for t in strack_pool - ] - app_sim = appearance_similarity(track_feats, det_embeddings) - # Proximity uses raw standard IoU (before score fusion), matching BoT-SORT. - # Reuse the association matrix when it is already plain IoU. - if isinstance(self.iou, IoU): - proximity_iou = iou_sim_raw - else: - proximity_iou = self._standard_iou_matrix(strack_pool, high_boxes) - similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) - else: - similarity_matrix = iou_sim_fused + similarity_matrix = self._association_similarity( + strack_pool, high_boxes, high_scores, det_embeddings + ) matched, unmatched_pool, unmatched_high = self._get_associated_indices( similarity_matrix, self.minimum_iou_threshold_first_assoc @@ -329,7 +311,6 @@ def update( # Step 2: associate low-confidence detections to remaining *tracked* tracks # only (excluding lost tracks, following the original ByteTrack). - # No score fusing or ReID in second association (upstream bot_sort.py). remaining_tracked = [strack_pool[i] for i in unmatched_pool if strack_pool[i].time_since_update == 1] iou_matrix = self._get_iou_matrix(remaining_tracked, low_boxes) matched, _, unmatched_low = self._get_associated_indices(iou_matrix, self.minimum_iou_threshold_second_assoc) @@ -358,25 +339,12 @@ def update( if len(unconfirmed_tracks) > 0 and len(unmatched_high_list) > 0: uh_boxes = high_boxes[unmatched_high_list] uh_scores = high_scores[unmatched_high_list] - - iou_matrix = self._get_iou_matrix(unconfirmed_tracks, uh_boxes) - iou_sim_raw = self.iou.normalize_for_fusion(iou_matrix) - iou_sim_fused = _fuse_score(iou_sim_raw, uh_scores) - - if det_embeddings is not None: - uh_embeddings = det_embeddings[unmatched_high_list] - track_feats = [ - t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None - for t in unconfirmed_tracks - ] - app_sim = appearance_similarity(track_feats, uh_embeddings) - if isinstance(self.iou, IoU): - proximity_iou = iou_sim_raw - else: - proximity_iou = self._standard_iou_matrix(unconfirmed_tracks, uh_boxes) - similarity_matrix = self._fuse_botsort_reid(iou_sim_fused, app_sim, proximity_iou) - else: - similarity_matrix = iou_sim_fused + uh_embeddings = ( + det_embeddings[unmatched_high_list] if det_embeddings is not None else None + ) + similarity_matrix = self._association_similarity( + unconfirmed_tracks, uh_boxes, uh_scores, uh_embeddings + ) matched_uc, unmatched_uc_indices, remaining_uh = self._get_associated_indices( similarity_matrix, self.minimum_iou_threshold_unconfirmed_assoc @@ -451,35 +419,46 @@ def _assign_track_detection( out_det_indices.append(global_det_index) out_tracker_ids.append(track.tracker_id) - def _fuse_botsort_reid( + def _association_similarity( self, - association_similarity: np.ndarray, - appearance_similarity: np.ndarray, - proximity_iou_similarity: np.ndarray, + tracklets: list[BoTSORTTracklet], + boxes: np.ndarray, + scores: np.ndarray, + embeddings: np.ndarray | None, ) -> np.ndarray: - """Fuse IoU and appearance using BoT-SORT ``bot_sort.py`` min-cost ReID.""" + """Score-fused association similarity, with optional BoT-SORT ReID fusion.""" + iou_sim_raw = self.iou.normalize_for_fusion(self._get_iou_matrix(tracklets, boxes)) + iou_sim_fused = _fuse_score(iou_sim_raw, scores) + if embeddings is None or len(tracklets) == 0: + return iou_sim_fused + + track_feats = [ + t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None + for t in tracklets + ] + proximity_iou = ( + iou_sim_raw if isinstance(self.iou, IoU) else self._get_iou_matrix(tracklets, boxes, metric=IoU()) + ) return fuse_botsort_reid_association( - association_similarity, - appearance_similarity, - proximity_iou_similarity=proximity_iou_similarity, + iou_sim_fused, + appearance_similarity(track_feats, embeddings), + proximity_iou_similarity=proximity_iou, proximity_threshold=self.proximity_threshold, appearance_threshold=self.appearance_threshold, ) - def _standard_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: - """Standard IoU for ReID proximity when association uses a variant metric.""" - if len(tracklets) == 0: - tracklet_boxes = np.empty((0, 4)) - else: - tracklet_boxes = np.array([tracklet.get_state_bbox() for tracklet in tracklets]) - return IoU().compute(tracklet_boxes, detections) - - def _get_iou_matrix(self, tracklets: list[BoTSORTTracklet], detections: np.ndarray) -> np.ndarray: + def _get_iou_matrix( + self, + tracklets: list[BoTSORTTracklet], + detections: np.ndarray, + *, + metric: BaseIoU | None = None, + ) -> np.ndarray: if len(tracklets) == 0: tracklet_boxes = np.empty((0, 4)) else: tracklet_boxes = np.array([tracklet.get_state_bbox() for tracklet in tracklets]) - return self.iou.compute(tracklet_boxes, detections) + return (metric or self.iou).compute(tracklet_boxes, detections) def _get_associated_indices( self, diff --git a/src/trackers/core/botsort/tracklet.py b/src/trackers/core/botsort/tracklet.py index 0d5dd7cba..7ebf11071 100644 --- a/src/trackers/core/botsort/tracklet.py +++ b/src/trackers/core/botsort/tracklet.py @@ -58,7 +58,7 @@ def __init__( # Count initial bbox as first successful update so that # number_of_successful_updates starts at 1. self.number_of_successful_updates = 1 - # Optional appearance feature bank, populated by BoTSORTTracker. + # Optional appearance feature bank, set by BoTSORTTracker when ReID is enabled. self.feature_bank: FeatureBank | None = None def _configure_initial_noise(self, bbox: np.ndarray) -> None: From 9b9116b715e61a7a23fae976b91f066b8d38fb22 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:09:21 +0000 Subject: [PATCH 46/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/trackers/core/botsort/tracker.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index 1f78893db..6b3d2efe0 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -291,9 +291,7 @@ def update( # Lost tracks are included here (following the original ByteTrack), and # IoU is fused with detection scores. strack_pool = confirmed_tracks + lost_tracks - similarity_matrix = self._association_similarity( - strack_pool, high_boxes, high_scores, det_embeddings - ) + similarity_matrix = self._association_similarity(strack_pool, high_boxes, high_scores, det_embeddings) matched, unmatched_pool, unmatched_high = self._get_associated_indices( similarity_matrix, self.minimum_iou_threshold_first_assoc @@ -339,12 +337,8 @@ def update( if len(unconfirmed_tracks) > 0 and len(unmatched_high_list) > 0: uh_boxes = high_boxes[unmatched_high_list] uh_scores = high_scores[unmatched_high_list] - uh_embeddings = ( - det_embeddings[unmatched_high_list] if det_embeddings is not None else None - ) - similarity_matrix = self._association_similarity( - unconfirmed_tracks, uh_boxes, uh_scores, uh_embeddings - ) + uh_embeddings = det_embeddings[unmatched_high_list] if det_embeddings is not None else None + similarity_matrix = self._association_similarity(unconfirmed_tracks, uh_boxes, uh_scores, uh_embeddings) matched_uc, unmatched_uc_indices, remaining_uh = self._get_associated_indices( similarity_matrix, self.minimum_iou_threshold_unconfirmed_assoc From 36e4248f8229831aef133a87563dba0f9d5bcd5d Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Wed, 22 Jul 2026 12:16:36 -0300 Subject: [PATCH 47/54] refactor(cli): single path for ReID validation and model load Drop the duplicate prerequisite helper and reject --tracker.reid.architecture without --tracker.reid.model so bare architecture cannot load random weights. Co-authored-by: Cursor --- src/trackers/scripts/track.py | 25 ++++++------------------- tests/scripts/test_track.py | 12 ++++++++++++ 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/trackers/scripts/track.py b/src/trackers/scripts/track.py index 84f8673f8..28268afd9 100644 --- a/src/trackers/scripts/track.py +++ b/src/trackers/scripts/track.py @@ -348,11 +348,6 @@ def run_track(args: argparse.Namespace) -> int: track_id_filter = _resolve_track_id_filter(args.track_ids) - reid_error = _validate_reid_cli_prerequisites(args) - if reid_error is not None: - print(reid_error, file=sys.stderr) - return 1 - # Create tracker tracker_params = _extract_tracker_params(args.tracker, args) tracker_params, reid_error = _apply_reid_tracker_params(args.tracker, args, tracker_params) @@ -653,20 +648,6 @@ def _reid_requested(args: argparse.Namespace) -> bool: return bool(getattr(args, "tracker_reid_enable", False)) or getattr(args, "tracker_reid_model", None) is not None -def _validate_reid_cli_prerequisites(args: argparse.Namespace) -> str | None: - """Validate ReID CLI options before loading any checkpoint.""" - if not _reid_requested(args): - return None - if args.tracker != "botsort": - return f"Error: --tracker.reid.* options apply only to --tracker botsort, got {args.tracker!r}." - if args.source is None: - return ( - "Error: ReID-enabled BoT-SORT requires --source (video/webcam/images) " - "so appearance embeddings can be extracted from frames." - ) - return None - - def _apply_reid_tracker_params( tracker_id: str, args: argparse.Namespace, @@ -696,6 +677,12 @@ def _apply_reid_tracker_params( device = getattr(args, "tracker_reid_device", DEFAULT_DEVICE) model_source = getattr(args, "tracker_reid_model", None) architecture = getattr(args, "tracker_reid_architecture", None) + if architecture is not None and model_source is None: + return params, ( + "Error: --tracker.reid.architecture requires --tracker.reid.model " + "(bare weights need a checkpoint path)." + ) + load_kwargs: dict[str, object] = {"device": device} if model_source is not None: load_kwargs["source"] = model_source diff --git a/tests/scripts/test_track.py b/tests/scripts/test_track.py index cd1b8732d..739197cfa 100644 --- a/tests/scripts/test_track.py +++ b/tests/scripts/test_track.py @@ -264,6 +264,18 @@ def test_rejects_non_botsort_tracker(self) -> None: _, error = _apply_reid_tracker_params("bytetrack", args, {}) assert error is not None and "botsort" in error + def test_architecture_requires_model(self) -> None: + args = argparse.Namespace( + tracker_reid_enable=True, + tracker_reid_model=None, + tracker_reid_device="cpu", + tracker_reid_architecture="osnet_x0_25", + source="video.mp4", + ) + params, error = _apply_reid_tracker_params("botsort", args, {}) + assert error is not None and "--tracker.reid.model" in error + assert params == {} + def test_help_lists_reid_flags(self) -> None: parser = argparse.ArgumentParser() subparsers = parser.add_subparsers() From 2f461540b78adcf67c0f1bd0dff4aea4a79cf4b5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:17:03 +0000 Subject: [PATCH 48/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/trackers/scripts/track.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/trackers/scripts/track.py b/src/trackers/scripts/track.py index 28268afd9..414b5f921 100644 --- a/src/trackers/scripts/track.py +++ b/src/trackers/scripts/track.py @@ -679,8 +679,7 @@ def _apply_reid_tracker_params( architecture = getattr(args, "tracker_reid_architecture", None) if architecture is not None and model_source is None: return params, ( - "Error: --tracker.reid.architecture requires --tracker.reid.model " - "(bare weights need a checkpoint path)." + "Error: --tracker.reid.architecture requires --tracker.reid.model (bare weights need a checkpoint path)." ) load_kwargs: dict[str, object] = {"device": device} From 0ee51cb772861b7b626a551b74985facc5902b80 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Thu, 23 Jul 2026 17:11:35 -0300 Subject: [PATCH 49/54] fix(reid): pin standalone reid package and harden optional-extra CI Use the published package name `reid` (git@main until PyPI), keep unit tests free of `--extra reid`, and clean up ReID association glue/docs/tests for the cutover. Co-authored-by: Cursor --- .github/workflows/ci-tests.yml | 2 +- README.md | 2 +- notebooks/eval_trackers_reid.ipynb | 121 ++++++------------------- pyproject.toml | 4 +- src/trackers/core/base.py | 6 +- src/trackers/core/reid/__init__.py | 8 +- src/trackers/core/reid/feature_bank.py | 8 +- tests/core/test_botsort_reid.py | 14 ++- tests/scripts/test_track.py | 13 ++- uv.lock | 38 ++++---- 10 files changed, 83 insertions(+), 133 deletions(-) diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index acaf4b93a..9ca4d9f38 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -30,7 +30,7 @@ jobs: prune-cache: ${{ matrix.os != 'windows-latest' }} - name: 🚀 Install Packages - run: uv sync --frozen --group dev --extra reid + run: uv sync --frozen --group dev - name: 🧪 Run the Import test run: uv run python -c "import trackers" diff --git a/README.md b/README.md index f33ee5a23..544f3f5d3 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Keeping track of objects across video frames is one of those problems that sound - **Benchmarked across four datasets.** MOT17, SportsMOT, SoccerNet, and DanceTrack — at default parameters and after hyperparameter tuning, so you know what to expect before you deploy. - **Tunable out of the box.** Built-in Optuna-based hyperparameter search via `trackers tune` so you can optimize for your specific scene and detector. - **Camera motion compensation.** BoT-SORT handles moving cameras natively, keeping track IDs stable even when the whole frame shifts. -- **Optional appearance ReID.** BoT-SORT can fuse visual embeddings with motion for harder association scenes — install `trackers[reid]` (pulls in the standalone `roboflow-reid` package) and pass a `reid.ReIDModel` as `reid_model`. +- **Optional appearance ReID.** BoT-SORT can fuse visual embeddings with motion for harder association scenes: install `trackers[reid]` (pulls in the standalone [`reid`](https://github.com/roboflow/re-ID) package) and pass a `reid.ReIDModel` as `reid_model`. ## Install diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index 7dc91ff18..d5040d7f2 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -20,7 +20,7 @@ "\n", "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", "\n", "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" ] @@ -32,11 +32,11 @@ "source": [ "## 1. Setup\n", "\n", - "Install **trackers** (this PR branch) and **reid** (`import reid`) from git. Colab already ships CUDA PyTorch, so install both packages with `--no-deps` and pull notebook deps separately.\n", + "Install **trackers** with the ReID extra (pulls in the standalone `reid` package),\n", + "plus notebook-only deps (`matplotlib`, `scikit-learn`, `gdown`).\n", "\n", - "> **Private repos:** you need a GitHub PAT with **repo read** access (classic token scope `repo`). For the `roboflow` org, authorize SSO if prompted. The Colab install cell asks for your token via `getpass`; nothing is saved in this notebook.\n", - "\n", - "**Local:** skip the install cell if your venv already has editable installs (`pip install -e .` in trackers, `pip install -e .` in re-ID). Uncomment the local lines in the cell if you want the notebook to install for you.\n" + "**Local:** skip the install cell if your venv already has `trackers[reid]`\n", + "(`pip install 'trackers[reid]'` or `uv sync --extra reid`).\n" ] }, { @@ -46,7 +46,6 @@ "metadata": {}, "outputs": [], "source": [ - "import getpass\n", "import subprocess\n", "import sys\n", "\n", @@ -57,14 +56,7 @@ "except ImportError:\n", " IN_COLAB_INSTALL = False\n", "\n", - "REID_BRANCH = \"feat/port-model-stack\"\n", - "TRACKERS_BRANCH = \"feat/core/reid-consume-reid-package\"\n", - "\n", "if IN_COLAB_INSTALL:\n", - " TOKEN = getpass.getpass(\"GitHub PAT (repo read): \")\n", - " REID_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/re-ID.git@{REID_BRANCH}\"\n", - " TRACKERS_REF = f\"git+https://x-access-token:{TOKEN}@github.com/roboflow/trackers.git@{TRACKERS_BRANCH}\"\n", - "\n", " cmds = [\n", " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", " [\n", @@ -73,60 +65,21 @@ " \"pip\",\n", " \"install\",\n", " \"-q\",\n", - " \"timm\",\n", - " \"huggingface-hub\",\n", - " \"safetensors\",\n", - " \"gdown\",\n", + " \"trackers[reid]\",\n", " \"matplotlib\",\n", " \"scikit-learn\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " REID_REF,\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"--no-cache-dir\",\n", - " \"--force-reinstall\",\n", - " \"--no-deps\",\n", - " f\"trackers @ {TRACKERS_REF}\",\n", - " ],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"supervision\",\n", - " \"scipy\",\n", - " \"opencv-python-headless\",\n", - " \"rich\",\n", - " \"requests\",\n", - " \"pydeprecate\",\n", + " \"gdown\",\n", " ],\n", " ]\n", " for cmd in cmds:\n", " subprocess.run(cmd, check=True) # noqa: S603\n", - "\n", - " del TOKEN\n", - " print(\"Installed reid + trackers from git.\")\n", + " print(\"Installed trackers[reid] and notebook deps.\")\n", "else:\n", - " print(\"Local kernel: skipping git install.\")\n", - " print(\"Ensure reid and trackers are importable (editable installs or uv sync --extra reid).\")\n", - " # Optional local editable installs:\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"..\"], check=True)\n", - " # subprocess.run([sys.executable, \"-m\", \"pip\", \"install\", \"-e\", \"../../re-ID\"], check=True)" + " print(\"Local kernel: skipping install.\")\n", + " print(\n", + " \"Ensure trackers[reid] is importable \"\n", + " \"(pip install 'trackers[reid]' or uv sync --extra reid).\"\n", + " )\n" ] }, { @@ -196,8 +149,8 @@ "\n", "| `REID_ENCODER` | Training | Input |\n", "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" ] }, { @@ -217,7 +170,7 @@ "else:\n", " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", "\n", - "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", "print(reid_model.preprocessing.describe())" ] }, @@ -682,8 +635,8 @@ "r = fmt_metrics(result_reid)\n", "print(\n", " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", ")\n", "\n", "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", @@ -711,10 +664,10 @@ ")\n", "print(\n", " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", ")\n", "\n", @@ -740,11 +693,11 @@ ")\n", "print(\n", " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", ")" ] @@ -829,7 +782,7 @@ "source": [ "### 8. Visual comparison - largest ReID gain sequence\n", "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", "(from the runs above). On Colab the mp4 is downloaded automatically.\n", "" ] @@ -898,13 +851,13 @@ " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", "\n", "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", "\n", "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", "\n", "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", @@ -950,7 +903,7 @@ "ffmpeg = shutil.which(\"ffmpeg\")\n", "if ffmpeg is not None:\n", " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", - " result = subprocess.run(\n", + " result = subprocess.run( # noqa: S603\n", " [\n", " ffmpeg,\n", " \"-y\",\n", @@ -976,22 +929,6 @@ "if IN_COLAB:\n", " files.download(str(out_path))" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "ac864a15", - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "aa51e6b5", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/pyproject.toml b/pyproject.toml index 78ca9b06c..f92696a56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,8 +47,8 @@ dependencies = [ [project.optional-dependencies] detection = ["inference-models>=0.19.0"] tune = ["optuna>=3.0.0"] -# Private git pin for review. Swap to PyPI before merge (test CI uses --extra reid). -reid = ["roboflow-reid @ git+https://github.com/roboflow/re-ID.git@feat/port-model-stack"] +# Temporary git pin until reid is on PyPI. Swap to reid>=0.1.0,<0.2 before merge. +reid = ["reid @ git+https://github.com/roboflow/re-ID.git@main"] [project.scripts] trackers = "trackers.scripts.__main__:main" diff --git a/src/trackers/core/base.py b/src/trackers/core/base.py index c1f855b25..a070fb11f 100644 --- a/src/trackers/core/base.py +++ b/src/trackers/core/base.py @@ -37,7 +37,11 @@ class ParameterInfo: class TrackerParameters(dict[str, ParameterInfo]): - """Tracker parameter mapping with CLI-only filtering for IoU metrics.""" + """Tracker parameter mapping that hides non-flaggable constructor args from the CLI. + + Omits IoU metric objects and injection-only parameters such as ``reid_model`` + from CLI flag generation while keeping them available on the tracker itself. + """ def items(self) -> Iterator[tuple[str, ParameterInfo]]: # type: ignore[override] try: diff --git a/src/trackers/core/reid/__init__.py b/src/trackers/core/reid/__init__.py index 721311dd4..c8fbe85fa 100644 --- a/src/trackers/core/reid/__init__.py +++ b/src/trackers/core/reid/__init__.py @@ -4,13 +4,7 @@ # Licensed under the Apache License, Version 2.0 [see LICENSE for details] # ------------------------------------------------------------------------ -"""NumPy-only appearance-ReID association glue. - -This package intentionally contains no model stack. The encoder, weights, -preprocessing, and gallery evaluation live in the standalone ``reid`` package -(``pip install 'trackers[reid]'``); import ``ReIDModel`` and evaluation helpers -from there. Everything exported here is importable without torch. -""" +"""Appearance-ReID association helpers for multi-object trackers.""" from __future__ import annotations diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index 24dc92dfc..6222fc362 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -10,7 +10,7 @@ import numpy as np -_NORM_EPS = 1e-12 +from trackers.core.reid.appearance import _l2_normalize class FeatureBank: @@ -70,9 +70,3 @@ def _require_embedding(embedding: np.ndarray) -> np.ndarray: if not np.all(np.isfinite(flat)): raise ValueError("embedding must contain only finite values") return flat - - -def _l2_normalize(vec: np.ndarray) -> np.ndarray: - """Return a unit-norm float32 vector (zero vectors are returned unchanged).""" - norm = float(np.linalg.norm(vec)) - return (vec / max(norm, _NORM_EPS)).astype(np.float32) diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index cb3d0ed23..eba334b67 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -65,7 +65,11 @@ def test_botsort_import_does_not_load_reid_model_stack() -> None: [ sys.executable, "-c", - ("import sys; import trackers.core.botsort.tracker; assert 'reid' not in sys.modules"), + ( + "import sys; import trackers.core.botsort.tracker; " + "assert 'reid' not in sys.modules; " + "assert 'torch' not in sys.modules" + ), ], check=False, capture_output=True, @@ -75,6 +79,8 @@ def test_botsort_import_does_not_load_reid_model_stack() -> None: class TestFeatureBank: + """Unit tests for ``FeatureBank`` L2 + EMA behavior.""" + def test_first_update_normalizes_embedding(self) -> None: # BoT-SORT STrack.update_features: L2-normalize before storage. bank = FeatureBank(alpha=0.9) @@ -122,6 +128,8 @@ def test_shape_change_raises(self) -> None: class TestAppearanceSimilarity: + """Unit tests for cosine ``appearance_similarity`` and embedding extraction.""" + def test_identical_vectors_are_one(self) -> None: similarity = appearance_similarity( [np.array([1.0, 0.0], dtype=np.float32)], @@ -186,6 +194,8 @@ def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.n class TestFuseBotsortReidAssociation: + """Unit tests for BoT-SORT IoU/appearance fusion gates.""" + def test_appearance_can_win_when_proximity_passes(self) -> None: # Association IoU 0.63 clears the proximity gate (needs IoU > 1 - 0.5 = 0.5), # so a strong appearance score can beat it (0.63 → 0.9). @@ -224,6 +234,8 @@ def test_proximity_uses_standard_iou_not_giou(self) -> None: class TestBoTSORTTrackerReID: + """Integration-style tests for BoT-SORT tracker appearance association.""" + def test_rejects_invalid_reid_ema_alpha(self) -> None: with pytest.raises(ValueError, match="reid_ema_alpha"): BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder(), reid_ema_alpha=1.5) diff --git a/tests/scripts/test_track.py b/tests/scripts/test_track.py index 739197cfa..7eac07eb6 100644 --- a/tests/scripts/test_track.py +++ b/tests/scripts/test_track.py @@ -211,6 +211,8 @@ def from_pretrained(cls, **kwargs: object) -> _FakeReIDModel: class TestReidTrackCli: + """CLI wiring for optional BoT-SORT ReID model loading.""" + def test_model_source_implies_enable(self) -> None: args = argparse.Namespace( tracker_reid_enable=False, @@ -235,7 +237,7 @@ def test_passes_architecture(self, monkeypatch: pytest.MonkeyPatch, tmp_path: Pa from types import ModuleType fake_reid = ModuleType("reid") - fake_reid.ReIDModel = _FakeReIDModel + setattr(fake_reid, "ReIDModel", _FakeReIDModel) monkeypatch.setitem(sys.modules, "reid", fake_reid) weights = tmp_path / "weights.pth" @@ -264,7 +266,14 @@ def test_rejects_non_botsort_tracker(self) -> None: _, error = _apply_reid_tracker_params("bytetrack", args, {}) assert error is not None and "botsort" in error - def test_architecture_requires_model(self) -> None: + def test_architecture_requires_model(self, monkeypatch: pytest.MonkeyPatch) -> None: + import sys + from types import ModuleType + + fake_reid = ModuleType("reid") + setattr(fake_reid, "ReIDModel", object) + monkeypatch.setitem(sys.modules, "reid", fake_reid) + args = argparse.Namespace( tracker_reid_enable=True, tracker_reid_model=None, diff --git a/uv.lock b/uv.lock index 2c49af460..fdb9024ae 100644 --- a/uv.lock +++ b/uv.lock @@ -3300,6 +3300,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/e4/a3b9480c78cf8ee86626cb06f8d931d74d775897d44201ccb813097ae697/regex-2026.1.15-cp314-cp314t-win_arm64.whl", hash = "sha256:ca89c5e596fc05b015f27561b3793dc2fa0917ea0d7507eebb448efd35274a70", size = 274837, upload-time = "2026-01-14T23:17:23.146Z" }, ] +[[package]] +name = "reid" +version = "0.1.0" +source = { git = "https://github.com/roboflow/re-ID.git?rev=main#d88d77cd7407d3a0158fbcff79eafa4e060234ea" } +dependencies = [ + { name = "gdown" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "safetensors" }, + { name = "supervision" }, + { name = "timm" }, + { name = "torch" }, + { name = "torchvision" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -3429,23 +3446,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/25/b208c5683343959b670dc001595f2f3737e051da617f66c31f7c4fa93abc/rich-14.3.3-py3-none-any.whl", hash = "sha256:793431c1f8619afa7d3b52b2cdec859562b950ea0d4b6b505397612db8d5362d", size = 310458, upload-time = "2026-02-19T17:23:13.732Z" }, ] -[[package]] -name = "roboflow-reid" -version = "0.1.0" -source = { git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack#b6da83541adfc7a93eb962efc85860eb3c0e0583" } -dependencies = [ - { name = "gdown" }, - { name = "huggingface-hub" }, - { name = "numpy" }, - { name = "opencv-python" }, - { name = "pillow" }, - { name = "safetensors" }, - { name = "supervision" }, - { name = "timm" }, - { name = "torch" }, - { name = "torchvision" }, -] - [[package]] name = "safetensors" version = "0.7.0" @@ -4212,7 +4212,7 @@ detection = [ { name = "inference-models" }, ] reid = [ - { name = "roboflow-reid" }, + { name = "reid" }, ] tune = [ { name = "optuna" }, @@ -4252,9 +4252,9 @@ requires-dist = [ { name = "opencv-python", specifier = ">=4.8.0" }, { name = "optuna", marker = "extra == 'tune'", specifier = ">=3.0.0" }, { name = "pydeprecate", specifier = ">=0.7.0" }, + { name = "reid", marker = "extra == 'reid'", git = "https://github.com/roboflow/re-ID.git?rev=main" }, { name = "requests", specifier = ">=2.28.0" }, { name = "rich", specifier = ">=13.0.0" }, - { name = "roboflow-reid", marker = "extra == 'reid'", git = "https://github.com/roboflow/re-ID.git?rev=feat%2Fport-model-stack" }, { name = "scipy", specifier = ">=1.13.1" }, { name = "supervision", specifier = ">=0.26.1" }, ] From a2556cb43a4c40a14f62f04880048e96353933ce Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:15:23 +0000 Subject: [PATCH 50/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 1903 ++++++++++++++-------------- 1 file changed, 950 insertions(+), 953 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index d5040d7f2..e34fa1bb3 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,955 +1,952 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "2d522414", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ] - }, - { - "cell_type": "markdown", - "id": "7bec6c65", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** with the ReID extra (pulls in the standalone `reid` package),\n", - "plus notebook-only deps (`matplotlib`, `scikit-learn`, `gdown`).\n", - "\n", - "**Local:** skip the install cell if your venv already has `trackers[reid]`\n", - "(`pip install 'trackers[reid]'` or `uv sync --extra reid`).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc2b8d8", - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"trackers[reid]\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " \"gdown\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - " print(\"Installed trackers[reid] and notebook deps.\")\n", - "else:\n", - " print(\"Local kernel: skipping install.\")\n", - " print(\n", - " \"Ensure trackers[reid] is importable \"\n", - " \"(pip install 'trackers[reid]' or uv sync --extra reid).\"\n", - " )\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c2e60ad", - "metadata": {}, - "outputs": [], - "source": [ - "import shutil\n", - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.frames import load_mot_frame_image\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ] - }, - { - "cell_type": "markdown", - "id": "a54bb5ed", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc892d6", - "metadata": {}, - "outputs": [], - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "29afb2e0", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea423a9", - "metadata": {}, - "outputs": [], - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ] - }, - { - "cell_type": "markdown", - "id": "b57560b2", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f38487ea", - "metadata": {}, - "outputs": [], - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ] - }, - { - "cell_type": "markdown", - "id": "823b2696", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d09332f1", - "metadata": {}, - "outputs": [], - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ] - }, - { - "cell_type": "markdown", - "id": "ad28e88f", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6612c281", - "metadata": {}, - "outputs": [], - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ] - }, - { - "cell_type": "markdown", - "id": "b0ea623e", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ] - }, - { - "cell_type": "markdown", - "id": "43321292", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d16f6483", - "metadata": {}, - "outputs": [], - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8ee1ac84", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d448e555", - "metadata": {}, - "outputs": [], - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "8de54c38", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a4f4194", - "metadata": {}, - "outputs": [], - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", - "if COMPARE_MAX_FRAMES is not None:\n", - " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", - "\n", - "compare_fps = COMPARE_FPS\n", - "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", - "if seqinfo.is_file():\n", - " for line in seqinfo.read_text().splitlines():\n", - " if line.startswith(\"frameRate=\"):\n", - " compare_fps = int(line.split(\"=\", 1)[1])\n", - " break\n", - "\n", - "sample = load_mot_frame_image(img_dir, 1)\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", - " for frame_idx in range(1, n_frames + 1):\n", - " frame = load_mot_frame_image(img_dir, frame_idx)\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "ffmpeg = shutil.which(\"ffmpeg\")\n", - "if ffmpeg is not None:\n", - " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", - " result = subprocess.run( # noqa: S603\n", - " [\n", - " ffmpeg,\n", - " \"-y\",\n", - " \"-i\",\n", - " str(out_path),\n", - " \"-c:v\",\n", - " \"libx264\",\n", - " \"-pix_fmt\",\n", - " \"yuv420p\",\n", - " \"-movflags\",\n", - " \"+faststart\",\n", - " \"-an\",\n", - " str(tmp),\n", - " ],\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " if result.returncode == 0:\n", - " tmp.replace(out_path)\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "2d522414", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bec6c65", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** with the ReID extra (pulls in the standalone `reid` package),\n", + "plus notebook-only deps (`matplotlib`, `scikit-learn`, `gdown`).\n", + "\n", + "**Local:** skip the install cell if your venv already has `trackers[reid]`\n", + "(`pip install 'trackers[reid]'` or `uv sync --extra reid`).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"trackers[reid]\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " \"gdown\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + " print(\"Installed trackers[reid] and notebook deps.\")\n", + "else:\n", + " print(\"Local kernel: skipping install.\")\n", + " print(\"Ensure trackers[reid] is importable (pip install 'trackers[reid]' or uv sync --extra reid).\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.frames import load_mot_frame_image\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a54bb5ed", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", + "metadata": {}, + "outputs": [], + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "29afb2e0", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", + "metadata": {}, + "outputs": [], + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b57560b2", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f38487ea", + "metadata": {}, + "outputs": [], + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "id": "823b2696", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d09332f1", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ad28e88f", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6612c281", + "metadata": {}, + "outputs": [], + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0ea623e", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ] + }, + { + "cell_type": "markdown", + "id": "43321292", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16f6483", + "metadata": {}, + "outputs": [], + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8ee1ac84", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d448e555", + "metadata": {}, + "outputs": [], + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "8de54c38", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", + "metadata": {}, + "outputs": [], + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", + "if COMPARE_MAX_FRAMES is not None:\n", + " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", + "\n", + "compare_fps = COMPARE_FPS\n", + "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", + "if seqinfo.is_file():\n", + " for line in seqinfo.read_text().splitlines():\n", + " if line.startswith(\"frameRate=\"):\n", + " compare_fps = int(line.split(\"=\", 1)[1])\n", + " break\n", + "\n", + "sample = load_mot_frame_image(img_dir, 1)\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", + " for frame_idx in range(1, n_frames + 1):\n", + " frame = load_mot_frame_image(img_dir, frame_idx)\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "ffmpeg = shutil.which(\"ffmpeg\")\n", + "if ffmpeg is not None:\n", + " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", + " result = subprocess.run( # noqa: S603\n", + " [\n", + " ffmpeg,\n", + " \"-y\",\n", + " \"-i\",\n", + " str(out_path),\n", + " \"-c:v\",\n", + " \"libx264\",\n", + " \"-pix_fmt\",\n", + " \"yuv420p\",\n", + " \"-movflags\",\n", + " \"+faststart\",\n", + " \"-an\",\n", + " str(tmp),\n", + " ],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " if result.returncode == 0:\n", + " tmp.replace(out_path)\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From a7ebc4526cfa483fb585f5ce8afce08e8ea81f2b Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Thu, 23 Jul 2026 17:17:09 -0300 Subject: [PATCH 51/54] refactor(reid): slim FeatureBank and ReID CLI wiring Drop redundant embedding validation, dead reset/is_initialized API, and defensive getattr; keep device="auto" for reid to resolve. Co-authored-by: Cursor --- src/trackers/core/botsort/tracker.py | 5 +--- src/trackers/core/reid/encoder.py | 7 +----- src/trackers/core/reid/feature_bank.py | 32 ++------------------------ src/trackers/scripts/track.py | 11 ++++----- tests/core/test_botsort_reid.py | 6 ++--- 5 files changed, 12 insertions(+), 49 deletions(-) diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index 6b3d2efe0..296927f72 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -426,10 +426,7 @@ def _association_similarity( if embeddings is None or len(tracklets) == 0: return iou_sim_fused - track_feats = [ - t.feature_bank.feature if t.feature_bank is not None and t.feature_bank.is_initialized else None - for t in tracklets - ] + track_feats = [None if t.feature_bank is None else t.feature_bank.feature for t in tracklets] proximity_iou = ( iou_sim_raw if isinstance(self.iou, IoU) else self._get_iou_matrix(tracklets, boxes, metric=IoU()) ) diff --git a/src/trackers/core/reid/encoder.py b/src/trackers/core/reid/encoder.py index 7279c01ba..bfa2efc08 100644 --- a/src/trackers/core/reid/encoder.py +++ b/src/trackers/core/reid/encoder.py @@ -15,12 +15,7 @@ class ReIDEncoder(Protocol): - """Appearance encoder used for tracking association. - - Trackers only depend on ``extract_features``. ``reid.ReIDModel`` structurally - satisfies this protocol, and custom or test encoders may implement it without - depending on the full model stack. - """ + """Encoder with ``extract_features(detections, frame)``.""" def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: """Return appearance embeddings for each detection box. diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index 6222fc362..1969d34c0 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -14,16 +14,7 @@ class FeatureBank: - """Per-track EMA appearance embedding, kept on the unit hypersphere. - - Matches BoT-SORT's ``STrack.update_features`` - (https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py): - L2-normalize the incoming embedding, blend with EMA momentum ``alpha``, - then L2-normalize the result again so the stored template stays unit-norm. - - Args: - alpha: EMA momentum in ``[0, 1]``. - """ + """Per-track EMA unit embedding (L2 before and after blend).""" def __init__(self, alpha: float = 0.9) -> None: if not 0.0 <= alpha <= 1.0: @@ -36,14 +27,9 @@ def feature(self) -> np.ndarray | None: """Current stored unit embedding, or ``None`` if never updated.""" return None if self._feature is None else self._feature.copy() - @property - def is_initialized(self) -> bool: - """``True`` after the first update.""" - return self._feature is not None - def update(self, embedding: np.ndarray) -> None: """Blend an L2-normalized embedding into the stored unit feature.""" - cleaned = _l2_normalize(_require_embedding(embedding)) + cleaned = _l2_normalize(embedding) if self._feature is None: self._feature = cleaned @@ -56,17 +42,3 @@ def update(self, embedding: np.ndarray) -> None: blended = self._alpha * self._feature + (1.0 - self._alpha) * cleaned self._feature = _l2_normalize(blended) - - def reset(self) -> None: - """Clear the stored feature.""" - self._feature = None - - -def _require_embedding(embedding: np.ndarray) -> np.ndarray: - """Return a finite 1-D float32 vector.""" - flat = np.asarray(embedding, dtype=np.float32).reshape(-1) - if flat.size == 0: - raise ValueError("embedding must be non-empty") - if not np.all(np.isfinite(flat)): - raise ValueError("embedding must contain only finite values") - return flat diff --git a/src/trackers/scripts/track.py b/src/trackers/scripts/track.py index 414b5f921..5568bcd2a 100644 --- a/src/trackers/scripts/track.py +++ b/src/trackers/scripts/track.py @@ -645,7 +645,7 @@ def _run_model(model: AnyModel, frame: np.ndarray, confidence: float) -> sv.Dete def _reid_requested(args: argparse.Namespace) -> bool: - return bool(getattr(args, "tracker_reid_enable", False)) or getattr(args, "tracker_reid_model", None) is not None + return bool(args.tracker_reid_enable) or args.tracker_reid_model is not None def _apply_reid_tracker_params( @@ -660,7 +660,7 @@ def _apply_reid_tracker_params( if tracker_id != "botsort": return params, (f"Error: --tracker.reid.* options apply only to --tracker botsort, got {tracker_id!r}.") - if getattr(args, "source", None) is None: + if args.source is None: return params, ( "Error: ReID-enabled BoT-SORT requires --source (video/webcam/images) " "so appearance embeddings can be extracted from frames." @@ -674,15 +674,14 @@ def _apply_reid_tracker_params( "Install with: pip install 'trackers[reid]'" ) - device = getattr(args, "tracker_reid_device", DEFAULT_DEVICE) - model_source = getattr(args, "tracker_reid_model", None) - architecture = getattr(args, "tracker_reid_architecture", None) + model_source = args.tracker_reid_model + architecture = args.tracker_reid_architecture if architecture is not None and model_source is None: return params, ( "Error: --tracker.reid.architecture requires --tracker.reid.model (bare weights need a checkpoint path)." ) - load_kwargs: dict[str, object] = {"device": device} + load_kwargs: dict[str, object] = {"device": args.tracker_reid_device} if model_source is not None: load_kwargs["source"] = model_source if architecture is not None: diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py index eba334b67..4e899e1bb 100644 --- a/tests/core/test_botsort_reid.py +++ b/tests/core/test_botsort_reid.py @@ -113,7 +113,7 @@ def test_non_finite_embedding_raises(self) -> None: bank = FeatureBank() with pytest.raises(ValueError, match="finite"): bank.update(np.array([1.0, np.nan], dtype=np.float32)) - assert not bank.is_initialized + assert bank.feature is None def test_shape_change_raises(self) -> None: bank = FeatureBank() @@ -249,7 +249,7 @@ def test_feature_bank_initializes_on_spawn(self) -> None: tracker = BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder()) tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame()) bank = tracker.tracks[0].feature_bank - assert bank is not None and bank.is_initialized + assert bank is not None and bank.feature is not None def test_appearance_changes_assignment_vs_geometry_only(self) -> None: identity = _norm(np.array([1.0, 0.0, 0.0, 0.0])) @@ -349,4 +349,4 @@ def test_real_reid_model_runs_over_frames(self) -> None: assert len(tracker.tracks) == 1 bank = tracker.tracks[0].feature_bank - assert bank is not None and bank.is_initialized + assert bank is not None and bank.feature is not None From 68ee592b0920368591aca701c26e943fa12ae70a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:19:23 +0000 Subject: [PATCH 52/54] =?UTF-8?q?fix(pre=5Fcommit):=20=F0=9F=8E=A8=20auto?= =?UTF-8?q?=20format=20pre-commit=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- notebooks/eval_trackers_reid.ipynb | 1900 ++++++++++++++-------------- 1 file changed, 950 insertions(+), 950 deletions(-) diff --git a/notebooks/eval_trackers_reid.ipynb b/notebooks/eval_trackers_reid.ipynb index e34fa1bb3..fb829f9cc 100644 --- a/notebooks/eval_trackers_reid.ipynb +++ b/notebooks/eval_trackers_reid.ipynb @@ -1,952 +1,952 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "2d522414", - "metadata": {}, - "source": [ - "# Tracker ReID evaluation on MOT17 val\n", - "\n", - "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", - "\n", - "| Config | Tracker | CMC | ReID | Fusion |\n", - "|---|---|---|---|---|\n", - "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", - "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", - "\n", - "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", - "\n", - "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", - "\n", - "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", - "\n", - "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest ΔHOTA (Colab auto-download).\n", - "\n", - "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" - ] - }, - { - "cell_type": "markdown", - "id": "7bec6c65", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "Install **trackers** with the ReID extra (pulls in the standalone `reid` package),\n", - "plus notebook-only deps (`matplotlib`, `scikit-learn`, `gdown`).\n", - "\n", - "**Local:** skip the install cell if your venv already has `trackers[reid]`\n", - "(`pip install 'trackers[reid]'` or `uv sync --extra reid`).\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6bc2b8d8", - "metadata": {}, - "outputs": [], - "source": [ - "import subprocess\n", - "import sys\n", - "\n", - "try:\n", - " import google.colab # noqa: F401\n", - "\n", - " IN_COLAB_INSTALL = True\n", - "except ImportError:\n", - " IN_COLAB_INSTALL = False\n", - "\n", - "if IN_COLAB_INSTALL:\n", - " cmds = [\n", - " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"pip\",\n", - " \"install\",\n", - " \"-q\",\n", - " \"trackers[reid]\",\n", - " \"matplotlib\",\n", - " \"scikit-learn\",\n", - " \"gdown\",\n", - " ],\n", - " ]\n", - " for cmd in cmds:\n", - " subprocess.run(cmd, check=True) # noqa: S603\n", - " print(\"Installed trackers[reid] and notebook deps.\")\n", - "else:\n", - " print(\"Local kernel: skipping install.\")\n", - " print(\"Ensure trackers[reid] is importable (pip install 'trackers[reid]' or uv sync --extra reid).\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6c2e60ad", - "metadata": {}, - "outputs": [], - "source": [ - "import shutil\n", - "import subprocess\n", - "import sys\n", - "import warnings\n", - "import zipfile\n", - "from pathlib import Path\n", - "\n", - "import cv2\n", - "import gdown\n", - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "import supervision as sv\n", - "import torch\n", - "from IPython.display import Video\n", - "from IPython.display import display as ipy_display\n", - "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", - "from sklearn.decomposition import PCA\n", - "\n", - "from trackers import BoTSORTTracker\n", - "from trackers.eval import evaluate_mot_sequences\n", - "from trackers.eval.box import box_iou\n", - "from trackers.eval.results import BenchmarkResult\n", - "from trackers.io.frames import load_mot_frame_image\n", - "from trackers.io.mot import _MOTOutput, load_mot_file\n", - "\n", - "warnings.filterwarnings(\"ignore\")\n", - "\n", - "try:\n", - " from google.colab import files\n", - "\n", - " IN_COLAB = True\n", - " REPO_ROOT = Path(\"/content\")\n", - "except ImportError:\n", - " files = None\n", - " IN_COLAB = False\n", - " REPO_ROOT = Path(\"..\").resolve()\n", - "\n", - "VAL_SEQUENCES = [\n", - " \"MOT17-02-FRCNN\",\n", - " \"MOT17-04-FRCNN\",\n", - " \"MOT17-05-FRCNN\",\n", - " \"MOT17-09-FRCNN\",\n", - " \"MOT17-10-FRCNN\",\n", - " \"MOT17-11-FRCNN\",\n", - " \"MOT17-13-FRCNN\",\n", - "]\n", - "\n", - "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", - "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" - ] - }, - { - "cell_type": "markdown", - "id": "a54bb5ed", - "metadata": {}, - "source": [ - "## 2. ReID model\n", - "\n", - "| `REID_ENCODER` | Training | Input |\n", - "|---|---|---|\n", - "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384×128 |\n", - "| `osnet_msmt17` | MSMT17 combineall | 256×128 |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "bbc892d6", - "metadata": {}, - "outputs": [], - "source": [ - "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", - "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", - "\n", - "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", - " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", - "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", - " reid_model = ReIDModel.from_pretrained()\n", - "else:\n", - " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", - "\n", - "print(f\"Encoder: {REID_ENCODER} | θ_emb: {REID_APPEARANCE_THRESHOLD}\")\n", - "print(reid_model.preprocessing.describe())" - ] - }, - { - "cell_type": "markdown", - "id": "29afb2e0", - "metadata": {}, - "source": [ - "## 3. Download data\n", - "\n", - "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", - "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "5ea423a9", - "metadata": {}, - "outputs": [], - "source": [ - "FORCE_DOWNLOAD = False\n", - "\n", - "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", - "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", - "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", - "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", - "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", - "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", - "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", - "\n", - "\n", - "def yolox_det_path(seq: str) -> Path:\n", - " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", - "\n", - "\n", - "def mot17_val_ready() -> bool:\n", - " return all(\n", - " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", - " )\n", - "\n", - "\n", - "def yolox_ready() -> bool:\n", - " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", - "\n", - "\n", - "if FORCE_DOWNLOAD or not mot17_val_ready():\n", - " subprocess.run( # noqa: S603\n", - " [\n", - " sys.executable,\n", - " \"-m\",\n", - " \"trackers.scripts\",\n", - " \"download\",\n", - " \"mot17\",\n", - " \"--split\",\n", - " \"val\",\n", - " \"--asset\",\n", - " \"annotations,frames\",\n", - " \"-o\",\n", - " str(REPO_ROOT),\n", - " ],\n", - " check=True,\n", - " )\n", - "else:\n", - " print(\"MOT17 val already present.\")\n", - "\n", - "if FORCE_DOWNLOAD or not yolox_ready():\n", - " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", - " print(\"Downloading YOLOX val detections...\")\n", - " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", - " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", - " zf.extractall(YOLOX_DIR)\n", - "else:\n", - " print(\"YOLOX detections already present.\")\n", - "\n", - "SEQUENCE_PATHS: dict[str, dict] = {}\n", - "for seq in VAL_SEQUENCES:\n", - " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", - " img = MOT17_VAL / seq / \"img1\"\n", - " det = yolox_det_path(seq)\n", - " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", - " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", - " continue\n", - " n_frames = len(list(img.glob(\"*.jpg\")))\n", - " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", - " print(f\" {seq}: {n_frames} frames\")\n", - "\n", - "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", - "if not ACTIVE_SEQUENCES:\n", - " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", - "\n", - "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", - "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", - "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" - ] - }, - { - "cell_type": "markdown", - "id": "b57560b2", - "metadata": {}, - "source": [ - "## 4. Tracking helpers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "f38487ea", - "metadata": {}, - "outputs": [], - "source": [ - "RERUN = {\n", - " \"botsort_baseline\": True,\n", - " \"botsort_reid\": True,\n", - "}\n", - "\n", - "\n", - "def _yolox_frame_offset(det_path: Path) -> int:\n", - " min_frame = None\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0]))\n", - " min_frame = frame if min_frame is None else min(min_frame, frame)\n", - " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", - "\n", - "\n", - "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", - " offset = _yolox_frame_offset(det_path)\n", - " by_frame: dict[int, list[list[float]]] = {}\n", - " with det_path.open() as f:\n", - " for line in f:\n", - " parts = line.strip().split(\",\")\n", - " if len(parts) < 6:\n", - " continue\n", - " frame = int(float(parts[0])) - offset\n", - " if frame < 1:\n", - " continue\n", - " x1, y1, x2, y2, score = map(float, parts[1:6])\n", - " if score <= 0:\n", - " continue\n", - " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", - " return {\n", - " frame: sv.Detections(\n", - " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", - " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", - " )\n", - " for frame, boxes in by_frame.items()\n", - " }\n", - "\n", - "\n", - "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", - " a = result.aggregate\n", - " return (\n", - " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", - " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", - " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", - " (a.CLEAR.IDSW if a.CLEAR else 0),\n", - " )\n", - "\n", - "\n", - "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", - " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", - " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", - "\n", - "\n", - "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " pred_dir.mkdir(parents=True, exist_ok=True)\n", - "\n", - " for seq in ACTIVE_SEQUENCES:\n", - " spec = SEQUENCE_PATHS[seq]\n", - " dets = load_yolox_dets(spec[\"det\"])\n", - " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - " tracker = factory()\n", - "\n", - " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", - " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", - " frame = None\n", - " if use_frames and frame_idx <= len(images):\n", - " frame = cv2.imread(str(images[frame_idx - 1]))\n", - " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", - " if tracked.tracker_id is not None:\n", - " tracked = tracked[tracked.tracker_id != -1]\n", - " out.write(frame_idx, tracked)\n", - " print(f\" {seq}: {spec['n_frames']} frames\")\n", - "\n", - " return pred_dir\n", - "\n", - "\n", - "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", - " result = evaluate_mot_sequences(\n", - " gt_dir=MOT17_VAL,\n", - " tracker_dir=pred_dir,\n", - " seqmap=SEQMAP_PATH,\n", - " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", - " )\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " cache.parent.mkdir(parents=True, exist_ok=True)\n", - " result.save(cache)\n", - " return result\n", - "\n", - "\n", - "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", - " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", - " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", - " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", - "\n", - " ran = False\n", - " if RERUN.get(name, True) or not preds_ok:\n", - " print(f\"Running {name}...\")\n", - " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", - " ran = True\n", - " else:\n", - " print(f\"Using cached preds: {pred_dir}\")\n", - "\n", - " if not ran and cache.exists():\n", - " print(f\"Using cached eval: {cache}\")\n", - " return BenchmarkResult.load(cache)\n", - "\n", - " print(f\"Evaluating {name}...\")\n", - " return evaluate(name, pred_dir)\n", - "\n", - "\n", - "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", - " if len(det_xyxy) == 0:\n", - " return np.array([], dtype=np.int64)\n", - " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", - " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", - " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", - " if len(gt_xyxy) == 0:\n", - " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", - " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", - " for i in range(len(det_xyxy)):\n", - " j = int(np.argmax(ious[i]))\n", - " if ious[i, j] >= min_iou:\n", - " out[i] = int(gt_ids[j])\n", - " return out" - ] - }, - { - "cell_type": "markdown", - "id": "823b2696", - "metadata": {}, - "source": [ - "## 5. Run trackers\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d09332f1", - "metadata": {}, - "outputs": [], - "source": [ - "EXPERIMENTS = [\n", - " (\n", - " \"botsort_baseline\",\n", - " \"BoT-SORT (baseline)\",\n", - " lambda: BoTSORTTracker(enable_cmc=True),\n", - " True,\n", - " ),\n", - " (\n", - " \"botsort_reid\",\n", - " \"BoT-SORT + ReID\",\n", - " lambda: BoTSORTTracker(\n", - " enable_cmc=True,\n", - " reid_model=reid_model,\n", - " reid_ema_alpha=0.9,\n", - " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", - " ),\n", - " True,\n", - " ),\n", - "]\n", - "\n", - "results: dict[str, BenchmarkResult] = {}\n", - "for name, label, factory, use_frames in EXPERIMENTS:\n", - " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", - " print_metrics(label, results[name])\n", - " print()\n", - "\n", - "result_baseline = results[\"botsort_baseline\"]\n", - "result_reid = results[\"botsort_reid\"]" - ] - }, - { - "cell_type": "markdown", - "id": "ad28e88f", - "metadata": {}, - "source": [ - "## 6. ReID embedding visualization (optional)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6612c281", - "metadata": {}, - "outputs": [], - "source": [ - "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", - "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", - "\n", - "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", - "gt_by_frame = load_mot_file(spec[\"gt\"])\n", - "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", - "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", - "\n", - "crops, embeddings, gt_ids = [], [], []\n", - "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", - " dets = dets_by_frame.get(frame_idx)\n", - " gt = gt_by_frame.get(frame_idx)\n", - " if dets is None or gt is None or len(dets) == 0:\n", - " continue\n", - " dets = dets[dets.confidence >= 0.5]\n", - " if len(dets) == 0:\n", - " continue\n", - " bgr = cv2.imread(str(images[frame_idx - 1]))\n", - " if bgr is None:\n", - " continue\n", - " matched = match_dets_to_gt(gt, dets.xyxy)\n", - " feats = reid_model.extract_features(dets, bgr)\n", - " for i in range(len(dets)):\n", - " if matched[i] < 0:\n", - " continue\n", - " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", - " if crop.size == 0:\n", - " continue\n", - " crops.append(crop[:, :, ::-1])\n", - " embeddings.append(feats[i])\n", - " gt_ids.append(int(matched[i]))\n", - "\n", - "if not embeddings:\n", - " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", - "\n", - "emb = np.stack(embeddings)\n", - "labels = np.array(gt_ids)\n", - "if len(emb) > VIZ_MAX_POINTS:\n", - " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", - " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", - "\n", - "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", - "unique = np.unique(labels)\n", - "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", - "\n", - "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", - "for pid in unique:\n", - " m = labels == pid\n", - " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", - "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", - "ax_pca.grid(True, alpha=0.3)\n", - "if len(unique) <= 12:\n", - " ax_pca.legend(fontsize=8)\n", - "\n", - "n_show = min(len(crops), VIZ_MAX_CROPS)\n", - "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", - "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", - "for k in range(n_show):\n", - " r, c = divmod(k, ncols)\n", - " tile = cv2.resize(crops[k], (32, 64))\n", - " y, x = r * 64, c * 32\n", - " mosaic[y : y + 64, x : x + 32] = tile\n", - " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", - " mosaic[y : y + 2, x : x + 32] = rgb\n", - " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", - "\n", - "ax_crop.imshow(mosaic)\n", - "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", - "ax_crop.axis(\"off\")\n", - "plt.tight_layout()\n", - "plt.show()\n", - "print(f\"{len(coords)} points, {len(unique)} GT ids\")" - ] - }, - { - "cell_type": "markdown", - "id": "b0ea623e", - "metadata": {}, - "source": [ - "## 7. Results\n", - "\n", - "**7.1-7.2** BoT-SORT vs published references.\n" - ] - }, - { - "cell_type": "markdown", - "id": "43321292", - "metadata": {}, - "source": [ - "### 7.1 BoT-SORT - reference targets\n", - "\n", - "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", - "\n", - "| Config | HOTA | IDF1 |\n", - "|---|---:|---:|\n", - "| No re-ID | 68.43 | 80.92 |\n", - "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", - "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", - "\n", - "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", - "\n", - "| Method | HOTA | MOTA | IDF1 |\n", - "|---|---:|---:|---:|\n", - "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", - "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", - "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d16f6483", - "metadata": {}, - "outputs": [], - "source": [ - "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", - "# MOTA is not reported for the YOLOX setup in that study.\n", - "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", - "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", - "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", - "\n", - "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", - "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", - "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", - "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", - "\n", - "\n", - "def fmt_ref_metric(value: float | None) -> str:\n", - " return f\"{value:6.2f}\" if value is not None else \" -\"\n", - "\n", - "\n", - "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", - " s = result.sequences.get(seq)\n", - " if s is None:\n", - " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", - " return (\n", - " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", - " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", - " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", - " s.CLEAR.IDSW if s.CLEAR else 0,\n", - " )\n", - "\n", - "\n", - "botsort_rows = [\n", - " (\"BoT-SORT (baseline)\", result_baseline),\n", - " (\"BoT-SORT + ReID\", result_reid),\n", - "]\n", - "\n", - "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", - "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", - "print(\"-\" * 72)\n", - "for label, res in botsort_rows:\n", - " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", - " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", - "\n", - "b = fmt_metrics(result_baseline)\n", - "r = fmt_metrics(result_reid)\n", - "print(\n", - " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", - " f\"ΔHOTA {r[0] - b[0]:+6.2f} ΔMOTA {r[3] - b[3]:+6.2f} \"\n", - " f\"ΔIDF1 {r[4] - b[4]:+6.2f} ΔIDSW {int(r[5] - b[5]):+5d}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'Reference (no re-ID)':<28} \"\n", - " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", - " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", - " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", - " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", - " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", - " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs reference study\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", - ")\n", - "\n", - "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", - "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", - "print(\"-\" * 52)\n", - "print(\n", - " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", - " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", - " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", - " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", - ")\n", - "print(\n", - " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", - " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", - " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", - ")\n", - "print(\n", - " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", - " f\" ΔHOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", - " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", - " f\" ΔMOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", - " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", - " f\" ΔIDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", - " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "8ee1ac84", - "metadata": {}, - "source": [ - "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d448e555", - "metadata": {}, - "outputs": [], - "source": [ - "REID_STUDY_PER_SEQ = {\n", - " \"MOT17-02\": {\n", - " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", - " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", - " },\n", - " \"MOT17-04\": {\n", - " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", - " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", - " },\n", - " \"MOT17-05\": {\n", - " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", - " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", - " },\n", - " \"MOT17-09\": {\n", - " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", - " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", - " },\n", - " \"MOT17-10\": {\n", - " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", - " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", - " },\n", - " \"MOT17-11\": {\n", - " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", - " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", - " },\n", - " \"MOT17-13\": {\n", - " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", - " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", - " },\n", - "}\n", - "\n", - "\n", - "def ref_seq_key(seq: str) -> str:\n", - " parts = seq.split(\"-\")\n", - " return f\"{parts[0]}-{parts[1]}\"\n", - "\n", - "\n", - "for seq in ACTIVE_SEQUENCES:\n", - " key = ref_seq_key(seq)\n", - " ref = REID_STUDY_PER_SEQ.get(key, {})\n", - " print(seq)\n", - " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", - " for label, res in botsort_rows:\n", - " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", - " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", - " ref_vals = ref.get(ref_key, {})\n", - " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", - " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", - " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", - " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", - " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", - " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", - " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", - " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", - " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", - " print()" - ] - }, - { - "cell_type": "markdown", - "id": "8de54c38", - "metadata": {}, - "source": [ - "### 8. Visual comparison - largest ReID gain sequence\n", - "\n", - "Side-by-side **baseline vs +ReID** video for the val sequence with the largest ΔHOTA\n", - "(from the runs above). On Colab the mp4 is downloaded automatically.\n", - "" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "9a4f4194", - "metadata": {}, - "outputs": [], - "source": [ - "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", - "COMPARE_SEQ: str | None = None\n", - "COMPARE_FPS = 30\n", - "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", - "\n", - "\n", - "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", - " frame = mot.get(frame_idx)\n", - " if frame is None:\n", - " return sv.Detections.empty()\n", - " active = frame.ids >= 0\n", - " if not np.any(active):\n", - " return sv.Detections.empty()\n", - " return sv.Detections(\n", - " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", - " tracker_id=frame.ids[active].astype(int),\n", - " confidence=frame.confidences[active].astype(np.float32),\n", - " )\n", - "\n", - "\n", - "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", - " if len(detections) == 0:\n", - " return frame_bgr\n", - " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", - " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", - " labels = [str(int(tid)) for tid in detections.tracker_id]\n", - " return sv.LabelAnnotator(\n", - " color=palette,\n", - " color_lookup=lookup,\n", - " text_color=sv.Color.BLACK,\n", - " text_scale=0.5,\n", - " ).annotate(scene, detections, labels=labels)\n", - "\n", - "\n", - "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", - " out = frame.copy()\n", - " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", - " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", - " x, y, pad, bar = 12, 12, 10, 6\n", - " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", - " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", - " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", - " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", - " return out\n", - "\n", - "\n", - "seq_gains: list[tuple[str, float, float, float]] = []\n", - "for seq in ACTIVE_SEQUENCES:\n", - " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", - " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", - " if h_b == h_b and h_r == h_r:\n", - " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", - "\n", - "if not seq_gains:\n", - " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", - "\n", - "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", - "print(\"Per-sequence ReID ΔHOTA (largest first):\")\n", - "for seq, dh, di, _ in seq_gains:\n", - " print(f\" {seq:<20} ΔHOTA {dh:+6.2f} ΔIDF1 {di:+6.2f}\")\n", - "\n", - "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", - "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", - "print(f\"\\nRendering comparison for {COMPARE_SEQ} (ΔHOTA {delta_hota:+.2f}, ΔIDF1 {delta_idf1:+.2f})\")\n", - "\n", - "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", - "if not pred_base.is_file() or not pred_reid.is_file():\n", - " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", - "\n", - "mot_base = load_mot_file(pred_base)\n", - "mot_reid = load_mot_file(pred_reid)\n", - "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", - "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", - "if COMPARE_MAX_FRAMES is not None:\n", - " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", - "\n", - "compare_fps = COMPARE_FPS\n", - "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", - "if seqinfo.is_file():\n", - " for line in seqinfo.read_text().splitlines():\n", - " if line.startswith(\"frameRate=\"):\n", - " compare_fps = int(line.split(\"=\", 1)[1])\n", - " break\n", - "\n", - "sample = load_mot_frame_image(img_dir, 1)\n", - "h, w = sample.shape[:2]\n", - "\n", - "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", - "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", - "\n", - "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", - " for frame_idx in range(1, n_frames + 1):\n", - " frame = load_mot_frame_image(img_dir, frame_idx)\n", - " left = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", - " \"BASELINE (NO REID)\",\n", - " (0, 165, 255),\n", - " )\n", - " right = _panel_badge(\n", - " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", - " \"BOT-SORT + REID\",\n", - " (80, 200, 120),\n", - " )\n", - " sink.write_frame(np.hstack([left, right]))\n", - "\n", - "ffmpeg = shutil.which(\"ffmpeg\")\n", - "if ffmpeg is not None:\n", - " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", - " result = subprocess.run( # noqa: S603\n", - " [\n", - " ffmpeg,\n", - " \"-y\",\n", - " \"-i\",\n", - " str(out_path),\n", - " \"-c:v\",\n", - " \"libx264\",\n", - " \"-pix_fmt\",\n", - " \"yuv420p\",\n", - " \"-movflags\",\n", - " \"+faststart\",\n", - " \"-an\",\n", - " str(tmp),\n", - " ],\n", - " capture_output=True,\n", - " text=True,\n", - " )\n", - " if result.returncode == 0:\n", - " tmp.replace(out_path)\n", - "\n", - "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", - "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", - "if IN_COLAB:\n", - " files.download(str(out_path))" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 + "cells": [ + { + "cell_type": "markdown", + "id": "2d522414", + "metadata": {}, + "source": [ + "# Tracker ReID evaluation on MOT17 val\n", + "\n", + "Compare BoT-SORT with and without a **reid** encoder on the MOT17 val-half split (YOLOX detections, TrackEval metrics).\n", + "\n", + "| Config | Tracker | CMC | ReID | Fusion |\n", + "|---|---|---|---|---|\n", + "| Baseline | BoT-SORT | yes (sparseOptFlow) | no | geometry + CMC |\n", + "| + ReID | BoT-SORT | yes | yes | min-cost (x5) |\n", + "\n", + "**Defaults:** `fastreid_mot17_sbs50`, `REID_APPEARANCE_THRESHOLD=0.2` (MOT17 re-ID study Table 8).\n", + "\n", + "**Data:** `trackers download mot17 --split val --asset annotations,frames` + [YOLOX val detections](https://drive.google.com/file/d/1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT).\n", + "\n", + "**Outputs:** `trackers_reid_outputs/` - set `RERUN[name]=False` to reuse cached preds.\n", + "\n", + "Section 8 builds a side-by-side baseline vs +ReID video for the sequence with the largest \u0394HOTA (Colab auto-download).\n", + "\n", + "> **Runtime:** T4 GPU (`Runtime -> Change runtime type`).\n" + ] + }, + { + "cell_type": "markdown", + "id": "7bec6c65", + "metadata": {}, + "source": [ + "## 1. Setup\n", + "\n", + "Install **trackers** with the ReID extra (pulls in the standalone `reid` package),\n", + "plus notebook-only deps (`matplotlib`, `scikit-learn`, `gdown`).\n", + "\n", + "**Local:** skip the install cell if your venv already has `trackers[reid]`\n", + "(`pip install 'trackers[reid]'` or `uv sync --extra reid`).\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6bc2b8d8", + "metadata": {}, + "outputs": [], + "source": [ + "import subprocess\n", + "import sys\n", + "\n", + "try:\n", + " import google.colab # noqa: F401\n", + "\n", + " IN_COLAB_INSTALL = True\n", + "except ImportError:\n", + " IN_COLAB_INSTALL = False\n", + "\n", + "if IN_COLAB_INSTALL:\n", + " cmds = [\n", + " [sys.executable, \"-m\", \"pip\", \"install\", \"-q\", \"--upgrade\", \"pip\"],\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"pip\",\n", + " \"install\",\n", + " \"-q\",\n", + " \"trackers[reid]\",\n", + " \"matplotlib\",\n", + " \"scikit-learn\",\n", + " \"gdown\",\n", + " ],\n", + " ]\n", + " for cmd in cmds:\n", + " subprocess.run(cmd, check=True) # noqa: S603\n", + " print(\"Installed trackers[reid] and notebook deps.\")\n", + "else:\n", + " print(\"Local kernel: skipping install.\")\n", + " print(\"Ensure trackers[reid] is importable (pip install 'trackers[reid]' or uv sync --extra reid).\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6c2e60ad", + "metadata": {}, + "outputs": [], + "source": [ + "import shutil\n", + "import subprocess\n", + "import sys\n", + "import warnings\n", + "import zipfile\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import gdown\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from IPython.display import Video\n", + "from IPython.display import display as ipy_display\n", + "from reid import DEFAULT_MODEL, FASTREID_MOT17_SBS50, ReIDModel\n", + "from sklearn.decomposition import PCA\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.eval.box import box_iou\n", + "from trackers.eval.results import BenchmarkResult\n", + "from trackers.io.frames import load_mot_frame_image\n", + "from trackers.io.mot import _MOTOutput, load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "try:\n", + " from google.colab import files\n", + "\n", + " IN_COLAB = True\n", + " REPO_ROOT = Path(\"/content\")\n", + "except ImportError:\n", + " files = None\n", + " IN_COLAB = False\n", + " REPO_ROOT = Path(\"..\").resolve()\n", + "\n", + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")" + ] + }, + { + "cell_type": "markdown", + "id": "a54bb5ed", + "metadata": {}, + "source": [ + "## 2. ReID model\n", + "\n", + "| `REID_ENCODER` | Training | Input |\n", + "|---|---|---|\n", + "| `fastreid_mot17_sbs50` (default) | MOT17 train-half | 384\u00d7128 |\n", + "| `osnet_msmt17` | MSMT17 combineall | 256\u00d7128 |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "bbc892d6", + "metadata": {}, + "outputs": [], + "source": [ + "REID_ENCODER = \"fastreid_mot17_sbs50\"\n", + "REID_APPEARANCE_THRESHOLD = 0.2 # MOT17 re-ID study Table 8; BoT-SORT paper default 0.25\n", + "\n", + "if REID_ENCODER == FASTREID_MOT17_SBS50:\n", + " reid_model = ReIDModel.from_pretrained(FASTREID_MOT17_SBS50)\n", + "elif REID_ENCODER in (DEFAULT_MODEL, \"osnet_msmt17\"):\n", + " reid_model = ReIDModel.from_pretrained()\n", + "else:\n", + " raise ValueError(f\"Unknown REID_ENCODER: {REID_ENCODER!r}\")\n", + "\n", + "print(f\"Encoder: {REID_ENCODER} | \u03b8_emb: {REID_APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ] + }, + { + "cell_type": "markdown", + "id": "29afb2e0", + "metadata": {}, + "source": [ + "## 3. Download data\n", + "\n", + "MOT17 val GT + frames via `trackers download`. YOLOX val detections via gdown\n", + "(BoT-SORT / ByteTrack eval protocol). YOLOX frame IDs are remapped to 1...N.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5ea423a9", + "metadata": {}, + "outputs": [], + "source": [ + "FORCE_DOWNLOAD = False\n", + "\n", + "MOT17_VAL = REPO_ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = REPO_ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_VAL_DIR = YOLOX_DIR / \"val\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = REPO_ROOT / \"trackers_reid_outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "\n", + "def yolox_det_path(seq: str) -> Path:\n", + " return YOLOX_VAL_DIR / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + "\n", + "\n", + "def mot17_val_ready() -> bool:\n", + " return all(\n", + " (MOT17_VAL / seq / \"gt\" / \"gt.txt\").is_file() and (MOT17_VAL / seq / \"img1\").is_dir() for seq in VAL_SEQUENCES\n", + " )\n", + "\n", + "\n", + "def yolox_ready() -> bool:\n", + " return YOLOX_VAL_DIR.is_dir() and len(list(YOLOX_VAL_DIR.glob(\"MOT17-*_val.txt\"))) >= len(VAL_SEQUENCES)\n", + "\n", + "\n", + "if FORCE_DOWNLOAD or not mot17_val_ready():\n", + " subprocess.run( # noqa: S603\n", + " [\n", + " sys.executable,\n", + " \"-m\",\n", + " \"trackers.scripts\",\n", + " \"download\",\n", + " \"mot17\",\n", + " \"--split\",\n", + " \"val\",\n", + " \"--asset\",\n", + " \"annotations,frames\",\n", + " \"-o\",\n", + " str(REPO_ROOT),\n", + " ],\n", + " check=True,\n", + " )\n", + "else:\n", + " print(\"MOT17 val already present.\")\n", + "\n", + "if FORCE_DOWNLOAD or not yolox_ready():\n", + " YOLOX_DIR.mkdir(parents=True, exist_ok=True)\n", + " print(\"Downloading YOLOX val detections...\")\n", + " gdown.download(id=YOLOX_GDRIVE_ID, output=str(YOLOX_ZIP), quiet=False)\n", + " with zipfile.ZipFile(YOLOX_ZIP) as zf:\n", + " zf.extractall(YOLOX_DIR)\n", + "else:\n", + " print(\"YOLOX detections already present.\")\n", + "\n", + "SEQUENCE_PATHS: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " gt = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " img = MOT17_VAL / seq / \"img1\"\n", + " det = yolox_det_path(seq)\n", + " if not (gt.is_file() and img.is_dir() and det.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX det\")\n", + " continue\n", + " n_frames = len(list(img.glob(\"*.jpg\")))\n", + " SEQUENCE_PATHS[seq] = {\"gt\": gt, \"img\": img, \"det\": det, \"n_frames\": n_frames}\n", + " print(f\" {seq}: {n_frames} frames\")\n", + "\n", + "ACTIVE_SEQUENCES = list(SEQUENCE_PATHS)\n", + "if not ACTIVE_SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready - re-run downloads above.\")\n", + "\n", + "SEQMAP_PATH = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP_PATH.write_text(\"name\\n\" + \"\\n\".join(ACTIVE_SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(ACTIVE_SEQUENCES)} sequences -> outputs in {OUTPUT_ROOT}\")" + ] + }, + { + "cell_type": "markdown", + "id": "b57560b2", + "metadata": {}, + "source": [ + "## 4. Tracking helpers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "f38487ea", + "metadata": {}, + "outputs": [], + "source": [ + "RERUN = {\n", + " \"botsort_baseline\": True,\n", + " \"botsort_reid\": True,\n", + "}\n", + "\n", + "\n", + "def _yolox_frame_offset(det_path: Path) -> int:\n", + " min_frame = None\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0]))\n", + " min_frame = frame if min_frame is None else min(min_frame, frame)\n", + " return (min_frame - 1) if min_frame and min_frame > 1 else 0\n", + "\n", + "\n", + "def load_yolox_dets(det_path: Path) -> dict[int, sv.Detections]:\n", + " offset = _yolox_frame_offset(det_path)\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " with det_path.open() as f:\n", + " for line in f:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame = int(float(parts[0])) - offset\n", + " if frame < 1:\n", + " continue\n", + " x1, y1, x2, y2, score = map(float, parts[1:6])\n", + " if score <= 0:\n", + " continue\n", + " by_frame.setdefault(frame, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.array(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.array(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def fmt_metrics(result: BenchmarkResult) -> tuple[float, float, float, float, float, int]:\n", + " a = result.aggregate\n", + " return (\n", + " (a.HOTA.HOTA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.AssA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.HOTA.DetA * 100 if a.HOTA else float(\"nan\")),\n", + " (a.CLEAR.MOTA * 100 if a.CLEAR else float(\"nan\")),\n", + " (a.Identity.IDF1 * 100 if a.Identity else float(\"nan\")),\n", + " (a.CLEAR.IDSW if a.CLEAR else 0),\n", + " )\n", + "\n", + "\n", + "def print_metrics(label: str, result: BenchmarkResult) -> None:\n", + " hota, _assa, _deta, mota, idf1, idsw = fmt_metrics(result)\n", + " print(f\"{label}: HOTA {hota:6.2f} MOTA {mota:6.2f} IDF1 {idf1:6.2f} IDSW {idsw}\")\n", + "\n", + "\n", + "def run_tracking(name: str, factory, *, use_frames: bool) -> Path:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " pred_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + " for seq in ACTIVE_SEQUENCES:\n", + " spec = SEQUENCE_PATHS[seq]\n", + " dets = load_yolox_dets(spec[\"det\"])\n", + " images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + " tracker = factory()\n", + "\n", + " with _MOTOutput(pred_dir / f\"{seq}.txt\") as out:\n", + " for frame_idx in range(1, spec[\"n_frames\"] + 1):\n", + " frame = None\n", + " if use_frames and frame_idx <= len(images):\n", + " frame = cv2.imread(str(images[frame_idx - 1]))\n", + " tracked = tracker.update(dets.get(frame_idx, sv.Detections.empty()), frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " out.write(frame_idx, tracked)\n", + " print(f\" {seq}: {spec['n_frames']} frames\")\n", + "\n", + " return pred_dir\n", + "\n", + "\n", + "def evaluate(name: str, pred_dir: Path) -> BenchmarkResult:\n", + " result = evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=pred_dir,\n", + " seqmap=SEQMAP_PATH,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " cache.parent.mkdir(parents=True, exist_ok=True)\n", + " result.save(cache)\n", + " return result\n", + "\n", + "\n", + "def load_or_run(name: str, factory, *, use_frames: bool) -> BenchmarkResult:\n", + " pred_dir = OUTPUT_ROOT / name / \"preds\"\n", + " cache = OUTPUT_ROOT / name / \"eval_results.json\"\n", + " preds_ok = pred_dir.exists() and all((pred_dir / f\"{s}.txt\").exists() for s in ACTIVE_SEQUENCES)\n", + "\n", + " ran = False\n", + " if RERUN.get(name, True) or not preds_ok:\n", + " print(f\"Running {name}...\")\n", + " pred_dir = run_tracking(name, factory, use_frames=use_frames)\n", + " ran = True\n", + " else:\n", + " print(f\"Using cached preds: {pred_dir}\")\n", + "\n", + " if not ran and cache.exists():\n", + " print(f\"Using cached eval: {cache}\")\n", + " return BenchmarkResult.load(cache)\n", + "\n", + " print(f\"Evaluating {name}...\")\n", + " return evaluate(name, pred_dir)\n", + "\n", + "\n", + "def match_dets_to_gt(gt_frame, det_xyxy: np.ndarray, min_iou: float = 0.5) -> np.ndarray:\n", + " if len(det_xyxy) == 0:\n", + " return np.array([], dtype=np.int64)\n", + " gt_xyxy = sv.xywh_to_xyxy(gt_frame.boxes)\n", + " keep = (gt_frame.confidences > 0) & (gt_frame.classes == 1)\n", + " gt_xyxy, gt_ids = gt_xyxy[keep], gt_frame.ids[keep]\n", + " if len(gt_xyxy) == 0:\n", + " return np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " ious = box_iou(det_xyxy.astype(np.float64), gt_xyxy.astype(np.float64))\n", + " out = np.full(len(det_xyxy), -1, dtype=np.int64)\n", + " for i in range(len(det_xyxy)):\n", + " j = int(np.argmax(ious[i]))\n", + " if ious[i, j] >= min_iou:\n", + " out[i] = int(gt_ids[j])\n", + " return out" + ] + }, + { + "cell_type": "markdown", + "id": "823b2696", + "metadata": {}, + "source": [ + "## 5. Run trackers\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d09332f1", + "metadata": {}, + "outputs": [], + "source": [ + "EXPERIMENTS = [\n", + " (\n", + " \"botsort_baseline\",\n", + " \"BoT-SORT (baseline)\",\n", + " lambda: BoTSORTTracker(enable_cmc=True),\n", + " True,\n", + " ),\n", + " (\n", + " \"botsort_reid\",\n", + " \"BoT-SORT + ReID\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " appearance_threshold=REID_APPEARANCE_THRESHOLD,\n", + " ),\n", + " True,\n", + " ),\n", + "]\n", + "\n", + "results: dict[str, BenchmarkResult] = {}\n", + "for name, label, factory, use_frames in EXPERIMENTS:\n", + " results[name] = load_or_run(name, factory, use_frames=use_frames)\n", + " print_metrics(label, results[name])\n", + " print()\n", + "\n", + "result_baseline = results[\"botsort_baseline\"]\n", + "result_reid = results[\"botsort_reid\"]" + ] + }, + { + "cell_type": "markdown", + "id": "ad28e88f", + "metadata": {}, + "source": [ + "## 6. ReID embedding visualization (optional)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6612c281", + "metadata": {}, + "outputs": [], + "source": [ + "VIZ_SEQ = \"MOT17-02-FRCNN\"\n", + "VIZ_STRIDE, VIZ_MAX_FRAMES, VIZ_MAX_POINTS, VIZ_MAX_CROPS = 5, 40, 300, 24\n", + "\n", + "spec = SEQUENCE_PATHS[VIZ_SEQ]\n", + "gt_by_frame = load_mot_file(spec[\"gt\"])\n", + "dets_by_frame = load_yolox_dets(spec[\"det\"])\n", + "images = sorted(spec[\"img\"].glob(\"*.jpg\"))\n", + "\n", + "crops, embeddings, gt_ids = [], [], []\n", + "for frame_idx in list(range(1, spec[\"n_frames\"] + 1, VIZ_STRIDE))[:VIZ_MAX_FRAMES]:\n", + " dets = dets_by_frame.get(frame_idx)\n", + " gt = gt_by_frame.get(frame_idx)\n", + " if dets is None or gt is None or len(dets) == 0:\n", + " continue\n", + " dets = dets[dets.confidence >= 0.5]\n", + " if len(dets) == 0:\n", + " continue\n", + " bgr = cv2.imread(str(images[frame_idx - 1]))\n", + " if bgr is None:\n", + " continue\n", + " matched = match_dets_to_gt(gt, dets.xyxy)\n", + " feats = reid_model.extract_features(dets, bgr)\n", + " for i in range(len(dets)):\n", + " if matched[i] < 0:\n", + " continue\n", + " crop = sv.crop_image(bgr, dets.xyxy[i].astype(int))\n", + " if crop.size == 0:\n", + " continue\n", + " crops.append(crop[:, :, ::-1])\n", + " embeddings.append(feats[i])\n", + " gt_ids.append(int(matched[i]))\n", + "\n", + "if not embeddings:\n", + " raise RuntimeError(\"No matched crops - try another sequence or lower confidence threshold\")\n", + "\n", + "emb = np.stack(embeddings)\n", + "labels = np.array(gt_ids)\n", + "if len(emb) > VIZ_MAX_POINTS:\n", + " idx = np.linspace(0, len(emb) - 1, VIZ_MAX_POINTS, dtype=int)\n", + " emb, labels, crops = emb[idx], labels[idx], [crops[i] for i in idx]\n", + "\n", + "coords = PCA(n_components=2, random_state=0).fit_transform(emb)\n", + "unique = np.unique(labels)\n", + "colors = {pid: plt.colormaps[\"tab20\"](i % 20) for i, pid in enumerate(unique)}\n", + "\n", + "fig, (ax_pca, ax_crop) = plt.subplots(1, 2, figsize=(14, 6))\n", + "for pid in unique:\n", + " m = labels == pid\n", + " ax_pca.scatter(coords[m, 0], coords[m, 1], s=28, alpha=0.85, color=colors[pid], label=f\"id {pid}\")\n", + "ax_pca.set(title=f\"{VIZ_SEQ} - PCA by GT id\", xlabel=\"PC1\", ylabel=\"PC2\")\n", + "ax_pca.grid(True, alpha=0.3)\n", + "if len(unique) <= 12:\n", + " ax_pca.legend(fontsize=8)\n", + "\n", + "n_show = min(len(crops), VIZ_MAX_CROPS)\n", + "ncols, nrows = 6, int(np.ceil(n_show / 6))\n", + "mosaic = np.full((nrows * 64, ncols * 32, 3), 255, dtype=np.uint8)\n", + "for k in range(n_show):\n", + " r, c = divmod(k, ncols)\n", + " tile = cv2.resize(crops[k], (32, 64))\n", + " y, x = r * 64, c * 32\n", + " mosaic[y : y + 64, x : x + 32] = tile\n", + " rgb = (np.array(colors[labels[k]])[:3] * 255).astype(np.uint8)\n", + " mosaic[y : y + 2, x : x + 32] = rgb\n", + " mosaic[y + 62 : y + 64, x : x + 32] = rgb\n", + "\n", + "ax_crop.imshow(mosaic)\n", + "ax_crop.set(title=f\"Sample crops ({n_show})\")\n", + "ax_crop.axis(\"off\")\n", + "plt.tight_layout()\n", + "plt.show()\n", + "print(f\"{len(coords)} points, {len(unique)} GT ids\")" + ] + }, + { + "cell_type": "markdown", + "id": "b0ea623e", + "metadata": {}, + "source": [ + "## 7. Results\n", + "\n", + "**7.1-7.2** BoT-SORT vs published references.\n" + ] + }, + { + "cell_type": "markdown", + "id": "43321292", + "metadata": {}, + "source": [ + "### 7.1 BoT-SORT - reference targets\n", + "\n", + "**Primary - [*Does Re-ID Really Help in Multi-Object Tracking?*](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (2025). BoT-SORT + YOLOX + MOT17 FastReID, app th=0.2. Combined val scores from **Table 8 (HOTA)** and **Table 13 (IDF1)**; MOTA is not reported for this YOLOX setup.\n", + "\n", + "| Config | HOTA | IDF1 |\n", + "|---|---:|---:|\n", + "| No re-ID | 68.43 | 80.92 |\n", + "| MOT17 FastReID, app th=0.2 | 68.95 | 81.98 |\n", + "| **ReID d (reference)** | **+0.52** | **+1.06** |\n", + "\n", + "**Secondary - [BoT-SORT paper](https://arxiv.org/abs/2206.14651)** (Table 1, MOT17 val):\n", + "\n", + "| Method | HOTA | MOTA | IDF1 |\n", + "|---|---:|---:|---:|\n", + "| BoT-SORT | 69.11 | 78.39 | 81.53 |\n", + "| BoT-SORT + ReID | 69.17 | 78.46 | 82.07 |\n", + "| **ReID d (BoT-SORT paper)** | **+0.06** | **+0.07** | **+0.54** |\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d16f6483", + "metadata": {}, + "outputs": [], + "source": [ + "# MOT17 re-ID study reference - Table 8 (HOTA) + Table 13 (IDF1), COMBINED row.\n", + "# MOTA is not reported for the YOLOX setup in that study.\n", + "REID_STUDY_NO_REID = {\"hota\": 68.428, \"mota\": None, \"idf1\": 80.92}\n", + "REID_STUDY_MOT17_TH02 = {\"hota\": 68.951, \"mota\": None, \"idf1\": 81.984}\n", + "REID_STUDY_REID_DELTA = {k: REID_STUDY_MOT17_TH02[k] - REID_STUDY_NO_REID[k] for k in (\"hota\", \"idf1\")}\n", + "\n", + "# BoT-SORT paper Table 1 (MOT17 val, YOLOX).\n", + "BOTSORT_PAPER = {\"hota\": 69.11, \"mota\": 78.39, \"idf1\": 81.53}\n", + "BOTSORT_PAPER_REID = {\"hota\": 69.17, \"mota\": 78.46, \"idf1\": 82.07}\n", + "BOTSORT_PAPER_REID_DELTA = {k: BOTSORT_PAPER_REID[k] - BOTSORT_PAPER[k] for k in BOTSORT_PAPER}\n", + "\n", + "\n", + "def fmt_ref_metric(value: float | None) -> str:\n", + " return f\"{value:6.2f}\" if value is not None else \" -\"\n", + "\n", + "\n", + "def seq_metrics(result: BenchmarkResult, seq: str) -> tuple[float, float, float, int]:\n", + " s = result.sequences.get(seq)\n", + " if s is None:\n", + " return float(\"nan\"), float(\"nan\"), float(\"nan\"), 0\n", + " return (\n", + " s.HOTA.HOTA * 100 if s.HOTA else float(\"nan\"),\n", + " s.HOTA.AssA * 100 if s.HOTA else float(\"nan\"),\n", + " s.Identity.IDF1 * 100 if s.Identity else float(\"nan\"),\n", + " s.CLEAR.IDSW if s.CLEAR else 0,\n", + " )\n", + "\n", + "\n", + "botsort_rows = [\n", + " (\"BoT-SORT (baseline)\", result_baseline),\n", + " (\"BoT-SORT + ReID\", result_reid),\n", + "]\n", + "\n", + "print(\"BoT-SORT - trackers (aggregate, all val sequences)\")\n", + "print(f\"{'Config':<28} {'HOTA':>6} {'AssA':>6} {'DetA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 72)\n", + "for label, res in botsort_rows:\n", + " hota, assa, deta, mota, idf1, idsw = fmt_metrics(res)\n", + " print(f\"{label:<28} {hota:6.2f} {assa:6.2f} {deta:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "b = fmt_metrics(result_baseline)\n", + "r = fmt_metrics(result_reid)\n", + "print(\n", + " f\"\\nBoT-SORT ReID uplift (trackers): \"\n", + " f\"\u0394HOTA {r[0] - b[0]:+6.2f} \u0394MOTA {r[3] - b[3]:+6.2f} \"\n", + " f\"\u0394IDF1 {r[4] - b[4]:+6.2f} \u0394IDSW {int(r[5] - b[5]):+5d}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs MOT17 re-ID study (primary - Table 8 + Table 13)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'Reference (no re-ID)':<28} \"\n", + " f\"{REID_STUDY_NO_REID['hota']:6.2f} {fmt_ref_metric(REID_STUDY_NO_REID['mota'])} \"\n", + " f\"{REID_STUDY_NO_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - REID_STUDY_NO_REID['hota']:+5.2f} \"\n", + " f\"{'-':>6} {b[4] - REID_STUDY_NO_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'Reference (MOT17 th=0.2)':<28} \"\n", + " f\"{REID_STUDY_MOT17_TH02['hota']:6.2f} {fmt_ref_metric(REID_STUDY_MOT17_TH02['mota'])} \"\n", + " f\"{REID_STUDY_MOT17_TH02['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - REID_STUDY_MOT17_TH02['hota']:+5.2f} \"\n", + " f\"{'-':>6} {r[4] - REID_STUDY_MOT17_TH02['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs reference study\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} reference {REID_STUDY_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - REID_STUDY_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} reference -\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} reference {REID_STUDY_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - REID_STUDY_REID_DELTA['idf1']:+6.2f}\"\n", + ")\n", + "\n", + "print(\"\\nBoT-SORT vs BoT-SORT paper Table 1 (secondary)\")\n", + "print(f\"{'':28} {'HOTA':>6} {'MOTA':>6} {'IDF1':>6}\")\n", + "print(\"-\" * 52)\n", + "print(\n", + " f\"{'BoT-SORT paper':<28} {BOTSORT_PAPER['hota']:6.2f} {BOTSORT_PAPER['mota']:6.2f} {BOTSORT_PAPER['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (baseline)':<28} {b[0]:6.2f} {b[3]:6.2f} {b[4]:6.2f} \"\n", + " f\" d {b[0] - BOTSORT_PAPER['hota']:+5.2f} \"\n", + " f\"{b[3] - BOTSORT_PAPER['mota']:+5.2f} {b[4] - BOTSORT_PAPER['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"{'BoT-SORT paper + ReID':<28} {BOTSORT_PAPER_REID['hota']:6.2f} \"\n", + " f\"{BOTSORT_PAPER_REID['mota']:6.2f} {BOTSORT_PAPER_REID['idf1']:6.2f}\"\n", + ")\n", + "print(\n", + " f\"{'trackers (+ ReID)':<28} {r[0]:6.2f} {r[3]:6.2f} {r[4]:6.2f} \"\n", + " f\" d {r[0] - BOTSORT_PAPER_REID['hota']:+5.2f} \"\n", + " f\"{r[3] - BOTSORT_PAPER_REID['mota']:+5.2f} {r[4] - BOTSORT_PAPER_REID['idf1']:+5.2f}\"\n", + ")\n", + "print(\n", + " f\"\\nReID uplift vs BoT-SORT paper\\n\"\n", + " f\" \u0394HOTA trackers {r[0] - b[0]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['hota']:+6.2f} \"\n", + " f\"gap {(r[0] - b[0]) - BOTSORT_PAPER_REID_DELTA['hota']:+6.2f}\\n\"\n", + " f\" \u0394MOTA trackers {r[3] - b[3]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['mota']:+6.2f} \"\n", + " f\"gap {(r[3] - b[3]) - BOTSORT_PAPER_REID_DELTA['mota']:+6.2f}\\n\"\n", + " f\" \u0394IDF1 trackers {r[4] - b[4]:+6.2f} BoT-SORT paper {BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f} \"\n", + " f\"gap {(r[4] - b[4]) - BOTSORT_PAPER_REID_DELTA['idf1']:+6.2f}\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8ee1ac84", + "metadata": {}, + "source": [ + "### 7.2 BoT-SORT - per-sequence vs reference (Table 8 HOTA + Table 13 IDF1)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d448e555", + "metadata": {}, + "outputs": [], + "source": [ + "REID_STUDY_PER_SEQ = {\n", + " \"MOT17-02\": {\n", + " \"no_reid\": {\"hota\": 47.131, \"idf1\": 56.968},\n", + " \"mot17_th02\": {\"hota\": 49.304, \"idf1\": 60.0},\n", + " },\n", + " \"MOT17-04\": {\n", + " \"no_reid\": {\"hota\": 78.976, \"idf1\": 91.021},\n", + " \"mot17_th02\": {\"hota\": 79.046, \"idf1\": 90.864},\n", + " },\n", + " \"MOT17-05\": {\n", + " \"no_reid\": {\"hota\": 60.078, \"idf1\": 75.124},\n", + " \"mot17_th02\": {\"hota\": 61.469, \"idf1\": 77.969},\n", + " },\n", + " \"MOT17-09\": {\n", + " \"no_reid\": {\"hota\": 67.941, \"idf1\": 79.985},\n", + " \"mot17_th02\": {\"hota\": 65.878, \"idf1\": 78.832},\n", + " },\n", + " \"MOT17-10\": {\n", + " \"no_reid\": {\"hota\": 57.204, \"idf1\": 76.157},\n", + " \"mot17_th02\": {\"hota\": 59.565, \"idf1\": 81.087},\n", + " },\n", + " \"MOT17-11\": {\n", + " \"no_reid\": {\"hota\": 66.697, \"idf1\": 77.326},\n", + " \"mot17_th02\": {\"hota\": 66.699, \"idf1\": 77.326},\n", + " },\n", + " \"MOT17-13\": {\n", + " \"no_reid\": {\"hota\": 69.833, \"idf1\": 89.533},\n", + " \"mot17_th02\": {\"hota\": 69.791, \"idf1\": 89.431},\n", + " },\n", + "}\n", + "\n", + "\n", + "def ref_seq_key(seq: str) -> str:\n", + " parts = seq.split(\"-\")\n", + " return f\"{parts[0]}-{parts[1]}\"\n", + "\n", + "\n", + "for seq in ACTIVE_SEQUENCES:\n", + " key = ref_seq_key(seq)\n", + " ref = REID_STUDY_PER_SEQ.get(key, {})\n", + " print(seq)\n", + " print(f\" {'Config':<28} {'HOTA':>6} {'IDF1':>6} {'IDSW':>5} {'Ref H':>6} {'dH':>6} {'Ref I':>6} {'dI':>6}\")\n", + " for label, res in botsort_rows:\n", + " hota, assa, idf1, idsw = seq_metrics(res, seq)\n", + " ref_key = \"no_reid\" if \"baseline\" in label else \"mot17_th02\"\n", + " ref_vals = ref.get(ref_key, {})\n", + " ref_hota = ref_vals.get(\"hota\", float(\"nan\"))\n", + " ref_idf1 = ref_vals.get(\"idf1\", float(\"nan\"))\n", + " delta_h = hota - ref_hota if ref_hota == ref_hota else float(\"nan\")\n", + " delta_i = idf1 - ref_idf1 if ref_idf1 == ref_idf1 else float(\"nan\")\n", + " ref_h_s = f\"{ref_hota:6.2f}\" if ref_hota == ref_hota else \" n/a\"\n", + " ref_i_s = f\"{ref_idf1:6.2f}\" if ref_idf1 == ref_idf1 else \" n/a\"\n", + " delta_h_s = f\"{delta_h:+6.2f}\" if delta_h == delta_h else \" n/a\"\n", + " delta_i_s = f\"{delta_i:+6.2f}\" if delta_i == delta_i else \" n/a\"\n", + " print(f\" {label:<28} {hota:6.2f} {idf1:6.2f} {idsw:5d} {ref_h_s} {delta_h_s} {ref_i_s} {delta_i_s}\")\n", + " print()" + ] + }, + { + "cell_type": "markdown", + "id": "8de54c38", + "metadata": {}, + "source": [ + "### 8. Visual comparison - largest ReID gain sequence\n", + "\n", + "Side-by-side **baseline vs +ReID** video for the val sequence with the largest \u0394HOTA\n", + "(from the runs above). On Colab the mp4 is downloaded automatically.\n", + "" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9a4f4194", + "metadata": {}, + "outputs": [], + "source": [ + "# Auto-pick the sequence with the largest HOTA gain (override with COMPARE_SEQ = \"MOT17-02-FRCNN\").\n", + "COMPARE_SEQ: str | None = None\n", + "COMPARE_FPS = 30\n", + "COMPARE_MAX_FRAMES: int | None = None # None = full sequence\n", + "\n", + "\n", + "def _mot_frame_to_detections(mot: dict, frame_idx: int) -> sv.Detections:\n", + " frame = mot.get(frame_idx)\n", + " if frame is None:\n", + " return sv.Detections.empty()\n", + " active = frame.ids >= 0\n", + " if not np.any(active):\n", + " return sv.Detections.empty()\n", + " return sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame.boxes[active]).astype(np.float32),\n", + " tracker_id=frame.ids[active].astype(int),\n", + " confidence=frame.confidences[active].astype(np.float32),\n", + " )\n", + "\n", + "\n", + "def _annotate_tracks(frame_bgr: np.ndarray, detections: sv.Detections) -> np.ndarray:\n", + " if len(detections) == 0:\n", + " return frame_bgr\n", + " palette, lookup = sv.ColorPalette.DEFAULT, sv.ColorLookup.TRACK\n", + " scene = sv.BoxAnnotator(color=palette, color_lookup=lookup, thickness=2).annotate(frame_bgr, detections)\n", + " labels = [str(int(tid)) for tid in detections.tracker_id]\n", + " return sv.LabelAnnotator(\n", + " color=palette,\n", + " color_lookup=lookup,\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " ).annotate(scene, detections, labels=labels)\n", + "\n", + "\n", + "def _panel_badge(frame: np.ndarray, text: str, accent: tuple[int, int, int]) -> np.ndarray:\n", + " out = frame.copy()\n", + " font, scale, thick = cv2.FONT_HERSHEY_SIMPLEX, 0.7, 2\n", + " (tw, th), _ = cv2.getTextSize(text, font, scale, thick)\n", + " x, y, pad, bar = 12, 12, 10, 6\n", + " w, h = tw + 2 * pad + bar + 8, th + 2 * pad\n", + " cv2.rectangle(out, (x, y), (x + w, y + h), (24, 24, 28), -1)\n", + " cv2.rectangle(out, (x + 6, y + 6), (x + 6 + bar, y + h - 6), accent, -1)\n", + " cv2.putText(out, text, (x + bar + pad + 4, y + pad + th), font, scale, (245, 245, 245), thick, cv2.LINE_AA)\n", + " return out\n", + "\n", + "\n", + "seq_gains: list[tuple[str, float, float, float]] = []\n", + "for seq in ACTIVE_SEQUENCES:\n", + " h_b, _, i_b, _ = seq_metrics(result_baseline, seq)\n", + " h_r, _, i_r, _ = seq_metrics(result_reid, seq)\n", + " if h_b == h_b and h_r == h_r:\n", + " seq_gains.append((seq, h_r - h_b, i_r - i_b, h_r))\n", + "\n", + "if not seq_gains:\n", + " raise RuntimeError(\"No per-sequence metrics - run sections 5-7 first.\")\n", + "\n", + "seq_gains.sort(key=lambda row: row[1], reverse=True)\n", + "print(\"Per-sequence ReID \u0394HOTA (largest first):\")\n", + "for seq, dh, di, _ in seq_gains:\n", + " print(f\" {seq:<20} \u0394HOTA {dh:+6.2f} \u0394IDF1 {di:+6.2f}\")\n", + "\n", + "COMPARE_SEQ = COMPARE_SEQ or seq_gains[0][0]\n", + "delta_hota, delta_idf1 = next((dh, di) for s, dh, di, _ in seq_gains if s == COMPARE_SEQ)\n", + "print(f\"\\nRendering comparison for {COMPARE_SEQ} (\u0394HOTA {delta_hota:+.2f}, \u0394IDF1 {delta_idf1:+.2f})\")\n", + "\n", + "pred_base = OUTPUT_ROOT / \"botsort_baseline\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "pred_reid = OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{COMPARE_SEQ}.txt\"\n", + "if not pred_base.is_file() or not pred_reid.is_file():\n", + " raise FileNotFoundError(f\"Missing preds for {COMPARE_SEQ}:\\n {pred_base}\\n {pred_reid}\")\n", + "\n", + "mot_base = load_mot_file(pred_base)\n", + "mot_reid = load_mot_file(pred_reid)\n", + "img_dir = SEQUENCE_PATHS[COMPARE_SEQ][\"img\"]\n", + "n_frames = SEQUENCE_PATHS[COMPARE_SEQ][\"n_frames\"]\n", + "if COMPARE_MAX_FRAMES is not None:\n", + " n_frames = min(n_frames, COMPARE_MAX_FRAMES)\n", + "\n", + "compare_fps = COMPARE_FPS\n", + "seqinfo = img_dir.parent / \"seqinfo.ini\"\n", + "if seqinfo.is_file():\n", + " for line in seqinfo.read_text().splitlines():\n", + " if line.startswith(\"frameRate=\"):\n", + " compare_fps = int(line.split(\"=\", 1)[1])\n", + " break\n", + "\n", + "sample = load_mot_frame_image(img_dir, 1)\n", + "h, w = sample.shape[:2]\n", + "\n", + "out_path = OUTPUT_ROOT / f\"compare_{COMPARE_SEQ}_baseline_vs_reid.mp4\"\n", + "video_info = sv.VideoInfo(width=w * 2, height=h, fps=compare_fps, total_frames=n_frames)\n", + "\n", + "with sv.VideoSink(target_path=str(out_path), video_info=video_info) as sink:\n", + " for frame_idx in range(1, n_frames + 1):\n", + " frame = load_mot_frame_image(img_dir, frame_idx)\n", + " left = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_base, frame_idx)),\n", + " \"BASELINE (NO REID)\",\n", + " (0, 165, 255),\n", + " )\n", + " right = _panel_badge(\n", + " _annotate_tracks(frame.copy(), _mot_frame_to_detections(mot_reid, frame_idx)),\n", + " \"BOT-SORT + REID\",\n", + " (80, 200, 120),\n", + " )\n", + " sink.write_frame(np.hstack([left, right]))\n", + "\n", + "ffmpeg = shutil.which(\"ffmpeg\")\n", + "if ffmpeg is not None:\n", + " tmp = out_path.with_name(f\"{out_path.stem}.web.mp4\")\n", + " result = subprocess.run( # noqa: S603\n", + " [\n", + " ffmpeg,\n", + " \"-y\",\n", + " \"-i\",\n", + " str(out_path),\n", + " \"-c:v\",\n", + " \"libx264\",\n", + " \"-pix_fmt\",\n", + " \"yuv420p\",\n", + " \"-movflags\",\n", + " \"+faststart\",\n", + " \"-an\",\n", + " str(tmp),\n", + " ],\n", + " capture_output=True,\n", + " text=True,\n", + " )\n", + " if result.returncode == 0:\n", + " tmp.replace(out_path)\n", + "\n", + "print(f\"Wrote {out_path} ({n_frames} frames @ {compare_fps} fps)\")\n", + "ipy_display(Video(str(out_path), embed=True, width=min(960, w * 2)))\n", + "if IN_COLAB:\n", + " files.download(str(out_path))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } From eba091644e44b1647c366931a062596c6655c26c Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Thu, 23 Jul 2026 17:19:54 -0300 Subject: [PATCH 53/54] docs(reid): cite BoT-SORT update_features on FeatureBank Co-authored-by: Cursor --- src/trackers/core/reid/feature_bank.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py index 1969d34c0..5f5439285 100644 --- a/src/trackers/core/reid/feature_bank.py +++ b/src/trackers/core/reid/feature_bank.py @@ -14,7 +14,11 @@ class FeatureBank: - """Per-track EMA unit embedding (L2 before and after blend).""" + """Per-track EMA unit embedding (L2 before and after blend). + + Follows BoT-SORT ``STrack.update_features`` + (https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py). + """ def __init__(self, alpha: float = 0.9) -> None: if not 0.0 <= alpha <= 1.0: From d7c04238737671405f3a3a83ebb6ccc955e0cfc5 Mon Sep 17 00:00:00 2001 From: Alex Bodner Date: Tue, 28 Jul 2026 14:13:16 -0300 Subject: [PATCH 54/54] feat(benchmark): wire optional BoT-SORT ReID into Codabench track flow Pass REID_ENCODER / APPEARANCE_THRESHOLD through Makefile to track_split, map MOT17 det stems to FRCNN frame folders, and install trackers[tune,reid]. Co-authored-by: Cursor --- benchmark/Makefile | 12 +++++-- benchmark/README.md | 10 ++++-- benchmark/scripts/datasets.py | 36 +++++++++++++++++++-- benchmark/scripts/track_split.py | 54 +++++++++++++++++++++++++++++--- 4 files changed, 98 insertions(+), 14 deletions(-) diff --git a/benchmark/Makefile b/benchmark/Makefile index 29994ca6d..6fb169cd2 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -48,6 +48,9 @@ OBJECTIVE ?= HOTA THRESHOLD ?= 0.5 SEED ?= FIXED_PARAMS ?= +# Optional BoT-SORT appearance: set REID_ENCODER=fastreid_mot17_sbs50 to enable. +REID_ENCODER ?= +APPEARANCE_THRESHOLD ?= 0.2 CODABENCH_URL ?= https://www.codabench.org CODABENCH_TOKEN ?= @@ -91,7 +94,7 @@ help: @echo "MOT benchmark workflow — run from \`cd benchmark\`" @echo "" @echo "Targets:" - @echo " setup Install \`trackers[tune]\` from $(REPO_ROOT)" + @echo " setup Install \`trackers[tune,reid]\` from $(REPO_ROOT)" @echo " data-check Print present/missing assets under $(DATA_ROOT)" @echo " prep Prep one dataset (DATASET=...) into $(PREP_DIR)" @echo " prep-all Prep every dataset" @@ -109,10 +112,11 @@ help: @echo " poll Poll an existing Codabench submission (SUBMISSION_ID=, DATASET=, CONFIG=)" @echo " clean Remove $(PREP_DIR) and $(OUTPUT_DIR)" @echo "" + @echo "BoT-SORT ReID: pass REID_ENCODER=fastreid_mot17_sbs50 APPEARANCE_THRESHOLD=0.2" @echo "Codabench upload requires CODABENCH_TOKEN. See README for data setup." setup: - $(UV) pip install -e "$(REPO_ROOT)[tune]" + $(UV) pip install -e "$(REPO_ROOT)[tune,reid]" data-check: @$(PYTHON) scripts/data_check.py --data-root "$(DATA_ROOT)" @@ -162,7 +166,9 @@ _track-and-score: --tracker $(TRACKER) --dataset $(DATASET) --split $(SCORE_SPLIT) \ --data-root "$(DATA_ROOT)" --prep-dir "$(PREP_DIR)" \ --output-dir "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/$(CONFIG)" \ - $(if $(filter tuned,$(CONFIG)),--params "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json",) + $(if $(filter tuned,$(CONFIG)),--params "$(OUTPUT_DIR)/$(TRACKER)/$(DATASET)/best_params.json",) \ + $(if $(REID_ENCODER),--reid "$(REID_ENCODER)",) \ + $(if $(and $(REID_ENCODER),$(APPEARANCE_THRESHOLD)),--appearance-threshold $(APPEARANCE_THRESHOLD),) @if [ "$(DATASET)" = "soccernet" ]; then \ trackers eval \ --gt-dir "$(PREP_DIR)/soccernet/test/gt" \ diff --git a/benchmark/README.md b/benchmark/README.md index a8e48dc8c..5ea358457 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -69,7 +69,7 @@ Point `DATA_ROOT` at the folder that directly contains `mot17/`, `sportsmot/`, e $DATA_ROOT/ mot17/MOT17_yolox_dets/{val,test}/... mot17/TrackEval/data/gt/MOT17_yolox_val/train_val/... - mot17/{val,test}//img1/... # BoT-SORT CMC only + mot17/{val,test}//img1/... # BoT-SORT CMC / ReID sportsmot/sportsmot_yolox_dets/{val,test}/... sportsmot/TrackEval/data/gt/sportsmot/val/... dancetrack/dancetrack_yolox_dets/{train,val,test}/... @@ -120,7 +120,7 @@ Run from `benchmark/`. Pass variables on the command line or export them first ( | Target | Description | | ------------------------------ | ------------------------------------------------------------------------------------------------------ | -| `setup` | Install `trackers[tune]` from the repo root | +| `setup` | Install `trackers[tune,reid]` from the repo root | | `data-check` | Print present/missing assets under `DATA_ROOT` | | `prep` | Prep one dataset (`DATASET=…`) into `benchmark_prep/` | | `prep-all` | Prep all four datasets | @@ -184,6 +184,8 @@ Single dataset or step: make prep DATASET=mot17 make tune TRACKER=bytetrack DATASET=mot17 N_TRIALS=50 make track-default TRACKER=bytetrack DATASET=mot17 +make track-default TRACKER=botsort DATASET=mot17 \ + REID_ENCODER=fastreid_mot17_sbs50 APPEARANCE_THRESHOLD=0.2 make track-tuned TRACKER=bytetrack DATASET=mot17 make upload TRACKER=bytetrack DATASET=mot17 CONFIG=tuned make collect TRACKER=bytetrack @@ -205,8 +207,10 @@ make clean | `CODABENCH_TOKEN` | — | Required for Codabench datasets | | `PREP_DIR` | `./benchmark_prep` | Prepared flat MOT dets/GT | | `OUTPUT_DIR` | `./benchmark_outputs` | Params, preds, scores, tables | +| `REID_ENCODER` | — | Optional BoT-SORT ReID alias/path (e.g. `fastreid_mot17_sbs50`); requires `trackers[reid]` | +| `APPEARANCE_THRESHOLD` | `0.2` | Passed with `REID_ENCODER` as BoT-SORT `appearance_threshold` | -BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories when present. +BoT-SORT sets `FIXED_PARAMS={"enable_cmc": true}` and uses frame directories when CMC and/or `REID_ENCODER` is set. ## Notes diff --git a/benchmark/scripts/datasets.py b/benchmark/scripts/datasets.py index 2733ff1a6..0afee3ddf 100644 --- a/benchmark/scripts/datasets.py +++ b/benchmark/scripts/datasets.py @@ -55,9 +55,25 @@ def _soccernet_seq(stem: str) -> str: def _mot17_val_seq(stem: str) -> str: + """Map YOLOX val det stem (e.g. MOT17-02_val) to the FRCNN GT/image folder name.""" return stem.split("_")[0] + "-FRCNN" +def _mot17_frame_seq(stem: str) -> str: + """Map any MOT17 seq id to the FRCNN folder that holds shared ``img1`` frames. + + YOLOX test dets are ``MOT17-01``; official sequences are ``MOT17-01-{FRCNN,SDP,DPM}`` + with frames only under the FRCNN copy (SDP/DPM are detector variants of the same video). + """ + base = stem.split("_", 1)[0] + for suf in _MOT17_SUFFIXES: + suffix = f"-{suf}" + if base.endswith(suffix): + base = base[: -len(suffix)] + break + return f"{base}-FRCNN" + + @dataclass(frozen=True) class SplitPaths: det_dir: Path @@ -66,6 +82,7 @@ class SplitPaths: images_dir: Path | None seqmap: Path | None seq_name_fn: Callable[[str], str] | None = None + image_seq_name_fn: Callable[[str], str] | None = None def split_paths(data_root: Path, dataset: str, split: str) -> SplitPaths: @@ -114,9 +131,18 @@ def split_paths(data_root: Path, dataset: str, split: str) -> SplitPaths: root / "mot17/val", seqmap if seqmap.is_file() else None, _mot17_val_seq, + _mot17_frame_seq, ) if split == "test": - return SplitPaths(root / "mot17/MOT17_yolox_dets/test", "xyxy", None, root / "mot17/test", None) + return SplitPaths( + root / "mot17/MOT17_yolox_dets/test", + "xyxy", + None, + root / "mot17/test", + None, + None, + _mot17_frame_seq, + ) raise ValueError(f"unknown (dataset, split): ({dataset!r}, {split!r})") @@ -131,8 +157,12 @@ def job_dir(output_dir: Path, tracker: str, dataset: str) -> Path: def needs_frames(tracker: str, params: dict) -> bool: - """Whether tracking requires source frames (e.g. BoT-SORT with CMC enabled).""" - return tracker == "botsort" and bool(params.get("enable_cmc", False)) + """Whether tracking requires source frames (CMC and/or ReID appearance).""" + if tracker != "botsort": + return False + if bool(params.get("enable_cmc", False)): + return True + return params.get("reid_model") is not None def mot17_server_filenames() -> tuple[tuple[str, ...], tuple[str, ...], tuple[str, ...]]: diff --git a/benchmark/scripts/track_split.py b/benchmark/scripts/track_split.py index 807d965bb..ef6e04d30 100644 --- a/benchmark/scripts/track_split.py +++ b/benchmark/scripts/track_split.py @@ -23,6 +23,12 @@ --data-root ./data --prep-dir ./benchmark_prep \ --output-dir ./benchmark_outputs/sort/mot17/default \ [--params best_params.json] + + # BoT-SORT + official FastReID SBS (requires `pip install 'trackers[reid]'`): + python track_split.py --tracker botsort --dataset mot17 --split test \ + --data-root ./data --prep-dir ./benchmark_prep \ + --output-dir ./benchmark_outputs/botsort/mot17/default \ + --reid fastreid_mot17_sbs50 --appearance-threshold 0.2 """ from __future__ import annotations @@ -72,6 +78,22 @@ def _resolve_params(tracker_id: str, *, params_file: Path | None) -> dict[str, A return _init_kwargs(tracker_id, merged) +def _load_reid_model(source: str, appearance_threshold: float | None) -> dict[str, Any]: + """Load a ``reid.ReIDModel`` and optional appearance threshold overrides.""" + try: + from reid import ReIDModel + except ImportError as exc: + raise ImportError( + "BoT-SORT ReID requires the standalone reid package. " + "Install with: pip install 'trackers[reid]'" + ) from exc + + overrides: dict[str, Any] = {"reid_model": ReIDModel.from_pretrained(source)} + if appearance_threshold is not None: + overrides["appearance_threshold"] = appearance_threshold + return overrides + + def _build(tracker_id: str, params: dict[str, Any]) -> BaseTracker: info = BaseTracker._lookup_tracker(tracker_id) if info is None: @@ -88,19 +110,39 @@ def main(argv: list[str] | None = None) -> int: p.add_argument("--prep-dir", type=Path, required=True) p.add_argument("--output-dir", type=Path, required=True, help="Predictions root: writes pred/.txt under here.") p.add_argument("--params", type=Path, default=None, help="Optional tuned best_params.json") + p.add_argument( + "--reid", + default=None, + help="Optional reid alias/path for BoT-SORT (e.g. fastreid_mot17_sbs50).", + ) + p.add_argument( + "--appearance-threshold", + type=float, + default=None, + help="BoT-SORT appearance_threshold when --reid is set (e.g. 0.2).", + ) args = p.parse_args(argv) + if args.reid is not None and args.tracker != "botsort": + print(f"--reid is only supported for botsort (got {args.tracker!r})", file=sys.stderr) + return 1 + if args.appearance_threshold is not None and args.reid is None: + print("--appearance-threshold requires --reid", file=sys.stderr) + return 1 + dets_dir = args.prep_dir / args.dataset / args.split / "dets" if not dets_dir.is_dir(): print(f"missing prepared dets: {dets_dir} (run `make prep DATASET={args.dataset}`)", file=sys.stderr) return 1 params = _resolve_params(args.tracker, params_file=args.params) - images_dir = ( - split_paths(args.data_root, args.dataset, args.split).images_dir if needs_frames(args.tracker, params) else None - ) + if args.reid is not None: + params.update(_load_reid_model(args.reid, args.appearance_threshold)) + + paths = split_paths(args.data_root, args.dataset, args.split) + images_dir = paths.images_dir if needs_frames(args.tracker, params) else None if images_dir is not None and not images_dir.is_dir(): - print(f"missing frames for CMC: {images_dir}", file=sys.stderr) + print(f"missing frames for CMC/ReID: {images_dir}", file=sys.stderr) return 1 pred_dir = args.output_dir / "pred" @@ -108,13 +150,15 @@ def main(argv: list[str] | None = None) -> int: tracker = _build(args.tracker, params) for det_path in sorted(dets_dir.glob("*.txt")): seq = det_path.stem + # Pred names stay as det stems (MOT17-01); frames may live under MOT17-01-FRCNN. + image_seq = paths.image_seq_name_fn(seq) if paths.image_seq_name_fn is not None else seq tracker.reset() _run_tracker_on_detections( tracker, det_path, pred_dir / f"{seq}.txt", images_dir=images_dir, - seq_name=seq, + seq_name=image_seq, ) print(f" tracked {seq}") return 0