diff --git a/.github/workflows/ci-integrations.yml b/.github/workflows/ci-integrations.yml index 185333621..f44d7faf8 100644 --- a/.github/workflows/ci-integrations.yml +++ b/.github/workflows/ci-integrations.yml @@ -28,7 +28,7 @@ jobs: activate-environment: true - name: 🚀 Install Packages - run: uv sync --frozen --group dev + run: uv sync --frozen --group dev --extra reid - name: 🧪 Run Integration Tests diff --git a/.github/workflows/ci-reid-prerelease.yml b/.github/workflows/ci-reid-prerelease.yml new file mode 100644 index 000000000..dc6793fa5 --- /dev/null +++ b/.github/workflows/ci-reid-prerelease.yml @@ -0,0 +1,31 @@ +name: ReID Pre-release Compatibility + +on: + schedule: + - cron: "17 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +jobs: + reid-integration-smoke: + name: ReID Integration Smoke + timeout-minutes: 15 + runs-on: ubuntu-latest + + steps: + - name: 📥 Checkout the repository + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: 🐍 Install uv and set Python version + uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 + with: + python-version: "3.12" + activate-environment: true + + - name: 🚀 Install Packages with the latest allowed reid pre-release + run: uv sync --group dev --extra reid --upgrade-package reid + + - name: 🧪 Smoke-test the trackers to reid boundary + run: uv run pytest tests/core/test_botsort_reid.py::TestBoTSORTTrackerReID::test_real_reid_model_runs_over_frames -v --tb=short diff --git a/README.md b/README.md index b88c008ce..0a2335fe6 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ - **Benchmarked across four datasets.** MOT17, SportsMOT, SoccerNet, and DanceTrack — at default parameters and after hyperparameter tuning (McByte: defaults only, by design), so you know what to expect before you deploy. - **Tunable with one extra.** Optuna-based hyperparameter search via `trackers tune` (`pip install "trackers[tune]"`) so you can optimize for your specific scene and detector. - **Camera motion compensation.** BoT-SORT and McByte handle moving cameras natively, keeping track IDs stable even when the whole frame shifts. +- **Optional appearance ReID.** BoT-SORT can fuse visual embeddings with motion for harder association scenes: install `trackers[reid]` (pulls in the [`reid`](https://github.com/roboflow/re-ID) package), pass a `reid.ReIDModel` as `reid_model`, and supply `frame=` to `update()`. ## Install diff --git a/docs/adr/0001-model-backend-externalization.md b/docs/adr/0001-model-backend-externalization.md new file mode 100644 index 000000000..ba48d6cfb --- /dev/null +++ b/docs/adr/0001-model-backend-externalization.md @@ -0,0 +1,33 @@ +--- +title: 'ADR: Model-backend externalization: vendor vs external package' +description: Decision criteria for keeping model-backend integration in Trackers or consuming it from a separate package. +--- + +# Model-backend externalization: vendor vs external package + +- Status: Accepted +- Date: 2026-08-14 + +## Context + +Trackers needs learned-model backends without making them part of its core tracking API. The existing mask pipeline keeps its SAM and Cutie integration in `trackers.core.masks`: Trackers owns checkpoint discovery, preprocessing, inference lifecycle, and conversion to the mask protocols. The upstream model packages and weights remain optional dependencies; “vendor” here means owning the backend integration code, not copying upstream model source. + +Appearance ReID has a different boundary. Trackers owns the `ReIDEncoder` protocol, feature bank, association helpers, and threshold-analysis tools. Model architectures, pretrained checkpoints, preprocessing, training, and gallery evaluation live in the standalone `reid` package, installed by the `reid` extra. + +## Decision + +Keep the ReID model backend in the external `reid` package. Trackers consumes it through the small `ReIDEncoder.extract_features(detections, frame)` contract and does not duplicate model-loading or model-catalog code. + +Choose the ownership boundary for future learned backends using these criteria: + +- Keep integration in Trackers when it is tightly coupled to tracker state or frame-to-frame lifecycle, and a small, stable set of backends implements a Trackers-owned protocol. +- Use an external package when model architectures, training, preprocessing, checkpoint catalogs, or release cadence form a substantial product surface independent of multi-object tracking. +- In both cases, import heavy optional dependencies lazily, expose a lightweight protocol from Trackers, and keep tracker behavior usable without the extra. +- Revisit an in-repo backend if its integration grows an independently useful model API; revisit an external backend if its boundary cannot express required tracker lifecycle semantics without leaking implementation details. + +## Consequences + +- `trackers` stays focused on tracking and association while `reid` can evolve models and training independently. +- Users install `trackers[reid]` for the supported ReID backend; custom encoders may implement `ReIDEncoder` without depending on that package. +- Compatibility is enforced at the protocol and optional-dependency version range, so cross-package changes require coordinated tests and releases. +- The masks and ReID layouts remain intentionally asymmetric: mask lifecycle adapters are Trackers-owned, while ReID model implementations are external. diff --git a/docs/api/reid.md b/docs/api/reid.md new file mode 100644 index 000000000..7026e0563 --- /dev/null +++ b/docs/api/reid.md @@ -0,0 +1,75 @@ +--- +description: Python API reference for the ReID encoder protocol, feature bank, appearance association utilities, and threshold-selection plots in Roboflow Trackers. +--- + +# ReID API + +Requires the `reid` extra (`pip install "trackers[reid]"`, see the [install guide](../guides/install.md)). + +This page covers the `ReIDEncoder` protocol, `FeatureBank`, appearance association helpers, and the threshold-selection plots in `trackers.core.reid`. For enabling appearance on BoT-SORT and for benchmark results, see the [ReID appearance guide](../guides/reid.md). Model loading and gallery evaluation are in the standalone [`reid`](https://github.com/roboflow/re-ID) package. + +## ReIDEncoder + +::: trackers.core.reid.encoder.ReIDEncoder + +## FeatureBank + +::: trackers.core.reid.feature_bank.FeatureBank + +## appearance_similarity + +::: trackers.core.reid.appearance.appearance_similarity + +## extract_detection_embeddings + +::: trackers.core.reid.appearance.extract_detection_embeddings + +## Choosing a threshold + +Measure your own encoder on your own footage instead of inheriting a threshold from a paper. These helpers embed a labeled dataset, sample the distances a tracker actually sees, plot them, and report separability. Plotting needs `matplotlib`, which ships with the `reid` extra. + +Both plot functions take their reference lines as `ThresholdLines`, either a sequence of values or a mapping from value to annotation. + +```python +from trackers.core.reid import ( + extract_ground_truth_embeddings, + plot_appearance_distances, + plot_frame_gap_sweep, + sample_appearance_distances, + sweep_frame_gap, +) + +embeddings, ids, frame_ids, sequence_ids = extract_ground_truth_embeddings(model, "mot17/val", keep_classes=(1,)) +distances = sample_appearance_distances(embeddings, ids, frame_ids, sequence_ids) +same_id_rate, different_id_rate = distances.rates_at(0.25) +plot_appearance_distances(distances, thresholds={0.20: "selected", 0.25: "default"}) +plot_frame_gap_sweep(sweep_frame_gap(embeddings, ids, frame_ids, sequence_ids)) +``` + +### extract_ground_truth_embeddings + +::: trackers.core.reid.appearance.extract_ground_truth_embeddings + +### AppearanceDistances + +::: trackers.core.reid.thresholds.AppearanceDistances + +### sample_appearance_distances + +::: trackers.core.reid.thresholds.sample_appearance_distances + +### sweep_frame_gap + +::: trackers.core.reid.thresholds.sweep_frame_gap + +### roc_auc + +::: trackers.core.reid.thresholds.roc_auc + +### plot_appearance_distances + +::: trackers.core.reid.thresholds.plot_appearance_distances + +### plot_frame_gap_sweep + +::: trackers.core.reid.thresholds.plot_frame_gap_sweep diff --git a/docs/assets/reid/mot17-fastreid-appearance-distances-vs-gap.png b/docs/assets/reid/mot17-fastreid-appearance-distances-vs-gap.png new file mode 100644 index 000000000..8303d8fda Binary files /dev/null and b/docs/assets/reid/mot17-fastreid-appearance-distances-vs-gap.png differ diff --git a/docs/assets/reid/mot17-fastreid-appearance-distances.png b/docs/assets/reid/mot17-fastreid-appearance-distances.png new file mode 100644 index 000000000..68f93c7e9 Binary files /dev/null and b/docs/assets/reid/mot17-fastreid-appearance-distances.png differ diff --git a/docs/assets/reid/soccernet-osnet-appearance-distances-vs-gap.png b/docs/assets/reid/soccernet-osnet-appearance-distances-vs-gap.png new file mode 100644 index 000000000..75dd587e1 Binary files /dev/null and b/docs/assets/reid/soccernet-osnet-appearance-distances-vs-gap.png differ diff --git a/docs/assets/reid/soccernet-osnet-appearance-distances.png b/docs/assets/reid/soccernet-osnet-appearance-distances.png new file mode 100644 index 000000000..61858b26d Binary files /dev/null and b/docs/assets/reid/soccernet-osnet-appearance-distances.png differ diff --git a/docs/cookbooks/how-to-add-reid-to-trackers.ipynb b/docs/cookbooks/how-to-add-reid-to-trackers.ipynb new file mode 100644 index 000000000..74cf9972d --- /dev/null +++ b/docs/cookbooks/how-to-add-reid-to-trackers.ipynb @@ -0,0 +1,531 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "[![Roboflow Notebooks](https://media.roboflow.com/notebooks/template/bannertest2-2.png?ik-sdk-version=javascript-1.4.3&updatedAt=1672932710194)](https://github.com/roboflow/notebooks)\n", + "\n", + "# How to Add ReID to Trackers\n", + "\n", + "[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/roboflow/trackers/blob/develop/docs/cookbooks/how-to-add-reid-to-trackers.ipynb)\n", + "\n", + "BoT-SORT associates a detection with a track by how much their boxes overlap. When two people cross, the boxes overlap about equally and geometry alone picks the wrong one. Appearance ReID adds a second opinion: an encoder turns each crop into an embedding, and the appearance distance between a track and a detection breaks the tie.\n", + "\n", + "This notebook enables ReID on BoT-SORT with the [`reid`](https://reid.roboflow.com/latest/) package, measures what it buys on MOT17 val-half against the same run without it, and then calibrates `reid_appearance_threshold` on the data instead of inheriting it from a paper.\n", + "\n", + "Runtime: about 30 minutes on a Colab T4, most of it encoding crops." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup\n", + "\n", + "### Check GPU availability\n", + "\n", + "An encoder runs on every high-confidence detection in every frame, so a GPU is the difference between minutes and hours. If the cell below fails, switch the runtime type to GPU." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!nvidia-smi" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install dependencies\n", + "\n", + "The `reid` extra pulls in the encoder package and matplotlib. `gdown` fetches the YOLOX detections from Google Drive." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "!pip install -q \"trackers[reid]\" gdown" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Imports" + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "import warnings\n", + "from pathlib import Path\n", + "\n", + "import cv2\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import supervision as sv\n", + "import torch\n", + "from reid import FASTREID_MOT17_SBS50, ReIDModel\n", + "\n", + "from trackers import BoTSORTTracker\n", + "from trackers.core.reid import (\n", + " extract_ground_truth_embeddings,\n", + " plot_appearance_distances,\n", + " plot_frame_gap_sweep,\n", + " sample_appearance_distances,\n", + " sweep_frame_gap,\n", + ")\n", + "from trackers.eval import evaluate_mot_sequences\n", + "from trackers.io.frames import load_mot_frame_image\n", + "from trackers.io.mot import load_mot_file\n", + "\n", + "warnings.filterwarnings(\"ignore\")\n", + "\n", + "device = torch.cuda.get_device_name(0) if torch.cuda.is_available() else \"cpu\"\n", + "print(f\"PyTorch {torch.__version__} | CUDA {torch.cuda.is_available()} | {device}\")\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Download MOT17 val-half and YOLOX detections\n", + "\n", + "Both runs share one set of detections, so any difference in the metrics comes from association rather than detection quality. The YOLOX detections are the ones the BoT-SORT paper used, which is what makes the numbers at the end comparable to published results." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "ROOT = Path(\"mot17-reid\")\n", + "MOT17_VAL = ROOT / \"mot17\" / \"val\"\n", + "YOLOX_DIR = ROOT / \"MOT17_yolox_dets\"\n", + "YOLOX_ZIP = YOLOX_DIR / \"yolox_detections_MOT17.zip\"\n", + "YOLOX_GDRIVE_ID = \"1BuXtPWf8QbPU_y1i2xY2IbTE-rj3l6qT\"\n", + "OUTPUT_ROOT = ROOT / \"outputs\"\n", + "OUTPUT_ROOT.mkdir(parents=True, exist_ok=True)\n", + "\n", + "!trackers download --name mot17 --split val --asset annotations,frames --output {ROOT}\n", + "\n", + "!mkdir -p {YOLOX_DIR}\n", + "!gdown {YOLOX_GDRIVE_ID} -O {YOLOX_ZIP}\n", + "!unzip -qo {YOLOX_ZIP} -d {YOLOX_DIR}" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "VAL_SEQUENCES = [\n", + " \"MOT17-02-FRCNN\",\n", + " \"MOT17-04-FRCNN\",\n", + " \"MOT17-05-FRCNN\",\n", + " \"MOT17-09-FRCNN\",\n", + " \"MOT17-10-FRCNN\",\n", + " \"MOT17-11-FRCNN\",\n", + " \"MOT17-13-FRCNN\",\n", + "]\n", + "\n", + "SEQUENCES: dict[str, dict] = {}\n", + "for seq in VAL_SEQUENCES:\n", + " ground_truth = MOT17_VAL / seq / \"gt\" / \"gt.txt\"\n", + " images = MOT17_VAL / seq / \"img1\"\n", + " detections = YOLOX_DIR / \"val\" / f\"{seq.replace('-FRCNN', '')}_val.txt\"\n", + " if not (ground_truth.is_file() and images.is_dir() and detections.is_file()):\n", + " print(f\" skip {seq}: missing gt, img1, or YOLOX detections\")\n", + " continue\n", + " frames = sorted(images.glob(\"*.jpg\"))\n", + " SEQUENCES[seq] = {\n", + " \"ground_truth\": ground_truth,\n", + " \"images\": images,\n", + " \"detections\": detections,\n", + " \"frames\": frames,\n", + " }\n", + " print(f\" {seq}: {len(frames)} frames\")\n", + "\n", + "if not SEQUENCES:\n", + " raise RuntimeError(\"No sequences ready, re-run the download cell above.\")\n", + "\n", + "SEQMAP = OUTPUT_ROOT / \"MOT17-val.txt\"\n", + "SEQMAP.write_text(\"name\\n\" + \"\\n\".join(SEQUENCES) + \"\\n\")\n", + "print(f\"\\n{len(SEQUENCES)} sequences ready\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Load the ReID encoder\n", + "\n", + "`fastreid_mot17_sbs50` was trained on MOT17, so it is the encoder to beat on this benchmark. Any other id from the `reid` package, or your own checkpoint, drops in here unchanged.\n", + "\n", + "`reid_appearance_threshold` is the distance above which BoT-SORT stops trusting appearance. It defaults to 0.25, the value the BoT-SORT paper uses. We start from 0.2 instead and check that choice against the data further down." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "REID_ENCODER = FASTREID_MOT17_SBS50\n", + "APPEARANCE_THRESHOLD = 0.2\n", + "\n", + "reid_model = ReIDModel.from_pretrained(REID_ENCODER)\n", + "print(f\"Encoder: {REID_ENCODER} | reid_appearance_threshold: {APPEARANCE_THRESHOLD}\")\n", + "print(reid_model.preprocessing.describe())" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Run BoT-SORT with and without ReID\n", + "\n", + "Passing `reid_model` is the whole change. Everything else, detections included, is held fixed between the two runs.\n", + "\n", + "`reid_ema_alpha` controls how much of a track's stored appearance survives each update: 0.9 keeps 90% of the running average, so a single blurry crop cannot overwrite a track's identity." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "def load_yolox_detections(path: Path) -> dict[int, sv.Detections]:\n", + " \"\"\"Load YOLOX detections (`frame,x1,y1,x2,y2,score`) keyed by 1-based frame index.\"\"\"\n", + " rows = []\n", + " with path.open() as handle:\n", + " for line in handle:\n", + " parts = line.strip().split(\",\")\n", + " if len(parts) < 6:\n", + " continue\n", + " frame, x1, y1, x2, y2, score = map(float, parts[:6])\n", + " if score > 0:\n", + " rows.append((int(frame), x1, y1, x2, y2, score))\n", + " if not rows:\n", + " return {}\n", + " # Some files start at frame 0 and some at 1; align both to 1.\n", + " offset = min(frame for frame, *_ in rows) - 1\n", + " by_frame: dict[int, list[list[float]]] = {}\n", + " for frame, x1, y1, x2, y2, score in rows:\n", + " by_frame.setdefault(frame - offset, []).append([x1, y1, x2, y2, score])\n", + " return {\n", + " frame: sv.Detections(\n", + " xyxy=np.asarray(boxes, dtype=np.float32)[:, :4],\n", + " confidence=np.asarray(boxes, dtype=np.float32)[:, 4],\n", + " )\n", + " for frame, boxes in by_frame.items()\n", + " }\n", + "\n", + "\n", + "def write_mot_row(handle, frame_idx: int, detections: sv.Detections) -> None:\n", + " \"\"\"Append one frame of tracks as MOT rows: `frame,id,x,y,w,h,conf,-1,-1,-1`.\"\"\"\n", + " if detections.tracker_id is None:\n", + " return\n", + " for box, track_id in zip(detections.xyxy, detections.tracker_id):\n", + " x1, y1, x2, y2 = box\n", + " handle.write(f\"{frame_idx},{int(track_id)},{x1:.2f},{y1:.2f},{x2 - x1:.2f},{y2 - y1:.2f},1,-1,-1,-1\\n\")\n", + "\n", + "\n", + "def run(name: str, build_tracker) -> object:\n", + " \"\"\"Track every sequence with a fresh tracker, then score the predictions.\"\"\"\n", + " prediction_dir = OUTPUT_ROOT / name / \"preds\"\n", + " prediction_dir.mkdir(parents=True, exist_ok=True)\n", + " for seq, spec in SEQUENCES.items():\n", + " detections_by_frame = load_yolox_detections(spec[\"detections\"])\n", + " tracker = build_tracker()\n", + " with (prediction_dir / f\"{seq}.txt\").open(\"w\") as handle:\n", + " for frame_idx, image_path in enumerate(spec[\"frames\"], start=1):\n", + " frame = cv2.imread(str(image_path))\n", + " detections = detections_by_frame.get(frame_idx, sv.Detections.empty())\n", + " tracked = tracker.update(detections, frame)\n", + " if tracked.tracker_id is not None:\n", + " tracked = tracked[tracked.tracker_id != -1]\n", + " write_mot_row(handle, frame_idx, tracked)\n", + " print(f\" {name}: {seq} done\")\n", + " return evaluate_mot_sequences(\n", + " gt_dir=MOT17_VAL,\n", + " tracker_dir=prediction_dir,\n", + " seqmap=SEQMAP,\n", + " metrics=[\"CLEAR\", \"HOTA\", \"Identity\"],\n", + " )\n", + "\n", + "\n", + "baseline = run(\"botsort\", lambda: BoTSORTTracker(enable_cmc=True))\n", + "with_reid = run(\n", + " \"botsort_reid\",\n", + " lambda: BoTSORTTracker(\n", + " enable_cmc=True,\n", + " reid_model=reid_model,\n", + " reid_ema_alpha=0.9,\n", + " reid_appearance_threshold=APPEARANCE_THRESHOLD,\n", + " ),\n", + ")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Compare the two runs\n", + "\n", + "IDF1 and AssA are the metrics to watch. Both reward keeping one identity on one person for the whole sequence, which is exactly what appearance is there to protect, while MOTA is dominated by detection quality and barely moves.\n", + "\n", + "The reference row comes from a [MOT17 re-ID study](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) (Table 8 for HOTA, Table 13 for IDF1) that ran the same encoder, detections and threshold, so your uplift should land near theirs." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "STUDY_NO_REID = {\"hota\": 68.43, \"idf1\": 80.92}\n", + "STUDY_REID = {\"hota\": 68.95, \"idf1\": 81.98}\n", + "\n", + "\n", + "def metrics(result) -> tuple[float, float, float, float, int]:\n", + " aggregate = result.aggregate\n", + " return (\n", + " aggregate.HOTA.HOTA * 100,\n", + " aggregate.HOTA.AssA * 100,\n", + " aggregate.CLEAR.MOTA * 100,\n", + " aggregate.Identity.IDF1 * 100,\n", + " aggregate.CLEAR.IDSW,\n", + " )\n", + "\n", + "\n", + "print(f\"{'Config':<26} {'HOTA':>6} {'AssA':>6} {'MOTA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 68)\n", + "for label, result in ((\"BoT-SORT\", baseline), (\"BoT-SORT + ReID\", with_reid)):\n", + " hota, assa, mota, idf1, idsw = metrics(result)\n", + " print(f\"{label:<26} {hota:6.2f} {assa:6.2f} {mota:6.2f} {idf1:6.2f} {idsw:5d}\")\n", + "\n", + "before, after = metrics(baseline), metrics(with_reid)\n", + "print(\n", + " f\"\\nReID uplift: HOTA {after[0] - before[0]:+.2f} IDF1 {after[3] - before[3]:+.2f} \"\n", + " f\"IDSW {after[4] - before[4]:+d}\"\n", + ")\n", + "print(\n", + " f\"Reference study: HOTA {STUDY_REID['hota'] - STUDY_NO_REID['hota']:+.2f} \"\n", + " f\"IDF1 {STUDY_REID['idf1'] - STUDY_NO_REID['idf1']:+.2f}\"\n", + ")\n", + "\n", + "print(f\"\\n{'Sequence':<18} {'HOTA':>6} {'AssA':>6} {'IDF1':>6} {'IDSW':>5}\")\n", + "print(\"-\" * 50)\n", + "for seq in SEQUENCES:\n", + " plain, reid = baseline.sequences[seq], with_reid.sequences[seq]\n", + " print(\n", + " f\"{seq:<18} {reid.HOTA.HOTA * 100:6.2f} {reid.HOTA.AssA * 100:6.2f} \"\n", + " f\"{reid.Identity.IDF1 * 100:6.2f} {reid.CLEAR.IDSW:5d}\"\n", + " f\" (IDSW without ReID: {plain.CLEAR.IDSW})\"\n", + " )" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Choose an appearance threshold\n", + "\n", + "BoT-SORT gates appearance on `d_app = 0.5 * (1 - cosine_similarity)`, ignoring the appearance term whenever the distance exceeds `reid_appearance_threshold`. Where to put that threshold depends on your encoder and your footage, so measure it rather than inherit it.\n", + "\n", + "What matters is which pairs you measure. A tracker only ever compares crops from the same video within a few dozen frames of each other, so those are the pairs to sample: same-ID pairs it should accept, and different-ID pairs from the same window that could steal the match. `sample_appearance_distances` draws exactly those, giving every sequence an equal quota and every identity an equal chance, so one crowded sequence or one long track cannot decide the answer.\n", + "\n", + "First, embed the ground-truth crops. This is the slow cell." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "# MOT17 marks pedestrians as class 1; confidence 0 rows are ignore-flagged.\n", + "embeddings, ids, frame_ids, sequence_ids = extract_ground_truth_embeddings(\n", + " reid_model,\n", + " MOT17_VAL,\n", + " sequences=list(SEQUENCES),\n", + " keep_classes=(1,),\n", + ")\n", + "\n", + "print(f\"pool: {len(embeddings)} crops, {len(np.unique(ids))} identities\")\n" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now sample the pairs and look at the two distributions. A workable threshold sits in the valley between them: high enough to accept most same-ID pairs, low enough to reject the different-ID ones.\n", + "\n", + "`rates_at` puts a number on that trade-off. There is no correct answer for both columns at once, and which way to lean depends on whether a lost track or a swapped ID hurts you more, which is why nothing here picks a threshold for you." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "distances = sample_appearance_distances(\n", + " embeddings,\n", + " ids,\n", + " frame_ids,\n", + " sequence_ids,\n", + " same_id_pairs=5000,\n", + " different_id_pairs=10000,\n", + " minimum_frame_gap=1,\n", + " maximum_frame_gap=30, # the default lost_track_buffer, i.e. one second at 30 FPS\n", + ")\n", + "\n", + "print(f\"separability (ROC AUC): {distances.roc_auc:.3f}\")\n", + "print(f\"\\n{'\u03b8':>6} {'same-ID accepted':>17} {'different-ID accepted':>22}\")\n", + "for threshold in (0.10, 0.20, 0.25, 0.30):\n", + " same_id_rate, different_id_rate = distances.rates_at(threshold)\n", + " print(f\"{threshold:6.2f} {same_id_rate:16.1%} {different_id_rate:21.1%}\")\n", + "\n", + "plot_appearance_distances(\n", + " distances,\n", + " thresholds={APPEARANCE_THRESHOLD: \"selected\", 0.25: \"default\"},\n", + " title=f\"{REID_ENCODER} on MOT17 val ground truth\",\n", + ")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## How far the threshold carries\n", + "\n", + "The histogram above fixes the frame gap at 30, so it describes re-association within a second. It says nothing about finding a track again after a longer occlusion, which is the case appearance is supposed to rescue.\n", + "\n", + "`sweep_frame_gap` repeats the sampling across widening gaps. Watch the same-ID band drift upward while the different-ID band stays put: identities become harder to recognise as time passes, but strangers do not become easier to confuse. A threshold tuned on adjacent frames therefore turns into a threshold that quietly refuses to re-find anything.\n", + "\n", + "The lower panel reports ROC AUC, the chance that a random same-ID pair scores closer than a random different-ID pair. It summarises every possible threshold at once, so it separates \"the encoder has degraded\" from \"our threshold is in the wrong place\"." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "sweep = sweep_frame_gap(embeddings, ids, frame_ids, sequence_ids, pairs_per_class=8000)\n", + "\n", + "print(f\"{'frame gap':>10} {'ROC AUC':>8} {'same-ID < \u03b8':>12} {'different-ID < \u03b8':>17}\")\n", + "for band in sweep:\n", + " same_id_rate, different_id_rate = band.rates_at(APPEARANCE_THRESHOLD)\n", + " print(f\"{band.label:>10} {band.roc_auc:8.3f} {same_id_rate:11.1%} {different_id_rate:16.1%}\")\n", + "\n", + "plot_frame_gap_sweep(\n", + " sweep,\n", + " thresholds={APPEARANCE_THRESHOLD: \"selected\", 0.25: \"default\"},\n", + " title=f\"{REID_ENCODER}: separability vs frame gap\",\n", + ")\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you raise `lost_track_buffer` to recover tracks after long occlusions, raise `reid_appearance_threshold` with it and re-read the different-ID column, otherwise the longer buffer buys nothing." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Sample tracked frames\n", + "\n", + "Finally, look at the output. Track ids should stay pinned to the same person across all four frames." + ] + }, + { + "cell_type": "code", + "metadata": {}, + "source": [ + "VIZ_SEQ = next(iter(SEQUENCES))\n", + "VIZ_FRAMES = (1, 30, 60, 90)\n", + "\n", + "predictions = load_mot_file(OUTPUT_ROOT / \"botsort_reid\" / \"preds\" / f\"{VIZ_SEQ}.txt\")\n", + "box_annotator = sv.BoxAnnotator(thickness=2, color_lookup=sv.ColorLookup.TRACK)\n", + "label_annotator = sv.LabelAnnotator(\n", + " text_color=sv.Color.BLACK,\n", + " text_scale=0.5,\n", + " color_lookup=sv.ColorLookup.TRACK,\n", + ")\n", + "\n", + "figure, axes = plt.subplots(2, 2, figsize=(12, 8))\n", + "for ax, frame_idx in zip(axes.ravel(), VIZ_FRAMES):\n", + " frame = load_mot_frame_image(SEQUENCES[VIZ_SEQ][\"images\"], frame_idx)\n", + " frame_data = predictions.get(frame_idx)\n", + " scene = frame\n", + " if frame_data is not None and len(frame_data.ids) > 0:\n", + " detections = sv.Detections(\n", + " xyxy=sv.xywh_to_xyxy(frame_data.boxes).astype(np.float32),\n", + " tracker_id=frame_data.ids.astype(int),\n", + " )\n", + " scene = box_annotator.annotate(frame.copy(), detections)\n", + " scene = label_annotator.annotate(\n", + " scene, detections, labels=[str(int(track_id)) for track_id in detections.tracker_id]\n", + " )\n", + " ax.imshow(scene[:, :, ::-1])\n", + " ax.set_title(f\"{VIZ_SEQ} frame {frame_idx}\")\n", + " ax.axis(\"off\")\n", + "\n", + "figure.suptitle(\"BoT-SORT + ReID\", y=1.01)\n", + "figure.tight_layout()\n", + "plt.show()" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next steps\n", + "\n", + "You ran BoT-SORT with and without appearance ReID on MOT17, and calibrated the threshold on the pairs a tracker actually sees rather than on a paper's default.\n", + "\n", + "To take this to your own footage: swap the encoder id for one trained closer to your domain, re-run the two threshold cells on a labeled slice of your data, and read the different-ID column before trusting the number. A pedestrian encoder on, say, soccer footage compresses every distance into a narrow band and needs a much tighter threshold, which the [ReID appearance guide](https://trackers.roboflow.com/latest/guides/reid/) walks through with SoccerNet numbers." + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "provenance": [], + "gpuType": "T4" + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/docs/evaluations/results.md b/docs/evaluations/results.md index b8448e054..9909d84b4 100644 --- a/docs/evaluations/results.md +++ b/docs/evaluations/results.md @@ -28,7 +28,7 @@ Pedestrian tracking with crowded scenes and frequent occlusions. Strongly tests !!! info - Parameters were tuned on the validation set. Results are reported on the test set via Codabench submission. Detections come from a YOLOX model. + Parameters were tuned on the validation set. Results are reported on the test set via Codabench submission. Detections come from a YOLOX model. BoT-SORT rows are CMC without appearance; for CMC + FastReID on MOT17 and OSNet MSMT17 on SoccerNet see [BoT-SORT with and without ReID](../guides/reid.md#bot-sort-with-and-without-reid). === "Default" @@ -216,7 +216,7 @@ Long sequences with dense interactions and partial occlusions. Tests long-term I !!! info - Parameters were tuned on the train set. Results are reported on the test set. SoccerNet-tracking has no validation split. This dataset provides oracle (ground-truth) detections. + Parameters were tuned on the train set. Results are reported on the test set. SoccerNet-tracking has no validation split. This dataset provides oracle (ground-truth) detections. The BoT-SORT row is CMC without appearance; for OSNet MSMT17 appearance on this split see [BoT-SORT with and without ReID](../guides/reid.md#bot-sort-with-and-without-reid). === "Default" diff --git a/docs/guides/install.md b/docs/guides/install.md index a077ded07..dd8242249 100644 --- a/docs/guides/install.md +++ b/docs/guides/install.md @@ -70,6 +70,24 @@ The `detection` extra installs `inference-models`, enabling the CLI to run detec uv pip install "trackers[detection]" ``` +### ReID (BoT-SORT appearance) + +The `reid` extra installs the standalone [`reid`](https://github.com/roboflow/re-ID) package, which brings PyTorch, timm, Hugging Face Hub, safetensors, Pillow, and gdown for ReID model loading (OSNet, FastReID SBS, and `timm:` backbones) and BoT-SORT appearance association. + +=== "pip" + + ```bash + pip install "trackers[reid]" + ``` + +=== "uv" + + ```bash + uv pip install "trackers[reid]" + ``` + +Use via `from reid import ReIDModel` and `BoTSORTTracker(reid_model=...)`, or via CLI flags such as `--reid.enable` and `--reid.architecture` on `trackers track` command (BoT-SORT only). + !!! tip "GPU Acceleration" For GPU support, ensure PyTorch is installed with CUDA or MPS. diff --git a/docs/guides/reid.md b/docs/guides/reid.md new file mode 100644 index 000000000..5fedbbf33 --- /dev/null +++ b/docs/guides/reid.md @@ -0,0 +1,164 @@ +--- +title: ReID Appearance — BoT-SORT Appearance Association | Trackers +description: Use ReID appearance association with BoT-SORT in Roboflow Trackers, from model loading to appearance threshold selection, with MOT17 and SoccerNet results. +--- + +# ReID Appearance + +BoT-SORT can fuse appearance embeddings with IoU during association. Embeddings come from a model in the standalone [`reid`](https://github.com/roboflow/re-ID) package. See the [ReID API](../api/reid.md) for the association helpers. + +**What you'll learn:** + +- How to enable appearance association on BoT-SORT +- Which parameters control the appearance gate +- How to pick `reid_appearance_threshold` for your encoder and domain +- What ReID changes on MOT17 and SoccerNet + +--- + +## Install + +```bash +pip install "trackers[reid]" +``` + +For extra contents and other options, see the [install guide](install.md). + +--- + +## Quickstart + +```python +from reid import ReIDModel + +from trackers import BoTSORTTracker + +reid_model = ReIDModel.from_pretrained("fastreid_mot17_sbs50") +tracker = BoTSORTTracker(reid_model=reid_model, reid_appearance_threshold=0.2) +``` + +!!! warning "A frame is required when ReID is enabled" + + Pass the current video frame as `tracker.update(detections, frame=frame_bgr)`. When `reid_model` is set, `update()` raises if `frame` is omitted. + +For the model catalog and fine-tuning, see the [`reid` training guide](https://github.com/roboflow/re-ID/blob/main/docs/learn/train.md). + +--- + +## Key Parameters + +| Parameter | Purpose | Tuning guidance | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------- | +| `reid_model` | Appearance encoder queried during association. | Leave unset for IoU and CMC only. Pick a checkpoint trained on your object domain where possible. | +| `reid_ema_alpha` | EMA momentum for a track's appearance feature. | Default 0.9. Higher keeps a stable long-term identity; lower adapts faster to appearance change but drifts more. | +| `reid_appearance_threshold` | Maximum appearance distance `d_app` for appearance to lower a pair's matching cost. | BoT-SORT paper default 0.25. Calibrate per encoder and domain, see below. | +| `reid_proximity_threshold` | IoU gate applied before appearance (`IoU ≥ 1 - reid_proximity_threshold`), from true IoU even with GIoU/DIoU/CIoU. | Default 0.5. Lower restricts how far apart a pair may be before appearance stops contributing. | + +--- + +## Choosing an appearance threshold + +BoT-SORT fuses costs as `min(d_iou, d_app)` with `d_app = 0.5 * (1 - cos_sim)`, and discards the appearance term when `d_app` exceeds `reid_appearance_threshold` (paper default 0.25) or when the pair fails the `reid_proximity_threshold` IoU gate. Appearance can therefore only lower a pair's cost, never veto a geometric match. Pick θ on a labeled split with the encoder you will track with: + +1. Embed GT crops. +2. Histogram `d_app` for association-local pairs: same video only, with frame gap bounded by the lost-track horizon (default 30 frames). Positives are same-ID; negatives are different-ID that could co-compete. Sample both classes with the same per-sequence quota, otherwise one crowded sequence decides the answer. +3. Choose θ so most same-ID pairs fall below it and most different-ID pairs fall above it. + +All three steps ship with Trackers, so you can run them on your own footage. `extract_ground_truth_embeddings` reads any MOT-format dataset, meaning a `gt/gt.txt` and an `img1` folder per sequence, and returns each crop's embedding alongside the identity, frame and sequence it came from: + +```python +from trackers.core.reid import ( + extract_ground_truth_embeddings, + plot_appearance_distances, + sample_appearance_distances, +) + +embeddings, ids, frame_ids, sequence_ids = extract_ground_truth_embeddings(model, "mot17/val", keep_classes=(1,)) +distances = sample_appearance_distances(embeddings, ids, frame_ids, sequence_ids) +for threshold in (0.10, 0.20, 0.25): + same_id_rate, different_id_rate = distances.rates_at(threshold) + print(f"θ={threshold:.2f}: same-ID {same_id_rate:.1%}, different-ID {different_id_rate:.1%}") + +plot_appearance_distances(distances, thresholds={0.20: "selected", 0.25: "default"}) +``` + +See the [ReID API reference](../api/reid.md#choosing-a-threshold) for the full signatures. The figures on this page were produced with these helpers on MOT17 val and SoccerNet test ground truth. To reproduce them end to end, from download to calibrated threshold, open the [ReID cookbook](https://colab.research.google.com/github/roboflow/trackers/blob/develop/docs/cookbooks/how-to-add-reid-to-trackers.ipynb) in Colab. + +**MOT17 val, `fastreid_mot17_sbs50`.** Same-ID distances peak near 0 and different-ID near 0.4. On association-local GT crop pairs (5000 same-ID, 10000 different-ID, frame gap 1 to 30), θ=0.2 keeps 68% of same-ID pairs while passing 1.1% of different-ID pairs. Raising θ to the BoT-SORT default 0.25 recovers same-ID pairs (79%) but nearly triples the different-ID pairs it admits (2.9%), which is why 0.2 is the better operating point here ([MOT17 re-ID study](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) Table 8 uses the same threshold). + +![FastReID MOT17 SBS on MOT17 val GT](../assets/reid/mot17-fastreid-appearance-distances.png) + +**SoccerNet test, `osnet_x1_0_msmt17_combineall`.** A pedestrian encoder on soccer footage squeezes every distance into a narrow range: same-ID pairs peak near 0.05 and different-ID pairs near 0.20 (similar kits). The two shapes still separate, but the scale no longer matches the thresholds BoT-SORT was tuned with. On association-local GT crop pairs (5000 same-ID, 10000 different-ID, frame gap 1 to 30), θ=0.2 admits 96% of same-ID pairs but also 49% of different-ID pairs, and tracking stays flat against CMC-only. θ=0.1 holds different-ID pairs to 9%, yet appearance still assists a mix of correct and same-kit pairs and costs HOTA and IDF1 (see the SoccerNet table below). Calibrate θ on your own domain rather than carrying 0.2 or 0.25 across. + +![OSNet MSMT17 on SoccerNet test GT](../assets/reid/soccernet-osnet-appearance-distances.png) + +--- + +## How far the threshold carries + +A histogram fixes one frame gap, so it only describes re-association over that horizon. Sweeping the gap shows how long a track can stay lost before appearance stops helping to re-find it. `sweep_frame_gap` repeats the sampling above across widening bands, and `plot_frame_gap_sweep` draws the result: + +```python +from trackers.core.reid import plot_frame_gap_sweep, sweep_frame_gap + +sweep = sweep_frame_gap(embeddings, ids, frame_ids, sequence_ids) +plot_frame_gap_sweep(sweep, thresholds={0.20: "selected", 0.25: "default"}) +``` + +On MOT17 val, different-ID distances barely move with the gap: the median stays near 0.41 and the 10th percentile near 0.31 from a 1-frame gap out to 240 frames. Same-ID distances spread steadily, from a median of 0.04 at a 1-frame gap to 0.20 across the 16 to 30 band and 0.28 beyond 120 frames. + +ROC AUC below is the chance that a random same-ID pair scores closer than a random different-ID pair: 1.0 means the two never cross, 0.5 means appearance carries no information, and its complement is how often a same-ID pair sits farther apart than a different-ID one. It is the area under the curve traced by sweeping θ from 0 to 1 and plotting the two rates next to it, so it summarises every threshold instead of the single one we ship. + +It is not the area where the shaded bands cross in the figure. That is two percentile ranges intersecting, which ignores where the mass sits and which side is closer; at a 1-frame gap the bands never touch yet the AUC is 0.998 rather than 1.0. The two rates beside it evaluate the default 0.25 and the 0.2 this page argues for, rather than deriving a third. + +| Frame gap | ROC AUC | same-ID below 0.2 | different-ID below 0.2 | +| :--------- | :-----: | :---------------: | :--------------------: | +| 1 | 0.998 | 98.0% | 1.7% | +| 2 to 5 | 0.987 | 87.6% | 1.6% | +| 6 to 15 | 0.957 | 67.4% | 1.1% | +| 16 to 30 | 0.929 | 51.4% | 1.1% | +| 31 to 60 | 0.899 | 39.8% | 0.9% | +| 61 to 120 | 0.865 | 31.8% | 0.8% | +| 121 to 240 | 0.854 | 28.7% | 0.8% | + +![FastReID MOT17 SBS separability vs frame gap](../assets/reid/mot17-fastreid-appearance-distances-vs-gap.png) + +Two things follow. First, a threshold validated on adjacent frames says little about re-association: at θ=0.2 appearance helps 98% of same-ID pairs one frame apart but only 51% across the default 30-frame lost-track buffer. Second, the price of a tight θ over long gaps is missed re-associations rather than extra wrong ones, because the different-ID rate stays near 1% throughout. If you raise `lost_track_buffer` to recover tracks after long occlusions, raise `reid_appearance_threshold` with it and re-check the different-ID column. + +The cross-domain encoder fails differently. On SoccerNet the different-ID rate at θ=0.2 stays between 44% and 51% at every gap, so the frame gap is not what limits it; the encoder simply cannot separate players in matching kits at any horizon. Widening the gap costs same-ID pairs (99.6% down to 87.0%) without ever making the different-ID side usable, which is why θ has to come down to about 0.1 on this domain instead of being traded against the gap. + +![OSNet MSMT17 separability vs frame gap](../assets/reid/soccernet-osnet-appearance-distances-vs-gap.png) + +--- + +## BoT-SORT with and without ReID + +### MOT17 test + +YOLOX detections, CMC on, Codabench MOT17 test (same protocol as the [benchmark results](../evaluations/results.md) default table). ReID: `fastreid_mot17_sbs50`, `reid_appearance_threshold=0.2` ([MOT17 re-ID study](https://www-sop.inria.fr/members/Francois.Bremond/Postscript/Tomasz__SCCAI_2025.pdf) Table 8). + +| Config | HOTA | IDF1 | MOTA | +| :-------------- | :------: | :------: | :------: | +| BoT-SORT | 63.7 | 78.7 | **79.2** | +| BoT-SORT + ReID | **63.9** | **79.2** | **79.2** | + +### MOT17 val-half + +YOLOX detections, CMC on, MOT17 val-half split, same encoder and threshold, scored with `trackers eval`. + +| Config | HOTA | IDF1 | MOTA | +| :-------------- | :------: | :------: | :------: | +| BoT-SORT | 68.9 | 81.2 | 78.3 | +| BoT-SORT + ReID | **69.1** | **81.9** | **78.4** | + +The MOT17 re-ID study reports 68.43 HOTA / 80.92 IDF1 without ReID and 68.95 / 81.98 with, on the same split at `reid_appearance_threshold=0.2` (Table 8 and Table 13; MOTA is not reported for that YOLOX setup). + +### SoccerNet test (OSNet MSMT17) + +Oracle detections, CMC on, SoccerNet-tracking test (same protocol as the [benchmark results](../evaluations/results.md) default table). ReID: `osnet_x1_0_msmt17_combineall` (MSMT17 pretrained), so this is a cross-domain encoder on soccer footage. + +| Config | HOTA | IDF1 | MOTA | +| :------------------------------ | :------: | :------: | :------: | +| BoT-SORT | 84.5 | 79.3 | **96.6** | +| BoT-SORT + OSNet MSMT17 (θ=0.2) | **84.6** | **79.4** | **96.6** | +| BoT-SORT + OSNet MSMT17 (θ=0.1) | 82.9 | 77.7 | 96.5 | diff --git a/docs/llms.txt b/docs/llms.txt index 32ddfcea1..0e6c16853 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -12,6 +12,7 @@ - [Track Objects](https://trackers.roboflow.com/latest/guides/track/): CLI and Python API - [Evaluate Trackers](https://trackers.roboflow.com/latest/evaluations/evaluate/): HOTA, IDF1, MOTA metrics guide - [Detection Quality Matters](https://trackers.roboflow.com/latest/guides/detection-quality/): How detector quality affects tracking +- [ReID Appearance](https://trackers.roboflow.com/latest/guides/reid/): BoT-SORT appearance association, threshold selection, MOT17 and SoccerNet results - [SORT](https://trackers.roboflow.com/latest/trackers/sort/): Kalman + Hungarian algorithm tracker - [ByteTrack](https://trackers.roboflow.com/latest/trackers/bytetrack/): Low-confidence detection association tracker - [OC-SORT](https://trackers.roboflow.com/latest/trackers/ocsort/): Observation-centric re-update tracker @@ -20,6 +21,7 @@ - [Trackers API](https://trackers.roboflow.com/latest/api/trackers/) - [Motion API](https://trackers.roboflow.com/latest/api/motion/) +- [ReID API](https://trackers.roboflow.com/latest/api/reid/) - [Evals API](https://trackers.roboflow.com/latest/api/evals/) - [Datasets API](https://trackers.roboflow.com/latest/api/datasets/) - [I/O API](https://trackers.roboflow.com/latest/api/io/) diff --git a/docs/trackers/botsort.md b/docs/trackers/botsort.md index 3d65cfe64..9c94ca432 100644 --- a/docs/trackers/botsort.md +++ b/docs/trackers/botsort.md @@ -52,6 +52,10 @@ BoT-SORT keeps the same tracking-by-detection backbone as [ByteTrack](bytetrack. | `high_conf_det_threshold` | Confidence split between stage-1 and stage-2 detections. | 0.5-0.7 common. Higher shifts more detections to recovery stage; lower gives stage-1 broader coverage. | | `enable_cmc` | Enables camera motion compensation before association. | Keep enabled for moving-camera footage (sports, drone, handheld). Disable mainly for static cameras if you need maximal speed. | +## ReID appearance (optional) + +BoT-SORT can fuse appearance embeddings with IoU during association via an optional `reid_model`. Install, usage, parameters, and MOT17 with/without ReID scores are on the [ReID appearance](../guides/reid.md) page. + ## Run on video, webcam, or RTSP stream These examples use `opencv-python` for decoding and display. Replace ``, ``, and `` with your inputs. `` is usually 0 for the default camera. diff --git a/mkdocs.yml b/mkdocs.yml index 9330d9a91..b48f9d686 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -134,6 +134,7 @@ plugins: - mkdocstrings: handlers: python: + paths: [src] options: # Controls whether to show symbol types in the table of contents show_symbol_type_toc: true @@ -166,8 +167,11 @@ nav: - Tune Trackers: guides/tune.md - IoU Variants: guides/iou.md - State Estimators: guides/state-estimators.md + - ReID Appearance: guides/reid.md - Developer: - Inspect Mask Pipeline: guides/inspect.md + - Architecture Decisions: + - "Model-backend externalization": adr/0001-model-backend-externalization.md - Evaluations: - Download Datasets: evaluations/download.md - Benchmark Trackers: evaluations/evaluate.md @@ -184,6 +188,7 @@ nav: - API Reference: - Trackers: api/trackers.md - Motion: api/motion.md + - ReID: api/reid.md - Datasets: api/datasets.md - Evals: api/evals.md - I/O: api/io.md diff --git a/pyproject.toml b/pyproject.toml index 0933def3f..bc8064b05 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,6 +48,7 @@ dependencies = [ [project.optional-dependencies] detection = ["inference-models>=0.19.0"] tune = ["optuna>=3.0.0"] +reid = ["reid>=0.1.0.dev0,<0.2", "matplotlib>=3.7.0"] mask = [ "torch", "torchvision", @@ -233,5 +234,7 @@ module = [ "rfdetr.*", "supervision", "supervision.*", + "reid", + "reid.*", ] ignore_missing_imports = true diff --git a/src/trackers/__init__.py b/src/trackers/__init__.py index a2229bf2b..2a50a1b18 100644 --- a/src/trackers/__init__.py +++ b/src/trackers/__init__.py @@ -12,6 +12,21 @@ from trackers.core.cbiou.tracker import CBIoUTracker from trackers.core.mcbyte.tracker import McByteMaskConfig, McByteTracker from trackers.core.ocsort.tracker import OCSORTTracker +from trackers.core.reid import ( + DEFAULT_FRAME_GAP_BANDS, + AppearanceDistances, + FeatureBank, + ReIDEncoder, + ThresholdLines, + appearance_similarity, + extract_detection_embeddings, + extract_ground_truth_embeddings, + plot_appearance_distances, + plot_frame_gap_sweep, + roc_auc, + sample_appearance_distances, + sweep_frame_gap, +) from trackers.core.sort.tracker import SORTTracker from trackers.datasets.download import download_dataset from trackers.datasets.manifest import Dataset, DatasetAsset, DatasetSplit @@ -29,6 +44,8 @@ __all__ = [ "CMC", + "DEFAULT_FRAME_GAP_BANDS", + "AppearanceDistances", "BIoU", "BaseIoU", "BoTSORTTracker", @@ -43,6 +60,7 @@ "Dataset", "DatasetAsset", "DatasetSplit", + "FeatureBank", "GIoU", "HomographyTransformation", "IdentityTransformation", @@ -52,10 +70,20 @@ "MotionAwareTraceAnnotator", "MotionEstimator", "OCSORTTracker", + "ReIDEncoder", "SORTTracker", + "ThresholdLines", + "appearance_similarity", "download_dataset", + "extract_detection_embeddings", + "extract_ground_truth_embeddings", "frames_from_source", "load_mot_file", + "plot_appearance_distances", + "plot_frame_gap_sweep", + "roc_auc", + "sample_appearance_distances", + "sweep_frame_gap", "xcycsr_to_xyxy", "xyxy_to_xcycsr", ] diff --git a/src/trackers/cli/_parser.py b/src/trackers/cli/_parser.py index f115acc27..a056be883 100644 --- a/src/trackers/cli/_parser.py +++ b/src/trackers/cli/_parser.py @@ -25,6 +25,7 @@ DetectionOptions, FilterOptions, OutputOptions, + ReIDOptions, ShowOptions, TrackerOptions, track_command, @@ -39,6 +40,7 @@ (DetectionOptions, "detection"), (FilterOptions, "filters"), (TrackerOptions, "tracker"), + (ReIDOptions, "reid"), (OutputOptions, "output"), (ShowOptions, "show"), ) @@ -212,6 +214,19 @@ def _add_track_arguments(parser: ArgumentParser) -> list[str]: ) added_args = ["source"] for option_class, nested_key in _TRACK_OPTION_GROUPS: + if option_class is ReIDOptions: + added_args.extend(parser.add_class_arguments(option_class, nested_key, skip={"enable"})) + # jsonargparse handles ``bool | None`` as a value-taking type hint. + # Registering the field as ``bool`` preserves the paired bare flags + # while its explicit default retains the omitted state. + parser.add_argument( + "--reid.enable", + type=bool, + default=None, + help="Explicitly enable or disable appearance association.", + ) + added_args.append("reid.enable") + continue added_args.extend(parser.add_class_arguments(option_class, nested_key)) # Registered as a plain boolean rather than a bare ``store_true`` so it gains # the same ``--no_display`` half every other boolean option has. diff --git a/src/trackers/cli/track.py b/src/trackers/cli/track.py index 115106dd3..a8c49fc82 100644 --- a/src/trackers/cli/track.py +++ b/src/trackers/cli/track.py @@ -65,6 +65,32 @@ class DetectionOptions: api_key: str | None = None +@dataclass +class ReIDOptions: + """Appearance ReID settings, for trackers that accept an encoder. + + Requires the ``reid`` extra (``pip install "trackers[reid]"``). BoT-SORT is + the only tracker that accepts an encoder today. + + Attributes: + enable: Explicitly enable or disable appearance association. When + omitted, any non-default ReID option enables it. + model: Checkpoint source: curated alias, ``hf://`` URL, local path, or + ``save_pretrained`` directory. Implies enable and cannot be combined + with an explicit disable. Default alias when omitted: + ``osnet_x1_0_msmt17_combineall``. + device: ReID compute device: ``auto``, ``cpu``, ``cuda``, ``mps``. + architecture: Backbone for bare ``.pth``/``.safetensors`` weights (e.g. + ``osnet_x1_0``, ``fastreid_sbs_resnest50``). Required when ``model`` + points at a bare weights file. + """ + + enable: bool | None = None + model: str | None = None + device: str = DEFAULT_DEVICE + architecture: str | None = None + + @dataclass class FilterOptions: """Detection and track filters. @@ -130,10 +156,11 @@ class ShowOptions: _CLI_PARAMETER_RENAMES = {"mask_config": "mask"} _CLI_PARAMETER_RENAMES_REVERSED = {short: long for long, short in _CLI_PARAMETER_RENAMES.items()} -# Tracker registry parameters that never reach the CLI. ``mask_manager`` takes -# a live ``MaskManager`` instance, not a value a command line can express; -# ``enable_mask_manager`` is its CLI-facing switch. -_EXCLUDED_TRACKER_PARAMETERS = frozenset({"mask_manager"}) +# Tracker registry parameters that never reach the CLI. Both take a live object +# rather than a value a command line can express: ``mask_manager`` a +# ``MaskManager``, whose CLI-facing switch is ``enable_mask_manager``, and +# ``reid_model`` an encoder, built from the ``ReIDOptions`` group instead. +_EXCLUDED_TRACKER_PARAMETERS = frozenset({"mask_manager", "reid_model"}) def _abbreviate_parameter_name(name: str) -> str: @@ -325,6 +352,7 @@ def track_command( detection: DetectionOptions | None = None, filters: FilterOptions | None = None, tracker: TrackerOptions | None = None, # type: ignore[valid-type] + reid: ReIDOptions | None = None, output: OutputOptions | None = None, display: bool = False, show: ShowOptions | None = None, @@ -338,6 +366,8 @@ def track_command( filters: Class and track-ID filters applied to detections and tracks. tracker: Algorithm ID plus optional parameter overrides; only fields matching the chosen tracker's ``__init__`` are forwarded. + reid: Appearance ReID encoder options. Requires ``source``, since + embeddings are extracted from frames. output: Output paths. display: Show a live preview window during tracking. show: Annotation elements to draw on each frame. @@ -351,6 +381,8 @@ def track_command( filters = FilterOptions() if tracker is None: tracker = TrackerOptions() + if reid is None: + reid = ReIDOptions() if output is None: output = OutputOptions() if show is None: @@ -363,6 +395,13 @@ def track_command( if needs_frames and source is None: print("Error: --source is required when using --output.video or --display.", file=sys.stderr) return 1 + if _reid_requested(reid) and source is None: + print( + "Error: ReID requires --source (video/webcam/images) so appearance " + "embeddings can be extracted from frames.", + file=sys.stderr, + ) + return 1 if output.video: _validate_output_path(_resolve_video_output_path(output.video), overwrite=output.overwrite) @@ -372,7 +411,7 @@ def track_command( # Built before the detection model so an unknown tracker ID is rejected # without first paying for a model download and load. try: - tracker_obj = _init_tracker(tracker) + tracker_obj = _init_tracker(tracker, reid) except ValueError as e: print(f"Error: {e}", file=sys.stderr) return 1 @@ -704,7 +743,64 @@ def _warn_dropped_tracker_overrides(tracker_id: str, dropped: list[str]) -> None ) -def _init_tracker(params: TrackerOptions | None) -> BaseTracker: # type: ignore[valid-type] +def _reid_requested(reid: ReIDOptions) -> bool: + """Whether the command line asked for appearance association. + + Raises: + ValueError: If a checkpoint is combined with an explicit disable. + """ + if reid.enable is False and reid.model is not None: + raise ValueError("--reid.model cannot be combined with --reid.no_enable or --reid.enable false.") + if reid.enable is not None: + return reid.enable + return reid.model is not None or reid.architecture is not None or reid.device != DEFAULT_DEVICE + + +def _load_reid_model(reid: ReIDOptions) -> Any: + """Build the ReID encoder described by ``reid``. + + Args: + reid: Encoder selection and loading options. + + Returns: + A ``reid.ReIDModel`` satisfying the ``ReIDEncoder`` protocol. + + Raises: + ImportError: If the ``reid`` package is present but one of its imports + fails. + ValueError: If the ``reid`` extra is not installed, ``architecture`` was + given without ``model``, or the checkpoint fails to load. + """ + try: + from reid import ReIDModel + except ImportError as exc: + if exc.name != "reid": + raise + raise ValueError( + "ReID tracking requires the optional `trackers[reid]` extra.\nInstall with: pip install 'trackers[reid]'" + ) from exc + + if reid.architecture is not None and reid.model is None: + raise ValueError("--reid.architecture requires --reid.model (bare weights need a checkpoint path).") + + load_kwargs: dict[str, Any] = {"device": reid.device} + if reid.model is not None: + load_kwargs["source"] = reid.model + if reid.architecture is not None: + load_kwargs["architecture"] = reid.architecture + + try: + return ReIDModel.from_pretrained(**load_kwargs) + except KeyboardInterrupt: + raise + except (OSError, ValueError, RuntimeError) as exc: + raise ValueError(f"Failed to load ReID model: {exc}") from exc + + +def _init_tracker( + params: TrackerOptions | None, # type: ignore[valid-type] + reid: ReIDOptions | None = None, +) -> BaseTracker: """Create a tracker instance from the registry. ``params.name`` selects the algorithm; every other field is a parameter @@ -715,12 +811,18 @@ def _init_tracker(params: TrackerOptions | None) -> BaseTracker: # type: ignore Args: params: Tracker selection and parameter overrides. + reid: Appearance options. When ReID is requested, the encoder is built + here and injected as the ``reid_model`` keyword, mirroring how + ``iou_variant`` becomes the ``iou`` keyword. Loading happens after + the tracker ID resolves, so a bad ID fails before a checkpoint + download. Returns: Initialised tracker instance. Raises: - ValueError: If ``params.name`` is not registered. + ValueError: If ``params.name`` is not registered, the chosen tracker + does not accept an encoder, or the encoder fails to load. """ raw = _tracker_options_as_dict(params) tracker_id = raw.pop("name", DEFAULT_TRACKER) @@ -742,6 +844,10 @@ def _init_tracker(params: TrackerOptions | None) -> BaseTracker: # type: ignore UserWarning, stacklevel=2, ) + if reid is not None and _reid_requested(reid): + if "reid_model" not in accepted: + raise ValueError(f"--reid.* options apply only to a tracker that accepts an encoder, got '{tracker_id}'.") + kwargs["reid_model"] = _load_reid_model(reid) return info.tracker_class(**kwargs) diff --git a/src/trackers/core/botsort/tracker.py b/src/trackers/core/botsort/tracker.py index faafb9318..b902a7a95 100644 --- a/src/trackers/core/botsort/tracker.py +++ b/src/trackers/core/botsort/tracker.py @@ -14,6 +14,10 @@ from trackers.core.base import BaseTracker from trackers.core.botsort.tracklet import BoTSORTTracklet from trackers.core.botsort.utils import _fuse_score, get_alive_tracklets +from trackers.core.reid.appearance import appearance_similarity, extract_detection_embeddings +from trackers.core.reid.encoder import ReIDEncoder +from trackers.core.reid.feature_bank import FeatureBank +from trackers.core.reid.fusion import fuse_botsort_reid_association from trackers.utils.cmc import CMC, CMCConfig, CMCMethod from trackers.utils.detections import default_confidences from trackers.utils.iou import BaseIoU, IoU @@ -33,11 +37,11 @@ class BoTSORTTracker(BaseTracker): 3) Split tracks into confirmed, unconfirmed, and lost 4) Apply camera motion compensation to predicted tracks 5) Associate high-confidence detections to confirmed + lost tracks - (IoU fused with detection scores + assignment) + (IoU fused with detection scores, optional appearance) 6) Associate low-confidence detections to remaining tracks - (excluding lost tracks) + (excluding lost tracks; geometry only) 7) Match remaining unmatched high-confidence detections to unconfirmed tracks - and remove unmatched unconfirmed tracks + (optional appearance) and remove unmatched unconfirmed tracks 8) Spawn new tracks from still unmatched high-confidence detections (instantly activated on the very first frame) 9) Remove tracks that have been lost for too long @@ -83,13 +87,26 @@ class BoTSORTTracker(BaseTracker): Passing ``None`` (the default) is equivalent to ``IoU()`` and is provided for backward compatibility with existing code that did not supply an ``iou`` argument. + reid_model: Optional appearance encoder (``ReIDEncoder``) for appearance + association. Pass a ``reid.ReIDModel`` in normal use. Requires + ``frame`` in :meth:`update`. When ``None`` (default), behaviour + matches the geometry-only BoT-SORT baseline. + reid_ema_alpha: EMA momentum for track appearance features. Default ``0.9``. + reid_appearance_threshold: Appearance distance gate. Drops the appearance term + when the halved cosine distance ``0.5 * (1 - cos_sim)`` exceeds this + value, leaving the pair scored on geometry alone. Default ``0.25`` + (BoT-SORT ``appearance_thresh``). + reid_proximity_threshold: Standard-IoU distance gate applied before appearance + is used. Computed from true IoU even when ``iou`` is GIoU/DIoU/CIoU. + Default ``0.5`` (BoT-SORT ``proximity_thresh``; requires + ``IoU >= 1 - reid_proximity_threshold``). Notes: - Positive `maximum_frames_without_update` values are scaled by ``frame_rate`` and rounded up to at least one missed frame. Explicit zero-buffer configurations remain zero. - - When CMC is enabled, pass the current video frame via the ``frame`` - argument of :meth:`update`. + - When CMC or ReID is enabled, pass the current video frame via the + ``frame`` argument of :meth:`update`. """ tracker_id = "botsort" @@ -123,6 +140,10 @@ def __init__( instant_first_frame_activation: bool = True, state_estimator_class: type[BaseStateEstimator] = XCYCWHStateEstimator, iou: BaseIoU | None = None, + reid_model: ReIDEncoder | None = None, + reid_ema_alpha: float = 0.9, + reid_appearance_threshold: float = 0.25, + reid_proximity_threshold: float = 0.5, ) -> None: self.maximum_frames_without_update = self._compute_maximum_frames_without_update( lost_track_buffer=lost_track_buffer, @@ -145,6 +166,17 @@ def __init__( self.enable_cmc = enable_cmc self.cmc = CMC(CMCConfig(method=cmc_method, downscale=cmc_downscale)) if enable_cmc else None + self.reid_model = reid_model + if not 0.0 <= reid_ema_alpha <= 1.0: + raise ValueError(f"reid_ema_alpha must be in [0, 1], got {reid_ema_alpha}") + self.reid_ema_alpha = reid_ema_alpha + if not 0.0 <= reid_appearance_threshold <= 1.0: + raise ValueError(f"reid_appearance_threshold must be in [0, 1], got {reid_appearance_threshold}") + if not 0.0 <= reid_proximity_threshold <= 1.0: + raise ValueError(f"reid_proximity_threshold must be in [0, 1], got {reid_proximity_threshold}") + self.reid_appearance_threshold = reid_appearance_threshold + self.reid_proximity_threshold = reid_proximity_threshold + self._init_timestamp_state(frame_rate) def update( @@ -181,11 +213,15 @@ def update( the last state. Notes: - - If CMC is enabled, pass the current video frame via ``frame`` so the - tracker can estimate a global affine transform and warp predicted - track states before association. When ``frame=None`` and - ``enable_cmc=True``, CMC is silently skipped for that step. + - If CMC or ReID is enabled, pass the current video frame via ``frame`` + so the tracker can estimate a global affine transform and/or extract + appearance embeddings before association. When ``frame=None`` and + ``enable_cmc=True``, CMC is silently skipped for that step; ReID + requires ``frame`` and raises when it is ``None``. """ + if self.reid_model is not None and frame is None: + raise ValueError(f"{type(self).__name__}.update() requires frame when reid_model is set.") + timing = self._predict_timing(timestamp) if timing.skip_update: return self._detections_for_skipped_update(detections) @@ -254,38 +290,47 @@ def update( # redundant, so all stages read boxes from this map keyed by ``id()``. predicted_state_boxes = {id(track): track.get_state_bbox() for track in self.tracks} + det_embeddings: np.ndarray | None = None + if self.reid_model is not None and frame is not None: + if len(high_boxes) > 0: + det_embeddings = extract_detection_embeddings(self.reid_model, frame, high_boxes) + # Step 1: associate high-confidence detections to confirmed + lost tracks. # Lost tracks are included here (following the original ByteTrack), and # IoU is fused with detection scores. strack_pool = confirmed_tracks + lost_tracks - iou_matrix = self._get_iou_matrix(strack_pool, high_boxes, predicted_state_boxes) - iou_matrix = _fuse_score(self.iou.normalize_for_fusion(iou_matrix), high_scores) + similarity_matrix = self._association_similarity( + strack_pool, high_boxes, predicted_state_boxes, high_scores, det_embeddings + ) matched, unmatched_pool, unmatched_high = self._get_associated_indices( - iou_matrix, self.minimum_iou_threshold_first_assoc + similarity_matrix, self.minimum_iou_threshold_first_assoc ) for row, col in matched: - track = strack_pool[row] - track.update(high_boxes[col]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(high_indices[col])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + strack_pool[row], + high_boxes[col], + det_embeddings[col] if det_embeddings is not None else None, + int(high_indices[col]), + out_det_indices, + out_tracker_ids, + ) # Step 2: associate low-confidence detections to remaining *tracked* tracks # only (excluding lost tracks, following the original ByteTrack). - # No score fusing in second association. remaining_tracked = [strack_pool[i] for i in unmatched_pool if strack_pool[i].time_since_update == 1] iou_matrix = self._get_iou_matrix(remaining_tracked, low_boxes, predicted_state_boxes) matched, _, unmatched_low = self._get_associated_indices(iou_matrix, self.minimum_iou_threshold_second_assoc) for row, col in matched: - track = remaining_tracked[row] - track.update(low_boxes[col]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(low_indices[col])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + remaining_tracked[row], + low_boxes[col], + None, + int(low_indices[col]), + out_det_indices, + out_tracker_ids, + ) # Unmatched low-confidence detections for det_local_idx in sorted(unmatched_low): @@ -301,21 +346,25 @@ def update( if len(unconfirmed_tracks) > 0 and len(unmatched_high_list) > 0: uh_boxes = high_boxes[unmatched_high_list] uh_scores = high_scores[unmatched_high_list] + uh_embeddings = det_embeddings[unmatched_high_list] if det_embeddings is not None else None + similarity_matrix = self._association_similarity( + unconfirmed_tracks, uh_boxes, predicted_state_boxes, uh_scores, uh_embeddings + ) - iou_matrix = self._get_iou_matrix(unconfirmed_tracks, uh_boxes, predicted_state_boxes) - iou_matrix = _fuse_score(self.iou.normalize_for_fusion(iou_matrix), uh_scores) matched_uc, unmatched_uc_indices, remaining_uh = self._get_associated_indices( - iou_matrix, self.minimum_iou_threshold_unconfirmed_assoc + similarity_matrix, self.minimum_iou_threshold_unconfirmed_assoc ) for row, col in matched_uc: - track = unconfirmed_tracks[row] orig_high_idx = unmatched_high_list[col] - track.update(high_boxes[orig_high_idx]) - if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: - track.tracker_id = self._allocate_tracker_id() - out_det_indices.append(int(high_indices[orig_high_idx])) - out_tracker_ids.append(track.tracker_id) + self._assign_track_detection( + unconfirmed_tracks[row], + high_boxes[orig_high_idx], + det_embeddings[orig_high_idx] if det_embeddings is not None else None, + int(high_indices[orig_high_idx]), + out_det_indices, + out_tracker_ids, + ) # Only remaining unmatched high-conf dets proceed to spawning unmatched_high = [unmatched_high_list[i] for i in remaining_uh] @@ -335,6 +384,7 @@ def update( out_det_indices, out_tracker_ids, is_first_frame=(self.frame_id == 1), + det_embeddings=det_embeddings, ) # Full lifecycle prune: removes immature+unmatched and any remaining expired @@ -356,11 +406,59 @@ def update( result.tracker_id = np.array(out_tracker_ids, dtype=int) return result + def _assign_track_detection( + self, + track: BoTSORTTracklet, + bbox: np.ndarray, + embedding: np.ndarray | None, + global_det_index: int, + out_det_indices: list[int], + out_tracker_ids: list[int], + ) -> None: + """Update a track from a matched detection and record output indices.""" + track.update(bbox) + if track.feature_bank is not None and embedding is not None: + track.feature_bank.update(embedding, normalized=True) + if track.number_of_successful_updates >= self.minimum_consecutive_frames and track.tracker_id == -1: + track.tracker_id = self._allocate_tracker_id() + out_det_indices.append(global_det_index) + out_tracker_ids.append(track.tracker_id) + + def _association_similarity( + self, + tracklets: list[BoTSORTTracklet], + boxes: np.ndarray, + tracklet_boxes_by_id: dict[int, np.ndarray], + scores: np.ndarray, + embeddings: np.ndarray | None, + ) -> np.ndarray: + """Score-fused association similarity, with optional BoT-SORT ReID fusion.""" + iou_sim_raw = self.iou.normalize_for_fusion(self._get_iou_matrix(tracklets, boxes, tracklet_boxes_by_id)) + iou_sim_fused = _fuse_score(iou_sim_raw, scores) + if embeddings is None or len(tracklets) == 0: + return iou_sim_fused + + track_feats = [None if t.feature_bank is None else t.feature_bank.feature for t in tracklets] + proximity_iou = ( + iou_sim_raw + if isinstance(self.iou, IoU) + else self._get_iou_matrix(tracklets, boxes, tracklet_boxes_by_id, metric=IoU()) + ) + return fuse_botsort_reid_association( + iou_sim_fused, + appearance_similarity(track_feats, embeddings, det_embeddings_normalized=True), + proximity_iou_similarity=proximity_iou, + reid_proximity_threshold=self.reid_proximity_threshold, + reid_appearance_threshold=self.reid_appearance_threshold, + ) + def _get_iou_matrix( self, tracklets: list[BoTSORTTracklet], detections: np.ndarray, tracklet_boxes_by_id: dict[int, np.ndarray], + *, + metric: BaseIoU | None = None, ) -> np.ndarray: """Compute IoU similarity between tracklet states and detection boxes. @@ -370,6 +468,9 @@ def _get_iou_matrix( tracklet_boxes_by_id: Mapping from ``id(track)`` to the track's predicted state bbox, computed once per ``update()`` and reused across association stages to avoid recomputing ``get_state_bbox``. + metric: IoU variant to score with, defaulting to ``self.iou``. Passed + explicitly as plain ``IoU()`` for the ReID proximity gate, which + is defined on true IoU even when association uses GIoU/DIoU/CIoU. Raises: KeyError: If a tracklet passed in is absent from ``tracklet_boxes_by_id`` @@ -387,7 +488,7 @@ def _get_iou_matrix( "tracklet_boxes_by_id must contain every tracklet passed to this helper " "(it is built from self.tracks once per update())" ) from exc - return self.iou.compute(tracklet_boxes, detections) + return (metric or self.iou).compute(tracklet_boxes, detections) def _get_associated_indices( self, @@ -436,6 +537,7 @@ def _spawn_new_tracks( out_det_indices: list[int], out_tracker_ids: list[int], is_first_frame: bool = False, + det_embeddings: np.ndarray | None = None, ) -> None: """Create new tracklets from unmatched high-confidence detections. @@ -453,6 +555,10 @@ def _spawn_new_tracks( initial_bbox=detection_boxes[global_idx], state_estimator_class=self.state_estimator_class, ) + if self.reid_model is not None: + tracklet.feature_bank = FeatureBank(self.reid_ema_alpha) + if det_embeddings is not None: + tracklet.feature_bank.update(det_embeddings[det_local_idx], normalized=True) if is_first_frame and self.instant_first_frame_activation: tracklet.tracker_id = self._allocate_tracker_id() self.tracks.append(tracklet) diff --git a/src/trackers/core/botsort/tracklet.py b/src/trackers/core/botsort/tracklet.py index 2e0597e57..5740cc37d 100644 --- a/src/trackers/core/botsort/tracklet.py +++ b/src/trackers/core/botsort/tracklet.py @@ -6,6 +6,8 @@ from __future__ import annotations +from typing import TYPE_CHECKING + import numpy as np from trackers.utils.base_tracklet import BaseTracklet @@ -19,6 +21,9 @@ XYXYStateEstimator, ) +if TYPE_CHECKING: + from trackers.core.reid.feature_bank import FeatureBank + class BoTSORTTracklet(BaseTracklet): """Tracklet for the BoT-SORT tracker. @@ -53,6 +58,8 @@ def __init__( # Count initial bbox as first successful update so that # number_of_successful_updates starts at 1. self.number_of_successful_updates = 1 + # Optional appearance feature bank, set by BoTSORTTracker when ReID is enabled. + self.feature_bank: FeatureBank | None = None def _configure_initial_noise(self, bbox: np.ndarray) -> None: """Set initial P, Q, R based on the first detection's size.""" diff --git a/src/trackers/core/reid/__init__.py b/src/trackers/core/reid/__init__.py new file mode 100644 index 000000000..c56bdd240 --- /dev/null +++ b/src/trackers/core/reid/__init__.py @@ -0,0 +1,43 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Appearance-ReID association plus offline threshold selection and plotting tools.""" + +from __future__ import annotations + +from trackers.core.reid.appearance import ( + appearance_similarity, + extract_detection_embeddings, + extract_ground_truth_embeddings, +) +from trackers.core.reid.encoder import ReIDEncoder +from trackers.core.reid.feature_bank import FeatureBank +from trackers.core.reid.thresholds import ( + DEFAULT_FRAME_GAP_BANDS, + AppearanceDistances, + ThresholdLines, + plot_appearance_distances, + plot_frame_gap_sweep, + roc_auc, + sample_appearance_distances, + sweep_frame_gap, +) + +__all__ = [ + "DEFAULT_FRAME_GAP_BANDS", + "AppearanceDistances", + "FeatureBank", + "ReIDEncoder", + "ThresholdLines", + "appearance_similarity", + "extract_detection_embeddings", + "extract_ground_truth_embeddings", + "plot_appearance_distances", + "plot_frame_gap_sweep", + "roc_auc", + "sample_appearance_distances", + "sweep_frame_gap", +] diff --git a/src/trackers/core/reid/appearance.py b/src/trackers/core/reid/appearance.py new file mode 100644 index 000000000..bfead0283 --- /dev/null +++ b/src/trackers/core/reid/appearance.py @@ -0,0 +1,243 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Appearance embedding helpers for tracker association.""" + +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +import numpy as np +import supervision as sv + +from trackers.core.reid.encoder import ReIDEncoder +from trackers.io.frames import load_mot_frame_image, resolve_mot_frame_path +from trackers.io.mot import load_mot_file + +_NORM_EPS = 1e-12 + + +def _require_embedding_matrix(embeddings: np.ndarray) -> np.ndarray: + """Return a finite float32 embedding matrix.""" + cleaned = np.asarray(embeddings, dtype=np.float32) + if cleaned.ndim != 2: + raise ValueError(f"embeddings must be 2-D, got shape {cleaned.shape}") + if cleaned.size > 0 and not np.all(np.isfinite(cleaned)): + raise ValueError("embeddings must contain only finite values") + return cleaned + + +def _l2_normalize(embedding: np.ndarray) -> np.ndarray: + """Return an L2-normalised 1-D vector.""" + flat = np.asarray(embedding, dtype=np.float64).reshape(-1) + if flat.size == 0: + raise ValueError("embedding must be non-empty") + if not np.all(np.isfinite(flat)): + raise ValueError("embedding must contain only finite values") + norm = float(np.linalg.norm(flat)) + return (flat / max(norm, _NORM_EPS)).astype(np.float32) + + +def _l2_normalize_rows(embeddings: np.ndarray) -> np.ndarray: + """L2-normalise each row in an embedding matrix.""" + if embeddings.size == 0: + return embeddings + mat = embeddings.astype(np.float64) + norms = np.linalg.norm(mat, axis=1, keepdims=True) + return (mat / np.maximum(norms, _NORM_EPS)).astype(np.float32) + + +def extract_detection_embeddings( + model: ReIDEncoder, + frame: np.ndarray, + boxes: np.ndarray, +) -> np.ndarray: + """Extract appearance embeddings for detection boxes. + + Args: + model: Encoder that returns one embedding per detection. + frame: BGR image with shape ``(H, W, C)``. + boxes: Detection boxes in ``xyxy`` format with shape ``(N, 4)``. + + Returns: + Float32 embedding matrix with shape ``(N, D)``. Returns shape ``(0, 0)`` + when ``boxes`` is empty without calling ``model``. + + Raises: + ValueError: If the encoder output is not a finite 2-D matrix or its row + count does not match the number of boxes. + + Example: + >>> class Encoder: + ... def extract_features(self, detections, frame): + ... return np.ones((len(detections), 2), dtype=np.float32) + >>> frame = np.zeros((8, 8, 3), dtype=np.uint8) + >>> boxes = np.array([[0.0, 0.0, 4.0, 4.0]], dtype=np.float32) + >>> extract_detection_embeddings(Encoder(), frame, boxes) + array([[0.70710677, 0.70710677]], dtype=float32) + """ + if len(boxes) == 0: + return np.empty((0, 0), dtype=np.float32) + embeddings = _require_embedding_matrix(model.extract_features(sv.Detections(xyxy=boxes), frame)) + if embeddings.shape[0] != len(boxes): + raise ValueError(f"embedding rows ({embeddings.shape[0]}) must match detection boxes ({len(boxes)})") + return _l2_normalize_rows(embeddings) + + +def extract_ground_truth_embeddings( + model: ReIDEncoder, + dataset_root: str | Path, + *, + sequences: Sequence[str] | None = None, + keep_classes: Sequence[int] | None = None, + frame_stride: int = 1, +) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Embed every ground-truth crop in a MOT-format dataset. + + Walks ``{dataset_root}/{sequence}/gt/gt.txt`` against the frames in + ``{dataset_root}/{sequence}/img1``. Rows flagged ignore (confidence ``0``) + are always dropped. Identities are renumbered across sequences, so the same + track number in two videos stays two identities. + + The four returned arrays are what + :func:`trackers.core.reid.sample_appearance_distances` expects, which is how + an ``appearance_threshold`` gets calibrated on a new dataset. + + Args: + model: Encoder to embed crops with. + dataset_root: Directory holding one folder per sequence. + sequences: Sequences to read. Defaults to every one found. + keep_classes: MOT class ids to keep, e.g. ``(1,)`` for MOT17 + pedestrians. Defaults to every class, which suits single-class + datasets such as SoccerNet. + frame_stride: Embed every Nth frame. Raising it trims a dense dataset, + at the cost of the smallest frame gaps. + + Returns: + ``(embeddings, ids, frame_ids, sequence_ids)``, aligned row-wise. + + Raises: + FileNotFoundError: If no sequence under ``dataset_root`` has a + ``gt/gt.txt``, or a named sequence is missing one. + ValueError: If no crop survived the filters. + + Examples: + >>> from trackers.core.reid import extract_ground_truth_embeddings # doctest: +SKIP + >>> + >>> crops = extract_ground_truth_embeddings( # doctest: +SKIP + ... model, "mot17/val", keep_classes=(1,) + ... ) + """ + root = Path(dataset_root) + names = sorted(p.parent.parent.name for p in root.glob("*/gt/gt.txt")) if sequences is None else list(sequences) + if not names: + raise FileNotFoundError(f"no sequences with gt/gt.txt under {root}") + + embeddings: list[np.ndarray] = [] + ids: list[int] = [] + frame_ids: list[int] = [] + sequence_ids: list[int] = [] + identity_by_key: dict[str, int] = {} + + for sequence_id, name in enumerate(names): + ground_truth = load_mot_file(root / name / "gt" / "gt.txt") + frame_dir = root / name / "img1" + for frame_id in range(1, max(ground_truth) + 1, frame_stride): + rows = ground_truth.get(frame_id) + if rows is None: + continue + keep = rows.confidences > 0 + if keep_classes is not None: + keep &= np.isin(rows.classes, list(keep_classes)) + if not keep.any(): + continue + frame_path = resolve_mot_frame_path(frame_dir, frame_id) + if frame_path is None: + continue + boxes = sv.xywh_to_xyxy(rows.boxes[keep]).astype(np.float32) + features = extract_detection_embeddings(model, load_mot_frame_image(frame_dir, frame_id), boxes) + for feature, track_id in zip(features, rows.ids[keep], strict=True): + key = f"{name}_{int(track_id)}" + embeddings.append(feature) + ids.append(identity_by_key.setdefault(key, len(identity_by_key))) + frame_ids.append(frame_id) + sequence_ids.append(sequence_id) + + if not embeddings: + raise ValueError(f"no ground-truth crops under {root} survived the class and ignore-flag filters") + return ( + np.stack(embeddings), + np.asarray(ids, dtype=np.int64), + np.asarray(frame_ids, dtype=np.int64), + np.asarray(sequence_ids, dtype=np.int64), + ) + + +def appearance_similarity( + track_features: Sequence[np.ndarray | None], + det_embeddings: np.ndarray, + *, + det_embeddings_normalized: bool = False, +) -> np.ndarray: + """Compute cosine similarities between track and detection embeddings. + + Args: + track_features: Sequence of ``T`` track features, each with shape ``(D,)``. + Entries may be ``None`` when a track has no appearance feature. + det_embeddings: Detection embedding matrix with shape ``(N, D)``. + det_embeddings_normalized: Whether detection rows are already validated + unit embeddings from :func:`extract_detection_embeddings`. + + Returns: + Float32 similarity matrix with shape ``(T, N)``. A ``None`` track feature + produces an all-zero row. + + Raises: + ValueError: If detection embeddings are not a finite 2-D matrix, or a + track feature is empty, non-finite, or has the wrong dimension. + + Example: + >>> tracks = [np.array([1.0, 0.0], dtype=np.float32), None] + >>> detections = np.array([[1.0, 0.0], [0.0, 1.0]], dtype=np.float32) + >>> appearance_similarity(tracks, detections) + array([[1., 0.], + [0., 0.]], dtype=float32) + """ + n_tracks = len(track_features) + if det_embeddings_normalized: + det_embeddings = np.asarray(det_embeddings, dtype=np.float32) + else: + det_embeddings = _l2_normalize_rows(_require_embedding_matrix(det_embeddings)) + n_dets = det_embeddings.shape[0] + similarity = np.zeros((n_tracks, n_dets), dtype=np.float32) + + if n_tracks == 0 or n_dets == 0: + return similarity + + embed_dim = det_embeddings.shape[1] + track_rows: list[np.ndarray] = [] + kept_indices: list[int] = [] + for track_idx, feature in enumerate(track_features): + if feature is None: + continue + flat = np.asarray(feature, dtype=np.float32).reshape(-1) + if flat.shape[0] != embed_dim: + raise ValueError( + f"track feature dim {flat.shape[0]} does not match detection " + f"embedding dim {embed_dim} (track index {track_idx})" + ) + track_rows.append(flat) + kept_indices.append(track_idx) + + if not track_rows: + return similarity + + normalized_track_rows = _l2_normalize_rows(_require_embedding_matrix(np.stack(track_rows))) + cosine_similarities = (normalized_track_rows @ det_embeddings.T).astype(np.float32) + similarity[kept_indices] = cosine_similarities + + return similarity diff --git a/src/trackers/core/reid/encoder.py b/src/trackers/core/reid/encoder.py new file mode 100644 index 000000000..bfa2efc08 --- /dev/null +++ b/src/trackers/core/reid/encoder.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Encoder protocol for ReID association.""" + +from __future__ import annotations + +from typing import Protocol + +import numpy as np +import supervision as sv + + +class ReIDEncoder(Protocol): + """Encoder with ``extract_features(detections, frame)``.""" + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + """Return appearance embeddings for each detection box. + + Args: + detections: Boxes to embed (``xyxy``). + frame: BGR frame the detections were produced on. + + Returns: + Float32 array of shape ``(N, D)``, or ``(0, 0)`` when empty. + """ + ... diff --git a/src/trackers/core/reid/feature_bank.py b/src/trackers/core/reid/feature_bank.py new file mode 100644 index 000000000..67703ff7a --- /dev/null +++ b/src/trackers/core/reid/feature_bank.py @@ -0,0 +1,62 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Per-track exponential moving average feature bank.""" + +from __future__ import annotations + +import numpy as np + +from trackers.core.reid.appearance import _l2_normalize + + +class FeatureBank: + """Per-track EMA unit embedding (L2 before and after blend). + + Follows the per-track feature update in BoT-SORT + (https://github.com/NirAharon/BoT-SORT/blob/main/tracker/bot_sort.py). + """ + + def __init__(self, alpha: float = 0.9) -> None: + if not 0.0 <= alpha <= 1.0: + raise ValueError(f"alpha must be in [0, 1], got {alpha}") + self._alpha = alpha + self._feature: np.ndarray | None = None + + @property + def feature(self) -> np.ndarray | None: + """Current stored unit embedding, or ``None`` if never updated.""" + return None if self._feature is None else self._feature.copy() + + def update(self, embedding: np.ndarray, *, normalized: bool = False) -> None: + """Blend an embedding into the stored unit feature. + + Args: + embedding: Raw embedding, or a unit embedding when ``normalized`` is + true. + normalized: Skip input normalization for validated output from + ``extract_detection_embeddings``. + + Raises: + ValueError: If the embedding is empty, non-finite, or changes shape. + """ + cleaned = np.asarray(embedding, dtype=np.float32).reshape(-1) + if cleaned.size == 0: + raise ValueError("embedding must be non-empty") + if not normalized: + cleaned = _l2_normalize(cleaned) + + if self._feature is None: + self._feature = cleaned.copy() + return + + if self._feature.shape != cleaned.shape: + raise ValueError( + f"embedding shape {cleaned.shape} does not match stored feature shape {self._feature.shape}" + ) + + blended = self._alpha * self._feature + (1.0 - self._alpha) * cleaned + self._feature = _l2_normalize(blended) diff --git a/src/trackers/core/reid/fusion.py b/src/trackers/core/reid/fusion.py new file mode 100644 index 000000000..f76637f0b --- /dev/null +++ b/src/trackers/core/reid/fusion.py @@ -0,0 +1,67 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ +# +# Adapted from NirAharon/BoT-SORT (MIT) +# Copyright (c) 2022 Nir Aharon +# Source: https://github.com/NirAharon/BoT-SORT +# Reference: tracker/bot_sort.py (ReID appearance-IoU cost fusion) +# ------------------------------------------------------------------------ + +"""Appearance-IoU fusion methods for ReID association. + +Fusion methods are numpy-only and take track-detection similarity matrices, so they are reusable across trackers rather +than tied to any single one. +""" + +from __future__ import annotations + +import numpy as np + + +def fuse_botsort_reid_association( + association_similarity: np.ndarray, + appearance_similarity: np.ndarray, + *, + reid_proximity_threshold: float, + reid_appearance_threshold: float, + proximity_iou_similarity: np.ndarray | None = None, +) -> np.ndarray: + """Fuse IoU and appearance the way BoT-SORT ``bot_sort.py`` does. + + Computes ``min(association_cost, capped_appearance_cost)`` with proximity + and appearance gates, then returns the corresponding similarity matrix + (``1 - cost``). + + ``proximity_iou_similarity`` is the standard-IoU gate (defaults to + ``association_similarity``). Pass it separately when association uses + GIoU/DIoU/CIoU so proximity still uses plain IoU. + + Args: + association_similarity: Geometry-based track-detection similarities with + shape ``(T, N)``. + appearance_similarity: Cosine similarities for the same pairs with shape + ``(T, N)``. + reid_proximity_threshold: Maximum standard-IoU distance at which appearance + may lower the association cost. + reid_appearance_threshold: Maximum appearance cost allowed to contribute to + the fused association. + proximity_iou_similarity: Standard-IoU similarities with shape ``(T, N)``. + Defaults to ``association_similarity``. + + Returns: + Fused track-detection similarities with shape ``(T, N)``, obtained from + ``1 - min(d_iou, d_app)`` after applying both gates. + """ + if proximity_iou_similarity is None: + proximity_iou_similarity = association_similarity + + d_iou = 1.0 - association_similarity + d_iou_proximity = 1.0 - proximity_iou_similarity + d_app = 0.5 * (1.0 - appearance_similarity) + d_app = np.where(d_app > reid_appearance_threshold, 1.0, d_app) + d_app = np.where(d_iou_proximity > reid_proximity_threshold, 1.0, d_app) + fused_cost = np.minimum(d_iou, d_app) + return 1.0 - fused_cost diff --git a/src/trackers/core/reid/thresholds.py b/src/trackers/core/reid/thresholds.py new file mode 100644 index 000000000..1a2144f15 --- /dev/null +++ b/src/trackers/core/reid/thresholds.py @@ -0,0 +1,543 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Association-local appearance distance sampling for threshold selection.""" + +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Hashable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any + +import numpy as np + +from trackers.core.reid.appearance import _l2_normalize_rows, _require_embedding_matrix + +if TYPE_CHECKING: + from matplotlib.figure import Figure + +DEFAULT_FRAME_GAP_BANDS: tuple[tuple[int, int], ...] = ( + (1, 1), + (2, 5), + (6, 15), + (16, 30), + (31, 60), + (61, 120), + (121, 240), +) + +_SAME_ID_COLOR = "#3366CC" +_DIFFERENT_ID_COLOR = "#DC3912" + +_THRESHOLD_STYLES = (("#111111", "--", 1.6), ("#666666", ":", 1.5)) + +ThresholdLines = Sequence[float] | Mapping[float, str] + + +class _NoPairsInBand(ValueError): + """Signal that valid inputs contain no sampleable pairs in one gap band.""" + + +@dataclass(frozen=True) +class AppearanceDistances: + """Sampled appearance distances for one frame-gap band. + + Distances are ``0.5 * (1 - cosine_similarity)``, the term BoT-SORT gates on + with ``reid_appearance_threshold``. + + Attributes: + same_id: Distances between two crops of the same identity. + different_id: Distances between crops of two different identities. + minimum_frame_gap: Lower bound of the band the pairs were drawn from. + maximum_frame_gap: Upper bound of the band the pairs were drawn from. + """ + + same_id: np.ndarray + different_id: np.ndarray + minimum_frame_gap: int + maximum_frame_gap: int + + @property + def label(self) -> str: + """Gap band as an axis label, e.g. ``"1"`` or ``"6-15"``.""" + if self.minimum_frame_gap == self.maximum_frame_gap: + return str(self.minimum_frame_gap) + return f"{self.minimum_frame_gap}-{self.maximum_frame_gap}" + + @property + def roc_auc(self) -> float: + """Threshold-free separability of the two classes. + + See :func:`roc_auc`. + """ + return roc_auc(self.same_id, self.different_id) + + def rates_at(self, threshold: float) -> tuple[float, float]: + """Return the match rate of both classes at one candidate threshold. + + A pair counts as accepted at or below the threshold, matching the gate in + :func:`trackers.core.reid.fusion.fuse_botsort_reid_association`, which discards + appearance only once the distance *exceeds* ``reid_appearance_threshold``. + + Args: + threshold: Candidate ``reid_appearance_threshold``. + + Returns: + ``(same_id_rate, different_id_rate)``: the fraction of same-ID pairs + accepted, to be maximised, and the fraction of different-ID pairs + accepted, to be minimised. + """ + return ( + float(np.mean(self.same_id <= threshold)), + float(np.mean(self.different_id <= threshold)), + ) + + +def roc_auc(same_id: np.ndarray, different_id: np.ndarray) -> float: + """Return the probability that a same-ID pair scores closer than a different-ID pair. + + Ties count as half. Equivalent to the area under the curve traced by sweeping + the threshold from 0 to 1 and plotting the two rates from + :meth:`AppearanceDistances.rates_at` against each other, which is why it + summarises every threshold instead of one chosen operating point. + + Args: + same_id: Distances between crops of the same identity. + different_id: Distances between crops of different identities. + + Returns: + ``1.0`` when every same-ID pair is closer than every different-ID pair, + ``0.5`` when appearance carries no information, and ``0.0`` when the two + classes are ordered the wrong way round. + + Raises: + ValueError: If either distance array is empty. + """ + if len(same_id) == 0 or len(different_id) == 0: + raise ValueError("both distance arrays must be non-empty") + # Compares every pair, so O(n*m); a rank-based form would scale better if pair counts ever grow. + same = same_id[:, None] + different = different_id[None, :] + return float(np.mean(same < different) + 0.5 * np.mean(same == different)) + + +def _window(frames: np.ndarray, anchor_frame: int, minimum_frame_gap: int, maximum_frame_gap: int) -> np.ndarray: + """Positions into sorted ``frames`` lying in the gap band before or after ``anchor_frame``.""" + before = np.arange( + np.searchsorted(frames, anchor_frame - maximum_frame_gap, side="left"), + np.searchsorted(frames, anchor_frame - minimum_frame_gap, side="right"), + ) + after = np.arange( + np.searchsorted(frames, anchor_frame + minimum_frame_gap, side="left"), + np.searchsorted(frames, anchor_frame + maximum_frame_gap, side="right"), + ) + return np.concatenate((before, after)) + + +class _SequenceIndex: + """Frame-sorted crop index for one sequence, with binary-search gap lookup. + + A *slot* is a position in the frame-sorted order, not a crop index. + + Attributes: + order: Crop indexes sorted by frame. + frames: Frame number per slot. + ids: Identity per slot. + tracks: Slots per identity, frame-sorted because the sort is stable. + track_of: Index into ``tracks`` per slot, so drawing never re-hashes a label. + """ + + def __init__(self, crop_indexes: np.ndarray, ids: np.ndarray, frame_ids: np.ndarray) -> None: + order = np.argsort(frame_ids[crop_indexes], kind="stable") + self.order = crop_indexes[order] + self.frames = frame_ids[self.order] + self.ids = ids[self.order] + tracks: list[list[int]] = [] + track_by_id: dict[Hashable, int] = {} + self.track_of = np.empty(len(self.ids), dtype=np.intp) + for slot, identity in enumerate(self.ids): + track = track_by_id.setdefault(identity, len(tracks)) + if track == len(tracks): + tracks.append([]) + tracks[track].append(slot) + self.track_of[slot] = track + self.tracks = [np.asarray(slots) for slots in tracks] + + def get_candidates( + self, anchor: int, minimum_frame_gap: int, maximum_frame_gap: int, *, same_id: bool + ) -> np.ndarray: + """Slots inside the gap band around ``anchor`` that may pair with it.""" + anchor_frame = int(self.frames[anchor]) + if same_id: + slots = self.tracks[self.track_of[anchor]] + return slots[_window(self.frames[slots], anchor_frame, minimum_frame_gap, maximum_frame_gap)] + candidates = _window(self.frames, anchor_frame, minimum_frame_gap, maximum_frame_gap) + return candidates[self.ids[candidates] != self.ids[anchor]] + + def get_anchor_groups(self, minimum_frame_gap: int, maximum_frame_gap: int, *, same_id: bool) -> list[np.ndarray]: + """Anchors that have a candidate in the band, grouped so that every group is drawn equally often. + + Same-ID anchors group by identity, so a long track cannot dominate the sample. Different-ID anchors form a + single group, i.e. uniform over crops. + """ + grouped = self.tracks if same_id else [np.arange(len(self.order))] + eligible = ( + np.asarray( + [ + s + for s in map(int, slots) + if len(self.get_candidates(s, minimum_frame_gap, maximum_frame_gap, same_id=same_id)) + ] + ) + for slots in grouped + ) + return [group for group in eligible if len(group)] + + +def _split_quota(total: int, bucket_count: int) -> list[int]: + """Spread ``total`` draws as evenly as possible over ``bucket_count`` buckets.""" + base, remainder = divmod(total, bucket_count) + return [base + (1 if index < remainder else 0) for index in range(bucket_count)] + + +def _draw_distances( + rng: np.random.Generator, + indexes: Mapping[Hashable, _SequenceIndex], + embeddings: np.ndarray, + *, + same_id: bool, + total_pairs: int, + minimum_frame_gap: int, + maximum_frame_gap: int, +) -> np.ndarray: + """Draw pairs of one class, splitting the quota equally over the sequences that hold any.""" + active = [ + (index, groups) + for index in indexes.values() + if (groups := index.get_anchor_groups(minimum_frame_gap, maximum_frame_gap, same_id=same_id)) + ] + if not active: + return np.asarray([], dtype=np.float64) + + distances: list[float] = [] + for (index, groups), quota in zip(active, _split_quota(total_pairs, len(active)), strict=True): + for _ in range(quota): + group = groups[int(rng.integers(len(groups)))] + anchor = int(group[int(rng.integers(len(group)))]) + candidates = index.get_candidates(anchor, minimum_frame_gap, maximum_frame_gap, same_id=same_id) + candidate = int(candidates[int(rng.integers(len(candidates)))]) + first, second = index.order[anchor], index.order[candidate] + distances.append(0.5 * (1.0 - float(embeddings[first] @ embeddings[second]))) + return np.asarray(distances, dtype=np.float64) + + +def sample_appearance_distances( + embeddings: np.ndarray, + ids: np.ndarray, + frame_ids: np.ndarray, + sequence_ids: np.ndarray, + *, + same_id_pairs: int = 5000, + different_id_pairs: int = 5000, + minimum_frame_gap: int = 1, + maximum_frame_gap: int = 30, + seed: int = 0, +) -> AppearanceDistances: + """Sample same-ID and different-ID crop pairs inside one frame-gap band. + + Anchors are filtered to those that have a candidate in the active gap band + before sampling, and every sequence holding any gets an equal quota. Within a + sequence, same-ID pairs pick an identity uniformly so that long tracks cannot + dominate, different-ID pairs pick an anchor crop uniformly, and both then pick + a candidate uniformly inside the band. + + Args: + embeddings: Appearance embeddings, shape ``(N, D)``. Normalised here, so + either raw or unit-length input works. + ids: Hashable scalar identity label per embedding, shape ``(N,)``. + frame_ids: Frame number per embedding, shape ``(N,)``. + sequence_ids: Hashable scalar sequence or video label per embedding, shape ``(N,)``. + same_id_pairs: Same-ID pairs to draw across all sequences. + different_id_pairs: Different-ID pairs to draw across all sequences. + minimum_frame_gap: Smallest allowed frame gap. Must be at least 1: a gap + of 0 lets a crop pair with itself, which is what puts the spike at + distance 0 in the original BoT-SORT figure. + maximum_frame_gap: Largest allowed frame gap, i.e. the association + horizon being measured. + seed: Seed for the pair-drawing generator. + + Returns: + The sampled distances for this band. + + Raises: + TypeError: If identity or sequence labels are not hashable. + ValueError: If the arrays disagree in length, the gap band is invalid, or + either class yielded no pairs at all. + """ + if minimum_frame_gap < 1 or maximum_frame_gap < minimum_frame_gap: + raise ValueError( + f"invalid frame gap band [{minimum_frame_gap}, {maximum_frame_gap}], expected 1 <= minimum <= maximum" + ) + embeddings = _require_embedding_matrix(embeddings) + ids = np.asarray(ids) + frame_ids = np.asarray(frame_ids) + sequence_ids = np.asarray(sequence_ids) + lengths = {len(embeddings), len(ids), len(frame_ids), len(sequence_ids)} + if len(lengths) != 1: + raise ValueError(f"embeddings, ids, frame_ids and sequence_ids must have equal length, got {sorted(lengths)}") + if len(embeddings) == 0: + raise ValueError("embeddings, ids, frame_ids and sequence_ids must contain at least one row") + + normalized = _l2_normalize_rows(embeddings) + rng = np.random.default_rng(seed) + crop_indexes_by_sequence: dict[Hashable, list[int]] = defaultdict(list) + for crop_index, sequence in enumerate(sequence_ids): + crop_indexes_by_sequence[sequence].append(crop_index) + indexes = { + sequence: _SequenceIndex(np.asarray(crop_indexes), ids, frame_ids) + for sequence, crop_indexes in crop_indexes_by_sequence.items() + } + + same_id = _draw_distances( + rng, + indexes, + normalized, + same_id=True, + total_pairs=same_id_pairs, + minimum_frame_gap=minimum_frame_gap, + maximum_frame_gap=maximum_frame_gap, + ) + different_id = _draw_distances( + rng, + indexes, + normalized, + same_id=False, + total_pairs=different_id_pairs, + minimum_frame_gap=minimum_frame_gap, + maximum_frame_gap=maximum_frame_gap, + ) + if len(same_id) == 0 or len(different_id) == 0: + raise _NoPairsInBand( + f"no association-local pairs in frame gap band [{minimum_frame_gap}, {maximum_frame_gap}]; " + "widen the band or check that ids, frame_ids and sequence_ids line up with the embeddings" + ) + return AppearanceDistances( + same_id=same_id, + different_id=different_id, + minimum_frame_gap=minimum_frame_gap, + maximum_frame_gap=maximum_frame_gap, + ) + + +def sweep_frame_gap( + embeddings: np.ndarray, + ids: np.ndarray, + frame_ids: np.ndarray, + sequence_ids: np.ndarray, + *, + gap_bands: Sequence[tuple[int, int]] = DEFAULT_FRAME_GAP_BANDS, + pairs_per_class: int = 5000, + seed: int = 0, +) -> list[AppearanceDistances]: + """Measure separability inside each frame-gap band in turn. + + A threshold tuned on consecutive frames says nothing about re-finding a track + after an occlusion, so this reports how far the same threshold carries as the + gap widens. Bands that hold no pairs are skipped rather than raising, since a + short dataset legitimately has no 240-frame gaps. + + Args: + embeddings: Appearance embeddings, shape ``(N, D)``. + ids: Identity label per embedding. + frame_ids: Frame number per embedding. + sequence_ids: Sequence or video label per embedding. + gap_bands: ``(minimum, maximum)`` frame gaps to measure. + pairs_per_class: Pairs drawn per class within each band. + seed: Seed for the pair-drawing generator. + + Returns: + One entry per band that yielded pairs, in ``gap_bands`` order. + + Raises: + ValueError: If the input arrays or a requested gap band are invalid. + """ + sweep: list[AppearanceDistances] = [] + for minimum_frame_gap, maximum_frame_gap in gap_bands: + try: + sweep.append( + sample_appearance_distances( + embeddings, + ids, + frame_ids, + sequence_ids, + same_id_pairs=pairs_per_class, + different_id_pairs=pairs_per_class, + minimum_frame_gap=minimum_frame_gap, + maximum_frame_gap=maximum_frame_gap, + seed=seed, + ) + ) + except _NoPairsInBand: + continue + return sweep + + +def _threshold_lines(thresholds: ThresholdLines) -> Iterator[tuple[float, dict[str, Any]]]: + """Yield ``(value, line keyword arguments)`` for each reference line.""" + notes = thresholds if isinstance(thresholds, Mapping) else {} + for index, value in enumerate(thresholds): + note = notes.get(value) + color, linestyle, linewidth = _THRESHOLD_STYLES[min(index, len(_THRESHOLD_STYLES) - 1)] + label = f"θ = {value:.2f}" + (f" ({note})" if note else "") + yield value, {"label": label, "color": color, "ls": linestyle, "lw": linewidth} + + +def plot_appearance_distances( + distances: AppearanceDistances, + *, + thresholds: ThresholdLines = (0.25,), + title: str | None = None, +) -> Figure: + """Plot the two distance distributions with candidate thresholds marked. + + Where the two histograms overlap is where no threshold can separate them. + + Args: + distances: Output of :func:`sample_appearance_distances`. + thresholds: Candidate thresholds to draw as vertical reference lines. Pass a + mapping to annotate them, e.g. ``{0.20: "selected", 0.25: "default"}``. + The first is drawn dashed black and the rest recede into grey. + title: Figure title. Defaults to naming the gap band that was sampled. + + Returns: + The figure the distances were drawn on. + """ + import matplotlib.pyplot as plt + + figure, ax = plt.subplots(figsize=(8, 4.5)) + bins = np.linspace(0.0, 1.0, 51).tolist() + + for values, color, name in ( + (distances.same_id, _SAME_ID_COLOR, "same ID"), + (distances.different_id, _DIFFERENT_ID_COLOR, "different ID"), + ): + ax.hist( + values, + bins=bins, + weights=np.full(len(values), 1.0 / len(values)), + alpha=0.65, + color=color, + label=f"{name} (n={len(values)})", + ) + for value, style in _threshold_lines(thresholds): + ax.axvline(value, **style) + + gap = f"{distances.label} frame gap" + ax.set( + xlabel="appearance distance (0.5 * (1 - cosine similarity))", + ylabel="probability", + title=title if title is not None else f"appearance distances, {gap}", + xlim=(0.0, 1.0), + ) + ax.legend(frameon=False, fontsize=9) + ax.grid(True, alpha=0.25) + figure.tight_layout() + return figure + + +def plot_frame_gap_sweep( + sweep: Sequence[AppearanceDistances], + *, + thresholds: ThresholdLines = (0.25,), + percentiles: tuple[int, int] = (10, 90), + title: str | None = None, +) -> Figure: + """Plot how separability degrades as the frame gap widens. + + The upper panel tracks each class's median and percentile band against the + gap; the lower panel tracks :func:`roc_auc`, which answers the same question + without committing to a threshold. Note that the two panels do not measure + the same thing: the shaded bands can sit clear of each other while the AUC is + still short of 1.0, because percentile ranges ignore where the mass sits. + + Args: + sweep: Output of :func:`sweep_frame_gap`. + thresholds: Candidate thresholds to draw as horizontal reference lines. Pass a + mapping to annotate them, e.g. ``{0.20: "selected", 0.25: "default"}``. + percentiles: ``(low, high)`` bounds of the band shaded around each class's + median. Keep it symmetric so both classes are read the same way. + title: Figure title. + + Returns: + The figure the sweep was drawn on. + + Raises: + ValueError: If ``sweep`` is empty. + """ + if len(sweep) == 0: + raise ValueError("sweep is empty; nothing to plot") + import matplotlib.pyplot as plt + + low_percentile, high_percentile = percentiles + positions = np.arange(len(sweep)) + + figure, (ax_distance, ax_auc) = plt.subplots( + 2, 1, figsize=(8, 6.5), sharex=True, gridspec_kw={"height_ratios": [2.2, 1.0]} + ) + for values, color, name in ( + ([band.same_id for band in sweep], _SAME_ID_COLOR, "same ID"), + ([band.different_id for band in sweep], _DIFFERENT_ID_COLOR, "different ID"), + ): + quantiles = np.array([np.percentile(band, [low_percentile, 50, high_percentile]) for band in values]) + ax_distance.fill_between(positions, quantiles[:, 0], quantiles[:, 2], color=color, alpha=0.22) + ax_distance.plot(positions, quantiles[:, 1], color=color, marker="o", lw=2, label=name) + for value, style in _threshold_lines(thresholds): + ax_distance.axhline(value, **style) + + ax_distance.set(ylabel="appearance distance") + ax_distance.set_title( + f"line = median, shaded = {low_percentile}th to {high_percentile}th percentile", + fontsize=8.5, + color="#333333", + pad=4, + ) + ax_distance.legend(loc="lower right", fontsize=9, ncol=2, framealpha=0.92, edgecolor="none") + ax_distance.grid(True, alpha=0.25) + + auc_values = [band.roc_auc for band in sweep] + ax_auc.plot(positions, auc_values, color="#111111", marker="s", lw=2) + for position, auc in zip(positions, auc_values, strict=True): + ax_auc.annotate( + f"{auc:.3f}", + (position, auc), + textcoords="offset points", + xytext=(0, 7), + ha="center", + fontsize=7.5, + color="#111111", + ) + ax_auc.axhline(0.5, color="#999999", ls=":", lw=1.2) + ax_auc.set( + xlabel="frames between the two crops", + ylabel="separability", + ylim=(0.42, 1.12), + xticks=positions, + xticklabels=[band.label for band in sweep], + ) + ax_auc.set_title( + "take one same-ID and one different-ID pair at random: how often is the same-ID one closer?" + "\n1.0 = always, 0.5 = coin flip. Counts every sampled pair, not the shaded overlap above.", + fontsize=8, + color="#333333", + pad=4, + ) + ax_auc.grid(True, alpha=0.25) + + if title is not None: + figure.suptitle(title, y=0.995) + figure.tight_layout() + return figure diff --git a/tests/cli/test_main.py b/tests/cli/test_main.py index 05bc0e8aa..4aa595344 100644 --- a/tests/cli/test_main.py +++ b/tests/cli/test_main.py @@ -767,6 +767,25 @@ def test_display_gained_a_negative_half(self, parser: _CLIParser) -> None: assert parser.parse_args(["--display"]).display is True assert parser.parse_args(["--no_display"]).display is False + @pytest.mark.parametrize( + ("arguments", "expected"), + [ + pytest.param([], None, id="omitted"), + pytest.param(["--reid.enable"], True, id="positive"), + pytest.param(["--reid.no_enable"], False, id="negative"), + pytest.param(["--reid.enable", "false"], False, id="explicit_false"), + ], + ) + def test_reid_enable_preserves_tristate( + self, + parser: _CLIParser, + arguments: list[str], + expected: bool | None, + ) -> None: + parsed = parser.instantiate_classes(parser.parse_args(arguments)) + + assert parsed.reid.enable is expected + @pytest.mark.parametrize( ("arguments", "expected"), [ diff --git a/tests/cli/test_track.py b/tests/cli/test_track.py index 7b82a9ca4..4369b30f7 100644 --- a/tests/cli/test_track.py +++ b/tests/cli/test_track.py @@ -6,8 +6,12 @@ from __future__ import annotations +import builtins +from collections.abc import Mapping, Sequence from dataclasses import fields +from pathlib import Path from typing import ClassVar +from unittest.mock import Mock import numpy as np import pytest @@ -15,6 +19,8 @@ from trackers.cli.track import ( _EXCLUDED_TRACKER_PARAMETERS, + DetectionOptions, + ReIDOptions, ShowOptions, TrackerOptions, _abbreviate_parameter_name, @@ -23,12 +29,16 @@ _format_labels, _init_annotators, _init_tracker, + _load_reid_model, + _reid_requested, _resolve_class_filter, _resolve_track_id_filter, _resolve_tracker_kwargs, _tracker_options_as_dict, + track_command, ) from trackers.core.base import BaseTracker +from trackers.core.botsort.tracker import BoTSORTTracker class TestInitAnnotators: @@ -350,3 +360,174 @@ def test_unsupported_override_is_dropped_with_a_warning(self) -> None: tracker = _init_tracker(options) assert not hasattr(tracker, "minimum_mask_coverage") + + +class _FakeReIDModel: + """Stand-in for ``reid.ReIDModel`` that records its loading kwargs.""" + + last_kwargs: ClassVar[dict | None] = None + + @classmethod + def from_pretrained(cls, **kwargs: object) -> _FakeReIDModel: + cls.last_kwargs = dict(kwargs) + return cls() + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + return np.zeros((len(detections), 8), dtype=np.float32) + + +@pytest.fixture +def fake_reid_module(monkeypatch: pytest.MonkeyPatch) -> type[_FakeReIDModel]: + """Install a fake ``reid`` module so loading needs no checkpoint download.""" + import sys + from types import ModuleType + + module = ModuleType("reid") + module.ReIDModel = _FakeReIDModel # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "reid", module) + _FakeReIDModel.last_kwargs = None + return _FakeReIDModel + + +class TestReIDOptions: + """CLI wiring for optional appearance-encoder loading.""" + + def test_model_source_implies_enable(self) -> None: + """Naming a checkpoint is enough; --reid.enable is not also required.""" + assert _reid_requested(ReIDOptions(model="osnet_x1_0_msmt17_combineall")) + + @pytest.mark.parametrize( + "options", + [ + pytest.param(ReIDOptions(architecture="osnet_x1_0"), id="architecture"), + pytest.param(ReIDOptions(device="cpu"), id="device"), + ], + ) + def test_non_default_options_imply_enable(self, options: ReIDOptions) -> None: + assert _reid_requested(options) + + def test_explicit_disable_conflicts_with_model(self) -> None: + with pytest.raises(ValueError, match="cannot be combined"): + _reid_requested(ReIDOptions(enable=False, model="osnet_x1_0_msmt17_combineall")) + + def test_absent_by_default(self) -> None: + """Default options leave ReID off, so geometry-only tracking is unchanged.""" + assert not _reid_requested(ReIDOptions()) + + def test_track_command_rejects_reid_without_source(self, capsys: pytest.CaptureFixture[str]) -> None: + """The command entry point rejects ReID when MOT input supplies no frames.""" + exit_code = track_command( + detection=DetectionOptions(mot_file=Path("detections.txt")), + reid=ReIDOptions(enable=True), + ) + + assert exit_code == 1 + assert capsys.readouterr().err == ( + "Error: ReID requires --source (video/webcam/images) so appearance embeddings " + "can be extracted from frames.\n" + ) + + def test_missing_optional_extra_reports_install_command(self, monkeypatch: pytest.MonkeyPatch) -> None: + """A missing ReID dependency is translated into actionable CLI guidance.""" + import sys + from types import ModuleType + + monkeypatch.setitem(sys.modules, "reid", ModuleType("reid")) + + with pytest.raises(ValueError) as exc_info: + _load_reid_model(ReIDOptions(enable=True)) + + assert str(exc_info.value) == ( + "ReID tracking requires the optional `trackers[reid]` extra.\nInstall with: pip install 'trackers[reid]'" + ) + assert isinstance(exc_info.value.__cause__, ImportError) + + @pytest.mark.parametrize( + "load_error", + [ + pytest.param(OSError("checkpoint unavailable"), id="os_error"), + pytest.param(ValueError("checkpoint unavailable"), id="value_error"), + pytest.param(RuntimeError("checkpoint unavailable"), id="runtime_error"), + ], + ) + def test_model_load_errors_include_reid_context( + self, + load_error: Exception, + fake_reid_module: type[_FakeReIDModel], + monkeypatch: pytest.MonkeyPatch, + ) -> None: + """Expected loader failures identify the ReID checkpoint operation.""" + monkeypatch.setattr(fake_reid_module, "from_pretrained", Mock(side_effect=load_error)) + + with pytest.raises(ValueError, match="Failed to load ReID model: checkpoint unavailable") as exc_info: + _load_reid_model(ReIDOptions(enable=True)) + + assert exc_info.value.__cause__ is load_error + + def test_encoder_reaches_the_tracker(self, fake_reid_module: type[_FakeReIDModel]) -> None: + """A requested encoder is injected as the tracker's reid_model.""" + tracker = _init_tracker(TrackerOptions(name="botsort"), ReIDOptions(enable=True, device="cpu")) + + assert isinstance(tracker, BoTSORTTracker) + assert isinstance(tracker.reid_model, _FakeReIDModel) + assert fake_reid_module.last_kwargs == {"device": "cpu"} + + def test_architecture_is_forwarded(self, fake_reid_module: type[_FakeReIDModel], tmp_path: Path) -> None: + """Bare weights pass both source and architecture through to the loader.""" + weights = tmp_path / "weights.pth" + weights.touch() + + _init_tracker( + TrackerOptions(name="botsort"), + ReIDOptions(model=str(weights), device="cpu", architecture="osnet_x1_0"), + ) + + assert fake_reid_module.last_kwargs is not None + assert fake_reid_module.last_kwargs["architecture"] == "osnet_x1_0" + assert fake_reid_module.last_kwargs["source"] == str(weights) + + def test_architecture_requires_model(self, fake_reid_module: type[_FakeReIDModel]) -> None: + """An architecture with no checkpoint to apply it to is rejected.""" + with pytest.raises(ValueError, match=r"--reid\.architecture requires --reid\.model"): + _init_tracker(TrackerOptions(name="botsort"), ReIDOptions(enable=True, architecture="osnet_x0_25")) + + def test_tracker_without_encoder_support_is_rejected(self, fake_reid_module: type[_FakeReIDModel]) -> None: + """Requesting ReID on a geometry-only tracker fails loudly.""" + with pytest.raises(ValueError, match=r"--reid\.\* options apply only to a tracker that accepts an encoder"): + _init_tracker(TrackerOptions(name="bytetrack"), ReIDOptions(enable=True)) + + def test_transitive_reid_import_error_is_preserved(self, monkeypatch: pytest.MonkeyPatch) -> None: + original_import = builtins.__import__ + + def fail_reid_import( + name: str, + global_vars: Mapping[str, object] | None = None, + local_vars: Mapping[str, object] | None = None, + fromlist: Sequence[str] | None = None, + level: int = 0, + ) -> object: + if name == "reid": + raise ImportError("No module named 'broken_dependency'", name="broken_dependency") + return original_import(name, global_vars, local_vars, fromlist, level) + + monkeypatch.setattr(builtins, "__import__", fail_reid_import) + + with pytest.raises(ImportError, match="broken_dependency") as exc_info: + _load_reid_model(ReIDOptions(enable=True)) + + assert exc_info.value.name == "broken_dependency" + + def test_reid_model_is_not_a_cli_flag(self) -> None: + """The encoder is built from ReIDOptions, so it must not become a --tracker flag.""" + assert "reid_model" in _EXCLUDED_TRACKER_PARAMETERS + assert "reid_model" not in {field.name for field in fields(TrackerOptions)} + + def test_appearance_parameters_stay_cli_reachable(self) -> None: + """The three scalar ReID knobs are still generated from the registry.""" + option_fields = {field.name for field in fields(TrackerOptions)} + assert { + "reid_appearance_threshold", + "reid_ema_alpha", + "reid_proximity_threshold", + } <= option_fields + assert {"appearance_threshold", "proximity_threshold"}.isdisjoint(option_fields) diff --git a/tests/core/test_botsort_reid.py b/tests/core/test_botsort_reid.py new file mode 100644 index 000000000..7a509123b --- /dev/null +++ b/tests/core/test_botsort_reid.py @@ -0,0 +1,288 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""BoT-SORT ReID association and fusion tests.""" + +from __future__ import annotations + +import subprocess +import sys + +import numpy as np +import pytest +import supervision as sv + +from trackers.core.botsort.tracker import BoTSORTTracker +from trackers.core.reid.fusion import fuse_botsort_reid_association + + +def _detection(xyxy: tuple[float, float, float, float], conf: float = 0.9) -> sv.Detections: + return sv.Detections( + xyxy=np.array([xyxy], dtype=np.float32), + confidence=np.array([conf], dtype=np.float32), + ) + + +def _frame(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + + +def _norm(vec: np.ndarray) -> np.ndarray: + vec = vec.astype(np.float32) + return vec / np.linalg.norm(vec) + + +class _KeyedReIDEncoder: + """Deterministic embeddings keyed by detection top-left corner.""" + + def __init__(self, table: dict[tuple[int, int], np.ndarray] | None = None) -> None: + self.table = table or {} + self.calls = 0 + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + self.calls += 1 + if len(detections) == 0: + return np.empty((0, 0), dtype=np.float32) + rows = [] + for box in detections.xyxy: + key = (round(float(box[0])), round(float(box[1]))) + rows.append(self.table.get(key, _norm(np.array([float(box[0]), float(box[1]), 1.0, 0.0])))) + return np.stack(rows) + + +def test_botsort_import_does_not_load_reid_model_stack() -> None: + """Importing BoT-SORT must not pull the heavy ``reid`` package (torch etc.).""" + result = subprocess.run( + [ + sys.executable, + "-c", + ( + "import sys; import trackers.core.botsort.tracker; " + "assert 'reid' not in sys.modules; " + "assert 'torch' not in sys.modules" + ), + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + + +class TestFuseBotsortReidAssociation: + """Unit tests for BoT-SORT IoU/appearance fusion gates.""" + + def test_appearance_can_win_when_proximity_passes(self) -> None: + # Association IoU 0.63 clears the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # so a strong appearance score can beat it (0.63 → 0.9). + fused = fuse_botsort_reid_association( + np.array([[0.63]], dtype=np.float32), + np.array([[0.8]], dtype=np.float32), + reid_proximity_threshold=0.5, + reid_appearance_threshold=0.25, + ) + assert fused[0, 0] == pytest.approx(0.9) + + def test_low_proximity_ignores_appearance(self) -> None: + # Association IoU 0.36 fails the proximity gate (needs IoU > 1 - 0.5 = 0.5), + # so appearance is discarded even though it is strong (0.9). Score stays IoU-only. + iou_only = np.array([[0.36]], dtype=np.float32) + fused = fuse_botsort_reid_association( + iou_only, + np.array([[0.9]], dtype=np.float32), + reid_proximity_threshold=0.5, + reid_appearance_threshold=0.25, + ) + assert fused[0, 0] == pytest.approx(float(iou_only[0, 0])) + + def test_proximity_uses_standard_iou_not_giou(self) -> None: + # Association score uses a high GIoU-like value (0.80), but standard IoU is + # only 0.35 and fails the proximity gate, so appearance must not be used. + association_iou = np.array([[0.80]], dtype=np.float32) + fused = fuse_botsort_reid_association( + association_iou, + np.array([[0.95]], dtype=np.float32), + reid_proximity_threshold=0.5, + reid_appearance_threshold=0.25, + proximity_iou_similarity=np.array([[0.35]], dtype=np.float32), + ) + assert fused[0, 0] == pytest.approx(float(association_iou[0, 0])) + + +class TestBoTSORTTrackerReID: + """Integration-style tests for BoT-SORT tracker appearance association.""" + + def test_rejects_invalid_reid_ema_alpha(self) -> None: + with pytest.raises(ValueError, match="reid_ema_alpha"): + BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder(), reid_ema_alpha=1.5) + + @pytest.mark.parametrize("parameter", ["reid_appearance_threshold", "reid_proximity_threshold"]) + @pytest.mark.parametrize("value", [-0.01, 1.01]) + def test_rejects_invalid_association_thresholds(self, parameter: str, value: float) -> None: + with pytest.raises(ValueError, match=parameter): + # mypy cannot narrow a dynamic dict against typed kwargs. + BoTSORTTracker(enable_cmc=False, **{parameter: value}) # type: ignore[arg-type] + + @pytest.mark.parametrize("parameter", ["reid_appearance_threshold", "reid_proximity_threshold"]) + @pytest.mark.parametrize("value", [0.0, 1.0]) + def test_accepts_association_threshold_boundaries(self, parameter: str, value: float) -> None: + # mypy cannot narrow a dynamic dict against typed kwargs. + tracker = BoTSORTTracker(enable_cmc=False, **{parameter: value}) # type: ignore[arg-type] + + assert getattr(tracker, parameter) == value + + def test_requires_frame_when_reid_enabled(self) -> None: + tracker = BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder()) + with pytest.raises(ValueError, match="requires frame"): + tracker.update(_detection((10.0, 10.0, 30.0, 30.0))) + + assert tracker.frame_id == 0 + tracked = tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame()) + np.testing.assert_array_equal(tracked.tracker_id, [0]) + + def test_feature_bank_initializes_on_spawn(self) -> None: + tracker = BoTSORTTracker(enable_cmc=False, reid_model=_KeyedReIDEncoder()) + tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame()) + bank = tracker.tracks[0].feature_bank + assert bank is not None and bank.feature is not None + + def test_unconfirmed_appearance_match_updates_feature_bank(self) -> None: + """An appearance-only unconfirmed match blends its detection feature.""" + initial_feature = _norm(np.array([1.0, 0.0, 0.0, 0.0])) + matched_feature = _norm(np.array([0.8, 0.6, 0.0, 0.0])) + encoder = _KeyedReIDEncoder({(10, 10): initial_feature}) + tracker = BoTSORTTracker( + enable_cmc=False, + instant_first_frame_activation=False, + reid_model=encoder, + reid_ema_alpha=0.5, + minimum_iou_threshold_unconfirmed_assoc=0.85, + ) + detection = _detection((10.0, 10.0, 30.0, 30.0), conf=0.8) + + tracker.update(detection, frame=_frame(2)) + track = tracker.tracks[0] + assert track.tracker_id == -1 + bank = track.feature_bank + assert bank is not None + feature_after_spawn = bank.feature + assert feature_after_spawn is not None + np.testing.assert_allclose(feature_after_spawn, initial_feature) + + # Geometry fused with confidence is only 0.8, below the 0.85 gate; + # cosine appearance raises the fused similarity to 0.9 and permits the match. + encoder.table[(10, 10)] = matched_feature + result = tracker.update(detection, frame=_frame(3)) + + assert len(tracker.tracks) == 1 + assert tracker.tracks[0] is track + assert result.tracker_id is not None + assert result.tracker_id.tolist() == [track.tracker_id] + expected_feature = _norm(0.5 * initial_feature + 0.5 * matched_feature) + feature_after_match = bank.feature + assert feature_after_match is not None + np.testing.assert_allclose(feature_after_match, expected_feature, rtol=1e-6, atol=1e-7) + + def test_appearance_changes_assignment_vs_geometry_only(self) -> None: + identity = _norm(np.array([1.0, 0.0, 0.0, 0.0])) + impostor = _norm(np.array([0.0, 1.0, 0.0, 0.0])) + + class _PhaseEncoder: + phase = 1 + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + rows = [] + for box in detections.xyxy: + key = (round(float(box[0])), round(float(box[1]))) + if self.phase == 1: + rows.append(identity) + elif key == (10, 10): + rows.append(impostor) + else: + rows.append(identity) + return np.stack(rows) + + encoder = _PhaseEncoder() + frame = _frame(1) + geo = BoTSORTTracker( + enable_cmc=False, + minimum_iou_threshold_first_assoc=0.01, + reid_appearance_threshold=0.6, + reid_proximity_threshold=0.99, + ) + geo.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=frame) + + reid = BoTSORTTracker( + enable_cmc=False, + minimum_iou_threshold_first_assoc=0.01, + reid_appearance_threshold=0.6, + reid_proximity_threshold=0.99, + reid_model=encoder, + ) + reid.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=frame) + track_id = int(reid.tracks[0].tracker_id) + + competitors = sv.Detections( + xyxy=np.array([[10.0, 10.0, 30.0, 30.0], [14.0, 14.0, 34.0, 34.0]], dtype=np.float32), + confidence=np.array([0.9, 0.9], dtype=np.float32), + ) + encoder.phase = 2 + geo_out = geo.update(competitors, frame=frame) + reid_out = reid.update(competitors, frame=frame) + + def _matched_xy(out: sv.Detections, tid: int) -> tuple[float, float]: + assert out.tracker_id is not None + box = out.xyxy[out.tracker_id == tid][0] + return float(box[0]), float(box[1]) + + assert _matched_xy(geo_out, int(geo.tracks[0].tracker_id)) == (10.0, 10.0) + assert _matched_xy(reid_out, track_id) == (14.0, 14.0) + + def test_low_confidence_stage_does_not_update_feature_bank(self) -> None: + model = _KeyedReIDEncoder({(10, 10): _norm(np.array([1.0, 0.0, 0.0, 0.0]))}) + tracker = BoTSORTTracker( + enable_cmc=False, + reid_model=model, + high_conf_det_threshold=0.8, + minimum_iou_threshold_second_assoc=0.01, + ) + tracker.update(_detection((10.0, 10.0, 30.0, 30.0)), frame=_frame(4)) + bank = tracker.tracks[0].feature_bank + assert bank is not None + before = bank.feature + assert before is not None + + calls_after_high = model.calls + tracker.update(_detection((12.0, 12.0, 32.0, 32.0), conf=0.5), frame=_frame(4)) + assert model.calls == calls_after_high + after = bank.feature + assert after is not None + np.testing.assert_allclose(before, after) + + @pytest.mark.integration + def test_real_reid_model_runs_over_frames(self) -> None: + """Smoke the ``trackers`` → ``reid`` boundary with a real encoder.""" + import reid + + reid_model = reid.ReIDModel.from_pretrained(architecture="osnet_x0_25", device="cpu") + tracker = BoTSORTTracker(enable_cmc=False, reid_model=reid_model) + + rng = np.random.default_rng(0) + box = np.array([30.0, 30.0, 70.0, 90.0], dtype=np.float32) + for _ in range(3): + frame = rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + detections = sv.Detections( + xyxy=box[None, :].copy(), + confidence=np.array([0.9], dtype=np.float32), + ) + result = tracker.update(detections, frame=frame) + assert result.tracker_id is not None + box = box + np.array([2.0, 1.0, 2.0, 1.0], dtype=np.float32) + + assert len(tracker.tracks) == 1 + bank = tracker.tracks[0].feature_bank + assert bank is not None and bank.feature is not None diff --git a/tests/core/test_reid_appearance.py b/tests/core/test_reid_appearance.py new file mode 100644 index 000000000..125094aab --- /dev/null +++ b/tests/core/test_reid_appearance.py @@ -0,0 +1,178 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Appearance similarity and embedding extraction tests.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import cv2 +import numpy as np +import pytest +import supervision as sv + +import trackers +from trackers.core import reid +from trackers.core.reid.appearance import ( + appearance_similarity, + extract_detection_embeddings, + extract_ground_truth_embeddings, +) + + +def _frame(seed: int = 0) -> np.ndarray: + rng = np.random.default_rng(seed) + return rng.integers(0, 255, (128, 128, 3), dtype=np.uint8) + + +def test_reid_public_api_is_exported_from_package_root() -> None: + """Every ReID subpackage export is available from the stable root API.""" + assert set(reid.__all__) <= set(trackers.__all__) + for name in reid.__all__: + assert getattr(trackers, name) is getattr(reid, name) + + +class _MeanIntensityEncoder: + """Encoder returning one row per box, tagged with the frame's mean intensity.""" + + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + return np.full((len(detections), 2), float(frame.mean()), dtype=np.float32) + + +def _write_sequence(root: Path, name: str, rows: list[tuple[int, int, int, int]], frames: int = 3) -> None: + """Write a MOT sequence where each row is ``(frame, track_id, confidence, class)``.""" + (root / name / "gt").mkdir(parents=True) + (root / name / "img1").mkdir(parents=True) + lines = [ + f"{frame},{track_id},0,0,10,10,{confidence},{class_id},1" for frame, track_id, confidence, class_id in rows + ] + (root / name / "gt" / "gt.txt").write_text("\n".join(lines)) + for frame_id in range(1, frames + 1): + cv2.imwrite(str(root / name / "img1" / f"{frame_id:06d}.jpg"), _frame(frame_id)) + + +class TestAppearanceSimilarity: + """Unit tests for cosine ``appearance_similarity`` and embedding extraction.""" + + def test_identical_vectors_are_one(self) -> None: + similarity = appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[1.0]], atol=1e-6) + + def test_orthogonal_vectors_are_zero(self) -> None: + similarity = appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[0.0, 1.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[0.0]], atol=1e-6) + + def test_normalizes_both_sides(self) -> None: + similarity = appearance_similarity( + [np.array([3.0, 4.0], dtype=np.float32)], + np.array([[6.0, 8.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[1.0]], atol=1e-6) + + def test_none_track_yields_zero_row(self) -> None: + similarity = appearance_similarity( + [None, np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + np.testing.assert_allclose(similarity, [[0.0], [1.0]], atol=1e-6) + + def test_empty_inputs_return_empty_matrix(self) -> None: + assert appearance_similarity([], np.empty((0, 4), dtype=np.float32)).shape == (0, 0) + assert appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.empty((0, 2), dtype=np.float32), + ).shape == (1, 0) + + def test_non_finite_detection_rows_raise(self) -> None: + with pytest.raises(ValueError, match="finite"): + appearance_similarity( + [np.array([1.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0], [np.nan, 1.0]], dtype=np.float32), + ) + + def test_incompatible_track_dimensions_raise(self) -> None: + with pytest.raises(ValueError, match="dim"): + appearance_similarity( + [np.array([1.0, 0.0, 0.0], dtype=np.float32)], + np.array([[1.0, 0.0]], dtype=np.float32), + ) + + def test_extract_detection_embeddings_requires_one_row_per_box(self) -> None: + # Encoder must return embeddings.shape[0] == len(boxes). + class _WrongLengthEncoder: + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + return np.empty((0, 4), dtype=np.float32) + + with pytest.raises(ValueError, match="rows"): + extract_detection_embeddings( + _WrongLengthEncoder(), + _frame(), + np.array([[0.0, 0.0, 10.0, 10.0]], dtype=np.float32), + ) + + def test_extract_detection_embeddings_normalizes_all_rows(self) -> None: + class _RawEncoder: + def extract_features(self, detections: sv.Detections, frame: np.ndarray) -> np.ndarray: + return np.array([[3.0, 4.0], [0.0, 0.0]], dtype=np.float32) + + embeddings = extract_detection_embeddings( + _RawEncoder(), + _frame(), + np.array([[0.0, 0.0, 10.0, 10.0], [20.0, 20.0, 30.0, 30.0]], dtype=np.float32), + ) + + np.testing.assert_allclose(embeddings, [[0.6, 0.8], [0.0, 0.0]], atol=1e-6) + + +class TestExtractGroundTruthEmbeddings: + """Unit tests for reading a MOT-format dataset into labeled crop embeddings.""" + + def test_identities_are_renumbered_across_sequences(self, tmp_path: Path) -> None: + # Track 1 in two sequences is two people, so the ids must not collapse. + rows = [(1, 1, 1, 1), (1, 2, 1, 1), (2, 1, 1, 1), (2, 2, 1, 1)] + _write_sequence(tmp_path, "seq_a", rows) + _write_sequence(tmp_path, "seq_b", rows) + + embeddings, ids, frame_ids, sequence_ids = extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path) + + assert embeddings.shape == (8, 2) + assert set(zip(sequence_ids.tolist(), ids.tolist())) == {(0, 0), (0, 1), (1, 2), (1, 3)} + np.testing.assert_array_equal(np.unique(frame_ids), [1, 2]) + + def test_ignored_rows_and_unwanted_classes_are_dropped(self, tmp_path: Path) -> None: + _write_sequence(tmp_path, "seq_a", [(1, 1, 1, 1), (1, 2, 0, 1), (1, 3, 1, 7)]) + + _, every_class, _, _ = extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path) + _, pedestrians, _, _ = extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path, keep_classes=(1,)) + + assert len(every_class) == 2 + assert len(pedestrians) == 1 + + def test_frame_stride_subsamples_frames(self, tmp_path: Path) -> None: + _write_sequence(tmp_path, "seq_a", [(frame, 1, 1, 1) for frame in (1, 2, 3)]) + + _, _, frame_ids, _ = extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path, frame_stride=2) + + np.testing.assert_array_equal(frame_ids, [1, 3]) + + def test_frames_without_images_are_skipped(self, tmp_path: Path) -> None: + _write_sequence(tmp_path, "seq_a", [(frame, 1, 1, 1) for frame in (1, 2, 3)], frames=2) + + _, _, frame_ids, _ = extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path) + + np.testing.assert_array_equal(frame_ids, [1, 2]) + + def test_dataset_without_annotations_raises(self, tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError, match=re.escape("gt/gt.txt")): + extract_ground_truth_embeddings(_MeanIntensityEncoder(), tmp_path) diff --git a/tests/core/test_reid_feature_bank.py b/tests/core/test_reid_feature_bank.py new file mode 100644 index 000000000..c9c00519c --- /dev/null +++ b/tests/core/test_reid_feature_bank.py @@ -0,0 +1,63 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Per-track appearance feature bank tests.""" + +from __future__ import annotations + +import numpy as np +import pytest + +from trackers.core.reid.feature_bank import FeatureBank + + +class TestFeatureBank: + """Unit tests for ``FeatureBank`` L2 + EMA behavior.""" + + def test_first_update_normalizes_embedding(self) -> None: + # BoT-SORT normalizes the embedding before storing it. + bank = FeatureBank(alpha=0.9) + bank.update(np.array([3.0, 4.0], dtype=np.float32)) + feature = bank.feature + assert feature is not None + np.testing.assert_allclose(feature, [0.6, 0.8], atol=1e-6) + + def test_blends_on_unit_sphere(self) -> None: + # BoT-SORT: EMA on unit vectors, then L2-normalize the blend again. + bank = FeatureBank(alpha=0.75) + bank.update(np.array([1.0, 0.0], dtype=np.float32)) + bank.update(np.array([0.0, 1.0], dtype=np.float32)) + feature = bank.feature + assert feature is not None + # 0.75*[1,0] + 0.25*[0,1] = [0.75, 0.25], then / ||.|| + expected = np.array([0.75, 0.25], dtype=np.float32) + expected /= np.linalg.norm(expected) + np.testing.assert_allclose(feature, expected, atol=1e-6) + np.testing.assert_allclose(np.linalg.norm(feature), 1.0, atol=1e-6) + + def test_zero_embedding_is_accepted(self) -> None: + bank = FeatureBank() + bank.update(np.zeros(8, dtype=np.float32)) + feature = bank.feature + assert feature is not None + np.testing.assert_allclose(feature, 0.0) + + def test_non_finite_embedding_raises(self) -> None: + bank = FeatureBank() + with pytest.raises(ValueError, match="finite"): + bank.update(np.array([1.0, np.nan], dtype=np.float32)) + assert bank.feature is None + + def test_shape_change_raises(self) -> None: + bank = FeatureBank() + bank.update(np.array([1.0, 0.0], dtype=np.float32)) + before = bank.feature + assert before is not None + with pytest.raises(ValueError, match="shape"): + bank.update(np.array([1.0, 0.0, 0.0], dtype=np.float32)) + after = bank.feature + assert after is not None + np.testing.assert_allclose(after, before) diff --git a/tests/core/test_reid_thresholds.py b/tests/core/test_reid_thresholds.py new file mode 100644 index 000000000..bb1548a30 --- /dev/null +++ b/tests/core/test_reid_thresholds.py @@ -0,0 +1,266 @@ +# ------------------------------------------------------------------------ +# Trackers +# Copyright (c) 2026 Roboflow. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 [see LICENSE for details] +# ------------------------------------------------------------------------ + +"""Appearance threshold-selection sampling and metrics tests.""" + +from __future__ import annotations + +import subprocess +import sys + +import numpy as np +import pytest + +from trackers.core.reid.thresholds import ( + AppearanceDistances, + plot_appearance_distances, + plot_frame_gap_sweep, + roc_auc, + sample_appearance_distances, + sweep_frame_gap, +) + +# Two sequences, two identities, one crop of each identity in frames 1 to 6. The two +# identity vectors are orthogonal, so a same-ID pair is exactly 0 apart and a +# different-ID pair exactly 0.5, whichever frames the sampler happens to draw. +_FIRST_IDENTITY = [1.0, 0.0] +_SECOND_IDENTITY = [0.0, 1.0] +_EMBEDDINGS = np.array([_FIRST_IDENTITY, _SECOND_IDENTITY] * 12, dtype=np.float32) +_IDS = np.array([0, 1] * 12) +_FRAME_IDS = np.array([1, 1, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6] * 2) +_SEQUENCE_IDS = np.array([0] * 12 + [1] * 12) +_DATASET = (_EMBEDDINGS, _IDS, _FRAME_IDS, _SEQUENCE_IDS) + + +class TestRocAuc: + """Unit tests for the threshold-free separability metric.""" + + def test_disjoint_distributions_score_one(self) -> None: + assert roc_auc(np.array([0.0, 0.1]), np.array([0.5, 0.6])) == pytest.approx(1.0) + + def test_reversed_distributions_score_zero(self) -> None: + assert roc_auc(np.array([0.5, 0.6]), np.array([0.0, 0.1])) == pytest.approx(0.0) + + def test_identical_distributions_are_a_coin_flip(self) -> None: + """All ties, so every comparison counts as half.""" + values = np.array([0.2, 0.2, 0.2]) + assert roc_auc(values, values) == pytest.approx(0.5) + + def test_ties_count_as_half(self) -> None: + # One same-ID value below both, one exactly equal to one of them. + assert roc_auc(np.array([0.0, 0.5]), np.array([0.5, 0.9])) == pytest.approx(0.875) + + +class TestSampleAppearanceDistances: + """Unit tests for association-local pair sampling.""" + + def test_draws_the_requested_pairs_from_each_class(self) -> None: + distances = sample_appearance_distances(*_DATASET, same_id_pairs=8, different_id_pairs=10) + + assert len(distances.same_id) == 8 + assert len(distances.different_id) == 10 + np.testing.assert_allclose(distances.same_id, 0.0, atol=1e-6) + np.testing.assert_allclose(distances.different_id, 0.5, atol=1e-6) + assert distances.roc_auc == pytest.approx(1.0) + + def test_zero_frame_gap_is_rejected(self) -> None: + """A gap of 0 would let a crop pair with itself and fake a spike at distance 0.""" + with pytest.raises(ValueError, match="invalid frame gap band"): + sample_appearance_distances(*_DATASET, minimum_frame_gap=0) + + def test_empty_dataset_is_rejected(self) -> None: + embeddings = np.empty((0, 2), dtype=np.float32) + empty_labels = np.array([], dtype=int) + + with pytest.raises(ValueError, match="at least one row"): + sample_appearance_distances(embeddings, empty_labels, empty_labels, empty_labels) + + def test_string_sequence_and_identity_labels_are_supported(self) -> None: + embeddings = np.array([_FIRST_IDENTITY, _SECOND_IDENTITY] * 2, dtype=np.float32) + + distances = sample_appearance_distances( + embeddings, + np.array(["person-a", "person-b"] * 2), + np.array([1, 1, 2, 2]), + np.array(["camera-a"] * 4), + same_id_pairs=4, + different_id_pairs=4, + maximum_frame_gap=1, + ) + + np.testing.assert_allclose(distances.same_id, 0.0, atol=1e-6) + np.testing.assert_allclose(distances.different_id, 0.5, atol=1e-6) + + def test_distinct_non_integer_identity_labels_are_not_merged(self) -> None: + embeddings = np.array([_FIRST_IDENTITY, _SECOND_IDENTITY] * 2, dtype=np.float32) + + distances = sample_appearance_distances( + embeddings, + np.array([1.2, 1.8] * 2), + np.array([1, 1, 2, 2]), + np.zeros(4), + same_id_pairs=32, + different_id_pairs=4, + maximum_frame_gap=1, + ) + + np.testing.assert_allclose(distances.same_id, 0.0, atol=1e-6) + np.testing.assert_allclose(distances.different_id, 0.5, atol=1e-6) + + def test_labels_that_never_equal_themselves_are_still_sampleable(self) -> None: + """A NaN label is its own identity, so sampling must not look a track up by its label.""" + embeddings = np.array([_FIRST_IDENTITY, _SECOND_IDENTITY] * 2, dtype=np.float32) + + distances = sample_appearance_distances( + embeddings, + np.array([np.nan, 1.0] * 2), + np.array([1, 1, 2, 2]), + np.zeros(4), + same_id_pairs=4, + different_id_pairs=4, + maximum_frame_gap=1, + ) + + # Only the 1.0 track pairs with itself; the two NaN crops are distinct identities. + np.testing.assert_allclose(distances.same_id, 0.0, atol=1e-6) + + def test_every_sequence_gets_an_equal_quota(self) -> None: + """The per-sequence split is what stops one crowded sequence deciding the answer. + + Both identities in the second sequence are given the same embedding, so a different-ID pair drawn there measures + 0 while one from the first sequence measures 0.5. Asking for two pairs must produce one of each. + """ + embeddings = _EMBEDDINGS.copy() + embeddings[12:] = _FIRST_IDENTITY + + distances = sample_appearance_distances( + embeddings, + _IDS, + _FRAME_IDS, + _SEQUENCE_IDS, + same_id_pairs=2, + different_id_pairs=2, + ) + + np.testing.assert_allclose(sorted(distances.different_id), [0.0, 0.5], atol=1e-6) + + def test_pair_quotas_are_redistributed_over_sequences_valid_for_the_gap_band(self) -> None: + embeddings = np.array( + [_FIRST_IDENTITY, _SECOND_IDENTITY, _FIRST_IDENTITY, _SECOND_IDENTITY] * 2, + dtype=np.float32, + ) + + distances = sample_appearance_distances( + embeddings, + np.array([0, 1, 0, 1] * 2), + np.array([1, 1, 100, 100, 1, 1, 2, 2]), + np.array(["unpairable"] * 4 + ["pairable"] * 4), + same_id_pairs=8, + different_id_pairs=8, + maximum_frame_gap=1, + ) + + assert len(distances.same_id) == 8 + assert len(distances.different_id) == 8 + + def test_same_id_sampling_is_uniform_over_identities_valid_for_the_gap_band(self) -> None: + isolated_frames = np.arange(100, 1100, 10) + embeddings = np.array( + [_FIRST_IDENTITY, _FIRST_IDENTITY, _SECOND_IDENTITY] + + [_FIRST_IDENTITY, _SECOND_IDENTITY] + + [_FIRST_IDENTITY] * len(isolated_frames), + dtype=np.float32, + ) + ids = np.array(["dense", "dense", "single"] + ["sparse"] * (2 + len(isolated_frames))) + frame_ids = np.concatenate(([1, 2, 1, 10, 11], isolated_frames)) + + distances = sample_appearance_distances( + embeddings, + ids, + frame_ids, + np.zeros(len(ids)), + same_id_pairs=2000, + different_id_pairs=4, + maximum_frame_gap=1, + seed=7, + ) + + assert len(distances.same_id) == 2000 + assert np.mean(distances.same_id) == pytest.approx(0.25, abs=0.03) + + +class TestAppearanceDistances: + """Unit tests for what a sampled band reports about itself.""" + + @pytest.mark.parametrize( + ("threshold", "expected"), + [ + (0.05, (0.0, 0.0)), + # On the boundary: the tracker keeps appearance at exactly the threshold. + (0.1, (0.5, 0.0)), + (0.15, (0.5, 0.0)), + (0.45, (1.0, 0.5)), + (1.0, (1.0, 1.0)), + ], + ) + def test_rates_at_counts_both_classes(self, threshold: float, expected: tuple[float, float]) -> None: + distances = AppearanceDistances( + same_id=np.array([0.1, 0.3]), + different_id=np.array([0.4, 0.6]), + minimum_frame_gap=1, + maximum_frame_gap=1, + ) + assert distances.rates_at(threshold) == expected + + @pytest.mark.parametrize(("band", "expected"), [((1, 1), "1"), ((6, 15), "6-15")]) + def test_label_describes_the_gap_band(self, band: tuple[int, int], expected: str) -> None: + distances = AppearanceDistances(np.array([0.1]), np.array([0.5]), *band) + assert distances.label == expected + + +def test_sweep_skips_bands_the_data_cannot_fill() -> None: + """Six frames per sequence hold no gap wider than five, so the later bands drop out.""" + sweep = sweep_frame_gap(*_DATASET, pairs_per_class=4) + + assert [band.label for band in sweep] == ["1", "2-5"] + + +def test_sweep_rejects_mismatched_dataset_lengths() -> None: + with pytest.raises(ValueError, match="equal length"): + sweep_frame_gap(_EMBEDDINGS[:-1], _IDS, _FRAME_IDS, _SEQUENCE_IDS) + + +def test_both_plots_build() -> None: + """Both figures build, and the histogram displays its complete data range.""" + distances = AppearanceDistances( + same_id=np.array([0.0, 0.2]), + different_id=np.array([0.5, 1.0]), + minimum_frame_gap=1, + maximum_frame_gap=1, + ) + sweep = sweep_frame_gap(*_DATASET, pairs_per_class=4) + + histogram = plot_appearance_distances(distances, thresholds={0.2: "selected"}) + lower_bound, upper_bound = histogram.axes[0].get_xlim() + + assert lower_bound <= min(np.min(distances.same_id), np.min(distances.different_id)) + assert upper_bound >= max(np.max(distances.same_id), np.max(distances.different_id)) + assert plot_frame_gap_sweep(sweep) is not None + + +def test_importing_the_tracker_does_not_pull_matplotlib() -> None: + """Plotting is opt-in, so the tracking path must not pay for matplotlib.""" + result = subprocess.run( + [ + sys.executable, + "-c", + "import sys; import trackers.core.botsort.tracker; assert 'matplotlib' not in sys.modules", + ], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr diff --git a/uv.lock b/uv.lock index 4e783f374..704c96066 100644 --- a/uv.lock +++ b/uv.lock @@ -250,13 +250,26 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/37/fb6973edeb700f6e3d6ff222400602ab1830446c25c7b4676d8de93e65b8/backrefs-5.8-py39-none-any.whl", hash = "sha256:a66851e4533fb5b371aa0628e1fee1af05135616b86140c9d787a2ffdf4b8fdc", size = 380336, upload-time = "2025-02-25T16:53:29.858Z" }, ] +[[package]] +name = "beautifulsoup4" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "soupsieve" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/65/318323f98dbee45d42dff61d8f047181bc6f2268a9068cfad035a46be5af/beautifulsoup4-4.15.0.tar.gz", hash = "sha256:288e3ca7d54b06f2ac191970bc275c1939cb46d450b255bf6718b04aa37ab4f7", size = 632571, upload-time = "2026-06-07T16:44:20.453Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/c6/92fcd42f1ba33e1184263f25bfabf3d27c383410470f169e4b8163bf9c17/beautifulsoup4-4.15.0-py3-none-any.whl", hash = "sha256:d6f88de62e1d4e38ecb1077eb9724cd0eff29d2a08ca16a401e9b9e93f117cf9", size = 109924, upload-time = "2026-06-07T16:44:21.566Z" }, +] + [[package]] name = "bitsandbytes" version = "0.47.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy" }, - { name = "torch" }, + { name = "numpy", marker = "sys_platform != 'darwin'" }, + { name = "torch", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/aa/eb/477d6b5602f469c7305fd43eec71d890c39909f615c1d7138f6e7d226eff/bitsandbytes-0.47.0-py3-none-manylinux_2_24_aarch64.whl", hash = "sha256:2f805b76891a596025e9e13318b675d08481b9ee650d65e5d2f9d844084c6521", size = 30004641, upload-time = "2025-08-11T18:51:20.524Z" }, @@ -333,7 +346,7 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x' and sys_platform == 'darwin'", ] dependencies = [ - { name = "pycparser" }, + { name = "pycparser", marker = "python_full_version < '3.14' and sys_platform == 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/97/c783634659c2920c3fc70419e3af40972dbaf758daa229a7d6ea6135c90d/cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824", size = 516621, upload-time = "2024-09-04T20:45:21.852Z" } wheels = [ @@ -371,7 +384,7 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x' and sys_platform != 'darwin'", ] dependencies = [ - { name = "pycparser", marker = "implementation_name != 'PyPy'" }, + { name = "pycparser", marker = "(python_full_version >= '3.14' and implementation_name != 'PyPy') or (implementation_name != 'PyPy' and sys_platform != 'darwin')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/eb/56/b1ba7935a17738ae8453301356628e8147c79dbb825bcbc73dc7401f9846/cffi-2.0.0.tar.gz", hash = "sha256:44d1b5909021139fe36001ae048dbdde8214afa20200eda0f64c068cac5d5529", size = 523588, upload-time = "2025-09-08T23:24:04.541Z" } wheels = [ @@ -623,8 +636,8 @@ name = "cryptography" version = "50.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_python_implementation != 'PyPy'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "cffi", version = "2.0.0", source = { registry = "https://pypi.org/simple" }, marker = "platform_machine != 's390x' and platform_python_implementation != 'PyPy' and sys_platform != 'darwin'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11' and platform_machine != 's390x' and sys_platform != 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/de/41/6cbdcf9142d00fe82836fbb51e503e58088575cf7a0fe1dbff6695bf0840/cryptography-50.0.0.tar.gz", hash = "sha256:eeac2acb5a20ed25e0ad6d1df9891a520b78b404266b6d11778f25d5d691a6c9", size = 880201, upload-time = "2026-07-31T14:25:10.11Z" } wheels = [ @@ -685,7 +698,7 @@ name = "cuda-bindings" version = "13.2.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder" }, + { name = "cuda-pathfinder", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/1a/fe/7351d7e586a8b4c9f89731bfe4cf0148223e8f9903ff09571f78b3fb0682/cuda_bindings-13.2.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:08b395f79cb89ce0cd8effff07c4a1e20101b873c256a1aeb286e8fd7bd0f556", size = 5744254, upload-time = "2026-03-11T00:12:29.798Z" }, @@ -720,43 +733,43 @@ wheels = [ [package.optional-dependencies] cublas = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cudart = [ - { name = "nvidia-cuda-runtime", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-runtime", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cufft = [ - { name = "nvidia-cufft", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufft", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cufile = [ - { name = "nvidia-cufile", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cufile", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux')" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-cupti", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] curand = [ - { name = "nvidia-curand", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-curand", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cusolver = [ - { name = "nvidia-cublas", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusolver", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cublas", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusolver", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cusparse", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvjitlink", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-cuda-nvrtc", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "platform_machine == 'aarch64' or platform_machine == 'x86_64'" }, + { name = "nvidia-nvtx", marker = "(platform_machine == 'aarch64' and sys_platform == 'linux') or (platform_machine == 'x86_64' and sys_platform == 'linux') or (platform_machine == 'AMD64' and sys_platform == 'win32')" }, ] [[package]] @@ -837,7 +850,7 @@ name = "exceptiongroup" version = "1.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0b/9f/a65090624ecf468cdca03533906e7c69ed7588582240cfe7cc9e770b50eb/exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88", size = 29749, upload-time = "2025-05-10T17:42:51.123Z" } wheels = [ @@ -931,6 +944,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ab/6e/81d47999aebc1b155f81eca4477a616a70f238a2549848c38983f3c22a82/ftfy-6.3.1-py3-none-any.whl", hash = "sha256:7c70eb532015cd2f9adb53f101fb6c7945988d023a085d127d1573dc49dd0083", size = 44821, upload-time = "2024-10-26T00:50:33.425Z" }, ] +[[package]] +name = "gdown" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beautifulsoup4" }, + { name = "filelock" }, + { name = "requests", extra = ["socks"] }, + { name = "tqdm" }, + { name = "typing-extensions", marker = "python_full_version < '3.12'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6a/b5/a45f62f20664031bf74a6aeb6f8d8cd5910e411bf90d756bd6b09bdc6c35/gdown-6.1.0.tar.gz", hash = "sha256:361c6e04c6ca335df50b9d71f40bcfe9ab70fb26a1b0e890a427267781389553", size = 269670, upload-time = "2026-05-30T11:56:21.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/56/a99f0f159cce5b26d267317d436afee184f45fc7911938757d7cbbd2d10c/gdown-6.1.0-py3-none-any.whl", hash = "sha256:38a36a94275b8272f684db469bbd73b4d1f64cbbc1751bcb993a1b2be8f013c8", size = 19216, upload-time = "2026-05-30T11:56:20.016Z" }, +] + [[package]] name = "ghp-import" version = "2.1.0" @@ -1239,7 +1268,7 @@ name = "importlib-metadata" version = "8.7.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "zipp" }, + { name = "zipp", marker = "(python_full_version < '3.12' and platform_machine != 's390x') or (python_full_version < '3.11' and platform_machine == 's390x')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } wheels = [ @@ -1325,7 +1354,7 @@ name = "jaraco-classes" version = "3.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools" }, + { name = "more-itertools", marker = "platform_machine != 's390x'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } wheels = [ @@ -1337,7 +1366,7 @@ name = "jaraco-context" version = "6.0.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "backports-tarfile", marker = "python_full_version < '3.12'" }, + { name = "backports-tarfile", marker = "python_full_version < '3.12' and platform_machine != 's390x'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/df/ad/f3777b81bf0b6e7bc7514a1656d3e637b2e8e15fab2ce3235730b3e7a4e6/jaraco_context-6.0.1.tar.gz", hash = "sha256:9bae4ea555cf0b14938dc0aee7c9f32ed303aa20a3b73e7dc80111628792d1b3", size = 13912, upload-time = "2024-08-20T03:39:27.358Z" } wheels = [ @@ -1349,7 +1378,7 @@ name = "jaraco-functools" version = "4.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "more-itertools" }, + { name = "more-itertools", marker = "platform_machine != 's390x'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ab/23/9894b3df5d0a6eb44611c36aec777823fc2e07740dabbd0b810e19594013/jaraco_functools-4.1.0.tar.gz", hash = "sha256:70f7e0e2ae076498e212562325e805204fc092d7b4c17e0e86c959e249701a9d", size = 19159, upload-time = "2024-09-27T19:47:09.122Z" } wheels = [ @@ -1400,13 +1429,13 @@ name = "keyring" version = "25.6.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.12'" }, - { name = "jaraco-classes" }, - { name = "jaraco-context" }, - { name = "jaraco-functools" }, - { name = "jeepney", marker = "sys_platform == 'linux'" }, - { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, - { name = "secretstorage", marker = "sys_platform == 'linux'" }, + { name = "importlib-metadata", marker = "python_full_version < '3.12' and platform_machine != 's390x'" }, + { name = "jaraco-classes", marker = "platform_machine != 's390x'" }, + { name = "jaraco-context", marker = "platform_machine != 's390x'" }, + { name = "jaraco-functools", marker = "platform_machine != 's390x'" }, + { name = "jeepney", marker = "platform_machine != 's390x' and sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "platform_machine != 's390x' and sys_platform == 'win32'" }, + { name = "secretstorage", marker = "platform_machine != 's390x' and sys_platform == 'linux'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/70/09/d904a6e96f76ff214be59e7aa6ef7190008f52a0ab6689760a98de0bf37d/keyring-25.6.0.tar.gz", hash = "sha256:0b39998aa941431eb3d9b0d4b2460bc773b9df6fed7621c2dfb291a7e0187a66", size = 62750, upload-time = "2024-12-25T15:26:45.782Z" } wheels = [ @@ -2124,7 +2153,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc" }, + { name = "nvidia-cuda-nvrtc", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -2163,7 +2192,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, + { name = "nvidia-cublas", marker = "sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -2175,7 +2204,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -2205,9 +2234,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, + { name = "nvidia-cublas", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, + { name = "nvidia-cusparse", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -2219,7 +2248,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink" }, + { name = "nvidia-nvjitlink", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -2935,6 +2964,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bd/24/12818598c362d7f300f18e74db45963dbcb85150324092410c8b49405e42/pyproject_hooks-1.2.0-py3-none-any.whl", hash = "sha256:9e5c6bfa8dcc30091c74b0cf803c81fdd29d94f01992a7707bc97babb1141913", size = 10216, upload-time = "2024-09-29T09:24:11.978Z" }, ] +[[package]] +name = "pysocks" +version = "1.7.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bd/11/293dd436aea955d45fc4e8a35b6ae7270f5b8e00b53cf6c024c83b657a11/PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0", size = 284429, upload-time = "2019-09-20T02:07:35.714Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, +] + [[package]] name = "pytest" version = "9.1.1" @@ -3409,6 +3447,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/95/e4/a3b9480c78cf8ee86626cb06f8d931d74d775897d44201ccb813097ae697/regex-2026.1.15-cp314-cp314t-win_arm64.whl", hash = "sha256:ca89c5e596fc05b015f27561b3793dc2fa0917ea0d7507eebb448efd35274a70", size = 274837, upload-time = "2026-01-14T23:17:23.146Z" }, ] +[[package]] +name = "reid" +version = "0.1.0.dev0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "gdown" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "opencv-python" }, + { name = "pillow" }, + { name = "safetensors" }, + { name = "supervision" }, + { name = "timm" }, + { name = "torch" }, + { name = "torchvision" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/08/17/9cda7631bf369ed3933cba949356be1991a2871d2cd90366199ace3d1792/reid-0.1.0.dev0.tar.gz", hash = "sha256:56b3319c7b01e3224d1858209fa079d923ca5a6e8a76cc52feb28f0fc67cc6d9", size = 44919, upload-time = "2026-07-29T15:33:05.48Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c4/b1/d1109a065a2796e9358df341e6d63ca9ce022387fc13912a561d859df94b/reid-0.1.0.dev0-py3-none-any.whl", hash = "sha256:4f850116264892ea55f7a7978d3f7f7a26e8449f42d93dead56caa719c458b30", size = 41937, upload-time = "2026-07-29T15:33:04.192Z" }, +] + [[package]] name = "requests" version = "2.34.2" @@ -3424,6 +3483,11 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, ] +[package.optional-dependencies] +socks = [ + { name = "pysocks" }, +] + [[package]] name = "requests-file" version = "3.0.1" @@ -3685,8 +3749,8 @@ name = "secretstorage" version = "3.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cryptography" }, - { name = "jeepney" }, + { name = "cryptography", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, + { name = "jeepney", marker = "platform_machine != 's390x' and sys_platform != 'darwin'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/53/a4/f48c9d79cb507ed1373477dbceaba7401fd8a23af63b837fa61f1dcd3691/SecretStorage-3.3.3.tar.gz", hash = "sha256:2403533ef369eca6d2ba81718576c5e0f564d5cca1b58f73a8b23e7d4eeebd77", size = 19739, upload-time = "2022-08-13T16:22:46.976Z" } wheels = [ @@ -3941,6 +4005,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/d4/59e74daffcb57a07668852eeeb6035af9f32cbfd7a1d2511f17d2fe6a738/smmap-5.0.3-py3-none-any.whl", hash = "sha256:c106e05d5a61449cf6ba9a1e650227ecfb141590d2a98412103ff35d89fc7b2f", size = 24390, upload-time = "2026-03-09T03:43:24.361Z" }, ] +[[package]] +name = "soupsieve" +version = "2.9.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/69/99/a6ca3beb3ccacb41fb3321d8a60e5566f9e6467601ef8eba6a17e1b89778/soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74", size = 122445, upload-time = "2026-08-07T00:57:24.801Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/dc/ad025c1ee131eba60c69f4dd5779b18fcf1e6b21a343e2162a84d5d133c7/soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823", size = 37370, upload-time = "2026-08-07T00:57:23.524Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.49" @@ -4047,7 +4120,7 @@ resolution-markers = [ "python_full_version < '3.11' and platform_machine == 's390x' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/44/d0/18fed0fc0916578a4463f775b0fbd9c5fed2392152d039df2fb533bfdd5d/tifffile-2025.5.10.tar.gz", hash = "sha256:018335d34283aa3fd8c263bae5c3c2b661ebc45548fde31504016fcae7bf1103", size = 365290, upload-time = "2025-05-10T19:22:34.386Z" } wheels = [ @@ -4081,7 +4154,7 @@ resolution-markers = [ "python_full_version == '3.11.*' and platform_machine == 's390x' and sys_platform != 'darwin'", ] dependencies = [ - { name = "numpy" }, + { name = "numpy", marker = "python_full_version >= '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/94/32/38498d2a1a5d70f33f6c3909bbad48557c9a54b0e33a9307ff06b6d416ba/tifffile-2026.1.28.tar.gz", hash = "sha256:537ae6466a8bb555c336108bb1878d8319d52c9c738041d3349454dea6956e1c", size = 374675, upload-time = "2026-01-29T05:17:24.992Z" } wheels = [ @@ -4321,6 +4394,10 @@ mask = [ { name = "torch" }, { name = "torchvision" }, ] +reid = [ + { name = "matplotlib" }, + { name = "reid" }, +] tune = [ { name = "optuna" }, ] @@ -4357,10 +4434,12 @@ mypy-types = [ requires-dist = [ { name = "inference-models", marker = "extra == 'detection'", specifier = ">=0.19.0" }, { name = "jsonargparse", specifier = ">=4.48.0,<5" }, + { name = "matplotlib", marker = "extra == 'reid'", specifier = ">=3.7.0" }, { name = "numpy", specifier = ">=2.0.2" }, { name = "opencv-python", specifier = ">=4.8.0" }, { name = "optuna", marker = "extra == 'tune'", specifier = ">=3.0.0" }, { name = "pydeprecate", specifier = ">=0.8.0" }, + { name = "reid", marker = "extra == 'reid'", specifier = ">=0.1.0.dev0,<0.2" }, { name = "requests", specifier = ">=2.28.0" }, { name = "rf-cutie", extras = ["inference"], marker = "extra == 'mask'", specifier = ">=1.0.0" }, { name = "rf-segment-anything", marker = "extra == 'mask'", specifier = ">=1.0" }, @@ -4370,7 +4449,7 @@ requires-dist = [ { name = "torch", marker = "extra == 'mask'" }, { name = "torchvision", marker = "extra == 'mask'" }, ] -provides-extras = ["detection", "tune", "mask"] +provides-extras = ["detection", "tune", "reid", "mask"] [package.metadata.requires-dev] build = [