From 40bcea2b5a208084ce0efe244c6898c62124852e Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 20 Jul 2026 15:06:56 -0700 Subject: [PATCH 1/7] fix: render session detail captures from EXIF-stripped large thumbnails Portrait photos are typically stored as landscape pixels plus an EXIF Orientation tag. Browsers force-apply that rotation to cross-origin images, while detections, image dimensions, and crops all live in the raw (unrotated) pixel space - so the session detail view drew bounding boxes 90 degrees out of place on portrait captures. This regressed in PR #1339, which switched the view from the medium thumbnail to the original image URL for zoom quality. Thumbnails are re-encoded without EXIF metadata, so they always render in the same pixel space as the boxes. Add a "large" (2560px) thumbnail size for zoom quality and point the session detail view at it, falling back to the original URL when thumbnails are unavailable. Co-Authored-By: Claude --- ami/main/tests.py | 45 +++++++++++++++++++ config/settings/base.py | 9 +++- ui/src/data-services/models/capture.ts | 12 +++++ .../pages/session-details/session-details.tsx | 2 +- 4 files changed, 65 insertions(+), 3 deletions(-) diff --git a/ami/main/tests.py b/ami/main/tests.py index a0d0253a1..2fe2122ba 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -262,6 +262,51 @@ def test_thumbnail_new_with_size(self): self.assertEqual(thumb.height, 768) self.assertEqual(response.headers["Location"], f"/media/{thumb.path}") + def test_thumbnail_new_with_large_size(self): + response = self.client.get(f"/api/v2/captures/thumbnails/{self.first_capture.pk}/?label=large") + self.assertEqual(response.status_code, 302) + thumb = self.first_capture.thumbnails.get(label="large") + # ``width`` stores the requested spec width for the regen gate, not the + # encoder output. The test fixture is 1024px wide and PIL's thumbnail() + # never upscales, so the stored file keeps its original dimensions. + self.assertEqual(thumb.width, 2560) + self.assertEqual(thumb.height, 768) + self.assertEqual(response.headers["Location"], f"/media/{thumb.path}") + + def test_thumbnail_strips_exif_orientation(self): + """Thumbnails must stay in the source file's raw pixel space so detection + bounding boxes (stored in raw coordinates) align when overlaid in the UI. + + A photo shot in portrait is typically stored as landscape pixels plus an + EXIF Orientation tag, and browsers force-apply that rotation to + cross-origin images. The thumbnail pipeline must neither rotate the + pixels nor propagate the tag to the output file, otherwise the session + detail view renders boxes 90° out of place (the pre-thumbnail behavior). + """ + from django.core.files.storage import default_storage + + from ami.utils import s3 + + ORIENTATION_TAG = 274 # EXIF tag id for Orientation + source = Image.new("RGB", (320, 240), (10, 120, 30)) + exif = source.getexif() + exif[ORIENTATION_TAG] = 6 # stored landscape, rotate 90° CW to view + buffer = BytesIO() + source.save(buffer, format="JPEG", exif=exif) + + assert self.deployment.data_source is not None + s3.write_file(self.deployment.data_source.config, self.first_capture.path, buffer.getvalue()) + + thumb = self.first_capture.find_or_generate_thumbnail_for_label("large") + + with default_storage.open(thumb.path) as f: + output = Image.open(f) + output.load() + + # Pixels stay in raw (landscape) space and the orientation tag is gone. + self.assertEqual(output.size, (320, 240)) + self.assertIsNone(output.getexif().get(ORIENTATION_TAG)) + def test_thumbnail_blank_path_row_regenerates(self): """A row with an empty ``path`` (failed or interrupted generation) must trigger regeneration, not redirect to the storage root. diff --git a/config/settings/base.py b/config/settings/base.py index d90899e91..5fbfcdcd1 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -589,5 +589,10 @@ def _celery_result_backend_url(redis_url): JOB_LOG_PERSIST_ENABLED = env.bool("JOB_LOG_PERSIST_ENABLED", default=True) # type: ignore[no-untyped-call] -# Sizes for Source Image Thumbnails -THUMBNAILS = {"STORAGE_PREFIX": "thumbnails/", "SIZES": {"small": {"width": 240}, "medium": {"width": 1024}}} +# Sizes for Source Image Thumbnails. The "large" size backs the zoomable session +# detail view; thumbnails are re-encoded without EXIF metadata, so they render in +# the same pixel space as detection bounding boxes (EXIF-rotated originals do not). +THUMBNAILS = { + "STORAGE_PREFIX": "thumbnails/", + "SIZES": {"small": {"width": 240}, "medium": {"width": 1024}, "large": {"width": 2560}}, +} diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index 4d6d7b303..538eaca91 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -183,6 +183,18 @@ export class Capture { return this._capture.url } + // Preferred source for the zoomable session detail view: large thumbnails are + // re-encoded without EXIF metadata, so they render in the same pixel space as + // detection bounding boxes. The raw original may carry an EXIF rotation that + // browsers force-apply to cross-origin images, misplacing the boxes. + get thumbnailLarge(): string { + if (this._capture.thumbnails?.large) { + return this._capture.thumbnails.large + } + + return this._capture.url + } + get src(): string { return this._capture.url } diff --git a/ui/src/pages/session-details/session-details.tsx b/ui/src/pages/session-details/session-details.tsx index a55dd1ca2..09d51d2bf 100644 --- a/ui/src/pages/session-details/session-details.tsx +++ b/ui/src/pages/session-details/session-details.tsx @@ -169,7 +169,7 @@ const Content = ({ session }: { session: SessionDetails }) => { detections={activeCapture?.detections ?? []} height={activeCapture?.height ?? session.firstCapture.height} showDetections={settings.showDetections} - src={activeCapture?.url} + src={activeCapture?.thumbnailLarge} transformRef={transformRef} width={activeCapture?.width ?? session.firstCapture.width} /> From 3405963f6d774681fc9913f9944bbcdeffefafe6 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 20 Jul 2026 16:42:43 -0700 Subject: [PATCH 2/7] fix: skip detection boxes when capture dimensions are unknown Box coordinates are stored in the original image's pixel space, and the session detail view now renders a downscaled thumbnail, so the previous fallback to the rendered image's natural size would draw boxes at the wrong scale for captures with no stored width/height. Skip drawing instead. Also document in the thumbnail generator that EXIF must not propagate to the output, since box alignment depends on it. Co-Authored-By: Claude --- ami/main/models.py | 5 +++++ .../pages/session-details/capture/capture.tsx | 20 ++++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index 3662b4107..e3d8384ca 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2519,6 +2519,11 @@ def find_or_generate_thumbnail_for_label(self, label): img.thumbnail(new_size) buffer = BytesIO() + # EXIF must NOT be copied to the output (no ``exif=`` argument): the UI + # overlays detection boxes, which are stored in raw pixel coordinates, on + # these thumbnails. An EXIF Orientation tag would make browsers rotate the + # rendered pixels out of that coordinate space. Pinned by + # test_thumbnail_strips_exif_orientation. img.save(buffer, format="JPEG", progressive=True, optimize=True, quality=82) contents = buffer.getvalue() file_size = len(contents) diff --git a/ui/src/pages/session-details/capture/capture.tsx b/ui/src/pages/session-details/capture/capture.tsx index 6e2848ee7..7ef9a1a8a 100644 --- a/ui/src/pages/session-details/capture/capture.tsx +++ b/ui/src/pages/session-details/capture/capture.tsx @@ -93,23 +93,25 @@ export const Capture = ({ const boxWidth = boxRight - boxLeft const boxHeight = boxBottom - boxTop - const _width = width ?? naturalSize?.width - const _height = height ?? naturalSize?.height - - if (!_width || !_height) { + // Box coordinates are in the original image's pixel space, so they can + // only be scaled against the capture's stored dimensions. The rendered + // image may be a downscaled thumbnail, so its natural size is not a + // valid substitute — without stored dimensions, skip drawing boxes + // rather than drawing them at the wrong scale. + if (!width || !height) { return result } result[detection.id] = { - width: `${(boxWidth / _width) * 100}%`, - height: `${(boxHeight / _height) * 100}%`, - top: `${(boxTop / _height) * 100}%`, - left: `${(boxLeft / _width) * 100}%`, + width: `${(boxWidth / width) * 100}%`, + height: `${(boxHeight / height) * 100}%`, + top: `${(boxTop / height) * 100}%`, + left: `${(boxLeft / width) * 100}%`, } return result }, {}), - [width, height, naturalSize, detections] + [width, height, detections] ) const ratio = useMemo(() => { From 1a15510a077c94f34042d1d868433fa817cbd8c2 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 20 Jul 2026 16:44:12 -0700 Subject: [PATCH 3/7] refactor: trim comments to the load-bearing facts Co-Authored-By: Claude --- ami/main/models.py | 7 ++----- ami/main/tests.py | 9 ++------- config/settings/base.py | 4 +--- ui/src/data-services/models/capture.ts | 6 ++---- ui/src/pages/session-details/capture/capture.tsx | 7 ++----- 5 files changed, 9 insertions(+), 24 deletions(-) diff --git a/ami/main/models.py b/ami/main/models.py index e3d8384ca..370f7e323 100644 --- a/ami/main/models.py +++ b/ami/main/models.py @@ -2519,11 +2519,8 @@ def find_or_generate_thumbnail_for_label(self, label): img.thumbnail(new_size) buffer = BytesIO() - # EXIF must NOT be copied to the output (no ``exif=`` argument): the UI - # overlays detection boxes, which are stored in raw pixel coordinates, on - # these thumbnails. An EXIF Orientation tag would make browsers rotate the - # rendered pixels out of that coordinate space. Pinned by - # test_thumbnail_strips_exif_orientation. + # No ``exif=`` argument: detection boxes are overlaid on thumbnails in raw + # pixel coordinates, so the EXIF Orientation tag must not propagate. img.save(buffer, format="JPEG", progressive=True, optimize=True, quality=82) contents = buffer.getvalue() file_size = len(contents) diff --git a/ami/main/tests.py b/ami/main/tests.py index 2fe2122ba..996db345a 100644 --- a/ami/main/tests.py +++ b/ami/main/tests.py @@ -275,13 +275,8 @@ def test_thumbnail_new_with_large_size(self): def test_thumbnail_strips_exif_orientation(self): """Thumbnails must stay in the source file's raw pixel space so detection - bounding boxes (stored in raw coordinates) align when overlaid in the UI. - - A photo shot in portrait is typically stored as landscape pixels plus an - EXIF Orientation tag, and browsers force-apply that rotation to - cross-origin images. The thumbnail pipeline must neither rotate the - pixels nor propagate the tag to the output file, otherwise the session - detail view renders boxes 90° out of place (the pre-thumbnail behavior). + boxes (stored in raw coordinates) align when overlaid in the UI: neither + rotate the pixels nor propagate the EXIF Orientation tag to the output. """ from django.core.files.storage import default_storage diff --git a/config/settings/base.py b/config/settings/base.py index 5fbfcdcd1..01883d0eb 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -589,9 +589,7 @@ def _celery_result_backend_url(redis_url): JOB_LOG_PERSIST_ENABLED = env.bool("JOB_LOG_PERSIST_ENABLED", default=True) # type: ignore[no-untyped-call] -# Sizes for Source Image Thumbnails. The "large" size backs the zoomable session -# detail view; thumbnails are re-encoded without EXIF metadata, so they render in -# the same pixel space as detection bounding boxes (EXIF-rotated originals do not). +# Sizes for Source Image Thumbnails ("large" backs the zoomable session detail view) THUMBNAILS = { "STORAGE_PREFIX": "thumbnails/", "SIZES": {"small": {"width": 240}, "medium": {"width": 1024}, "large": {"width": 2560}}, diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index 538eaca91..c87189957 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -183,10 +183,8 @@ export class Capture { return this._capture.url } - // Preferred source for the zoomable session detail view: large thumbnails are - // re-encoded without EXIF metadata, so they render in the same pixel space as - // detection bounding boxes. The raw original may carry an EXIF rotation that - // browsers force-apply to cross-origin images, misplacing the boxes. + // EXIF-free large thumbnail for the zoomable session detail view; detection + // boxes align with its pixels, unlike the EXIF-rotated original file. get thumbnailLarge(): string { if (this._capture.thumbnails?.large) { return this._capture.thumbnails.large diff --git a/ui/src/pages/session-details/capture/capture.tsx b/ui/src/pages/session-details/capture/capture.tsx index 7ef9a1a8a..a4dc631bb 100644 --- a/ui/src/pages/session-details/capture/capture.tsx +++ b/ui/src/pages/session-details/capture/capture.tsx @@ -93,11 +93,8 @@ export const Capture = ({ const boxWidth = boxRight - boxLeft const boxHeight = boxBottom - boxTop - // Box coordinates are in the original image's pixel space, so they can - // only be scaled against the capture's stored dimensions. The rendered - // image may be a downscaled thumbnail, so its natural size is not a - // valid substitute — without stored dimensions, skip drawing boxes - // rather than drawing them at the wrong scale. + // Boxes are in the original image's pixel space and the rendered image + // may be a downscaled thumbnail, so only stored dimensions can scale them. if (!width || !height) { return result } From 33dfac9b273808b2baf8302e1d4943dcabc724d4 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Mon, 20 Jul 2026 20:59:57 -0700 Subject: [PATCH 4/7] feat: progressive zoom-resolution loading in the session detail view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Load the session detail capture through a resolution ladder (medium 1024 → large 2560 → original) driven by zoom demand: container width × devicePixelRatio × zoom scale. The initial tier is picked from the display (medium on a 1x screen, large on retina), and zooming in promotes to higher tiers with a debounced fetch, a blurry-first crossfade, and a corner loading pill. A zoom readout shows the current level relative to the original's pixels (100% = one image pixel per device pixel) and maxScale is raised so large originals can be inspected past 100%. The original tier is guarded against browser-applied EXIF rotation: if its rendered dimensions are the stored dimensions transposed, the upgrade is refused and the view stays on the EXIF-free large thumbnail, keeping detection boxes aligned (the bug PR #1374 fixed). Closes #1373 Co-Authored-By: Claude --- .../capture/capture-tiers.test.ts | 146 +++++++++++++ .../session-details/capture/capture-tiers.ts | 119 +++++++++++ .../pages/session-details/capture/capture.tsx | 156 +++++++++++--- .../capture/useCaptureTiers.ts | 196 ++++++++++++++++++ .../pages/session-details/session-details.tsx | 10 +- ui/src/utils/language.ts | 2 + 6 files changed, 597 insertions(+), 32 deletions(-) create mode 100644 ui/src/pages/session-details/capture/capture-tiers.test.ts create mode 100644 ui/src/pages/session-details/capture/capture-tiers.ts create mode 100644 ui/src/pages/session-details/capture/useCaptureTiers.ts diff --git a/ui/src/pages/session-details/capture/capture-tiers.test.ts b/ui/src/pages/session-details/capture/capture-tiers.test.ts new file mode 100644 index 000000000..ae4e24f5d --- /dev/null +++ b/ui/src/pages/session-details/capture/capture-tiers.test.ts @@ -0,0 +1,146 @@ +import { + buildTierLadder, + isTransposed, + pickTier, + TierSources, +} from './capture-tiers' + +const SOURCES: TierSources = { + medium: 'https://example.org/thumbnails/medium/capture.jpg', + large: 'https://example.org/thumbnails/large/capture.jpg', + original: 'https://example.org/captures/capture.jpg', +} + +describe('buildTierLadder', () => { + test('full ladder for a large original: medium, large, original', () => { + const tiers = buildTierLadder(SOURCES, 4096) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: 4096, isOriginal: true }, + ]) + }) + + test('drops the original when the large thumbnail already covers it', () => { + // A 2000px original produces a 2000px "large" thumbnail (thumbnails are + // never upscaled), so the EXIF-risky original adds no resolution. + const tiers = buildTierLadder(SOURCES, 2000) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2000, isOriginal: false }, + ]) + }) + + test('collapses to a single tier when the original is small', () => { + const tiers = buildTierLadder(SOURCES, 800) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 800, isOriginal: false }, + ]) + }) + + test('keeps the original as an unknown-width top tier when dimensions are missing', () => { + const tiers = buildTierLadder(SOURCES, null) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: null, isOriginal: true }, + ]) + }) + + test('missing medium thumbnail falls back to original and sorts by real width', () => { + // When a thumbnail size is missing, the model getters fall back to the + // original URL — that tier's real width is the original's, not 1024. + const sources = { ...SOURCES, medium: SOURCES.original } + const tiers = buildTierLadder(sources, 4096) + + expect(tiers).toEqual([ + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: 4096, isOriginal: true }, + ]) + }) + + test('all thumbnails missing yields a single original tier', () => { + const sources = { + medium: SOURCES.original, + large: SOURCES.original, + original: SOURCES.original, + } + const tiers = buildTierLadder(sources, 3000) + + expect(tiers).toEqual([ + { src: SOURCES.original, width: 3000, isOriginal: true }, + ]) + }) +}) + +describe('pickTier', () => { + const tiers = buildTierLadder(SOURCES, 4096) + + test('accepts up to 20% upscale before stepping up', () => { + // 1024 * 1.2 = 1228.8 — a 1200px demand stays on medium. + expect(pickTier(tiers, 1200)?.src).toBe(SOURCES.medium) + expect(pickTier(tiers, 1300)?.src).toBe(SOURCES.large) + }) + + test('picks medium on a 1x laptop and large on a retina display', () => { + expect(pickTier(tiers, 1100 * 1)?.src).toBe(SOURCES.medium) + expect(pickTier(tiers, 1100 * 2)?.src).toBe(SOURCES.large) + }) + + test('high zoom demand reaches the original', () => { + expect(pickTier(tiers, 4000)?.src).toBe(SOURCES.original) + }) + + test('demand beyond every tier returns the top tier', () => { + expect(pickTier(tiers, 100000)?.src).toBe(SOURCES.original) + }) + + test('unknown-width tier satisfies any demand', () => { + const openEnded = buildTierLadder(SOURCES, null) + expect(pickTier(openEnded, 100000)?.src).toBe(SOURCES.original) + }) + + test('returns null for an empty ladder', () => { + expect(pickTier([], 1000)).toBeNull() + }) +}) + +describe('isTransposed', () => { + test('detects a browser-rotated portrait original', () => { + // Stored dimensions are raw pixel space; the browser applied the EXIF + // Orientation tag and swapped the rendered dimensions. + expect( + isTransposed({ width: 3456, height: 4608 }, { width: 4608, height: 3456 }) + ).toBe(true) + }) + + test('matching dimensions are not transposed', () => { + expect( + isTransposed({ width: 4608, height: 3456 }, { width: 4608, height: 3456 }) + ).toBe(false) + }) + + test('square images can never be flagged', () => { + expect( + isTransposed({ width: 2000, height: 2000 }, { width: 2000, height: 2000 }) + ).toBe(false) + }) + + test('unknown stored dimensions are never flagged', () => { + expect( + isTransposed({ width: 3456, height: 4608 }, { width: null, height: null }) + ).toBe(false) + }) + + test('a downscaled thumbnail is not flagged even for portrait captures', () => { + // Thumbnails keep raw orientation and different absolute dimensions, so + // the exact-swap check does not fire. + expect( + isTransposed({ width: 1920, height: 2560 }, { width: 3456, height: 4608 }) + ).toBe(false) + }) +}) diff --git a/ui/src/pages/session-details/capture/capture-tiers.ts b/ui/src/pages/session-details/capture/capture-tiers.ts new file mode 100644 index 000000000..552785910 --- /dev/null +++ b/ui/src/pages/session-details/capture/capture-tiers.ts @@ -0,0 +1,119 @@ +/** + * Resolution ladder for the zoomable session detail capture. + * + * The view keeps a ladder of image tiers — medium (1024) → large (2560) → + * original — and requests a higher tier whenever the zoom level demands more + * pixels than the current tier provides. Demand is expressed in device pixels: + * container CSS width × devicePixelRatio × zoom scale. See issue #1373. + */ + +export interface TierSources { + medium: string + large: string + original: string +} + +export interface CaptureTier { + src: string + /** Native pixel width; null when unknown (original with no stored dimensions). */ + width: number | null + /** + * True when src points at the original file, which may carry an EXIF + * Orientation tag that browsers force-apply, rotating the pixels out of the + * coordinate space the detection boxes are drawn in. + */ + isOriginal: boolean +} + +/** Must match THUMBNAILS["SIZES"] in config/settings/base.py. */ +export const THUMBNAIL_WIDTHS = { + medium: 1024, + large: 2560, +} + +/** + * Accept up to 20% upscale before requesting the next tier, so a display + * marginally wider than a tier does not force the bigger download. + */ +export const TIER_UPSCALE_TOLERANCE = 1.2 + +export const buildTierLadder = ( + sources: TierSources, + captureWidth: number | null +): CaptureTier[] => { + // Thumbnails are never upscaled, so a tier's real width is capped by the + // original's. A missing thumbnail size falls back to the original URL, in + // which case the tier's real width is the original's, whatever the nominal + // thumbnail size says. + const tierFor = (src: string, thumbnailWidth: number): CaptureTier => { + if (src === sources.original) { + return { src, width: captureWidth, isOriginal: true } + } + return { + src, + width: captureWidth + ? Math.min(thumbnailWidth, captureWidth) + : thumbnailWidth, + isOriginal: false, + } + } + + const candidates = [ + tierFor(sources.medium, THUMBNAIL_WIDTHS.medium), + tierFor(sources.large, THUMBNAIL_WIDTHS.large), + { src: sources.original, width: captureWidth, isOriginal: true }, + ] + + const unique = candidates.filter( + (candidate, index) => + candidates.findIndex((other) => other.src === candidate.src) === index + ) + + // Sort by real width (unknown last) and keep only tiers that add resolution. + const sorted = [...unique].sort( + (a, b) => (a.width ?? Infinity) - (b.width ?? Infinity) + ) + + return sorted.filter((tier, index) => { + if (index === 0) { + return true + } + const previous = sorted[index - 1] + if (previous.width === null) { + return false + } + return tier.width === null || tier.width > previous.width + }) +} + +/** + * Smallest tier that satisfies the demand (within the upscale tolerance); + * the top tier when nothing does. + */ +export const pickTier = ( + tiers: CaptureTier[], + demand: number +): CaptureTier | null => { + for (const tier of tiers) { + if (tier.width === null || demand <= tier.width * TIER_UPSCALE_TOLERANCE) { + return tier + } + } + + return tiers.length ? tiers[tiers.length - 1] : null +} + +/** + * True when the rendered image dimensions are the stored dimensions swapped — + * the signature of a browser-applied EXIF rotation. Detection boxes are drawn + * in the stored (raw pixel) space, so a transposed image must not be shown. + */ +export const isTransposed = ( + natural: { width: number; height: number }, + stored: { width: number | null; height: number | null } +): boolean => + stored.width !== null && + stored.height !== null && + stored.width !== stored.height && + natural.width === stored.height && + natural.height === stored.width diff --git a/ui/src/pages/session-details/capture/capture.tsx b/ui/src/pages/session-details/capture/capture.tsx index a4dc631bb..aa27f47d3 100644 --- a/ui/src/pages/session-details/capture/capture.tsx +++ b/ui/src/pages/session-details/capture/capture.tsx @@ -7,7 +7,7 @@ import { OccurrenceDetails, TABS, } from 'pages/occurrence-details/occurrence-details' -import { useLayoutEffect, useMemo, useRef, useState } from 'react' +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { ReactZoomPanPinchRef, TransformComponent, @@ -16,10 +16,17 @@ import { import { SCORE_THRESHOLDS } from 'utils/constants' import { STRING, translate } from 'utils/language' import { useActiveOccurrences } from '../hooks/useActiveOccurrences' +import { TierSources } from './capture-tiers' import styles from './capture.module.scss' +import { useCaptureTiers } from './useCaptureTiers' const FALLBACK_RATIO = 16 / 9 +// react-zoom-pan-pinch's default maxScale; raised dynamically so large +// originals can always be inspected past 100% of their native pixels. +const DEFAULT_MAX_SCALE = 8 +const MAX_OVERZOOM = 2 + interface BoxStyle { width: string height: string @@ -32,7 +39,7 @@ interface CaptureProps { detections: CaptureDetection[] height: number | null showDetections?: boolean - src?: string + sources?: TierSources transformRef: React.RefObject width: number | null } @@ -42,44 +49,57 @@ export const Capture = ({ detections, height, showDetections, - src, + sources, transformRef, width, }: CaptureProps) => { + const wrapperRef = useRef(null) const [naturalSize, setNaturalSize] = useState<{ width: number height: number }>() - const imageRef = useRef(null) const [isLoading, setIsLoading] = useState() const [renderOverlay, setRenderOverlay] = useState() + const [containerWidth, setContainerWidth] = useState(0) + const [scale, setScale] = useState(1) + + const { + displayed, + incoming, + incomingLoaded, + updateDemand, + onIncomingLoad, + onIncomingError, + } = useCaptureTiers({ sources, captureWidth: width, captureHeight: height }) + const dpr = window.devicePixelRatio || 1 + + // The container width drives both the tier demand and the zoom readout. useLayoutEffect(() => { - if (!imageRef.current) { + const element = wrapperRef.current + if (!element) { return } + const measure = () => + setContainerWidth(element.getBoundingClientRect().width) + measure() + const observer = new ResizeObserver(measure) + observer.observe(element) + return () => observer.disconnect() + }, []) + + useEffect(() => { + if (containerWidth) { + updateDemand(containerWidth * dpr * scale) + } + }, [containerWidth, dpr, scale, updateDemand]) + useEffect(() => { + // Show the spinner whenever the active capture changes; the previous + // image stays visible underneath until the new tier loads. setIsLoading(true) setNaturalSize(undefined) - - if (src) { - imageRef.current.src = src - imageRef.current.onload = () => { - if (imageRef.current?.width && imageRef.current.height) { - setNaturalSize({ - width: imageRef.current.naturalWidth, - height: imageRef.current.naturalHeight, - }) - } - setIsLoading(false) - } - - imageRef.current.onerror = () => { - setNaturalSize(undefined) - setIsLoading(false) - } - } - }, [src]) + }, [sources?.original]) useLayoutEffect(() => { // Ugly hack to make overlay correct on first render @@ -112,25 +132,85 @@ export const Capture = ({ ) const ratio = useMemo(() => { - if (naturalSize) { - return naturalSize.width / naturalSize.height - } - + // Stored dimensions first: they match the detection box space and stay + // constant across tiers, so the layout never shifts on a tier swap. if (width && height) { return width / height } + if (naturalSize) { + return naturalSize.width / naturalSize.height + } + return FALLBACK_RATIO }, [width, height, naturalSize]) + const maxScale = useMemo(() => { + if (width && containerWidth) { + const fullResolutionScale = width / (containerWidth * dpr) + return Math.max(DEFAULT_MAX_SCALE, fullResolutionScale * MAX_OVERZOOM) + } + + return DEFAULT_MAX_SCALE + }, [width, containerWidth, dpr]) + + // Zoom level relative to the original image's pixels: 100% = one image + // pixel per device pixel. + const zoomPercent = useMemo(() => { + if (!width || !containerWidth) { + return null + } + + return Math.round(((containerWidth * dpr * scale) / width) * 100) + }, [width, containerWidth, dpr, scale]) + return ( -
- +
+ setScale(state.scale)} + ref={transformRef} + > - + { + const image = event.currentTarget + if (image.naturalWidth && image.naturalHeight) { + setNaturalSize({ + width: image.naturalWidth, + height: image.naturalHeight, + }) + } + setIsLoading(false) + }} + onError={() => { + setNaturalSize(undefined) + setIsLoading(false) + }} + /> + {incoming ? ( + onIncomingLoad(event.currentTarget)} + onError={onIncomingError} + /> + ) : null}
+ {zoomPercent !== null ? ( + + {zoomPercent}% + + ) : null} + {incoming && !incomingLoaded ? ( + + + {translate(STRING.LOADING_HIGHER_RESOLUTION)}... + + ) : null} {isLoading ? (
diff --git a/ui/src/pages/session-details/capture/useCaptureTiers.ts b/ui/src/pages/session-details/capture/useCaptureTiers.ts new file mode 100644 index 000000000..fff4ec786 --- /dev/null +++ b/ui/src/pages/session-details/capture/useCaptureTiers.ts @@ -0,0 +1,196 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { + buildTierLadder, + CaptureTier, + isTransposed, + pickTier, + TierSources, +} from './capture-tiers' + +/** Let a pinch/scroll gesture settle before any higher-tier fetch fires. */ +const PROMOTE_DEBOUNCE_MS = 300 + +/** Keep in sync with the duration-300 class on the incoming image. */ +const CROSSFADE_MS = 300 + +/** + * Owns the resolution ladder state for the session detail capture: which tier + * is displayed, which higher tier is loading, and the crossfade between them. + * + * The caller reports pixel demand (container width × devicePixelRatio × zoom + * scale) via updateDemand; the hook debounces promotion so a zoom gesture + * settles before a fetch fires. Tiers are only ever promoted — a downloaded + * tier is never traded back for a blurrier one. + */ +export const useCaptureTiers = ({ + sources, + captureWidth, + captureHeight, +}: { + sources?: TierSources + captureWidth: number | null + captureHeight: number | null +}) => { + const [displayed, setDisplayed] = useState(null) + const [incoming, setIncoming] = useState(null) + const [incomingLoaded, setIncomingLoaded] = useState(false) + + const medium = sources?.medium + const large = sources?.large + const original = sources?.original + + const tiers = useMemo( + () => + medium && large && original + ? buildTierLadder({ medium, large, original }, captureWidth) + : [], + [medium, large, original, captureWidth] + ) + + // Mirror state into refs so the debounce timer always acts on fresh values. + const tiersRef = useRef(tiers) + tiersRef.current = tiers + const displayedRef = useRef(displayed) + displayedRef.current = displayed + const incomingRef = useRef(incoming) + incomingRef.current = incoming + const incomingLoadedRef = useRef(incomingLoaded) + incomingLoadedRef.current = incomingLoaded + const storedDimensionsRef = useRef({ + width: captureWidth, + height: captureHeight, + }) + storedDimensionsRef.current = { width: captureWidth, height: captureHeight } + + const demandRef = useRef(0) + // Tiers that failed to load or turned out EXIF-rotated; skipped for the + // rest of this capture's ladder so promotion does not retry in a loop. + const unusableSrcsRef = useRef>(new Set()) + const promoteTimerRef = useRef>() + const commitTimerRef = useRef>() + + const selectTier = useCallback((demand: number) => { + const usable = tiersRef.current.filter( + (tier) => !unusableSrcsRef.current.has(tier.src) + ) + return pickTier(usable, demand) + }, []) + + // Reset the ladder when navigating to another capture. While the capture is + // loading (original undefined) the previous image stays displayed, matching + // the pre-ladder behavior of keeping the old frame under the spinner. + useEffect(() => { + if (!original) { + return + } + clearTimeout(promoteTimerRef.current) + clearTimeout(commitTimerRef.current) + unusableSrcsRef.current = new Set() + setIncoming(null) + setIncomingLoaded(false) + // Demand is already known when navigating between captures; on first mount + // it is 0 until the container is measured, and updateDemand picks the + // initial tier as soon as the measurement arrives. + setDisplayed(demandRef.current > 0 ? selectTier(demandRef.current) : null) + }, [original, selectTier]) + + const evaluatePromotion = useCallback(() => { + const current = displayedRef.current + const target = selectTier(demandRef.current) + if (!current || !target) { + return + } + + const ladder = tiersRef.current + const currentIndex = ladder.findIndex((tier) => tier.src === current.src) + const targetIndex = ladder.findIndex((tier) => tier.src === target.src) + + if (targetIndex > currentIndex) { + if (incomingRef.current?.src !== target.src) { + clearTimeout(commitTimerRef.current) + setIncoming(target) + setIncomingLoaded(false) + } + } else if (incomingRef.current && !incomingLoadedRef.current) { + // Zoomed back out before the upgrade arrived — drop the request. A tier + // that already finished loading is committed regardless (never demote). + setIncoming(null) + } + }, [selectTier]) + + const updateDemand = useCallback( + (demand: number) => { + demandRef.current = demand + + if (!displayedRef.current) { + const initial = selectTier(demand) + if (initial) { + setDisplayed(initial) + } + return + } + + clearTimeout(promoteTimerRef.current) + promoteTimerRef.current = setTimeout( + evaluatePromotion, + PROMOTE_DEBOUNCE_MS + ) + }, + [evaluatePromotion, selectTier] + ) + + const onIncomingLoad = useCallback((image: HTMLImageElement) => { + const tier = incomingRef.current + if (!tier) { + return + } + + // A browser-applied EXIF rotation would misalign the detection boxes (the + // bug PR #1374 fixed), so refuse the upgrade and stay on the current tier. + if ( + isTransposed( + { width: image.naturalWidth, height: image.naturalHeight }, + storedDimensionsRef.current + ) + ) { + unusableSrcsRef.current.add(tier.src) + setIncoming(null) + setIncomingLoaded(false) + return + } + + setIncomingLoaded(true) + commitTimerRef.current = setTimeout(() => { + setDisplayed(tier) + setIncoming(null) + setIncomingLoaded(false) + }, CROSSFADE_MS + 50) + }, []) + + const onIncomingError = useCallback(() => { + const tier = incomingRef.current + if (!tier) { + return + } + unusableSrcsRef.current.add(tier.src) + setIncoming(null) + setIncomingLoaded(false) + }, []) + + useEffect( + () => () => { + clearTimeout(promoteTimerRef.current) + clearTimeout(commitTimerRef.current) + }, + [] + ) + + return { + displayed, + incoming, + incomingLoaded, + updateDemand, + onIncomingLoad, + onIncomingError, + } +} diff --git a/ui/src/pages/session-details/session-details.tsx b/ui/src/pages/session-details/session-details.tsx index 09d51d2bf..860084443 100644 --- a/ui/src/pages/session-details/session-details.tsx +++ b/ui/src/pages/session-details/session-details.tsx @@ -169,7 +169,15 @@ const Content = ({ session }: { session: SessionDetails }) => { detections={activeCapture?.detections ?? []} height={activeCapture?.height ?? session.firstCapture.height} showDetections={settings.showDetections} - src={activeCapture?.thumbnailLarge} + sources={ + activeCapture + ? { + medium: activeCapture.thumbnailMedium, + large: activeCapture.thumbnailLarge, + original: activeCapture.src, + } + : undefined + } transformRef={transformRef} width={activeCapture?.width ?? session.firstCapture.width} /> diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index f2be6618c..5f16f04f5 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -311,6 +311,7 @@ export enum STRING { LATEST_OCCURRENCES, LEAVE_TEAM, LOADING_DATA, + LOADING_HIGHER_RESOLUTION, MACHINE_PREDICTION_SCORE, MACHINE_SUGGESTION, MANAGE_ACCESS_FOR, @@ -719,6 +720,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.LATEST_OCCURRENCES]: 'Latest occurrences', [STRING.LEAVE_TEAM]: 'Leave team', [STRING.LOADING_DATA]: 'Loading data', + [STRING.LOADING_HIGHER_RESOLUTION]: 'Loading higher resolution', [STRING.MACHINE_PREDICTION_SCORE]: 'Machine prediction score\n{{score}}', [STRING.MACHINE_SUGGESTION]: 'Machine suggestion', [STRING.MANAGE_ACCESS_FOR]: 'Manage access for {{user}}.', From f25111c1f31848c59fb592593d029745f0351594 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 21 Jul 2026 06:30:16 -0700 Subject: [PATCH 5/7] refactor: assume thumbnails exist and skip missing sizes in the zoom ladder MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thumbnail sizes are generated on request, so a missing URL means the capture's storage is likely unreachable — the ladder now skips that tier instead of aliasing it to the original file. This removes the URL-identity width derivation, sorting, and dedupe that only existed to handle the model getters' original-URL fallback, and replaces the thumbnailLarge getter with a thumbnailSizes getter that has no fallback. Co-Authored-By: Claude --- ui/src/data-services/models/capture.ts | 15 ++-- .../capture/capture-tiers.test.ts | 16 ++-- .../session-details/capture/capture-tiers.ts | 80 +++++++++---------- .../capture/useCaptureTiers.ts | 2 +- .../pages/session-details/session-details.tsx | 3 +- 5 files changed, 55 insertions(+), 61 deletions(-) diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index c87189957..238367ef1 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -183,14 +183,15 @@ export class Capture { return this._capture.url } - // EXIF-free large thumbnail for the zoomable session detail view; detection - // boxes align with its pixels, unlike the EXIF-rotated original file. - get thumbnailLarge(): string { - if (this._capture.thumbnails?.large) { - return this._capture.thumbnails.large + // EXIF-free thumbnail URLs for the session detail zoom ladder, with no + // original-file fallback: thumbnails are generated on request, so a missing + // size usually means the source file is unreachable too, and the ladder + // simply skips that tier. + get thumbnailSizes(): { medium?: string; large?: string } { + return { + medium: this._capture.thumbnails?.medium ?? undefined, + large: this._capture.thumbnails?.large ?? undefined, } - - return this._capture.url } get src(): string { diff --git a/ui/src/pages/session-details/capture/capture-tiers.test.ts b/ui/src/pages/session-details/capture/capture-tiers.test.ts index ae4e24f5d..d1b3f3c7b 100644 --- a/ui/src/pages/session-details/capture/capture-tiers.test.ts +++ b/ui/src/pages/session-details/capture/capture-tiers.test.ts @@ -51,11 +51,10 @@ describe('buildTierLadder', () => { ]) }) - test('missing medium thumbnail falls back to original and sorts by real width', () => { - // When a thumbnail size is missing, the model getters fall back to the - // original URL — that tier's real width is the original's, not 1024. - const sources = { ...SOURCES, medium: SOURCES.original } - const tiers = buildTierLadder(sources, 4096) + test('a missing thumbnail size is skipped as a tier', () => { + // Thumbnails are generated on request, so a missing size means the + // capture's storage is likely unreachable — skip it rather than retry. + const tiers = buildTierLadder({ ...SOURCES, medium: undefined }, 4096) expect(tiers).toEqual([ { src: SOURCES.large, width: 2560, isOriginal: false }, @@ -64,12 +63,7 @@ describe('buildTierLadder', () => { }) test('all thumbnails missing yields a single original tier', () => { - const sources = { - medium: SOURCES.original, - large: SOURCES.original, - original: SOURCES.original, - } - const tiers = buildTierLadder(sources, 3000) + const tiers = buildTierLadder({ original: SOURCES.original }, 3000) expect(tiers).toEqual([ { src: SOURCES.original, width: 3000, isOriginal: true }, diff --git a/ui/src/pages/session-details/capture/capture-tiers.ts b/ui/src/pages/session-details/capture/capture-tiers.ts index 552785910..612086807 100644 --- a/ui/src/pages/session-details/capture/capture-tiers.ts +++ b/ui/src/pages/session-details/capture/capture-tiers.ts @@ -8,8 +8,10 @@ */ export interface TierSources { - medium: string - large: string + // Thumbnail sizes are generated on request, so a missing URL means the + // capture's storage is likely unreachable; the ladder skips that tier. + medium?: string + large?: string original: string } @@ -42,48 +44,46 @@ export const buildTierLadder = ( captureWidth: number | null ): CaptureTier[] => { // Thumbnails are never upscaled, so a tier's real width is capped by the - // original's. A missing thumbnail size falls back to the original URL, in - // which case the tier's real width is the original's, whatever the nominal - // thumbnail size says. - const tierFor = (src: string, thumbnailWidth: number): CaptureTier => { - if (src === sources.original) { - return { src, width: captureWidth, isOriginal: true } - } - return { - src, - width: captureWidth - ? Math.min(thumbnailWidth, captureWidth) - : thumbnailWidth, + // original's. + const cap = (thumbnailWidth: number) => + captureWidth ? Math.min(thumbnailWidth, captureWidth) : thumbnailWidth + + const candidates: CaptureTier[] = [] + if (sources.medium) { + candidates.push({ + src: sources.medium, + width: cap(THUMBNAIL_WIDTHS.medium), isOriginal: false, - } + }) } + if (sources.large) { + candidates.push({ + src: sources.large, + width: cap(THUMBNAIL_WIDTHS.large), + isOriginal: false, + }) + } + candidates.push({ + src: sources.original, + width: captureWidth, + isOriginal: true, + }) - const candidates = [ - tierFor(sources.medium, THUMBNAIL_WIDTHS.medium), - tierFor(sources.large, THUMBNAIL_WIDTHS.large), - { src: sources.original, width: captureWidth, isOriginal: true }, - ] - - const unique = candidates.filter( - (candidate, index) => - candidates.findIndex((other) => other.src === candidate.src) === index - ) - - // Sort by real width (unknown last) and keep only tiers that add resolution. - const sorted = [...unique].sort( - (a, b) => (a.width ?? Infinity) - (b.width ?? Infinity) - ) - - return sorted.filter((tier, index) => { - if (index === 0) { - return true + // Keep only tiers that add resolution over the previous one — this drops + // the EXIF-risky original whenever the large thumbnail already covers its + // full size. + return candidates.reduce((ladder: CaptureTier[], tier) => { + const previous = ladder[ladder.length - 1] + if ( + previous && + previous.width !== null && + tier.width !== null && + tier.width <= previous.width + ) { + return ladder } - const previous = sorted[index - 1] - if (previous.width === null) { - return false - } - return tier.width === null || tier.width > previous.width - }) + return [...ladder, tier] + }, []) } /** diff --git a/ui/src/pages/session-details/capture/useCaptureTiers.ts b/ui/src/pages/session-details/capture/useCaptureTiers.ts index fff4ec786..ac8aede40 100644 --- a/ui/src/pages/session-details/capture/useCaptureTiers.ts +++ b/ui/src/pages/session-details/capture/useCaptureTiers.ts @@ -41,7 +41,7 @@ export const useCaptureTiers = ({ const tiers = useMemo( () => - medium && large && original + original ? buildTierLadder({ medium, large, original }, captureWidth) : [], [medium, large, original, captureWidth] diff --git a/ui/src/pages/session-details/session-details.tsx b/ui/src/pages/session-details/session-details.tsx index 860084443..00ddf0594 100644 --- a/ui/src/pages/session-details/session-details.tsx +++ b/ui/src/pages/session-details/session-details.tsx @@ -172,8 +172,7 @@ const Content = ({ session }: { session: SessionDetails }) => { sources={ activeCapture ? { - medium: activeCapture.thumbnailMedium, - large: activeCapture.thumbnailLarge, + ...activeCapture.thumbnailSizes, original: activeCapture.src, } : undefined From 4ab6209deca94bea617df44d7f8acdf380aa122d Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Tue, 21 Jul 2026 18:01:45 -0700 Subject: [PATCH 6/7] refactor: drop the unused thumbnailMedium getter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The getter's only consumer was the session detail image source, which #1339 switched to the original URL in the same commit that renamed the getter to camelCase. It has had no callers since. The medium thumbnail size itself is still in use — it is the base tier of the session detail zoom ladder. What this removes is only the unused single-URL-with-fallback shape, leaving the model's thumbnail accessors matched to real consumers: thumbnailSmall for the captures list and thumbnailSizes for the zoom ladder. Co-Authored-By: Claude --- ui/src/data-services/models/capture.ts | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index 238367ef1..22f786d5f 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -175,14 +175,6 @@ export class Capture { return this._capture.url } - get thumbnailMedium(): string { - if (this._capture.thumbnails?.medium) { - return this._capture.thumbnails.medium - } - - return this._capture.url - } - // EXIF-free thumbnail URLs for the session detail zoom ladder, with no // original-file fallback: thumbnails are generated on request, so a missing // size usually means the source file is unreachable too, and the ladder From 33b1c951c064630669cc7cd0b3ac513681b610c7 Mon Sep 17 00:00:00 2001 From: Michael Bunsen Date: Wed, 22 Jul 2026 20:17:07 -0700 Subject: [PATCH 7/7] refactor: name the crossfade commit buffer instead of a bare + 50 Extract the 50ms margin after the crossfade into a named COMMIT_BUFFER_MS constant with a comment explaining why it exists: the commit must land after the opacity transition finishes, or the incoming image is promoted a frame early and the layer swap flashes. Addresses a review note about the magic number. Co-Authored-By: Claude --- .../pages/session-details/capture/useCaptureTiers.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ui/src/pages/session-details/capture/useCaptureTiers.ts b/ui/src/pages/session-details/capture/useCaptureTiers.ts index ac8aede40..420080f4b 100644 --- a/ui/src/pages/session-details/capture/useCaptureTiers.ts +++ b/ui/src/pages/session-details/capture/useCaptureTiers.ts @@ -13,6 +13,14 @@ const PROMOTE_DEBOUNCE_MS = 300 /** Keep in sync with the duration-300 class on the incoming image. */ const CROSSFADE_MS = 300 +/** + * Extra margin past the crossfade before committing the tier, so the opacity + * transition finishes before the incoming image is promoted to the base layer + * and the old layer unmounts. Without it the swap can land a frame early and + * flash. + */ +const COMMIT_BUFFER_MS = 50 + /** * Owns the resolution ladder state for the session detail capture: which tier * is displayed, which higher tier is loading, and the crossfade between them. @@ -164,7 +172,7 @@ export const useCaptureTiers = ({ setDisplayed(tier) setIncoming(null) setIncomingLoaded(false) - }, CROSSFADE_MS + 50) + }, CROSSFADE_MS + COMMIT_BUFFER_MS) }, []) const onIncomingError = useCallback(() => {