diff --git a/ui/src/data-services/models/capture.ts b/ui/src/data-services/models/capture.ts index c87189957..22f786d5f 100644 --- a/ui/src/data-services/models/capture.ts +++ b/ui/src/data-services/models/capture.ts @@ -175,22 +175,15 @@ export class Capture { return this._capture.url } - get thumbnailMedium(): string { - if (this._capture.thumbnails?.medium) { - return this._capture.thumbnails.medium + // EXIF-free thumbnail URLs for the session detail zoom ladder, with no + // original-file fallback: thumbnails are generated on request, so a missing + // size usually means the source file is unreachable too, and the ladder + // simply skips that tier. + get thumbnailSizes(): { medium?: string; large?: string } { + return { + medium: this._capture.thumbnails?.medium ?? undefined, + large: this._capture.thumbnails?.large ?? undefined, } - - return this._capture.url - } - - // EXIF-free large thumbnail for the zoomable session detail view; detection - // boxes align with its pixels, unlike the EXIF-rotated original file. - get thumbnailLarge(): string { - if (this._capture.thumbnails?.large) { - return this._capture.thumbnails.large - } - - return this._capture.url } get src(): string { diff --git a/ui/src/pages/session-details/capture/capture-tiers.test.ts b/ui/src/pages/session-details/capture/capture-tiers.test.ts new file mode 100644 index 000000000..d1b3f3c7b --- /dev/null +++ b/ui/src/pages/session-details/capture/capture-tiers.test.ts @@ -0,0 +1,140 @@ +import { + buildTierLadder, + isTransposed, + pickTier, + TierSources, +} from './capture-tiers' + +const SOURCES: TierSources = { + medium: 'https://example.org/thumbnails/medium/capture.jpg', + large: 'https://example.org/thumbnails/large/capture.jpg', + original: 'https://example.org/captures/capture.jpg', +} + +describe('buildTierLadder', () => { + test('full ladder for a large original: medium, large, original', () => { + const tiers = buildTierLadder(SOURCES, 4096) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: 4096, isOriginal: true }, + ]) + }) + + test('drops the original when the large thumbnail already covers it', () => { + // A 2000px original produces a 2000px "large" thumbnail (thumbnails are + // never upscaled), so the EXIF-risky original adds no resolution. + const tiers = buildTierLadder(SOURCES, 2000) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2000, isOriginal: false }, + ]) + }) + + test('collapses to a single tier when the original is small', () => { + const tiers = buildTierLadder(SOURCES, 800) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 800, isOriginal: false }, + ]) + }) + + test('keeps the original as an unknown-width top tier when dimensions are missing', () => { + const tiers = buildTierLadder(SOURCES, null) + + expect(tiers).toEqual([ + { src: SOURCES.medium, width: 1024, isOriginal: false }, + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: null, isOriginal: true }, + ]) + }) + + test('a missing thumbnail size is skipped as a tier', () => { + // Thumbnails are generated on request, so a missing size means the + // capture's storage is likely unreachable — skip it rather than retry. + const tiers = buildTierLadder({ ...SOURCES, medium: undefined }, 4096) + + expect(tiers).toEqual([ + { src: SOURCES.large, width: 2560, isOriginal: false }, + { src: SOURCES.original, width: 4096, isOriginal: true }, + ]) + }) + + test('all thumbnails missing yields a single original tier', () => { + const tiers = buildTierLadder({ original: SOURCES.original }, 3000) + + expect(tiers).toEqual([ + { src: SOURCES.original, width: 3000, isOriginal: true }, + ]) + }) +}) + +describe('pickTier', () => { + const tiers = buildTierLadder(SOURCES, 4096) + + test('accepts up to 20% upscale before stepping up', () => { + // 1024 * 1.2 = 1228.8 — a 1200px demand stays on medium. + expect(pickTier(tiers, 1200)?.src).toBe(SOURCES.medium) + expect(pickTier(tiers, 1300)?.src).toBe(SOURCES.large) + }) + + test('picks medium on a 1x laptop and large on a retina display', () => { + expect(pickTier(tiers, 1100 * 1)?.src).toBe(SOURCES.medium) + expect(pickTier(tiers, 1100 * 2)?.src).toBe(SOURCES.large) + }) + + test('high zoom demand reaches the original', () => { + expect(pickTier(tiers, 4000)?.src).toBe(SOURCES.original) + }) + + test('demand beyond every tier returns the top tier', () => { + expect(pickTier(tiers, 100000)?.src).toBe(SOURCES.original) + }) + + test('unknown-width tier satisfies any demand', () => { + const openEnded = buildTierLadder(SOURCES, null) + expect(pickTier(openEnded, 100000)?.src).toBe(SOURCES.original) + }) + + test('returns null for an empty ladder', () => { + expect(pickTier([], 1000)).toBeNull() + }) +}) + +describe('isTransposed', () => { + test('detects a browser-rotated portrait original', () => { + // Stored dimensions are raw pixel space; the browser applied the EXIF + // Orientation tag and swapped the rendered dimensions. + expect( + isTransposed({ width: 3456, height: 4608 }, { width: 4608, height: 3456 }) + ).toBe(true) + }) + + test('matching dimensions are not transposed', () => { + expect( + isTransposed({ width: 4608, height: 3456 }, { width: 4608, height: 3456 }) + ).toBe(false) + }) + + test('square images can never be flagged', () => { + expect( + isTransposed({ width: 2000, height: 2000 }, { width: 2000, height: 2000 }) + ).toBe(false) + }) + + test('unknown stored dimensions are never flagged', () => { + expect( + isTransposed({ width: 3456, height: 4608 }, { width: null, height: null }) + ).toBe(false) + }) + + test('a downscaled thumbnail is not flagged even for portrait captures', () => { + // Thumbnails keep raw orientation and different absolute dimensions, so + // the exact-swap check does not fire. + expect( + isTransposed({ width: 1920, height: 2560 }, { width: 3456, height: 4608 }) + ).toBe(false) + }) +}) diff --git a/ui/src/pages/session-details/capture/capture-tiers.ts b/ui/src/pages/session-details/capture/capture-tiers.ts new file mode 100644 index 000000000..612086807 --- /dev/null +++ b/ui/src/pages/session-details/capture/capture-tiers.ts @@ -0,0 +1,119 @@ +/** + * Resolution ladder for the zoomable session detail capture. + * + * The view keeps a ladder of image tiers — medium (1024) → large (2560) → + * original — and requests a higher tier whenever the zoom level demands more + * pixels than the current tier provides. Demand is expressed in device pixels: + * container CSS width × devicePixelRatio × zoom scale. See issue #1373. + */ + +export interface TierSources { + // Thumbnail sizes are generated on request, so a missing URL means the + // capture's storage is likely unreachable; the ladder skips that tier. + medium?: string + large?: string + original: string +} + +export interface CaptureTier { + src: string + /** Native pixel width; null when unknown (original with no stored dimensions). */ + width: number | null + /** + * True when src points at the original file, which may carry an EXIF + * Orientation tag that browsers force-apply, rotating the pixels out of the + * coordinate space the detection boxes are drawn in. + */ + isOriginal: boolean +} + +/** Must match THUMBNAILS["SIZES"] in config/settings/base.py. */ +export const THUMBNAIL_WIDTHS = { + medium: 1024, + large: 2560, +} + +/** + * Accept up to 20% upscale before requesting the next tier, so a display + * marginally wider than a tier does not force the bigger download. + */ +export const TIER_UPSCALE_TOLERANCE = 1.2 + +export const buildTierLadder = ( + sources: TierSources, + captureWidth: number | null +): CaptureTier[] => { + // Thumbnails are never upscaled, so a tier's real width is capped by the + // original's. + const cap = (thumbnailWidth: number) => + captureWidth ? Math.min(thumbnailWidth, captureWidth) : thumbnailWidth + + const candidates: CaptureTier[] = [] + if (sources.medium) { + candidates.push({ + src: sources.medium, + width: cap(THUMBNAIL_WIDTHS.medium), + isOriginal: false, + }) + } + if (sources.large) { + candidates.push({ + src: sources.large, + width: cap(THUMBNAIL_WIDTHS.large), + isOriginal: false, + }) + } + candidates.push({ + src: sources.original, + width: captureWidth, + isOriginal: true, + }) + + // Keep only tiers that add resolution over the previous one — this drops + // the EXIF-risky original whenever the large thumbnail already covers its + // full size. + return candidates.reduce((ladder: CaptureTier[], tier) => { + const previous = ladder[ladder.length - 1] + if ( + previous && + previous.width !== null && + tier.width !== null && + tier.width <= previous.width + ) { + return ladder + } + return [...ladder, tier] + }, []) +} + +/** + * Smallest tier that satisfies the demand (within the upscale tolerance); + * the top tier when nothing does. + */ +export const pickTier = ( + tiers: CaptureTier[], + demand: number +): CaptureTier | null => { + for (const tier of tiers) { + if (tier.width === null || demand <= tier.width * TIER_UPSCALE_TOLERANCE) { + return tier + } + } + + return tiers.length ? tiers[tiers.length - 1] : null +} + +/** + * True when the rendered image dimensions are the stored dimensions swapped — + * the signature of a browser-applied EXIF rotation. Detection boxes are drawn + * in the stored (raw pixel) space, so a transposed image must not be shown. + */ +export const isTransposed = ( + natural: { width: number; height: number }, + stored: { width: number | null; height: number | null } +): boolean => + stored.width !== null && + stored.height !== null && + stored.width !== stored.height && + natural.width === stored.height && + natural.height === stored.width diff --git a/ui/src/pages/session-details/capture/capture.tsx b/ui/src/pages/session-details/capture/capture.tsx index a4dc631bb..aa27f47d3 100644 --- a/ui/src/pages/session-details/capture/capture.tsx +++ b/ui/src/pages/session-details/capture/capture.tsx @@ -7,7 +7,7 @@ import { OccurrenceDetails, TABS, } from 'pages/occurrence-details/occurrence-details' -import { useLayoutEffect, useMemo, useRef, useState } from 'react' +import { useEffect, useLayoutEffect, useMemo, useRef, useState } from 'react' import { ReactZoomPanPinchRef, TransformComponent, @@ -16,10 +16,17 @@ import { import { SCORE_THRESHOLDS } from 'utils/constants' import { STRING, translate } from 'utils/language' import { useActiveOccurrences } from '../hooks/useActiveOccurrences' +import { TierSources } from './capture-tiers' import styles from './capture.module.scss' +import { useCaptureTiers } from './useCaptureTiers' const FALLBACK_RATIO = 16 / 9 +// react-zoom-pan-pinch's default maxScale; raised dynamically so large +// originals can always be inspected past 100% of their native pixels. +const DEFAULT_MAX_SCALE = 8 +const MAX_OVERZOOM = 2 + interface BoxStyle { width: string height: string @@ -32,7 +39,7 @@ interface CaptureProps { detections: CaptureDetection[] height: number | null showDetections?: boolean - src?: string + sources?: TierSources transformRef: React.RefObject width: number | null } @@ -42,44 +49,57 @@ export const Capture = ({ detections, height, showDetections, - src, + sources, transformRef, width, }: CaptureProps) => { + const wrapperRef = useRef(null) const [naturalSize, setNaturalSize] = useState<{ width: number height: number }>() - const imageRef = useRef(null) const [isLoading, setIsLoading] = useState() const [renderOverlay, setRenderOverlay] = useState() + const [containerWidth, setContainerWidth] = useState(0) + const [scale, setScale] = useState(1) + + const { + displayed, + incoming, + incomingLoaded, + updateDemand, + onIncomingLoad, + onIncomingError, + } = useCaptureTiers({ sources, captureWidth: width, captureHeight: height }) + const dpr = window.devicePixelRatio || 1 + + // The container width drives both the tier demand and the zoom readout. useLayoutEffect(() => { - if (!imageRef.current) { + const element = wrapperRef.current + if (!element) { return } + const measure = () => + setContainerWidth(element.getBoundingClientRect().width) + measure() + const observer = new ResizeObserver(measure) + observer.observe(element) + return () => observer.disconnect() + }, []) + + useEffect(() => { + if (containerWidth) { + updateDemand(containerWidth * dpr * scale) + } + }, [containerWidth, dpr, scale, updateDemand]) + useEffect(() => { + // Show the spinner whenever the active capture changes; the previous + // image stays visible underneath until the new tier loads. setIsLoading(true) setNaturalSize(undefined) - - if (src) { - imageRef.current.src = src - imageRef.current.onload = () => { - if (imageRef.current?.width && imageRef.current.height) { - setNaturalSize({ - width: imageRef.current.naturalWidth, - height: imageRef.current.naturalHeight, - }) - } - setIsLoading(false) - } - - imageRef.current.onerror = () => { - setNaturalSize(undefined) - setIsLoading(false) - } - } - }, [src]) + }, [sources?.original]) useLayoutEffect(() => { // Ugly hack to make overlay correct on first render @@ -112,25 +132,85 @@ export const Capture = ({ ) const ratio = useMemo(() => { - if (naturalSize) { - return naturalSize.width / naturalSize.height - } - + // Stored dimensions first: they match the detection box space and stay + // constant across tiers, so the layout never shifts on a tier swap. if (width && height) { return width / height } + if (naturalSize) { + return naturalSize.width / naturalSize.height + } + return FALLBACK_RATIO }, [width, height, naturalSize]) + const maxScale = useMemo(() => { + if (width && containerWidth) { + const fullResolutionScale = width / (containerWidth * dpr) + return Math.max(DEFAULT_MAX_SCALE, fullResolutionScale * MAX_OVERZOOM) + } + + return DEFAULT_MAX_SCALE + }, [width, containerWidth, dpr]) + + // Zoom level relative to the original image's pixels: 100% = one image + // pixel per device pixel. + const zoomPercent = useMemo(() => { + if (!width || !containerWidth) { + return null + } + + return Math.round(((containerWidth * dpr * scale) / width) * 100) + }, [width, containerWidth, dpr, scale]) + return ( -
- +
+ setScale(state.scale)} + ref={transformRef} + > - + { + const image = event.currentTarget + if (image.naturalWidth && image.naturalHeight) { + setNaturalSize({ + width: image.naturalWidth, + height: image.naturalHeight, + }) + } + setIsLoading(false) + }} + onError={() => { + setNaturalSize(undefined) + setIsLoading(false) + }} + /> + {incoming ? ( + onIncomingLoad(event.currentTarget)} + onError={onIncomingError} + /> + ) : null}
+ {zoomPercent !== null ? ( + + {zoomPercent}% + + ) : null} + {incoming && !incomingLoaded ? ( + + + {translate(STRING.LOADING_HIGHER_RESOLUTION)}... + + ) : null} {isLoading ? (
diff --git a/ui/src/pages/session-details/capture/useCaptureTiers.ts b/ui/src/pages/session-details/capture/useCaptureTiers.ts new file mode 100644 index 000000000..420080f4b --- /dev/null +++ b/ui/src/pages/session-details/capture/useCaptureTiers.ts @@ -0,0 +1,204 @@ +import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { + buildTierLadder, + CaptureTier, + isTransposed, + pickTier, + TierSources, +} from './capture-tiers' + +/** Let a pinch/scroll gesture settle before any higher-tier fetch fires. */ +const PROMOTE_DEBOUNCE_MS = 300 + +/** Keep in sync with the duration-300 class on the incoming image. */ +const CROSSFADE_MS = 300 + +/** + * Extra margin past the crossfade before committing the tier, so the opacity + * transition finishes before the incoming image is promoted to the base layer + * and the old layer unmounts. Without it the swap can land a frame early and + * flash. + */ +const COMMIT_BUFFER_MS = 50 + +/** + * Owns the resolution ladder state for the session detail capture: which tier + * is displayed, which higher tier is loading, and the crossfade between them. + * + * The caller reports pixel demand (container width × devicePixelRatio × zoom + * scale) via updateDemand; the hook debounces promotion so a zoom gesture + * settles before a fetch fires. Tiers are only ever promoted — a downloaded + * tier is never traded back for a blurrier one. + */ +export const useCaptureTiers = ({ + sources, + captureWidth, + captureHeight, +}: { + sources?: TierSources + captureWidth: number | null + captureHeight: number | null +}) => { + const [displayed, setDisplayed] = useState(null) + const [incoming, setIncoming] = useState(null) + const [incomingLoaded, setIncomingLoaded] = useState(false) + + const medium = sources?.medium + const large = sources?.large + const original = sources?.original + + const tiers = useMemo( + () => + original + ? buildTierLadder({ medium, large, original }, captureWidth) + : [], + [medium, large, original, captureWidth] + ) + + // Mirror state into refs so the debounce timer always acts on fresh values. + const tiersRef = useRef(tiers) + tiersRef.current = tiers + const displayedRef = useRef(displayed) + displayedRef.current = displayed + const incomingRef = useRef(incoming) + incomingRef.current = incoming + const incomingLoadedRef = useRef(incomingLoaded) + incomingLoadedRef.current = incomingLoaded + const storedDimensionsRef = useRef({ + width: captureWidth, + height: captureHeight, + }) + storedDimensionsRef.current = { width: captureWidth, height: captureHeight } + + const demandRef = useRef(0) + // Tiers that failed to load or turned out EXIF-rotated; skipped for the + // rest of this capture's ladder so promotion does not retry in a loop. + const unusableSrcsRef = useRef>(new Set()) + const promoteTimerRef = useRef>() + const commitTimerRef = useRef>() + + const selectTier = useCallback((demand: number) => { + const usable = tiersRef.current.filter( + (tier) => !unusableSrcsRef.current.has(tier.src) + ) + return pickTier(usable, demand) + }, []) + + // Reset the ladder when navigating to another capture. While the capture is + // loading (original undefined) the previous image stays displayed, matching + // the pre-ladder behavior of keeping the old frame under the spinner. + useEffect(() => { + if (!original) { + return + } + clearTimeout(promoteTimerRef.current) + clearTimeout(commitTimerRef.current) + unusableSrcsRef.current = new Set() + setIncoming(null) + setIncomingLoaded(false) + // Demand is already known when navigating between captures; on first mount + // it is 0 until the container is measured, and updateDemand picks the + // initial tier as soon as the measurement arrives. + setDisplayed(demandRef.current > 0 ? selectTier(demandRef.current) : null) + }, [original, selectTier]) + + const evaluatePromotion = useCallback(() => { + const current = displayedRef.current + const target = selectTier(demandRef.current) + if (!current || !target) { + return + } + + const ladder = tiersRef.current + const currentIndex = ladder.findIndex((tier) => tier.src === current.src) + const targetIndex = ladder.findIndex((tier) => tier.src === target.src) + + if (targetIndex > currentIndex) { + if (incomingRef.current?.src !== target.src) { + clearTimeout(commitTimerRef.current) + setIncoming(target) + setIncomingLoaded(false) + } + } else if (incomingRef.current && !incomingLoadedRef.current) { + // Zoomed back out before the upgrade arrived — drop the request. A tier + // that already finished loading is committed regardless (never demote). + setIncoming(null) + } + }, [selectTier]) + + const updateDemand = useCallback( + (demand: number) => { + demandRef.current = demand + + if (!displayedRef.current) { + const initial = selectTier(demand) + if (initial) { + setDisplayed(initial) + } + return + } + + clearTimeout(promoteTimerRef.current) + promoteTimerRef.current = setTimeout( + evaluatePromotion, + PROMOTE_DEBOUNCE_MS + ) + }, + [evaluatePromotion, selectTier] + ) + + const onIncomingLoad = useCallback((image: HTMLImageElement) => { + const tier = incomingRef.current + if (!tier) { + return + } + + // A browser-applied EXIF rotation would misalign the detection boxes (the + // bug PR #1374 fixed), so refuse the upgrade and stay on the current tier. + if ( + isTransposed( + { width: image.naturalWidth, height: image.naturalHeight }, + storedDimensionsRef.current + ) + ) { + unusableSrcsRef.current.add(tier.src) + setIncoming(null) + setIncomingLoaded(false) + return + } + + setIncomingLoaded(true) + commitTimerRef.current = setTimeout(() => { + setDisplayed(tier) + setIncoming(null) + setIncomingLoaded(false) + }, CROSSFADE_MS + COMMIT_BUFFER_MS) + }, []) + + const onIncomingError = useCallback(() => { + const tier = incomingRef.current + if (!tier) { + return + } + unusableSrcsRef.current.add(tier.src) + setIncoming(null) + setIncomingLoaded(false) + }, []) + + useEffect( + () => () => { + clearTimeout(promoteTimerRef.current) + clearTimeout(commitTimerRef.current) + }, + [] + ) + + return { + displayed, + incoming, + incomingLoaded, + updateDemand, + onIncomingLoad, + onIncomingError, + } +} diff --git a/ui/src/pages/session-details/session-details.tsx b/ui/src/pages/session-details/session-details.tsx index 09d51d2bf..00ddf0594 100644 --- a/ui/src/pages/session-details/session-details.tsx +++ b/ui/src/pages/session-details/session-details.tsx @@ -169,7 +169,14 @@ const Content = ({ session }: { session: SessionDetails }) => { detections={activeCapture?.detections ?? []} height={activeCapture?.height ?? session.firstCapture.height} showDetections={settings.showDetections} - src={activeCapture?.thumbnailLarge} + sources={ + activeCapture + ? { + ...activeCapture.thumbnailSizes, + original: activeCapture.src, + } + : undefined + } transformRef={transformRef} width={activeCapture?.width ?? session.firstCapture.width} /> diff --git a/ui/src/utils/language.ts b/ui/src/utils/language.ts index f2be6618c..5f16f04f5 100644 --- a/ui/src/utils/language.ts +++ b/ui/src/utils/language.ts @@ -311,6 +311,7 @@ export enum STRING { LATEST_OCCURRENCES, LEAVE_TEAM, LOADING_DATA, + LOADING_HIGHER_RESOLUTION, MACHINE_PREDICTION_SCORE, MACHINE_SUGGESTION, MANAGE_ACCESS_FOR, @@ -719,6 +720,7 @@ const ENGLISH_STRINGS: { [key in STRING]: string } = { [STRING.LATEST_OCCURRENCES]: 'Latest occurrences', [STRING.LEAVE_TEAM]: 'Leave team', [STRING.LOADING_DATA]: 'Loading data', + [STRING.LOADING_HIGHER_RESOLUTION]: 'Loading higher resolution', [STRING.MACHINE_PREDICTION_SCORE]: 'Machine prediction score\n{{score}}', [STRING.MACHINE_SUGGESTION]: 'Machine suggestion', [STRING.MANAGE_ACCESS_FOR]: 'Manage access for {{user}}.',