From 5ec870377c69393c492354efb19681457d1ad235 Mon Sep 17 00:00:00 2001 From: hm21 Date: Thu, 30 Jul 2026 09:52:51 +0200 Subject: [PATCH 1/3] fix: honor EXIF orientation when decoding caller-supplied images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `ChromaKey.backgroundImage` or an image `VideoLayer` given a photo straight from the gallery rendered sideways. A phone stores a portrait shot as landscape pixels plus an EXIF `Orientation` tag, and the decoders on the render path dropped that tag — so the same input came out upright on macOS (`NSImage` bakes the tag in) but rotated 90° on Android (`BitmapFactory`) and iOS (`UIImage` parses the tag, `.cgImage` then hands back the stored pixels). That cross-platform disagreement was the core defect; the contract is now that an encoded image renders the way the user sees it in their photo library. - Android: extract `StopMotionGenerator`'s existing orientation handling into a shared `ImageOrientation` helper and decode the chroma-key background and image layers through it. `probe()` returns the orientation-corrected bounds plus the orientation, which is what `ChromaKeyEffect`'s header-only probe needs to size its background texture. `read()` normalizes anything outside the eight defined values — including the `UNDEFINED` a tagless file reports — to `NORMAL`, so callers get a definite orientation. - Darwin: add `decodeOrientedImage` and use it for both sites. It decodes via `CGImageSource`, which returns the stored pixels on iOS and macOS alike, and applies the tag itself, so the `#if os()` split disappears instead of being patched per half. The extent origin is renormalized to zero, which the overlay positioning already assumes. - Swap `android.media.ExifInterface` for `androidx.exifinterface`: it covers HEIF/AVIF/WebP as well as JPEG, and being plain Java it is reachable from the JVM unit tests, where the framework class is stubbed to return 0. Tests build their fixtures at runtime, because the usual encoders bake the orientation into the pixels and drop the tag: a minimal APP1 EXIF segment is spliced into a JPEG by hand instead. The Swift tests use a four-colour quadrant image so they pin which way the pixels turned, not just that the extent swapped, and both platforms have a counter-test that an untagged image is left alone — otherwise "rotate everything" would satisfy the first assertion. Verified non-vacuous: neutering the Darwin helper fails exactly the two orientation tests while both counter-tests keep passing. The Android unit tests run on the JVM with no Robolectric, where `BitmapFactory` returns null, so `ImageOrientationTest` drives `probeOf` — the seam `probe` hands its header-decoded size to. It covers the real EXIF parse and the size contract; the pixel decode itself is covered by the Swift tests and the example integration tests. --- CHANGELOG.md | 3 + android/build.gradle | 7 +- .../render/helpers/ApplyImageLayer.kt | 16 +- .../render/helpers/ChromaKeyEffect.kt | 36 +-- .../stopmotion/StopMotionGenerator.kt | 95 +----- .../src/shared/media/ImageOrientation.kt | 177 +++++++++++ .../src/shared/media/ImageOrientationTest.kt | 175 +++++++++++ .../render/helpers/DecodeOrientedImage.swift | 48 +++ .../render/utils/VideoCompositor.swift | 37 +-- example/ios/RunnerTests/RunnerTests.swift | 286 ++++++++++++++++++ example/macos/RunnerTests/RunnerTests.swift | 286 ++++++++++++++++++ pubspec.yaml | 2 +- 12 files changed, 1024 insertions(+), 144 deletions(-) create mode 100644 android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt create mode 100644 android/src/test/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientationTest.kt create mode 100644 darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/DecodeOrientedImage.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index f2962d2..66a5b87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 2.11.1 +- **FIX**(android, iOS, macOS): A `ChromaKey.backgroundImage` or image `VideoLayer` fed a photo straight from the gallery no longer renders sideways. Phones store a portrait shot as landscape pixels plus an EXIF `Orientation` tag, and the decoders dropped it — so the same image came out upright on macOS but rotated 90° on Android and iOS. All three now orient on decode. + ## 2.11.0 - **FEAT**(android, iOS, macOS): Add `ChromaKey` — green-screen removal with a soft edge and spill suppression. The keyed area becomes a `backgroundColor`, a `backgroundImage`, or (in a `VideoComposition`) the layer below. Settable on `VideoRenderData`, `VideoLayer` and `VideoSegment`, resolved per clip as segment → layer → global. H.264/HEVC carry no alpha, so on the single-track `videoSegments` path a key without a background is flattened to black. - **FEAT**: `ChromaKey.autoDetect(video)` measures the key color and `similarity` off the footage, for any saturated screen hue. `ChromaKey.detect` returns the raw measurement. Costs one thumbnail decode, no render. diff --git a/android/build.gradle b/android/build.gradle index 4f5e078..9007a1c 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -57,7 +57,12 @@ android { implementation("androidx.media3:media3-inspector-frame:$media3_version") implementation("androidx.media3:media3-effect:$media3_version") implementation("androidx.media3:media3-muxer:$media3_version") - + + // EXIF metadata. Preferred over android.media.ExifInterface: it covers + // HEIF/AVIF/WebP/PNG as well as JPEG, and it is plain Java, so the + // orientation parsing is reachable from the JVM unit tests. + implementation("androidx.exifinterface:exifinterface:1.4.1") + // Coroutines implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt index 97c3862..13a35ee 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt @@ -2,7 +2,6 @@ package ch.waio.pro_video_editor.src.features.render.helpers import RENDER_TAG import android.graphics.Bitmap -import android.graphics.BitmapFactory import android.graphics.Matrix import androidx.media3.common.Effect import androidx.media3.common.util.UnstableApi @@ -15,6 +14,7 @@ import androidx.media3.effect.StaticOverlaySettings import androidx.media3.effect.TimestampWrapper import ch.waio.pro_video_editor.src.features.render.models.ImageLayer import ch.waio.pro_video_editor.src.shared.logging.PluginLog as Log +import ch.waio.pro_video_editor.src.shared.media.ImageOrientation /** * Applies static image overlay on video. @@ -148,12 +148,16 @@ fun applyTimedImageLayers( "Layer: animated GIF with ${gifFrames.size} frame(s), loop=${layer.loop}" ) } else { - val options = BitmapFactory.Options().apply { - inPreferredConfig = Bitmap.Config.ARGB_8888 - } - val layerBitmap = BitmapFactory.decodeByteArray( - imageBytes, 0, imageBytes.size, options + // Decoded through ImageOrientation so a gallery photo carrying an + // EXIF orientation is laid in the way the user sees it, not as the + // sideways pixels it is stored as. + val layerBitmap = ImageOrientation.decode( + imageBytes, config = Bitmap.Config.ARGB_8888 ) + if (layerBitmap == null) { + Log.e(RENDER_TAG, "Failed to decode image layer") + continue + } val prepared = prepareOverlay(layerBitmap, layer, videoWidth, videoHeight) bitmapOverlay = if (hasAnimations) { diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt index 3c24e3f..3282859 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt @@ -15,6 +15,7 @@ import androidx.media3.effect.GlEffect import androidx.media3.effect.GlShaderProgram import ch.waio.pro_video_editor.src.features.render.models.ChromaKeyConfig import ch.waio.pro_video_editor.src.shared.logging.PluginLog as Log +import ch.waio.pro_video_editor.src.shared.media.ImageOrientation /** * Removes a solid-colored background ("green screen") from every frame. @@ -70,17 +71,13 @@ class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { * Bounds-only probe of the background image. * * Decoding just the header tells us whether the bytes are usable — and - * how large they are — without holding a full-size bitmap from - * construction until the first draw. The real decode happens in - * [drawFrame], where there is a GL context to size it against. + * how large they are once their EXIF orientation is honored — without + * holding a full-size bitmap from construction until the first draw. The + * real decode happens in [drawFrame], where there is a GL context to + * size it against. */ - private val backgroundBounds: BitmapFactory.Options? = - config.backgroundImageData?.let { bytes -> - BitmapFactory.Options().apply { - inJustDecodeBounds = true - BitmapFactory.decodeByteArray(bytes, 0, bytes.size, this) - }.takeIf { it.outWidth > 0 && it.outHeight > 0 } - } + private val backgroundProbe: ImageOrientation.Probe? = + config.backgroundImageData?.let { ImageOrientation.probe(it) } /** * Background texture id. A 1x1 placeholder is uploaded when there is no @@ -90,7 +87,7 @@ class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { private var backgroundTexId: Int = -1 private val bgMode: Int = when { - backgroundBounds != null -> BG_IMAGE + backgroundProbe != null -> BG_IMAGE config.backgroundColor != null -> BG_COLOR // An image was asked for but its bytes will not decode. Falling // through to BG_TRANSPARENT would quietly un-key the clip on the @@ -180,7 +177,7 @@ class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { } init { - if (backgroundBounds == null && config.backgroundImageData != null) { + if (backgroundProbe == null && config.backgroundImageData != null) { Log.w( RENDER_TAG, "Chroma key: the background image could not be decoded; " + @@ -287,21 +284,26 @@ class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { */ private fun decodeBackgroundWithinTextureLimit(): Bitmap? { val bytes = config.backgroundImageData ?: return null - val bounds = backgroundBounds ?: return null + val probe = backgroundProbe ?: return null val limit = maxTextureSize() var sampleSize = 1 - while (bounds.outWidth / sampleSize > limit || - bounds.outHeight / sampleSize > limit - ) { + // `probe` is oriented and `inSampleSize` applies to the stored pixels, + // but an orientation only ever exchanges the two dimensions, so the + // pair is over the limit either way round. + while (probe.width / sampleSize > limit || probe.height / sampleSize > limit) { sampleSize *= 2 } - val decoded = BitmapFactory.decodeByteArray( + val raw = BitmapFactory.decodeByteArray( bytes, 0, bytes.size, BitmapFactory.Options().apply { inSampleSize = sampleSize } ) ?: return null + // A portrait photo is stored as landscape pixels plus an EXIF tag; + // without this the background would be keyed in sideways. + val decoded = ImageOrientation.apply(raw, probe.orientation) + // inSampleSize only halves, so one more exact pass may be needed. if (decoded.width <= limit && decoded.height <= limit) return decoded val scale = minOf( diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/stopmotion/StopMotionGenerator.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/stopmotion/StopMotionGenerator.kt index 71a81aa..159c7d3 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/stopmotion/StopMotionGenerator.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/stopmotion/StopMotionGenerator.kt @@ -3,9 +3,6 @@ package ch.waio.pro_video_editor.src.features.stopmotion import RENDER_TAG import android.content.Context import android.graphics.Bitmap -import android.graphics.BitmapFactory -import android.graphics.Matrix -import android.media.ExifInterface import android.net.Uri import android.os.Handler import android.os.Looper @@ -30,7 +27,7 @@ import applyBitrate import ch.waio.pro_video_editor.src.features.render.models.RenderJobHandle import ch.waio.pro_video_editor.src.features.stopmotion.models.StopMotionConfig import ch.waio.pro_video_editor.src.shared.logging.PluginLog as Log -import java.io.ByteArrayInputStream +import ch.waio.pro_video_editor.src.shared.media.ImageOrientation import java.io.File import java.util.concurrent.atomic.AtomicBoolean import java.util.concurrent.atomic.AtomicReference @@ -105,9 +102,9 @@ class StopMotionGenerator(private val context: Context) { var targetWidth = config.width ?: 0 var targetHeight = config.height ?: 0 if (targetWidth <= 0 || targetHeight <= 0) { - val (w, h) = orientedBounds(config.frames[0].imageData) - targetWidth = w - targetHeight = h + val probe = ImageOrientation.probe(config.frames[0].imageData) + targetWidth = probe?.width ?: 2 + targetHeight = probe?.height ?: 2 } targetWidth = evenize(targetWidth) targetHeight = evenize(targetHeight) @@ -117,7 +114,8 @@ class StopMotionGenerator(private val context: Context) { config.frames.forEachIndexed { index, frame -> if (shouldStopPolling.get()) return@Thread - val bitmap = decodeOriented(frame.imageData, targetWidth, targetHeight) + val bitmap = ImageOrientation + .decode(frame.imageData, targetWidth, targetHeight) ?: throw IllegalStateException("Failed to decode frame $index") val file = File( @@ -298,87 +296,6 @@ class StopMotionGenerator(private val context: Context) { }) } - /** Returns the orientation-corrected pixel size of an encoded image. */ - private fun orientedBounds(data: ByteArray): Pair { - val opts = BitmapFactory.Options().apply { inJustDecodeBounds = true } - BitmapFactory.decodeByteArray(data, 0, data.size, opts) - val w = if (opts.outWidth > 0) opts.outWidth else 2 - val h = if (opts.outHeight > 0) opts.outHeight else 2 - return when (readExifOrientation(data)) { - ExifInterface.ORIENTATION_ROTATE_90, - ExifInterface.ORIENTATION_ROTATE_270, - ExifInterface.ORIENTATION_TRANSPOSE, - ExifInterface.ORIENTATION_TRANSVERSE -> Pair(h, w) - else -> Pair(w, h) - } - } - - /** - * Decodes an encoded image downscaled to roughly [reqW]×[reqH] and applies - * its EXIF orientation, so portrait photos are not rendered sideways. - */ - private fun decodeOriented(data: ByteArray, reqW: Int, reqH: Int): Bitmap? { - val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } - BitmapFactory.decodeByteArray(data, 0, data.size, bounds) - if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null - - val opts = BitmapFactory.Options().apply { - inSampleSize = calcInSampleSize(bounds.outWidth, bounds.outHeight, reqW, reqH) - } - val raw = BitmapFactory.decodeByteArray(data, 0, data.size, opts) ?: return null - return applyOrientation(raw, readExifOrientation(data)) - } - - private fun readExifOrientation(data: ByteArray): Int { - return try { - ExifInterface(ByteArrayInputStream(data)) - .getAttributeInt( - ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL - ) - } catch (e: Exception) { - ExifInterface.ORIENTATION_NORMAL - } - } - - private fun applyOrientation(bitmap: Bitmap, orientation: Int): Bitmap { - val matrix = Matrix() - when (orientation) { - ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) - ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) - ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) - ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) - ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) - ExifInterface.ORIENTATION_TRANSPOSE -> { - matrix.postRotate(90f) - matrix.postScale(-1f, 1f) - } - ExifInterface.ORIENTATION_TRANSVERSE -> { - matrix.postRotate(270f) - matrix.postScale(-1f, 1f) - } - else -> return bitmap - } - val rotated = Bitmap.createBitmap( - bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true - ) - if (rotated != bitmap) bitmap.recycle() - return rotated - } - - /** Largest power-of-two sample size that keeps the image ≥ the target size. */ - private fun calcInSampleSize(srcW: Int, srcH: Int, reqW: Int, reqH: Int): Int { - if (reqW <= 0 || reqH <= 0) return 1 - var sample = 1 - var halfW = srcW / 2 - var halfH = srcH / 2 - while (halfW >= reqW && halfH >= reqH) { - sample *= 2 - halfW /= 2 - halfH /= 2 - } - return sample - } - /** Rounds a dimension down to the nearest even value, minimum 2. */ private fun evenize(value: Int): Int { val v = if (value <= 0) 2 else value diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt new file mode 100644 index 0000000..d6be3cc --- /dev/null +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt @@ -0,0 +1,177 @@ +package ch.waio.pro_video_editor.src.shared.media + +import android.graphics.Bitmap +import android.graphics.BitmapFactory +import android.graphics.Matrix +import androidx.exifinterface.media.ExifInterface +import java.io.ByteArrayInputStream + +/** + * EXIF-aware decoding of caller-supplied encoded images. + * + * A photo straight from a phone's gallery stores a portrait shot as *landscape* + * pixels plus an EXIF `Orientation` tag saying how to turn them. [BitmapFactory] + * decodes the pixels and drops the tag, so the image renders sideways. + * + * Every path that takes encoded bytes from the caller — the chroma-key + * background, image layers, stop-motion frames — decodes through here, so they + * agree with each other and with Darwin's `decodeOrientedImage`, which + * normalizes the same way. The contract is: an encoded image renders the way + * the user sees it in their photo library. + */ +internal object ImageOrientation { + + /** The eight orientations EXIF defines; anything else means "no transform". */ + private val DEFINED_ORIENTATIONS = setOf( + ExifInterface.ORIENTATION_NORMAL, + ExifInterface.ORIENTATION_FLIP_HORIZONTAL, + ExifInterface.ORIENTATION_ROTATE_180, + ExifInterface.ORIENTATION_FLIP_VERTICAL, + ExifInterface.ORIENTATION_TRANSPOSE, + ExifInterface.ORIENTATION_ROTATE_90, + ExifInterface.ORIENTATION_TRANSVERSE, + ExifInterface.ORIENTATION_ROTATE_270, + ) + + /** An encoded image's displayed size together with its EXIF orientation. */ + data class Probe(val width: Int, val height: Int, val orientation: Int) + + /** + * Header-only probe: the **orientation-corrected** pixel size of [data] plus + * the orientation itself, without holding a full-size bitmap. + * + * Returns null when the bytes do not decode. + */ + fun probe(data: ByteArray): Probe? { + val opts = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(data, 0, data.size, opts) + if (opts.outWidth <= 0 || opts.outHeight <= 0) return null + return probeOf(opts.outWidth, opts.outHeight, data) + } + + /** + * [probe] for stored dimensions that are already known, skipping the header + * decode. + */ + fun probeOf(storedWidth: Int, storedHeight: Int, data: ByteArray): Probe { + val orientation = read(data) + val (width, height) = orientedSize(storedWidth, storedHeight, orientation) + return Probe(width, height, orientation) + } + + /** + * The EXIF orientation stored in [data], always one of the eight defined + * values. + * + * A file with no tag reports `ORIENTATION_UNDEFINED`, and unreadable bytes + * report nothing at all; both come back as [ExifInterface.ORIENTATION_NORMAL] + * so callers get a definite orientation rather than a value they have to + * range-check themselves. + */ + fun read(data: ByteArray): Int { + val orientation = try { + ExifInterface(ByteArrayInputStream(data)).getAttributeInt( + ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL + ) + } catch (e: Exception) { + ExifInterface.ORIENTATION_NORMAL + } + return if (orientation in DEFINED_ORIENTATIONS) { + orientation + } else { + ExifInterface.ORIENTATION_NORMAL + } + } + + /** Whether [orientation] exchanges the image's width and height. */ + fun swapsDimensions(orientation: Int): Boolean = when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90, + ExifInterface.ORIENTATION_ROTATE_270, + ExifInterface.ORIENTATION_TRANSPOSE, + ExifInterface.ORIENTATION_TRANSVERSE -> true + + else -> false + } + + /** [width]×[height] as it is displayed once [orientation] has been applied. */ + fun orientedSize(width: Int, height: Int, orientation: Int): Pair = + if (swapsDimensions(orientation)) Pair(height, width) else Pair(width, height) + + /** + * Decodes [data] and applies its EXIF orientation, so portrait photos are not + * rendered sideways. + * + * When both [reqWidth] and [reqHeight] are positive the bitmap is decoded + * downscaled toward that size instead of at full resolution. [config], when + * given, is passed on as `inPreferredConfig`. + */ + fun decode( + data: ByteArray, + reqWidth: Int = 0, + reqHeight: Int = 0, + config: Bitmap.Config? = null, + ): Bitmap? { + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(data, 0, data.size, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null + + val opts = BitmapFactory.Options().apply { + inSampleSize = sampleSizeFor(bounds.outWidth, bounds.outHeight, reqWidth, reqHeight) + if (config != null) inPreferredConfig = config + } + val raw = BitmapFactory.decodeByteArray(data, 0, data.size, opts) ?: return null + return apply(raw, read(data)) + } + + /** + * Applies [orientation] to [bitmap]. + * + * Returns a new bitmap and recycles [bitmap] when the orientation calls for a + * transform; returns [bitmap] untouched otherwise (including for the + * "undefined" orientation a file without the tag reports). + */ + fun apply(bitmap: Bitmap, orientation: Int): Bitmap { + val matrix = Matrix() + when (orientation) { + ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) + ExifInterface.ORIENTATION_ROTATE_180 -> matrix.postRotate(180f) + ExifInterface.ORIENTATION_ROTATE_270 -> matrix.postRotate(270f) + ExifInterface.ORIENTATION_FLIP_HORIZONTAL -> matrix.postScale(-1f, 1f) + ExifInterface.ORIENTATION_FLIP_VERTICAL -> matrix.postScale(1f, -1f) + ExifInterface.ORIENTATION_TRANSPOSE -> { + matrix.postRotate(90f) + matrix.postScale(-1f, 1f) + } + + ExifInterface.ORIENTATION_TRANSVERSE -> { + matrix.postRotate(270f) + matrix.postScale(-1f, 1f) + } + + else -> return bitmap + } + val transformed = Bitmap.createBitmap( + bitmap, 0, 0, bitmap.width, bitmap.height, matrix, true + ) + if (transformed !== bitmap) bitmap.recycle() + return transformed + } + + /** + * Largest power-of-two sample size that keeps the image ≥ the target size. + * + * A non-positive target means "no downscale", i.e. a sample size of 1. + */ + fun sampleSizeFor(srcWidth: Int, srcHeight: Int, reqWidth: Int, reqHeight: Int): Int { + if (reqWidth <= 0 || reqHeight <= 0) return 1 + var sample = 1 + var halfWidth = srcWidth / 2 + var halfHeight = srcHeight / 2 + while (halfWidth >= reqWidth && halfHeight >= reqHeight) { + sample *= 2 + halfWidth /= 2 + halfHeight /= 2 + } + return sample + } +} diff --git a/android/src/test/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientationTest.kt b/android/src/test/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientationTest.kt new file mode 100644 index 0000000..c4c6159 --- /dev/null +++ b/android/src/test/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientationTest.kt @@ -0,0 +1,175 @@ +package ch.waio.pro_video_editor.src.shared.media + +import androidx.exifinterface.media.ExifInterface +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertTrue + +/** + * Pins the EXIF-orientation contract for caller-supplied images. + * + * A phone stores a portrait photo as *landscape* pixels plus an `Orientation` + * tag; every path that takes encoded bytes from the caller — the chroma-key + * background, image layers, stop-motion frames — decodes through + * [ImageOrientation] so the image renders the way the user sees it in their + * photo library. iOS and macOS pin the same contract for `decodeOrientedImage` + * in `DecodeOrientedImageTests`. + * + * The fixtures are built here rather than checked in because the usual encoders + * bake the orientation into the pixels and drop the tag, so a "landscape pixels + * + Orientation=6" image cannot be produced by round-tripping one; the APP1 EXIF + * segment is written out by hand in [jpegWithOrientation]. + * + * Scope: this is a JVM unit test, where `BitmapFactory` returns null for + * everything, so it drives [ImageOrientation.probeOf] — the seam + * [ImageOrientation.probe] hands its header-decoded size to, and the one + * `ChromaKeyEffect` sizes its background texture from — with the stored size + * supplied directly. The pixel decode itself is covered on-device by the Swift + * tests and the example integration tests. + */ +internal class ImageOrientationTest { + + /** A stored frame that is landscape, as a portrait phone photo is. */ + private val storedWidth = 1600 + private val storedHeight = 1200 + + @Test + fun taggedImageReportsItsOrientation() { + val tagged = jpegWithOrientation(ExifInterface.ORIENTATION_ROTATE_90) + + assertEquals(ExifInterface.ORIENTATION_ROTATE_90, ImageOrientation.read(tagged)) + } + + @Test + fun taggedImageSwapsTheDimensionsOfTheStoredPixels() { + val tagged = jpegWithOrientation(ExifInterface.ORIENTATION_ROTATE_90) + + val probe = ImageOrientation.probeOf(storedWidth, storedHeight, tagged) + + assertEquals(storedHeight, probe.width) + assertEquals(storedWidth, probe.height) + assertEquals(ExifInterface.ORIENTATION_ROTATE_90, probe.orientation) + } + + /** + * The counter-test: without it, "rotate everything" would satisfy + * [taggedImageSwapsTheDimensionsOfTheStoredPixels]. + */ + @Test + fun untaggedImageIsNotRotated() { + val untagged = jpegWithOrientation(null) + + val probe = ImageOrientation.probeOf(storedWidth, storedHeight, untagged) + + assertEquals(storedWidth, probe.width) + assertEquals(storedHeight, probe.height) + assertEquals(ExifInterface.ORIENTATION_NORMAL, probe.orientation) + assertFalse(ImageOrientation.swapsDimensions(probe.orientation)) + } + + /** + * Bytes that are not an image, and tags outside the eight defined values, + * must not be treated as rotated. + */ + @Test + fun unreadableOrNonsenseOrientationsFallBackToNoRotation() { + val cases = mapOf( + "garbage" to byteArrayOf(0x00, 0x01, 0x02, 0x03), + "empty" to ByteArray(0), + "out-of-range tag" to jpegWithOrientation(42), + ) + + for ((label, bytes) in cases) { + assertEquals(ExifInterface.ORIENTATION_NORMAL, ImageOrientation.read(bytes), label) + val probe = ImageOrientation.probeOf(storedWidth, storedHeight, bytes) + assertEquals(storedWidth, probe.width, label) + assertEquals(storedHeight, probe.height, label) + } + } + + @Test + fun onlyQuarterTurnsExchangeWidthAndHeight() { + val swapping = listOf( + ExifInterface.ORIENTATION_ROTATE_90, + ExifInterface.ORIENTATION_ROTATE_270, + ExifInterface.ORIENTATION_TRANSPOSE, + ExifInterface.ORIENTATION_TRANSVERSE, + ) + val keeping = listOf( + ExifInterface.ORIENTATION_UNDEFINED, + ExifInterface.ORIENTATION_NORMAL, + ExifInterface.ORIENTATION_ROTATE_180, + ExifInterface.ORIENTATION_FLIP_HORIZONTAL, + ExifInterface.ORIENTATION_FLIP_VERTICAL, + ) + + for (orientation in swapping) { + assertTrue(ImageOrientation.swapsDimensions(orientation), "orientation $orientation") + assertEquals( + Pair(storedHeight, storedWidth), + ImageOrientation.orientedSize(storedWidth, storedHeight, orientation), + "orientation $orientation" + ) + } + for (orientation in keeping) { + assertFalse(ImageOrientation.swapsDimensions(orientation), "orientation $orientation") + assertEquals( + Pair(storedWidth, storedHeight), + ImageOrientation.orientedSize(storedWidth, storedHeight, orientation), + "orientation $orientation" + ) + } + } + + /** + * Every one of the eight EXIF orientations must round-trip through the APP1 + * segment, so a photo tagged with any of them is read as itself. + */ + @Test + fun everyOrientationRoundTripsThroughTheExifSegment() { + for (orientation in 1..8) { + assertEquals( + orientation, + ImageOrientation.read(jpegWithOrientation(orientation)), + "orientation $orientation" + ) + } + } + + @Test + fun sampleSizeHalvesUntilTheTargetWouldBeUndershot() { + // 1600×1200 into 400×300: two halvings land exactly on the target, a + // third would fall below it. + assertEquals(4, ImageOrientation.sampleSizeFor(1600, 1200, 400, 300)) + assertEquals(1, ImageOrientation.sampleSizeFor(1600, 1200, 1600, 1200)) + // A non-positive target means "full resolution". + assertEquals(1, ImageOrientation.sampleSizeFor(1600, 1200, 0, 0)) + } + + /** + * A JPEG container carrying nothing but an APP1 EXIF segment declaring + * [orientation] — or no segment at all when it is null. + * + * Only the metadata matters here: the JVM test harness has no `BitmapFactory` + * to decode pixels with, and [ImageOrientation.read] looks at the APP1 + * segment alone. + */ + private fun jpegWithOrientation(orientation: Int?): ByteArray { + val soi = byteArrayOf(0xFF.toByte(), 0xD8.toByte()) + val eoi = byteArrayOf(0xFF.toByte(), 0xD9.toByte()) + val app1 = if (orientation == null) ByteArray(0) else byteArrayOf( + 0xFF.toByte(), 0xE1.toByte(), 0x00, 0x22, // APP1, length 34 = 2 + 32 + 0x45, 0x78, 0x69, 0x66, 0x00, 0x00, // "Exif\0\0" + 0x4D, 0x4D, 0x00, 0x2A, // TIFF header, big-endian + 0x00, 0x00, 0x00, 0x08, // IFD0 sits 8 bytes in + 0x00, 0x01, // one entry + 0x01, 0x12, // tag 0x0112: Orientation + 0x00, 0x03, // type 3: SHORT + 0x00, 0x00, 0x00, 0x01, // count 1 + 0x00, orientation.toByte(), 0x00, 0x00, // the value, left-aligned + 0x00, 0x00, 0x00, 0x00, // no next IFD + ) + return soi + app1 + eoi + } +} diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/DecodeOrientedImage.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/DecodeOrientedImage.swift new file mode 100644 index 0000000..2a7dfd4 --- /dev/null +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/helpers/DecodeOrientedImage.swift @@ -0,0 +1,48 @@ +import CoreGraphics +import CoreImage +import Foundation +import ImageIO + +/// Decodes encoded image `data` into a `CIImage` with its EXIF orientation applied. +/// +/// Callers hand the plugin whatever their photo library gave them, and a phone +/// stores a portrait shot as *landscape* pixels plus an `Orientation` tag saying +/// how to turn them. The obvious decoders disagree about that tag: +/// `UIImage(data:)` parses it but `.cgImage` hands back the stored pixels — so an +/// image built from it renders sideways — while `NSImage` bakes it in. Same +/// input, different output per platform. +/// +/// `CGImageSource` is used instead because it never applies the tag on either +/// platform: it returns the stored pixels, and the orientation is applied here. +/// So iOS and macOS agree with each other and with Android, which normalizes the +/// same way in `ImageOrientation`. The contract is: an encoded image renders the +/// way the user sees it in their photo library. +/// +/// Returns nil when the bytes do not decode. +func decodeOrientedImage(_ data: Data) -> CIImage? { + guard let source = CGImageSourceCreateWithData(data as CFData, nil), + let cgImage = CGImageSourceCreateImageAtIndex(source, 0, nil) + else { return nil } + + let image = CIImage(cgImage: cgImage) + let orientation = exifOrientation(of: source) + guard orientation != .up else { return image } + + // The rotation can push the extent off its original origin, and callers read + // `extent` as a plain size sitting at that origin, so put it back. + let oriented = image.oriented(orientation) + return oriented.transformed( + by: CGAffineTransform( + translationX: image.extent.origin.x - oriented.extent.origin.x, + y: image.extent.origin.y - oriented.extent.origin.y)) +} + +/// The EXIF orientation `source` declares, or `.up` when it declares none. +private func exifOrientation(of source: CGImageSource) -> CGImagePropertyOrientation { + guard + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], + let raw = properties[kCGImagePropertyOrientation] as? UInt32, + let orientation = CGImagePropertyOrientation(rawValue: raw) + else { return .up } + return orientation +} diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift index 2d88396..cb53525 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/render/utils/VideoCompositor.swift @@ -3,12 +3,6 @@ import CoreImage import Foundation import ImageIO -#if os(iOS) - import UIKit -#elseif os(macOS) - import AppKit -#endif - struct ImageLayer { /// Decoded frames: one for a static image, several for an animated GIF. let frames: [CIImage] @@ -268,21 +262,10 @@ class VideoCompositor: NSObject, AVVideoCompositing { frameEndsUs = gif.frameEndsUs totalDurationUs = gif.totalUs } else { - // Static image: decode the single frame. - #if os(iOS) - guard let uiImage = UIImage(data: layer.imageData), - let cgImage = uiImage.cgImage - else { - continue - } - #elseif os(macOS) - guard let nsImage = NSImage(data: layer.imageData), - let cgImage = nsImage.cgImage(forProposedRect: nil, context: nil, hints: nil) - else { - continue - } - #endif - frames = [CIImage(cgImage: cgImage)] + // Static image: decode the single frame, honoring its EXIF orientation + // so a gallery photo is laid in upright rather than sideways. + guard let image = decodeOrientedImage(layer.imageData) else { continue } + frames = [image] frameEndsUs = [0] totalDurationUs = 0 } @@ -495,15 +478,9 @@ class VideoCompositor: NSObject, AVVideoCompositing { guard let data = config.backgroundImageData else { return nil } if let cached = chromaBackgroundCache[config.cacheKey] { return cached } - #if os(iOS) - guard let image = UIImage(data: data), let cgImage = image.cgImage else { return nil } - #elseif os(macOS) - guard let image = NSImage(data: data), - let cgImage = image.cgImage(forProposedRect: nil, context: nil, hints: nil) - else { return nil } - #endif - - let ciImage = CIImage(cgImage: cgImage) + // Oriented on decode: a portrait background photo is stored as landscape + // pixels plus an EXIF tag, and would otherwise be keyed in rotated 90°. + guard let ciImage = decodeOrientedImage(data) else { return nil } chromaBackgroundCache[config.cacheKey] = ciImage return ciImage } diff --git a/example/ios/RunnerTests/RunnerTests.swift b/example/ios/RunnerTests/RunnerTests.swift index 105db37..f51953e 100644 --- a/example/ios/RunnerTests/RunnerTests.swift +++ b/example/ios/RunnerTests/RunnerTests.swift @@ -508,3 +508,289 @@ enum ThumbnailTimestampFixture { return bestIndex } } + + +// MARK: - EXIF orientation on caller-supplied images + +/// `decodeOrientedImage` must honor the EXIF `Orientation` tag. +/// +/// A phone stores a portrait photo as *landscape* pixels plus a tag saying how +/// to turn them, and the chroma-key background and image layers are fed exactly +/// those bytes. This is the decode that keeps them upright, and that keeps iOS, +/// macOS and Android agreeing on the same input — Android pins the same contract +/// in `ImageOrientationTest`. +/// +/// The fixtures are built here rather than checked in because the usual encoders +/// bake the orientation into the pixels and drop the tag, so a "landscape pixels +/// + Orientation=6" image cannot be produced by round-tripping one. The APP1 +/// EXIF segment is spliced in by hand instead; see +/// `OrientedImageFixture.tagging(_:orientation:)`. +class DecodeOrientedImageTests: XCTestCase { + + /// Stored (pre-orientation) size: deliberately landscape, so an orientation + /// that is honored is visible as a portrait result. + private let storedWidth = 32 + private let storedHeight = 16 + + /// EXIF 6: "the 0th row is the visual right side" — display by rotating the + /// stored pixels 90° clockwise. + private let rotate90 = OrientedImageFixture.orientationRotate90 + + private func untaggedJpeg() throws -> Data { + try XCTUnwrap(OrientedImageFixture.quadrantJpeg(width: storedWidth, height: storedHeight)) + } + + private func taggedJpeg() throws -> Data { + OrientedImageFixture.tagging(try untaggedJpeg(), orientation: rotate90) + } + + /// Guards the hand-written APP1 segment itself. Without this, a splice that + /// silently produced no tag would make every "not rotated" assertion below + /// pass for the wrong reason. + func testFixtureCarriesTheOrientationTagItClaimsTo() throws { + XCTAssertEqual(OrientedImageFixture.declaredOrientation(try taggedJpeg()), rotate90) + XCTAssertEqual(OrientedImageFixture.declaredOrientation(try untaggedJpeg()), nil) + } + + func testExifOrientationSwapsTheDecodedDimensions() throws { + let image = try XCTUnwrap(decodeOrientedImage(try taggedJpeg())) + + // Swapped relative to the pixels actually stored in the file. + XCTAssertEqual(image.extent.width, CGFloat(storedHeight)) + XCTAssertEqual(image.extent.height, CGFloat(storedWidth)) + XCTAssertEqual(image.extent.origin, .zero) + } + + /// The counter-test: without it, "rotate everything" would satisfy the + /// assertion above. + func testUntaggedImageIsNotRotated() throws { + let image = try XCTUnwrap(decodeOrientedImage(try untaggedJpeg())) + + XCTAssertEqual(image.extent.width, CGFloat(storedWidth)) + XCTAssertEqual(image.extent.height, CGFloat(storedHeight)) + XCTAssertEqual(image.extent.origin, .zero) + } + + /// Extent alone would also be satisfied by a decode that swapped the size and + /// left the pixels where they were, so check where the quadrants landed. + /// + /// Rotating the stored image 90° clockwise sends top-left → top-right, + /// top-right → bottom-right, bottom-right → bottom-left, bottom-left → + /// top-left. + func testExifOrientationRotatesThePixelsAndNotJustTheExtent() throws { + let image = try XCTUnwrap(decodeOrientedImage(try taggedJpeg())) + let quadrants = try XCTUnwrap(OrientedImageFixture.quadrants(of: image)) + + XCTAssertEqual(quadrants.topLeft, .bottomLeft) + XCTAssertEqual(quadrants.topRight, .topLeft) + XCTAssertEqual(quadrants.bottomLeft, .bottomRight) + XCTAssertEqual(quadrants.bottomRight, .topRight) + } + + func testUntaggedImageKeepsItsQuadrantsWhereTheyWere() throws { + let image = try XCTUnwrap(decodeOrientedImage(try untaggedJpeg())) + let quadrants = try XCTUnwrap(OrientedImageFixture.quadrants(of: image)) + + XCTAssertEqual(quadrants.topLeft, .topLeft) + XCTAssertEqual(quadrants.topRight, .topRight) + XCTAssertEqual(quadrants.bottomLeft, .bottomLeft) + XCTAssertEqual(quadrants.bottomRight, .bottomRight) + } + + func testUndecodableBytesReturnNil() { + XCTAssertNil(decodeOrientedImage(Data([0x00, 0x01, 0x02, 0x03]))) + XCTAssertNil(decodeOrientedImage(Data())) + } +} + +// MARK: - EXIF orientation test fixtures + +/// Builds and reads back the EXIF-orientation fixtures. +/// Shared by the iOS and macOS RunnerTests. +enum OrientedImageFixture { + + /// EXIF `Orientation` = 6, i.e. display by rotating the stored pixels 90° CW. + static let orientationRotate90: UInt32 = 6 + + /// Which corner of the *stored* image a color came from. Each quadrant of the + /// fixture gets its own color, so a rotation is readable as a permutation. + enum Corner: CaseIterable { + case topLeft, topRight, bottomLeft, bottomRight + + var rgb: (r: UInt8, g: UInt8, b: UInt8) { + switch self { + case .topLeft: return (255, 0, 0) // red + case .topRight: return (0, 255, 0) // green + case .bottomLeft: return (0, 0, 255) // blue + case .bottomRight: return (255, 255, 0) // yellow + } + } + } + + struct Quadrants { + let topLeft: Corner + let topRight: Corner + let bottomLeft: Corner + let bottomRight: Corner + } + + /// A JPEG whose four quadrants are four distinct flat colors. + /// + /// Built from raw top-down raster bytes, so "row 0 is the top" is a property of + /// the fixture rather than something the test has to assume about a drawing + /// context. Encoded at maximum quality: the quadrant centers are sampled far + /// from the color edges, so what ringing survives cannot flip a classification. + static func quadrantJpeg(width: Int, height: Int) -> Data? { + var raster = [UInt8]() + raster.reserveCapacity(width * height * 4) + for row in 0.. Data { + let app1: [UInt8] = [ + 0xFF, 0xE1, 0x00, 0x22, // APP1, length 34 = 2 + 32 bytes of payload + 0x45, 0x78, 0x69, 0x66, 0x00, 0x00, // "Exif\0\0" + 0x4D, 0x4D, 0x00, 0x2A, // TIFF header, big-endian ("MM") + 0x00, 0x00, 0x00, 0x08, // IFD0 sits 8 bytes in + 0x00, 0x01, // one entry + 0x01, 0x12, // tag 0x0112: Orientation + 0x00, 0x03, // type 3: SHORT + 0x00, 0x00, 0x00, 0x01, // count 1 + 0x00, UInt8(orientation), 0x00, 0x00, // the value, left-aligned + 0x00, 0x00, 0x00, 0x00, // no next IFD + ] + + var bytes = [UInt8](jpeg) + bytes.insert(contentsOf: app1, at: app1InsertionPoint(bytes)) + return Data(bytes) + } + + /// The EXIF orientation `jpeg` declares, or nil when it declares none. + static func declaredOrientation(_ jpeg: Data) -> UInt32? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return nil } + return properties[kCGImagePropertyOrientation] as? UInt32 + } + + /// Classifies the four quadrant centers of `image` back to the stored corner + /// each color came from. + static func quadrants(of image: CIImage) -> Quadrants? { + guard let raster = raster(of: image) else { return nil } + let (pixels, width, height) = raster + guard width >= 2, height >= 2 else { return nil } + + func corner(atFractionX fx: Double, y fy: Double) -> Corner { + let col = min(width - 1, Int(Double(width) * fx)) + let row = min(height - 1, Int(Double(height) * fy)) + let offset = (row * width + col) * 4 + return nearestCorner((pixels[offset], pixels[offset + 1], pixels[offset + 2])) + } + + return Quadrants( + topLeft: corner(atFractionX: 0.25, y: 0.25), + topRight: corner(atFractionX: 0.75, y: 0.25), + bottomLeft: corner(atFractionX: 0.25, y: 0.75), + bottomRight: corner(atFractionX: 0.75, y: 0.75)) + } + + /// Which corner of the stored image the pixel at (`col`, `row`) belongs to, + /// with row 0 the top. + private static func corner(col: Int, row: Int, width: Int, height: Int) -> Corner { + let isLeft = col < width / 2 + let isTop = row < height / 2 + if isTop { return isLeft ? .topLeft : .topRight } + return isLeft ? .bottomLeft : .bottomRight + } + + /// `image`'s pixels in raster order, row 0 the top. + private static func raster(of image: CIImage) -> ( + pixels: [UInt8], width: Int, height: Int + )? { + let context = CIContext(options: [.workingColorSpace: NSNull()]) + guard let cgImage = context.createCGImage(image, from: image.extent) else { return nil } + + let width = cgImage.width + let height = cgImage.height + var pixels = [UInt8](repeating: 0, count: width * height * 4) + guard + let bitmap = CGContext( + data: &pixels, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: width * 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) + else { return nil } + + bitmap.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height)) + return (pixels, width, height) + } + + /// The quadrant color closest to `color`, so JPEG ringing does not matter. + private static func nearestCorner(_ color: (r: UInt8, g: UInt8, b: UInt8)) -> Corner { + var best = Corner.topLeft + var bestDistance = Int.max + for candidate in Corner.allCases { + let rgb = candidate.rgb + let dr = Int(color.r) - Int(rgb.r) + let dg = Int(color.g) - Int(rgb.g) + let db = Int(color.b) - Int(rgb.b) + let distance = dr * dr + dg * dg + db * db + if distance < bestDistance { + bestDistance = distance + best = candidate + } + } + return best + } + + /// Where an APP1 segment may be inserted: after SOI, and after a leading JFIF + /// APP0 if the encoder wrote one — putting it first would leave a JFIF file + /// whose APP0 no longer follows SOI. Being the *first* APP1 is what matters, + /// since that is the one a reader takes. + private static func app1InsertionPoint(_ bytes: [UInt8]) -> Int { + var index = 2 // past SOI (FFD8) + while index + 4 <= bytes.count, bytes[index] == 0xFF, bytes[index + 1] == 0xE0 { + index += 2 + (Int(bytes[index + 2]) << 8 | Int(bytes[index + 3])) + } + return index + } +} diff --git a/example/macos/RunnerTests/RunnerTests.swift b/example/macos/RunnerTests/RunnerTests.swift index 7f93076..61ee921 100644 --- a/example/macos/RunnerTests/RunnerTests.swift +++ b/example/macos/RunnerTests/RunnerTests.swift @@ -783,3 +783,289 @@ class ChromaKeyMathTests: XCTestCase { } } + + +// MARK: - EXIF orientation on caller-supplied images + +/// `decodeOrientedImage` must honor the EXIF `Orientation` tag. +/// +/// A phone stores a portrait photo as *landscape* pixels plus a tag saying how +/// to turn them, and the chroma-key background and image layers are fed exactly +/// those bytes. This is the decode that keeps them upright, and that keeps iOS, +/// macOS and Android agreeing on the same input — Android pins the same contract +/// in `ImageOrientationTest`. +/// +/// The fixtures are built here rather than checked in because the usual encoders +/// bake the orientation into the pixels and drop the tag, so a "landscape pixels +/// + Orientation=6" image cannot be produced by round-tripping one. The APP1 +/// EXIF segment is spliced in by hand instead; see +/// `OrientedImageFixture.tagging(_:orientation:)`. +class DecodeOrientedImageTests: XCTestCase { + + /// Stored (pre-orientation) size: deliberately landscape, so an orientation + /// that is honored is visible as a portrait result. + private let storedWidth = 32 + private let storedHeight = 16 + + /// EXIF 6: "the 0th row is the visual right side" — display by rotating the + /// stored pixels 90° clockwise. + private let rotate90 = OrientedImageFixture.orientationRotate90 + + private func untaggedJpeg() throws -> Data { + try XCTUnwrap(OrientedImageFixture.quadrantJpeg(width: storedWidth, height: storedHeight)) + } + + private func taggedJpeg() throws -> Data { + OrientedImageFixture.tagging(try untaggedJpeg(), orientation: rotate90) + } + + /// Guards the hand-written APP1 segment itself. Without this, a splice that + /// silently produced no tag would make every "not rotated" assertion below + /// pass for the wrong reason. + func testFixtureCarriesTheOrientationTagItClaimsTo() throws { + XCTAssertEqual(OrientedImageFixture.declaredOrientation(try taggedJpeg()), rotate90) + XCTAssertEqual(OrientedImageFixture.declaredOrientation(try untaggedJpeg()), nil) + } + + func testExifOrientationSwapsTheDecodedDimensions() throws { + let image = try XCTUnwrap(decodeOrientedImage(try taggedJpeg())) + + // Swapped relative to the pixels actually stored in the file. + XCTAssertEqual(image.extent.width, CGFloat(storedHeight)) + XCTAssertEqual(image.extent.height, CGFloat(storedWidth)) + XCTAssertEqual(image.extent.origin, .zero) + } + + /// The counter-test: without it, "rotate everything" would satisfy the + /// assertion above. + func testUntaggedImageIsNotRotated() throws { + let image = try XCTUnwrap(decodeOrientedImage(try untaggedJpeg())) + + XCTAssertEqual(image.extent.width, CGFloat(storedWidth)) + XCTAssertEqual(image.extent.height, CGFloat(storedHeight)) + XCTAssertEqual(image.extent.origin, .zero) + } + + /// Extent alone would also be satisfied by a decode that swapped the size and + /// left the pixels where they were, so check where the quadrants landed. + /// + /// Rotating the stored image 90° clockwise sends top-left → top-right, + /// top-right → bottom-right, bottom-right → bottom-left, bottom-left → + /// top-left. + func testExifOrientationRotatesThePixelsAndNotJustTheExtent() throws { + let image = try XCTUnwrap(decodeOrientedImage(try taggedJpeg())) + let quadrants = try XCTUnwrap(OrientedImageFixture.quadrants(of: image)) + + XCTAssertEqual(quadrants.topLeft, .bottomLeft) + XCTAssertEqual(quadrants.topRight, .topLeft) + XCTAssertEqual(quadrants.bottomLeft, .bottomRight) + XCTAssertEqual(quadrants.bottomRight, .topRight) + } + + func testUntaggedImageKeepsItsQuadrantsWhereTheyWere() throws { + let image = try XCTUnwrap(decodeOrientedImage(try untaggedJpeg())) + let quadrants = try XCTUnwrap(OrientedImageFixture.quadrants(of: image)) + + XCTAssertEqual(quadrants.topLeft, .topLeft) + XCTAssertEqual(quadrants.topRight, .topRight) + XCTAssertEqual(quadrants.bottomLeft, .bottomLeft) + XCTAssertEqual(quadrants.bottomRight, .bottomRight) + } + + func testUndecodableBytesReturnNil() { + XCTAssertNil(decodeOrientedImage(Data([0x00, 0x01, 0x02, 0x03]))) + XCTAssertNil(decodeOrientedImage(Data())) + } +} + +// MARK: - EXIF orientation test fixtures + +/// Builds and reads back the EXIF-orientation fixtures. +/// Shared by the iOS and macOS RunnerTests. +enum OrientedImageFixture { + + /// EXIF `Orientation` = 6, i.e. display by rotating the stored pixels 90° CW. + static let orientationRotate90: UInt32 = 6 + + /// Which corner of the *stored* image a color came from. Each quadrant of the + /// fixture gets its own color, so a rotation is readable as a permutation. + enum Corner: CaseIterable { + case topLeft, topRight, bottomLeft, bottomRight + + var rgb: (r: UInt8, g: UInt8, b: UInt8) { + switch self { + case .topLeft: return (255, 0, 0) // red + case .topRight: return (0, 255, 0) // green + case .bottomLeft: return (0, 0, 255) // blue + case .bottomRight: return (255, 255, 0) // yellow + } + } + } + + struct Quadrants { + let topLeft: Corner + let topRight: Corner + let bottomLeft: Corner + let bottomRight: Corner + } + + /// A JPEG whose four quadrants are four distinct flat colors. + /// + /// Built from raw top-down raster bytes, so "row 0 is the top" is a property of + /// the fixture rather than something the test has to assume about a drawing + /// context. Encoded at maximum quality: the quadrant centers are sampled far + /// from the color edges, so what ringing survives cannot flip a classification. + static func quadrantJpeg(width: Int, height: Int) -> Data? { + var raster = [UInt8]() + raster.reserveCapacity(width * height * 4) + for row in 0.. Data { + let app1: [UInt8] = [ + 0xFF, 0xE1, 0x00, 0x22, // APP1, length 34 = 2 + 32 bytes of payload + 0x45, 0x78, 0x69, 0x66, 0x00, 0x00, // "Exif\0\0" + 0x4D, 0x4D, 0x00, 0x2A, // TIFF header, big-endian ("MM") + 0x00, 0x00, 0x00, 0x08, // IFD0 sits 8 bytes in + 0x00, 0x01, // one entry + 0x01, 0x12, // tag 0x0112: Orientation + 0x00, 0x03, // type 3: SHORT + 0x00, 0x00, 0x00, 0x01, // count 1 + 0x00, UInt8(orientation), 0x00, 0x00, // the value, left-aligned + 0x00, 0x00, 0x00, 0x00, // no next IFD + ] + + var bytes = [UInt8](jpeg) + bytes.insert(contentsOf: app1, at: app1InsertionPoint(bytes)) + return Data(bytes) + } + + /// The EXIF orientation `jpeg` declares, or nil when it declares none. + static func declaredOrientation(_ jpeg: Data) -> UInt32? { + guard let source = CGImageSourceCreateWithData(jpeg as CFData, nil), + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any] + else { return nil } + return properties[kCGImagePropertyOrientation] as? UInt32 + } + + /// Classifies the four quadrant centers of `image` back to the stored corner + /// each color came from. + static func quadrants(of image: CIImage) -> Quadrants? { + guard let raster = raster(of: image) else { return nil } + let (pixels, width, height) = raster + guard width >= 2, height >= 2 else { return nil } + + func corner(atFractionX fx: Double, y fy: Double) -> Corner { + let col = min(width - 1, Int(Double(width) * fx)) + let row = min(height - 1, Int(Double(height) * fy)) + let offset = (row * width + col) * 4 + return nearestCorner((pixels[offset], pixels[offset + 1], pixels[offset + 2])) + } + + return Quadrants( + topLeft: corner(atFractionX: 0.25, y: 0.25), + topRight: corner(atFractionX: 0.75, y: 0.25), + bottomLeft: corner(atFractionX: 0.25, y: 0.75), + bottomRight: corner(atFractionX: 0.75, y: 0.75)) + } + + /// Which corner of the stored image the pixel at (`col`, `row`) belongs to, + /// with row 0 the top. + private static func corner(col: Int, row: Int, width: Int, height: Int) -> Corner { + let isLeft = col < width / 2 + let isTop = row < height / 2 + if isTop { return isLeft ? .topLeft : .topRight } + return isLeft ? .bottomLeft : .bottomRight + } + + /// `image`'s pixels in raster order, row 0 the top. + private static func raster(of image: CIImage) -> ( + pixels: [UInt8], width: Int, height: Int + )? { + let context = CIContext(options: [.workingColorSpace: NSNull()]) + guard let cgImage = context.createCGImage(image, from: image.extent) else { return nil } + + let width = cgImage.width + let height = cgImage.height + var pixels = [UInt8](repeating: 0, count: width * height * 4) + guard + let bitmap = CGContext( + data: &pixels, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: width * 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue) + else { return nil } + + bitmap.draw(cgImage, in: CGRect(x: 0, y: 0, width: width, height: height)) + return (pixels, width, height) + } + + /// The quadrant color closest to `color`, so JPEG ringing does not matter. + private static func nearestCorner(_ color: (r: UInt8, g: UInt8, b: UInt8)) -> Corner { + var best = Corner.topLeft + var bestDistance = Int.max + for candidate in Corner.allCases { + let rgb = candidate.rgb + let dr = Int(color.r) - Int(rgb.r) + let dg = Int(color.g) - Int(rgb.g) + let db = Int(color.b) - Int(rgb.b) + let distance = dr * dr + dg * dg + db * db + if distance < bestDistance { + bestDistance = distance + best = candidate + } + } + return best + } + + /// Where an APP1 segment may be inserted: after SOI, and after a leading JFIF + /// APP0 if the encoder wrote one — putting it first would leave a JFIF file + /// whose APP0 no longer follows SOI. Being the *first* APP1 is what matters, + /// since that is the one a reader takes. + private static func app1InsertionPoint(_ bytes: [UInt8]) -> Int { + var index = 2 // past SOI (FFD8) + while index + 4 <= bytes.count, bytes[index] == 0xFF, bytes[index + 1] == 0xE0 { + index += 2 + (Int(bytes[index + 2]) << 8 | Int(bytes[index + 3])) + } + return index + } +} diff --git a/pubspec.yaml b/pubspec.yaml index 9fa7e82..c33c811 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: pro_video_editor description: "A Flutter video editor: Seamlessly enhance your videos with user-friendly editing features." -version: 2.11.0 +version: 2.11.1 homepage: https://github.com/hm21/pro_video_editor/ repository: https://github.com/hm21/pro_video_editor/ documentation: https://github.com/hm21/pro_video_editor/ From 480abc3069e043ca5b6665e3754e1aa01e237547 Mon Sep 17 00:00:00 2001 From: hm21 Date: Thu, 30 Jul 2026 13:04:50 +0200 Subject: [PATCH 2/3] fix: address review findings on EXIF orientation decoding MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Decode image layers no larger than `prepareOverlay` is about to scale them to. A full-resolution decode of a phone photo cost a second full-resolution copy once the EXIF orientation had to be turned, which is where an overlay runs out of memory — and the result was scaled down immediately afterwards anyway. `overlayDecodeSize` mirrors `prepareOverlay`'s own branching so a positioned layer, which is laid out from its own pixel dimensions, still keeps its full resolution. `ImageOrientation.decode` now maps the requested size back into stored space before choosing `inSampleSize`, which measures the stored pixels; on a rotated image the two were compared across mismatched axes and settled one step too conservative. Darwin's stop-motion decode applies the orientation on its fallback too. `CGImageSourceCreateImageAtIndex` returns the stored pixels with the tag dropped, so a failed thumbnail decode put a portrait frame back on its side — the one path left contradicting the contract this branch establishes. Also: `apply` renamed to `orient`, since it took ownership of its argument and read as Kotlin's scope function at the call site; the two identical "failed to decode" log lines separated; `DEFINED_ORIENTATIONS` unboxed; the CHANGELOG entry cut to the user-facing effect; exifinterface to 1.4.2. Tests: `OverlayDecodeSizeTest` pins the decode-size branching against `prepareOverlay`, and a grayscale fixture covers the one case `NSImage` used to hide on macOS — `CGImageSource` hands back the file's own color space, and the render path's `CIContext` does no color management. Android 24 pass, macOS 27, iOS 15, flutter analyze/format/test clean. --- CHANGELOG.md | 2 +- android/build.gradle | 2 +- .../render/helpers/ApplyImageLayer.kt | 35 ++++- .../render/helpers/ChromaKeyEffect.kt | 2 +- .../src/shared/media/ImageOrientation.kt | 34 +++-- .../render/helpers/OverlayDecodeSizeTest.kt | 87 +++++++++++ .../stopmotion/StopMotionGenerator.swift | 20 ++- example/ios/RunnerTests/RunnerTests.swift | 137 +++++++++++++++++- example/macos/RunnerTests/RunnerTests.swift | 137 +++++++++++++++++- 9 files changed, 430 insertions(+), 26 deletions(-) create mode 100644 android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/OverlayDecodeSizeTest.kt diff --git a/CHANGELOG.md b/CHANGELOG.md index 66a5b87..0ed1b39 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 2.11.1 -- **FIX**(android, iOS, macOS): A `ChromaKey.backgroundImage` or image `VideoLayer` fed a photo straight from the gallery no longer renders sideways. Phones store a portrait shot as landscape pixels plus an EXIF `Orientation` tag, and the decoders dropped it — so the same image came out upright on macOS but rotated 90° on Android and iOS. All three now orient on decode. +- **FIX**(android, iOS, macOS): A `ChromaKey.backgroundImage` or image `VideoLayer` fed a photo straight from the gallery no longer renders sideways — the EXIF `Orientation` tag is now honored on all three platforms. ## 2.11.0 - **FEAT**(android, iOS, macOS): Add `ChromaKey` — green-screen removal with a soft edge and spill suppression. The keyed area becomes a `backgroundColor`, a `backgroundImage`, or (in a `VideoComposition`) the layer below. Settable on `VideoRenderData`, `VideoLayer` and `VideoSegment`, resolved per clip as segment → layer → global. H.264/HEVC carry no alpha, so on the single-track `videoSegments` path a key without a background is flattened to black. diff --git a/android/build.gradle b/android/build.gradle index 9007a1c..640b841 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -61,7 +61,7 @@ android { // EXIF metadata. Preferred over android.media.ExifInterface: it covers // HEIF/AVIF/WebP/PNG as well as JPEG, and it is plain Java, so the // orientation parsing is reachable from the JVM unit tests. - implementation("androidx.exifinterface:exifinterface:1.4.1") + implementation("androidx.exifinterface:exifinterface:1.4.2") // Coroutines implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.7.3") diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt index 13a35ee..44d9b34 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ApplyImageLayer.kt @@ -151,11 +151,20 @@ fun applyTimedImageLayers( // Decoded through ImageOrientation so a gallery photo carrying an // EXIF orientation is laid in the way the user sees it, not as the // sideways pixels it is stored as. + // + // `prepareOverlay` scales the result down straight away, so it is + // decoded no larger than it will end up: a full-resolution decode + // of a phone photo costs a second full-resolution copy when the + // orientation has to be turned, which is where an overlay OOMs. + val (reqWidth, reqHeight) = overlayDecodeSize(layer, videoWidth, videoHeight) val layerBitmap = ImageOrientation.decode( - imageBytes, config = Bitmap.Config.ARGB_8888 + imageBytes, + reqWidth = reqWidth, + reqHeight = reqHeight, + config = Bitmap.Config.ARGB_8888, ) if (layerBitmap == null) { - Log.e(RENDER_TAG, "Failed to decode image layer") + Log.e(RENDER_TAG, "Layer: image bytes did not decode; skipping layer") continue } val prepared = prepareOverlay(layerBitmap, layer, videoWidth, videoHeight) @@ -212,6 +221,28 @@ private data class PreparedOverlay( val overlaySettings: StaticOverlaySettings, ) +/** + * The size [prepareOverlay] will first scale a decoded layer down to, in + * displayed pixels, or `0 × 0` when it keeps the image at its natural size. + * + * Mirrors [prepareOverlay]'s own branching, so decoding to this size cannot cost + * resolution the overlay would otherwise have kept. A layer with an explicit + * size is scaled to it; a layer with neither an explicit size nor a position is + * stretched over the whole frame; a positioned layer without an explicit size is + * laid out from its own pixel dimensions and so must not be sampled down. + */ +internal fun overlayDecodeSize( + layer: VideoSequenceBuilder.ImageLayerConfig, + videoWidth: Int, + videoHeight: Int, +): Pair { + val width = layer.width + val height = layer.height + if (width != null && height != null) return Pair(width.toInt(), height.toInt()) + if (layer.x == null && layer.y == null) return Pair(videoWidth, videoHeight) + return Pair(0, 0) +} + /** * Scales, positions, unpremultiplies and rotates a single overlay [rawBitmap] * according to [layer], returning the final bitmap plus its anchor/settings. diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt index 3282859..564f5c9 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/ChromaKeyEffect.kt @@ -302,7 +302,7 @@ class ChromaKeyEffect(private val config: ChromaKeyConfig) : GlEffect { // A portrait photo is stored as landscape pixels plus an EXIF tag; // without this the background would be keyed in sideways. - val decoded = ImageOrientation.apply(raw, probe.orientation) + val decoded = ImageOrientation.orient(raw, probe.orientation) // inSampleSize only halves, so one more exact pass may be needed. if (decoded.width <= limit && decoded.height <= limit) return decoded diff --git a/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt b/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt index d6be3cc..60d980c 100644 --- a/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt +++ b/android/src/main/kotlin/ch/waio/pro_video_editor/src/shared/media/ImageOrientation.kt @@ -22,7 +22,7 @@ import java.io.ByteArrayInputStream internal object ImageOrientation { /** The eight orientations EXIF defines; anything else means "no transform". */ - private val DEFINED_ORIENTATIONS = setOf( + private val DEFINED_ORIENTATIONS = intArrayOf( ExifInterface.ORIENTATION_NORMAL, ExifInterface.ORIENTATION_FLIP_HORIZONTAL, ExifInterface.ORIENTATION_ROTATE_180, @@ -73,7 +73,7 @@ internal object ImageOrientation { ExifInterface(ByteArrayInputStream(data)).getAttributeInt( ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL ) - } catch (e: Exception) { + } catch (_: Exception) { ExifInterface.ORIENTATION_NORMAL } return if (orientation in DEFINED_ORIENTATIONS) { @@ -102,8 +102,11 @@ internal object ImageOrientation { * rendered sideways. * * When both [reqWidth] and [reqHeight] are positive the bitmap is decoded - * downscaled toward that size instead of at full resolution. [config], when - * given, is passed on as `inPreferredConfig`. + * downscaled toward that size instead of at full resolution — worth passing + * whenever the caller is going to scale the result down anyway, since it + * saves both the oversized decode and the oversized rotation copy. The + * request is in *displayed* pixels, i.e. after the orientation. [config], + * when given, is passed on as `inPreferredConfig`. */ fun decode( data: ByteArray, @@ -115,22 +118,31 @@ internal object ImageOrientation { BitmapFactory.decodeByteArray(data, 0, data.size, bounds) if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return null + // `inSampleSize` measures the stored pixels, so the request has to be + // turned back into stored space first — the same swap, since exchanging + // the two dimensions is its own inverse. + val orientation = read(data) + val (srcReqWidth, srcReqHeight) = orientedSize(reqWidth, reqHeight, orientation) + val opts = BitmapFactory.Options().apply { - inSampleSize = sampleSizeFor(bounds.outWidth, bounds.outHeight, reqWidth, reqHeight) + inSampleSize = + sampleSizeFor(bounds.outWidth, bounds.outHeight, srcReqWidth, srcReqHeight) if (config != null) inPreferredConfig = config } val raw = BitmapFactory.decodeByteArray(data, 0, data.size, opts) ?: return null - return apply(raw, read(data)) + return orient(raw, orientation) } /** - * Applies [orientation] to [bitmap]. + * Turns [bitmap] the way [orientation] says it should be displayed. * - * Returns a new bitmap and recycles [bitmap] when the orientation calls for a - * transform; returns [bitmap] untouched otherwise (including for the - * "undefined" orientation a file without the tag reports). + * **Takes ownership of [bitmap].** Returns a new bitmap and recycles + * [bitmap] when the orientation calls for a transform; returns [bitmap] + * untouched otherwise (including for the "undefined" orientation a file + * without the tag reports). Either way the caller must use the return value + * and treat [bitmap] as gone. */ - fun apply(bitmap: Bitmap, orientation: Int): Bitmap { + fun orient(bitmap: Bitmap, orientation: Int): Bitmap { val matrix = Matrix() when (orientation) { ExifInterface.ORIENTATION_ROTATE_90 -> matrix.postRotate(90f) diff --git a/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/OverlayDecodeSizeTest.kt b/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/OverlayDecodeSizeTest.kt new file mode 100644 index 0000000..3830a99 --- /dev/null +++ b/android/src/test/kotlin/ch/waio/pro_video_editor/src/features/render/helpers/OverlayDecodeSizeTest.kt @@ -0,0 +1,87 @@ +package ch.waio.pro_video_editor.src.features.render.helpers + +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Pins [overlayDecodeSize] to the branching [prepareOverlay] actually does. + * + * An image layer is decoded no larger than the size it is about to be scaled + * down to, because a full-resolution decode of a phone photo costs a *second* + * full-resolution copy when its EXIF orientation has to be turned — the point + * where an overlay runs out of memory. That only holds while this function and + * `prepareOverlay` agree on which layers get scaled and to what; if they drift, + * a layer starts being sampled below the resolution it keeps, so the two are + * pinned together here. + */ +internal class OverlayDecodeSizeTest { + + private val videoWidth = 1920 + private val videoHeight = 1080 + + private fun layer( + width: Double? = null, + height: Double? = null, + x: Int? = null, + y: Int? = null, + ) = VideoSequenceBuilder.ImageLayerConfig( + imageBytes = null, + scaleX = null, + scaleY = null, + width = width, + height = height, + x = x, + y = y, + ) + + @Test + fun explicitSizeIsTheDecodeTarget() { + val size = overlayDecodeSize( + layer(width = 480.0, height = 270.0, x = 100, y = 50), videoWidth, videoHeight + ) + + assertEquals(Pair(480, 270), size) + } + + /** An explicit size wins even when the layer is also stretched. */ + @Test + fun explicitSizeWinsOverTheFrame() { + val size = overlayDecodeSize( + layer(width = 480.0, height = 270.0), videoWidth, videoHeight + ) + + assertEquals(Pair(480, 270), size) + } + + @Test + fun aStretchedLayerIsDecodedToTheFrame() { + val size = overlayDecodeSize(layer(), videoWidth, videoHeight) + + assertEquals(Pair(videoWidth, videoHeight), size) + } + + /** + * The case that must *not* downscale: a positioned layer with no explicit + * size is laid out from its own pixel dimensions, so sampling it down would + * shrink the rendered overlay. + */ + @Test + fun aPositionedLayerWithoutASizeKeepsItsOwnResolution() { + assertEquals(Pair(0, 0), overlayDecodeSize(layer(x = 40, y = 80), videoWidth, videoHeight)) + assertEquals(Pair(0, 0), overlayDecodeSize(layer(x = 40), videoWidth, videoHeight)) + assertEquals(Pair(0, 0), overlayDecodeSize(layer(y = 80), videoWidth, videoHeight)) + } + + /** Half a size is no size: both dimensions are needed to scale to one. */ + @Test + fun aHalfSpecifiedSizeIsNotADecodeTarget() { + assertEquals( + Pair(0, 0), + overlayDecodeSize(layer(width = 480.0, x = 10, y = 10), videoWidth, videoHeight) + ) + assertEquals( + Pair(videoWidth, videoHeight), + overlayDecodeSize(layer(height = 270.0), videoWidth, videoHeight) + ) + } +} diff --git a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/stopmotion/StopMotionGenerator.swift b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/stopmotion/StopMotionGenerator.swift index 25a743b..df1ad51 100644 --- a/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/stopmotion/StopMotionGenerator.swift +++ b/darwin/pro_video_editor/Sources/pro_video_editor/src/shared/features/stopmotion/StopMotionGenerator.swift @@ -1,5 +1,6 @@ import AVFoundation import CoreGraphics +import CoreImage import Foundation import ImageIO @@ -215,10 +216,25 @@ internal enum StopMotionGenerator { options[kCGImageSourceThumbnailMaxPixelSize] = maxPixelSize } - return CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) - ?? CGImageSourceCreateImageAtIndex(source, 0, nil) + if let thumbnail = CGImageSourceCreateThumbnailAtIndex(source, 0, options as CFDictionary) { + return thumbnail + } + + // Fallback. `CGImageSourceCreateImageAtIndex` hands back the *stored* pixels + // and drops the tag, so going straight to it would undo the one thing the + // thumbnail above was doing for us and put a portrait frame back on its + // side. `decodeOrientedImage` applies the orientation the same way the + // render path does, at the cost of the full-size decode this path was + // trying to avoid — acceptable, since it only runs when the thumbnail + // decode has already failed. + guard let oriented = decodeOrientedImage(data) else { return nil } + return orientationContext.createCGImage(oriented, from: oriented.extent) } + /// Renders the orientation fallback in [decodeImage]. Kept off the render + /// path's context so a stop-motion job never contends with an export. + private static let orientationContext = CIContext(options: [.workingColorSpace: NSNull()]) + /// Rounds a dimension down to the nearest even value (codec requirement), /// with a minimum of 2. private static func evenize(_ value: Int) -> Int { diff --git a/example/ios/RunnerTests/RunnerTests.swift b/example/ios/RunnerTests/RunnerTests.swift index f51953e..4faeca8 100644 --- a/example/ios/RunnerTests/RunnerTests.swift +++ b/example/ios/RunnerTests/RunnerTests.swift @@ -509,7 +509,6 @@ enum ThumbnailTimestampFixture { } } - // MARK: - EXIF orientation on caller-supplied images /// `decodeOrientedImage` must honor the EXIF `Orientation` tag. @@ -597,6 +596,41 @@ class DecodeOrientedImageTests: XCTestCase { XCTAssertEqual(quadrants.bottomRight, .bottomRight) } + /// The case `NSImage` used to paper over on macOS: it rasterized every source + /// into RGB before the compositor saw it, whereas `CGImageSource` hands back + /// the file's own color space — so a one-component grayscale JPEG now reaches + /// CoreImage as gray, and the render path's `CIContext` does no color + /// management (`workingColorSpace: NSNull`). Pins that such an image still + /// lands on the right pixels, and still turns. + /// + /// Classified by rank, not by value: a transfer function may move the levels, + /// but it must not reorder them, and `grayQuadrants` refuses to guess when two + /// of them come back equal — which is what a flattened or blank decode looks + /// like. + func testGrayscaleImageDecodesAndStillOrients() throws { + let untagged = try XCTUnwrap( + OrientedImageFixture.grayscaleQuadrantJpeg(width: storedWidth, height: storedHeight)) + let tagged = OrientedImageFixture.tagging(untagged, orientation: rotate90) + + let plain = try XCTUnwrap(decodeOrientedImage(untagged)) + XCTAssertEqual(plain.extent.width, CGFloat(storedWidth)) + XCTAssertEqual(plain.extent.height, CGFloat(storedHeight)) + let asStored = try XCTUnwrap(OrientedImageFixture.grayQuadrants(of: plain)) + XCTAssertEqual(asStored.topLeft, .topLeft) + XCTAssertEqual(asStored.topRight, .topRight) + XCTAssertEqual(asStored.bottomLeft, .bottomLeft) + XCTAssertEqual(asStored.bottomRight, .bottomRight) + + let rotated = try XCTUnwrap(decodeOrientedImage(tagged)) + XCTAssertEqual(rotated.extent.width, CGFloat(storedHeight)) + XCTAssertEqual(rotated.extent.height, CGFloat(storedWidth)) + let asDisplayed = try XCTUnwrap(OrientedImageFixture.grayQuadrants(of: rotated)) + XCTAssertEqual(asDisplayed.topLeft, .bottomLeft) + XCTAssertEqual(asDisplayed.topRight, .topLeft) + XCTAssertEqual(asDisplayed.bottomLeft, .bottomRight) + XCTAssertEqual(asDisplayed.bottomRight, .topRight) + } + func testUndecodableBytesReturnNil() { XCTAssertNil(decodeOrientedImage(Data([0x00, 0x01, 0x02, 0x03]))) XCTAssertNil(decodeOrientedImage(Data())) @@ -606,7 +640,10 @@ class DecodeOrientedImageTests: XCTestCase { // MARK: - EXIF orientation test fixtures /// Builds and reads back the EXIF-orientation fixtures. -/// Shared by the iOS and macOS RunnerTests. +/// +/// Duplicated in the iOS and macOS RunnerTests: they are separate test targets +/// with no shared source directory, as `ThumbnailTimestampFixture` already is. +/// Keep the two copies in step. enum OrientedImageFixture { /// EXIF `Orientation` = 6, i.e. display by rotating the stored pixels 90° CW. @@ -625,6 +662,18 @@ enum OrientedImageFixture { case .bottomRight: return (255, 255, 0) // yellow } } + + /// The grayscale fixture's level for this corner. Strictly increasing in + /// the order the cases are declared, which is what lets `grayQuadrants` + /// classify by rank instead of by value. + var gray: UInt8 { + switch self { + case .topLeft: return 16 + case .topRight: return 96 + case .bottomLeft: return 176 + case .bottomRight: return 248 + } + } } struct Quadrants { @@ -638,8 +687,7 @@ enum OrientedImageFixture { /// /// Built from raw top-down raster bytes, so "row 0 is the top" is a property of /// the fixture rather than something the test has to assume about a drawing - /// context. Encoded at maximum quality: the quadrant centers are sampled far - /// from the color edges, so what ringing survives cannot flip a classification. + /// context. static func quadrantJpeg(width: Int, height: Int) -> Data? { var raster = [UInt8]() raster.reserveCapacity(width * height * 4) @@ -665,6 +713,43 @@ enum OrientedImageFixture { intent: .defaultIntent) else { return nil } + return encodeJpeg(image) + } + + /// A one-component grayscale JPEG whose four quadrants are four distinct gray + /// levels — the same layout as `quadrantJpeg`, in a color space that is not + /// RGB. + static func grayscaleQuadrantJpeg(width: Int, height: Int) -> Data? { + var raster = [UInt8]() + raster.reserveCapacity(width * height) + for row in 0.. Data? { let encoded = NSMutableData() guard let destination = CGImageDestinationCreateWithData( @@ -730,6 +815,50 @@ enum OrientedImageFixture { bottomRight: corner(atFractionX: 0.75, y: 0.75)) } + /// Classifies the four quadrant centers of a grayscale `image` back to the + /// stored corner each level came from, by **rank** rather than by value: the + /// decode may put the levels through a transfer function, but it must not + /// reorder them. + /// + /// Returns nil when two centers come back equal — a flattened or blank decode + /// would otherwise be ranked into some arbitrary permutation and could pass by + /// luck. + static func grayQuadrants(of image: CIImage) -> Quadrants? { + guard let raster = raster(of: image) else { return nil } + let (pixels, width, height) = raster + guard width >= 2, height >= 2 else { return nil } + + func level(atFractionX fx: Double, y fy: Double) -> UInt8 { + let col = min(width - 1, Int(Double(width) * fx)) + let row = min(height - 1, Int(Double(height) * fy)) + return pixels[(row * width + col) * 4] + } + + // Positions in the order Quadrants declares them. + let levels = [ + level(atFractionX: 0.25, y: 0.25), + level(atFractionX: 0.75, y: 0.25), + level(atFractionX: 0.25, y: 0.75), + level(atFractionX: 0.75, y: 0.75), + ] + guard Set(levels).count == levels.count else { return nil } + + // Corner.allCases is declared darkest-first, matching Corner.gray. + let darkestFirst = Corner.allCases + var corners = [Corner](repeating: .topLeft, count: levels.count) + for (rank, position) in levels.enumerated() + .sorted(by: { $0.element < $1.element }) + .map({ $0.offset }) + .enumerated() + { + corners[position] = darkestFirst[rank] + } + + return Quadrants( + topLeft: corners[0], topRight: corners[1], + bottomLeft: corners[2], bottomRight: corners[3]) + } + /// Which corner of the stored image the pixel at (`col`, `row`) belongs to, /// with row 0 the top. private static func corner(col: Int, row: Int, width: Int, height: Int) -> Corner { diff --git a/example/macos/RunnerTests/RunnerTests.swift b/example/macos/RunnerTests/RunnerTests.swift index 61ee921..5795943 100644 --- a/example/macos/RunnerTests/RunnerTests.swift +++ b/example/macos/RunnerTests/RunnerTests.swift @@ -784,7 +784,6 @@ class ChromaKeyMathTests: XCTestCase { } - // MARK: - EXIF orientation on caller-supplied images /// `decodeOrientedImage` must honor the EXIF `Orientation` tag. @@ -872,6 +871,41 @@ class DecodeOrientedImageTests: XCTestCase { XCTAssertEqual(quadrants.bottomRight, .bottomRight) } + /// The case `NSImage` used to paper over on macOS: it rasterized every source + /// into RGB before the compositor saw it, whereas `CGImageSource` hands back + /// the file's own color space — so a one-component grayscale JPEG now reaches + /// CoreImage as gray, and the render path's `CIContext` does no color + /// management (`workingColorSpace: NSNull`). Pins that such an image still + /// lands on the right pixels, and still turns. + /// + /// Classified by rank, not by value: a transfer function may move the levels, + /// but it must not reorder them, and `grayQuadrants` refuses to guess when two + /// of them come back equal — which is what a flattened or blank decode looks + /// like. + func testGrayscaleImageDecodesAndStillOrients() throws { + let untagged = try XCTUnwrap( + OrientedImageFixture.grayscaleQuadrantJpeg(width: storedWidth, height: storedHeight)) + let tagged = OrientedImageFixture.tagging(untagged, orientation: rotate90) + + let plain = try XCTUnwrap(decodeOrientedImage(untagged)) + XCTAssertEqual(plain.extent.width, CGFloat(storedWidth)) + XCTAssertEqual(plain.extent.height, CGFloat(storedHeight)) + let asStored = try XCTUnwrap(OrientedImageFixture.grayQuadrants(of: plain)) + XCTAssertEqual(asStored.topLeft, .topLeft) + XCTAssertEqual(asStored.topRight, .topRight) + XCTAssertEqual(asStored.bottomLeft, .bottomLeft) + XCTAssertEqual(asStored.bottomRight, .bottomRight) + + let rotated = try XCTUnwrap(decodeOrientedImage(tagged)) + XCTAssertEqual(rotated.extent.width, CGFloat(storedHeight)) + XCTAssertEqual(rotated.extent.height, CGFloat(storedWidth)) + let asDisplayed = try XCTUnwrap(OrientedImageFixture.grayQuadrants(of: rotated)) + XCTAssertEqual(asDisplayed.topLeft, .bottomLeft) + XCTAssertEqual(asDisplayed.topRight, .topLeft) + XCTAssertEqual(asDisplayed.bottomLeft, .bottomRight) + XCTAssertEqual(asDisplayed.bottomRight, .topRight) + } + func testUndecodableBytesReturnNil() { XCTAssertNil(decodeOrientedImage(Data([0x00, 0x01, 0x02, 0x03]))) XCTAssertNil(decodeOrientedImage(Data())) @@ -881,7 +915,10 @@ class DecodeOrientedImageTests: XCTestCase { // MARK: - EXIF orientation test fixtures /// Builds and reads back the EXIF-orientation fixtures. -/// Shared by the iOS and macOS RunnerTests. +/// +/// Duplicated in the iOS and macOS RunnerTests: they are separate test targets +/// with no shared source directory, as `ThumbnailTimestampFixture` already is. +/// Keep the two copies in step. enum OrientedImageFixture { /// EXIF `Orientation` = 6, i.e. display by rotating the stored pixels 90° CW. @@ -900,6 +937,18 @@ enum OrientedImageFixture { case .bottomRight: return (255, 255, 0) // yellow } } + + /// The grayscale fixture's level for this corner. Strictly increasing in + /// the order the cases are declared, which is what lets `grayQuadrants` + /// classify by rank instead of by value. + var gray: UInt8 { + switch self { + case .topLeft: return 16 + case .topRight: return 96 + case .bottomLeft: return 176 + case .bottomRight: return 248 + } + } } struct Quadrants { @@ -913,8 +962,7 @@ enum OrientedImageFixture { /// /// Built from raw top-down raster bytes, so "row 0 is the top" is a property of /// the fixture rather than something the test has to assume about a drawing - /// context. Encoded at maximum quality: the quadrant centers are sampled far - /// from the color edges, so what ringing survives cannot flip a classification. + /// context. static func quadrantJpeg(width: Int, height: Int) -> Data? { var raster = [UInt8]() raster.reserveCapacity(width * height * 4) @@ -940,6 +988,43 @@ enum OrientedImageFixture { intent: .defaultIntent) else { return nil } + return encodeJpeg(image) + } + + /// A one-component grayscale JPEG whose four quadrants are four distinct gray + /// levels — the same layout as `quadrantJpeg`, in a color space that is not + /// RGB. + static func grayscaleQuadrantJpeg(width: Int, height: Int) -> Data? { + var raster = [UInt8]() + raster.reserveCapacity(width * height) + for row in 0.. Data? { let encoded = NSMutableData() guard let destination = CGImageDestinationCreateWithData( @@ -1005,6 +1090,50 @@ enum OrientedImageFixture { bottomRight: corner(atFractionX: 0.75, y: 0.75)) } + /// Classifies the four quadrant centers of a grayscale `image` back to the + /// stored corner each level came from, by **rank** rather than by value: the + /// decode may put the levels through a transfer function, but it must not + /// reorder them. + /// + /// Returns nil when two centers come back equal — a flattened or blank decode + /// would otherwise be ranked into some arbitrary permutation and could pass by + /// luck. + static func grayQuadrants(of image: CIImage) -> Quadrants? { + guard let raster = raster(of: image) else { return nil } + let (pixels, width, height) = raster + guard width >= 2, height >= 2 else { return nil } + + func level(atFractionX fx: Double, y fy: Double) -> UInt8 { + let col = min(width - 1, Int(Double(width) * fx)) + let row = min(height - 1, Int(Double(height) * fy)) + return pixels[(row * width + col) * 4] + } + + // Positions in the order Quadrants declares them. + let levels = [ + level(atFractionX: 0.25, y: 0.25), + level(atFractionX: 0.75, y: 0.25), + level(atFractionX: 0.25, y: 0.75), + level(atFractionX: 0.75, y: 0.75), + ] + guard Set(levels).count == levels.count else { return nil } + + // Corner.allCases is declared darkest-first, matching Corner.gray. + let darkestFirst = Corner.allCases + var corners = [Corner](repeating: .topLeft, count: levels.count) + for (rank, position) in levels.enumerated() + .sorted(by: { $0.element < $1.element }) + .map({ $0.offset }) + .enumerated() + { + corners[position] = darkestFirst[rank] + } + + return Quadrants( + topLeft: corners[0], topRight: corners[1], + bottomLeft: corners[2], bottomRight: corners[3]) + } + /// Which corner of the stored image the pixel at (`col`, `row`) belongs to, /// with row 0 the top. private static func corner(col: Int, row: Int, width: Int, height: Int) -> Corner { From 56662cbabd46a54ddb9617a6e89a1ef51a04990e Mon Sep 17 00:00:00 2001 From: hm21 Date: Thu, 30 Jul 2026 18:22:44 +0200 Subject: [PATCH 3/3] fix: simplify video composition comments and remove unsupported features --- .../features/render/video_renderer_page.dart | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/example/lib/features/render/video_renderer_page.dart b/example/lib/features/render/video_renderer_page.dart index e8da882..dacada8 100644 --- a/example/lib/features/render/video_renderer_page.dart +++ b/example/lib/features/render/video_renderer_page.dart @@ -275,9 +275,9 @@ class _VideoRendererPageState extends State { /// /// Exercises, in a single render: /// - A 3-layer [VideoComposition] on an explicit canvas with a background. - /// - Base layer: two trimmed clips joined by an intra-layer dissolve - /// transition, the second sped up and at reduced volume. - /// - A reversed, muted picture-in-picture layer that enters after 2s. + /// - Base layer: two trimmed clips played back-to-back, the second at + /// reduced volume. + /// - A muted picture-in-picture layer that enters after 2s. /// - A semi-transparent secondary video placed bottom-left. /// - Image overlays: a timed sticker with fade in/out animations. /// - A timed warm color filter over the first 6 seconds. @@ -299,37 +299,31 @@ class _VideoRendererPageState extends State { canvasSize: meta.resolution, backgroundColor: Colors.black, layers: [ - // Base layer: two trimmed clips, dissolve between them, the second - // sped up to 1.5x at half volume. + // Base layer: two trimmed clips back-to-back, the second at half + // volume. Transitions and playbackSpeed are not supported inside a + // composition — see the transition demos for those. VideoLayer( clips: [ VideoSegment( video: _video, startTime: Duration.zero, endTime: const Duration(seconds: 5), - transition: const ClipTransition( - type: ClipTransitionType.dissolve, - duration: Duration(milliseconds: 800), - curve: AnimationCurve.easeInOut, - ), ), VideoSegment( video: _video, startTime: const Duration(seconds: 10), endTime: const Duration(seconds: 16), - playbackSpeed: 1.5, volume: 0.5, ), ], ), - // Reversed, muted picture-in-picture, top-right, enters at 2s. + // Muted picture-in-picture, top-right, enters at 2s. VideoLayer( clips: [ VideoSegment( video: _video, startTime: const Duration(seconds: 4), endTime: const Duration(seconds: 9), - reverseVideo: true, volume: 0, timelineStart: const Duration(seconds: 2), ),