Add experimental session replay capturing on iOS (UE 5.8+)

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Features
 
+- Add experimental session replay capturing on iOS (UE 5.8+) ([#1462](https://github.com/getsentry/sentry-unreal/pull/1462))
 - Add integration with Session Replay product for native platforms ([#1445](https://github.com/getsentry/sentry-unreal/pull/1445))
 
 ### Fixes

plugin-dev/Source/Sentry/Private/Apple/AppleSentrySubsystem.cpp

@@ -253,10 +253,39 @@ void FAppleSentrySubsystem::InitWithSettings(const USentrySettings* settings, co
 	// Wait synchronously until sentry-cocoa initialization finished in main thread
 	dispatch_group_wait(sentryDispatchGroup, DISPATCH_TIME_FOREVER);
 	dispatch_release(sentryDispatchGroup);
+
+	if (IsEnabled())
+	{
+#ifdef USE_SENTRY_SESSION_REPLAY
+		if (settings->AttachSessionReplay)
+		{
+			SessionReplayId = FGuid::NewGuid().ToString(EGuidFormats::Digits).ToLower();
+
+			SessionReplay = MakeUnique<FSentrySessionReplayRecorder>();
+			if (SessionReplay->Initialize(settings, SessionReplayId, GetReplayPath()))
+			{
+				SetContext(TEXT("replay"), { { TEXT("replay_id"), FSentryVariant(SessionReplayId) } });
+			}
+			else
+			{
+				SessionReplay.Reset();
+				SessionReplayId.Reset();
+			}
+		}
+#endif
+	}
 }
 
 void FAppleSentrySubsystem::Close()
 {
+#ifdef USE_SENTRY_SESSION_REPLAY
+	if (SessionReplay)
+	{
+		SessionReplay->Shutdown();
+		SessionReplay.Reset();
+	}
+#endif
+
 	[SENTRY_APPLE_CLASS(SentryObjCSDK) flush:0];
 	[SENTRY_APPLE_CLASS(SentryObjCSDK) close];
 }
@@ -799,4 +828,13 @@ FString FAppleSentrySubsystem::GetLatestSessionReplay() const
 	return Replays[0];
 }
 
+#ifdef USE_SENTRY_SESSION_REPLAY
+FString FAppleSentrySubsystem::GetReplayPath() const
+{
+	const FString ReplayId = FGuid::NewGuid().ToString(EGuidFormats::DigitsWithHyphens).ToLower();
+	const FString ReplayPath = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("SentryReplays"), FString::Printf(TEXT("replay-%s.mp4"), *ReplayId));
+	return FPaths::ConvertRelativePathToFull(ReplayPath);
+}
+#endif
+
 #endif // !USE_SENTRY_NATIVE

plugin-dev/Source/Sentry/Private/Apple/AppleSentrySubsystem.h

@@ -6,6 +6,10 @@
 
 #include "Interface/SentrySubsystemInterface.h"
 
+#ifdef USE_SENTRY_SESSION_REPLAY
+#include "SessionReplay/SentrySessionReplayRecorder.h"
+#endif
+
 class FAppleSentrySubsystem : public ISentrySubsystem
 {
 public:
@@ -80,6 +84,15 @@ class FAppleSentrySubsystem : public ISentrySubsystem
 	int32 maxAttachmentSize = 0;
 
 	FString PrevSessionReplayPath;
+
+private:
+#ifdef USE_SENTRY_SESSION_REPLAY
+	FString GetReplayPath() const;
+
+	FString SessionReplayId;
+
+	TUniquePtr<FSentrySessionReplayRecorder> SessionReplay;
+#endif
 };
 
 #endif // !USE_SENTRY_NATIVE

plugin-dev/Source/Sentry/Private/Mac/MacSentrySubsystem.cpp

@@ -116,31 +116,11 @@ void FMacSentrySubsystem::InitWithSettings(const USentrySettings* settings, cons
 				TryCaptureScreenshot();
 			});
 		}
-
-#ifdef USE_SENTRY_SESSION_REPLAY
-		if (settings->AttachSessionReplay)
-		{
-			SessionReplayId = FGuid::NewGuid().ToString(EGuidFormats::DigitsWithHyphens).ToLower();
-			SessionReplay = MakeUnique<FSentrySessionReplayRecorder>();
-			if (!SessionReplay->Initialize(settings, SessionReplayId, GetReplayPath()))
-			{
-				SessionReplay.Reset();
-			}
-		}
-#endif
 	}
 }
 
 void FMacSentrySubsystem::Close()
 {
-#ifdef USE_SENTRY_SESSION_REPLAY
-	if (SessionReplay)
-	{
-		SessionReplay->Shutdown();
-		SessionReplay.Reset();
-	}
-#endif
-
 	if (OnHandleSystemErrorDelegateHandle.IsValid())
 	{
 		FCoreDelegates::OnHandleSystemError.Remove(OnHandleSystemErrorDelegateHandle);
@@ -150,14 +130,6 @@ void FMacSentrySubsystem::Close()
 	FAppleSentrySubsystem::Close();
 }
 
-#ifdef USE_SENTRY_SESSION_REPLAY
-FString FMacSentrySubsystem::GetReplayPath() const
-{
-	const FString ReplayPath = FPaths::Combine(FPaths::ProjectSavedDir(), TEXT("SentryReplays"), FString::Printf(TEXT("replay-%s.mp4"), *SessionReplayId));
-	return FPaths::ConvertRelativePathToFull(ReplayPath);
-}
-#endif
-
 TSharedPtr<ISentryId> FMacSentrySubsystem::CaptureEnsure(const FString& type, const FString& message)
 {
 	TSharedPtr<ISentryId> id = FAppleSentrySubsystem::CaptureEnsure(type, message);

plugin-dev/Source/Sentry/Private/Mac/MacSentrySubsystem.h

@@ -31,10 +31,6 @@ class FMacSentrySubsystem : public FGenericPlatformSentrySubsystem
 
 #include "Apple/AppleSentrySubsystem.h"
 
-#ifdef USE_SENTRY_SESSION_REPLAY
-#include "SessionReplay/SentrySessionReplayRecorder.h"
-#endif
-
 class FMacSentrySubsystem : public FAppleSentrySubsystem
 {
 public:
@@ -53,14 +49,6 @@ class FMacSentrySubsystem : public FAppleSentrySubsystem
 
 private:
 	FDelegateHandle OnHandleSystemErrorDelegateHandle;
-
-#ifdef USE_SENTRY_SESSION_REPLAY
-	FString GetReplayPath() const;
-
-	FString SessionReplayId;
-
-	TUniquePtr<FSentrySessionReplayRecorder> SessionReplay;
-#endif
 };
 
 #endif

plugin-dev/Source/Sentry/Private/SessionReplay/SentryBackBufferCapture.cpp

@@ -117,12 +117,12 @@ void FSentryBackBufferCapture::CaptureBackBuffer_RenderThread(const FTextureRHIR
 	FTextureRHIRef ScratchTex = AcquireCachedTexture_RenderThread(Scratch, W, H, BackBuffer->GetFormat(),
 		SrvRt, ERHIAccess::SRVGraphics, TEXT("SentrySessionReplayScratch"));
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	FTextureRHIRef ConvertedTex = AcquireCachedTexture_RenderThread(Converted, W, H, PF_B8G8R8A8,
 		SrvRt, ERHIAccess::SRVGraphics, TEXT("SentrySessionReplayConverted"));
 #endif
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	// VT requires pool slot to have CPUReadback flag so it can be read via Metal's getBytes().
 	// Note: Metal forbids RT|CPUReadback on the same texture so an extra copy is needed (step 3)
 	constexpr ETextureCreateFlags PoolFlags = ETextureCreateFlags::CPUReadback;
@@ -149,7 +149,7 @@ void FSentryBackBufferCapture::CaptureBackBuffer_RenderThread(const FTextureRHIR
 		return;
 	}
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	if (!ConvertedTex.IsValid())
 	{
 		return;
@@ -161,7 +161,7 @@ void FSentryBackBufferCapture::CaptureBackBuffer_RenderThread(const FTextureRHIR
 		return;
 	}
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	FTextureRHIRef DrawTarget = ConvertedTex;
 #else
 	FTextureRHIRef DrawTarget = EncoderTex;
@@ -202,8 +202,8 @@ void FSentryBackBufferCapture::CaptureBackBuffer_RenderThread(const FTextureRHIR
 		GraphBuilder.Execute();
 	}
 
-#if PLATFORM_MAC
-	// Step 3 (Mac only): hardware-copy Converted -> EncoderPool slot. Both are BGRA8 same-format
+#if PLATFORM_APPLE
+	// Step 3 (Apple only): hardware-copy Converted -> EncoderPool slot. Both are BGRA8 same-format
 	{
 		FRHITransitionInfo Transitions[] = {
 			FRHITransitionInfo(DrawTarget.GetReference(), ERHIAccess::Unknown, ERHIAccess::CopySrc),
@@ -217,7 +217,7 @@ void FSentryBackBufferCapture::CaptureBackBuffer_RenderThread(const FTextureRHIR
 	}
 #endif
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	RHICmdList.Transition(FRHITransitionInfo(EncoderTex.GetReference(), ERHIAccess::CopyDest, ERHIAccess::CPURead));
 #else
 	RHICmdList.Transition(FRHITransitionInfo(EncoderTex.GetReference(), ERHIAccess::Unknown, ERHIAccess::SRVGraphics));

plugin-dev/Source/Sentry/Private/SessionReplay/SentryBackBufferCapture.h

@@ -28,9 +28,9 @@ class SWindow;
  *   2. AddDrawTexturePass scratch -> a BGRA8 destination. When the scratch
  *      format is already BGRA8 this stays a hardware copy; otherwise the
  *      engine's built-in pixel-shader path converts HDR/10-bit to BGRA8.
- *      Destination is the encoder pool slot on Windows; on Mac it's the
+ *      Destination is the encoder pool slot on Windows; on Apple platforms it's the
  *      "converted" texture because Metal forbids RenderTargetable | CPUReadback.
- *   3. Mac only: hardware-copy converted -> encoder pool slot (CPUReadback BGRA8).
+ *   3. Apple only: hardware-copy converted -> encoder pool slot (CPUReadback BGRA8).
  * The pool slot is then handed to the encoder.
  */
 class FSentryBackBufferCapture
@@ -94,7 +94,7 @@ class FSentryBackBufferCapture
 	// don't carry the SRV flag, so they can't be sampled in a shader directly
 	FCachedTexture Scratch;
 
-	// BGRA8 RenderTargetable texture. Used on Mac as the draw pass output
+	// BGRA8 RenderTargetable texture. Used on Apple platforms as the draw pass output
 	// before the final hardware copy into the CPUReadback EncoderPool slot
 	FCachedTexture Converted;
 

plugin-dev/Source/Sentry/Private/SessionReplay/SentryVideoEncoder.cpp

@@ -8,6 +8,7 @@
 #include "SentrySessionReplayRecorder.h"
 
 #include "HAL/Event.h"
+#include "HAL/PlatformMisc.h"
 #include "HAL/RunnableThread.h"
 #include "Misc/ScopeLock.h"
 #include "RHI.h"
@@ -27,6 +28,9 @@ FSentryVideoEncoder::FSentryVideoEncoder(FSentrySessionReplayRecorder& InRecorde
 	, BitrateBps(InBitrateKbps * 1000)
 	, FragmentSeconds(InFragmentSeconds)
 {
+#if PLATFORM_IOS
+	ExpectedOrientation = FPlatformMisc::GetDeviceOrientation();
+#endif
 }
 
 FSentryVideoEncoder::~FSentryVideoEncoder()
@@ -175,7 +179,7 @@ uint32 FSentryVideoEncoder::Run()
 			}
 
 			// VT interprets SendFrame's timestamp as microseconds (see Engine's VideoEncoderVT.hpp)
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 			static constexpr double SendTimestampScale = 1'000'000.0;
 #else
 			static constexpr double SendTimestampScale = 1'000.0;
@@ -184,8 +188,8 @@ uint32 FSentryVideoEncoder::Run()
 			const double TimestampSeconds = FMath::Max(0.0, Frame.CaptureTimeSeconds - CaptureTimeBaseSeconds);
 			const double ScaledTimestamp = TimestampSeconds * SendTimestampScale;
 
-#if PLATFORM_MAC
-			// Restart the encoder every hour of recording. On Mac this stays well clear of the
+#if PLATFORM_APPLE
+			// Restart the encoder every hour of recording. On Apple platforms this stays well clear of the
 			// uint32-microseconds wrap at ~71 min which would otherwise feed VT a backward PTS
 			// and corrupt all subsequent fragments. On Windows the natural wrap is at ~49 days
 			// so realistically periodic refresh of encoder state won't be needed there
@@ -237,11 +241,33 @@ uint32 FSentryVideoEncoder::Run()
 	return 0;
 }
 
+bool FSentryVideoEncoder::ShouldSwapDimensions(uint32 ResourceWidth, uint32 ResourceHeight) const
+{
+	switch (ExpectedOrientation)
+	{
+	case EDeviceScreenOrientation::Portrait:
+	case EDeviceScreenOrientation::PortraitUpsideDown:
+		return ResourceWidth > ResourceHeight;
+	case EDeviceScreenOrientation::LandscapeLeft:
+	case EDeviceScreenOrientation::LandscapeRight:
+		return ResourceHeight > ResourceWidth;
+	default:
+		return false;
+	}
+}
+
 bool FSentryVideoEncoder::EnsureEncoderOpen(uint32 ResourceWidth, uint32 ResourceHeight)
 {
 	if (bEncoderOpen)
 	{
-		if ((ResourceWidth != Width || ResourceHeight != Height) && !bResolutionChanged)
+		const bool bSameSize = ResourceWidth == Width && ResourceHeight == Height;
+
+		// Transposed frames are expected only when orientation tracking is active (iOS);
+		// elsewhere a transposed size is a genuine resolution change
+		const bool bSameTransposedSize = ExpectedOrientation != EDeviceScreenOrientation::Unknown &&
+										 ResourceWidth == Height && ResourceHeight == Width;
+
+		if (!bSameSize && !bSameTransposedSize && !bResolutionChanged)
 		{
 			UE_LOG(LogSentrySdk, Warning, TEXT("Session replay: capture resolution changed from %ux%u to %ux%u; recording stays locked to the original size and may be cropped or black."),
 				Width, Height, ResourceWidth, ResourceHeight);
@@ -255,6 +281,15 @@ bool FSentryVideoEncoder::EnsureEncoderOpen(uint32 ResourceWidth, uint32 Resourc
 		return false;
 	}
 
+	// If the first frame arrives in the device's native (transposed) dimensions
+	// swap them so the video is written with the correct resolution from the start
+	if (ShouldSwapDimensions(ResourceWidth, ResourceHeight))
+	{
+		UE_LOG(LogSentrySdk, Log, TEXT("Session replay: first frame is %ux%u but the app runs in %s orientation; opening the encoder with swapped dimensions."),
+			ResourceWidth, ResourceHeight, ResourceHeight > ResourceWidth ? TEXT("landscape") : TEXT("portrait"));
+		Swap(ResourceWidth, ResourceHeight);
+	}
+
 	FVideoEncoderConfigH264 Config;
 	Config.Width = ResourceWidth;
 	Config.Height = ResourceHeight;
@@ -270,7 +305,7 @@ bool FSentryVideoEncoder::EnsureEncoderOpen(uint32 ResourceWidth, uint32 Resourc
 	Config.bFillData = 0;
 	Config.MultipassMode = EMultipassMode::Disabled;
 
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 	// Work around a VT bug where H.264 Auto maps to a null EntropyCodingMode
 	// causing a crash in CFStringGetLength. Use CABAC instead (supported by Main/High profiles)
 	Config.EntropyCodingMode = EH264EntropyCodingMode::CABAC;
@@ -393,7 +428,7 @@ void FSentryVideoEncoder::DrainPackets()
 		// when the source renders below the configured target rate). A skipped
 		// packet never updates the marker, so its interval folds into the next
 		// sample. The first sample falls back to a nominal 1/Framerate
-#if PLATFORM_MAC
+#if PLATFORM_APPLE
 		const uint32 PacketTimestampMs = static_cast<uint32>(FPlatformTime::ToMilliseconds64(Packet.Timestamp));
 #else
 		const uint32 PacketTimestampMs = static_cast<uint32>(Packet.Timestamp);

plugin-dev/Source/Sentry/Private/SessionReplay/SentryVideoEncoder.h

@@ -61,6 +61,9 @@ class FSentryVideoEncoder : public FRunnable
 	static constexpr int32 MaxQueueDepth = 5;
 
 private:
+	// Checks if frame dimensions match with the app's fixed screen orientation
+	bool ShouldSwapDimensions(uint32 ResourceWidth, uint32 ResourceHeight) const;
+
 	bool EnsureEncoderOpen(uint32 ResourceWidth, uint32 ResourceHeight);
 
 	// Pulls available packets from the encoder, converts them to AVCC samples and emits a fragment at each keyframe boundary
@@ -72,14 +75,19 @@ class FSentryVideoEncoder : public FRunnable
 	// Tears down the current encoder and resets per-encoder state so the next frame
 	// re-baselines against a fresh VT timestamp origin and republishes a new init
 	// segment. Used to avoid uint32 overflow of the SendFrame timestamp (~71 min of
-	// microseconds on Mac, ~49 days of milliseconds on Windows). Must be called only
+	// microseconds on Apple platforms, ~49 days of milliseconds on Windows). Must be called only
 	// from the encoder thread
 	void Restart();
 
 	FSentrySessionReplayRecorder& Recorder;
 
 	bool bEncoderOpen = false;
 	bool bResolutionChanged = false;
+
+	// Screen orientation the app runs in, captured once at construction (iOS only)
+	// and assumed constant for the session. Unknown disables orientation handling
+	EDeviceScreenOrientation ExpectedOrientation = EDeviceScreenOrientation::Unknown;
+
 	TSharedPtr<TVideoEncoder<FVideoResourceRHI>> Encoder;
 
 	bool bFirstFrameValidated = false;