diff --git a/src/main/kotlin/graphics/scenery/Camera.kt b/src/main/kotlin/graphics/scenery/Camera.kt index 191316195..414625cc9 100644 --- a/src/main/kotlin/graphics/scenery/Camera.kt +++ b/src/main/kotlin/graphics/scenery/Camera.kt @@ -2,14 +2,16 @@ package graphics.scenery import graphics.scenery.primitives.TextBoard import graphics.scenery.attribute.material.HasMaterial +import graphics.scenery.attribute.populatesubo.DefaultPopulatesUBO +import graphics.scenery.attribute.populatesubo.HasCustomPopulatesUBO +import graphics.scenery.attribute.populatesubo.HasPopulatesUBO +import graphics.scenery.attribute.populatesubo.PopulatesUBO import graphics.scenery.attribute.renderable.HasRenderable import graphics.scenery.attribute.spatial.DefaultSpatial import graphics.scenery.attribute.spatial.HasCustomSpatial +import graphics.scenery.backends.UBO import graphics.scenery.net.Networkable -import graphics.scenery.utils.extensions.minus -import graphics.scenery.utils.extensions.plus -import graphics.scenery.utils.extensions.times -import graphics.scenery.utils.extensions.xyz +import graphics.scenery.utils.extensions.* import org.joml.* import java.lang.Math import java.util.concurrent.atomic.AtomicInteger @@ -26,7 +28,11 @@ import kotlin.math.tan * @constructor Creates a new camera with default position and right-handed * coordinate system. */ -open class Camera : DefaultNode("Camera"), HasRenderable, HasMaterial, HasCustomSpatial { +open class Camera : DefaultNode("Camera"), + HasRenderable, + HasMaterial, + HasCustomSpatial, + HasCustomPopulatesUBO { /** Enum class for camera projection types */ enum class ProjectionType { @@ -88,22 +94,30 @@ open class Camera : DefaultNode("Camera"), HasRenderable, HasMaterial, HasCustom /** Disables culling for this camera. */ var disableCulling: Boolean = false + /** The eye count (aka number of generated viewports) of the current camera. */ + var eyeCount = 1 + protected set + var wantsSync = true override fun wantsSync(): Boolean = wantsSync init { - this.nodeType = "Camera" this.viewSpaceTripod = cameraTripod() this.name = "Camera-${counter.incrementAndGet()}" addSpatial() addRenderable() addMaterial() + addPopulatesUBO() } override fun createSpatial(): CameraSpatial { return CameraSpatial(this) } + override fun createPopulatesUBO(): PopulatesUBO { + return CameraUBOPopulator(this) + } + override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) { if (fresh !is Camera) throw IllegalArgumentException("Update called with object of foreign class") super.update(fresh, getNetworkable, additionalData) @@ -367,6 +381,19 @@ open class Camera : DefaultNode("Camera"), HasRenderable, HasMaterial, HasCustom protected val counter = AtomicInteger(0) } + open class CameraUBOPopulator(open val cam: Camera): DefaultPopulatesUBO() { + override fun populate(ubo: UBO) { + val camSpatial = cam.spatial() + + ubo.add("projection0", { camSpatial.projection.applyVulkanCoordinateSystem() }) + ubo.add("projection1", { camSpatial.projection.applyVulkanCoordinateSystem() }) + ubo.add("inverseProjection0", { camSpatial.projection.applyVulkanCoordinateSystem().invert() }) + ubo.add("inverseProjection1", { camSpatial.projection.applyVulkanCoordinateSystem().invert() }) + ubo.add("headShift", { Matrix4f().identity() }) + ubo.add("IPD", { 0.05f }) + ubo.add("stereoEnabled", { 0 }) + } + } open class CameraSpatial(val camera: Camera): DefaultSpatial(camera) { diff --git a/src/main/kotlin/graphics/scenery/DefaultNode.kt b/src/main/kotlin/graphics/scenery/DefaultNode.kt index 5ed5499b6..872242dd4 100644 --- a/src/main/kotlin/graphics/scenery/DefaultNode.kt +++ b/src/main/kotlin/graphics/scenery/DefaultNode.kt @@ -48,7 +48,6 @@ open class DefaultNode(name: String = "Node") : Node, Networkable { return true } - override var nodeType = "Node" override var boundingBox: OrientedBoundingBox? = null override val logger by lazyLogger() diff --git a/src/main/kotlin/graphics/scenery/DetachedHeadCamera.kt b/src/main/kotlin/graphics/scenery/DetachedHeadCamera.kt index 78d39ee72..65cef8850 100644 --- a/src/main/kotlin/graphics/scenery/DetachedHeadCamera.kt +++ b/src/main/kotlin/graphics/scenery/DetachedHeadCamera.kt @@ -1,7 +1,11 @@ package graphics.scenery +import graphics.scenery.attribute.populatesubo.HasCustomPopulatesUBO +import graphics.scenery.attribute.populatesubo.PopulatesUBO import graphics.scenery.backends.Display +import graphics.scenery.backends.UBO import graphics.scenery.controls.TrackerInput +import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem import graphics.scenery.utils.extensions.plus import graphics.scenery.utils.extensions.times import org.joml.Matrix4f @@ -20,7 +24,11 @@ import kotlin.reflect.KProperty * @author Ulrik Günther */ -class DetachedHeadCamera(@Transient var tracker: TrackerInput? = null) : Camera() { +open class DetachedHeadCamera(@Transient var tracker: TrackerInput? = null) : Camera(), HasCustomPopulatesUBO { + + init { + eyeCount = 2 + } override var width: Int = 0 get() = if (tracker != null && tracker is Display && tracker?.initializedAndWorking() == true) { @@ -104,12 +112,36 @@ class DetachedHeadCamera(@Transient var tracker: TrackerInput? = null) : Camera( val headOrientation: Quaternionf by HeadOrientationDelegate() init { - this.nodeType = "Camera" - this.name = "DetachedHeadCamera-${tracker ?: "${counter.getAndIncrement()}"}" + name = "DetachedHeadCamera-${tracker ?: "${counter.getAndIncrement()}"}" } override fun createSpatial(): CameraSpatial = DetachedHeadCameraSpatial(this) - + + override fun createPopulatesUBO(): PopulatesUBO = DetachedHeadCameraUBOPopulator(this) + open class DetachedHeadCameraUBOPopulator(override val cam: DetachedHeadCamera): Camera.CameraUBOPopulator(cam) { + override fun populate(ubo: UBO) { + val camSpatial = cam.spatial() + val hmd = (cam.tracker as? Display) + + (0 until cam.eyeCount).forEach { eye -> + ubo.add("projection$eye", { + (hmd?.getEyeProjection(eye, cam.nearPlaneDistance, cam.farPlaneDistance) + ?: camSpatial.projection).applyVulkanCoordinateSystem() + }) + ubo.add("inverseProjection$eye", { + (hmd?.getEyeProjection(eye, cam.nearPlaneDistance, cam.farPlaneDistance) + ?: camSpatial.projection).applyVulkanCoordinateSystem().invert() + }) + } + ubo.add("headShift", { hmd?.getHeadToEyeTransform(0) ?: Matrix4f().identity() }) + ubo.add("IPD", { hmd?.getIPD() ?: 0.05f }) + ubo.add("stereoEnabled", { cam.stereoEnabled }) + } + } + + var stereoEnabled = false + internal set + class DetachedHeadCameraSpatial(private val cam: DetachedHeadCamera) : Camera.CameraSpatial(cam) { override var projection: Matrix4f = Matrix4f().identity() diff --git a/src/main/kotlin/graphics/scenery/Node.kt b/src/main/kotlin/graphics/scenery/Node.kt index db4a5b648..9a3cf9774 100644 --- a/src/main/kotlin/graphics/scenery/Node.kt +++ b/src/main/kotlin/graphics/scenery/Node.kt @@ -24,7 +24,6 @@ import kotlin.collections.ArrayList interface Node : Networkable { var name: String - var nodeType: String /** Children of the Node. */ var children: CopyOnWriteArrayList /** Other nodes that have linked transforms. */ diff --git a/src/main/kotlin/graphics/scenery/attribute/populatesubo/DefaultPopulatesUBO.kt b/src/main/kotlin/graphics/scenery/attribute/populatesubo/DefaultPopulatesUBO.kt new file mode 100644 index 000000000..d1ab8bf93 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/attribute/populatesubo/DefaultPopulatesUBO.kt @@ -0,0 +1,8 @@ +package graphics.scenery.attribute.populatesubo + +import graphics.scenery.backends.UBO + +open class DefaultPopulatesUBO: PopulatesUBO { + override fun populate(ubo: UBO) { + } +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasCustomPopulatesUBO.kt b/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasCustomPopulatesUBO.kt new file mode 100644 index 000000000..da062331d --- /dev/null +++ b/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasCustomPopulatesUBO.kt @@ -0,0 +1,15 @@ +package graphics.scenery.attribute.populatesubo + +import graphics.scenery.Node + +interface HasCustomPopulatesUBO: Node { + fun createPopulatesUBO(): PopulatesUBO + + fun addPopulatesUBO() { + addAttribute(PopulatesUBO::class.java, createPopulatesUBO()) + } + + fun populatesUBO(): T { + return getAttribute(PopulatesUBO::class.java) + } +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasPopulatesUBO.kt b/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasPopulatesUBO.kt new file mode 100644 index 000000000..c7ad029ab --- /dev/null +++ b/src/main/kotlin/graphics/scenery/attribute/populatesubo/HasPopulatesUBO.kt @@ -0,0 +1,7 @@ +package graphics.scenery.attribute.populatesubo + +interface HasPopulatesUBO: HasCustomPopulatesUBO { + override fun createPopulatesUBO(): PopulatesUBO { + return DefaultPopulatesUBO() + } +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/attribute/populatesubo/PopulatesUBO.kt b/src/main/kotlin/graphics/scenery/attribute/populatesubo/PopulatesUBO.kt new file mode 100644 index 000000000..cc07b6c47 --- /dev/null +++ b/src/main/kotlin/graphics/scenery/attribute/populatesubo/PopulatesUBO.kt @@ -0,0 +1,7 @@ +package graphics.scenery.attribute.populatesubo + +import graphics.scenery.backends.UBO + +interface PopulatesUBO { + fun populate(ubo: UBO) +} \ No newline at end of file diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt index a02576d58..f1435f3b0 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderer.kt @@ -8,6 +8,7 @@ import graphics.scenery.attribute.renderable.Renderable import graphics.scenery.backends.* import graphics.scenery.textures.Texture import graphics.scenery.utils.* +import graphics.scenery.utils.extensions.applyVulkanCoordinateSystem import kotlinx.coroutines.* import org.joml.* import org.lwjgl.PointerBuffer @@ -28,7 +29,6 @@ import org.lwjgl.vulkan.KHRWin32Surface.VK_KHR_WIN32_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.KHRXlibSurface.VK_KHR_XLIB_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.MVKMacosSurface.VK_MVK_MACOS_SURFACE_EXTENSION_NAME import org.lwjgl.vulkan.VK10.* -import java.awt.BorderLayout import java.awt.image.BufferedImage import java.awt.image.DataBufferByte import java.io.File @@ -39,7 +39,6 @@ import java.util.* import java.util.concurrent.* import java.util.concurrent.locks.ReentrantLock import javax.imageio.ImageIO -import javax.swing.JFrame import kotlin.concurrent.thread import kotlin.concurrent.withLock import kotlin.reflect.full.* @@ -387,12 +386,6 @@ open class VulkanRenderer(hub: Hub, private var renderConfig: RenderConfigReader.RenderConfig private var flow: List = listOf() - private val vulkanProjectionFix = - Matrix4f( - 1.0f, 0.0f, 0.0f, 0.0f, - 0.0f, -1.0f, 0.0f, 0.0f, - 0.0f, 0.0f, 0.5f, 0.0f, - 0.0f, 0.0f, 0.5f, 1.0f) final override var renderConfigFile: String = "" set(config) { @@ -1727,12 +1720,21 @@ open class VulkanRenderer(hub: Hub, profiler?.end() + val cameraConfig = (0 until activeCamera.eyeCount).map { eye -> + VulkanRenderpass.CameraConfig( + view = Matrix4f(activeCamera.spatial().getTransformationForEye(eye)), + projection = Matrix4f(activeCamera.spatial().projection), + eye = eye + ) + }.toMutableList() + flow.take(flow.size - 1).forEachIndexed { i, t -> val si = submitInfo[i] profiler?.begin("Renderer.$t") logger.trace("Running pass {}", t) val target = renderpasses[t]!! val commandBuffer = target.commandBuffer + target.cameraConfiguration = cameraConfig if (commandBuffer.submitted) { commandBuffer.waitForFence() @@ -1793,6 +1795,7 @@ open class VulkanRenderer(hub: Hub, profiler?.begin("Renderer.${renderpasses.keys.last()}") val viewportPass = renderpasses.values.last() val viewportCommandBuffer = viewportPass.commandBuffer + viewportPass.cameraConfiguration = cameraConfig logger.trace("Running viewport pass {}", renderpasses.keys.last()) @@ -2064,14 +2067,6 @@ open class VulkanRenderer(hub: Hub, instanceMasters.isNotEmpty() } - fun Matrix4f.applyVulkanCoordinateSystem(): Matrix4f { - val m = Matrix4f(vulkanProjectionFix) - m.mul(this) - - return m - } - - private fun getDescriptorCache(): TimestampedConcurrentHashMap> { @Suppress("UNCHECKED_CAST") return scene.metadata.getOrPut("DescriptorCache") { @@ -2109,26 +2104,8 @@ open class VulkanRenderer(hub: Hub, buffers.VRParameters.reset() val vrUbo = defaultUBOs["VRParameters"]!! - vrUbo.add("projection0", { - (hmd?.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance) - ?: camSpatial.projection).applyVulkanCoordinateSystem() - }) - vrUbo.add("projection1", { - (hmd?.getEyeProjection(1, cam.nearPlaneDistance, cam.farPlaneDistance) - ?: camSpatial.projection).applyVulkanCoordinateSystem() - }) - vrUbo.add("inverseProjection0", { - (hmd?.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance) - ?: camSpatial.projection).applyVulkanCoordinateSystem().invert() - }) - vrUbo.add("inverseProjection1", { - (hmd?.getEyeProjection(1, cam.nearPlaneDistance, cam.farPlaneDistance) - ?: camSpatial.projection).applyVulkanCoordinateSystem().invert() - }) - vrUbo.add("headShift", { hmd?.getHeadToEyeTransform(0) ?: Matrix4f().identity() }) - vrUbo.add("IPD", { hmd?.getIPD() ?: 0.05f }) - vrUbo.add("stereoEnabled", { renderConfig.stereoEnabled.toInt() }) - + (cam as? DetachedHeadCamera)?.stereoEnabled = renderConfig.stereoEnabled + cam.populatesUBO().populate(vrUbo) updated = vrUbo.populate() buffers.UBOs.reset() diff --git a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt index 8420d1470..a5dfdb43a 100644 --- a/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt +++ b/src/main/kotlin/graphics/scenery/backends/vulkan/VulkanRenderpass.kt @@ -10,6 +10,7 @@ import graphics.scenery.attribute.material.Material import graphics.scenery.backends.* import graphics.scenery.utils.lazyLogger import graphics.scenery.utils.RingBuffer +import org.joml.Matrix4f import org.joml.Vector2f import org.joml.Vector4f import org.lwjgl.system.MemoryUtil.* @@ -107,6 +108,24 @@ open class VulkanRenderpass(val name: String, var config: RenderConfigReader.Ren /** Whether this renderpass will render to the viewport or to a [VulkanFramebuffer] */ var isViewportRenderpass = false + /** Data class for storing this passes' camera matrices. These are stored per-eye of the camera. */ + data class CameraConfig(val view: Matrix4f, val projection: Matrix4f, val eye: Int) + /** A list of this passes' camera configuration, backed by a RingBuffer. + * For each element in the ring buffer, there will be multiple [CameraConfig]s stored, + * each corresponding to an eye of the camera at hand. + */ + var cameraConfiguration: MutableList + get() { + return cameraConfigurationBacking.get() + } + + set(cc) { + cameraConfigurationBacking.put(cc) + } + + private var cameraConfigurationBacking = RingBuffer(size = ringBufferSize, + default = { mutableListOf() }) + /** The number of command buffers to keep in the [RingBuffer] [commandBufferBacking]. */ var commandBufferCount = 3 set(count) { diff --git a/src/main/kotlin/graphics/scenery/utils/Statistics.kt b/src/main/kotlin/graphics/scenery/utils/Statistics.kt index ea1ecf46c..42c828bf5 100644 --- a/src/main/kotlin/graphics/scenery/utils/Statistics.kt +++ b/src/main/kotlin/graphics/scenery/utils/Statistics.kt @@ -6,6 +6,7 @@ import kotlinx.coroutines.GlobalScope import kotlinx.coroutines.asCoroutineDispatcher import kotlinx.coroutines.launch import java.util.* +import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentLinkedDeque import java.util.concurrent.Executors @@ -29,7 +30,7 @@ class Statistics(override var hub: Hub?) : Hubable { */ inner class StatisticData(val isTime: Boolean) { /** Data storage */ - var data: Deque = ConcurrentLinkedDeque() + var data: Queue = ArrayBlockingQueue(dataSize) /** Returns the average of the [data] */ fun avg(): Float = data.sum() / data.size @@ -60,7 +61,7 @@ class Statistics(override var hub: Hub?) : Hubable { } /** Returns all the stats about [data] formatted as string */ - override fun toString(): String = "${avg().inMillisecondsIfTime()}/${min().inMillisecondsIfTime()}/${max().inMillisecondsIfTime()}/${stddev().inMillisecondsIfTime()}/${data.first.inMillisecondsIfTime()}" + override fun toString(): String = "${avg().inMillisecondsIfTime()}/${min().inMillisecondsIfTime()}/${max().inMillisecondsIfTime()}/${stddev().inMillisecondsIfTime()}/${data.first().inMillisecondsIfTime()}" } protected var stats = ConcurrentHashMap() @@ -72,15 +73,12 @@ class Statistics(override var hub: Hub?) : Hubable { fun add(name: String, value: Float, isTime: Boolean = true) { GlobalScope.launch(threadContext.asCoroutineDispatcher()) { stats.computeIfAbsent(name) { - val d = StatisticData(isTime) - d.data.push(value) - d + StatisticData(isTime) }.let { if(it.data.size >= dataSize) { - it.data.removeLast() + it.data.poll() } - - it.data.push(value) + it.data.add(value) } } } diff --git a/src/main/kotlin/graphics/scenery/utils/extensions/VectorMath.kt b/src/main/kotlin/graphics/scenery/utils/extensions/VectorMath.kt index 327b8e586..d182eb69a 100644 --- a/src/main/kotlin/graphics/scenery/utils/extensions/VectorMath.kt +++ b/src/main/kotlin/graphics/scenery/utils/extensions/VectorMath.kt @@ -200,4 +200,17 @@ fun Matrix4f.compare(right: Matrix4fc, explainDiff: Boolean): Boolean { return true } +private val vulkanProjectionFix = + Matrix4f( + 1.0f, 0.0f, 0.0f, 0.0f, + 0.0f, -1.0f, 0.0f, 0.0f, + 0.0f, 0.0f, 0.5f, 0.0f, + 0.0f, 0.0f, 0.5f, 1.0f) +fun Matrix4f.applyVulkanCoordinateSystem(): Matrix4f { + val m = Matrix4f(vulkanProjectionFix) + m.mul(this) + + return m +} +