From 708cd1f8a920aba30f92c029c32f487edc02b8eb Mon Sep 17 00:00:00 2001 From: Ulrik Guenther Date: Tue, 25 Jun 2024 10:19:49 +0200 Subject: [PATCH 1/3] UpdatableTexture: Use CopyOnWriteArrayList for storing updates --- src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt b/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt index c9f555cd9..0a1ae7078 100644 --- a/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt +++ b/src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt @@ -5,6 +5,7 @@ import net.imglib2.type.numeric.integer.UnsignedByteType import org.joml.Vector3i import org.lwjgl.system.MemoryUtil import java.nio.ByteBuffer +import java.util.concurrent.CopyOnWriteArrayList class UpdatableTexture( dimensions: Vector3i, @@ -26,7 +27,7 @@ class UpdatableTexture( data class TextureUpdate(val extents: TextureExtents, val contents: ByteBuffer, var consumed: Boolean = false, var deallocate: Boolean = false) /** List of [TextureUpdate]s for the currently active texture. */ - private var updates: ArrayList = ArrayList() + private var updates: CopyOnWriteArrayList = CopyOnWriteArrayList() fun addUpdate(update: TextureUpdate) { updates.add(update) From 7d0db106e1a516406ea92b860b5bc7e89f6c8bae Mon Sep 17 00:00:00 2001 From: Ulrik Guenther Date: Tue, 25 Jun 2024 10:20:39 +0200 Subject: [PATCH 2/3] VolumeManager: Run update routine asynchronously from renderer (WIP) --- .../graphics/scenery/volumes/VolumeManager.kt | 72 +++++++++++-------- 1 file changed, 43 insertions(+), 29 deletions(-) diff --git a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt index bd5b0cbe6..9feea33a1 100644 --- a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt +++ b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt @@ -29,6 +29,9 @@ import graphics.scenery.attribute.material.Material import graphics.scenery.attribute.renderable.DefaultRenderable import graphics.scenery.attribute.renderable.HasRenderable import graphics.scenery.attribute.renderable.Renderable +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.newFixedThreadPoolContext import net.imglib2.realtransform.AffineTransform3D import net.imglib2.type.numeric.ARGBType import net.imglib2.type.numeric.integer.UnsignedByteType @@ -44,6 +47,10 @@ import java.nio.IntBuffer import java.util.* import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.ForkJoinPool +import java.util.concurrent.locks.ReentrantLock +import kotlin.concurrent.thread +import kotlin.concurrent.withLock +import kotlin.coroutines.CoroutineContext import kotlin.math.max import kotlin.math.min import kotlin.system.measureTimeMillis @@ -682,46 +689,53 @@ class VolumeManager( override fun createRenderable(): Renderable { return object: DefaultRenderable(this) { + val updateLock: ReentrantLock = ReentrantLock() + private val VolumeManagerDispatcher = newFixedThreadPoolContext(1, "VolumeManagerWorker") + /** * Pre-draw routine to be called by the rendered just before drawing. * Updates texture cache and used blocks. */ + @Synchronized override fun preDraw(): Boolean { - logger.debug("Running predraw") - context.bindTexture(textureCache) - - if (nodes.any { it.transferFunction.stale }) { - transferFunctionTextures.clear() - val keys = material().textures.filter { it.key.startsWith("transferFunction") }.keys - keys.forEach { material().textures.remove(it) } - renderStateUpdated = true - } + CoroutineScope(VolumeManagerDispatcher).launch { + updateLock.withLock { + logger.debug("Running predraw") + context.bindTexture(textureCache) + + if(nodes.any { it.transferFunction.stale }) { + transferFunctionTextures.clear() + val keys = material().textures.filter { it.key.startsWith("transferFunction") }.keys + keys.forEach { material().textures.remove(it) } + renderStateUpdated = true + } - if (renderStateUpdated) { - updateRenderState() - needAtLeastNumVolumes(renderStacksStates.size) - renderStateUpdated = false - } + if(renderStateUpdated) { + updateRenderState() + needAtLeastNumVolumes(renderStacksStates.size) + renderStateUpdated = false + } - var repaint = true - val blockUpdateDuration = measureTimeMillis { - if (!freezeRequiredBlocks) { - try { - updateBlocks(context) - } catch (e: RuntimeException) { - logger.warn("Probably ran out of data, corrupt BDV file? $e") - e.printStackTrace() + var repaint = true + val blockUpdateDuration = measureTimeMillis { + if(!freezeRequiredBlocks) { + try { + updateBlocks(context) + } catch(e: RuntimeException) { + logger.warn("Probably ran out of data, corrupt BDV file? $e") + e.printStackTrace() + } + } } - } - } - logger.debug("Block updates took {}ms", blockUpdateDuration) + logger.debug("Block updates took {}ms", blockUpdateDuration) - context.runDeferredBindings() - if (repaint) { - context.runTextureUpdates() + context.runDeferredBindings() + if(repaint) { + context.runTextureUpdates() + } + } } - return readyToRender() } } From bdcce201ca1f264c173aad594c5bca1b2dc5042f Mon Sep 17 00:00:00 2001 From: Kyle Harrington Date: Tue, 2 Jul 2024 22:03:46 -0400 Subject: [PATCH 3/3] wip: an attempt at doing multires volume rendering with just scenery --- .../kotlin/graphics/scenery/volumes/Volume.kt | 24 +++++++++++ .../graphics/scenery/volumes/VolumeManager.kt | 43 +++++++++++++++++++ .../scenery/backends/shaders/Volume.frag | 22 ++++++++-- 3 files changed, 86 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/graphics/scenery/volumes/Volume.kt b/src/main/kotlin/graphics/scenery/volumes/Volume.kt index cd947033f..126502b98 100644 --- a/src/main/kotlin/graphics/scenery/volumes/Volume.kt +++ b/src/main/kotlin/graphics/scenery/volumes/Volume.kt @@ -46,6 +46,7 @@ import net.imagej.ops.OpService import net.imglib2.RandomAccessibleInterval import net.imglib2.Volatile import net.imglib2.histogram.Histogram1d +import net.imglib2.img.cell.CellImg import net.imglib2.realtransform.AffineTransform3D import net.imglib2.type.numeric.ARGBType import net.imglib2.type.numeric.NumericType @@ -315,6 +316,17 @@ open class Volume( } } + var resolutionLevels: List = listOf() + var currentResolutionLevel: Int = 0 + var activeBlocks: MutableSet = mutableSetOf() + + data class ResolutionLevel( + val cellImg: CellImg<*, *>, + val blockDimensions: Vector3i + ) + + data class BlockKey(val level: Int, val position: Vector3i) + /** * Enum class for selecting a rendering method. */ @@ -404,6 +416,18 @@ open class Volume( } } + fun switchResolutionLevel(level: Int, camera: Camera) { + if (level in resolutionLevels.indices) { + currentResolutionLevel = level + activeBlocks.clear() + updateVisibleBlocks(camera) + } + } + + open fun updateVisibleBlocks(camera: Camera) { + // Implement logic to determine which blocks are visible + // and should be loaded for the current resolution level + } override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) { if (fresh !is Volume) throw IllegalArgumentException("Update called with object of foreign class") diff --git a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt index 9feea33a1..a020390d1 100644 --- a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt +++ b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt @@ -927,6 +927,49 @@ class VolumeManager( needAtLeastNumVolumes(renderStacksStates.size) } + class VolumeTextureCache { + private val cache = mutableMapOf() + + fun contains(key: Volume.BlockKey): Boolean = key in cache + + fun put(key: Volume.BlockKey, texture: Texture) { + cache[key] = texture + } + + fun get(key: Volume.BlockKey): Texture? = cache[key] + + fun clear() { + cache.clear() + } + } + + private val volumeTextureCache: VolumeTextureCache = VolumeTextureCache() + + fun update(camera: Camera) { + children.filterIsInstance().forEach { volume -> + val bestLevel = determineBestLevel(volume, camera) + volume.switchResolutionLevel(bestLevel, camera) + loadVisibleBlocks(volume) + updateShaderUniforms(volume)// TODO write this + } + } + + private fun loadVisibleBlocks(volume: Volume) { + val resLevel = volume.resolutionLevels[volume.currentResolutionLevel] + volume.activeBlocks.forEach { blockKey -> + if (!volumeTextureCache.contains(blockKey)) { + val cellImg = resLevel.cellImg + val block = cellImg.getAt(blockKey.position.x, blockKey.position.y, blockKey.position.z) + val texture = Texture(block)// TODO no matching signature + volumeTextureCache.put(blockKey, texture) + } + } + } + + private fun determineBestLevel(volume: Volume, camera: Camera): Int { + // Implement logic to choose the best resolution level + } + /** * Requests re-rendering. */ diff --git a/src/main/resources/graphics/scenery/backends/shaders/Volume.frag b/src/main/resources/graphics/scenery/backends/shaders/Volume.frag index d8f4bfdd9..98f92ed01 100644 --- a/src/main/resources/graphics/scenery/backends/shaders/Volume.frag +++ b/src/main/resources/graphics/scenery/backends/shaders/Volume.frag @@ -77,8 +77,13 @@ layout(set = 5, binding = 0) uniform ShaderProperties { int occlusionSteps; float maxOcclusionDistance; float time; + int numActiveBlocks; + vec3 blockPositions[MAX_BLOCKS]; + vec3 blockDimensions; }; +layout(set = 4, binding = 0) uniform sampler3D volumeBlocks[MAX_BLOCKS]; + layout(push_constant) uniform currentEye_t { int eye; } currentEye; @@ -336,7 +341,7 @@ void main() if(renderingMethod == 0) { float opacity = 1.0f; for(int i = 0; i <= steps; i++, pos += vecstep) { - float volumeSample = texture(VolumeTextures, pos.xyz).r * dataRangeMax; + float volumeSample = sampleVolume(pos.xyz).r * dataRangeMax; newVal = clamp(ta * volumeSample + tb,0.f,1.f); colVal = max(colVal, opacity*newVal); @@ -355,7 +360,7 @@ void main() // Maximum Intensity Projection else if(renderingMethod == 1) { for(int i = 0; i <= steps; i++, pos += vecstep) { - float volumeSample = texture(VolumeTextures, pos.xyz).r * dataRangeMax; + float volumeSample = sampleVolume(pos.xyz).r * dataRangeMax; float newVal = clamp(ta * volumeSample + tb,0.f,1.f); colVal = max(colVal, newVal); } @@ -371,7 +376,7 @@ void main() pos += vec3(random(vec3(Vertex.textureCoord.s, Vertex.textureCoord.t, time)))/10000.0f; for(int i = 0; i <= steps; i++, pos += vecstep) { - float rawSample = texture(VolumeTextures, pos.xyz).r; + float rawSample = sampleVolume(pos.xyz).r; float volumeSample = rawSample * dataRangeMax; float shadowing = 0.0f; volumeSample = clamp(ta * volumeSample + tb,0.f,1.f); @@ -410,3 +415,14 @@ void main() } } +vec4 sampleVolume(vec3 texCoord) { + for (int i = 0; i < numActiveBlocks; i++) { + vec3 blockMin = blockPositions[i]; + vec3 blockMax = blockMin + blockDimensions; + if (all(greaterThanEqual(texCoord, blockMin)) && all(lessThan(texCoord, blockMax))) { + vec3 localTexCoord = (texCoord - blockMin) / blockDimensions; + return texture(volumeBlocks[i], localTexCoord); + } + } + return vec4(0.0); // Return transparent if not in any block +}