Progressive volume rendering in scenery

src/main/kotlin/graphics/scenery/textures/UpdatableTexture.kt

@@ -5,6 +5,7 @@ import net.imglib2.type.numeric.integer.UnsignedByteType
 import org.joml.Vector3i
 import org.lwjgl.system.MemoryUtil
 import java.nio.ByteBuffer
+import java.util.concurrent.CopyOnWriteArrayList
 
 class UpdatableTexture(
     dimensions: Vector3i,
@@ -26,7 +27,7 @@ class UpdatableTexture(
     data class TextureUpdate(val extents: TextureExtents, val contents: ByteBuffer, var consumed: Boolean = false, var deallocate: Boolean = false)
 
     /** List of [TextureUpdate]s for the currently active texture. */
-    private var updates: ArrayList<TextureUpdate> = ArrayList()
+    private var updates: CopyOnWriteArrayList<TextureUpdate> = CopyOnWriteArrayList()
 
     fun addUpdate(update: TextureUpdate) {
         updates.add(update)

src/main/kotlin/graphics/scenery/volumes/Volume.kt

@@ -46,6 +46,7 @@ import net.imagej.ops.OpService
 import net.imglib2.RandomAccessibleInterval
 import net.imglib2.Volatile
 import net.imglib2.histogram.Histogram1d
+import net.imglib2.img.cell.CellImg
 import net.imglib2.realtransform.AffineTransform3D
 import net.imglib2.type.numeric.ARGBType
 import net.imglib2.type.numeric.NumericType
@@ -315,6 +316,17 @@ open class Volume(
         }
     }
 
+    var resolutionLevels: List<ResolutionLevel> = listOf()
+    var currentResolutionLevel: Int = 0
+    var activeBlocks: MutableSet<BlockKey> = mutableSetOf()
+
+    data class ResolutionLevel(
+        val cellImg: CellImg<*, *>,
+        val blockDimensions: Vector3i
+    )
+
+    data class BlockKey(val level: Int, val position: Vector3i)
+
     /**
      * Enum class for selecting a rendering method.
      */
@@ -404,6 +416,18 @@ open class Volume(
         }
     }
 
+    fun switchResolutionLevel(level: Int, camera: Camera) {
+        if (level in resolutionLevels.indices) {
+            currentResolutionLevel = level
+            activeBlocks.clear()
+            updateVisibleBlocks(camera)
+        }
+    }
+
+    open fun updateVisibleBlocks(camera: Camera) {
+        // Implement logic to determine which blocks are visible
+        // and should be loaded for the current resolution level
+    }
 
     override fun update(fresh: Networkable, getNetworkable: (Int) -> Networkable, additionalData: Any?) {
         if (fresh !is Volume) throw IllegalArgumentException("Update called with object of foreign class")

src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt

@@ -29,6 +29,9 @@ import graphics.scenery.attribute.material.Material
 import graphics.scenery.attribute.renderable.DefaultRenderable
 import graphics.scenery.attribute.renderable.HasRenderable
 import graphics.scenery.attribute.renderable.Renderable
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+import kotlinx.coroutines.newFixedThreadPoolContext
 import net.imglib2.realtransform.AffineTransform3D
 import net.imglib2.type.numeric.ARGBType
 import net.imglib2.type.numeric.integer.UnsignedByteType
@@ -44,6 +47,10 @@ import java.nio.IntBuffer
 import java.util.*
 import java.util.concurrent.CopyOnWriteArrayList
 import java.util.concurrent.ForkJoinPool
+import java.util.concurrent.locks.ReentrantLock
+import kotlin.concurrent.thread
+import kotlin.concurrent.withLock
+import kotlin.coroutines.CoroutineContext
 import kotlin.math.max
 import kotlin.math.min
 import kotlin.system.measureTimeMillis
@@ -682,46 +689,53 @@ class VolumeManager(
 
     override fun createRenderable(): Renderable {
         return object: DefaultRenderable(this) {
+            val updateLock: ReentrantLock = ReentrantLock()
+            private val VolumeManagerDispatcher = newFixedThreadPoolContext(1, "VolumeManagerWorker")
+
             /**
              * Pre-draw routine to be called by the rendered just before drawing.
              * Updates texture cache and used blocks.
              */
+            @Synchronized
             override fun preDraw(): Boolean {
-                logger.debug("Running predraw")
-                context.bindTexture(textureCache)
-
-                if (nodes.any { it.transferFunction.stale }) {
-                    transferFunctionTextures.clear()
-                    val keys = material().textures.filter { it.key.startsWith("transferFunction") }.keys
-                    keys.forEach { material().textures.remove(it) }
-                    renderStateUpdated = true
-                }
+                CoroutineScope(VolumeManagerDispatcher).launch {
+                    updateLock.withLock {
+                        logger.debug("Running predraw")
+                        context.bindTexture(textureCache)
+
+                        if(nodes.any { it.transferFunction.stale }) {
+                            transferFunctionTextures.clear()
+                            val keys = material().textures.filter { it.key.startsWith("transferFunction") }.keys
+                            keys.forEach { material().textures.remove(it) }
+                            renderStateUpdated = true
+                        }
 
-                if (renderStateUpdated) {
-                    updateRenderState()
-                    needAtLeastNumVolumes(renderStacksStates.size)
-                    renderStateUpdated = false
-                }
+                        if(renderStateUpdated) {
+                            updateRenderState()
+                            needAtLeastNumVolumes(renderStacksStates.size)
+                            renderStateUpdated = false
+                        }
 
-                var repaint = true
-                val blockUpdateDuration = measureTimeMillis {
-                    if (!freezeRequiredBlocks) {
-                        try {
-                            updateBlocks(context)
-                        } catch (e: RuntimeException) {
-                            logger.warn("Probably ran out of data, corrupt BDV file? $e")
-                            e.printStackTrace()
+                        var repaint = true
+                        val blockUpdateDuration = measureTimeMillis {
+                            if(!freezeRequiredBlocks) {
+                                try {
+                                    updateBlocks(context)
+                                } catch(e: RuntimeException) {
+                                    logger.warn("Probably ran out of data, corrupt BDV file? $e")
+                                    e.printStackTrace()
+                                }
+                            }
                         }
-                    }
-                }
 
-                logger.debug("Block updates took {}ms", blockUpdateDuration)
+                        logger.debug("Block updates took {}ms", blockUpdateDuration)
 
-                context.runDeferredBindings()
-                if (repaint) {
-                    context.runTextureUpdates()
+                        context.runDeferredBindings()
+                        if(repaint) {
+                            context.runTextureUpdates()
+                        }
+                    }
                 }
-
                 return readyToRender()
             }
         }
@@ -913,6 +927,49 @@ class VolumeManager(
         needAtLeastNumVolumes(renderStacksStates.size)
     }
 
+    class VolumeTextureCache {
+        private val cache = mutableMapOf<Volume.BlockKey, Texture>()
+
+        fun contains(key: Volume.BlockKey): Boolean = key in cache
+
+        fun put(key: Volume.BlockKey, texture: Texture) {
+            cache[key] = texture
+        }
+
+        fun get(key: Volume.BlockKey): Texture? = cache[key]
+
+        fun clear() {
+            cache.clear()
+        }
+    }
+
+    private val volumeTextureCache: VolumeTextureCache = VolumeTextureCache()
+
+    fun update(camera: Camera) {
+        children.filterIsInstance<Volume>().forEach { volume ->
+            val bestLevel = determineBestLevel(volume, camera)
+            volume.switchResolutionLevel(bestLevel, camera)
+            loadVisibleBlocks(volume)
+            updateShaderUniforms(volume)// TODO write this
+        }
+    }
+
+    private fun loadVisibleBlocks(volume: Volume) {
+        val resLevel = volume.resolutionLevels[volume.currentResolutionLevel]
+        volume.activeBlocks.forEach { blockKey ->
+            if (!volumeTextureCache.contains(blockKey)) {
+                val cellImg = resLevel.cellImg
+                val block = cellImg.getAt(blockKey.position.x, blockKey.position.y, blockKey.position.z)
+                val texture = Texture(block)// TODO no matching signature
+                volumeTextureCache.put(blockKey, texture)
+            }
+        }
+    }
+
+    private fun determineBestLevel(volume: Volume, camera: Camera): Int {
+        // Implement logic to choose the best resolution level
+    }
+
     /**
      * Requests re-rendering.
      */

src/main/resources/graphics/scenery/backends/shaders/Volume.frag

@@ -77,8 +77,13 @@ layout(set = 5, binding = 0) uniform ShaderProperties {
     int occlusionSteps;
     float maxOcclusionDistance;
     float time;
+    int numActiveBlocks;
+    vec3 blockPositions[MAX_BLOCKS];
+    vec3 blockDimensions;
 };
 
+layout(set = 4, binding = 0) uniform sampler3D volumeBlocks[MAX_BLOCKS];
+
 layout(push_constant) uniform currentEye_t {
     int eye;
 } currentEye;
@@ -336,7 +341,7 @@ void main()
     if(renderingMethod == 0) {
          float opacity = 1.0f;
          for(int i = 0; i <= steps; i++, pos += vecstep) {
-              float volumeSample = texture(VolumeTextures, pos.xyz).r * dataRangeMax;
+              float volumeSample = sampleVolume(pos.xyz).r * dataRangeMax;
               newVal = clamp(ta * volumeSample + tb,0.f,1.f);
               colVal = max(colVal, opacity*newVal);
 
@@ -355,7 +360,7 @@ void main()
     // Maximum Intensity Projection
     else if(renderingMethod == 1) {
          for(int i = 0; i <= steps; i++, pos += vecstep) {
-          float volumeSample = texture(VolumeTextures, pos.xyz).r * dataRangeMax;
+          float volumeSample = sampleVolume(pos.xyz).r * dataRangeMax;
           float newVal = clamp(ta * volumeSample + tb,0.f,1.f);
           colVal = max(colVal, newVal);
         }
@@ -371,7 +376,7 @@ void main()
         pos += vec3(random(vec3(Vertex.textureCoord.s, Vertex.textureCoord.t, time)))/10000.0f;
 
          for(int i = 0; i <= steps; i++, pos += vecstep) {
-            float rawSample = texture(VolumeTextures, pos.xyz).r;
+            float rawSample = sampleVolume(pos.xyz).r;
             float volumeSample = rawSample * dataRangeMax;
             float shadowing = 0.0f;
             volumeSample = clamp(ta * volumeSample + tb,0.f,1.f);
@@ -410,3 +415,14 @@ void main()
     }
 }
 
+vec4 sampleVolume(vec3 texCoord) {
+    for (int i = 0; i < numActiveBlocks; i++) {
+        vec3 blockMin = blockPositions[i];
+        vec3 blockMax = blockMin + blockDimensions;
+        if (all(greaterThanEqual(texCoord, blockMin)) && all(lessThan(texCoord, blockMax))) {
+            vec3 localTexCoord = (texCoord - blockMin) / blockDimensions;
+            return texture(volumeBlocks[i], localTexCoord);
+        }
+    }
+    return vec4(0.0);  // Return transparent if not in any block
+}