diff --git a/build.gradle.kts b/build.gradle.kts index f32e0691e..f7067f2fc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -116,7 +116,7 @@ dependencies { api("sc.fiji:bigdataviewer-core:10.4.14") api("sc.fiji:bigdataviewer-vistools:1.0.0-beta-28") - api("sc.fiji:bigvolumeviewer:0.3.3") { + api("sc.fiji:bigvolumeviewer:0.4.1") { exclude("org.jogamp.gluegen", "gluegen-rt") exclude("org.jogamp.jogl", "jogl-all") } diff --git a/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt b/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt index 4d7b3d30c..c6be05c3f 100644 --- a/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt +++ b/src/main/kotlin/graphics/scenery/volumes/SceneryContext.kt @@ -42,8 +42,10 @@ open class SceneryContext(val node: VolumeManager, val useCompute: Boolean = fal /** Factory for the autogenerated shaders. */ val factory = VolumeShaderFactory(useCompute) - /** Reference to the currently bound texture cache. */ - protected var currentlyBoundCache: Texture? = null + /** Reference to the currently bound R8 texture cache. */ + protected var currentlyBoundR8Cache: Texture? = null + /** Reference to the currently bound R16 texture cache. */ + protected var currentlyBoundR16Cache: Texture? = null /** Hashmap for references to the currently bound LUTs/texture atlases. */ protected var currentlyBoundTextures = ConcurrentHashMap() /** Hashmap for storing associations between [Texture] objects, texture slots and uniform names. */ @@ -353,26 +355,70 @@ open class SceneryContext(val node: VolumeManager, val useCompute: Boolean = fal val material = node.material() if (texture is TextureCache) { - if(currentlyBoundCache != null && material.textures["volumeCache"] == currentlyBoundCache && dimensionsMatch(texture, material.textures["volumeCache"])) { - return 0 + val cacheName = bindings[texture]?.uniformName + var db: (String) -> Unit = { _ -> } + if(texture.spec().format() == bvv.core.backend.Texture.InternalFormat.R8) { + db = { name: String -> + + if (!(currentlyBoundR8Cache != null && material.textures[name] == currentlyBoundR8Cache && dimensionsMatch( + texture, + material.textures[name] + )) + ) { + // logger.warn("Binding and updating cache $texture") + val gt = UpdatableTexture( + Vector3i(texture.texWidth(), texture.texHeight(), texture.texDepth()), + channels, + type, + null, + repeat.all(), + BorderColor.TransparentBlack, + normalized, + false, + minFilter = Texture.FilteringMode.Linear, + maxFilter = Texture.FilteringMode.Linear + ) + + material.textures[name] = gt + currentlyBoundR8Cache = gt + } + } + + } + else + { + db = { name: String -> + if (!(currentlyBoundR16Cache != null && material.textures[name] == currentlyBoundR16Cache && dimensionsMatch( + texture, + material.textures[name] + )) + ) { + // logger.warn("Binding and updating cache $texture") + val gt = UpdatableTexture( + Vector3i(texture.texWidth(), texture.texHeight(), texture.texDepth()), + channels, + type, + null, + repeat.all(), + BorderColor.TransparentBlack, + normalized, + false, + minFilter = Texture.FilteringMode.Linear, + maxFilter = Texture.FilteringMode.Linear + ) + + material.textures[name] = gt + currentlyBoundR16Cache = gt + } + } + } + if(cacheName == null) { + deferredBindings[texture] = db + return -1 + } else { + db.invoke(cacheName) } -// logger.warn("Binding and updating cache $texture") - val gt = UpdatableTexture( - Vector3i(texture.texWidth(), texture.texHeight(), texture.texDepth()), - channels, - type, - null, - repeat.all(), - BorderColor.TransparentBlack, - normalized, - false, - minFilter = Texture.FilteringMode.Linear, - maxFilter = Texture.FilteringMode.Linear) - - material.textures["volumeCache"] = gt - - currentlyBoundCache = gt } else { val textureName = bindings[texture]?.uniformName logger.debug("lutName is $textureName for $texture") @@ -385,7 +431,8 @@ open class SceneryContext(val node: VolumeManager, val useCompute: Boolean = fal */ if (!(material.textures[name] != null && currentlyBoundTextures[name] != null - && material.textures[name] == currentlyBoundTextures[name])) { + && material.textures[name] == currentlyBoundTextures[name])) + { val contents = when(texture) { is LookupTextureARGB -> null is VolumeManager.SimpleTexture2D -> texture.data @@ -459,7 +506,8 @@ open class SceneryContext(val node: VolumeManager, val useCompute: Boolean = fal fun clearCacheBindings() { val caches = bindings.filter { it is TextureCache } caches.map { bindings.remove(it.key) } - currentlyBoundCache = null + currentlyBoundR8Cache = null + currentlyBoundR16Cache = null } fun clearBindings() { diff --git a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt index 971fe6d6d..eb2816f36 100644 --- a/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt +++ b/src/main/kotlin/graphics/scenery/volumes/VolumeManager.kt @@ -75,13 +75,33 @@ class VolumeManager( var context = SceneryContext(this, useCompute) protected set - /** Texture cache. */ + private class TextureCacheAndPboChain( + format: Texture.InternalFormat, + blockSize: IntArray, + maxCacheSizeInMB: Int + ) { + val textureCache: TextureCache + val pboChain: PboChain + + init { + val cacheSpec = CacheSpec(format, blockSize) + val cacheGridDimensions = TextureCache.findSuitableGridSize(cacheSpec, maxCacheSizeInMB) + + textureCache = TextureCache(cacheGridDimensions, cacheSpec) + pboChain = PboChain(5, 100, textureCache) + } + + fun textureCache() = textureCache + fun pboChain() = pboChain + } + + /** Texture cache and PBO chain for 8-bit images. */ @Volatile - protected var textureCache: TextureCache + private var cacheR8: TextureCacheAndPboChain - /** PBO chain for temporary data storage. */ + /** Texture cache and PBO chain for 16-bit images. */ @Volatile - protected var pboChain: PboChain + private var cacheR16: TextureCacheAndPboChain /** Flexible [ShaderProperty] storage */ @ShaderProperty @@ -94,9 +114,6 @@ class VolumeManager( protected var transferFunctionTextures = HashMap, Texture>() protected var colorMapTextures = HashMap, Texture>() - /** Cache specification. */ - private val cacheSpec = CacheSpec(Texture.InternalFormat.R16, intArrayOf(32, 32, 32)) - private val renderStacksStates = CopyOnWriteArrayList() private data class StackState( @@ -185,10 +202,9 @@ class VolumeManager( val maxCacheSize = (hub?.get(SceneryElement.Settings) as? Settings)?.get("Renderer.MaxVolumeCacheSize", 512) ?: 512 - val cacheGridDimensions = TextureCache.findSuitableGridSize(cacheSpec, maxCacheSize) - textureCache = TextureCache(cacheGridDimensions, cacheSpec) + cacheR8 = TextureCacheAndPboChain(Texture.InternalFormat.R8,intArrayOf(32, 32, 32),maxCacheSize) - pboChain = PboChain(5, 100, textureCache) + cacheR16 = TextureCacheAndPboChain(Texture.InternalFormat.R16,intArrayOf(32, 32, 32),maxCacheSize) updateRenderState() needAtLeastNumVolumes(renderStacksStates.size) @@ -241,8 +257,7 @@ class VolumeManager( logger.debug("Updating effective shader program to $progvol") recreateMaterial(context) - progvol?.setTextureCache(textureCache) - progvol?.use(context) + progvol?.use(context) progvol?.setUniforms(context) // progvol?.bindSamplers(context) @@ -271,7 +286,7 @@ class VolumeManager( } while (outOfCoreVolumes.size < n) { - outOfCoreVolumes.add(VolumeBlocks(textureCache)) + outOfCoreVolumes.add(VolumeBlocks()) } val signatures = renderStacksStates.map { @@ -319,6 +334,11 @@ class VolumeManager( ) segments[SegmentType.SampleMultiresolutionVolume] = SegmentTemplate( "SampleBlockVolume.frag", + "volumeCache", + "blockSize", + "paddedBlockSize", + "cachePadOffset", + "cacheSize", "im", "sourcemin", "sourcemax", @@ -423,7 +443,6 @@ class VolumeManager( "InputZBuffer" ) - newProgvol.setTextureCache(textureCache) newProgvol.setDepthTextureName("InputZBuffer") logger.debug("Using program for $outOfCoreVolumeCount out-of-core volumes and $regularVolumeCount regular volumes") prog.add(newProgvol) @@ -445,57 +464,27 @@ class VolumeManager( } } - /** - * Updates the currently-used set of blocks using [context] to - * facilitate the updates on the GPU. - */ - @Synchronized - protected fun updateBlocks(context: SceneryContext): Boolean { - val currentProg = progvol - if (currentProg == null) { - logger.debug("Not updating blocks, no prog") - return false - } - - nodes.forEach { bdvNode -> - bdvNode.prepareNextFrame() - } - - val cam = nodes.firstOrNull()?.getScene()?.activeObserver ?: return false - val settings = hub?.get() ?: return false - - val hmd = hub?.getWorkingHMDDisplay()?.wantsVR(settings) - val vp = if(hmd != null) { - Matrix4f(hmd.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance)) - .mul(cam.spatial().getTransformation()) - } else { - Matrix4f(cam.spatial().projection) - .mul(cam.spatial().getTransformation()) - } - - // TODO: original might result in NULL, is this intended? - currentProg.use(context) - currentProg.setUniforms(context) + private fun updateBlocks(context: SceneryContext, + multiResStacks:List>, + volumes:List, + cachePBO: TextureCacheAndPboChain, + vpWidth: Int, + vp: Matrix4f): Boolean { var numTasks = 0 val fillTasksPerVolume = ArrayList() val taskCreationDuration = measureTimeMillis { - renderStacksStates.forEachIndexed { i, state -> - if (state.stack is MultiResolutionStack3D) { - val volume = outOfCoreVolumes[i] - - volume.init(state.stack, cam.width, vp) - + volumes.forEachIndexed { i, volume -> + volume.init(multiResStacks[i], cachePBO.textureCache, vpWidth, vp) val tasks = volume.fillTasks numTasks += tasks.size - fillTasksPerVolume.add(VolumeAndTasks(tasks, volume, state.stack.resolutions().size - 1)) - } + fillTasksPerVolume.add(VolumeAndTasks(tasks, volume, multiResStacks[i].resolutions().size - 1)) } } val fillTasksDuration = measureTimeMillis { - taskLoop@ while (numTasks > textureCache.maxNumTiles) { + taskLoop@ while (numTasks > cachePBO.textureCache.maxNumTiles) { fillTasksPerVolume.sortByDescending { it.numTasks() } for (vat in fillTasksPerVolume) { val baseLevel = vat.volume.baseLevel @@ -518,33 +507,91 @@ class VolumeManager( fillTasksPerVolume.forEach { fillTasks.addAll(it.tasks) } - logger.debug("Got ${fillTasks.size} fill tasks (vs max=${textureCache.maxNumTiles})") + logger.debug("Got ${fillTasks.size} fill tasks (vs max=${cachePBO.textureCache.maxNumTiles})") - if (fillTasks.size > textureCache.maxNumTiles) { - fillTasks.subList(textureCache.maxNumTiles, fillTasks.size).clear() + if (fillTasks.size > cachePBO.textureCache.maxNumTiles) { + fillTasks.subList(cachePBO.textureCache.maxNumTiles, fillTasks.size).clear() } - ProcessFillTasks.parallel(textureCache, pboChain, context, forkJoinPool, fillTasks) + ProcessFillTasks.parallel(cachePBO.textureCache, cachePBO.pboChain, context, forkJoinPool, fillTasks) // ProcessFillTasks.sequential(textureCache, pboChain, context, fillTasks) } // TODO: is repaint necessary? // var repaint = false val durationLutUpdate = measureTimeMillis { - renderStacksStates.forEachIndexed { i, state -> - if (state.stack is MultiResolutionStack3D) { - val volumeBlocks = outOfCoreVolumes[i] - val timestamp = textureCache.nextTimestamp() - volumeBlocks.makeLut(timestamp) - //val complete = volumeBlocks.makeLut(timestamp) - // if (!complete) { - // repaint = true - // } - context.bindTexture(volumeBlocks.lookupTexture) - volumeBlocks.lookupTexture.upload(context) + volumes.forEachIndexed { i, volumeBlocks -> + val timestamp = cachePBO.textureCache.nextTimestamp() + volumeBlocks.makeLut(timestamp) + //val complete = volumeBlocks.makeLut(timestamp) + // if (!complete) { + // repaint = true + // } + context.bindTexture(volumeBlocks.lookupTexture) + volumeBlocks.lookupTexture.upload(context) + } + } + + return true; + } + + /** + * Updates the currently-used set of blocks using [context] to + * facilitate the updates on the GPU. + */ + @Synchronized + protected fun updateBlocks(context: SceneryContext): Boolean { + val currentProg = progvol + if (currentProg == null) { + logger.debug("Not updating blocks, no prog") + return false + } + + nodes.forEach { bdvNode -> + bdvNode.prepareNextFrame() + } + + val cam = nodes.firstOrNull()?.getScene()?.activeObserver ?: return false + val settings = hub?.get() ?: return false + + val hmd = hub?.getWorkingHMDDisplay()?.wantsVR(settings) + val vp = if(hmd != null) { + Matrix4f(hmd.getEyeProjection(0, cam.nearPlaneDistance, cam.farPlaneDistance)) + .mul(cam.spatial().getTransformation()) + } else { + Matrix4f(cam.spatial().projection) + .mul(cam.spatial().getTransformation()) + } + + // TODO: original might result in NULL, is this intended? + currentProg.use(context) + currentProg.setUniforms(context) + + val multiResStacksR8 = mutableListOf>() + val volumesR8 = mutableListOf() + + val multiResStacksR16 = mutableListOf>() + val volumesR16 = mutableListOf() + + renderStacksStates.forEachIndexed { i, state -> + if (state.stack is MultiResolutionStack3D) { + val volume = outOfCoreVolumes[i] + if (bvv.core.blocks.TileAccess.getPrimitiveType(state.stack.type) == net.imglib2.type.PrimitiveType.BYTE) { + volumesR8.add(volume) + multiResStacksR8.add(state.stack) + } + if (bvv.core.blocks.TileAccess.getPrimitiveType(state.stack.type) == net.imglib2.type.PrimitiveType.SHORT) { + volumesR16.add(volume) + multiResStacksR16.add(state.stack) } } } + if( volumesR8.size > 0) { + updateBlocks(context, multiResStacksR8, volumesR8, cacheR8, cam.width, vp) + } + if(volumesR16.size > 0) { + updateBlocks(context, multiResStacksR16, volumesR16, cacheR16, cam.width, vp) + } var minWorldVoxelSize = Double.POSITIVE_INFINITY val ready = readyToRender() @@ -601,14 +648,14 @@ class VolumeManager( currentProg.bindSamplers(context) } - logger.debug( - "Task creation: {}ms, Fill task creation: {}ms, Fill task processing: {}ms, LUT update: {}ms, Bindings: {}ms", - taskCreationDuration, - fillTasksDuration, - durationFillTaskProcessing, - durationLutUpdate, - durationBinding - ) + // logger.debug( + // "Task creation: {}ms, Fill task creation: {}ms, Fill task processing: {}ms, LUT update: {}ms, Bindings: {}ms", + // taskCreationDuration, + // fillTasksDuration, + // durationFillTaskProcessing, + // durationLutUpdate, + // durationBinding + // ) // TODO: check if repaint can be made sufficient for triggering rendering return true } @@ -696,7 +743,7 @@ class VolumeManager( */ override fun preDraw(): Boolean { logger.debug("Running predraw") - context.bindTexture(textureCache) + //context.bindTexture(textureCache) if (nodes.any { it.transferFunction.stale }) { transferFunctionTextures.clear() @@ -930,11 +977,10 @@ class VolumeManager( fun recreateCache(newMaxSize: Int) { logger.warn("Recreating cache, new size: $newMaxSize MB ") - val cacheGridDimensions = TextureCache.findSuitableGridSize(cacheSpec, newMaxSize) - textureCache = TextureCache(cacheGridDimensions, cacheSpec) + cacheR8 = TextureCacheAndPboChain(Texture.InternalFormat.R8,intArrayOf(32, 32, 32),newMaxSize) + cacheR16 = TextureCacheAndPboChain(Texture.InternalFormat.R16,intArrayOf(32, 32, 32),newMaxSize) - pboChain = PboChain(5, 100, textureCache) prog.clear() // updateRenderState() diff --git a/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag b/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag index 5c6dba64a..c13402378 100644 --- a/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/AccumulateBlockVolume.frag @@ -5,7 +5,7 @@ uniform int sceneGraphVisibility; vis = vis && bool(sceneGraphVisibility); if (vis && step > localNear && step < localFar) { - vec4 x = sampleVolume(wpos, volumeCache, cacheSize, blockSize, paddedBlockSize, cachePadOffset); + vec4 x = sampleVolume(wpos); float newAlpha = x.a; vec3 newColor = x.rgb; diff --git a/src/main/resources/graphics/scenery/volumes/BDVVolume.frag b/src/main/resources/graphics/scenery/volumes/BDVVolume.frag index 07957ef00..1f4bf32c7 100644 --- a/src/main/resources/graphics/scenery/volumes/BDVVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/BDVVolume.frag @@ -12,15 +12,6 @@ uniform vec2 dsp; uniform float fwnw; uniform float nw; -uniform sampler3D volumeCache; - -// -- comes from CacheSpec ----- -uniform vec3 blockSize; -uniform vec3 paddedBlockSize; -uniform vec3 cachePadOffset; - -// -- comes from TextureCache -- -uniform vec3 cacheSize; // TODO: get from texture!? uniform mat4 transform; #pragma scenery verbatim diff --git a/src/main/resources/graphics/scenery/volumes/SampleBlockVolume.frag b/src/main/resources/graphics/scenery/volumes/SampleBlockVolume.frag index e7364a24d..2c8912c73 100644 --- a/src/main/resources/graphics/scenery/volumes/SampleBlockVolume.frag +++ b/src/main/resources/graphics/scenery/volumes/SampleBlockVolume.frag @@ -14,6 +14,16 @@ void intersectBoundingBox( vec4 wfront, vec4 wback, out float tnear, out float t intersectBox( mfront.xyz, (mback - mfront).xyz, sourcemin, sourcemax, tnear, tfar ); } +uniform sampler3D volumeCache; + +// -- comes from CacheSpec ----- +uniform vec3 blockSize; +uniform vec3 paddedBlockSize; +uniform vec3 cachePadOffset; + +// -- comes from TextureCache -- +uniform vec3 cacheSize; // TODO: get from texture!? + uniform usampler3D lutSampler; uniform sampler2D transferFunction; uniform sampler2D colorMap; @@ -21,7 +31,7 @@ uniform vec3 blockScales[ NUM_BLOCK_SCALES ]; uniform vec3 lutSize; uniform vec3 lutOffset; -vec4 sampleVolume( vec4 wpos, sampler3D volumeCache, vec3 cacheSize, vec3 blockSize, vec3 paddedBlockSize, vec3 padOffset ) +vec4 sampleVolume( vec4 wpos ) { bool cropping = slicingMode == 1 || slicingMode == 3; bool slicing = slicingMode == 2 || slicingMode == 3; @@ -52,7 +62,7 @@ vec4 sampleVolume( vec4 wpos, sampler3D volumeCache, vec3 cacheSize, vec3 blockS vec3 q = floor( pos / blockSize ) - lutOffset + 0.5; uvec4 lutv = texture( lutSampler, q / lutSize ); - vec3 B0 = lutv.xyz * paddedBlockSize + padOffset; + vec3 B0 = lutv.xyz * paddedBlockSize + cachePadOffset; vec3 sj = blockScales[ lutv.w ]; vec3 c0 = B0 + mod( pos * sj, blockSize ) + 0.5 * sj;