From fb03ab126f56fbf95a536832c744e63da66a2808 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Wed, 17 Jun 2026 16:30:08 +0200 Subject: [PATCH 1/3] Dynamic chunks for lidar meshing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Candice Bentéjac --- meshroom/aliceVision/LidarDecimating.py | 3 +- meshroom/aliceVision/LidarMeshing.py | 3 +- src/aliceVision/python/parallelization.py | 42 ++++++++++++++++++++++- 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/meshroom/aliceVision/LidarDecimating.py b/meshroom/aliceVision/LidarDecimating.py index 75151cc573..8f8d77bfbc 100644 --- a/meshroom/aliceVision/LidarDecimating.py +++ b/meshroom/aliceVision/LidarDecimating.py @@ -2,6 +2,7 @@ from meshroom.core import desc from meshroom.core.utils import VERBOSE_LEVEL +from pyalicevision import parallelization as avpar class LidarDecimating(desc.AVCommandLineNode): """ @@ -15,7 +16,7 @@ class LidarDecimating(desc.AVCommandLineNode): commandLine = "aliceVision_lidarDecimating {allParams}" - size = desc.StaticNodeSize(10) + size = avpar.DynamicJsonListSize("input") parallelization = desc.Parallelization(blockSize=1) commandLineRange = "--rangeStart {rangeStart} --rangeSize {rangeFullSize}" diff --git a/meshroom/aliceVision/LidarMeshing.py b/meshroom/aliceVision/LidarMeshing.py index abbeab98bf..bbe84c9bca 100644 --- a/meshroom/aliceVision/LidarMeshing.py +++ b/meshroom/aliceVision/LidarMeshing.py @@ -2,13 +2,14 @@ from meshroom.core import desc from meshroom.core.utils import VERBOSE_LEVEL +from pyalicevision import parallelization as avpar class LidarMeshing(desc.AVCommandLineNode): """This node creates a dense geometric surface representation of the Lidar measurements.""" commandLine = "aliceVision_lidarMeshing {allParams}" - size = desc.StaticNodeSize(10) + size = avpar.DynamicJsonListSize("input") parallelization = desc.Parallelization(blockSize=1) commandLineRange = "--rangeStart {rangeStart} --rangeSize {rangeFullSize}" diff --git a/src/aliceVision/python/parallelization.py b/src/aliceVision/python/parallelization.py index 9c0cf66ae9..3d0b8f25a8 100644 --- a/src/aliceVision/python/parallelization.py +++ b/src/aliceVision/python/parallelization.py @@ -178,4 +178,44 @@ def __call__(self, node): if not valid: raise RuntimeError(f"Failed to load file : {param.value}") - return max(1, len(imlist)) \ No newline at end of file + return max(1, len(imlist)) + +class DynamicJsonListSize(object): + """Compute parallelization size based on a JSON file containing an array of items. + + Reads the JSON file referenced by the given node parameter and returns the + number of elements in the root-level array. Returns 1 if the file cannot be + read, is not valid JSON, or its root element is not an array. + + Args: + param: Name of the node attribute that holds the path to the JSON file. + """ + + def __init__(self, param): + self._param = param + + def __call__(self, node): + """Compute the number of chunks from the JSON array size. + + Args: + node: The processing node whose attribute *param* points to a JSON file. + + Returns: + int: The number of elements in the root JSON array, or 1 if the file + cannot be loaded or the root is not an array. + """ + + import json + + param = node.attribute(self._param) + + try: + with open(param.value, 'r') as f: + data = json.load(f) + except Exception: + return 1 + + if not isinstance(data, list): + return 1 + + return max(1, len(data)) From 3ecb09e46972548a23a3e18a97671f171682950d Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 18 Jun 2026 09:41:11 +0200 Subject: [PATCH 2/3] e57 debug information --- src/aliceVision/dataio/E57Reader.hpp | 24 +++++++++++++++++++++++- src/software/convert/main_importE57.cpp | 2 +- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/src/aliceVision/dataio/E57Reader.hpp b/src/aliceVision/dataio/E57Reader.hpp index 167a30374e..e2b4b5cdeb 100644 --- a/src/aliceVision/dataio/E57Reader.hpp +++ b/src/aliceVision/dataio/E57Reader.hpp @@ -129,6 +129,27 @@ class E57Reader unsigned int readCount = 0; while ((readCount = datareader.read()) > 0) { + ALICEVISION_LOG_DEBUG("cartesianInvalidState pointer: " << static_cast(data3DPoints.cartesianInvalidState)); + ALICEVISION_LOG_DEBUG("cartesianX pointer: " << static_cast(data3DPoints.cartesianX)); + ALICEVISION_LOG_DEBUG("cartesianY pointer: " << static_cast(data3DPoints.cartesianY)); + ALICEVISION_LOG_DEBUG("cartesianZ pointer: " << static_cast(data3DPoints.cartesianZ)); + ALICEVISION_LOG_DEBUG("colorBlue pointer: " << static_cast(data3DPoints.colorBlue)); + ALICEVISION_LOG_DEBUG("colorGreen pointer: " << static_cast(data3DPoints.colorGreen)); + ALICEVISION_LOG_DEBUG("colorRed pointer: " << static_cast(data3DPoints.colorRed)); + ALICEVISION_LOG_DEBUG("columnIndex pointer: " << static_cast(data3DPoints.columnIndex)); + ALICEVISION_LOG_DEBUG("intensity pointer: " << static_cast(data3DPoints.intensity)); + ALICEVISION_LOG_DEBUG("isColorInvalid pointer: " << static_cast(data3DPoints.isColorInvalid)); + ALICEVISION_LOG_DEBUG("isIntensityInvalid pointer: " << static_cast(data3DPoints.isIntensityInvalid)); + ALICEVISION_LOG_DEBUG("isTimeStampInvalid pointer: " << static_cast(data3DPoints.isTimeStampInvalid)); + ALICEVISION_LOG_DEBUG("returnCount pointer: " << static_cast(data3DPoints.returnCount)); + ALICEVISION_LOG_DEBUG("returnIndex pointer: " << static_cast(data3DPoints.returnIndex)); + ALICEVISION_LOG_DEBUG("rowIndex pointer: " << static_cast(data3DPoints.rowIndex)); + ALICEVISION_LOG_DEBUG("sphericalAzimuth pointer: " << static_cast(data3DPoints.sphericalAzimuth)); + ALICEVISION_LOG_DEBUG("sphericalElevation pointer: " << static_cast(data3DPoints.sphericalElevation)); + ALICEVISION_LOG_DEBUG("sphericalInvalidState pointer: " << static_cast(data3DPoints.sphericalInvalidState)); + ALICEVISION_LOG_DEBUG("sphericalRange pointer: " << static_cast(data3DPoints.sphericalRange)); + ALICEVISION_LOG_DEBUG("timeStamp pointer: " << static_cast(data3DPoints.timeStamp)); + // Check input compatibility if (data3DPoints.sphericalRange != nullptr) { @@ -141,6 +162,7 @@ class E57Reader ALICEVISION_LOG_ERROR("Data contains no cartesian coordinates"); continue; } + if (data3DPoints.columnIndex == nullptr) { @@ -162,7 +184,7 @@ class E57Reader for (unsigned int pos = 0; pos < readCount; pos++) { - if (data3DPoints.cartesianInvalidState[pos]) + if (data3DPoints.cartesianInvalidState && data3DPoints.cartesianInvalidState[pos]) { continue; } diff --git a/src/software/convert/main_importE57.cpp b/src/software/convert/main_importE57.cpp index d139173007..4620a541b1 100644 --- a/src/software/convert/main_importE57.cpp +++ b/src/software/convert/main_importE57.cpp @@ -352,7 +352,7 @@ int aliceVision_main(int argc, char** argv) std::vector list; octree.visit(list); - ALICEVISION_LOG_INFO("Generating " << list.size() << "sub regions"); + ALICEVISION_LOG_INFO("Generating " << list.size() << " sub regions"); fuseCut::InputSet inputs; From 46d7c3b65c4e399bd9801436fcca25862733df46 Mon Sep 17 00:00:00 2001 From: Fabien Servant Date: Thu, 18 Jun 2026 10:54:01 +0200 Subject: [PATCH 3/3] Debug multiple File management --- src/aliceVision/dataio/E57Reader.hpp | 37 ++++++++++++++++--------- src/aliceVision/fuseCut/Octree.hpp | 16 +++++------ src/software/convert/main_importE57.cpp | 4 +-- 3 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/aliceVision/dataio/E57Reader.hpp b/src/aliceVision/dataio/E57Reader.hpp index e2b4b5cdeb..a90a8d5cea 100644 --- a/src/aliceVision/dataio/E57Reader.hpp +++ b/src/aliceVision/dataio/E57Reader.hpp @@ -31,6 +31,7 @@ class E57Reader : _paths(paths), _idPath(-1), _idMesh(-1), + _idMeshInFile(-1), _countMeshesForFile(0), _requiredIntensity(0.0) {} @@ -39,19 +40,23 @@ class E57Reader { _idPath = -1; _idMesh = -1; + _idMeshInFile = -1; _countMeshesForFile = 0; } bool getNext(Eigen::Vector3d& sensorPosition, std::vector& vertices, Eigen::Matrix& grid) { // Go to next mesh of current file + _idMeshInFile++; + + // Keep a unique counter for all files _idMesh++; // Load next file if needed - if (_idMesh >= static_cast(_countMeshesForFile)) + if (_idMeshInFile >= static_cast(_countMeshesForFile)) { _idPath++; - _idMesh = 0; + _idMeshInFile = 0; if (_idPath >= static_cast(_paths.size())) { return false; @@ -74,7 +79,7 @@ class E57Reader _countMeshesForFile = _reader->GetData3DCount(); } - if (_idMesh >= static_cast(_countMeshesForFile)) + if (_idMeshInFile >= static_cast(_countMeshesForFile)) { return false; } @@ -86,9 +91,9 @@ class E57Reader // Get header e57::Data3D scanHeader; - if (!_reader->ReadData3D(_idMesh, scanHeader)) + if (!_reader->ReadData3D(_idMeshInFile, scanHeader)) { - ALICEVISION_LOG_ERROR("Error reading mesh #" << _idMesh); + ALICEVISION_LOG_ERROR("Error reading mesh #" << _idMeshInFile); return false; } @@ -109,14 +114,14 @@ class E57Reader int64_t maxGroupSize = 0; bool isColumnIndex; - if (!_reader->GetData3DSizes(0, maxRows, maxColumns, countPoints, countGroups, maxGroupSize, isColumnIndex)) + if (!_reader->GetData3DSizes(_idMeshInFile, maxRows, maxColumns, countPoints, countGroups, maxGroupSize, isColumnIndex)) { - ALICEVISION_LOG_ERROR("Error reading content of mesh #" << _idMesh); + ALICEVISION_LOG_ERROR("Error reading content of mesh #" << _idMeshInFile); return false; } e57::Data3DPointsFloat data3DPoints(scanHeader); - e57::CompressedVectorReader datareader = _reader->SetUpData3DPointsData(_idMesh, countPoints, data3DPoints); + e57::CompressedVectorReader datareader = _reader->SetUpData3DPointsData(_idMeshInFile, countPoints, data3DPoints); // Prepare list of vertices vertices.clear(); @@ -181,6 +186,7 @@ class E57Reader ALICEVISION_LOG_ERROR("Data contains no intensities"); continue; } + for (unsigned int pos = 0; pos < readCount; pos++) { @@ -209,6 +215,7 @@ class E57Reader vertices.push_back(pi); } } + datareader.close(); sensorPosition = t; @@ -218,13 +225,16 @@ class E57Reader bool getNext(Eigen::Vector3d& sensorPosition) { // Go to next mesh of current file + _idMeshInFile++; + + // Keep a unique counter for all files _idMesh++; // Load next file if needed - if (_idMesh >= static_cast(_countMeshesForFile)) + if (_idMeshInFile >= static_cast(_countMeshesForFile)) { _idPath++; - _idMesh = 0; + _idMeshInFile = 0; if (_idPath >= static_cast(_paths.size())) { return false; @@ -247,7 +257,7 @@ class E57Reader _countMeshesForFile = _reader->GetData3DCount(); } - if (_idMesh >= static_cast(_countMeshesForFile)) + if (_idMeshInFile >= static_cast(_countMeshesForFile)) { return false; } @@ -259,9 +269,9 @@ class E57Reader // Get header e57::Data3D scanHeader; - if (!_reader->ReadData3D(_idMesh, scanHeader)) + if (!_reader->ReadData3D(_idMeshInFile, scanHeader)) { - ALICEVISION_LOG_ERROR("Error reading mesh #" << _idMesh); + ALICEVISION_LOG_ERROR("Error reading mesh #" << _idMeshInFile); return false; } @@ -286,6 +296,7 @@ class E57Reader int _idPath; int _idMesh; + int _idMeshInFile; size_t _countMeshesForFile; double _requiredIntensity; }; diff --git a/src/aliceVision/fuseCut/Octree.hpp b/src/aliceVision/fuseCut/Octree.hpp index b92a20dd34..96c6d1055f 100644 --- a/src/aliceVision/fuseCut/Octree.hpp +++ b/src/aliceVision/fuseCut/Octree.hpp @@ -136,6 +136,14 @@ class SimpleNode return; } + for (auto& item : _nodes) + { + if (item) + { + item->regroup(maxPointsPerNode); + } + } + if (count < maxPointsPerNode) { _count = count; @@ -148,14 +156,6 @@ class SimpleNode return; } - - for (auto& item : _nodes) - { - if (item) - { - item->regroup(maxPointsPerNode); - } - } } const size_t getCount() const diff --git a/src/software/convert/main_importE57.cpp b/src/software/convert/main_importE57.cpp index 4620a541b1..82c5a90e59 100644 --- a/src/software/convert/main_importE57.cpp +++ b/src/software/convert/main_importE57.cpp @@ -192,11 +192,11 @@ int aliceVision_main(int argc, char** argv) std::map cameras; while (reader.getNext(sensorPosition)) { - cameras[reader.getIdMesh()] = sensorPosition; - // Create pose for sfmData const int idMesh = reader.getIdMesh(); + cameras[idMesh] = sensorPosition; + Eigen::Vector3d correctedSensorPosition; correctedSensorPosition.x() = sensorPosition.x(); correctedSensorPosition.y() = -sensorPosition.z();