Skip to content

Parallel reading of serial basis files #316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
dylan-copeland merged 6 commits into from
Sep 16, 2025
Merged

Parallel reading of serial basis files #316

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
0bc6d4c
Enabling BasisReader to load a serial basis on each rank independentl…
dylan-copeland Mar 18, 2025
d7e2a50
Merge branch 'master' of github.com:LLNL/libROM into serial-reader
dylan-copeland Aug 31, 2025
df46dd3
Adding distributed flag to database read.
dylan-copeland Sep 3, 2025
b900a1b
Adding unit test to read a small serial file in parallel.
dylan-copeland Sep 4, 2025
f162ca5
Fixed new unit test.
dylan-copeland Sep 4, 2025
47678e0
Verify that BasisReader comm is NULL or WORLD.
dylan-copeland Sep 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 12 additions & 9 deletions lib/linalg/BasisReader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,21 @@ namespace CAROM {
BasisReader::BasisReader(
const std::string& base_file_name,
Database::formats db_format,
const int dim) :
const int dim,
MPI_Comm comm) :
d_dim(dim),
full_file_name(""),
base_file_name_(base_file_name),
d_format(db_format)
{
CAROM_ASSERT(!base_file_name.empty());
d_distributed = comm != MPI_COMM_NULL;

@dreamer2368 dreamer2368 Sep 2, 2025

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This concerns me a little bit, since libROM currently does not support custom MPI communicator other than MPI_COMM_WORLD.

Should we add a line of CAROM_VERIFY for checking if the communicator is MPI_COMM_WORLD?


int mpi_init;
MPI_Initialized(&mpi_init);
int rank;
if (mpi_init) {
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
if (mpi_init && d_distributed) {
MPI_Comm_rank(comm, &rank);
}
else {
rank = 0;
Expand All @@ -55,15 +57,15 @@ BasisReader::BasisReader(
We allow 0 local dimension. (global dimension still needs to be positive)
*/
std::vector<int> tmp;
d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD);
d_global_dim = get_global_offsets(d_dim, tmp, comm);
CAROM_VERIFY(d_dim >= 0);
CAROM_VERIFY(d_global_dim > 0);
d_database = new HDFDatabaseMPIO();
}
else
CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n");

d_database->open(full_file_name, "r", MPI_COMM_WORLD);
d_database->open(full_file_name, "r", comm);
}

BasisReader::~BasisReader()
Expand All @@ -78,7 +80,7 @@ BasisReader::getSpatialBasis()
int num_rows = getDim("basis");
int num_cols = getNumSamples("basis");

Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true);
Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, d_distributed);

d_database->getDoubleArray("spatial_basis",
Comment thread
ckendrick marked this conversation as resolved.
&spatial_basis_vectors->item(0, 0),
Expand Down Expand Up @@ -107,7 +109,8 @@ BasisReader::getSpatialBasis(
CAROM_VERIFY(start_col <= end_col && end_col <= num_cols);
int num_cols_to_read = end_col - start_col + 1;

Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true);
Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read,
d_distributed);
sprintf(tmp, "spatial_basis");
d_database->getDoubleArray(tmp,
&spatial_basis_vectors->item(0, 0),
Expand Down Expand Up @@ -324,7 +327,7 @@ BasisReader::getSnapshotMatrix()
int num_cols = getNumSamples("snapshot");

char tmp[100];
Matrix* snapshots = new Matrix(num_rows, num_cols, true);
Matrix* snapshots = new Matrix(num_rows, num_cols, d_distributed);
sprintf(tmp, "snapshot_matrix");
d_database->getDoubleArray(tmp,
&snapshots->item(0, 0),
Expand Down Expand Up @@ -353,7 +356,7 @@ BasisReader::getSnapshotMatrix(
int num_cols_to_read = end_col - start_col + 1;

char tmp[100];
Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true);
Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, d_distributed);
sprintf(tmp, "snapshot_matrix");
d_database->getDoubleArray(tmp,
&snapshots->item(0, 0),
Expand Down
5 changes: 4 additions & 1 deletion lib/linalg/BasisReader.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ class BasisReader {
BasisReader(
const std::string& base_file_name,
Database::formats db_format = Database::formats::HDF5,
const int dim = -1);
const int dim = -1,
MPI_Comm comm = MPI_COMM_WORLD);

/**
* @brief Destructor.
Expand Down Expand Up @@ -314,6 +315,8 @@ class BasisReader {
* If negative, use the dimension from the rank-specific local file.
*/
int d_global_dim;

bool d_distributed;
};

}
Expand Down