MPIO support for Basis classes

.github/workflows/run_tests/action.yml

@@ -23,6 +23,8 @@ runs:
           mpirun -n 3 --oversubscribe tests/test_StaticSVD
           ./tests/test_IncrementalSVDBrand
           mpirun -n 3 --oversubscribe tests/test_IncrementalSVDBrand
+          ./tests/test_HDFDatabase
+          mpirun -n 3 --oversubscribe tests/test_HDFDatabase
           ./tests/test_NNLS
           mpirun -n 3 --oversubscribe tests/test_NNLS
       shell: bash

CMakeLists.txt

@@ -265,6 +265,7 @@ if(GTEST_FOUND)
   set(unit_test_stems
     Vector
     Matrix
+    HDFDatabase
     DEIM
     DMD
     GNAT

docker/Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /$ENVDIR
 # install packages
 RUN sudo apt-get install -yq git
 RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
-RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-serial-dev
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev hdf5-tools
 RUN sudo apt-get install -yq vim
 RUN sudo apt-get install -yq git-lfs
 RUN sudo apt-get install -yq valgrind

lib/CMakeLists.txt

@@ -54,6 +54,7 @@ set(module_list
   hyperreduction/Hyperreduction
   utils/Database
   utils/HDFDatabase
+  utils/HDFDatabaseMPIO
   utils/CSVDatabase
   utils/Utilities
   utils/ParallelBuffer

lib/linalg/BasisGenerator.cpp

@@ -28,6 +28,7 @@ BasisGenerator::BasisGenerator(
     bool incremental,
     const std::string& basis_file_name,
     Database::formats file_format) :
+    d_dim(options.dim),
     d_incremental(incremental),
     d_basis_writer(0),
     d_basis_reader(0),
@@ -153,7 +154,7 @@ BasisGenerator::loadSamples(const std::string& base_file_name,
 
     if (d_basis_reader) delete d_basis_reader;
 
-    d_basis_reader = new BasisReader(base_file_name, db_format);
+    d_basis_reader = new BasisReader(base_file_name, db_format, d_dim);
     const Matrix* mat;
     const Vector* singular_vals;
 

lib/linalg/BasisGenerator.h

@@ -288,7 +288,7 @@ class BasisGenerator
     int
     getDim()
     {
-        return d_svd->getDim();
+        return d_dim;
     }
 
     /**
@@ -342,6 +342,13 @@ class BasisGenerator
      * @brief The number of processors being run on.
      */
     int d_num_procs;
+
+    /**
+     * @brief Dimension of the system on this processor.
+     *
+     * Equivalent to d_svd->getDim().
+     */
+    const int d_dim;
 };
 
 }

lib/linalg/BasisReader.cpp

@@ -12,18 +12,22 @@
 
 #include "BasisReader.h"
 #include "utils/HDFDatabase.h"
-#include "utils/CSVDatabase.h"
+#include "utils/HDFDatabaseMPIO.h"
 #include "Matrix.h"
 #include "Vector.h"
 #include "mpi.h"
+#include "utils/mpi_utils.h"
 
 namespace CAROM {
 
 BasisReader::BasisReader(
     const std::string& base_file_name,
-    Database::formats db_format) :
+    Database::formats db_format,
+    const int dim) :
+    d_dim(dim),
     full_file_name(""),
-    base_file_name_(base_file_name)
+    base_file_name_(base_file_name),
+    d_format(db_format)
 {
     CAROM_ASSERT(!base_file_name.empty());
 
@@ -37,17 +41,29 @@ BasisReader::BasisReader(
         rank = 0;
     }
 
-    char tmp[100];
-    sprintf(tmp, ".%06d", rank);
-    full_file_name = base_file_name + tmp;
-    if (db_format == Database::HDF5) {
+    full_file_name = base_file_name;
+
+    // Enforce hdf data format.
+    if (d_format == Database::HDF5)
+    {
         d_database = new HDFDatabase();
     }
-    else if (db_format == Database::CSV) {
-        d_database = new CSVDatabase();
+    else if (d_format == Database::HDF5_MPIO)
+    {
+        /*
+            For MPIO case, local dimension needs to be specified.
+            We allow 0 local dimension. (global dimension still needs to be positive)
+        */
+        std::vector<int> tmp;
+        d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD);
+        CAROM_VERIFY(d_dim >= 0);
+        CAROM_VERIFY(d_global_dim > 0);
+        d_database = new HDFDatabaseMPIO();
     }
+    else
+        CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n");
 
-    d_database->open(full_file_name, "r");
+    d_database->open_parallel(full_file_name, "r", MPI_COMM_WORLD);
 }
 
 BasisReader::~BasisReader()
@@ -64,9 +80,9 @@ BasisReader::getSpatialBasis()
 
     Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols, true);
 
-    d_database->getDoubleArray("spatial_basis",
-                               &spatial_basis_vectors->item(0, 0),
-                               num_rows*num_cols);
+    d_database->getDoubleArray_parallel("spatial_basis",
+                                        &spatial_basis_vectors->item(0, 0),
+                                        num_rows*num_cols);
     return spatial_basis_vectors;
 }
 
@@ -92,12 +108,12 @@ BasisReader::getSpatialBasis(
 
     Matrix* spatial_basis_vectors = new Matrix(num_rows, num_cols_to_read, true);
     sprintf(tmp, "spatial_basis");
-    d_database->getDoubleArray(tmp,
-                               &spatial_basis_vectors->item(0, 0),
-                               num_rows*num_cols_to_read,
-                               start_col - 1,
-                               num_cols_to_read,
-                               num_cols);
+    d_database->getDoubleArray_parallel(tmp,
+                                        &spatial_basis_vectors->item(0, 0),
+                                        num_rows*num_cols_to_read,
+                                        start_col - 1,
+                                        num_cols_to_read,
+                                        num_cols);
     return spatial_basis_vectors;
 }
 
@@ -135,7 +151,7 @@ BasisReader::getTemporalBasis()
     int num_cols = getNumSamples("temporal_basis");
 
     char tmp[100];
-    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols, true);
+    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols, false);
     sprintf(tmp, "temporal_basis");
     d_database->getDoubleArray(tmp,
                                &temporal_basis_vectors->item(0, 0),
@@ -163,7 +179,7 @@ BasisReader::getTemporalBasis(
     CAROM_VERIFY(start_col <= end_col && end_col <= num_cols);
     int num_cols_to_read = end_col - start_col + 1;
 
-    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols_to_read, true);
+    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols_to_read, false);
     sprintf(tmp, "temporal_basis");
     d_database->getDoubleArray(tmp,
                                &temporal_basis_vectors->item(0, 0),
@@ -268,7 +284,18 @@ BasisReader::getDim(
                 "temporal_basis_num_rows");
 
     d_database->getInteger(tmp, num_rows);
-    return num_rows;
+    /* only basis and snapshot are stored as distributed matrices */
+    if ((kind != "temporal_basis") && (d_format == Database::HDF5_MPIO))
+    {
+        /*
+            for MPIO database, return specified local dimension.
+            only checks the global dimension match.
+        */
+        CAROM_VERIFY(d_global_dim == num_rows);
+        return d_dim;
+    }
+    else
+        return num_rows;
 }
 
 int
@@ -298,11 +325,11 @@ BasisReader::getSnapshotMatrix()
     int num_cols = getNumSamples("snapshot");
 
     char tmp[100];
-    Matrix* snapshots = new Matrix(num_rows, num_cols, false);
+    Matrix* snapshots = new Matrix(num_rows, num_cols, true);
     sprintf(tmp, "snapshot_matrix");
-    d_database->getDoubleArray(tmp,
-                               &snapshots->item(0, 0),
-                               num_rows*num_cols);
+    d_database->getDoubleArray_parallel(tmp,
+                                        &snapshots->item(0, 0),
+                                        num_rows*num_cols);
     return snapshots;
 }
 
@@ -326,14 +353,14 @@ BasisReader::getSnapshotMatrix(
     int num_cols_to_read = end_col - start_col + 1;
 
     char tmp[100];
-    Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, false);
+    Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true);
     sprintf(tmp, "snapshot_matrix");
-    d_database->getDoubleArray(tmp,
-                               &snapshots->item(0, 0),
-                               num_rows*num_cols_to_read,
-                               start_col - 1,
-                               num_cols_to_read,
-                               num_cols);
+    d_database->getDoubleArray_parallel(tmp,
+                                        &snapshots->item(0, 0),
+                                        num_rows*num_cols_to_read,
+                                        start_col - 1,
+                                        num_cols_to_read,
+                                        num_cols);
     return snapshots;
 }
 }

lib/linalg/BasisReader.h

@@ -44,10 +44,13 @@ class BasisReader {
      * @param[in] db_format Format of the file to read.
      *                      One of the implemented file formats defined in
      *                      Database.
+     * @param[in] dim Dimension of the basis that will be read from a file.
+     *                If negative, will use the dimension from the rank-specific local file.
      */
     BasisReader(
         const std::string& base_file_name,
-        Database::formats db_format = Database::HDF5);
+        Database::formats db_format = Database::HDF5,
+        const int dim = -1);
 
     /**
      * @brief Destructor.
@@ -197,6 +200,11 @@ class BasisReader {
      *
      * @brief Returns the dimension of the system on this processor.
      *
+     * @param[in] kind  The target matrix of which dimension (row) is returned.
+     *                  "basis" - local dimension of spatial basis
+     *                  "snapshot" - local dimension of snapshot matrix
+     *                  "temporal_basis" - global dimension of temporal basis
+     *
      * @return The dimension of the system on this processor.
      */
     int
@@ -277,6 +285,11 @@ class BasisReader {
      */
     Database* d_database;
 
+    /**
+     * @brief The database being read from.
+     */
+    Database::formats d_format;
+
     /**
      * @brief Base file name stored for consistency between reading and writing.
      */
@@ -286,6 +299,20 @@ class BasisReader {
      * @brief Full file name of database including rank.
      */
     std::string full_file_name;
+
+    /**
+     * @brief Dimension of the basis on this processor.
+     *
+     * If negative, use the dimension from the rank-specific local file.
+     */
+    const int d_dim;
+
+    /**
+     * @brief Dimension of the basis on this processor.
+     *
+     * If negative, use the dimension from the rank-specific local file.
+     */
+    int d_global_dim;
 };
 
 }