MPIO support for Basis classes (#274)

* BasisWriter::writeBasis create/open a file and closes the file at the end.

* stylization.

* HDFDatabase::putIntegerArray - overwrites if the dataset exists.

* enforce single time interval in Options.

* HDFDatabase::putIntegerArray does not allow overwrite.

* BasisWriter::writeBasis always create the file, which will overwrite the exisiting file.

* add a header and stylization.

* remove increase time interval test, as time interval is fixed to 1.

* add an error message for a guidance.

* remove test_SVD from ci workflow.

* SVD::increaseTimeInterval - allow the initial time interval.

* minor fix in test_IncrementalSVDBrand.

* reflecting the comments.

* removed the concept of time interval in BasisReader. time argument remains for backward compatibility.

* BasisWriter: removed the concept of time interval.

* minor fix in BasisReader.

* SVD: removed the concept of time intervals.

* BasisGenerator: removed the concept of time interval.

* add test_SVD.cpp for resolving conflict.

* stylization.

* changed function signature of BasisGenerator::takeSample.

* rebased to resolve conflict.

* changed function signature for BasisReader::getSpatialBasis.

* changed function signature for BasisReader::getTemporalBasis.

* changed function signature for BasisReader::getSingularValues.

* changed function signature for BasisReader::getSnapshotMatrix.

* unit test with fapl_mpi.

* parallel integer array writing example.

* add timing for H5DWrite.

* 2d integer array parallel writing example.

* hdf5 parallel integer array reading example.

* add d_dim as const member variable of BasisGenerator/Reader.

* BasisReader: CSV format is never used. Enforce HDF5 format from now on.

* create/open_parallel function.

* stylization

* test_HDFDatabase: test for selective parallal I/O.

* parallel I/O routines within basis classes.

* HDFDatabaseMPIO class initial loading.

* rebase from generator-fix2

* deployed HDFDatabaseMPIO to basis classes.

* test for partial getSpatialBasis.

* test routien for scaling.

* add hdf database test in ci workflow.

* update doxygen description

* reflecting comments 1

* reflecting comments 2. changed interface.

* minor fix

* reflecting comments

* hdf5 parallel is optional. set compile-time if statements.

* minor fix

.github/workflows/run_tests/action.yml

@@ -23,6 +23,8 @@ runs:
           mpirun -n 3 --oversubscribe tests/test_StaticSVD
           ./tests/test_IncrementalSVDBrand
           mpirun -n 3 --oversubscribe tests/test_IncrementalSVDBrand
+          ./tests/test_HDFDatabase
+          mpirun -n 3 --oversubscribe tests/test_HDFDatabase
           ./tests/test_NNLS
           mpirun -n 3 --oversubscribe tests/test_NNLS
       shell: bash

CMakeLists.txt

@@ -265,6 +265,7 @@ if(GTEST_FOUND)
   set(unit_test_stems
     Vector
     Matrix
+    HDFDatabase
     DEIM
     DMD
     GNAT

docker/Dockerfile

@@ -10,7 +10,7 @@ WORKDIR /$ENVDIR
 # install packages
 RUN sudo apt-get install -yq git
 RUN sudo apt-get install --no-install-recommends -yq make gcc gfortran libssl-dev cmake
-RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-serial-dev
+RUN sudo apt-get install -yq libopenblas-dev libmpich-dev libblas-dev liblapack-dev libscalapack-mpi-dev libhdf5-mpi-dev hdf5-tools
 RUN sudo apt-get install -yq vim
 RUN sudo apt-get install -yq git-lfs
 RUN sudo apt-get install -yq valgrind

lib/CAROM_config.h.in

@@ -57,6 +57,9 @@
 /* Defined if you have HDF5 support */
 #cmakedefine01 CAROM_HAVE_HDF5
 
+/* Defined if you have parallel HDF5 support */
+#cmakedefine01 HDF5_IS_PARALLEL
+
 /* Define to 1 if you have the <inttypes.h> header file. */
 #cmakedefine01 CAROM_HAVE_INTTYPES_H
 

lib/CMakeLists.txt

@@ -54,6 +54,7 @@ set(module_list
   hyperreduction/Hyperreduction
   utils/Database
   utils/HDFDatabase
+  utils/HDFDatabaseMPIO
   utils/CSVDatabase
   utils/Utilities
   utils/ParallelBuffer

lib/linalg/BasisGenerator.cpp

@@ -31,6 +31,7 @@ BasisGenerator::BasisGenerator(
     bool incremental,
     const std::string& basis_file_name,
     Database::formats file_format) :
+    d_dim(options.dim),
     d_incremental(incremental),
     d_basis_writer(0),
     d_basis_reader(0),
@@ -157,7 +158,7 @@ BasisGenerator::loadSampleRange(const std::string& base_file_name,
 
     if (d_basis_reader) delete d_basis_reader;
 
-    d_basis_reader = new BasisReader(base_file_name, db_format);
+    d_basis_reader = new BasisReader(base_file_name, db_format, d_dim);
     const Matrix* mat;
     const Vector* singular_vals;
 

lib/linalg/BasisGenerator.h

@@ -64,7 +64,7 @@ class BasisGenerator
         Options options,
         bool incremental,
         const std::string& basis_file_name = "",
-        Database::formats file_format = Database::HDF5);
+        Database::formats file_format = Database::formats::HDF5);
 
     /**
      * @brief Destructor.
@@ -155,7 +155,7 @@ class BasisGenerator
                     const std::string& kind  = "basis",
                     int col_min = 0,
                     int col_max = 1e9,
-                    Database::formats db_format = Database::HDF5);
+                    Database::formats db_format = Database::formats::HDF5);
 
     /**
      * @brief Load previously saved sample (basis or state).
@@ -170,8 +170,8 @@ class BasisGenerator
     void
     loadSamples(const std::string& base_file_name,
                 const std::string& kind  = "basis",
-                int cutoff = 1e9,
-                Database::formats db_format = Database::HDF5);
+                int cut_off = 1e9,
+                Database::formats db_format = Database::formats::HDF5);
 
     /**
      * @brief Computes next time an svd sample is needed.
@@ -322,7 +322,7 @@ class BasisGenerator
     int
     getDim()
     {
-        return d_svd->getDim();
+        return d_dim;
     }
 
     /**
@@ -376,6 +376,13 @@ class BasisGenerator
      * @brief The number of processors being run on.
      */
     int d_num_procs;
+
+    /**
+     * @brief Dimension of the system on this processor.
+     *
+     * Equivalent to d_svd->getDim().
+     */
+    const int d_dim;
 };
 
 }

lib/linalg/BasisReader.cpp

@@ -12,18 +12,22 @@
 
 #include "BasisReader.h"
 #include "utils/HDFDatabase.h"
-#include "utils/CSVDatabase.h"
+#include "utils/HDFDatabaseMPIO.h"
 #include "Matrix.h"
 #include "Vector.h"
 #include "mpi.h"
+#include "utils/mpi_utils.h"
 
 namespace CAROM {
 
 BasisReader::BasisReader(
     const std::string& base_file_name,
-    Database::formats db_format) :
+    Database::formats db_format,
+    const int dim) :
+    d_dim(dim),
     full_file_name(""),
-    base_file_name_(base_file_name)
+    base_file_name_(base_file_name),
+    d_format(db_format)
 {
     CAROM_ASSERT(!base_file_name.empty());
 
@@ -37,17 +41,29 @@ BasisReader::BasisReader(
         rank = 0;
     }
 
-    char tmp[100];
-    sprintf(tmp, ".%06d", rank);
-    full_file_name = base_file_name + tmp;
-    if (db_format == Database::HDF5) {
+    full_file_name = base_file_name;
+
+    // Enforce hdf data format.
+    if (d_format == Database::formats::HDF5)
+    {
         d_database = new HDFDatabase();
     }
-    else if (db_format == Database::CSV) {
-        d_database = new CSVDatabase();
+    else if (d_format == Database::formats::HDF5_MPIO)
+    {
+        /*
+            For MPIO case, local dimension needs to be specified.
+            We allow 0 local dimension. (global dimension still needs to be positive)
+        */
+        std::vector<int> tmp;
+        d_global_dim = get_global_offsets(d_dim, tmp, MPI_COMM_WORLD);
+        CAROM_VERIFY(d_dim >= 0);
+        CAROM_VERIFY(d_global_dim > 0);
+        d_database = new HDFDatabaseMPIO();
     }
+    else
+        CAROM_ERROR("BasisWriter only supports HDF5/HDF5_MPIO data format!\n");
 
-    d_database->open(full_file_name, "r");
+    d_database->open(full_file_name, "r", MPI_COMM_WORLD);
 }
 
 BasisReader::~BasisReader()
@@ -66,7 +82,8 @@ BasisReader::getSpatialBasis()
 
     d_database->getDoubleArray("spatial_basis",
                                &spatial_basis_vectors->item(0, 0),
-                               num_rows*num_cols);
+                               num_rows*num_cols,
+                               true);
     return spatial_basis_vectors;
 }
 
@@ -97,7 +114,8 @@ BasisReader::getSpatialBasis(
                                num_rows*num_cols_to_read,
                                start_col - 1,
                                num_cols_to_read,
-                               num_cols);
+                               num_cols,
+                               true);
     return spatial_basis_vectors;
 }
 
@@ -135,7 +153,7 @@ BasisReader::getTemporalBasis()
     int num_cols = getNumSamples("temporal_basis");
 
     char tmp[100];
-    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols, true);
+    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols, false);
     sprintf(tmp, "temporal_basis");
     d_database->getDoubleArray(tmp,
                                &temporal_basis_vectors->item(0, 0),
@@ -163,7 +181,7 @@ BasisReader::getTemporalBasis(
     CAROM_VERIFY(start_col <= end_col && end_col <= num_cols);
     int num_cols_to_read = end_col - start_col + 1;
 
-    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols_to_read, true);
+    Matrix* temporal_basis_vectors = new Matrix(num_rows, num_cols_to_read, false);
     sprintf(tmp, "temporal_basis");
     d_database->getDoubleArray(tmp,
                                &temporal_basis_vectors->item(0, 0),
@@ -268,7 +286,18 @@ BasisReader::getDim(
                 "temporal_basis_num_rows");
 
     d_database->getInteger(tmp, num_rows);
-    return num_rows;
+    /* only basis and snapshot are stored as distributed matrices */
+    if ((kind != "temporal_basis") && (d_format == Database::formats::HDF5_MPIO))
+    {
+        /*
+            for MPIO database, return specified local dimension.
+            only checks the global dimension match.
+        */
+        CAROM_VERIFY(d_global_dim == num_rows);
+        return d_dim;
+    }
+    else
+        return num_rows;
 }
 
 int
@@ -298,11 +327,12 @@ BasisReader::getSnapshotMatrix()
     int num_cols = getNumSamples("snapshot");
 
     char tmp[100];
-    Matrix* snapshots = new Matrix(num_rows, num_cols, false);
+    Matrix* snapshots = new Matrix(num_rows, num_cols, true);
     sprintf(tmp, "snapshot_matrix");
     d_database->getDoubleArray(tmp,
                                &snapshots->item(0, 0),
-                               num_rows*num_cols);
+                               num_rows*num_cols,
+                               true);
     return snapshots;
 }
 
@@ -326,14 +356,15 @@ BasisReader::getSnapshotMatrix(
     int num_cols_to_read = end_col - start_col + 1;
 
     char tmp[100];
-    Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, false);
+    Matrix* snapshots = new Matrix(num_rows, num_cols_to_read, true);
     sprintf(tmp, "snapshot_matrix");
     d_database->getDoubleArray(tmp,
                                &snapshots->item(0, 0),
                                num_rows*num_cols_to_read,
                                start_col - 1,
                                num_cols_to_read,
-                               num_cols);
+                               num_cols,
+                               true);
     return snapshots;
 }
 }

lib/linalg/BasisReader.h

@@ -44,10 +44,13 @@ class BasisReader {
      * @param[in] db_format Format of the file to read.
      *                      One of the implemented file formats defined in
      *                      Database.
+     * @param[in] dim Number of rows of basis that will be read from a file.
+     *                If negative, will use the dimension from the rank-specific local file.
      */
     BasisReader(
         const std::string& base_file_name,
-        Database::formats db_format = Database::HDF5);
+        Database::formats db_format = Database::formats::HDF5,
+        const int dim = -1);
 
     /**
      * @brief Destructor.
@@ -197,6 +200,11 @@ class BasisReader {
      *
      * @brief Returns the dimension of the system on this processor.
      *
+     * @param[in] kind  Type of matrix whose row-dimension is returned.
+     *                  "basis" - local dimension of spatial basis
+     *                  "snapshot" - local dimension of snapshot matrix
+     *                  "temporal_basis" - global dimension of temporal basis
+     *
      * @return The dimension of the system on this processor.
      */
     int
@@ -277,6 +285,11 @@ class BasisReader {
      */
     Database* d_database;
 
+    /**
+     * @brief The database being read from.
+     */
+    Database::formats d_format;
+
     /**
      * @brief Base file name stored for consistency between reading and writing.
      */
@@ -286,6 +299,20 @@ class BasisReader {
      * @brief Full file name of database including rank.
      */
     std::string full_file_name;
+
+    /**
+     * @brief Dimension of the basis on this processor.
+     *
+     * If negative, use the dimension from the rank-specific local file.
+     */
+    const int d_dim;
+
+    /**
+     * @brief Dimension of the basis on this processor.
+     *
+     * If negative, use the dimension from the rank-specific local file.
+     */
+    int d_global_dim;
 };
 
 }