Rados wait_for_safe is deprecated. Only possible to do async IO and e…

…nsure persisted on flush, or wait for persist on every IO.
ecmwf · Jun 23, 2024 · fe324d1 · fe324d1
1 parent dd5729c
commit fe324d1
Show file tree

Hide file tree

Showing 12 changed files with 88 additions and 114 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -63,13 +63,11 @@ ecbuild_add_option( FEATURE RADOS_STORE_MULTIPART
 
 ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_FLUSH
                     DEFAULT OFF
-                    DESCRIPTION "Ensure writes are persisted in Rados storage on flush." )
+                    DESCRIPTION "Ensure writes/puts are persisted in Rados storage on flush rather than immediately." )
 
-ecbuild_add_option( FEATURE RADOS_BACKENDS_PERSIST_ON_WRITE
-                    CONDITION NOT fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
+ecbuild_add_option( FEATURE RADOS_ADMIN
                     DEFAULT OFF
-                    DESCRIPTION "Ensure every object write or kv put is persisted immediately." )
-
+                    DESCRIPTION "Have unit tests create pools automatically rather than using an existing pool specified in the ECKIT_RADOS_TEST_POOL cmake variable." )
 
 ### FDB backend in indexed filesystem with table-of-contents, i.e. TOC
 ### Supports Lustre parallel filesystem stripping control

diff --git a/src/fdb5/fdb5_config.h.in b/src/fdb5/fdb5_config.h.in
@@ -14,7 +14,7 @@
 #cmakedefine fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD
 #cmakedefine fdb5_HAVE_RADOS_STORE_MULTIPART
 #cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
-#cmakedefine fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
+#cmakedefine fdb5_HAVE_RADOS_ADMIN
 #cmakedefine fdb5_HAVE_TOCFDB
 #cmakedefine fdb5_HAVE_DUMMY_DAOS
 #cmakedefine fdb5_HAVE_DAOSFDB

diff --git a/src/fdb5/rados/README b/src/fdb5/rados/README
@@ -52,61 +52,46 @@ cmake options:
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF
 
 # pool per db, multiple fields per obj
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF
 
 # single pool, field per obj
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF
 
 # pool per db, field per obj
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=OFF \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF
 
-# single pool, multiple fields per obj, multipart
+# single pool, multiple fields per obj, multipart (default)
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF
 
 # single pool, multiple fields per obj, persist on flush
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON
 
 # single pool, field per obj, persist on flush
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
-
-# single pool, field per obj, persist on write
-cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
-    -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
-    -DENABLE_RADOS_STORE_OBJ_PER_FIELD=ON -DENABLE_RADOS_STORE_MULTIPART=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=OFF \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=ON
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON
 
 # single pool, multiple fields per obj, multipart, persist on flush
 cmake $src_dir -DENABLE_MEMFS=ON -DENABLE_AEC=OFF -DENABLE_RADOS=ON \
     -DENABLE_RADOSFDB=ON -DENABLE_RADOS_BACKENDS_SINGLE_POOL=ON \
     -DENABLE_RADOS_STORE_OBJ_PER_FIELD=OFF -DENABLE_RADOS_STORE_MULTIPART=ON \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON \
-    -DENABLE_RADOS_BACKENDS_PERSIST_ON_WRITE=OFF
+    -DENABLE_RADOS_BACKENDS_PERSIST_ON_FLUSH=ON
diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc
@@ -342,7 +342,6 @@ void RadosCatalogueWriter::flush() {
 #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
     for (IndexStore::iterator j = indexes_.begin(); j != indexes_.end(); ++j )
         j->second.flush();
-flush axis kvs if not done as part of RadosIndex::flush
     db_kv_->flush();
     root_kv_->flush();
 #endif

diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc
@@ -31,27 +31,17 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen
 
     readConfig(config, component, true);
 
-  #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    root_kv_.emplace(pool_, root_namespace_, "main_kv", true);
-    db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true);
-  #else
     root_kv_.emplace(pool_, root_namespace_, "main_kv");
     db_kv_.emplace(pool_, db_namespace_, "catalogue_kv");
-  #endif
 
 #else
 
     readConfig(config, component, true);
 
     db_pool_ = prefix_ + "_" + key.valuesToString();
 
-  #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    root_kv_.emplace(root_pool_, namespace_, "main_kv", true);
-    db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true);
-  #else
     root_kv_.emplace(root_pool_, namespace_, "main_kv");
     db_kv_.emplace(db_pool_, namespace_, "catalogue_kv");
-  #endif
 
 #endif
 
@@ -71,13 +61,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen
 
     readConfig(config, component, false);
 
-  #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    root_kv_.emplace(pool_, root_namespace_, "main_kv", true);
-    db_kv_.emplace(pool_, db_namespace_, "catalogue_kv", true);
-  #else
     root_kv_.emplace(pool_, root_namespace_, "main_kv");
     db_kv_.emplace(pool_, db_namespace_, "catalogue_kv");
-  #endif
 
 #else
 
@@ -91,13 +76,8 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen
     ASSERT(n > 1);
     prefix_ = parts[0];
 
-  #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    root_kv_.emplace(root_pool_, namespace_, "main_kv", true);
-    db_kv_.emplace(db_pool_, namespace_, "catalogue_kv", true);
-  #else
     root_kv_.emplace(root_pool_, namespace_, "main_kv");
     db_kv_.emplace(db_pool_, namespace_, "catalogue_kv");
-  #endif
 
 #endif
 
@@ -150,7 +130,7 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp
     root_pool_ = eckit::Resource<std::string>("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_);
 
     prefix_ = c.getString("pool_prefix", prefix_);
-    if (c.has(component)) prefix_ = c.getSubconfiguration(component).getString("pool_prefix", prefix_);
+    if (c.has(component)) prefix_ = c.getSubConfiguration(component).getString("pool_prefix", prefix_);
     ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores.");
 
 #endif

diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h
@@ -16,7 +16,7 @@
 #include "eckit/filesystem/URI.h"
 #include "eckit/utils/Optional.h"
 #include "eckit/io/rados/RadosKeyValue.h"
-#include "eckit/io/rados/RadosPersistentKeyValue.h"
+#include "eckit/io/rados/RadosAsyncKeyValue.h"
 
 #include "fdb5/fdb5_config.h"
 #include "fdb5/database/Key.h"
@@ -51,9 +51,9 @@ class RadosCommon {
     std::string namespace_;
 #endif
 
-#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH)
-    eckit::Optional<eckit::RadosPersistentKeyValue> root_kv_;
-    eckit::Optional<eckit::RadosPersistentKeyValue> db_kv_;
+#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH)
+    eckit::Optional<eckit::RadosAsyncKeyValue> root_kv_;
+    eckit::Optional<eckit::RadosAsyncKeyValue> db_kv_;
 #else
     eckit::Optional<eckit::RadosKeyValue> root_kv_;
     eckit::Optional<eckit::RadosKeyValue> db_kv_;

diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc
@@ -46,11 +46,7 @@ namespace fdb5 {
 RadosIndex::RadosIndex(const Key& key, const eckit::RadosNamespace& name) : 
     IndexBase(key, "radosKeyValue"), 
     location_(eckit::RadosKeyValue{name.pool().name(), name.name(), key.valuesToString()}, 0),
-#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    idx_kv_(location_.radosName().uri(), true) {
-#else
     idx_kv_(location_.radosName().uri()) {
-#endif
 
     /// @note: performed RPCs:
     /// - generate index kv oid (daos_obj_generate_oid)
@@ -83,11 +79,7 @@ RadosIndex::RadosIndex(const Key& key, const eckit::RadosKeyValue& name, bool re
 // #endif
     IndexBase(key, "radosKeyValue"),
     location_(name, 0),
-#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    idx_kv_(name.uri(), true) {
-#else
     idx_kv_(name.uri()) {
-#endif
 
     if (readAxes) updateAxes();
 
@@ -105,18 +97,6 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value)
 
     if (axis_kv == axis_kvs_.end()) {
         std::string kv_name = key().valuesToString() + std::string{"."} + axis;
-#ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-        axis_kvs_.emplace(
-            std::piecewise_construct, 
-            std::forward_as_tuple(axis), 
-            std::forward_as_tuple(
-                location_.radosName().nspace().pool().name(),
-                location_.radosName().nspace().name(),
-                kv_name,
-                true
-            )
-        );
-#else
         axis_kvs_.emplace(
             std::piecewise_construct, 
             std::forward_as_tuple(axis), 
@@ -126,7 +106,6 @@ void RadosIndex::putAxisValue(const std::string& axis, const std::string& value)
                 kv_name
             )
         );
-#endif
 
         axis_kv = axis_kvs_.find(axis);
     }

diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h
@@ -15,7 +15,7 @@
 
 #include "eckit/io/rados/RadosNamespace.h"
 #include "eckit/io/rados/RadosKeyValue.h"
-#include "eckit/io/rados/RadosPersistentKeyValue.h"
+#include "eckit/io/rados/RadosAsyncKeyValue.h"
 
 #include "fdb5/database/Index.h"
 #include "fdb5/rados/RadosIndexLocation.h"
@@ -82,9 +82,9 @@ class RadosIndex : public IndexBase {
 
     fdb5::RadosIndexLocation location_;
 
-#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE) || defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH)
-    eckit::RadosPersistentKeyValue idx_kv_;
-    std::map<std::string, eckit::RadosPersistetKeyValue> axis_kvs_;
+#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH)
+    eckit::RadosAsyncKeyValue idx_kv_;
+    std::map<std::string, eckit::RadosAsyncKeyValue> axis_kvs_;
 #else
     eckit::RadosKeyValue idx_kv_;
     std::map<std::string, eckit::RadosKeyValue> axis_kvs_;

diff --git a/src/fdb5/rados/RadosIndexLocation.h b/src/fdb5/rados/RadosIndexLocation.h
@@ -15,7 +15,7 @@
 
 #include "eckit/exception/Exceptions.h"
 #include "eckit/io/rados/RadosKeyValue.h"
-#include "eckit/io/rados/RadosPersistentKeyValue.h"
+#include "eckit/io/rados/RadosAsyncKeyValue.h"
 
 #include "fdb5/database/IndexLocation.h"
 

diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc
@@ -209,11 +209,9 @@ std::unique_ptr<FieldLocation> RadosStore::archive(const Key& key, const void *
   #endif
 
   #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
-    eckit::DataHandle* h = o.persistentDataHandle();
+    eckit::DataHandle* h = o.asyncDataHandle();
     ASSERT(handles_.size() < maxHandleBuffSize_);
     handles_.push_back(h);
-  #elif fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_WRITE
-    std::unique_ptr<eckit::DataHandle> h(o.persistentDataHandle(true));
   #else
     std::unique_ptr<eckit::DataHandle> h(o.dataHandle());
   #endif
@@ -471,15 +469,15 @@ eckit::DataHandle& RadosStore::getDataHandle(const Key& key, const eckit::RadosO
   #ifdef fdb5_HAVE_RADOS_STORE_MULTIPART
 
     #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
-    eckit::DataHandle *dh = name.persistentMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_);
+    eckit::DataHandle *dh = name.asyncMultipartWriteHandle(maxObjectSize_, maxAioBuffSize_, maxPartHandleBuffSize_);
     #else
     eckit::DataHandle *dh = name.multipartWriteHandle(maxObjectSize_);
     #endif
 
   #else
 
     #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH
-    eckit::DataHandle *dh = name.persistentDataHandle(false, maxAioBuffSize_);
+    eckit::DataHandle *dh = name.asyncDataHandle(maxAioBuffSize_);
     #else
     eckit::DataHandle *dh = name.dataHandle();
     #endif

diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt
@@ -11,7 +11,8 @@ if (HAVE_RADOSFDB)
     ecbuild_add_test( TARGET test_fdb5_rados_${_test}
                       SOURCES test_${_test}.cc
                       LIBS "${unit_test_libraries}"
-                      INCLUDES "${unit_test_include_dirs}" )
+                      INCLUDES "${unit_test_include_dirs}"
+                      ENVIRONMENT FDB_RADOS_TEST_POOL=${FDB_RADOS_TEST_POOL} )
 
   endforeach()