From 006bea57508d806cd7db4bc9e16a0e29b4a526a9 Mon Sep 17 00:00:00 2001 From: Nicolau Manubens Date: Tue, 25 Jun 2024 18:05:35 +0200 Subject: [PATCH] Added RadosCatalogue unit tests and several fixes. --- src/fdb5/CMakeLists.txt | 2 + src/fdb5/rados/RadosCatalogue.cc | 101 ++- src/fdb5/rados/RadosCatalogueWriter.cc | 10 +- src/fdb5/rados/RadosCommon.cc | 21 +- src/fdb5/rados/RadosCommon.h | 6 +- src/fdb5/rados/RadosEngine.cc | 291 ++++++++ src/fdb5/rados/RadosEngine.h | 97 +++ src/fdb5/rados/RadosFieldLocation.cc | 46 -- src/fdb5/rados/RadosIndex.cc | 4 +- src/fdb5/rados/RadosIndex.h | 1 + src/fdb5/rados/RadosStore.cc | 4 +- src/fdb5/rados/RadosStore.h | 2 - tests/fdb/rados/CMakeLists.txt | 1 + tests/fdb/rados/test_rados_catalogue.cc | 870 ++++++++++++++++++++++++ tests/fdb/rados/test_rados_store.cc | 115 ++-- 15 files changed, 1402 insertions(+), 169 deletions(-) create mode 100644 src/fdb5/rados/RadosEngine.cc create mode 100644 src/fdb5/rados/RadosEngine.h create mode 100644 tests/fdb/rados/test_rados_catalogue.cc diff --git a/src/fdb5/CMakeLists.txt b/src/fdb5/CMakeLists.txt index 8ccc6b39f..9f5841794 100644 --- a/src/fdb5/CMakeLists.txt +++ b/src/fdb5/CMakeLists.txt @@ -381,6 +381,8 @@ if( HAVE_RADOSFDB ) rados/RadosIndexLocation.h rados/RadosLazyFieldLocation.cc rados/RadosLazyFieldLocation.h + rados/RadosEngine.cc + rados/RadosEngine.h ) endif() diff --git a/src/fdb5/rados/RadosCatalogue.cc b/src/fdb5/rados/RadosCatalogue.cc index e6a84a05a..d45c57172 100644 --- a/src/fdb5/rados/RadosCatalogue.cc +++ b/src/fdb5/rados/RadosCatalogue.cc @@ -19,7 +19,7 @@ #include "fdb5/rados/RadosCatalogue.h" // #include "fdb5/daos/DaosName.h" // #include "fdb5/daos/DaosSession.h" -// #include "fdb5/daos/DaosIndex.h" +#include "fdb5/rados/RadosIndex.h" // #include "fdb5/daos/DaosWipeVisitor.h" // using namespace eckit; @@ -109,60 +109,53 @@ WipeVisitor* RadosCatalogue::wipeVisitor(const Store& store, const metkit::mars: std::vector RadosCatalogue::indexes(bool) const { - NOTIMP; + /// @note: sorted is not implemented as is not necessary in this backend. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db kv list keys (daos_kv_list) + + std::vector res; + + for (const auto& key : db_kv_->keys()) { + + /// @todo: document these well. Single source these reserved values. + /// Ensure where appropriate that user-provided keys do not collide. + if (key == "schema" || key == "key") continue; + + /// @note: performed RPCs: + /// - db kv get index location size (daos_kv_get without a buffer) + /// - db kv get index location (daos_kv_get) + std::vector v; + auto m = db_kv_->getMemoryStream(v, key, "DB kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + + /// @note: performed RPCs: + /// - index kv open (daos_kv_open) + /// - index kv get size (daos_kv_get without a buffer) + /// - index kv get key (daos_kv_get) + /// @note: the following three lines intend to check whether the index kv exists + /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, + /// so it is not useful on its own to check whether the index KV existed or not. + /// Instead, presence of a "key" key in the KV is used to determine if the index + /// KV existed. + eckit::RadosKeyValue index_kv{uri}; + std::optional index_key; + try { + std::vector data; + eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); + index_key.emplace(ms); + } catch (eckit::RadosEntityNotFoundException& e) { + continue; /// @note: the index_kv may not exist after a failed wipe + /// @todo: the index_kv may exist even if it does not have the "key" key + } + + res.push_back(Index(new fdb5::RadosIndex(index_key.value(), index_kv, false))); + + } -// /// @note: sorted is not implemented as is not necessary in this backend. - -// fdb5::DaosKeyValueName catalogue_kv_name{pool_, db_cont_, catalogue_kv_}; -// fdb5::DaosSession s{}; - -// /// @note: performed RPCs: -// /// - db kv open (daos_kv_open) -// /// - db kv list keys (daos_kv_list) -// fdb5::DaosKeyValue catalogue_kv{s, catalogue_kv_name}; /// @note: throws if not exists - -// std::vector res; - -// for (const auto& key : catalogue_kv.keys()) { - -// /// @todo: document these well. Single source these reserved values. -// /// Ensure where appropriate that user-provided keys do not collide. -// if (key == "schema" || key == "key") continue; - -// /// @note: performed RPCs: -// /// - db kv get index location size (daos_kv_get without a buffer) -// /// - db kv get index location (daos_kv_get) -// uint64_t size{catalogue_kv.size(key)}; -// std::vector v(size); -// catalogue_kv.get(key, v.data(), size); - -// fdb5::DaosKeyValueName index_kv_name{eckit::URI(std::string(v.begin(), v.end()))}; - -// /// @note: performed RPCs: -// /// - index kv open (daos_kv_open) -// /// - index kv get size (daos_kv_get without a buffer) -// /// - index kv get key (daos_kv_get) -// /// @note: the following three lines intend to check whether the index kv exists -// /// or not. The DaosKeyValue constructor calls kv open, which always succeeds, -// /// so it is not useful on its own to check whether the index KV existed or not. -// /// Instead, presence of a "key" key in the KV is used to determine if the index -// /// KV existed. -// fdb5::DaosKeyValue index_kv{s, index_kv_name}; -// std::optional index_key; -// try { -// std::vector data; -// eckit::MemoryStream ms = index_kv.getMemoryStream(data, "key", "index KV"); -// index_key.emplace(ms); -// } catch (fdb5::DaosEntityNotFoundException& e) { -// continue; /// @note: the index_kv may not exist after a failed wipe -// /// @todo: the index_kv may exist even if it does not have the "key" key -// } - -// res.push_back(Index(new fdb5::DaosIndex(index_key.value(), index_kv_name, false))); - -// } - -// return res; + return res; } diff --git a/src/fdb5/rados/RadosCatalogueWriter.cc b/src/fdb5/rados/RadosCatalogueWriter.cc index b84f434db..eb0a2471a 100644 --- a/src/fdb5/rados/RadosCatalogueWriter.cc +++ b/src/fdb5/rados/RadosCatalogueWriter.cc @@ -52,9 +52,13 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c /// @note: performed RPCs: /// - check if main kv contains db key (daos_kv_get without a buffer) + root_kv_->ensureCreated(); if (!root_kv_->has(db_name)) { /// create catalogue kv +#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + db_kv_->nspace().pool().ensureCreated(); +#endif db_kv_->ensureCreated(); /// write schema under "schema" @@ -68,7 +72,11 @@ RadosCatalogueWriter::RadosCatalogueWriter(const Key &key, const fdb5::Config& c eckit::FileHandle in(config_.schemaPath()); std::vector data; data.resize(in.size()); - in.read(&data[0], in.size()); + { + eckit::AutoClose ac{in}; + in.openForRead(); + in.read(&data[0], in.size()); + } db_kv_->put("schema", &data[0], data.size()); /// write dbKey under "key" diff --git a/src/fdb5/rados/RadosCommon.cc b/src/fdb5/rados/RadosCommon.cc index bf69ad6f7..778122712 100644 --- a/src/fdb5/rados/RadosCommon.cc +++ b/src/fdb5/rados/RadosCommon.cc @@ -27,10 +27,10 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - db_namespace_ = key.valuesToString(); - readConfig(config, component, true); + db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + root_kv_.emplace(pool_, root_namespace_, "main_kv"); db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); @@ -38,7 +38,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen readConfig(config, component, true); - db_pool_ = prefix_ + "_" + key.valuesToString(); + db_pool_ = pool_prefix_ + "_" + key.valuesToString(); root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); @@ -74,7 +74,7 @@ RadosCommon::RadosCommon(const fdb5::Config& config, const std::string& componen const auto parts = eckit::Tokenizer("_").tokenize(db_pool_); const auto n = parts.size(); ASSERT(n > 1); - prefix_ = parts[0]; + pool_prefix_ = parts[0]; root_kv_.emplace(root_pool_, namespace_, "main_kv"); db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); @@ -110,28 +110,35 @@ void RadosCommon::readConfig(const fdb5::Config& config, const std::string& comp pool_ = c.getString("pool", pool_); if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); } + root_namespace_ = c.getString("root_namespace", root_namespace_); if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); if (readPool) pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); + if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + #else if (readNamespace) namespace_ = "default"; root_pool_ = "root"; if (readNamespace) + namespace_ = c.getString("namespace", namespace_); if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + root_pool_ = c.getString("root_pool", root_pool_); if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); if (readNamespace) namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); - prefix_ = c.getString("pool_prefix", prefix_); - if (c.has(component)) prefix_ = c.getSubConfiguration(component).getString("pool_prefix", prefix_); - ASSERT_MSG(prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); + pool_prefix_ = c.getString("pool_prefix", pool_prefix_); + if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); #endif diff --git a/src/fdb5/rados/RadosCommon.h b/src/fdb5/rados/RadosCommon.h index 43ab61a1e..24b94bf51 100644 --- a/src/fdb5/rados/RadosCommon.h +++ b/src/fdb5/rados/RadosCommon.h @@ -61,10 +61,12 @@ class RadosCommon { eckit::Length maxObjectSize_; -#ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL private: // members - std::string prefix_; +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string nspace_prefix_; +#else + std::string pool_prefix_; #endif }; diff --git a/src/fdb5/rados/RadosEngine.cc b/src/fdb5/rados/RadosEngine.cc new file mode 100644 index 000000000..72238d852 --- /dev/null +++ b/src/fdb5/rados/RadosEngine.cc @@ -0,0 +1,291 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + + +#include "eckit/serialisation/MemoryStream.h" +#include "eckit/config/Resource.h" + +#include "fdb5/LibFdb5.h" +#include "fdb5/rados/RadosEngine.h" + +using namespace eckit; + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +std::string RadosEngine::name() const { + return RadosEngine::typeName(); +} + +// bool DaosEngine::canHandle(const eckit::URI& uri, const Config& config) const { + +// configureDaos(config); + +// if (uri.scheme() != "daos") +// return false; + +// fdb5::DaosName n{uri}; + +// if (!n.hasOID()) return false; + +// /// @todo: check containerName is not root_cont_. root_cont_ should be populated in +// /// configureDaos as done in DaosCommon +// // bool is_root_name = (n.containerName().find(root_cont_) != std::string::npos); +// bool is_root_name = false; +// bool is_store_name = (n.containerName().find("_") != std::string::npos); + +// /// @note: performed RPCs: +// /// - generate oids (daos_obj_generate_oid) +// /// - db kv open (daos_kv_open) + +// fdb5::DaosName n2{n.poolName(), n.containerName(), catalogue_kv_}; +// bool is_catalogue_kv = (!is_root_name && !is_store_name && (n.OID() == n2.OID())); + +// return is_catalogue_kv && n.exists(); + +// } + +std::vector RadosEngine::visitableLocations(const Key& key, const Config& config) const +{ + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + std::string component = "catalogue"; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + readConfig(config, component, true); + + // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + +#else + + readConfig(config, component, true); + + // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + +#endif + + /// --- + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + if (!root_kv_->exists()) return res; + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : root_kv_->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + std::vector v; + auto m = root_kv_->getMemoryStream(v, k, "root kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.match(key)) { + + Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +std::vector RadosEngine::visitableLocations(const metkit::mars::MarsRequest& request, const Config& config) const +{ + + /// @note: code mostly copied from DaosCommon + /// @note: should rather use DaosCommon, but can't inherit from it here as DaosEngine is + /// always instantiated even if daos is not used, and then DaosCommon would be unnecessarily + /// initialised. If owning a private instance of DaosCommon here, then the private members of + /// DaosCommon are not accessible from here + + std::string component = "catalogue"; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + readConfig(config, component, true); + + // db_namespace_ = nspace_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(pool_, root_namespace_, "main_kv"); + // db_kv_.emplace(pool_, db_namespace_, "catalogue_kv"); + +#else + + readConfig(config, component, true); + + // db_pool_ = pool_prefix_ + "_" + key.valuesToString(); + + root_kv_.emplace(root_pool_, namespace_, "main_kv"); + // db_kv_.emplace(db_pool_, namespace_, "catalogue_kv"); + +#endif + + /// --- + + std::vector res{}; + + /// @note: performed RPCs: + /// - main kv open (daos_kv_open) + + if (!root_kv_->exists()) return res; + + /// @note: performed RPCs: + /// - main kv list keys (daos_kv_list) + for (const auto& k : root_kv_->keys()) { + + try { + + /// @note: performed RPCs: + /// - main kv get db location size (daos_kv_get without a buffer) + /// - main kv get db location (daos_kv_get) + std::vector v; + auto m = root_kv_->getMemoryStream(v, k, "root kv"); + + eckit::URI uri(std::string(v.begin(), v.end())); + ASSERT(uri.scheme() == typeName()); + + /// @todo: this exact deserialisation is performed twice. Once here and once + /// in DaosCatalogue::(uri, ...). Try to avoid one. + + /// @note: performed RPCs: + /// - db kv open (daos_kv_open) + /// - db key get size (daos_kv_get without a buffer) + /// - db key get (daos_kv_get) + eckit::RadosKeyValue db_kv{uri}; /// @note: includes exist check + std::vector data; + eckit::MemoryStream ms = db_kv.getMemoryStream(data, "key", "DB kv"); + fdb5::Key db_key(ms); + + if (db_key.partialMatch(request)) { + + Log::debug() << " found match with " << root_kv_->uri() << " at key " << k << std::endl; + res.push_back(uri); + + } + + } catch (eckit::Exception& e) { + eckit::Log::error() << "Error loading FDB database " << k << " from " << root_kv_->uri() << std::endl; + eckit::Log::error() << e.what() << std::endl; + } + + } + + return res; + +} + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL +void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const { +#else +void RadosEngine::readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const { +#endif + + eckit::LocalConfiguration c{}; + + if (config.has("rados")) c = config.getSubConfiguration("rados"); + + // maxObjectSize_ = c.getInt("maxObjectSize", 0); + + std::string first_cap{component}; + first_cap[0] = toupper(component[0]); + + std::string all_caps{component}; + for (auto & c: all_caps) c = toupper(c); + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + + if (readPool) pool_ = "default"; + root_namespace_ = "root"; + + if (readPool) { + pool_ = c.getString("pool", pool_); + if (c.has(component)) pool_ = c.getSubConfiguration(component).getString("pool", pool_); + } + root_namespace_ = c.getString("root_namespace", root_namespace_); + if (c.has(component)) root_namespace_ = c.getSubConfiguration(component).getString("root_namespace", root_namespace_); + + if (readPool) + pool_ = eckit::Resource("fdbRados" + first_cap + "Pool;$FDB_RADOS_" + all_caps + "_POOL", pool_); + root_namespace_ = eckit::Resource("fdbRados" + first_cap + "RootNamespace;$FDB_RADOS_" + all_caps + "_ROOT_NAMESPACE", root_namespace_); + + nspace_prefix_ = c.getString("namespace_prefix", nspace_prefix_); + if (c.has(component)) nspace_prefix_ = c.getSubConfiguration(component).getString("namespace_prefix", nspace_prefix_); + ASSERT_MSG(nspace_prefix_.find("_") == std::string::npos, "The configured namespace prefix must not contain underscores."); + +#else + + if (readNamespace) namespace_ = "default"; + root_pool_ = "root"; + + if (readNamespace) + namespace_ = c.getString("namespace", namespace_); + if (c.has(component)) namespace_ = c.getSubConfiguration(component).getString("namespace", namespace_); + root_pool_ = c.getString("root_pool", root_pool_); + if (c.has(component)) root_pool_ = c.getSubConfiguration(component).getString("root_pool", root_pool_); + + if (readNamespace) + namespace_ = eckit::Resource("fdbRados" + first_cap + "Namespace;$FDB_RADOS_" + all_caps + "_NAMESPACE", namespace_); + root_pool_ = eckit::Resource("fdbRados" + first_cap + "RootPool;$FDB_RADOS_" + all_caps + "_ROOT_POOL", root_pool_); + + pool_prefix_ = c.getString("pool_prefix", pool_prefix_); + if (c.has(component)) pool_prefix_ = c.getSubConfiguration(component).getString("pool_prefix", pool_prefix_); + ASSERT_MSG(pool_prefix_.find("_") == std::string::npos, "The configured pool prefix must not contain underscores."); + +#endif + +} + +static EngineBuilder rados_builder; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosEngine.h b/src/fdb5/rados/RadosEngine.h new file mode 100644 index 000000000..b48f7dc2c --- /dev/null +++ b/src/fdb5/rados/RadosEngine.h @@ -0,0 +1,97 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/// @author Nicolau Manubens +/// @date Jun 2024 + +#pragma once + +#include "eckit/utils/Optional.h" +#include "eckit/io/rados/RadosKeyValue.h" +#include "eckit/io/rados/RadosAsyncKeyValue.h" + +#include "fdb5/database/Engine.h" +#include "fdb5/fdb5_config.h" + +namespace fdb5 { + +//---------------------------------------------------------------------------------------------------------------------- + +class RadosEngine : public fdb5::Engine { + +public: // methods + + RadosEngine() {}; + + static const char* typeName() { return "rados"; } + +protected: // methods + + virtual std::string name() const override; + + virtual std::string dbType() const override { NOTIMP; }; + + virtual eckit::URI location(const Key &key, const Config& config) const override { NOTIMP; }; + + virtual bool canHandle(const eckit::URI&, const Config&) const override { NOTIMP; }; + + virtual std::vector allLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual std::vector visitableLocations(const Key& key, const Config& config) const override; + virtual std::vector visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override; + + virtual std::vector writableLocations(const Key& key, const Config& config) const override { NOTIMP; }; + + virtual void print( std::ostream &out ) const override { NOTIMP; }; + +private: // methods + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + void readConfig(const fdb5::Config& config, const std::string& component, bool readPool) const; +#else + void readConfig(const fdb5::Config& config, const std::string& component, bool readNamespace) const; +#endif + +protected: // members + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + mutable std::string pool_; + mutable std::string root_namespace_; + // std::string db_namespace_; +#else + mutable std::string root_pool_; + // std::string db_pool_; + mutable std::string namespace_; +#endif + +#if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) + mutable eckit::Optional root_kv_; + // eckit::Optional db_kv_; +#else + mutable eckit::Optional root_kv_; + // eckit::Optional db_kv_; +#endif + + // eckit::Length maxObjectSize_; + +private: // members + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + mutable std::string nspace_prefix_; +#else + mutable std::string pool_prefix_; +#endif + +}; + +//---------------------------------------------------------------------------------------------------------------------- + + +} // namespace fdb5 diff --git a/src/fdb5/rados/RadosFieldLocation.cc b/src/fdb5/rados/RadosFieldLocation.cc index bafbbb99f..919d11f49 100644 --- a/src/fdb5/rados/RadosFieldLocation.cc +++ b/src/fdb5/rados/RadosFieldLocation.cc @@ -79,50 +79,4 @@ void RadosFieldLocation::visit(FieldLocationVisitor& visitor) const { //---------------------------------------------------------------------------------------------------------------------- -class RadosURIManager : public eckit::URIManager { - virtual bool query() override { return true; } - virtual bool fragment() override { return true; } - - // virtual eckit::PathName path(const eckit::URI& f) const override { return f.name(); } - - virtual bool exists(const eckit::URI& f) override { - - return eckit::RadosObject(f).exists(); - - } - - virtual eckit::DataHandle* newWriteHandle(const eckit::URI& f) override { - - return eckit::RadosObject(f).dataHandle(); - - } - - virtual eckit::DataHandle* newReadHandle(const eckit::URI& f) override { - - return eckit::RadosObject(f).dataHandle(); - - } - - virtual eckit::DataHandle* newReadHandle(const eckit::URI& f, const eckit::OffsetList& ol, const eckit::LengthList& ll) override { - - NOTIMP; - - } - - virtual std::string asString(const eckit::URI& uri) const override { - std::string q = uri.query(); - if (!q.empty()) - q = "?" + q; - std::string f = uri.fragment(); - if (!f.empty()) - f = "#" + f; - - return uri.scheme() + ":" + uri.name() + q + f; - } -public: - RadosURIManager(const std::string& name) : eckit::URIManager(name) {} -}; - -static RadosURIManager rados_uri_manager("rados"); - } // namespace fdb5 \ No newline at end of file diff --git a/src/fdb5/rados/RadosIndex.cc b/src/fdb5/rados/RadosIndex.cc index 3a15d610f..9cce9feb4 100644 --- a/src/fdb5/rados/RadosIndex.cc +++ b/src/fdb5/rados/RadosIndex.cc @@ -290,8 +290,8 @@ const std::vector RadosIndex::dataURIs() const { #ifdef fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH void RadosIndex::flush() { - for (auto axis : axis_kvs_) { - axis->second.flush(); + for (auto& axis : axis_kvs_) { + axis.second.flush(); } idx_kv_.flush(); diff --git a/src/fdb5/rados/RadosIndex.h b/src/fdb5/rados/RadosIndex.h index 73302309f..4762b4d5a 100644 --- a/src/fdb5/rados/RadosIndex.h +++ b/src/fdb5/rados/RadosIndex.h @@ -17,6 +17,7 @@ #include "eckit/io/rados/RadosKeyValue.h" #include "eckit/io/rados/RadosAsyncKeyValue.h" +#include "fdb5/fdb5_config.h" #include "fdb5/database/Index.h" #include "fdb5/rados/RadosIndexLocation.h" diff --git a/src/fdb5/rados/RadosStore.cc b/src/fdb5/rados/RadosStore.cc index 6fbcae491..437f14fed 100644 --- a/src/fdb5/rados/RadosStore.cc +++ b/src/fdb5/rados/RadosStore.cc @@ -224,7 +224,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * h->write(data, length); - return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key())); + return std::unique_ptr(new RadosFieldLocation(o.uri(), 0, length, fdb5::Key(nullptr, true))); #else @@ -251,7 +251,7 @@ std::unique_ptr RadosStore::archive(const Key& key, const void * ASSERT(len == length); - return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key())); + return std::unique_ptr(new RadosFieldLocation(o.uri(), offset, length, fdb5::Key(nullptr, true))); #endif diff --git a/src/fdb5/rados/RadosStore.h b/src/fdb5/rados/RadosStore.h index 33989328f..144f98d53 100644 --- a/src/fdb5/rados/RadosStore.h +++ b/src/fdb5/rados/RadosStore.h @@ -16,8 +16,6 @@ #include "eckit/io/rados/RadosObject.h" -#include "fdb5/fdb5_config.h" - #include "fdb5/database/Store.h" #include "fdb5/rules/Schema.h" diff --git a/tests/fdb/rados/CMakeLists.txt b/tests/fdb/rados/CMakeLists.txt index de2661407..30d4caf3b 100644 --- a/tests/fdb/rados/CMakeLists.txt +++ b/tests/fdb/rados/CMakeLists.txt @@ -2,6 +2,7 @@ if (HAVE_RADOSFDB) list( APPEND rados_tests rados_store + rados_catalogue ) list( APPEND unit_test_libraries fdb5 ) diff --git a/tests/fdb/rados/test_rados_catalogue.cc b/tests/fdb/rados/test_rados_catalogue.cc new file mode 100644 index 000000000..fab033e13 --- /dev/null +++ b/tests/fdb/rados/test_rados_catalogue.cc @@ -0,0 +1,870 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +// #include +// #include + +#include "eckit/config/Resource.h" +#include "eckit/testing/Test.h" +// #include "eckit/filesystem/URI.h" +#include "eckit/filesystem/PathName.h" +#include "eckit/filesystem/TmpFile.h" +// #include "eckit/filesystem/TmpDir.h" +// #include "eckit/io/FileHandle.h" +#include "eckit/io/MemoryHandle.h" +#include "eckit/config/YAMLConfiguration.h" +#include "eckit/io/PartHandle.h" + +// #include "metkit/mars/MarsRequest.h" + +#include "fdb5/fdb5_config.h" +// #include "fdb5/config/Config.h" +#include "fdb5/api/FDB.h" +#include "fdb5/api/helpers/FDBToolRequest.h" + +#include "fdb5/toc/TocStore.h" + +// #include "fdb5/daos/DaosSession.h" +// #include "fdb5/daos/DaosPool.h" +// #include "fdb5/daos/DaosArrayPartHandle.h" + +#include "fdb5/rados/RadosStore.h" +#include "fdb5/rados/RadosFieldLocation.h" +#include "fdb5/rados/RadosCatalogueWriter.h" +#include "fdb5/rados/RadosCatalogueReader.h" + +using namespace eckit::testing; +using namespace eckit; + +namespace { + + void deldir(eckit::PathName& p) { + if (!p.exists()) { + return; + } + + std::vector files; + std::vector dirs; + p.children(files, dirs); + + for (auto& f : files) { + f.unlink(); + } + for (auto& d : dirs) { + deldir(d); + } + + p.rmdir(); + }; + + void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } + } + +#ifdef fdb5_HAVE_RADOS_ADMIN + void ensureClean(const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listPools()) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosPool{name}.destroy(); + } + } + } +#endif + +} + +// temporary schema,spaces,root files common to all DAOS Catalogue tests + +eckit::TmpFile& schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::TmpFile& opt_schema_file() { + static eckit::TmpFile f{}; + return f; +} + +eckit::PathName& catalogue_tests_tmp_root() { + static eckit::PathName cd("./rados_catalogue_tests_fdb_root"); + return cd; +} + +namespace fdb { +namespace test { + +CASE( "Setup" ) { + +#if !defined(fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL) && !defined(fdb5_HAVE_RADOS_ADMIN) + throw eckit::Exception( + "RadosStore unit tests require Rados admin permissions to create pools if " + "RADOS_BACKENDS_SINGLE_POOL=OFF, and require enabling RADOS_ADMIN=ON."); +#endif + + // ensure fdb root directory exists. If not, then that root is + // registered as non existing and Catalogue/Store tests fail. + if (catalogue_tests_tmp_root().exists()) deldir(catalogue_tests_tmp_root()); + catalogue_tests_tmp_root().mkdir(); + ::setenv("FDB_ROOT_DIRECTORY", catalogue_tests_tmp_root().path().c_str(), 1); + + // prepare schema for tests involving DaosCatalogue + + std::string schema_str{"[ a, b [ c, d [ e, f ]]]"}; + + std::unique_ptr hs(schema_file().fileHandle()); + hs->openForWrite(schema_str.size()); + { + eckit::AutoClose closer(*hs); + hs->write(schema_str.data(), schema_str.size()); + } + + std::string opt_schema_str{"[ a, b [ c?, d [ e?, f ]]]"}; + + std::unique_ptr hs_opt(opt_schema_file().fileHandle()); + hs_opt->openForWrite(opt_schema_str.size()); + { + eckit::AutoClose closer(*hs_opt); + hs_opt->write(opt_schema_str.data(), opt_schema_str.size()); + } + + // this is necessary to avoid ~fdb/etc/fdb/schema being used where + // LibFdb5::instance().defaultConfig().schema() is called + // due to no specified schema file (e.g. in Key::registry()) + ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); + +} + +CASE("RadosCatalogue tests") { + + std::string test_id = "test-catalogue"; +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + #ifdef eckit_HAVE_RADOS_ADMIN + std::string pool = test_id; + eckit::RadosPool{pool}.ensureDestroyed(); + eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer + #else + std::string pool; + pool = eckit::Resource( + "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool + ); + EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); + #endif +#else + std::string prefix = test_id; + ensureClean(prefix); +#endif + + SECTION("DaosCatalogue archive (index) and retrieve without a Store") { + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " catalogue:\n" + " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" + }; +#else + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " catalogue:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + fdb5::Schema schema{schema_file()}; + + /// @note: a=11,b=22 instead of a=1,b=2 to avoid collision with potential parallel runs of store tests using a=1,b=2 + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}, schema.registry()); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}, schema.registry()); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}, schema.registry()); + + // archive + + std::unique_ptr loc(new fdb5::RadosFieldLocation( + eckit::URI{"rados", "test_uri"}, eckit::Offset(0), eckit::Length(1), fdb5::Key(nullptr, true) + )); + + { + fdb5::RadosCatalogueWriter dcatw{db_key, config}; + + // fdb5::DaosName db_cont{pool_name, db_key.valuesToString()}; + // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // EXPECT(db_cont.exists()); + // EXPECT(cat_kv.exists()); + + fdb5::Catalogue& cat = dcatw; + cat.selectIndex(index_key); + // fdb5::DaosKeyValueOID index_kv_oid{index_key.valuesToString(), OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName index_kv{pool_name, db_key.valuesToString(), index_kv_oid}; + // EXPECT(index_kv.exists()); + // EXPECT(cat_kv.has(index_key.valuesToString())); + + fdb5::CatalogueWriter& catw = dcatw; + catw.archive(field_key, std::move(loc)); + cat.flush(); + // EXPECT(index_kv.has(field_key.valuesToString())); + // fdb5::DaosKeyValueOID e_axis_kv_oid{index_key.valuesToString() + std::string{".e"}, OC_S1}; + // fdb5::DaosKeyValueName e_axis_kv{pool_name, db_key.valuesToString(), e_axis_kv_oid}; + // EXPECT(e_axis_kv.exists()); + // EXPECT(e_axis_kv.has("5")); + // fdb5::DaosKeyValueOID f_axis_kv_oid{index_key.valuesToString() + std::string{".f"}, OC_S1}; + // fdb5::DaosKeyValueName f_axis_kv{pool_name, db_key.valuesToString(), f_axis_kv_oid}; + // EXPECT(f_axis_kv.exists()); + // EXPECT(f_axis_kv.has("6")); + } + + // retrieve + + { + fdb5::RadosCatalogueReader dcatr{db_key, config}; + + fdb5::Catalogue& cat = dcatr; + cat.selectIndex(index_key); + + fdb5::Field f; + fdb5::CatalogueReader& catr = dcatr; + catr.retrieve(field_key, f); + EXPECT(f.location().uri().name() == eckit::URI("rados", "test_uri").name()); + EXPECT(f.location().offset() == eckit::Offset(0)); + EXPECT(f.location().length() == eckit::Length(1)); + } + + // // remove (manual deindex) + + // { + // fdb5::DaosCatalogueWriter dcatw{db_key, config}; + // fdb5::DaosName db_cont{dcatw.uri()}; + // std::ostream out(std::cout.rdbuf()); + + // fdb5::DaosCatalogue::remove(db_cont, out, out, true); + + // fdb5::DaosKeyValueOID cat_kv_oid{0, 0, OC_S1}; /// @todo: take oclass from config + // fdb5::DaosKeyValueName cat_kv{pool_name, db_key.valuesToString(), cat_kv_oid}; + // EXPECT_NOT(cat_kv.exists()); + // EXPECT_NOT(db_cont.exists()); + // } + + } + + SECTION("RadosCatalogue archive (index) and retrieve with a RadosStore") { + + // FDB configuration + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" + }; +#else + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "schema : " + schema_file().path() + "\n" + "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n" + }; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // schema + + fdb5::Schema schema{schema_file()}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // store data + + char data[] = "test"; + + fdb5::RadosStore rstore{schema, db_key, config}; + fdb5::Store& store = static_cast(rstore); + std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + + // index data + + { + fdb5::RadosCatalogueWriter rcatw{db_key, config}; + fdb5::Catalogue& cat = rcatw; + cat.deselectIndex(); + cat.selectIndex(index_key); + fdb5::CatalogueWriter& catw = rcatw; + catw.archive(field_key, std::move(loc)); + + /// flush store before flushing catalogue + rstore.flush(); // not necessary if using a DAOS store + } + + // find data + + fdb5::Field field; + { + fdb5::RadosCatalogueReader rcatr{db_key, config}; + fdb5::Catalogue& cat = rcatr; + cat.selectIndex(index_key); + fdb5::CatalogueReader& catr = rcatr; + catr.retrieve(field_key, field); + } + std::cout << "Read location: " << field.location() << std::endl; + + // retrieve data + + std::unique_ptr dh(store.retrieve(field)); + EXPECT(dynamic_cast(dh.get())); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // // deindex data + + // { + // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(dcat); + // std::ostream out(std::cout.rdbuf()); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } + + } + + // SECTION("DaosCatalogue archive (index) and retrieve with a TocStore") { + + // // FDB configuration + + // std::string config_str{ + // "spaces:\n" + // "- roots:\n" + // " - path: " + catalogue_tests_tmp_root().asString() + "\n" + // "schema : " + schema_file().path() + "\n" + // "daos:\n" + // " catalogue:\n" + // " pool: " + pool_name + "\n" + // " root_cont: " + root_cont_name + "\n" + // " client:\n" + // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + // }; + + // fdb5::Config config{YAMLConfiguration(config_str)}; + + // // schema + + // fdb5::Schema schema{schema_file()}; + + // // request + + // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + // fdb5::Key index_key({{"c", "3"}, {"d", "4"}}); + // fdb5::Key field_key({{"e", "5"}, {"f", "6"}}); + + // // store data + + // char data[] = "test"; + + // fdb5::TocStore tstore{schema, db_key, config}; + // fdb5::Store& store = static_cast(tstore); + // std::unique_ptr loc(store.archive(index_key, data, sizeof(data))); + // /// @todo: there are two cont create with label here + // /// @todo: again, daos_fini happening before cont and pool close + + // // index data + + // { + // fdb5::DaosCatalogueWriter dcatw{db_key, config}; + // fdb5::Catalogue& cat = dcatw; + // cat.deselectIndex(); + // cat.selectIndex(index_key); + // fdb5::CatalogueWriter& catw = dcatw; + // catw.archive(field_key, std::move(loc)); + + // /// flush store before flushing catalogue + // tstore.flush(); + // } + + // // find data + + // fdb5::Field field; + // { + // fdb5::DaosCatalogueReader dcatr{db_key, config}; + // fdb5::Catalogue& cat = dcatr; + // cat.selectIndex(index_key); + // fdb5::CatalogueReader& catr = dcatr; + // catr.retrieve(field_key, field); + // } + // std::cout << "Read location: " << field.location() << std::endl; + + // // retrieve data + + // std::unique_ptr dh(store.retrieve(field)); + + // std::vector test(dh->size()); + // dh->openForRead(); + // { + // eckit::AutoClose closer(*dh); + // dh->read(&test[0], test.size() - 3); + // } + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // // remove data + + // /// @todo: should DaosStore::remove accept full URIs to field arrays and remove the store container? + // eckit::PathName store_path{field.location().uri().path()}; + // std::ostream out(std::cout.rdbuf()); + // store.remove(field.location().uri(), out, out, false); + // EXPECT(store_path.exists()); + // store.remove(field.location().uri(), out, out, true); + // EXPECT_NOT(store_path.exists()); + + // // deindex data + + // { + // fdb5::DaosCatalogueWriter dcat{db_key, config}; + // fdb5::Catalogue& cat = static_cast(dcat); + // std::ostream out(std::cout.rdbuf()); + // metkit::mars::MarsRequest r = db_key.request("retrieve"); + // std::unique_ptr wv(cat.wipeVisitor(store, r, out, true, false, false)); + // cat.visitEntries(*wv, store, false); + // } + + // /// @todo: again, daos_fini happening before + + // } + + SECTION("Via FDB API with a Rados catalogue and store") { + + // FDB configuration + + std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + catalogue_tests_tmp_root().asString() + "\n" + "type: local\n" + "schema : " + schema_file().path() + "\n" + "engine: rados\n" + "store: rados\n" + "rados:\n" + }; + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; +#else + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; +#endif + + fdb5::Config config{YAMLConfiguration(config_str)}; + + // request + + fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"c", "3"}, {"d", "4"}}); + + fdb5::FDBToolRequest full_req{ + request_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest index_req{ + index_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest db_req{ + db_key.request("retrieve"), + false, + std::vector{"a", "b"} + }; + fdb5::FDBToolRequest all_req{ + metkit::mars::MarsRequest{}, + true, + std::vector{} + }; + + // initialise FDB + + fdb5::FDB fdb(config); + + // check FDB is empty + + size_t count; + fdb5::ListElement info; + + /// @todo: here, DaosManager is being configured with DAOS client config passed to FDB instance constructor. + // It happens in EntryVisitMechanism::visit when calling DB::open. Is this OK, or should this configuring + // rather happen as part of transforming a FieldLocation into a DataHandle? It is probably OK. One thing + // is to configure the DAOS client and the other thing is to initialise it. + auto listObject = fdb.list(db_req); + + count = 0; + while (listObject.next(info)) { + info.print(std::cout, true, true); + std::cout << std::endl; + ++count; + } + EXPECT(count == 0); + + // archive data + + char data[] = "test"; + + /// @todo: here, DaosManager is being reconfigured with identical config, and it happens again multiple times below. + // Should this be avoided? + fdb.archive(request_key, data, sizeof(data)); + fdb.flush(); + + // retrieve data + + metkit::mars::MarsRequest r = request_key.request("retrieve"); + std::unique_ptr dh(fdb.retrieve(r)); + + eckit::MemoryHandle mh; + dh->copyTo(mh); + EXPECT(mh.size() == eckit::Length(sizeof(data))); + EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + + // list all + + listObject = fdb.list(all_req); + count = 0; + while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + count++; + } + EXPECT(count == 1); + + // // wipe data + + // fdb5::WipeElement elem; + + // // dry run attempt to wipe with too specific request + + // auto wipeObject = fdb.wipe(full_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count == 0); + + // // dry run wipe index and store unit + // wipeObject = fdb.wipe(index_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + + // // dry run wipe database + // wipeObject = fdb.wipe(db_req); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + + // // ensure field still exists + // listObject = fdb.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 1); + + // // attempt to wipe with too specific request + // wipeObject = fdb.wipe(full_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count == 0); + // /// @todo: really needed? + // fdb.flush(); + + // // wipe index and store unit + // wipeObject = fdb.wipe(index_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + // /// @todo: really needed? + // fdb.flush(); + + // // ensure field does not exist + // listObject = fdb.list(full_req); + // count = 0; + // while (listObject.next(info)) count++; + // EXPECT(count == 0); + + // /// @todo: ensure index and corresponding container do not exist + // /// @todo: ensure DB still exists + // /// @todo: list db or index and expect count = 0? + + // // re-archive data + + // /// @note: FDB holds a LocalFDB which holds an Archiver which holds open DBs (DaosCatalogueWriters). + // /// If a whole DB is wiped, the top-level structures for that DB (main and catalogue KVs in this case) + // /// are deleted. If willing to archive again into that DB, the DB needs to be constructed again as the + // /// top-level structures are only generated as part of the DaosCatalogueWriter constructor. There is + // /// no way currently to destroy the open DBs held by FDB other than entirely destroying FDB. + // /// Alternatively, a separate FDB instance can be created. + // fdb5::FDB fdb2(config); + + // fdb2.archive(request_key, data, sizeof(data)); + + // fdb2.flush(); + + // listObject = fdb2.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 1); + + // // wipe full database + + // wipeObject = fdb2.wipe(db_req, true); + // count = 0; + // while (wipeObject.next(elem)) count++; + // EXPECT(count > 0); + // /// @todo: really needed? + // fdb2.flush(); + + // // ensure field does not exist + + // listObject = fdb2.list(full_req); + // count = 0; + // while (listObject.next(info)) { + // // info.print(std::cout, true, true); + // // std::cout << std::endl; + // count++; + // } + // EXPECT(count == 0); + + // /// @todo: ensure DB and corresponding pool do not exist + + // /// @todo: ensure new DaosSession has updated daos client config + + } + + // SECTION("OPTIONAL SCHEMA KEYS") { + + // // FDB configuration + + // ::setenv("FDB_SCHEMA_FILE", opt_schema_file().path().c_str(), 1); + + // std::string config_str{ + // "spaces:\n" + // "- roots:\n" + // " - path: " + catalogue_tests_tmp_root().asString() + "\n" + // "type: local\n" + // "schema : " + opt_schema_file().path() + "\n" + // "engine: daos\n" + // "store: daos\n" + // "daos:\n" + // " catalogue:\n" + // " pool: " + pool_name + "\n" + // " root_cont: " + root_cont_name + "\n" + // " store:\n" + // " pool: " + pool_name + "\n" + // " client:\n" + // " container_oids_per_alloc: " + std::to_string(container_oids_per_alloc) + // }; + + // fdb5::Config config{YAMLConfiguration(config_str)}; + + // // request + + // fdb5::Key request_key({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"f", "6"}}); + // fdb5::Key request_key2({{"a", "11"}, {"b", "22"}, {"d", "4"}, {"e", "5"}, {"f", "6"}}); + // fdb5::Key db_key({{"a", "11"}, {"b", "22"}}); + // fdb5::Key index_key({{"a", "11"}, {"b", "22"}, {"d", "4"}}); + + // fdb5::FDBToolRequest full_req{ + // request_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest full_req2{ + // request_key2.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest index_req{ + // index_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest db_req{ + // db_key.request("retrieve"), + // false, + // std::vector{"a", "b"} + // }; + // fdb5::FDBToolRequest all_req{ + // metkit::mars::MarsRequest{}, + // true, + // std::vector{} + // }; + + // // initialise FDB + + // fdb5::FDB fdb(config); + + // // check FDB is empty + + // size_t count; + // fdb5::ListElement info; + + // auto listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 0); + + // // archive data with incomplete key + + // char data[] = "test"; + + // fdb.archive(request_key, data, sizeof(data)); + + // fdb.flush(); + + // // list data + + // listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 1); + + // // retrieve data + + // { + // metkit::mars::MarsRequest r = request_key.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + // } + + // // archive data with complete key + + // char data2[] = "abcd"; + + // fdb.archive(request_key2, data2, sizeof(data)); + + // fdb.flush(); + + // // list data + + // listObject = fdb.list(db_req); + + // count = 0; + // while (listObject.next(info)) { + // info.print(std::cout, true, true); + // std::cout << std::endl; + // ++count; + // } + // EXPECT(count == 2); + + // // retrieve data + + // { + // metkit::mars::MarsRequest r = request_key.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data))); + // EXPECT(::memcmp(mh.data(), data, sizeof(data)) == 0); + // } + + // { + // metkit::mars::MarsRequest r = request_key2.request("retrieve"); + // std::unique_ptr dh(fdb.retrieve(r)); + + // eckit::MemoryHandle mh; + // dh->copyTo(mh); + // EXPECT(mh.size() == eckit::Length(sizeof(data2))); + // EXPECT(::memcmp(mh.data(), data2, sizeof(data2)) == 0); + // } + + // } + + // teardown rados + +#ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL + #ifdef eckit_HAVE_RADOS_ADMIN + eckit::RadosPool{pool}.ensureDestroyed(); + #else + ensureCleanNamespaces(pool, test_id); + #endif +#else + ensureClean(prefix); +#endif + +} + +} // namespace test +} // namespace fdb + +int main(int argc, char **argv) +{ + return run_tests ( argc, argv ); +} diff --git a/tests/fdb/rados/test_rados_store.cc b/tests/fdb/rados/test_rados_store.cc index 0b7f3eda1..020ea15f5 100644 --- a/tests/fdb/rados/test_rados_store.cc +++ b/tests/fdb/rados/test_rados_store.cc @@ -64,7 +64,14 @@ namespace { p.rmdir(); }; - // S3Config cfg("eu-central-1", "127.0.0.1", 8888); + void ensureCleanNamespaces(const std::string& pool, const std::string& prefix) { + ASSERT(prefix.length() > 3); + for (const std::string& name : eckit::RadosCluster::instance().listNamespaces(pool)) { + if (name.rfind(prefix, 0) == 0) { + eckit::RadosNamespace{pool, name}.destroy(); + } + } + } #ifdef fdb5_HAVE_RADOS_ADMIN void ensureClean(const std::string& prefix) { @@ -86,16 +93,6 @@ eckit::TmpFile& schema_file() { return f; } -eckit::TmpFile& spaces_file() { - static eckit::TmpFile f{}; - return f; -} - -eckit::TmpFile& roots_file() { - static eckit::TmpFile f{}; - return f; -} - eckit::PathName& store_tests_tmp_root() { static eckit::PathName sd("./rados_store_tests_fdb_root"); return sd; @@ -134,41 +131,16 @@ CASE( "Setup" ) { // due to no specified schema file (e.g. in Key::registry()) ::setenv("FDB_SCHEMA_FILE", schema_file().path().c_str(), 1); - // prepare scpaces - - std::string spaces_str{".* all Default"}; - - std::unique_ptr hsp(spaces_file().fileHandle()); - hsp->openForWrite(spaces_str.size()); - { - eckit::AutoClose closer(*hsp); - hsp->write(spaces_str.data(), spaces_str.size()); - } - - ::setenv("FDB_SPACES_FILE", spaces_file().path().c_str(), 1); - - // prepare roots - - std::string roots_str{store_tests_tmp_root().asString() + " all yes yes"}; - - std::unique_ptr hr(roots_file().fileHandle()); - hr->openForWrite(roots_str.size()); - { - eckit::AutoClose closer(*hr); - hr->write(roots_str.data(), roots_str.size()); - } - - ::setenv("FDB_ROOTS_FILE", roots_file().path().c_str(), 1); - } CASE("RadosStore tests") { SECTION("archive and retrieve") { + std::string test_id = "test-store1"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store1"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -177,19 +149,28 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "rados:\n" " store:\n" " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" }; #else - std::string prefix{"test-store1"}; - + std::string prefix = test_id; ensureClean(prefix); - std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" " pool_prefix: " + prefix + "\n" }; #endif @@ -251,9 +232,10 @@ CASE("RadosStore tests") { SECTION("with POSIX Catalogue") { + std::string test_id = "test-store2"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store2"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -262,21 +244,30 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "schema : " + schema_file().path() + "\n" "rados:\n" " store:\n" " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n" }; #else - std::string prefix{"test-store2"}; - + std::string prefix = test_id; ensureClean(prefix); - std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "schema : " + schema_file().path() + "\n" "rados:\n" + " namespace: default\n" + " root_pool: " + prefix + "_root\n" " pool_prefix: " + prefix + "\n" }; #endif @@ -376,9 +367,10 @@ CASE("RadosStore tests") { SECTION("VIA FDB API") { + std::string test_id = "test-store3"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store3"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -387,13 +379,17 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif #else - std::string prefix{"test-store3"}; + std::string prefix = test_id; ensureClean(prefix); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "type: local\n" "schema : " + schema_file().path() + "\n" "engine: toc\n" @@ -402,7 +398,9 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool_prefix: " + prefix + "\n"; + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -412,7 +410,9 @@ CASE("RadosStore tests") { config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n"; + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH) @@ -573,9 +573,10 @@ CASE("RadosStore tests") { // archive() fails as it expects a toc file to exist, but it has been removed by previous wipe SECTION("FDB API RE-STORE AND WIPE DB") { + std::string test_id = "test-store4"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL #ifdef eckit_HAVE_RADOS_ADMIN - std::string pool = "test-store4"; + std::string pool = test_id; eckit::RadosPool{pool}.ensureDestroyed(); eckit::RadosPool{pool}.ensureCreated(); /// @todo: auto pool destroyer #else @@ -584,13 +585,17 @@ CASE("RadosStore tests") { "fdbRadosTestPool;$FDB_RADOS_TEST_POOL", pool ); EXPECT(pool.length() > 0); + ensureCleanNamespaces(pool, test_id); #endif #else - std::string prefix{"test-store4"}; + std::string prefix = test_id; ensureClean(prefix); #endif std::string config_str{ + "spaces:\n" + "- roots:\n" + " - path: " + store_tests_tmp_root().asString() + "\n" "type: local\n" "schema : " + schema_file().path() + "\n" "engine: toc\n" @@ -599,7 +604,9 @@ CASE("RadosStore tests") { }; #ifndef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool_prefix: " + prefix + "\n"; + config_str += " namespace: default\n" + " root_pool: " + prefix + "_root\n" + " pool_prefix: " + prefix + "\n"; #endif #if defined(fdb5_HAVE_RADOS_STORE_MULTIPART) && ! defined(fdb5_HAVE_RADOS_STORE_OBJ_PER_FIELD) @@ -609,7 +616,9 @@ CASE("RadosStore tests") { config_str += " store:\n"; #ifdef fdb5_HAVE_RADOS_BACKENDS_SINGLE_POOL - config_str += " pool: " + pool + "\n"; + config_str += " pool: " + pool + "\n" + " root_namespace: " + test_id + "_root\n" + " namespace_prefix: " + test_id + "\n"; #endif #if defined(fdb5_HAVE_RADOS_BACKENDS_PERSIST_ON_FLUSH)