refactored to address PR comments

src/fdb5/api/FDB.cc

@@ -19,6 +19,7 @@
 #include "eckit/log/Log.h"
 #include "eckit/message/Message.h"
 #include "eckit/message/Reader.h"
+#include "eckit/utils/StringTools.h"
 
 #include "metkit/hypercube/HyperCubePayloaded.h"
 
@@ -104,15 +105,15 @@ void FDB::archive(const CanonicalKey& key, const void* data, size_t length) {
     // This is the API entrypoint. Keys supplied by the user may not have type registry info attached (so
     // serialisation won't work properly...)
     CanonicalKey keyInternal(key);
-    keyInternal.registry(config().schema().registry());
 
     // step in archival requests from the model is just an integer. We need to include the stepunit
     auto stepunit = keyInternal.find("stepunits");
     if (stepunit != keyInternal.end()) {
-        if (stepunit->second.size()>0 && stepunit->second[0]!='h') {
+        if (stepunit->second.size()>0 && static_cast<char>(tolower(stepunit->second[0])) != 'h') {
             auto step = keyInternal.find("step");
             if (step != keyInternal.end()) {
-                std::string canonicalStep = keyInternal.registry().lookupType("step").toKey("step", step->second+stepunit->second);
+                std::string canonicalStep = config().schema().registry()->lookupType("step").toKey("step", step->second + static_cast<char>(tolower(stepunit->second[0])));
+                keyInternal.set("step", canonicalStep);
             }
         }
         keyInternal.unset("stepunits");

src/fdb5/api/local/AxesVisitor.h

@@ -40,7 +40,7 @@ class AxesVisitor : public QueryVisitor<AxesElement> {
 //    bool preVisitDatabase(const eckit::URI& uri) override;
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override;
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/ControlVisitor.h

@@ -39,7 +39,7 @@ class ControlVisitor : public QueryVisitor<ControlElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/DumpVisitor.h

@@ -49,7 +49,7 @@ class DumpVisitor : public QueryVisitor<DumpElement> {
         return true;
     }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

src/fdb5/api/local/ListVisitor.h

@@ -80,7 +80,7 @@ struct ListVisitor : public QueryVisitor<ListElement> {
     }
 
     /// Test if entry matches the current request. If so, add to the output queue.
-    void visitDatum(const Field& field, const ApiKey& datumKey) override {
+    void visitDatum(const Field& field, const TypedKey& datumKey) override {
         ASSERT(currentCatalogue_);
         ASSERT(currentIndex_);
 

src/fdb5/api/local/MoveVisitor.h

@@ -44,7 +44,7 @@ class MoveVisitor : public QueryVisitor<MoveElement> {
 
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
 private: // members

src/fdb5/api/local/PurgeVisitor.cc

@@ -82,8 +82,6 @@ void PurgeVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void PurgeVisitor::visitDatum(const Field&, const ApiKey&) { NOTIMP; }
-
 void PurgeVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

src/fdb5/api/local/PurgeVisitor.h

@@ -48,7 +48,7 @@ class PurgeVisitor : public QueryVisitor<PurgeElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const ApiKey&) override;
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/StatsVisitor.cc

@@ -47,8 +47,6 @@ void StatsVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     internalVisitor_->visitDatum(field, keyFingerprint);
 }
 
-void StatsVisitor::visitDatum(const Field&, const ApiKey&) { NOTIMP; }
-
 void StatsVisitor::catalogueComplete(const Catalogue& catalogue) {
     internalVisitor_->catalogueComplete(catalogue);
 

src/fdb5/api/local/StatsVisitor.h

@@ -42,7 +42,7 @@ class StatsVisitor : public QueryVisitor<StatsElement> {
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
     void visitDatum(const Field& field, const std::string& keyFingerprint) override;
-    void visitDatum(const Field&, const ApiKey&) override;
+    void visitDatum(const Field&, const TypedKey&) override  { NOTIMP; }
 
 private: // members
 

src/fdb5/api/local/StatusVisitor.h

@@ -38,7 +38,7 @@ class StatusVisitor : public QueryVisitor<StatusElement> {
     bool visitEntries() override { return false; }
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override { queue_.emplace(catalogue); return true; }
     bool visitIndex(const Index&) override { NOTIMP; }
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
 
     void visitDatum(const Field& field, const std::string& keyFingerprint) override {
         EntryVisitor::visitDatum(field, keyFingerprint);

src/fdb5/api/local/WipeVisitor.h

@@ -50,7 +50,7 @@ class WipeVisitor : public QueryVisitor<WipeElement> {
     bool visitDatabase(const Catalogue& catalogue, const Store& store) override;
     bool visitIndex(const Index& index) override;
     void catalogueComplete(const Catalogue& catalogue) override;
-    void visitDatum(const Field&, const ApiKey&) override { NOTIMP; }
+    void visitDatum(const Field&, const TypedKey&) override { NOTIMP; }
     void visitDatum(const Field& field, const std::string& keyFingerprint) override { NOTIMP; }
 
     virtual void onDatabaseNotFound(const fdb5::DatabaseNotFoundException& e) override { throw e; }

src/fdb5/daos/DaosCatalogueReader.cc

@@ -122,7 +122,7 @@ bool DaosCatalogueReader::axis(const std::string &keyword, eckit::StringSet &s)
 
 }
 
-bool DaosCatalogueReader::retrieve(const ApiKey& key, Field& field) const {
+bool DaosCatalogueReader::retrieve(const TypedKey& key, Field& field) const {
 
     eckit::Log::debug<LibFdb5>() << "Trying to retrieve key " << key << std::endl;
     eckit::Log::debug<LibFdb5>() << "Scanning index " << current_.location() << std::endl;

src/fdb5/daos/DaosCatalogueReader.h

@@ -40,7 +40,7 @@ class DaosCatalogueReader : public DaosCatalogue, public CatalogueReader {
 
     bool axis(const std::string &keyword, eckit::StringSet &s) const override;
 
-    bool retrieve(const ApiKey& key, Field& field) const override;
+    bool retrieve(const TypedKey& key, Field& field) const override;
 
     void print( std::ostream &out ) const override { NOTIMP; }
 

src/fdb5/daos/DaosEngine.h

@@ -41,7 +41,7 @@ class DaosEngine : public fdb5::Engine {
 
     virtual bool canHandle(const eckit::URI&, const Config&) const override;
 
-    virtual std::vector<eckit::URI> allLocations(const CanonicalKey& key, const Config& config) const override { NOTIMP; };
+    // virtual std::vector<eckit::URI> allLocations(const CanonicalKey& key, const Config& config) const override { NOTIMP; };
 
     virtual std::vector<eckit::URI> visitableLocations(const CanonicalKey& key, const Config& config) const override;
     virtual std::vector<eckit::URI> visitableLocations(const metkit::mars::MarsRequest& rq, const Config& config) const override;

src/fdb5/daos/DaosIndex.cc

@@ -159,7 +159,7 @@ bool DaosIndex::get(const CanonicalKey& key, const CanonicalKey& remapKey, Field
 
 }
 
-void DaosIndex::add(const ApiKey& key, const Field &field) {
+void DaosIndex::add(const TypedKey& key, const Field &field) {
 
     eckit::MemoryHandle h{(size_t) PATH_MAX};
     eckit::HandleStream hs{h};

src/fdb5/daos/DaosIndex.h

@@ -48,7 +48,7 @@ class DaosIndex : public IndexBase {
     void visit(IndexLocationVisitor& visitor) const override { NOTIMP; }
 
     bool get( const CanonicalKey& key, const CanonicalKey& remapKey, Field &field ) const override;
-    void add( const ApiKey& key, const Field &field ) override;
+    void add( const TypedKey& key, const Field &field ) override;
     void flush() override { NOTIMP; }
     void encode(eckit::Stream& s, const int version) const override { NOTIMP; }
     void entries(EntryVisitor& visitor) const override;

src/fdb5/daos/DaosStore.cc

@@ -105,7 +105,7 @@ eckit::DataHandle* DaosStore::retrieve(Field& field) const {
 
 }
 
-std::unique_ptr<FieldLocation> DaosStore::archive(const CanonicalKey& key, const void *data, eckit::Length length) {
+std::unique_ptr<FieldLocation> DaosStore::archive(const CanonicalKey&, const void *data, eckit::Length length) {
 
     /// @note: performed RPCs:
     /// - open pool if not cached (daos_pool_connect) -- always skipped as it is cached after selectDatabase.

src/fdb5/daos/DaosStore.h

@@ -49,7 +49,7 @@ class DaosStore : public Store, public DaosCommon {
     bool exists() const override;
 
     eckit::DataHandle* retrieve(Field& field) const override;
-    std::unique_ptr<FieldLocation> archive(const CanonicalKey& key, const void *data, eckit::Length length) override;
+    std::unique_ptr<FieldLocation> archive(const CanonicalKey&, const void *data, eckit::Length length) override;
 
     void remove(const eckit::URI& uri, std::ostream& logAlways, std::ostream& logVerbose, bool doit) const override;
 

src/fdb5/database/ArchiveVisitor.cc

@@ -16,21 +16,19 @@
 
 namespace fdb5 {
 
-ArchiveVisitor::ArchiveVisitor(Archiver &owner, const CanonicalKey& field, const void *data, size_t size) :
-    BaseArchiveVisitor(owner, field),
+ArchiveVisitor::ArchiveVisitor(Archiver &owner, const CanonicalKey& initialFieldKey, const void *data, size_t size) :
+    BaseArchiveVisitor(owner, initialFieldKey),
     data_(data),
     size_(size) {
 }
 
-bool ArchiveVisitor::selectDatum(const ApiKey& key, const CanonicalKey& full) {
+bool ArchiveVisitor::selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) {
 
-    // eckit::Log::info() << "selectDatum " << key << ", " << full << " " << size_ << std::endl;
-    checkMissingKeys(full);
+    checkMissingKeys(fullComputedKey);
 
     ASSERT(current());
 
-    current()->archive(key, data_, size_);
-
+    current()->archive(datumKey, data_, size_);
 
     return true;
 }

src/fdb5/database/ArchiveVisitor.h

@@ -34,7 +34,7 @@ class ArchiveVisitor : public BaseArchiveVisitor {
 
 protected: // methods
 
-    virtual bool selectDatum(const ApiKey& key, const CanonicalKey& full) override;
+    virtual bool selectDatum(const TypedKey& datumKey, const TypedKey& fullComputedKey) override;
 
     virtual void print( std::ostream &out ) const override;
 

src/fdb5/database/Archiver.cc

@@ -45,7 +45,6 @@ void Archiver::archive(const CanonicalKey& key, BaseArchiveVisitor& visitor) {
 
     visitor.rule(nullptr);
 
-
     dbConfig_.schema().expand(key, visitor);
 
     const Rule* rule = visitor.rule();

src/fdb5/database/BaseArchiveVisitor.cc

@@ -17,30 +17,30 @@
 
 namespace fdb5 {
 
-BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const CanonicalKey& field) :
+BaseArchiveVisitor::BaseArchiveVisitor(Archiver &owner, const CanonicalKey& initialFieldKey) :
     WriteVisitor(owner.prev_),
     owner_(owner),
-    field_(field) {
+    initialFieldKey_(initialFieldKey) {
     checkMissingKeysOnWrite_ = eckit::Resource<bool>("checkMissingKeysOnWrite", true);
 }
 
-bool BaseArchiveVisitor::selectDatabase(const CanonicalKey& dbKey, const CanonicalKey&) {
-    LOG_DEBUG_LIB(LibFdb5) << "selectDatabase " << dbKey << std::endl;
+bool BaseArchiveVisitor::selectDatabase(const CanonicalKey& dbKey, const TypedKey& fullComputedKey) {
+    LOG_DEBUG_LIB(LibFdb5) << "BaseArchiveVisitor::selectDatabase " << dbKey << std::endl;
     owner_.current_ = &owner_.database(dbKey);
     owner_.current_->deselectIndex();
 
     return true;
 }
 
-bool BaseArchiveVisitor::selectIndex(const CanonicalKey& idxKey, const CanonicalKey&) {
+bool BaseArchiveVisitor::selectIndex(const CanonicalKey& idxKey, const TypedKey& fullComputedKey) {
     // eckit::Log::info() << "selectIndex " << idxKey << std::endl;
     ASSERT(owner_.current_);
     return owner_.current_->selectIndex(idxKey);
 }
 
-void BaseArchiveVisitor::checkMissingKeys(const CanonicalKey& full) {
+void BaseArchiveVisitor::checkMissingKeys(const TypedKey& fullComputedKey) {
     if (checkMissingKeysOnWrite_) {
-        field_.validateKeysOf(full);
+        fullComputedKey.validateKeys(initialFieldKey_);
     }
 }
 

src/fdb5/database/BaseArchiveVisitor.h

@@ -32,15 +32,15 @@ class BaseArchiveVisitor : public WriteVisitor {
 
 public: // methods
 
-    BaseArchiveVisitor(Archiver &owner, const CanonicalKey& field);
+    BaseArchiveVisitor(Archiver &owner, const CanonicalKey& initialFieldKey);
 
 protected: // methods
 
-    virtual bool selectDatabase(const CanonicalKey& dbKey, const CanonicalKey& full);
+    virtual bool selectDatabase(const CanonicalKey& dbKey, const TypedKey& fullComputedKey);
 
-    virtual bool selectIndex(const CanonicalKey& idxKey, const CanonicalKey& full);
+    virtual bool selectIndex(const CanonicalKey& idxKey, const TypedKey& fullComputedKey);
 
-    virtual void checkMissingKeys(const CanonicalKey& full);
+    virtual void checkMissingKeys(const TypedKey& fullComputedKey);
 
     virtual const Schema& databaseSchema() const;
 
@@ -50,7 +50,7 @@ class BaseArchiveVisitor : public WriteVisitor {
 
     Archiver &owner_;
 
-    const CanonicalKey& field_;
+    const CanonicalKey& initialFieldKey_;
 
     bool checkMissingKeysOnWrite_;
 };

src/fdb5/database/Catalogue.h

@@ -115,16 +115,16 @@ class CatalogueReader {
 public:
     virtual DbStats stats() const = 0;
     virtual bool axis(const std::string& keyword, eckit::StringSet& s) const = 0;
-    virtual bool retrieve(const ApiKey& key, Field& field) const = 0;
+    virtual bool retrieve(const TypedKey& key, Field& field) const = 0;
 };
 
 
 class CatalogueWriter {
 public:
     virtual const Index& currentIndex() = 0;
-    virtual void archive(const ApiKey& key, std::unique_ptr<FieldLocation> fieldLocation) = 0;
+    virtual void archive(const TypedKey& key, std::unique_ptr<FieldLocation> fieldLocation) = 0;
     virtual void overlayDB(const Catalogue& otherCatalogue, const std::set<std::string>& variableKeys, bool unmount) = 0;
-    virtual void index(const ApiKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
+    virtual void index(const TypedKey& key, const eckit::URI& uri, eckit::Offset offset, eckit::Length length) = 0;
     virtual void reconsolidate() = 0;
 };
 

src/fdb5/database/DB.cc

@@ -83,7 +83,7 @@ bool DB::axis(const std::string &keyword, eckit::StringSet &s) const {
     return cat->axis(keyword, s);
 }
 
-bool DB::inspect(const ApiKey& key, Field& field) {
+bool DB::inspect(const TypedKey& key, Field& field) {
 
     LOG_DEBUG_LIB(LibFdb5) << "Trying to retrieve key " << key << std::endl;
 
@@ -93,7 +93,7 @@ bool DB::inspect(const ApiKey& key, Field& field) {
     return cat->retrieve(key, field);
 }
 
-eckit::DataHandle *DB::retrieve(const ApiKey& key) {
+eckit::DataHandle *DB::retrieve(const TypedKey& key) {
 
     Field field;
     if (inspect(key, field)) {
@@ -103,7 +103,7 @@ eckit::DataHandle *DB::retrieve(const ApiKey& key) {
     return nullptr;
 }
 
-void DB::archive(const ApiKey& key, const void* data, eckit::Length length) {
+void DB::archive(const TypedKey& key, const void* data, eckit::Length length) {
 
     CatalogueWriter* cat = dynamic_cast<CatalogueWriter*>(catalogue_.get());
     ASSERT(cat);
@@ -164,7 +164,7 @@ void DB::reconsolidate() {
     cat->reconsolidate();
 }
 
-void DB::index(const ApiKey& key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length) {
+void DB::index(const TypedKey& key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length) {
     if (catalogue_->type() == TocEngine::typeName()) {
         CatalogueWriter* cat = dynamic_cast<CatalogueWriter*>(catalogue_.get());
         ASSERT(cat);

src/fdb5/database/DB.h

@@ -54,9 +54,9 @@ class DB final : public eckit::OwnedLock {
     const Schema& schema() const;
 
     bool axis(const std::string &keyword, eckit::StringSet &s) const;
-    bool inspect(const ApiKey& key, Field& field);
-    eckit::DataHandle *retrieve(const ApiKey& key);
-    void archive(const ApiKey& key, const void *data, eckit::Length length);
+    bool inspect(const TypedKey& key, Field& field);
+    eckit::DataHandle *retrieve(const TypedKey& key);
+    void archive(const TypedKey& key, const void *data, eckit::Length length);
 
     bool open();
     void flush();
@@ -79,7 +79,7 @@ class DB final : public eckit::OwnedLock {
 
     void visitEntries(EntryVisitor& visitor, bool sorted = false);
     /// Used for adopting & indexing external data to the TOC dir
-    void index(const ApiKey& key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length);
+    void index(const TypedKey& key, const eckit::PathName &path, eckit::Offset offset, eckit::Length length);
 
     // Control access properties of the DB
     void control(const ControlAction& action, const ControlIdentifiers& identifiers) const;

src/fdb5/database/Engine.h

@@ -56,7 +56,7 @@ class Engine : private eckit::NonCopyable {
     virtual eckit::URI location(const CanonicalKey& key, const Config& config) const = 0;
 
     /// Lists the roots that can be visited given a DB key
-    virtual std::vector<eckit::URI> allLocations(const CanonicalKey& key, const Config& config) const = 0;
+    // virtual std::vector<eckit::URI> allLocations(const CanonicalKey& key, const Config& config) const = 0;
 
     /// Lists the roots that can be visited given a DB key
     virtual std::vector<eckit::URI> visitableLocations(const CanonicalKey& key, const Config& config) const = 0;

src/fdb5/database/EntryVisitMechanism.cc

@@ -61,7 +61,7 @@ void EntryVisitor::visitDatum(const Field& field, const std::string& keyFingerpr
     ASSERT(currentCatalogue_);
     ASSERT(currentIndex_);
 
-    ApiKey key(keyFingerprint, currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key()));
+    TypedKey key(keyFingerprint, currentCatalogue_->schema().ruleFor(currentCatalogue_->key(), currentIndex_->key()));
     visitDatum(field, key);
 }