From a6b564d03c0069b64fd31b1449813e9bc6fafd54 Mon Sep 17 00:00:00 2001 From: Metin Cakircali Date: Wed, 30 Oct 2024 15:17:38 +0100 Subject: [PATCH] feat(FAM): added FamMap (WIP) --- src/eckit/CMakeLists.txt | 5 +- src/eckit/io/fam/FamHandle.cc | 1 + src/eckit/io/fam/FamHandle.h | 2 - src/eckit/io/fam/FamHashTable.cc | 47 ++++++++ src/eckit/io/fam/FamHashTable.h | 108 ++++++++++++++++++ src/eckit/io/fam/FamList.cc | 99 +++++++++++----- src/eckit/io/fam/FamList.h | 33 ++++-- src/eckit/io/fam/FamListIterator.cc | 8 +- src/eckit/io/fam/FamListIterator.h | 12 +- src/eckit/io/fam/FamMap.cc | 158 ++++++++++++++++++++++++++ src/eckit/io/fam/FamMap.h | 136 ++++++++++++++++++++++ src/eckit/io/fam/FamMapIterator.cc | 50 ++++++++ src/eckit/io/fam/FamMapIterator.h | 78 +++++++++++++ src/eckit/io/fam/FamObject.cc | 4 +- src/eckit/io/fam/FamObject.h | 6 +- src/eckit/io/fam/FamObjectName.h | 3 +- src/eckit/io/fam/FamProperty.h | 17 +-- src/eckit/io/fam/FamRegion.cc | 4 +- src/eckit/io/fam/FamRegion.h | 4 +- src/eckit/io/fam/FamRegionName.cc | 1 + src/eckit/io/fam/FamRegionName.h | 4 +- src/eckit/io/fam/FamURIManager.cc | 4 +- src/eckit/io/fam/detail/FamListNode.h | 58 ++++++++++ src/eckit/io/fam/detail/FamMapNode.h | 48 ++++++++ src/eckit/io/fam/detail/FamNode.h | 24 +--- tests/io/CMakeLists.txt | 2 +- 26 files changed, 820 insertions(+), 96 deletions(-) create mode 100644 src/eckit/io/fam/FamHashTable.cc create mode 100644 src/eckit/io/fam/FamHashTable.h create mode 100644 src/eckit/io/fam/FamMap.cc create mode 100644 src/eckit/io/fam/FamMap.h create mode 100644 src/eckit/io/fam/FamMapIterator.cc create mode 100644 src/eckit/io/fam/FamMapIterator.h create mode 100644 src/eckit/io/fam/detail/FamListNode.h create mode 100644 src/eckit/io/fam/detail/FamMapNode.h diff --git a/src/eckit/CMakeLists.txt b/src/eckit/CMakeLists.txt index 68a36abd9..d8e87d414 100644 --- a/src/eckit/CMakeLists.txt +++ b/src/eckit/CMakeLists.txt @@ -276,6 +276,10 @@ io/fam/FamList.cc io/fam/FamList.h io/fam/FamListIterator.cc io/fam/FamListIterator.h +io/fam/FamMap.cc +io/fam/FamMap.h +io/fam/FamMapIterator.cc +io/fam/FamMapIterator.h io/fam/FamName.cc io/fam/FamName.h io/fam/FamObject.cc @@ -1009,4 +1013,3 @@ endif() if( eckit_HAVE_ECKIT_GEO ) add_subdirectory( geo ) endif() - diff --git a/src/eckit/io/fam/FamHandle.cc b/src/eckit/io/fam/FamHandle.cc index 99e9cfc89..79e93385f 100644 --- a/src/eckit/io/fam/FamHandle.cc +++ b/src/eckit/io/fam/FamHandle.cc @@ -17,6 +17,7 @@ #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" +#include "eckit/io/fam/FamObject.h" #include "eckit/log/Log.h" namespace eckit { diff --git a/src/eckit/io/fam/FamHandle.h b/src/eckit/io/fam/FamHandle.h index b0b5a78cc..1f3fea44d 100644 --- a/src/eckit/io/fam/FamHandle.h +++ b/src/eckit/io/fam/FamHandle.h @@ -26,8 +26,6 @@ namespace eckit { -class FamObject; - //---------------------------------------------------------------------------------------------------------------------- class FamHandle: public DataHandle { diff --git a/src/eckit/io/fam/FamHashTable.cc b/src/eckit/io/fam/FamHashTable.cc new file mode 100644 index 000000000..f1873caa5 --- /dev/null +++ b/src/eckit/io/fam/FamHashTable.cc @@ -0,0 +1,47 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +#include "eckit/io/fam/FamHashTable.h" + +#include "detail/FamHashNode.h" +#include "eckit/io/fam/FamObject.h" +#include "eckit/io/fam/FamObjectName.h" +#include "eckit/io/fam/FamRegionName.h" + +// #include "detail/FamSessionDetail.h" +// #include "eckit/exception/Exceptions.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +FamHashTable::FamHashTable(const FamRegionName& regionName, const std::string& tableName): + region_ {regionName.lookup()}, + begin_ {initSentinel(tableName + "-hash-begin", sizeof(FamDescriptor))}, + count_ {initSentinel(tableName + "-hash-count", sizeof(size_type))} { } + +auto FamHashTable::initSentinel(const std::string& name, const fam::size_t size) const -> FamObject { + try { + return region_.allocateObject(size, name); + } catch (const AlreadyExists&) { + auto object = region_.lookupObject(name); + ASSERT(object.size() == size); + return object; + } +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamHashTable.h b/src/eckit/io/fam/FamHashTable.h new file mode 100644 index 000000000..9d3c8b47c --- /dev/null +++ b/src/eckit/io/fam/FamHashTable.h @@ -0,0 +1,108 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +/// @file FamHashTable.h +/// @author Metin Cakircali +/// @date Jul 2024 + +#pragma once + +#include "eckit/io/fam/FamObject.h" +#include "eckit/io/fam/FamRegion.h" +// #include "eckit/io/fam/FamRegionName.h" +// #include "eckit/io/fam/FamVector.h" +#include "eckit/io/fam/FamMapIterator.h" +#include "eckit/types/FixedString.h" + +#include +#include + +namespace eckit { + +class FamList; +// class FamRegion; +class FamRegionName; + +//---------------------------------------------------------------------------------------------------------------------- +// FAM HASHER + +/// @brief Hash functor. Override this to make a specialized hasher +template +struct FamHash { + auto operator()(const key_type& key) const noexcept -> std::size_t { + return std::hash {}(key.asString()); + /// @note example for a 3-level key + // const auto l1 = std::hash {}(key.firstLevel); + // const auto l2 = std::hash {}(key.secondLevel); + // const auto l3 = std::hash {}(key.thirdLevel); + // return l1 ^ (l2 ^ (l3 << 1)); + } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +/// @todo template: initial table size, key size, (also equal and/or hasher ?) + +/// @brief data structure is array of lists: FamVector< FamList > +// unsigned int index = key % table->size; + +class FamHashTable { + static constexpr auto keySize = 32; // template? + + static constexpr auto capacity = 1024; + +public: // types + using key_type = FixedString; + using hash_type = FamHash; + /// @todo char array ? + using value_type = char; + // using key_equal = key_equal; + using size_type = fam::size_t; + using difference_type = size_type; + + // using mapped_type = mapped_type; + // using allocator_type = allocator_type; + // using pointer = pointer; + // using const_pointer = const_pointer; + + using reference = value_type&; + using const_reference = const value_type&; + + using iterator = FamMapIterator; + using const_iterator = const FamMapIterator; + + // using local_iterator = local_iterator; + // using const_local_iterator = const_local_iterator; + + using node_type = FamList; + +public: // methods + FamHashTable(const FamRegionName& regionName, const std::string& tableName); + +private: // methods + auto initSentinel(const std::string& name, fam::size_t size) const -> FamObject; + +private: // members + FamRegion region_; + + FamObject begin_; + FamObject count_; + + std::array table_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamList.cc b/src/eckit/io/fam/FamList.cc index b9b3f25ef..9e758ec6a 100644 --- a/src/eckit/io/fam/FamList.cc +++ b/src/eckit/io/fam/FamList.cc @@ -15,7 +15,7 @@ #include "eckit/io/fam/FamList.h" -#include "detail/FamNode.h" +#include "detail/FamListNode.h" #include "eckit/exception/Exceptions.h" #include "eckit/io/fam/FamObject.h" #include "eckit/io/fam/FamRegion.h" @@ -28,35 +28,58 @@ namespace eckit { //---------------------------------------------------------------------------------------------------------------------- +namespace { + +auto initSentinel(const FamRegion& region, const std::string& objectName, const fam::size_t objectSize) -> FamObject { + try { + return region.allocateObject(objectSize, objectName); + } catch (const AlreadyExists&) { + auto object = region.lookupObject(objectName); + ASSERT(object.size() == objectSize); + return object; + } +} + +} // namespace + +//---------------------------------------------------------------------------------------------------------------------- + +FamList::FamList(const FamRegion& region, const Descriptor& desc): + region_ {region}, + head_ {region_.proxyObject(desc.head)}, + tail_ {region_.proxyObject(desc.tail)}, + size_ {region_.proxyObject(desc.size)} { + ASSERT(region.index() == desc.region); +} + FamList::FamList(const FamRegion& region, const std::string& listName): - region_ {region}, head_ {initSentinel(listName + "-head", sizeof(FamNode))}, - tail_ {initSentinel(listName + "-tail", sizeof(FamNode))}, - size_ {initSentinel(listName + "-size", sizeof(fam::size_t))} { + region_ {region}, + head_ {initSentinel(region_, listName + "-list-head", sizeof(FamListNode))}, + tail_ {initSentinel(region_, listName + "-list-tail", sizeof(FamListNode))}, + size_ {initSentinel(region_, listName + "-list-size", sizeof(size_type))} { // set head's next to tail's prev - if (FamNode::getNextOffset(head_) == 0) { head_.put(tail_.descriptor(), offsetof(FamNode, next)); } + if (FamListNode::getNextOffset(head_) == 0) { head_.put(tail_.descriptor(), offsetof(FamListNode, next)); } // set tail's prev to head's next - if (FamNode::getPrevOffset(tail_) == 0) { tail_.put(head_.descriptor(), offsetof(FamNode, prev)); } + if (FamListNode::getPrevOffset(tail_) == 0) { tail_.put(head_.descriptor(), offsetof(FamListNode, prev)); } } FamList::FamList(const FamRegionName& name): FamList(name.lookup(), name.path().objectName) { } FamList::~FamList() = default; -auto FamList::initSentinel(const std::string& name, const fam::size_t size) const -> FamObject { - try { - return region_.allocateObject(size, name); - } catch (const AlreadyExists&) { return region_.lookupObject(name); } +auto FamList::descriptor() const -> Descriptor { + return {region_.index(), head_.offset(), tail_.offset(), size_.offset()}; } //---------------------------------------------------------------------------------------------------------------------- // iterators auto FamList::begin() const -> iterator { - return {region_.proxyObject(FamNode::getNextOffset(head_))}; + return {region_.proxyObject(FamListNode::getNextOffset(head_))}; } auto FamList::cbegin() const -> const_iterator { - return {region_.proxyObject(FamNode::getNextOffset(head_))}; + return {region_.proxyObject(FamListNode::getNextOffset(head_))}; } auto FamList::end() const -> iterator { @@ -81,31 +104,49 @@ auto FamList::back() const -> Buffer { //---------------------------------------------------------------------------------------------------------------------- // modifiers -void FamList::push_front(const void* /* data */, const fam::size_t /* length */) { - NOTIMP; +void FamList::push_front(const void* data, const size_type length) { + // allocate an object + auto newObject = region_.allocateObject(sizeof(FamListNode) + length); + + // set new object's previous to head + newObject.put(head_.descriptor(), offsetof(FamListNode, prev)); + + // set head's next to new object + const auto prevOffset = head_.swap(offsetof(FamListNode, next.offset), newObject.offset()); + const auto oldObject = region_.proxyObject(prevOffset); + + // set old object's prev to new object + oldObject.put(newObject.descriptor(), offsetof(FamListNode, prev)); + // set new object's next to old object + newObject.put(oldObject.descriptor(), offsetof(FamListNode, next)); + + // finally put the data + newObject.put(length, offsetof(FamListNode, length)); + newObject.put(data, sizeof(FamListNode), length); + + // increment size + size_.add(0, 1UL); } -void FamList::push_back(const void* data, const fam::size_t length) { +void FamList::push_back(const void* data, const size_type length) { // allocate an object - auto newObject = region_.allocateObject(sizeof(FamNode) + length); + auto newObject = region_.allocateObject(sizeof(FamListNode) + length); // set new object's next to tail - newObject.put(tail_.descriptor(), offsetof(FamNode, next)); + newObject.put(tail_.descriptor(), offsetof(FamListNode, next)); // set tail's prev to new object - const auto prevOffset = tail_.swap(offsetof(FamNode, prev.offset), newObject.offset()); - - const auto oldObject = region_.proxyObject(prevOffset); + const auto prevOffset = tail_.swap(offsetof(FamListNode, prev.offset), newObject.offset()); + const auto oldObject = region_.proxyObject(prevOffset); // set old object's next to new object - oldObject.put(newObject.descriptor(), offsetof(FamNode, next)); - + oldObject.put(newObject.descriptor(), offsetof(FamListNode, next)); // set new object's prev to old object - newObject.put(oldObject.descriptor(), offsetof(FamNode, prev)); + newObject.put(oldObject.descriptor(), offsetof(FamListNode, prev)); - // finally the data - newObject.put(length, offsetof(FamNode, length)); - newObject.put(data, sizeof(FamNode), length); + // finally put the data + newObject.put(length, offsetof(FamListNode, length)); + newObject.put(data, sizeof(FamListNode), length); // increment size size_.add(0, 1UL); @@ -122,12 +163,12 @@ void FamList::pop_back() { //---------------------------------------------------------------------------------------------------------------------- // capacity -auto FamList::size() const -> fam::size_t { - return size_.get(0); +auto FamList::size() const -> size_type { + return size_.get(); } auto FamList::empty() const -> bool { - return (FamNode::getNextOffset(head_) == tail_.offset()); + return (FamListNode::getNextOffset(head_) == tail_.offset()); } //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/fam/FamList.h b/src/eckit/io/fam/FamList.h index 68b56d9bd..8fed0a4a4 100644 --- a/src/eckit/io/fam/FamList.h +++ b/src/eckit/io/fam/FamList.h @@ -20,31 +20,44 @@ #pragma once #include "eckit/io/fam/FamListIterator.h" -#include "eckit/io/fam/FamRegionName.h" +#include "eckit/io/fam/FamRegion.h" -#include -#include +#include #include namespace eckit { +class FamRegionName; + //---------------------------------------------------------------------------------------------------------------------- class FamList { public: // types + using size_type = fam::size_t; using iterator = FamListIterator; using const_iterator = FamListConstIterator; + struct Descriptor { + fam::index_t region; // region ID + fam::index_t head; // offset of head sentinel + fam::index_t tail; // offset of tail sentinel + fam::index_t size; // offset of size sentinel + }; + public: // methods - FamList(const FamRegion& region, const std::string& name); + FamList(const FamRegion& region, const Descriptor& desc); + + FamList(const FamRegion& region, const std::string& listName); FamList(const FamRegionName& name); ~FamList(); + auto descriptor() const -> Descriptor; + // capacity - auto size() const -> fam::size_t; + auto size() const -> size_type; auto empty() const -> bool; @@ -66,19 +79,17 @@ class FamList { // modifiers - void push_back(const void* data, fam::size_t length); + // void clear() noexcept; - void push_front(const void* data, fam::size_t length); + void push_back(const void* data, size_type length); + + void push_front(const void* data, size_type length); void pop_front(); void pop_back(); private: // methods - auto initSentinel(const std::string& name, fam::size_t size) const -> FamObject; - - // auto region() const -> FamRegion& { return region_; } - void print(std::ostream& out) const; friend std::ostream& operator<<(std::ostream& out, const FamList& list); diff --git a/src/eckit/io/fam/FamListIterator.cc b/src/eckit/io/fam/FamListIterator.cc index 80e885ddd..4e8b64289 100644 --- a/src/eckit/io/fam/FamListIterator.cc +++ b/src/eckit/io/fam/FamListIterator.cc @@ -15,7 +15,7 @@ #include "eckit/io/fam/FamListIterator.h" -#include "detail/FamNode.h" +#include "detail/FamListNode.h" #include "detail/FamSessionDetail.h" #include "eckit/exception/Exceptions.h" @@ -27,7 +27,7 @@ namespace eckit { FamListIterator::FamListIterator(const value_type& object): object_ {object} { } auto FamListIterator::operator++() -> FamListIterator& { - if (const auto next = FamNode::getNext(object_); next.offset > 0) { + if (const auto next = FamListNode::getNext(object_); next.offset > 0) { invalid_ = true; object_.replaceWith(next); } @@ -35,7 +35,7 @@ auto FamListIterator::operator++() -> FamListIterator& { } auto FamListIterator::operator--() -> FamListIterator& { - if (const auto prev = FamNode::getPrev(object_); prev.offset > 0) { + if (const auto prev = FamListNode::getPrev(object_); prev.offset > 0) { invalid_ = true; object_.replaceWith(prev); } @@ -51,7 +51,7 @@ auto FamListIterator::operator->() -> pointer { } auto FamListIterator::operator*() -> reference { - if (invalid_) { FamNode::getData(object_, data_); } + if (invalid_) { FamListNode::getData(object_, data_); } return data_; } diff --git a/src/eckit/io/fam/FamListIterator.h b/src/eckit/io/fam/FamListIterator.h index 5439d36af..e0721743a 100644 --- a/src/eckit/io/fam/FamListIterator.h +++ b/src/eckit/io/fam/FamListIterator.h @@ -30,9 +30,10 @@ namespace eckit { class FamListIterator { public: // types using iterator_category = std::bidirectional_iterator_tag; - using value_type = FamObject; - using pointer = FamObject*; - using reference = Buffer&; + + using value_type = FamObject; + using pointer = value_type*; + using reference = Buffer&; public: // methods FamListIterator(const value_type& object); @@ -64,8 +65,9 @@ class FamListIterator { class FamListConstIterator: public FamListIterator { using FamListIterator::FamListIterator; - using pointer = const FamObject*; - using reference = const Buffer&; + using value_type = FamListIterator::value_type; + using pointer = const value_type*; + using reference = const Buffer&; public: // methods auto operator->() -> pointer { return FamListIterator::operator->(); } diff --git a/src/eckit/io/fam/FamMap.cc b/src/eckit/io/fam/FamMap.cc new file mode 100644 index 000000000..27fadebd1 --- /dev/null +++ b/src/eckit/io/fam/FamMap.cc @@ -0,0 +1,158 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +#include "eckit/io/fam/FamMap.h" + +#include "detail/FamMapNode.h" +#include "eckit/exception/Exceptions.h" +#include "eckit/io/fam/FamObject.h" +#include "eckit/io/fam/FamRegion.h" +#include "eckit/io/fam/FamRegionName.h" + +#include +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +FamMap::FamMap(const FamRegionName& regionName, const std::string& tableName): + region_ {regionName.lookup()}, + root_ {initSentinel(tableName + "-map-root", sizeof(FamMapNode))}, + table_ {initSentinel(tableName + "-map-table", capacity * sizeof(FamMapNode))}, + count_ {initSentinel(tableName + "-map-count", sizeof(size_type))} { } + +auto FamMap::initSentinel(const std::string& objectName, const size_type objectSize) const -> FamObject { + try { + return region_.allocateObject(objectSize, objectName); + } catch (const AlreadyExists&) { + auto object = region_.lookupObject(objectName); + ASSERT(object.size() == objectSize); + return object; + } +} + +//---------------------------------------------------------------------------------------------------------------------- +// iterators + +auto FamMap::begin() const -> iterator { + return {region_, FamMapNode::getNextOffset(root_)}; +} + +auto FamMap::cbegin() const -> const_iterator { + return {region_, FamMapNode::getNextOffset(root_)}; +} + +auto FamMap::end() const -> iterator { + return {region_, 0}; +} + +auto FamMap::cend() const -> const_iterator { + return {region_, 0}; +} + +//---------------------------------------------------------------------------------------------------------------------- +// lookup + +auto FamMap::at(const key_type& key) -> reference { + NOTIMP; +} + +auto FamMap::at(const key_type& key) const -> const_reference { + NOTIMP; +} + +auto FamMap::find(const key_type& key) -> iterator { + NOTIMP; +} + +auto FamMap::find(const key_type& key) const -> const_iterator { + NOTIMP; +} + +auto FamMap::contains(const key_type& key) const -> bool { + NOTIMP; +} + +// auto FamMap::front() const -> Buffer { +// return std::move(*begin()); +// } +// +// auto FamMap::back() const -> Buffer { +// return std::move(*--end()); +// } + +//---------------------------------------------------------------------------------------------------------------------- +// modifiers + +auto FamMap::insert(const value_type& value) -> iterator { + NOTIMP; +} + +auto FamMap::insert(value_type&& value) -> iterator { + NOTIMP; +} + +// void FamMap::push_back(const void* data, const fam::size_t length) { +// // allocate an object +// auto newObject = region_.allocateObject(sizeof(FamNode) + length); +// +// // set new object's next to tail +// newObject.put(tail_.descriptor(), offsetof(FamNode, next)); +// +// // set tail's prev to new object +// const auto prevOffset = tail_.swap(offsetof(FamNode, prev.offset), newObject.offset()); +// +// const auto oldObject = region_.proxyObject(prevOffset); +// +// // set old object's next to new object +// oldObject.put(newObject.descriptor(), offsetof(FamNode, next)); +// +// // set new object's prev to old object +// newObject.put(oldObject.descriptor(), offsetof(FamNode, prev)); +// +// // finally the data +// newObject.put(length, offsetof(FamNode, length)); +// newObject.put(data, sizeof(FamNode), length); +// +// // increment size +// size_.add(0, 1UL); +// } + +//---------------------------------------------------------------------------------------------------------------------- +// capacity + +auto FamMap::size() const -> size_type { + return count_.get(); +} + +auto FamMap::empty() const -> bool { + return size() == 0; +} + +//---------------------------------------------------------------------------------------------------------------------- + +void FamMap::print(std::ostream& out) const { + out << "FamMap[capacity=" << capacity << ",region=" << region_ << ",root=" << root_ << ",count=" << count_ << ']'; +} + +std::ostream& operator<<(std::ostream& out, const FamMap& list) { + list.print(out); + return out; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamMap.h b/src/eckit/io/fam/FamMap.h new file mode 100644 index 000000000..68ca21dc3 --- /dev/null +++ b/src/eckit/io/fam/FamMap.h @@ -0,0 +1,136 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +/// @file FamMap.h +/// @author Metin Cakircali +/// @date Jul 2024 + +#pragma once + +#include "eckit/io/fam/FamHashTable.h" +// #include "eckit/io/fam/FamMapIterator.h" + +#include +#include + +namespace eckit { + +class FamRegionName; + +//---------------------------------------------------------------------------------------------------------------------- + +/// @brief FamMap is an associative key-value container on FAM. Each element is organized depending on the +/// hash value of its key. +class FamMap { + static constexpr auto keySize = 32; // template? + + static constexpr auto capacity = 1024; + +public: // types + using key_type = FixedString; + using hash_type = FamHash; + using value_type = char; + // using key_equal = key_equal; + using size_type = fam::size_t; + using difference_type = size_type; + + // using mapped_type = mapped_type; + // using allocator_type = allocator_type; + // using pointer = pointer; + // using const_pointer = const_pointer; + + using reference = value_type&; + using const_reference = const value_type&; + + using iterator = FamMapIterator; + using const_iterator = FamMapConstIterator; + + // using local_iterator = local_iterator; + // using const_local_iterator = const_local_iterator; + + using node_type = FamList; + + // using insert_return_type = std::pair; + +public: // methods + FamMap(const FamRegionName& regionName, const std::string& tableName); + + ~FamMap() = default; + + // capacity + + auto size() const -> size_type; + + auto empty() const -> bool; + + auto max_size() const noexcept -> size_type { return capacity; } + + // iterators + + auto begin() const -> iterator; + + auto cbegin() const -> const_iterator; + + auto end() const -> iterator; + + auto cend() const -> const_iterator; + + // lookup + + // Returns reference to the element with specified key. + // throws std::out_of_range if not found + auto at(const key_type& key) -> reference; + auto at(const key_type& key) const -> const_reference; + + // operator[] ? + + // size_type count( const Key& key ) const; + + auto find(const key_type& key) -> iterator; + auto find(const key_type& key) const -> const_iterator; + + auto contains(const key_type& key) const -> bool; + + // modifiers + + auto insert(const value_type& value) -> iterator; + auto insert(value_type&& value) -> iterator; + + // void push_back(const void* data, size_type length); + // void push_front(const void* data, size_type length); + // void pop_front(); + // void pop_back(); + + // void clear() noexcept; + +private: // methods + auto initSentinel(const std::string& name, size_type size) const -> FamObject; + + void print(std::ostream& out) const; + + friend std::ostream& operator<<(std::ostream& out, const FamMap& list); + +private: // members + FamRegion region_; + FamObject root_; + FamObject table_; + FamObject count_; + + // FamHashTable table_; +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamMapIterator.cc b/src/eckit/io/fam/FamMapIterator.cc new file mode 100644 index 000000000..8c0d65e09 --- /dev/null +++ b/src/eckit/io/fam/FamMapIterator.cc @@ -0,0 +1,50 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +#include "eckit/io/fam/FamMapIterator.h" + +#include "detail/FamMapNode.h" + +// #include "detail/FamSessionDetail.h" +// #include "eckit/exception/Exceptions.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +FamMapIterator::FamMapIterator(const FamRegion& region, const fam::index_t offset): + region_ {region}, node_ {region_.proxyObject(offset)} { } + +auto FamMapIterator::operator++() -> FamMapIterator& { + if (const auto next = FamMapNode::getNext(node_); next.region > 0) { + node_.replaceWith(next); + list_.reset(); + } + return *this; +} + +auto FamMapIterator::operator->() -> pointer { + if (list_) { list_ = FamMapNode::getList(region_, node_); } + return list_.get(); +} + +auto FamMapIterator::operator*() -> reference { + if (list_) { list_ = FamMapNode::getList(region_, node_); } + return *list_; +} + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamMapIterator.h b/src/eckit/io/fam/FamMapIterator.h new file mode 100644 index 000000000..ce359e634 --- /dev/null +++ b/src/eckit/io/fam/FamMapIterator.h @@ -0,0 +1,78 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +/// @file FamMapIterator.h +/// @author Metin Cakircali +/// @date Jul 2024 + +#pragma once + +#include "eckit/io/fam/FamList.h" +#include "eckit/io/fam/FamRegion.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- +// ITERATOR + +class FamMapIterator { +public: // types + using iterator_category = std::forward_iterator_tag; + + using value_type = FamList; + using pointer = value_type*; + using reference = value_type&; + +public: // methods + FamMapIterator(const FamRegion& region, fam::index_t offset); + + auto operator++() -> FamMapIterator&; + + auto operator==(const FamMapIterator& other) const -> bool { return other.node_ == node_; } + + auto operator!=(const FamMapIterator& other) const -> bool { return !operator==(other); } + + auto operator->() -> pointer; + + auto operator*() -> reference; + +private: // members + FamRegion region_; + FamObject node_; + + std::unique_ptr list_; +}; + +//---------------------------------------------------------------------------------------------------------------------- +// CONST ITERATOR + +class FamMapConstIterator: public FamMapIterator { + using FamMapIterator::FamMapIterator; + + using value_type = FamMapIterator::value_type; + using pointer = const value_type*; + using reference = const value_type&; + +public: // methods + auto operator->() -> pointer { return FamMapIterator::operator->(); } + + auto operator*() -> reference { return FamMapIterator::operator*(); } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/FamObject.cc b/src/eckit/io/fam/FamObject.cc index 7fa8e741c..259fbb1e9 100644 --- a/src/eckit/io/fam/FamObject.cc +++ b/src/eckit/io/fam/FamObject.cc @@ -72,11 +72,11 @@ auto FamObject::exists() const -> bool { //---------------------------------------------------------------------------------------------------------------------- // PROPERTIES -auto FamObject::regionId() const -> std::uint64_t { +auto FamObject::regionId() const -> fam::index_t { return object_->get_global_descriptor().regionId; } -auto FamObject::offset() const -> std::uint64_t { +auto FamObject::offset() const -> fam::index_t { return object_->get_global_descriptor().offset; } diff --git a/src/eckit/io/fam/FamObject.h b/src/eckit/io/fam/FamObject.h index 696a65b9a..2c949d4af 100644 --- a/src/eckit/io/fam/FamObject.h +++ b/src/eckit/io/fam/FamObject.h @@ -58,9 +58,9 @@ class FamObject { // properties - auto regionId() const -> std::uint64_t; + auto regionId() const -> fam::index_t; - auto offset() const -> std::uint64_t; + auto offset() const -> fam::index_t; auto descriptor() const -> FamDescriptor { return {regionId(), offset()}; } @@ -79,7 +79,7 @@ class FamObject { void get(void* buffer, fam::size_t offset, fam::size_t length) const; template - auto get(const fam::size_t offset) const -> T { + auto get(const fam::size_t offset = 0) const -> T { auto buffer = T {0}; get(&buffer, offset, sizeof(T)); return buffer; diff --git a/src/eckit/io/fam/FamObjectName.h b/src/eckit/io/fam/FamObjectName.h index 4f8afeb8e..2367e91be 100644 --- a/src/eckit/io/fam/FamObjectName.h +++ b/src/eckit/io/fam/FamObjectName.h @@ -22,7 +22,7 @@ #include "eckit/io/Length.h" #include "eckit/io/Offset.h" #include "eckit/io/fam/FamName.h" -#include "eckit/io/fam/FamObject.h" +#include "eckit/io/fam/FamProperty.h" #include #include @@ -30,6 +30,7 @@ namespace eckit { class DataHandle; +class FamObject; //---------------------------------------------------------------------------------------------------------------------- diff --git a/src/eckit/io/fam/FamProperty.h b/src/eckit/io/fam/FamProperty.h index b5ead8b5a..a567d2f1e 100644 --- a/src/eckit/io/fam/FamProperty.h +++ b/src/eckit/io/fam/FamProperty.h @@ -40,19 +40,20 @@ using FamRegionDescriptor = openfam::Fam_Region_Descriptor; //---------------------------------------------------------------------------------------------------------------------- -/// @note mirrors Fam_Global_Descriptor -struct FamDescriptor { - std::uint64_t region {0}; - std::uint64_t offset {0}; -}; - namespace fam { -using size_t = std::uint64_t; -using perm_t = mode_t; +using size_t = std::uint64_t; +using perm_t = mode_t; +using index_t = std::uint64_t; } // namespace fam +/// @note mirrors Fam_Global_Descriptor +struct FamDescriptor { + fam::index_t region {0}; + fam::index_t offset {0}; +}; + //---------------------------------------------------------------------------------------------------------------------- struct FamProperty { diff --git a/src/eckit/io/fam/FamRegion.cc b/src/eckit/io/fam/FamRegion.cc index 8e25ad3dd..c1fd28504 100644 --- a/src/eckit/io/fam/FamRegion.cc +++ b/src/eckit/io/fam/FamRegion.cc @@ -58,7 +58,7 @@ auto FamRegion::exists() const -> bool { //---------------------------------------------------------------------------------------------------------------------- // PROPERTIES -auto FamRegion::index() const -> std::uint64_t { +auto FamRegion::index() const -> fam::index_t { return region_->get_global_descriptor().regionId; } @@ -81,7 +81,7 @@ auto FamRegion::property() const -> FamProperty { //---------------------------------------------------------------------------------------------------------------------- // OBJECT factory methods -auto FamRegion::proxyObject(const std::uint64_t offset) const -> FamObject { +auto FamRegion::proxyObject(const fam::index_t offset) const -> FamObject { return session_->proxyObject(index(), offset); } diff --git a/src/eckit/io/fam/FamRegion.h b/src/eckit/io/fam/FamRegion.h index 503211397..4f9940357 100644 --- a/src/eckit/io/fam/FamRegion.h +++ b/src/eckit/io/fam/FamRegion.h @@ -47,7 +47,7 @@ class FamRegion { // properties - auto index() const -> std::uint64_t; + auto index() const -> fam::index_t; auto size() const -> fam::size_t; @@ -60,7 +60,7 @@ class FamRegion { // object methods [[nodiscard]] - auto proxyObject(std::uint64_t offset) const -> FamObject; + auto proxyObject(fam::index_t offset) const -> FamObject; [[nodiscard]] auto lookupObject(const std::string& objectName) const -> FamObject; diff --git a/src/eckit/io/fam/FamRegionName.cc b/src/eckit/io/fam/FamRegionName.cc index 323868c71..3d30e53ca 100644 --- a/src/eckit/io/fam/FamRegionName.cc +++ b/src/eckit/io/fam/FamRegionName.cc @@ -18,6 +18,7 @@ #include "eckit/config/LibEcKit.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/URI.h" +#include "eckit/io/fam/FamRegion.h" #include "eckit/io/fam/detail/FamSessionDetail.h" #include "eckit/log/Log.h" diff --git a/src/eckit/io/fam/FamRegionName.h b/src/eckit/io/fam/FamRegionName.h index e0a0bcae6..b48648c86 100644 --- a/src/eckit/io/fam/FamRegionName.h +++ b/src/eckit/io/fam/FamRegionName.h @@ -20,13 +20,15 @@ #pragma once #include "eckit/io/fam/FamObjectName.h" -#include "eckit/io/fam/FamRegion.h" +#include "eckit/io/fam/FamProperty.h" #include #include namespace eckit { +class FamRegion; + //---------------------------------------------------------------------------------------------------------------------- class FamRegionName: public FamName { diff --git a/src/eckit/io/fam/FamURIManager.cc b/src/eckit/io/fam/FamURIManager.cc index f798aadcc..4ed927320 100644 --- a/src/eckit/io/fam/FamURIManager.cc +++ b/src/eckit/io/fam/FamURIManager.cc @@ -20,8 +20,6 @@ namespace eckit { -const static FamURIManager manager(FamPath::scheme); - //---------------------------------------------------------------------------------------------------------------------- FamURIManager::FamURIManager(const std::string& name): URIManager(name) { } @@ -55,6 +53,8 @@ std::string FamURIManager::asString(const URI& uri) const { return uri.scheme() + ":" + uri.name() + query + fragment; } +static FamURIManager manager(FamPath::scheme); + //---------------------------------------------------------------------------------------------------------------------- } // namespace eckit diff --git a/src/eckit/io/fam/detail/FamListNode.h b/src/eckit/io/fam/detail/FamListNode.h new file mode 100644 index 000000000..84adb0a9b --- /dev/null +++ b/src/eckit/io/fam/detail/FamListNode.h @@ -0,0 +1,58 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +/// @file FamListNode.h +/// @author Metin Cakircali +/// @date Mar 2024 + +#pragma once + +#include "FamNode.h" +#include "eckit/io/Buffer.h" + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +struct FamListNode: public FamNode { + FamDescriptor prev; + fam::size_t length {0}; + + //------------------------------------------------------------------------------------------------------------------ + // HELPERS (DO NOT add any virtual function here) + + static auto getPrev(const FamObject& object) -> FamDescriptor { + return object.get(offsetof(FamListNode, prev)); + } + + static auto getPrevOffset(const FamObject& object) -> std::uint64_t { + return object.get(offsetof(FamListNode, prev.offset)); + } + + static auto getLength(const FamObject& object) -> fam::size_t { + return object.get(offsetof(FamListNode, length)); + } + + static void getData(const FamObject& object, Buffer& buffer) { + if (const auto length = getLength(object); length > 0) { + buffer.resize(length); + object.get(buffer.data(), sizeof(FamListNode), length); + } + } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/detail/FamMapNode.h b/src/eckit/io/fam/detail/FamMapNode.h new file mode 100644 index 000000000..b1fdb740f --- /dev/null +++ b/src/eckit/io/fam/detail/FamMapNode.h @@ -0,0 +1,48 @@ +/* + * (C) Copyright 1996- ECMWF. + * + * This software is licensed under the terms of the Apache Licence Version 2.0 + * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. + * In applying this licence, ECMWF does not waive the privileges and immunities + * granted to it by virtue of its status as an intergovernmental organisation nor + * does it submit to any jurisdiction. + */ + +/* + * This software was developed as part of the Horizon Europe programme funded project OpenCUBE + * (Grant agreement: 101092984) horizon-opencube.eu + */ + +/// @file FamMapNode.h +/// @author Metin Cakircali +/// @date Jul 2024 + +#pragma once + +#include "FamNode.h" +#include "eckit/io/fam/FamList.h" + +#include + +namespace eckit { + +//---------------------------------------------------------------------------------------------------------------------- + +struct FamMapNode: public FamNode { + FamList::Descriptor desc; + + //------------------------------------------------------------------------------------------------------------------ + // HELPERS (DO NOT add any virtual function here) + + static auto getDescriptor(const FamObject& object) -> FamList::Descriptor { + return object.get(offsetof(FamMapNode, desc)); + } + + static auto getList(const FamRegion& region, const FamObject& object) -> std::unique_ptr { + return std::make_unique(region, FamMapNode::getDescriptor(object)); + } +}; + +//---------------------------------------------------------------------------------------------------------------------- + +} // namespace eckit diff --git a/src/eckit/io/fam/detail/FamNode.h b/src/eckit/io/fam/detail/FamNode.h index d207e3a63..dc83f033c 100644 --- a/src/eckit/io/fam/detail/FamNode.h +++ b/src/eckit/io/fam/detail/FamNode.h @@ -19,9 +19,10 @@ #pragma once -#include "eckit/io/Buffer.h" #include "eckit/io/fam/FamObject.h" +#include // uint8_t + namespace eckit { //---------------------------------------------------------------------------------------------------------------------- @@ -29,8 +30,6 @@ namespace eckit { struct FamNode { std::uint8_t version {1}; // 1 byte FamDescriptor next; - FamDescriptor prev; - fam::size_t length {0}; //------------------------------------------------------------------------------------------------------------------ // HELPERS (DO NOT add any virtual function here) @@ -42,25 +41,6 @@ struct FamNode { static auto getNextOffset(const FamObject& object) -> std::uint64_t { return object.get(offsetof(FamNode, next.offset)); } - - static auto getPrev(const FamObject& object) -> FamDescriptor { - return object.get(offsetof(FamNode, prev)); - } - - static auto getPrevOffset(const FamObject& object) -> std::uint64_t { - return object.get(offsetof(FamNode, prev.offset)); - } - - static auto getLength(const FamObject& object) -> fam::size_t { - return object.get(offsetof(FamNode, length)); - } - - static void getData(const FamObject& object, Buffer& buffer) { - if (const auto length = getLength(object); length > 0) { - buffer.resize(length); - object.get(buffer.data(), sizeof(FamNode), length); - } - } }; //---------------------------------------------------------------------------------------------------------------------- diff --git a/tests/io/CMakeLists.txt b/tests/io/CMakeLists.txt index ab200747a..1c93bcd0f 100644 --- a/tests/io/CMakeLists.txt +++ b/tests/io/CMakeLists.txt @@ -54,7 +54,7 @@ ecbuild_add_test( TARGET eckit_test_fam LABELS openfam LIBS eckit ) -ecbuild_add_test( TARGET eckit_test_famlist +ecbuild_add_test( TARGET eckit_test_fam_list SOURCES test_famlist.cc CONDITION HAVE_OPENFAM LABELS openfam