From b0772f13d2b108b75761b70147d49383790d60dd Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Mon, 7 Jun 2021 12:55:58 -0400 Subject: [PATCH 01/18] vector header created --- include/shad/core/vector.h | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 include/shad/core/vector.h diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h new file mode 100644 index 00000000..e69de29b From f5b6e4ee629c844774574f47ada7488d7b44492b Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Tue, 8 Jun 2021 19:29:32 -0400 Subject: [PATCH 02/18] fix typo in core/array --- include/shad/core/array.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/shad/core/array.h b/include/shad/core/array.h index bd7aacda..937dcfb3 100644 --- a/include/shad/core/array.h +++ b/include/shad/core/array.h @@ -99,7 +99,7 @@ class array : public AbstractDataStructure> { auto This = array::GetPtr(std::get<0>(IDs)); auto Other = array::GetPtr(std::get<1>(IDs)); - std::copy(Other->chunk_, Other->chunk_ + chunk_size(), This->churk_); + std::copy(Other->chunk_, Other->chunk_ + chunk_size(), This->chunk_); }, std::make_pair(this->oid_, O.oid_)); return *this; From e7ca4a502351220db0b969f39f1194bb1b17aa94 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Tue, 8 Jun 2021 19:29:50 -0400 Subject: [PATCH 03/18] progress on vector, has parts not implemented. --- include/shad/core/vector.h | 349 +++++++++++++++++++++++++++++++++++++ 1 file changed, 349 insertions(+) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index e69de29b..70d5d760 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -0,0 +1,349 @@ +//===------------------------------------------------------------*- C++ -*-===// +// +// SHAD +// +// The Scalable High-performance Algorithms and Data Structure Library +// +//===----------------------------------------------------------------------===// +// +// Copyright 2018 Battelle Memorial Institute +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// +//===----------------------------------------------------------------------===// + +#ifndef INCLUDE_SHAD_CORE_VECTOR_H_ +#define INCLUDE_SHAD_CORE_VECTOR_H_ + +#include + +#include "shad/data_structures/abstract_data_structure.h" +#include "shad/distributed_iterator_traits.h" +#include "shad/runtime/runtime.h" + +namespace shad { + +namespace impl { +/// @brief Fixed size distributed array. +/// TODO change the descriptions +/// Section 21.3.7.1 of the C++ standard defines the ::array as a fixed-size +/// sequence of objects. An ::array should be a contiguous container (as +/// defined in section 21.2.1). According to that definition, contiguous +/// containers requires contiguous iterators. The definition of contiguous +/// iterators implies contiguous memory allocation for the sequence, and it +/// cannot be guaranteed in many distributed settings. Therefore, ::array +/// relaxes this requirement. +/// +/// @tparam T The type of the elements in the distributed array. +/// @tparam N The number of element in the distributed array. +template +class vector : public AbstractDataStructure> { +template + class BaseVectorRef; + template + class VectorRef; + + template + class vector_iterator; + + friend class AbstractDataStructure>; + + public: + /// @defgroup Types + /// @{ + + /// The type for the global identifier. + using ObjectID = typename AbstractDataStructure>::ObjectID; + + /// The type of the stored value. + using value_type = T; + /// The type used to represent size. + using size_type = std::size_t; + /// The type used to represent distances. + using difference_type = std::ptrdiff_t; + /// The type of references to the element in the array. + using reference = VectorRef; + /// The type for const references to element in the array. + using const_reference = VectorRef; + /// The type for pointer to ::value_type. + using pointer = value_type *; + /// The type for pointer to ::const_value_type + using const_pointer = const value_type *; + /// The type of iterators on the array. + using iterator = vector_iterator; + /// The type of const iterators on the array. + using const_iterator = vector_iterator; + + // using reverse_iterator = TBD; + // using const_reverse_iterator = TBD; + /// @} + + public: + /// @brief The copy assignment operator. + /// + /// @param O The right-hand side of the operator. + /// @return A reference to the left-hand side. + vector &operator=(const vector &O) { + rt::executeOnAll( + [](const std::pair &IDs) { + auto This = vector::GetPtr(std::get<0>(IDs)); + auto Other = vector::GetPtr(std::get<1>(IDs)); + + if (This->chunk_size() != Other->chunk_size()) + This->chunk_ = std::unique_ptr{new T[Other->chunk_size()]}; + This->p_ = Other->p_; + + std::copy(Other->chunk_, Other->chunk_ + chunk_size(), This->chunk_); + }, + std::make_pair(this->oid_, O.oid_)); + return *this; + } + + /// @brief Fill the array with an input value. + /// + /// @param v The input value used to fill the array. + void fill(const value_type &v) { + rt::executeOnAll( + [](const std::pair &args) { + auto This = vector::GetPtr(std::get<0>(args)); + auto value = std::get<1>(args); + + std::fill(This->chunk_.get(), This->chunk_.get() + chunk_size(), value); + }, + std::make_pair(this->oid_, v)); + } + + /// @brief Swap the content of two array. + /// + /// @param O The array to swap the content with. + void swap(vector &O) noexcept /* (std::is_nothrow_swappable_v) */ { + rt::executeOnAll( + [](const std::pair &IDs) { + auto This = vector::GetPtr(std::get<0>(IDs)); + auto Other = vector::GetPtr(std::get<1>(IDs)); + + std::swap(This->p_, Other->p_); + std::swap(This->chunk_, Other->chunk_); + }, + std::make_pair(this->oid_, O.oid_)); + } + + /// @defgroup Capacity + /// @{ + + /// @brief Empty test. + /// @return true if empty (N=0), and false otherwise. + [[nodiscard]] constexpr bool empty() const noexcept { return p_.back() == 0; } + + /// @brief The size of the container. + /// @return the size of the container (N). + constexpr size_type size() const noexcept { return p_.back(); } + + /// @brief The maximum size of the container. + /// @return the maximum size of the container (N). + constexpr size_type max_size() const noexcept { return p_.back(); } + /// @} + + /// @defgroup Element Access + /// @{ + + /// @brief Unchecked element access operator. + /// @return a ::reference to the n-th element in the array. + constexpr reference operator[](size_type n) {\ + const auto l = locate_index(n); + return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; + } + + /// @brief Unchecked element access operator. + /// @return a ::const_reference to the n-th element in the array. + constexpr const_reference operator[](size_type n) const { + const auto l = locate_index(n); + return const_reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; + } + + /// @brief Checked element access operator. + /// @return a ::reference to the n-th element in the vector. + constexpr reference at(size_type n) { + if (n >= size()) throw std::out_of_range("Vector::at()"); + return operator[](n); + } + + /// @brief Checked element access operator. + /// @return a ::const_reference to the n-th element in the vector. + constexpr const_reference at(size_type n) const { + if (n >= size()) throw std::out_of_range("Vector::at() const"); + return operator[](n); + } + + /// @brief the first element in the array. + /// @return a ::reference to the element in position 0. + constexpr reference front() { return *begin(); } + /// @brief the first element in the array. + /// @return a ::const_reference to the element in position 0. + constexpr const_reference front() const { return *cbegin(); } + + /// @brief the last element in the array. + /// @return a ::reference to the element in position N - 1. + constexpr reference back() { return *(end() - 1); } + /// @brief the last element in the array. + /// @return a ::const_reference to the element in position N - 1. + constexpr const_reference back() const { return *(cend() - 1); } + + /// @} + + /// @brief DataStructure identifier getter. + /// + /// Returns the global object identifier associated to a DataStructure + /// instance. + /// + /// @warning It must be implemented in the inheriting DataStructure. + ObjectID GetGlobalID() const { return oid_; } + + friend bool operator!=(const vector &LHS, const vector &RHS) { + bool result[rt::numLocalities()]; + + rt::Handle H; + for (auto &l : rt::allLocalities()) { + asyncExecuteAtWithRet( + H, l, + [](const rt::Handle &, const std::pair &IDs, + bool *result) { + auto LHS = vector::GetPtr(std::get<0>(IDs)); + auto RHS = vector::GetPtr(std::get<1>(IDs)); + + *result = LHS->size() != RHS->size(); + + if (!*result) { + if (LHS->p_ == RHS->p_) + *result = !std::equal(LHS->chunk_, LHS->chunk_ + LHS->chunk_size(), + RHS->chunk_); + else + throw std::logic_error("Not yet implemented!"); + } + }, + std::make_pair(LHS.GetGlobalID(), RHS.GetGlobalID()), + &result[static_cast(l)]); + } + + rt::waitForCompletion(H); + + for (size_t i = 1; i < rt::numLocalities(); ++i) result[0] |= result[i]; + + return result[0]; + } + + friend bool operator>=(const vector &LHS, const vector &RHS) { + bool result[rt::numLocalities()]; + + rt::Handle H; + for (auto &l : rt::allLocalities()) { + asyncExecuteAtWithRet( + H, l, + [](const rt::Handle &, const std::pair &IDs, + bool *result) { + auto LHS = vector::GetPtr(std::get<0>(IDs)); + auto RHS = vector::GetPtr(std::get<1>(IDs)); + + if (LHS->p_ != RHS->p_) + throw std::logic_error("Not yet implemented!"); + + *result = std::lexicographical_compare( + LHS->chunk_, LHS->chunk_ + LHS->chunk_size(), RHS->chunk_, + RHS->chunk_ + RHS->chunk_size(), std::greater_equal()); + }, + std::make_pair(LHS.GetGlobalID(), RHS.GetGlobalID()), + &result[static_cast(l)]); + } + + rt::waitForCompletion(H); + + for (size_t i = 1; i < rt::numLocalities(); ++i) { + result[0] &= result[i]; + } + + return result[0]; + } + + friend bool operator<=(const vector &LHS, const vector &RHS) { + bool result[rt::numLocalities()]; + + rt::Handle H; + for (auto &l : rt::allLocalities()) { + asyncExecuteAtWithRet( + H, l, + [](const rt::Handle &, const std::pair &IDs, + bool *result) { + auto LHS = vector::GetPtr(std::get<0>(IDs)); + auto RHS = vector::GetPtr(std::get<1>(IDs)); + + if (LHS->p_ != RHS->p_) + throw std::logic_error("Not yet implemented!"); + + *result = std::lexicographical_compare( + LHS->chunk_, LHS->chunk_ + LHS->chunk_size(), RHS->chunk_, + RHS->chunk_ + RHS->chunk_size(), std::less_equal()); + }, + std::make_pair(LHS.GetGlobalID(), RHS.GetGlobalID()), + &result[static_cast(l)]); + } + + rt::waitForCompletion(H); + + for (size_t i = 1; i < rt::numLocalities(); ++i) { + result[0] &= result[i]; + } + + return result[0]; + } + + protected: + + template + std::size_t lowerbound_index(Iter begin, Iter end, Int v) { + return std::distance(begin + 1, std::lower_bound(begin, end, v + 1)); + } + + std::size_t locate_index(std::size_t i) { + return lowerbound_index(p_.cbegin(), p_.cend(), i); + } + + constexpr std::uint32_t my_id() const { + return rt::thisLocality().uint32_t(); + } + + constexpr std::size_t chunk_size() const { + return p_[my_id() + 1] - p[my_id()]; + } + + /// @brief Constructor. + explicit vector(ObjectID oid, std::size_t N) : oid_{oid} { + p_.reserve(rt::numLocalities() + 1); + p_.emplace_back(0); + for (std::uint32_t i = 1; i <= rt::numLocalities(); i++) + p_.emplace_back(p_.back() + i * N / rt::numLocalities()); + chunk_ = std::unique_ptr{new T[chunk_size()]}; + } + + private: + std::vector p_; + std::unique_ptr chunk_; + ObjectID oid_; +}; + + + +} // namespace impl + +} // namespace shad + +#endif /* INCLUDE_SHAD_CORE_VECTOR_H_ */ \ No newline at end of file From 29e1d68a02028b1c01d2f1311ef7a32978ec032f Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Tue, 8 Jun 2021 23:39:46 -0400 Subject: [PATCH 04/18] more progress has been made, about to finish first draft --- include/shad/core/vector.h | 517 ++++++++++++++++++++++++++++++++++++- 1 file changed, 513 insertions(+), 4 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 70d5d760..33551c04 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -48,7 +48,7 @@ namespace impl { /// @tparam N The number of element in the distributed array. template class vector : public AbstractDataStructure> { -template + template class BaseVectorRef; template class VectorRef; @@ -159,7 +159,7 @@ template /// @brief Unchecked element access operator. /// @return a ::reference to the n-th element in the array. - constexpr reference operator[](size_type n) {\ + constexpr reference operator[](size_type n) { const auto l = locate_index(n); return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; } @@ -309,11 +309,11 @@ template protected: template - std::size_t lowerbound_index(Iter begin, Iter end, Int v) { + static constexpr difference_type lowerbound_index(Iter begin, Iter end, Int v) { return std::distance(begin + 1, std::lower_bound(begin, end, v + 1)); } - std::size_t locate_index(std::size_t i) { + constexpr difference_type locate_index(std::size_t i) { return lowerbound_index(p_.cbegin(), p_.cend(), i); } @@ -340,7 +340,516 @@ template ObjectID oid_; }; +template +template +class vector::BaseVectorRef { + public: + using value_type = U; + using ObjectID = typename vector::ObjectID; + using difference_type = typename vector::difference_type; + using pointer = typename vector::pointer; + + ObjectID oid_; + mutable pointer chunk_; + difference_type pos_; + rt::Locality loc_; + + BaseVectorRef(rt::Locality l, difference_type p, ObjectID oid, pointer chunk) + : oid_(oid), chunk_(chunk), pos_(p), loc_(l) {} + + BaseVectorRef(const BaseVectorRef &O) + : oid_(O.oid_), chunk_(O.chunk_), pos_(O.pos_), loc_(O.loc_) {} + + BaseVectorRef(BaseVectorRef &&O) + : oid_(std::move(O.oid_)), + chunk_(std::move(O.chunk_)), + pos_(std::move(O.pos_)), + loc_(std::move(O.loc_)) {} + + BaseVectorRef &operator=(const BaseVectorRef &O) { + oid_ = O.oid_; + chunk_ = O.chunk_; + pos_ = O.pos_; + loc_ = O.loc_; + return *this; + } + + BaseVectorRef &operator=(BaseVectorRef &&O) { + oid_ = std::move(O.oid_); + chunk_ = std::move(O.chunk_); + pos_ = std::move(O.pos_); + loc_ = std::move(O.loc_); + return *this; + } + + bool operator==(const BaseVectorRef &O) const { + if (oid_ == O.oid_ && pos_ == O.pos_ && loc_ == O.loc_) return true; + return this->get() == O.get(); + } + + value_type get() const { + bool local = this->loc_ == rt::thisLocality(); + if (local) { + if (chunk_ == nullptr) { + auto This = vector::GetPtr(oid_); + chunk_ = This->chunk_.get(); + } + return chunk_[pos_]; + } + + if (chunk_ != nullptr) { + value_type result; + rt::executeAtWithRet( + loc_, + [](const std::pair &args, T *result) { + *result = std::get<0>(args)[std::get<1>(args)]; + }, + std::make_pair(chunk_, pos_), &result); + return result; + } + + std::pair resultPair; + rt::executeAtWithRet(loc_, + [](const std::pair &args, + std::pair *result) { + auto This = vector::GetPtr(std::get<0>(args)); + result->first = This->chunk_[std::get<1>(args)]; + result->second = This->chunk_.get(); + }, + std::make_pair(oid_, pos_), &resultPair); + chunk_ = resultPair.second; + return resultPair.first; + } +}; + +template +template +class alignas(64) array::VectorRef + : public vector::template BaseVectorRef { + public: + using value_type = U; + using pointer = typename vector::pointer; + using difference_type = typename vector::difference_type; + using ObjectID = typename vector::ObjectID; + + VectorRef(rt::Locality l, difference_type p, ObjectID oid, pointer chunk) + : vector::template BaseVectorRef(l, p, oid, chunk) {} + + VectorRef(const VectorRef &O) : vector::template BaseVectorRef(O) {} + + VectorRef(VectorRef &&O) : Vector::template BaseVectorRef(O) {} + + VectorRef &operator=(const Ref &O) { + vector::template BaseVectorRef::operator=(O); + return *this; + } + + VectorRef &operator=(VectorRef &&O) { + vector::template BaseVectorRef::operator=(O); + return *this; + } + + operator value_type() const { // NOLINT + return vector::template BaseVectorRef::get(); + } + + bool operator==(const VectorRef &&v) const { + return vector::template BaseVectorRef::operator==(v); + } + VectorRef &operator=(const T &v) { + bool local = this->loc_ == rt::thisLocality(); + if (local) { + if (this->chunk_ == nullptr) { + auto This = vector::GetPtr(this->oid_); + this->chunk_ = This->chunk_.get(); + } + this->chunk_[this->pos_] = v; + return *this; + } + + if (this->chunk_ == nullptr) { + rt::executeAtWithRet( + this->loc_, + [](const std::tuple &args, + pointer *result) { + auto This = vector::GetPtr(std::get<0>(args)); + This->chunk_[std::get<1>(args)] = std::get<2>(args); + *result = This->chunk_.get(); + }, + std::make_tuple(this->oid_, this->pos_, v), &this->chunk_); + } else { + rt::executeAt(this->loc_, + [](const std::tuple &args) { + std::get<0>(args)[std::get<1>(args)] = std::get<2>(args); + }, + std::make_tuple(this->chunk_, this->pos_, v)); + } + return *this; + } + + friend std::ostream &operator<<(std::ostream &stream, const VectorRef i) { + stream << i.loc_ << " " << i.pos_ << " " << i.get(); + return stream; + } +}; + +template +template +class alignas(64) vector::VectorRef + : public vector::template BaseVectorRef { + public: + using value_type = const U; + using pointer = typename vector::pointer; + using difference_type = typename vector::difference_type; + using ObjectID = typename vector::ObjectID; + + VectorRef(rt::Locality l, difference_type p, ObjectID oid, pointer chunk) + : vector::template BaseVectorRef(l, p, oid, chunk) {} + + VectorRef(const VectorRef &O) : vector::template BaseVectorRef(O) {} + + VectorRef(VectorRef &&O) : vector::template BaseVectorRef(O) {} + + bool operator==(const VectorRef &&v) const { + return vector::template BaseVectorRef::operator==(v); + } + + VectorRef &operator=(const VectorRef &O) { + vector::template BaseVectorRef::operator=(O); + return *this; + } + + VectorRef &operator=(VectorRef &&O) { + vector::template BaseVectorRef::operator=(O); + return *this; + } + + operator value_type() const { // NOLINT + return vector::template BaseVectorRef::get(); + } + + friend std::ostream &operator<<(std::ostream &stream, const VectorRef i) { + stream << i.loc_ << " " << i.pos_; + return stream; + } +}; + +template +bool operator==(const vector &LHS, const vector &RHS) { + return !(LHS != RHS); +} + +template +bool operator<(const vector &LHS, const vector &RHS) { + return !(LHS >= RHS); +} + +template +bool operator>(const vector &LHS, const vector &RHS) { + return !(LHS <= RHS); +} + +template +template +class alignas(64) vector::vector_iterator { + public: + using reference = typename vector::template VectorRef; + using pointer = typename vector::pointer; + using difference_type = std::ptrdiff_t; + using value_type = typename vector::value_type; + using iterator_category = std::random_access_iterator_tag; + + using local_iterator_type = pointer; + + using distribution_range = std::vector>; + + /// @brief Constructor. + vector_iterator(rt::Locality &&l, difference_type offset, ObjectID oid, + pointer chunk, std::size_t *p_) + : locality_(l), offset_(offset), oid_(oid), chunk_(chunk), p_(p_) {} + + /// @brief Default constructor. + vector_iterator() + : vector_iterator(rt::Locality(0), -1, ObjectID::kNullID, nullptr, nullptr) {} + + /// @brief Copy constructor. + vector_iterator(const vector_iterator &O) + : locality_(O.locality_), + offset_(O.offset_), + oid_(O.oid_), + chunk_(O.chunk_), + p_(O.p_) {} + + /// @brief Move constructor. + vector_iterator(const vector_iterator &&O) noexcept + : locality_(std::move(O.locality_)), + offset_(std::move(O.offset_)), + oid_(std::move(O.oid_)), + chunk_(std::move(O.chunk_)), + p_(std::move(O.p_)) {} + + /// @brief Copy assignment operator. + vector_iterator &operator=(const vector_iterator &O) { + locality_ = O.locality_; + offset_ = O.offset_; + oid_ = O.oid_; + chunk_ = O.chunk_; + p_ = O.p_; + + return *this; + } + + /// @brief Move assignment operator. + vector_iterator &operator=(vector_iterator &&O) { + locality_ = std::move(O.locality_); + offset_ = std::move(O.offset_); + oid_ = std::move(O.oid_); + chunk_ = std::move(O.chunk_); + p_ = std::move(O.p_); + + return *this; + } + + bool operator==(const vector_iterator &O) const { + return locality_ == O.locality_ && oid_ == O.oid_ && offset_ == O.offset_; + } + + bool operator!=(const vector_iterator &O) const { return !(*this == O); } + + reference operator*() { + update_chunk_pointer(); + return reference(locality_, offset_, oid_, chunk_); + } + + vector_iterator &operator++() { + const auto l = locality_.get(); + const auto g_offset = p_[l] + offset_ + 1; + if (g_offset < p_[l + 1]) + ++offset_; + else { + const auto num_l = rt::numLocalities(); + while (l < num_l && g_offset >= p_[l + 1]) + l++; + if (l == num_l) { + locality_ = rt::Locality(num_l - 1); + offset_ = p_[num_l] - p_[num_l - 1]; + } + else { + locality_ = rt::Locality(l); + offset_ = 0; + } + } + return *this; + } + + vector_iterator operator++(int) { + vector_iterator tmp(*this); + operator++(); + return tmp; + } + + vector_iterator &operator--() { + if (offset_ > 0) + --offset_; + else { + auto l = locality_.get(); + const difference_type g_offset = p_[l] - 1; + if (g_offset < 0) { + locality_ = rt::Locality(0); + offset_ = -1; + } + else { + while(g_offset < p_[l - 1]) + l--; + locality_ = rt::locality(l - 1); + offset_ = p_[l] - p_[l - 1] - 1; + } + } + + return *this; + } + + vector_iterator operator--(int) { + vector_iterator tmp(*this); + operator--(); + return tmp; + } + + vector_iterator &operator+=(difference_type n) { + const auto l = locality_.get(); + const auto g_offset = p_[l] + offset_ + n; + if (p_[l] <= g_offset && g_offset < p_[l + 1]) + offset_ += n; + else { + const auto num_l = rt::numLocalities(); + const auto l = vector::lowerbound_index(p_, p_ + num_l + 1, g_offset); + if (l < 0) { + locality_ = rt::Locality(0); + offset_ = -1; + } + else if (l >= num_l) { + locality_ = rt::Locality(num_l - 1); + offset_ = p[num_l] - p[num_l - 1]; + } + else { + locality_ = rt::Locality(l); + offset_ = g_offset - p_[l]; + } + } + + return *this; + } + + vector_iterator &operator-=(difference_type n) { + return operator+=(-n); + } + + vector_iterator operator+(difference_type n) { + vector_iterator tmp(*this); + return tmp.operator+=(n); + } + + vector_iterator operator-(difference_type n) { + vector_iterator tmp(*this); + return tmp.operator-=(n); + } + + difference_type operator-(const vector_iterator &O) const { + if (oid_ != O.oid_) return std::numeric_limits::min(); + + return get_global_id() - O.get_global_id(); + } + + bool operator<(const vector_iterator &O) const { + if (oid_ != O.oid_ || locality_ > O.locality_) return false; + return locality_ < O.locality_ || offset_ < O.offset_; + } + + bool operator>(const vector_iterator &O) const { + if (oid_ != O.oid_ || locality_ < O.locality_) return false; + return locality_ > O.locality_ || offset_ > O.offset_; + } + + bool operator<=(const vector_iterator &O) const { return !(*this > O); } + + bool operator>=(const vector_iterator &O) const { return !(*this < O); } + + friend std::ostream &operator<<(std::ostream &stream, + const vector_iterator i) { + stream << i.locality_ << " " << i.offset_; + return stream; + } + + class local_iterator_range { + public: + local_iterator_range(local_iterator_type B, local_iterator_type E) + : begin_(B), end_(E) {} + local_iterator_type begin() { return begin_; } + local_iterator_type end() { return end_; } + + private: + local_iterator_type begin_; + local_iterator_type end_; + }; + + static local_iterator_range local_range(vector_iterator &B, + vector_iterator &E) { + auto vectorPtr = vector::GetPtr(B.oid_); + auto begin{vectorPtr->chunk_.get()}; + + if (B.oid_ != E.oid_ || rt::thisLocality() < B.locality_ || rt::thisLocality() > E.locality_) + return local_iterator_range(begin, begin); + + if (B.locality_ == rt::thisLocality()) + begin += B.offset_; + + const auto l = rt::thisLocality().get(); + const auto chunk = B.p_[l + 1] - B.p_[l]; + + auto end{vectorPtr->chunk_.get() + chunk}; + if (E.locality_ == rt::thisLocality()) + end = vectorPtr->chunk_.get() + E.offset_; + return local_iterator_range(begin, end); + } + + static distribution_range + distribution(array_iterator &begin, array_iterator &end) { + distribution_range result; + + // First block: + typename array_iterator::difference_type start_block_size = chunk_size(); + if (pivot_locality() != rt::Locality(0) && + begin.locality_ >= pivot_locality()) { + start_block_size -= 1; + } + if (begin.locality_ == end.locality_) + start_block_size = end.offset_; + result.push_back(std::make_pair(begin.locality_, + start_block_size - begin.offset_)); + + // Middle blocks: + for (auto locality = begin.locality_ + 1; + locality < end.locality_; ++locality) { + typename array_iterator::difference_type inner_block_size = chunk_size(); + if (pivot_locality() != rt::Locality(0) && + locality >= pivot_locality()) { + inner_block_size -= 1; + } + result.push_back(std::make_pair(locality, inner_block_size)); + } + + // Last block: + if (end.offset_ != 0 && begin.locality_ != end.locality_) + result.push_back(std::make_pair(end.locality_, end.offset_)); + + return result; + } + + static rt::localities_range localities(array_iterator &B, array_iterator &E) { + return rt::localities_range( + B.locality_, rt::Locality(static_cast(E.locality_) + 1)); + } + + static array_iterator iterator_from_local(array_iterator &B, + array_iterator &E, + local_iterator_type itr) { + if (rt::thisLocality() < B.locality_ || rt::thisLocality() > E.locality_) + return E; + + auto arrayPtr = array::GetPtr(B.oid_); + return array_iterator(rt::thisLocality(), + std::distance(arrayPtr->chunk_.get(), itr), B.oid_, + arrayPtr->chunk_.get()); + } + + protected: + constexpr difference_type get_global_id() const { + return p_[locality_.get()] + offset_; + } + + private: + void update_chunk_pointer() const { + if (locality_ == rt::thisLocality()) { + auto This = array::GetPtr(oid_); + chunk_ = This->chunk_.get(); + return; + } + + rt::executeAtWithRet(locality_, + [](const ObjectID &ID, pointer *result) { + auto This = array::GetPtr(ID); + *result = This->chunk_.get(); + }, + oid_, &chunk_); + } + + rt::Locality locality_; + ObjectID oid_; + difference_type offset_; + mutable pointer chunk_; + std::size_t *p_; +}; } // namespace impl From 647da274d1d7fd948b7dbe7c8262dcb403672c20 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Wed, 9 Jun 2021 14:08:51 -0400 Subject: [PATCH 05/18] vector full draft finished --- include/shad/core/vector.h | 284 +++++++++++++++++++++++++++++++++---- 1 file changed, 258 insertions(+), 26 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 33551c04..4bd835de 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -137,6 +137,68 @@ class vector : public AbstractDataStructure> { }, std::make_pair(this->oid_, O.oid_)); } + /// @defgroup Iterators + /// @{ + + /// @brief The iterator to the beginning of the sequence. + /// @return an ::iterator to the beginning of the sequence. + constexpr iterator begin() noexcept { + if (rt::thisLocality() == rt::Locality(0)) { + return iterator{rt::Locality(0), 0, oid_, chunk_.get(), p_}; + } + return iterator{rt::Locality(0), 0, oid_, nullptr, p_}; + } + + /// @brief The iterator to the beginning of the sequence. + /// @return a ::const_iterator to the beginning of the sequence. + constexpr const_iterator begin() const noexcept { return cbegin(); } + + /// @brief The iterator to the end of the sequence. + /// @return an ::iterator to the end of the sequence. + constexpr iterator end() noexcept { + const auto end_l = locate_index(p_.back()) - 1; + + difference_type pos = p_[end_l + 1] - p_[end_l]; + + rt::Locality last(end_l); + pointer chunk = last == rt::thisLocality() ? chunk_.get() : nullptr; + return iterator{std::forward(last), pos, oid_, chunk, p_}; + } + + /// @brief The iterator to the end of the sequence. + /// @return a ::const_iterator to the end of the sequence. + constexpr const_iterator end() const noexcept { return cend(); } + + /// @brief The iterator to the beginning of the sequence. + /// @return a ::const_iterator to the beginning of the sequence. + constexpr const_iterator cbegin() const noexcept { + if (rt::thisLocality() == rt::Locality(0)) { + return const_iterator{rt::Locality(0), 0, oid_, chunk_.get(), p_}; + } + + pointer chunk = nullptr; // TODO WHY? + rt::executeAtWithRet(rt::Locality(0), + [](const ObjectID &ID, pointer *result) { + auto This = vector::GetPtr(ID); + *result = This->chunk_.get(); + }, + GetGlobalID(), &chunk); + return const_iterator{rt::Locality(0), 0, oid_, chunk, p_}; + } + + /// @brief The iterator to the end of the sequence. + /// @return a ::const_iterator to the end of the sequence. + constexpr const_iterator cend() const noexcept { + const auto end_l = locate_index(p_.back()) - 1; + + difference_type pos = p_[end_l + 1] - p_[end_l]; + + rt::Locality last(end_l); + pointer chunk = last == rt::thisLocality() ? chunk_.get() : nullptr; + return const_iterator{std::forward(last), pos, oid_, chunk, p_}; + } + + /// @} /// @defgroup Capacity /// @{ @@ -623,7 +685,7 @@ class alignas(64) vector::vector_iterator { } vector_iterator &operator++() { - const auto l = locality_.get(); + const auto l = locality_.uint32_t(); const auto g_offset = p_[l] + offset_ + 1; if (g_offset < p_[l + 1]) ++offset_; @@ -653,7 +715,7 @@ class alignas(64) vector::vector_iterator { if (offset_ > 0) --offset_; else { - auto l = locality_.get(); + auto l = locality_.uint32_t(); const difference_type g_offset = p_[l] - 1; if (g_offset < 0) { locality_ = rt::Locality(0); @@ -677,7 +739,7 @@ class alignas(64) vector::vector_iterator { } vector_iterator &operator+=(difference_type n) { - const auto l = locality_.get(); + const auto l = locality_.uint32_t(); const auto g_offset = p_[l] + offset_ + n; if (p_[l] <= g_offset && g_offset < p_[l + 1]) offset_ += n; @@ -764,7 +826,7 @@ class alignas(64) vector::vector_iterator { if (B.locality_ == rt::thisLocality()) begin += B.offset_; - const auto l = rt::thisLocality().get(); + const auto l = rt::thisLocality().uint32_t(); const auto chunk = B.p_[l + 1] - B.p_[l]; auto end{vectorPtr->chunk_.get() + chunk}; @@ -774,15 +836,12 @@ class alignas(64) vector::vector_iterator { } static distribution_range - distribution(array_iterator &begin, array_iterator &end) { + distribution(vector_iterator &begin, vector_iterator &end) { distribution_range result; // First block: - typename array_iterator::difference_type start_block_size = chunk_size(); - if (pivot_locality() != rt::Locality(0) && - begin.locality_ >= pivot_locality()) { - start_block_size -= 1; - } + const auto begin_l = begin.locality_.uint32_t(); + const auto start_block_size = begin.p_[begin_l + 1] - begin.p_[begin_l]; if (begin.locality_ == end.locality_) start_block_size = end.offset_; result.push_back(std::make_pair(begin.locality_, @@ -791,11 +850,8 @@ class alignas(64) vector::vector_iterator { // Middle blocks: for (auto locality = begin.locality_ + 1; locality < end.locality_; ++locality) { - typename array_iterator::difference_type inner_block_size = chunk_size(); - if (pivot_locality() != rt::Locality(0) && - locality >= pivot_locality()) { - inner_block_size -= 1; - } + const auto mid_l = locality.uint32_t(); + const auto inner_block_size = begin.p_[mid_l + 1] - begin.p_[mid_l]; result.push_back(std::make_pair(locality, inner_block_size)); } @@ -806,39 +862,39 @@ class alignas(64) vector::vector_iterator { return result; } - static rt::localities_range localities(array_iterator &B, array_iterator &E) { + static rt::localities_range localities(vector_iterator &B, vector_iterator &E) { return rt::localities_range( - B.locality_, rt::Locality(static_cast(E.locality_) + 1)); + B.locality_, rt::Locality(static_cast(E.locality_) + (E.offset_ != 0 ? 1 : 0))); } - static array_iterator iterator_from_local(array_iterator &B, - array_iterator &E, + static vector_iterator iterator_from_local(vector_iterator &B, + vector_iterator &E, local_iterator_type itr) { if (rt::thisLocality() < B.locality_ || rt::thisLocality() > E.locality_) return E; - auto arrayPtr = array::GetPtr(B.oid_); - return array_iterator(rt::thisLocality(), - std::distance(arrayPtr->chunk_.get(), itr), B.oid_, - arrayPtr->chunk_.get()); + auto vectorPtr = vector::GetPtr(B.oid_); + return vector_iterator(rt::thisLocality(), + std::distance(vectorPtr->chunk_.get(), itr), B.oid_, + vectorPtr->chunk_.get(), B.p_); } protected: constexpr difference_type get_global_id() const { - return p_[locality_.get()] + offset_; + return p_[locality_.uint32_t()] + offset_; } private: void update_chunk_pointer() const { if (locality_ == rt::thisLocality()) { - auto This = array::GetPtr(oid_); + auto This = vector::GetPtr(oid_); chunk_ = This->chunk_.get(); return; } rt::executeAtWithRet(locality_, [](const ObjectID &ID, pointer *result) { - auto This = array::GetPtr(ID); + auto This = vector::GetPtr(ID); *result = This->chunk_.get(); }, oid_, &chunk_); @@ -853,6 +909,182 @@ class alignas(64) vector::vector_iterator { } // namespace impl +/// @brief Fixed size distributed array. +/// +/// Section 21.3.7.1 of the C++ standard defines the ::array as a fixed-size +/// sequence of objects. An ::array should be a contiguous container (as +/// defined in section 21.2.1). According to that definition, contiguous +/// containers requires contiguous iterators. The definition of contiguous +/// iterators implies contiguous memory allocation for the sequence, and it +/// cannot be guaranteed in many distributed settings. Therefore, ::array +/// relaxes this requirement. +/// +/// @tparam T The type of the elements in the distributed array. +template +class vector { + using array_t = impl::vector; + + public: + /// @defgroup Types + /// @{ + /// The type of the stored value. + using value_type = typename vector_t::value_type; + /// The type used to represent size. + using size_type = typename vector_t::size_type; + /// The type used to represent distances. + using difference_type = typename vector_t::difference_type; + /// The type of references to the element in the array. + using reference = typename vector_t::reference; + /// The type for const references to element in the array. + using const_reference = typename vector_t::const_reference; + /// The type for pointer to ::value_type. + using pointer = typename vector_t::pointer; + /// The type for pointer to ::const_value_type + using const_pointer = typename vector_t::const_pointer; + /// The type of iterators on the array. + using iterator = typename vector_t::iterator; + /// The type of const iterators on the array. + using const_iterator = typename vector_t::const_iterator; + // todo reverse_iterator + // todo const_reverse_iterator + /// @} + + public: + /// @brief Constructor. + explicit vector() { ptr = vector_t::Create(); } + + /// @brief Destructor. + ~vector() { vector_t::Destroy(impl()->GetGlobalID()); } + + /// @brief The copy assignment operator. + /// + /// @param O The right-hand side of the operator. + /// @return A reference to the left-hand side. + vector &operator=(const vector &O) { + impl()->operator=(*O.ptr); + return *this; + } + + /// @defgroup Element Access + /// @{ + + /// @brief Unchecked element access operator. + /// @return a ::reference to the n-th element in the array. + constexpr reference operator[](size_type n) { return impl()->operator[](n); } + + /// @brief Unchecked element access operator. + /// @return a ::const_reference to the n-th element in the array. + constexpr const_reference operator[](size_type n) const { + return impl()->operator[](n); + } + + /// @brief Checked element access operator. + /// @return a ::reference to the n-th element in the array. + constexpr reference at(size_type n) { return impl()->at(n); } + + /// @brief Checked element access operator. + /// @return a ::const_reference to the n-th element in the array. + constexpr const_reference at(size_type n) const { return impl()->at(n); } + + /// @brief the first element in the array. + /// @return a ::reference to the element in position 0. + constexpr reference front() { return impl()->front(); } + + /// @brief the first element in the array. + /// @return a ::const_reference to the element in position 0. + constexpr const_reference front() const { return impl()->front(); } + + /// @brief the last element in the array. + /// @return a ::reference to the element in position N - 1. + constexpr reference back() { return impl()->back(); } + + /// @brief the last element in the array. + /// @return a ::const_reference to the element in position N - 1. + constexpr const_reference back() const { return impl()->back(); } + /// @} + + /// @defgroup Iterators + /// @{ + /// @brief The iterator to the beginning of the sequence. + /// @return an ::iterator to the beginning of the sequence. + constexpr iterator begin() noexcept { return impl()->begin(); } + + /// @brief The iterator to the beginning of the sequence. + /// @return a ::const_iterator to the beginning of the sequence. + constexpr const_iterator begin() const noexcept { return impl()->begin(); } + + /// @brief The iterator to the end of the sequence. + /// @return an ::iterator to the end of the sequence. + constexpr iterator end() noexcept { return impl()->end(); } + + /// @brief The iterator to the end of the sequence. + /// @return a ::const_iterator to the end of the sequence. + constexpr const_iterator end() const noexcept { return impl()->end(); } + + /// @brief The iterator to the beginning of the sequence. + /// @return a ::const_iterator to the beginning of the sequence. + constexpr const_iterator cbegin() const noexcept { return impl()->cbegin(); } + + /// @brief The iterator to the end of the sequence. + /// @return a ::const_iterator to the end of the sequence. + constexpr const_iterator cend() const noexcept { return impl()->cend(); } + + // todo rbegin + // todo crbegin + // todo rend + // todo crend + /// @} + + /// @defgroup Capacity + /// @{ + /// @brief Empty test. + /// @return true if empty (N=0), and false otherwise. + constexpr bool empty() const noexcept { return impl()->empty(); } + + /// @brief The size of the container. + /// @return the size of the container (N). + constexpr size_type size() const noexcept { return impl()->size(); } + + /// @brief The maximum size of the container. + /// @return the maximum size of the container (N). + constexpr size_type max_size() const noexcept { return impl()->max_size(); } + /// @} + + /// @defgroup Operations + /// @{ + /// @brief Fill the vector with an input value. + /// + /// @param v The input value used to fill the vector. + void fill(const value_type &v) { impl()->fill(v); } + + /// @brief Swap the content of two vector. + /// + /// @param O The vector to swap the content with. + void swap(vector &O) noexcept /* (std::is_nothrow_swappable_v) */ { + impl()->swap(*O->ptr); + } + /// @} + + private: + std::shared_ptr ptr = nullptr; + + const vector_t *impl() const { return ptr.get(); } + + vector_t *impl() { return ptr.get(); } + + friend bool operator==(const vector &LHS, const vector &RHS) { + return *LHS.ptr == *RHS.ptr; + } + + friend bool operator<(const vector &LHS, const vector &RHS) { + return operator<(*LHS.ptr, *RHS.ptr); + } + + friend bool operator>(const vector &LHS, const vector &RHS) { + return operator>(*LHS.ptr, *RHS.ptr); + } +}; + } // namespace shad #endif /* INCLUDE_SHAD_CORE_VECTOR_H_ */ \ No newline at end of file From 96c69458a1643b9e76f201bc1a055d82e7bfbe3f Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Wed, 9 Jun 2021 14:19:13 -0400 Subject: [PATCH 06/18] adding vector tests --- test/unit_tests/core/shad_vector_test.cc | 174 +++++++++++++++++++++++ 1 file changed, 174 insertions(+) create mode 100644 test/unit_tests/core/shad_vector_test.cc diff --git a/test/unit_tests/core/shad_vector_test.cc b/test/unit_tests/core/shad_vector_test.cc new file mode 100644 index 00000000..cc6e0a27 --- /dev/null +++ b/test/unit_tests/core/shad_vector_test.cc @@ -0,0 +1,174 @@ +//===------------------------------------------------------------*- C++ -*-===// +// +// SHAD +// +// The Scalable High-performance Algorithms and Data Structure Library +// +//===----------------------------------------------------------------------===// +// +// Copyright 2018 Battelle Memorial Institute +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// +//===----------------------------------------------------------------------===// + +#include +#include + +#include "gtest/gtest.h" + +#include "shad/core/vector.h" +#include "shad/runtime/runtime.h" + +template +struct VectorTestPair { + using value_type = Type; + constexpr static std::size_t size = Size; +}; + +template +constexpr std::size_t VectorTestPair::size; + +template +class VectorTest : public ::testing::Test { + public: + using ElementType = typename PairType::value_type; + using VectorType = shad::vector; + using size_type = typename VectorType::size_type; + + void SetUp() { + for (size_type i = 0; i < vector_.size(); ++i) vector_.at(i) = i; + } + + protected: + VectorType vector_; +}; + +TYPED_TEST_CASE_P(VectorTest); + +TYPED_TEST_P(VectorTest, HasTypeInterface) { + using VectorType = typename VectorTest::VectorType; + ASSERT_TRUE((std::is_same::value)); + ASSERT_TRUE((std::is_integral::value)); + ASSERT_TRUE((std::is_integral::value)); + + ASSERT_TRUE((std::is_convertible::value)); + ASSERT_TRUE((std::is_convertible::value)); +} + +TYPED_TEST_P(VectorTest, Size) { + ASSERT_EQ(this->vector_.size(), TypeParam::size); + ASSERT_EQ(this->vector_.max_size(), TypeParam::size); +} + +TYPED_TEST_P(VectorTest, AccessMethods) { + using size_type = typename VectorTest::size_type; + using VectorType = typename VectorTest::VectorType; + + for (size_type i = 0; i < this->vector_.size(); ++i) { + ASSERT_EQ(this->vector_[i], i); + ASSERT_EQ(this->vector_.at(i), i); + ASSERT_EQ(this->vector_[i], this->vector_.at(i)); + } + + try { + this->vector_.at(this->vector_.size()); + FAIL(); + } catch (std::out_of_range &) { + SUCCEED(); + } catch (...) { + FAIL(); + } + + ASSERT_EQ(this->vector_[0], this->vector_.front()); + ASSERT_EQ(this->vector_[this->vector_.size() - 1], this->vector_.back()); + + // const_reference portion + using const_reference = typename VectorType::const_reference; + const VectorType &ref = this->vector_; + for (size_type i = 0; i < this->vector_.size(); ++i) { + const_reference first = ref[i]; + const_reference second = ref.at(i); + ASSERT_EQ(first, i); + ASSERT_EQ(second, i); + ASSERT_EQ(first, second); + } + + try { + const_reference _ = ref.at(this->vector_.size()); + FAIL(); + } catch (std::out_of_range &) { + SUCCEED(); + } catch (...) { + FAIL(); + } + + ASSERT_EQ(ref[0], ref.front()); + ASSERT_EQ(ref[ref.size() - 1], ref.back()); +} + +TYPED_TEST_P(VectorTest, IteratorMovements) { + using VectorType = typename VectorTest::VectorType; + + using iterator = typename VectorType::iterator; + + ssize_t i = 0; + for (iterator itr = this->vector_.begin(), end = this->vector_.end(); + itr != end; ++itr, ++i) { + ASSERT_EQ(*itr, this->vector_[i]); + } + + i = this->vector_.size() - 1; + for (iterator itr = this->vector_.end() - 1, end = this->vector_.begin(); + itr >= end; --itr, --i) { + ASSERT_EQ(*itr, this->vector_[i]); + } + + iterator b = this->vector_.begin(); + for (i = 0; i < this->vector_.size(); + i += shad::rt::numLocalities(), b += shad::rt::numLocalities()) { + ASSERT_EQ(*b, this->vector_[i]); + ASSERT_EQ(*(this->vector_.begin() + i), this->vector_[i]); + } + + iterator e = this->vector_.end() - 1; + for (i = 0; i < this->vector_.size(); + i -= shad::rt::numLocalities(), e -= shad::rt::numLocalities()) { + ASSERT_EQ(*e, this->vector_[this->vector_.size() - (i + 1)]); + ASSERT_EQ(*(this->vector_.end() - (i + 1)), + this->vector_[this->vector_.size() - (i + 1)]); + } + + size_t pivot = this->vector_.size() % shad::rt::numLocalities(); + size_t block = this->vector_.size() / shad::rt::numLocalities(); + size_t offset = 0; + for (size_t i = 0; i < shad::rt::numLocalities(); + ++i, offset += pivot != 0 && i >= pivot ? (block - 1) : block) { + if (offset < this->vector_.size()) + ASSERT_EQ(*(this->vector_.begin() + offset), this->vector_[offset]); + if (offset == this->vector_.size()) + ASSERT_EQ((this->vector_.begin() + offset), this->vector_.end()); + ASSERT_EQ(*(this->vector_.begin() + offset), offset); + } +} + +REGISTER_TYPED_TEST_CASE_P(VectorTest, HasTypeInterface, Size, AccessMethods, + IteratorMovements); + +using VectorTestTypes = + ::testing::Types, VectorTestPair, + VectorTestPair, VectorTestPair>; +INSTANTIATE_TYPED_TEST_CASE_P(ShadVector, VectorTest, VectorTestTypes); From 536824369e41da2a887c0e31509c7cf8acdd56bc Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 16:47:19 -0400 Subject: [PATCH 07/18] implemented pointer saving in vector --- include/shad/core/vector.h | 223 +++++++++-------------- test/unit_tests/core/CMakeLists.txt | 1 + test/unit_tests/core/shad_vector_test.cc | 4 +- 3 files changed, 94 insertions(+), 134 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 4bd835de..5b0c8e70 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -45,7 +45,6 @@ namespace impl { /// relaxes this requirement. /// /// @tparam T The type of the elements in the distributed array. -/// @tparam N The number of element in the distributed array. template class vector : public AbstractDataStructure> { template @@ -143,10 +142,7 @@ class vector : public AbstractDataStructure> { /// @brief The iterator to the beginning of the sequence. /// @return an ::iterator to the beginning of the sequence. constexpr iterator begin() noexcept { - if (rt::thisLocality() == rt::Locality(0)) { - return iterator{rt::Locality(0), 0, oid_, chunk_.get(), p_}; - } - return iterator{rt::Locality(0), 0, oid_, nullptr, p_}; + return iterator{rt::Locality(0), 0, oid_, ptrs_.data(), p_.data()}; } /// @brief The iterator to the beginning of the sequence. @@ -161,8 +157,8 @@ class vector : public AbstractDataStructure> { difference_type pos = p_[end_l + 1] - p_[end_l]; rt::Locality last(end_l); - pointer chunk = last == rt::thisLocality() ? chunk_.get() : nullptr; - return iterator{std::forward(last), pos, oid_, chunk, p_}; + + return iterator{std::forward(last), pos, oid_, ptrs_.data(), p_.data()}; } /// @brief The iterator to the end of the sequence. @@ -172,18 +168,7 @@ class vector : public AbstractDataStructure> { /// @brief The iterator to the beginning of the sequence. /// @return a ::const_iterator to the beginning of the sequence. constexpr const_iterator cbegin() const noexcept { - if (rt::thisLocality() == rt::Locality(0)) { - return const_iterator{rt::Locality(0), 0, oid_, chunk_.get(), p_}; - } - - pointer chunk = nullptr; // TODO WHY? - rt::executeAtWithRet(rt::Locality(0), - [](const ObjectID &ID, pointer *result) { - auto This = vector::GetPtr(ID); - *result = This->chunk_.get(); - }, - GetGlobalID(), &chunk); - return const_iterator{rt::Locality(0), 0, oid_, chunk, p_}; + return const_iterator{rt::Locality(0), 0, oid_, ptrs_.data(), p_.data()}; } /// @brief The iterator to the end of the sequence. @@ -194,8 +179,8 @@ class vector : public AbstractDataStructure> { difference_type pos = p_[end_l + 1] - p_[end_l]; rt::Locality last(end_l); - pointer chunk = last == rt::thisLocality() ? chunk_.get() : nullptr; - return const_iterator{std::forward(last), pos, oid_, chunk, p_}; + + return const_iterator{std::forward(last), pos, oid_, ptrs_.data(), p_.data()}; } /// @} @@ -222,7 +207,7 @@ class vector : public AbstractDataStructure> { /// @brief Unchecked element access operator. /// @return a ::reference to the n-th element in the array. constexpr reference operator[](size_type n) { - const auto l = locate_index(n); + const std::uint32_t l = locate_index(n); return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; } @@ -299,7 +284,7 @@ class vector : public AbstractDataStructure> { rt::waitForCompletion(H); - for (size_t i = 1; i < rt::numLocalities(); ++i) result[0] |= result[i]; + for (std::uint32_t i = 1; i < rt::numLocalities(); ++i) result[0] |= result[i]; return result[0]; } @@ -329,7 +314,7 @@ class vector : public AbstractDataStructure> { rt::waitForCompletion(H); - for (size_t i = 1; i < rt::numLocalities(); ++i) { + for (std::uint32_t i = 1; i < rt::numLocalities(); ++i) { result[0] &= result[i]; } @@ -361,7 +346,7 @@ class vector : public AbstractDataStructure> { rt::waitForCompletion(H); - for (size_t i = 1; i < rt::numLocalities(); ++i) { + for (std::uint32_t i = 1; i < rt::numLocalities(); ++i) { result[0] &= result[i]; } @@ -375,29 +360,54 @@ class vector : public AbstractDataStructure> { return std::distance(begin + 1, std::lower_bound(begin, end, v + 1)); } - constexpr difference_type locate_index(std::size_t i) { + constexpr difference_type locate_index(size_type i) const { return lowerbound_index(p_.cbegin(), p_.cend(), i); } - constexpr std::uint32_t my_id() const { - return rt::thisLocality().uint32_t(); - } - - constexpr std::size_t chunk_size() const { - return p_[my_id() + 1] - p[my_id()]; + constexpr size_type chunk_size() const { + return p_[to_int(rt::thisLocality()) + 1] - p_[to_int(rt::thisLocality())]; } /// @brief Constructor. - explicit vector(ObjectID oid, std::size_t N) : oid_{oid} { + explicit vector(ObjectID oid, size_type N) : oid_{oid} { p_.reserve(rt::numLocalities() + 1); p_.emplace_back(0); for (std::uint32_t i = 1; i <= rt::numLocalities(); i++) p_.emplace_back(p_.back() + i * N / rt::numLocalities()); chunk_ = std::unique_ptr{new T[chunk_size()]}; + + ptrs_.resize(rt::numLocalities()); + + // TODO might want to change to alltoall + rt::executeOnAll([](const std::tuple &args) { + const auto &[ID, master_l, master_ptrs] = args; + auto This = vector::GetPtr(ID); + if (rt::thisLocality() != master_l) { + const auto my_ptr = This->chunk_.get(); + rt::dma(master_l, master_ptrs + to_int(rt::thisLocality()), &my_ptr, 1); + } + else + This->ptrs_[to_int(rt::thisLocality())] = This->chunk_.get(); + }, + std::tuple{GetGlobalID(), rt::thisLocality(), ptrs_.data()}); + + rt::executeOnAll([](const std::tuple &args) { + const auto &[ID, master_l, master_ptrs] = args; + if (rt::thisLocality() != master_l) { + auto This = vector::GetPtr(ID); + rt::dma(This->ptrs_.data(), master_l, master_ptrs, rt::numLocalities()); + } + }, + std::tuple{GetGlobalID(), rt::thisLocality(), ptrs_.data()}); } private: - std::vector p_; + static constexpr std::uint32_t to_int(rt::Locality l) { + return static_cast(l); + } + + std::vector ptrs_; + std::vector p_; std::unique_ptr chunk_; ObjectID oid_; }; @@ -412,7 +422,7 @@ class vector::BaseVectorRef { using pointer = typename vector::pointer; ObjectID oid_; - mutable pointer chunk_; + pointer chunk_; difference_type pos_; rt::Locality loc_; @@ -450,43 +460,18 @@ class vector::BaseVectorRef { } value_type get() const { - bool local = this->loc_ == rt::thisLocality(); - if (local) { - if (chunk_ == nullptr) { - auto This = vector::GetPtr(oid_); - chunk_ = This->chunk_.get(); - } + if (this->loc_ == rt::thisLocality()) return chunk_[pos_]; - } - if (chunk_ != nullptr) { - value_type result; - rt::executeAtWithRet( - loc_, - [](const std::pair &args, T *result) { - *result = std::get<0>(args)[std::get<1>(args)]; - }, - std::make_pair(chunk_, pos_), &result); - return result; - } - - std::pair resultPair; - rt::executeAtWithRet(loc_, - [](const std::pair &args, - std::pair *result) { - auto This = vector::GetPtr(std::get<0>(args)); - result->first = This->chunk_[std::get<1>(args)]; - result->second = This->chunk_.get(); - }, - std::make_pair(oid_, pos_), &resultPair); - chunk_ = resultPair.second; - return resultPair.first; + value_type result; + rt::dma(&result, loc_, chunk_ + pos_, 1); + return result; } }; template template -class alignas(64) array::VectorRef +class alignas(64) vector::VectorRef : public vector::template BaseVectorRef { public: using value_type = U; @@ -495,13 +480,13 @@ class alignas(64) array::VectorRef using ObjectID = typename vector::ObjectID; VectorRef(rt::Locality l, difference_type p, ObjectID oid, pointer chunk) - : vector::template BaseVectorRef(l, p, oid, chunk) {} + : vector::template BaseVectorRef(l, p, oid, chunk) {} - VectorRef(const VectorRef &O) : vector::template BaseVectorRef(O) {} + VectorRef(const VectorRef &O) : vector::template BaseVectorRef(O) {} - VectorRef(VectorRef &&O) : Vector::template BaseVectorRef(O) {} + VectorRef(VectorRef &&O) : vector::template BaseVectorRef(O) {} - VectorRef &operator=(const Ref &O) { + VectorRef &operator=(const VectorRef &O) { vector::template BaseVectorRef::operator=(O); return *this; } @@ -522,31 +507,12 @@ class alignas(64) array::VectorRef VectorRef &operator=(const T &v) { bool local = this->loc_ == rt::thisLocality(); if (local) { - if (this->chunk_ == nullptr) { - auto This = vector::GetPtr(this->oid_); - this->chunk_ = This->chunk_.get(); - } this->chunk_[this->pos_] = v; return *this; } - if (this->chunk_ == nullptr) { - rt::executeAtWithRet( - this->loc_, - [](const std::tuple &args, - pointer *result) { - auto This = vector::GetPtr(std::get<0>(args)); - This->chunk_[std::get<1>(args)] = std::get<2>(args); - *result = This->chunk_.get(); - }, - std::make_tuple(this->oid_, this->pos_, v), &this->chunk_); - } else { - rt::executeAt(this->loc_, - [](const std::tuple &args) { - std::get<0>(args)[std::get<1>(args)] = std::get<2>(args); - }, - std::make_tuple(this->chunk_, this->pos_, v)); - } + rt::dma(this->loc_, this->chunk_ + this->pos_, &v, 1); + return *this; } @@ -556,7 +522,7 @@ class alignas(64) array::VectorRef } }; -template +template template class alignas(64) vector::VectorRef : public vector::template BaseVectorRef { @@ -614,7 +580,7 @@ bool operator>(const vector &LHS, const vector &RHS) { template template -class alignas(64) vector::vector_iterator { +class alignas(64) vector::vector_iterator { public: using reference = typename vector::template VectorRef; using pointer = typename vector::pointer; @@ -628,8 +594,8 @@ class alignas(64) vector::vector_iterator { /// @brief Constructor. vector_iterator(rt::Locality &&l, difference_type offset, ObjectID oid, - pointer chunk, std::size_t *p_) - : locality_(l), offset_(offset), oid_(oid), chunk_(chunk), p_(p_) {} + pointer const *ptrs, difference_type const *p_) + : locality_(l), offset_(offset), oid_(oid), ptrs_(ptrs), p_(p_) {} /// @brief Default constructor. vector_iterator() @@ -640,7 +606,7 @@ class alignas(64) vector::vector_iterator { : locality_(O.locality_), offset_(O.offset_), oid_(O.oid_), - chunk_(O.chunk_), + ptrs_(O.ptrs_), p_(O.p_) {} /// @brief Move constructor. @@ -648,7 +614,7 @@ class alignas(64) vector::vector_iterator { : locality_(std::move(O.locality_)), offset_(std::move(O.offset_)), oid_(std::move(O.oid_)), - chunk_(std::move(O.chunk_)), + ptrs_(std::move(O.ptrs_)), p_(std::move(O.p_)) {} /// @brief Copy assignment operator. @@ -656,7 +622,7 @@ class alignas(64) vector::vector_iterator { locality_ = O.locality_; offset_ = O.offset_; oid_ = O.oid_; - chunk_ = O.chunk_; + ptrs_ = O.ptrs_; p_ = O.p_; return *this; @@ -667,7 +633,7 @@ class alignas(64) vector::vector_iterator { locality_ = std::move(O.locality_); offset_ = std::move(O.offset_); oid_ = std::move(O.oid_); - chunk_ = std::move(O.chunk_); + ptrs_ = std::move(O.ptrs_); p_ = std::move(O.p_); return *this; @@ -680,12 +646,11 @@ class alignas(64) vector::vector_iterator { bool operator!=(const vector_iterator &O) const { return !(*this == O); } reference operator*() { - update_chunk_pointer(); - return reference(locality_, offset_, oid_, chunk_); + return reference(locality_, offset_, oid_, get_chunk()); } vector_iterator &operator++() { - const auto l = locality_.uint32_t(); + std::uint32_t l = locality_; const auto g_offset = p_[l] + offset_ + 1; if (g_offset < p_[l + 1]) ++offset_; @@ -715,7 +680,7 @@ class alignas(64) vector::vector_iterator { if (offset_ > 0) --offset_; else { - auto l = locality_.uint32_t(); + std::uint32_t l = locality_; const difference_type g_offset = p_[l] - 1; if (g_offset < 0) { locality_ = rt::Locality(0); @@ -724,7 +689,7 @@ class alignas(64) vector::vector_iterator { else { while(g_offset < p_[l - 1]) l--; - locality_ = rt::locality(l - 1); + locality_ = rt::Locality(l - 1); offset_ = p_[l] - p_[l - 1] - 1; } } @@ -739,7 +704,7 @@ class alignas(64) vector::vector_iterator { } vector_iterator &operator+=(difference_type n) { - const auto l = locality_.uint32_t(); + const std::uint32_t l = locality_; const auto g_offset = p_[l] + offset_ + n; if (p_[l] <= g_offset && g_offset < p_[l + 1]) offset_ += n; @@ -752,7 +717,7 @@ class alignas(64) vector::vector_iterator { } else if (l >= num_l) { locality_ = rt::Locality(num_l - 1); - offset_ = p[num_l] - p[num_l - 1]; + offset_ = p_[num_l] - p_[num_l - 1]; } else { locality_ = rt::Locality(l); @@ -817,8 +782,7 @@ class alignas(64) vector::vector_iterator { static local_iterator_range local_range(vector_iterator &B, vector_iterator &E) { - auto vectorPtr = vector::GetPtr(B.oid_); - auto begin{vectorPtr->chunk_.get()}; + auto begin{B.get_local_chunk()}; if (B.oid_ != E.oid_ || rt::thisLocality() < B.locality_ || rt::thisLocality() > E.locality_) return local_iterator_range(begin, begin); @@ -826,12 +790,12 @@ class alignas(64) vector::vector_iterator { if (B.locality_ == rt::thisLocality()) begin += B.offset_; - const auto l = rt::thisLocality().uint32_t(); + const std::uint32_t l = rt::thisLocality(); const auto chunk = B.p_[l + 1] - B.p_[l]; - auto end{vectorPtr->chunk_.get() + chunk}; + auto end{B.get_local_chunk() + chunk}; if (E.locality_ == rt::thisLocality()) - end = vectorPtr->chunk_.get() + E.offset_; + end = B.get_local_chunk() + E.offset_; return local_iterator_range(begin, end); } @@ -840,7 +804,7 @@ class alignas(64) vector::vector_iterator { distribution_range result; // First block: - const auto begin_l = begin.locality_.uint32_t(); + const std::uint32_t begin_l = begin.locality_; const auto start_block_size = begin.p_[begin_l + 1] - begin.p_[begin_l]; if (begin.locality_ == end.locality_) start_block_size = end.offset_; @@ -850,7 +814,7 @@ class alignas(64) vector::vector_iterator { // Middle blocks: for (auto locality = begin.locality_ + 1; locality < end.locality_; ++locality) { - const auto mid_l = locality.uint32_t(); + const std::uint32_t mid_l = locality; const auto inner_block_size = begin.p_[mid_l + 1] - begin.p_[mid_l]; result.push_back(std::make_pair(locality, inner_block_size)); } @@ -873,38 +837,31 @@ class alignas(64) vector::vector_iterator { if (rt::thisLocality() < B.locality_ || rt::thisLocality() > E.locality_) return E; - auto vectorPtr = vector::GetPtr(B.oid_); return vector_iterator(rt::thisLocality(), - std::distance(vectorPtr->chunk_.get(), itr), B.oid_, - vectorPtr->chunk_.get(), B.p_); + std::distance(B.get_local_chunk(), itr), B.oid_, + B.get_local_chunk(), B.p_); } protected: constexpr difference_type get_global_id() const { - return p_[locality_.uint32_t()] + offset_; + return p_[to_int(locality_)] + offset_; } - private: - void update_chunk_pointer() const { - if (locality_ == rt::thisLocality()) { - auto This = vector::GetPtr(oid_); - chunk_ = This->chunk_.get(); - return; - } + constexpr pointer get_chunk() const { + return ptrs_[to_int(locality_)]; + } - rt::executeAtWithRet(locality_, - [](const ObjectID &ID, pointer *result) { - auto This = vector::GetPtr(ID); - *result = This->chunk_.get(); - }, - oid_, &chunk_); + constexpr pointer get_local_chunk() const { + return ptrs_[to_int(rt::thisLocality())]; } + private: + rt::Locality locality_; ObjectID oid_; difference_type offset_; - mutable pointer chunk_; - std::size_t *p_; + pointer const *ptrs_; + difference_type const *p_; }; } // namespace impl @@ -922,7 +879,7 @@ class alignas(64) vector::vector_iterator { /// @tparam T The type of the elements in the distributed array. template class vector { - using array_t = impl::vector; + using vector_t = impl::vector; public: /// @defgroup Types @@ -951,7 +908,7 @@ class vector { public: /// @brief Constructor. - explicit vector() { ptr = vector_t::Create(); } + explicit vector(size_type N = 0) { ptr = vector_t::Create(N); } /// @brief Destructor. ~vector() { vector_t::Destroy(impl()->GetGlobalID()); } diff --git a/test/unit_tests/core/CMakeLists.txt b/test/unit_tests/core/CMakeLists.txt index fab5fce6..1384e23e 100755 --- a/test/unit_tests/core/CMakeLists.txt +++ b/test/unit_tests/core/CMakeLists.txt @@ -2,6 +2,7 @@ set(tests iterator_test for_test shad_array_test + shad_vector_test unordered_set_test unordered_map_test shad_algorithm_test diff --git a/test/unit_tests/core/shad_vector_test.cc b/test/unit_tests/core/shad_vector_test.cc index cc6e0a27..5bb395f8 100644 --- a/test/unit_tests/core/shad_vector_test.cc +++ b/test/unit_tests/core/shad_vector_test.cc @@ -43,9 +43,11 @@ template class VectorTest : public ::testing::Test { public: using ElementType = typename PairType::value_type; - using VectorType = shad::vector; + using VectorType = shad::vector; using size_type = typename VectorType::size_type; + VectorTest() : vector_(PairType::size) {} + void SetUp() { for (size_type i = 0; i < vector_.size(); ++i) vector_.at(i) = i; } From b059b3bc3fbae8be91be18d92918e1a6721bd594 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 22:02:07 -0400 Subject: [PATCH 08/18] buggy constructor --- include/shad/core/vector.h | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 5b0c8e70..5b1f65c3 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -371,34 +371,21 @@ class vector : public AbstractDataStructure> { /// @brief Constructor. explicit vector(ObjectID oid, size_type N) : oid_{oid} { p_.reserve(rt::numLocalities() + 1); - p_.emplace_back(0); - for (std::uint32_t i = 1; i <= rt::numLocalities(); i++) - p_.emplace_back(p_.back() + i * N / rt::numLocalities()); + for (std::uint32_t i = 0; i <= rt::numLocalities(); i++) + p_.emplace_back(i * N / rt::numLocalities()); chunk_ = std::unique_ptr{new T[chunk_size()]}; ptrs_.resize(rt::numLocalities()); - // TODO might want to change to alltoall - rt::executeOnAll([](const std::tuple &args) { - const auto &[ID, master_l, master_ptrs] = args; - auto This = vector::GetPtr(ID); - if (rt::thisLocality() != master_l) { - const auto my_ptr = This->chunk_.get(); - rt::dma(master_l, master_ptrs + to_int(rt::thisLocality()), &my_ptr, 1); - } - else - This->ptrs_[to_int(rt::thisLocality())] = This->chunk_.get(); - }, - std::tuple{GetGlobalID(), rt::thisLocality(), ptrs_.data()}); - - rt::executeOnAll([](const std::tuple &args) { - const auto &[ID, master_l, master_ptrs] = args; - if (rt::thisLocality() != master_l) { - auto This = vector::GetPtr(ID); - rt::dma(This->ptrs_.data(), master_l, master_ptrs, rt::numLocalities()); - } + std::cout << "NICELY DONE!" << std::endl; + + rt::executeOnAll([](const std::tuple &args) { + auto This = vector::GetPtr(std::get<0>(args)); + + This->ptrs_[to_int(std::get<1>(args))] = std::get<2>(args); }, - std::tuple{GetGlobalID(), rt::thisLocality(), ptrs_.data()}); + std::make_tuple(GetGlobalID(), rt::thisLocality(), chunk_.get())); + std::cout << "NICELY DONE!" << std::endl; } private: From fe06ad08a8ca5f2a293410d92fcebcf2d2ac1854 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 23:02:51 -0400 Subject: [PATCH 09/18] fixed vector --- include/shad/core/vector.h | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 5b1f65c3..13ceba22 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -88,6 +88,9 @@ class vector : public AbstractDataStructure> { /// @} public: + + constexpr pointer data() const { return chunk_.get(); } + /// @brief The copy assignment operator. /// /// @param O The right-hand side of the operator. @@ -208,14 +211,14 @@ class vector : public AbstractDataStructure> { /// @return a ::reference to the n-th element in the array. constexpr reference operator[](size_type n) { const std::uint32_t l = locate_index(n); - return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; + return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, ptrs_[l]}; } /// @brief Unchecked element access operator. /// @return a ::const_reference to the n-th element in the array. constexpr const_reference operator[](size_type n) const { const auto l = locate_index(n); - return const_reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, nullptr}; + return const_reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, ptrs_[l]}; } /// @brief Checked element access operator. @@ -376,27 +379,23 @@ class vector : public AbstractDataStructure> { chunk_ = std::unique_ptr{new T[chunk_size()]}; ptrs_.resize(rt::numLocalities()); - - std::cout << "NICELY DONE!" << std::endl; - - rt::executeOnAll([](const std::tuple &args) { - auto This = vector::GetPtr(std::get<0>(args)); - - This->ptrs_[to_int(std::get<1>(args))] = std::get<2>(args); - }, - std::make_tuple(GetGlobalID(), rt::thisLocality(), chunk_.get())); - std::cout << "NICELY DONE!" << std::endl; } private: + static constexpr std::uint32_t to_int(rt::Locality l) { return static_cast(l); } - std::vector ptrs_; std::vector p_; std::unique_ptr chunk_; ObjectID oid_; + std::vector ptrs_; + + public: + void set_ptrs_i(const rt::Locality l, const pointer l_chunk) { + ptrs_[to_int(l)] = l_chunk; + } }; template @@ -895,7 +894,18 @@ class vector { public: /// @brief Constructor. - explicit vector(size_type N = 0) { ptr = vector_t::Create(N); } + explicit vector(size_type N = 0) { + ptr = vector_t::Create(N); + rt::executeOnAll([](const typename vector_t::ObjectID &oid) { + auto This = vector_t::GetPtr(oid); + rt::executeOnAll([](const std::tuple &args) { + auto This = vector_t::GetPtr(std::get<0>(args)); + + This->set_ptrs_i(std::get<1>(args), std::get<2>(args)); + }, + std::make_tuple(This->GetGlobalID(), rt::thisLocality(), This->data())); + }, ptr->GetGlobalID()); + } /// @brief Destructor. ~vector() { vector_t::Destroy(impl()->GetGlobalID()); } From 751b962d9f97264776ed41ec97dcb61e9c0fca31 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 23:10:47 -0400 Subject: [PATCH 10/18] better way to fix it --- include/shad/core/vector.h | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 13ceba22..a7032f8b 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -89,8 +89,6 @@ class vector : public AbstractDataStructure> { public: - constexpr pointer data() const { return chunk_.get(); } - /// @brief The copy assignment operator. /// /// @param O The right-hand side of the operator. @@ -356,6 +354,18 @@ class vector : public AbstractDataStructure> { return result[0]; } + constexpr void fill_ptrs() { + rt::executeOnAll([](const ObjectID &oid) { + auto This = vector::GetPtr(oid); + rt::executeOnAll([](const std::tuple &args) { + auto This = vector::GetPtr(std::get<0>(args)); + + This->ptrs_[to_int(std::get<1>(args))] = std::get<2>(args); + }, + std::make_tuple(This->GetGlobalID(), rt::thisLocality(), This->chunk_.get())); + }, GetGlobalID()); + } + protected: template @@ -393,9 +403,6 @@ class vector : public AbstractDataStructure> { std::vector ptrs_; public: - void set_ptrs_i(const rt::Locality l, const pointer l_chunk) { - ptrs_[to_int(l)] = l_chunk; - } }; template @@ -896,15 +903,7 @@ class vector { /// @brief Constructor. explicit vector(size_type N = 0) { ptr = vector_t::Create(N); - rt::executeOnAll([](const typename vector_t::ObjectID &oid) { - auto This = vector_t::GetPtr(oid); - rt::executeOnAll([](const std::tuple &args) { - auto This = vector_t::GetPtr(std::get<0>(args)); - - This->set_ptrs_i(std::get<1>(args), std::get<2>(args)); - }, - std::make_tuple(This->GetGlobalID(), rt::thisLocality(), This->data())); - }, ptr->GetGlobalID()); + ptr->fill_ptrs(); } /// @brief Destructor. From a286f12e31a955f35cc5b3092c462fa044624f60 Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 23:13:46 -0400 Subject: [PATCH 11/18] remove unnecessary lines --- include/shad/core/vector.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index a7032f8b..0e50d6d9 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -401,8 +401,6 @@ class vector : public AbstractDataStructure> { std::unique_ptr chunk_; ObjectID oid_; std::vector ptrs_; - - public: }; template From 481e887b1ff1770246b3b4a4a64ca64e6da9021d Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Thu, 10 Jun 2021 23:34:50 -0400 Subject: [PATCH 12/18] minor fixes --- include/shad/core/vector.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 0e50d6d9..8b31189c 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -106,6 +106,8 @@ class vector : public AbstractDataStructure> { std::copy(Other->chunk_, Other->chunk_ + chunk_size(), This->chunk_); }, std::make_pair(this->oid_, O.oid_)); + + fill_ptrs(); return *this; } @@ -134,6 +136,7 @@ class vector : public AbstractDataStructure> { std::swap(This->p_, Other->p_); std::swap(This->chunk_, Other->chunk_); + std::swap(This->ptrs_, Other->ptrs_); }, std::make_pair(this->oid_, O.oid_)); } From 9f8ae4c06fdeeb234f3b837f18efe36a3b27e23b Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Fri, 11 Jun 2021 14:28:29 -0400 Subject: [PATCH 13/18] working on comments --- include/shad/core/vector.h | 76 +++++++++++++++++++------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 8b31189c..8503eee1 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -34,17 +34,17 @@ namespace shad { namespace impl { -/// @brief Fixed size distributed array. +/// @brief Distributed vector without resizing capabilities. /// TODO change the descriptions -/// Section 21.3.7.1 of the C++ standard defines the ::array as a fixed-size -/// sequence of objects. An ::array should be a contiguous container (as -/// defined in section 21.2.1). According to that definition, contiguous +/// Section XX.X.X.X of the C++ standard defines the ::vector as a dynamically +/// resizeable sequence of objects. A ::vector should be a contiguous container (as +/// defined in section XX.X.X). According to that definition, contiguous /// containers requires contiguous iterators. The definition of contiguous /// iterators implies contiguous memory allocation for the sequence, and it -/// cannot be guaranteed in many distributed settings. Therefore, ::array +/// cannot be guaranteed in many distributed settings. Therefore, ::vector /// relaxes this requirement. /// -/// @tparam T The type of the elements in the distributed array. +/// @tparam T The type of the elements in the distributed vector. template class vector : public AbstractDataStructure> { template @@ -70,17 +70,17 @@ class vector : public AbstractDataStructure> { using size_type = std::size_t; /// The type used to represent distances. using difference_type = std::ptrdiff_t; - /// The type of references to the element in the array. + /// The type of references to the element in the vector. using reference = VectorRef; - /// The type for const references to element in the array. + /// The type for const references to element in the vector. using const_reference = VectorRef; /// The type for pointer to ::value_type. using pointer = value_type *; /// The type for pointer to ::const_value_type using const_pointer = const value_type *; - /// The type of iterators on the array. + /// The type of iterators on the vector. using iterator = vector_iterator; - /// The type of const iterators on the array. + /// The type of const iterators on the vector. using const_iterator = vector_iterator; // using reverse_iterator = TBD; @@ -111,9 +111,9 @@ class vector : public AbstractDataStructure> { return *this; } - /// @brief Fill the array with an input value. + /// @brief Fill the vector with an input value. /// - /// @param v The input value used to fill the array. + /// @param v The input value used to fill the vector. void fill(const value_type &v) { rt::executeOnAll( [](const std::pair &args) { @@ -125,9 +125,9 @@ class vector : public AbstractDataStructure> { std::make_pair(this->oid_, v)); } - /// @brief Swap the content of two array. + /// @brief Swap the content of two vector. /// - /// @param O The array to swap the content with. + /// @param O The vector to swap the content with. void swap(vector &O) noexcept /* (std::is_nothrow_swappable_v) */ { rt::executeOnAll( [](const std::pair &IDs) { @@ -209,14 +209,14 @@ class vector : public AbstractDataStructure> { /// @{ /// @brief Unchecked element access operator. - /// @return a ::reference to the n-th element in the array. + /// @return a ::reference to the n-th element in the vector. constexpr reference operator[](size_type n) { const std::uint32_t l = locate_index(n); return reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, ptrs_[l]}; } /// @brief Unchecked element access operator. - /// @return a ::const_reference to the n-th element in the array. + /// @return a ::const_reference to the n-th element in the vector. constexpr const_reference operator[](size_type n) const { const auto l = locate_index(n); return const_reference{rt::Locality{l}, difference_type{n - p_[l]}, oid_, ptrs_[l]}; @@ -236,17 +236,17 @@ class vector : public AbstractDataStructure> { return operator[](n); } - /// @brief the first element in the array. + /// @brief the first element in the vector. /// @return a ::reference to the element in position 0. constexpr reference front() { return *begin(); } - /// @brief the first element in the array. + /// @brief the first element in the vector. /// @return a ::const_reference to the element in position 0. constexpr const_reference front() const { return *cbegin(); } - /// @brief the last element in the array. + /// @brief the last element in the vector. /// @return a ::reference to the element in position N - 1. constexpr reference back() { return *(end() - 1); } - /// @brief the last element in the array. + /// @brief the last element in the vector. /// @return a ::const_reference to the element in position N - 1. constexpr const_reference back() const { return *(cend() - 1); } @@ -860,17 +860,17 @@ class alignas(64) vector::vector_iterator { } // namespace impl -/// @brief Fixed size distributed array. +/// @brief Fixed size distributed vector. /// -/// Section 21.3.7.1 of the C++ standard defines the ::array as a fixed-size -/// sequence of objects. An ::array should be a contiguous container (as -/// defined in section 21.2.1). According to that definition, contiguous +/// Section XX.X.X.X of the C++ standard defines the ::vector as a dynamically +/// resizeable sequence of objects. A ::vector should be a contiguous container (as +/// defined in section XX.X.X). According to that definition, contiguous /// containers requires contiguous iterators. The definition of contiguous /// iterators implies contiguous memory allocation for the sequence, and it -/// cannot be guaranteed in many distributed settings. Therefore, ::array +/// cannot be guaranteed in many distributed settings. Therefore, ::vector /// relaxes this requirement. /// -/// @tparam T The type of the elements in the distributed array. +/// @tparam T The type of the elements in the distributed vector. template class vector { using vector_t = impl::vector; @@ -884,17 +884,17 @@ class vector { using size_type = typename vector_t::size_type; /// The type used to represent distances. using difference_type = typename vector_t::difference_type; - /// The type of references to the element in the array. + /// The type of references to the element in the vector. using reference = typename vector_t::reference; - /// The type for const references to element in the array. + /// The type for const references to element in the vector. using const_reference = typename vector_t::const_reference; /// The type for pointer to ::value_type. using pointer = typename vector_t::pointer; /// The type for pointer to ::const_value_type using const_pointer = typename vector_t::const_pointer; - /// The type of iterators on the array. + /// The type of iterators on the vector. using iterator = typename vector_t::iterator; - /// The type of const iterators on the array. + /// The type of const iterators on the vector. using const_iterator = typename vector_t::const_iterator; // todo reverse_iterator // todo const_reverse_iterator @@ -923,36 +923,36 @@ class vector { /// @{ /// @brief Unchecked element access operator. - /// @return a ::reference to the n-th element in the array. + /// @return a ::reference to the n-th element in the vector. constexpr reference operator[](size_type n) { return impl()->operator[](n); } /// @brief Unchecked element access operator. - /// @return a ::const_reference to the n-th element in the array. + /// @return a ::const_reference to the n-th element in the vector. constexpr const_reference operator[](size_type n) const { return impl()->operator[](n); } /// @brief Checked element access operator. - /// @return a ::reference to the n-th element in the array. + /// @return a ::reference to the n-th element in the vector. constexpr reference at(size_type n) { return impl()->at(n); } /// @brief Checked element access operator. - /// @return a ::const_reference to the n-th element in the array. + /// @return a ::const_reference to the n-th element in the vector. constexpr const_reference at(size_type n) const { return impl()->at(n); } - /// @brief the first element in the array. + /// @brief the first element in the vector. /// @return a ::reference to the element in position 0. constexpr reference front() { return impl()->front(); } - /// @brief the first element in the array. + /// @brief the first element in the vector. /// @return a ::const_reference to the element in position 0. constexpr const_reference front() const { return impl()->front(); } - /// @brief the last element in the array. + /// @brief the last element in the vector. /// @return a ::reference to the element in position N - 1. constexpr reference back() { return impl()->back(); } - /// @brief the last element in the array. + /// @brief the last element in the vector. /// @return a ::const_reference to the element in position N - 1. constexpr const_reference back() const { return impl()->back(); } /// @} From c9f7969a122721bc3839ed7321c7b042e52462cd Mon Sep 17 00:00:00 2001 From: Muhammed Fatih Balin Date: Fri, 9 Jul 2021 12:24:01 -0400 Subject: [PATCH 14/18] replacing missing section with C++ standard --- include/shad/core/vector.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/shad/core/vector.h b/include/shad/core/vector.h index 8503eee1..5f9a9c05 100644 --- a/include/shad/core/vector.h +++ b/include/shad/core/vector.h @@ -36,9 +36,9 @@ namespace shad { namespace impl { /// @brief Distributed vector without resizing capabilities. /// TODO change the descriptions -/// Section XX.X.X.X of the C++ standard defines the ::vector as a dynamically +/// The C++ standard defines the ::vector as a dynamically /// resizeable sequence of objects. A ::vector should be a contiguous container (as -/// defined in section XX.X.X). According to that definition, contiguous +/// defined in the C++ standard). According to that definition, contiguous /// containers requires contiguous iterators. The definition of contiguous /// iterators implies contiguous memory allocation for the sequence, and it /// cannot be guaranteed in many distributed settings. Therefore, ::vector From 777a964ba709cd207221e6419e1de65a68d622d0 Mon Sep 17 00:00:00 2001 From: Vito G Castellana Date: Thu, 3 Feb 2022 18:04:34 -0800 Subject: [PATCH 15/18] [#187] additional utils and stub for unit tests --- .../shad/extensions/data_types/data_types.h | 345 ++++++++++++++++-- test/unit_tests/CMakeLists.txt | 1 + .../extensions/data_types/CMakeLists.txt | 14 + .../extensions/data_types/data_types_test.cc | 71 ++++ 4 files changed, 407 insertions(+), 24 deletions(-) create mode 100644 test/unit_tests/extensions/data_types/CMakeLists.txt create mode 100644 test/unit_tests/extensions/data_types/data_types_test.cc diff --git a/include/shad/extensions/data_types/data_types.h b/include/shad/extensions/data_types/data_types.h index 33391148..b459949f 100644 --- a/include/shad/extensions/data_types/data_types.h +++ b/include/shad/extensions/data_types/data_types.h @@ -82,6 +82,16 @@ namespace data_types { /// @return Null encoded value for uint64_t. template <> constexpr uint64_t kNullValue = std::numeric_limits::max(); + + /// @brief Encoded null value for time_t (same as long). + /// @return Null encoded value for time_t (same as long). + template <> + constexpr time_t kNullValue = std::numeric_limits::max(); + + /// @brief Encoded null value for double. + /// @return Null encoded value for double. + template <> + constexpr double kNullValue = std::numeric_limits::max(); /// @brief Encode Function /// Available specializations: @@ -172,8 +182,8 @@ namespace data_types { } // namespace data_types -// METHODS SPECIALIZATION FOR UINT64 ENC_t -template<> +// ENCODE METHODS SPECIALIZATION FOR UINT64 ENC_t +template<> inline uint64_t data_types::encode(std::string &str) { @@ -183,7 +193,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -195,7 +205,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -207,7 +217,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -219,7 +229,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -231,7 +241,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -241,7 +251,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -263,7 +273,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -282,7 +292,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -301,7 +311,7 @@ uint64_t data_types::encode +template<> inline uint64_t data_types::encode(std::string &str) { @@ -320,6 +330,297 @@ uint64_t data_types::encode inline +double data_types::encode(std::string &str) { + double encval; + uint64_t value; + try { value = std::stoull(str); } + catch(...) { return kNullValue; } + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +double data_types::encode(std::string &str) { + double encval; + int64_t value; + try { value = stoll(str); } + catch(...) { return kNullValue; } + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +double data_types::encode(std::string &str) { + double encval; + float value; + try { value = stof(str); } + catch(...) { return kNullValue; } + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +double data_types::encode(std::string &str) { + double value; + try { value = stod(str); } + catch(...) { return kNullValue; } + return value; +} + +template<> inline +double data_types::encode(std::string &str) { + if (str.size() == 0) return kNullValue; + double encval = 1; + if ((str == "F") || (str == "f") || (str == "FALSE") + || (str == "false") || (str == "0")) encval = 0; + return encval; +} + + +template<> inline +double data_types::encode(std::string &str) { + double encval = 0; + memset(&encval, '\0', sizeof(encval)); + memcpy(&encval, str.c_str(), sizeof(encval)-1); + return encval; +} + +template<> inline +double data_types::encode(std::string &str) { + uint64_t val, value = 0; + std::string::iterator start = str.begin(); + for (unsigned i = 0; i < 4; i ++) { + std::string::iterator end = std::find(start, str.end(), '.'); + try { + val = std::stoull(std::string(start, end)); + } catch(...) { + return kNullValue; + } + if (val < 256) { + value = (value << 8) + val; start = end + 1; + } else { + return kNullValue; + } + } + double encval; + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +double data_types::encode(std::string &str) { + double value = 0; + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%Y-%m-%d", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + memcpy(&value, &t, sizeof(value)); + return value; +} + +template<> inline +double data_types::encode(std::string &str) { + double value = 0; + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%m/%d/%y", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + memcpy(&value, &t, sizeof(value)); + return value; +} + +template<> inline +double data_types::encode(std::string &str) { + double value = 0; + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%Y-%m-%dT%H:%M:%S", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + memcpy(&value, &t, sizeof(value)); + return value; +} + +// ENCODE METHODS SPECIALIZATION FOR TIME_T ENC_t (same as long) +template<> inline +time_t data_types::encode(std::string &str) { + time_t value; + try { value = std::stoul(str); } + catch(...) { value = kNullValue; } + return value; +} + +template<> inline +time_t data_types::encode(std::string &str) { + int64_t value; + try { value = stol(str); } + catch(...) { return kNullValue; } + return value; +} + +template<> inline +time_t data_types::encode(std::string &str) { + time_t encval; + float value; + try { value = stof(str); } + catch(...) { return kNullValue; } + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +time_t data_types::encode(std::string &str) { + time_t encval; + double value; + try { value = stod(str); } + catch(...) { return kNullValue; } + memcpy(&encval, &value, sizeof(value)); + return encval; +} + +template<> inline +time_t data_types::encode(std::string &str) { + if (str.size() == 0) return kNullValue; + time_t encval = 1; + if ((str == "F") || (str == "f") || (str == "FALSE") + || (str == "false") || (str == "0")) encval = 0; + return encval; +} + + +template<> inline +time_t data_types::encode(std::string &str) { + time_t encval = 0; + memset(&encval, '\0', sizeof(encval)); + memcpy(&encval, str.c_str(), sizeof(encval)-1); + return encval; +} + +template<> inline +time_t data_types::encode(std::string &str) { + time_t val, value = 0; + std::string::iterator start = str.begin(); + for (unsigned i = 0; i < 4; i ++) { + std::string::iterator end = std::find(start, str.end(), '.'); + try { + val = std::stoull(std::string(start, end)); + } catch(...) { + return kNullValue; + } + if (val < 256) { + value = (value << 8) + val; start = end + 1; + } else { + return kNullValue; + } + } + return value; +} + +template<> inline +time_t data_types::encode(std::string &str) { + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%Y-%m-%d", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + return t; +} + +template<> inline +time_t data_types::encode(std::string &str) { + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%m/%d/%y", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + return t; +} + +template<> inline +time_t data_types::encode(std::string &str) { + struct tm date{}; + date.tm_isdst = -1; + strptime(str.c_str(), "%Y-%m-%dT%H:%M:%S", &date); + time_t t; + try { + t = mktime(&date); + } + catch(...) { + return kNullValue; + } + return t; +} + template ENC_t data_types::encode(IN_t &in, data_types::data_t dt) { switch (dt) { @@ -349,7 +650,7 @@ ENC_t data_types::encode(IN_t &in, data_types::data_t dt) { return data_types::kNullValue; } -template<> +template<> inline std::string data_types::decode(uint64_t value) { @@ -357,7 +658,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -367,7 +668,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -377,7 +678,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -387,7 +688,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -398,7 +699,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -406,7 +707,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -416,7 +717,7 @@ std::string data_types::decode +template<> inline std::string data_types::decode(uint64_t value) { @@ -424,14 +725,10 @@ std::string data_types::decode +template <> inline uint64_t data_types::decode(uint64_t encvalue) { return encvalue; } - - - - } // namespace shad #endif // INCLUDE_SHAD_EXTENSIONS_DATA_TYPES_DATA_TYPES_H_ \ No newline at end of file diff --git a/test/unit_tests/CMakeLists.txt b/test/unit_tests/CMakeLists.txt index 0765bc83..e5ba6633 100644 --- a/test/unit_tests/CMakeLists.txt +++ b/test/unit_tests/CMakeLists.txt @@ -23,3 +23,4 @@ add_subdirectory(runtime) add_subdirectory(data_structures) add_subdirectory(core) add_subdirectory(extensions/graph_library) +add_subdirectory(extensions/data_types) diff --git a/test/unit_tests/extensions/data_types/CMakeLists.txt b/test/unit_tests/extensions/data_types/CMakeLists.txt new file mode 100644 index 00000000..56968fe0 --- /dev/null +++ b/test/unit_tests/extensions/data_types/CMakeLists.txt @@ -0,0 +1,14 @@ +set(tests + data_types_test +) + +foreach(t ${tests}) + add_executable(${t} ${t}.cc) + if (CLANG_TIDY_EXE) + set_target_properties( + ${t} PROPERTIES + CXX_CLANG_TIDY "${DO_CLANG_TIDY}") + endif() + target_link_libraries(${t} ${SHAD_RUNTIME_LIB} runtime shadtest_main) + add_test(NAME ${t} COMMAND ${SHAD_TEST_COMMAND} $) +endforeach(t) diff --git a/test/unit_tests/extensions/data_types/data_types_test.cc b/test/unit_tests/extensions/data_types/data_types_test.cc new file mode 100644 index 00000000..e0275aa6 --- /dev/null +++ b/test/unit_tests/extensions/data_types/data_types_test.cc @@ -0,0 +1,71 @@ +//===------------------------------------------------------------*- C++ -*-===// +// +// SHAD +// +// The Scalable High-performance Algorithms and Data Structure Library +// +//===----------------------------------------------------------------------===// +// +// Copyright 2018 Battelle Memorial Institute +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy +// of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. +// +//===----------------------------------------------------------------------===// + +#include + +#include "gtest/gtest.h" + +#include "shad/extensions/data_types/data_types.h" + +static const unsigned int kUintValue = 4242; +static const int kIntValue = -4242; +static const float kFloatValue = 42.42; +static const double kDoubleValue = 42.4242; + +static std::string kUintString = "4242"; +static std::string kIntString = "-4242"; +static std::string kFloatString = "42.42"; +static std::string kDoubleString = "42.4242"; +static std::string kString = "Mamba_24"; + + +class DataTypesTest : public ::testing::Test { + public: + DataTypesTest() {} + void SetUp() {} + void TearDown() {} +}; + +TEST_F(DataTypesTest, UintEncodeDecodeTest) { + using ENC_t = uint64_t; + using IN_t = std::string; + using DEC_t = std::string; + + auto encUint = shad::data_types::encode(kUintString); + auto encInt = shad::data_types::encode(kIntString); + auto encFloat = shad::data_types::encode(kFloatString); + auto encDouble = shad::data_types::encode(kDoubleString); + auto encChars = shad::data_types::encode(kString); + + auto decUint = shad::data_types::decode(encUint); + auto decInt = shad::data_types::decode(encInt); + auto decFloat = shad::data_types::decode(encFloat); + auto decDouble = shad::data_types::decode(encDouble); + ASSERT_EQ(decUint, kUintString); +} \ No newline at end of file From 4cc1cec0b10fd383877c87d1edf3e658e5d10479 Mon Sep 17 00:00:00 2001 From: Vito G Castellana Date: Thu, 3 Feb 2022 20:55:54 -0800 Subject: [PATCH 16/18] [#206] Implemented AsyncGetElements with DMA --- include/shad/data_structures/array.h | 52 ++++++++++++++++++- test/unit_tests/data_structures/array_test.cc | 33 +++++++++++- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/include/shad/data_structures/array.h b/include/shad/data_structures/array.h index f17fab50..169ecd93 100644 --- a/include/shad/data_structures/array.h +++ b/include/shad/data_structures/array.h @@ -458,6 +458,21 @@ class Array : public AbstractDataStructure> { data_[std::get<0>(entry)] = std::get<1>(entry); } + constexpr void FillPtrs() { + rt::executeOnAll([](const ObjectID &oid) { + auto This = Array::GetPtr(oid); + rt::executeOnAll([](const std::tuple &args) { + auto This = Array::GetPtr(std::get<0>(args)); + + This->ptrs_[(uint32_t)std::get<1>(args)] = std::get<2>(args); + }, + std::make_tuple(This->GetGlobalID(), rt::thisLocality(), This->data_.data())); + }, GetGlobalID()); + } + + void AsyncGetElements(rt::Handle& h, T* local_data, + const uint64_t idx, const uint64_t num_el); + protected: Array(ObjectID oid, size_t size, const T &initValue) : oid_(oid), @@ -467,7 +482,8 @@ class Array : public AbstractDataStructure> { : rt::numLocalities() - (size % rt::numLocalities())), data_(), dataDistribution_(), - buffers_(oid) { + buffers_(oid), + ptrs_(rt::numLocalities()) { rt::Locality pivot(pivot_); size_t start = 0; size_t chunkSize = size / rt::numLocalities(); @@ -498,6 +514,7 @@ class Array : public AbstractDataStructure> { std::vector data_; std::vector> dataDistribution_; BuffersVector buffers_; + std::vector ptrs_; struct InsertAtArgs { ObjectID oid; @@ -1073,6 +1090,39 @@ void Array::ForEach(ApplyFunT &&function, Args &... args) { rt::executeOnAll(feLambda, arguments); } +template +void Array::AsyncGetElements(rt::Handle& h, T* local_data, + const uint64_t idx, const uint64_t num_el) { + + size_t tgtPos = 0, firstPos = idx; + rt::Locality tgtLoc; + size_t remainingValues = num_el; + size_t chunkSize = 0; + T* tgtAddress; + + while (remainingValues > 0) { + if (firstPos < pivot_ * (size_ / rt::numLocalities())) { + tgtLoc = rt::Locality(firstPos / (size_ / rt::numLocalities())); + tgtPos = firstPos % (size_ / rt::numLocalities()); + chunkSize = + std::min((size_ / rt::numLocalities() - tgtPos), remainingValues); + } else { + size_t newPos = firstPos - (pivot_ * (size_ / rt::numLocalities())); + tgtLoc = + rt::Locality(pivot_ + newPos / ((size_ / rt::numLocalities() + 1))); + tgtPos = newPos % ((size_ / rt::numLocalities() + 1)); + chunkSize = + std::min((size_ / rt::numLocalities() + 1 - tgtPos), remainingValues); + } + + tgtAddress = ptrs_[(uint32_t)tgtLoc] + tgtPos; + rt::asyncDma(h, local_data, tgtLoc, tgtAddress, chunkSize); + local_data += chunkSize; + firstPos += chunkSize; + remainingValues -= chunkSize; + } +} + } // namespace shad #endif // INCLUDE_SHAD_DATA_STRUCTURES_ARRAY_H_ diff --git a/test/unit_tests/data_structures/array_test.cc b/test/unit_tests/data_structures/array_test.cc index 3fe491e6..d5fa6efb 100644 --- a/test/unit_tests/data_structures/array_test.cc +++ b/test/unit_tests/data_structures/array_test.cc @@ -122,7 +122,6 @@ TEST_F(ArrayTest, RangedAsyncInsertAndAsyncGet) { shad::rt::Handle handle; edsPtr->AsyncInsertAt(handle, 0, inputData_.data(), kArraySize); shad::rt::waitForCompletion(handle); - ; shad::rt::Handle handle2; for (size_t i = 0; i < kArraySize; i++) { @@ -136,6 +135,36 @@ TEST_F(ArrayTest, RangedAsyncInsertAndAsyncGet) { shad::Array::Destroy(edsPtr->GetGlobalID()); } +TEST_F(ArrayTest, RangedAsyncInsertAndAsyncGetElements) { + std::vector values(kArraySize); + + auto edsPtr = shad::Array::Create(kArraySize, kInitValue); + edsPtr->FillPtrs(); + shad::rt::Handle handle; + edsPtr->AsyncInsertAt(handle, 0, inputData_.data(), kArraySize); + shad::rt::waitForCompletion(handle); + + shad::rt::Handle handle2; + edsPtr->AsyncGetElements(handle2, values.data(), 0, kArraySize); + shad::rt::waitForCompletion(handle2); + + for (size_t i = 0; i < kArraySize; i++) { + ASSERT_EQ(values[i], i + 1); + } + + uint64_t to_insert2 = kArraySize/2; + uint64_t idx2 = kArraySize/6; + std::vector values2(to_insert2); + + edsPtr->AsyncGetElements(handle2, values2.data(), idx2, to_insert2); + shad::rt::waitForCompletion(handle2); + + for (size_t i = 0; i < to_insert2; i++) { + ASSERT_EQ(values2[i], i + idx2 + 1); + } + shad::Array::Destroy(edsPtr->GetGlobalID()); +} + TEST_F(ArrayTest, BufferedSyncInsertAndSyncGet) { auto edsPtr = shad::Array::Create(kArraySize, kInitValue); for (size_t i = 0; i < kArraySize; i++) { @@ -367,4 +396,4 @@ TEST_F(ArrayTest, AsyncInsertAsyncForEachAndAsyncGet) { ASSERT_EQ(values[i], i + 1 + (2 * kInitValue)); } shad::Array::Destroy(edsPtr->GetGlobalID()); -} +} \ No newline at end of file From c77ac933c485ce70605065617bd327c8adc39165 Mon Sep 17 00:00:00 2001 From: Vito G Castellana Date: Fri, 4 Feb 2022 19:06:19 -0800 Subject: [PATCH 17/18] [#206] Implemented AsyncApplyWithRetBuff in Array --- include/shad/data_structures/array.h | 48 +++++++++++++++++++ test/unit_tests/data_structures/array_test.cc | 41 ++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/include/shad/data_structures/array.h b/include/shad/data_structures/array.h index 169ecd93..a7c8082f 100644 --- a/include/shad/data_structures/array.h +++ b/include/shad/data_structures/array.h @@ -319,6 +319,11 @@ class Array : public AbstractDataStructure> { template void AsyncApply(rt::Handle &handle, const size_t pos, ApplyFunT &&function, Args &... args); + + template + void AsyncApplyWithRetBuff(rt::Handle &handle, const size_t pos, + ApplyFunT &&function, uint8_t* result, + uint32_t* resultSize, Args &... args); /// @brief Applies a user-defined function to every element /// in the specified range. @@ -617,6 +622,30 @@ class Array : public AbstractDataStructure> { std::get<4>(tuple), std::make_index_sequence{}); } + template + static void AsyncCallApplyWRBFun(rt::Handle &handle, ObjectID &oid, size_t pos, + size_t loffset, ApplyFunT function, + std::tuple &args, std::index_sequence, + uint8_t* result, uint32_t* resultSize) { + // Get a local instance on the remote node. + auto arrayPtr = Array::GetPtr(oid); + T &element = arrayPtr->data_[loffset]; + function(handle, pos, element, std::get(args)..., result, resultSize); + } + + template + static void AsyncApplyWRBFunWrapper(rt::Handle &handle, const Tuple &args, + uint8_t* result, uint32_t* resultSize) { + constexpr auto Size = std::tuple_size< + typename std::decay(args))>::type>::value; + + Tuple &tuple = const_cast(args); + + AsyncCallApplyWRBFun(handle, std::get<0>(tuple), std::get<1>(tuple), + std::get<2>(tuple), std::get<3>(tuple), + std::get<4>(tuple), std::make_index_sequence{}, result, resultSize); + } + template static void CallForEachInRangeFun(size_t i, ObjectID &oid, size_t pos, size_t lpos, ApplyFunT function, @@ -954,6 +983,25 @@ void Array::AsyncApply(rt::Handle &handle, const size_t pos, AsyncApplyFunWrapper, argsTuple); } +template +template +void Array::AsyncApplyWithRetBuff(rt::Handle &handle, const size_t pos, + ApplyFunT &&function, uint8_t* result, + uint32_t* resultSize, Args &... args) { + auto target = getTargetLocalityFromTargePosition(dataDistribution_, pos); + + using FunctionTy = void (*)(rt::Handle &, size_t, T &, Args & ..., uint8_t*, uint32_t*); + FunctionTy fn = std::forward(function); + using ArgsTuple = + std::tuple>; + ArgsTuple argsTuple{oid_, pos, target.second, fn, + std::tuple(args...)}; + + rt::asyncExecuteAtWithRetBuff(handle, target.first, + AsyncApplyWRBFunWrapper, argsTuple, + result, resultSize); +} + template template void Array::ForEachInRange(const size_t first, const size_t last, diff --git a/test/unit_tests/data_structures/array_test.cc b/test/unit_tests/data_structures/array_test.cc index d5fa6efb..3627c4bb 100644 --- a/test/unit_tests/data_structures/array_test.cc +++ b/test/unit_tests/data_structures/array_test.cc @@ -247,6 +247,15 @@ static void asyncApplyFun(shad::rt::Handle & /*unused*/, size_t i, size_t &elem, elem += kInitValue; } +static void asyncApplyWRBFun(shad::rt::Handle & /*unused*/, size_t i, size_t &elem, size_t &incr, + uint8_t* result, uint32_t* resultSize) { + ASSERT_EQ(incr, kInitValue); + ASSERT_EQ(elem, i + 1); + elem += kInitValue; + *resultSize = sizeof(elem); + memcpy(result, &elem, sizeof(elem)); +} + static void asyncApplyFunNoArgs(shad::rt::Handle & /*unused*/, size_t i, size_t &elem) { ASSERT_EQ(elem, i + kInitValue + 1); @@ -298,6 +307,38 @@ TEST_F(ArrayTest, AsyncInsertAsyncApplyAndAsyncGet) { shad::Array::Destroy(edsPtr->GetGlobalID()); } +TEST_F(ArrayTest, AsyncInsertAsyncApplyWRBAndAsyncGet) { + std::vector values(kArraySize); + auto edsPtr = shad::Array::Create(kArraySize, kInitValue); + + shad::rt::Handle handle; + for (size_t i = 0; i < kArraySize; i++) { + edsPtr->AsyncInsertAt(handle, i, i + 1); + } + shad::rt::waitForCompletion(handle); + + shad::rt::Handle handle2; + std::vector ret_values(kArraySize); + std::vector ret_sizes(kArraySize); + for (size_t i = 0; i < kArraySize; i++) { + edsPtr->AsyncApplyWithRetBuff(handle2, i, asyncApplyWRBFun, + (uint8_t*)(&ret_values[i]), + &ret_sizes[i], kInitValue); + } + shad::rt::waitForCompletion(handle2); + + shad::rt::Handle handle3; + for (size_t i = 0; i < kArraySize; i++) { + edsPtr->AsyncAt(handle3, i, &values[i]); + } + shad::rt::waitForCompletion(handle3); + for (size_t i = 0; i < kArraySize; i++) { + ASSERT_EQ(values[i], i + 1 + kInitValue); + ASSERT_EQ(ret_values[i], i + 1 + kInitValue); + } + shad::Array::Destroy(edsPtr->GetGlobalID()); +} + TEST_F(ArrayTest, AsyncInsertSyncForEachInRangeAndAsyncGet) { std::vector values(kArraySize); auto edsPtr = shad::Array::Create(kArraySize, kInitValue); From 8a2f031cf1a78664035d1bf88b13ac3782fde962 Mon Sep 17 00:00:00 2001 From: Vito G Castellana Date: Mon, 7 Feb 2022 18:02:08 -0800 Subject: [PATCH 18/18] [#187] additional tests for datatype utils --- include/shad/extensions/data_types/data_types.h | 2 +- .../extensions/data_types/data_types_test.cc | 15 ++++++++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/include/shad/extensions/data_types/data_types.h b/include/shad/extensions/data_types/data_types.h index b459949f..8b98177e 100644 --- a/include/shad/extensions/data_types/data_types.h +++ b/include/shad/extensions/data_types/data_types.h @@ -731,4 +731,4 @@ uint64_t data_types::decode(uint64_t encvalue) { } } // namespace shad -#endif // INCLUDE_SHAD_EXTENSIONS_DATA_TYPES_DATA_TYPES_H_ \ No newline at end of file +#endif // INCLUDE_SHAD_EXTENSIONS_DATA_TYPES_DATA_TYPES_H_ diff --git a/test/unit_tests/extensions/data_types/data_types_test.cc b/test/unit_tests/extensions/data_types/data_types_test.cc index e0275aa6..e8c22e4d 100644 --- a/test/unit_tests/extensions/data_types/data_types_test.cc +++ b/test/unit_tests/extensions/data_types/data_types_test.cc @@ -68,4 +68,17 @@ TEST_F(DataTypesTest, UintEncodeDecodeTest) { auto decFloat = shad::data_types::decode(encFloat); auto decDouble = shad::data_types::decode(encDouble); ASSERT_EQ(decUint, kUintString); -} \ No newline at end of file + ASSERT_EQ(decInt, kIntString); + // FIXME + // Decode to string of floats and double may use different precision from input string + // ASSERT_EQ(decFloat, kFloatString); + // ASSERT_EQ(decDouble, kDoubleString); + auto defUint = shad::data_types::decode(encUint); + auto defInt = shad::data_types::decode(encInt); + auto defFloat = shad::data_types::decode(encFloat); + auto defDouble = shad::data_types::decode(encDouble); + ASSERT_EQ(defUint, kUintValue); + ASSERT_EQ(defInt, kIntValue); + ASSERT_EQ(defFloat, kFloatValue); + ASSERT_EQ(defDouble, kDoubleValue); +}