From a4574ef34486a3e906951be0c0042ecc225f25b4 Mon Sep 17 00:00:00 2001 From: Aleksandr Lyapunov Date: Thu, 11 Jul 2024 18:05:49 +0300 Subject: [PATCH] mpp: implement container adapter Previously encode/decode expected the first argument (cont) to be tnt::Buffer of something with the same API. This commit introduces contaier adapter, so it is allowed now to encode to standard contiguous container (std::vector etc) and encode/decode just by pointer to data. Fix decode headers while we are here. --- src/mpp/ContAdapter.hpp | 273 +++++++++++++++++++++++++++++++++++++++ src/mpp/Dec.hpp | 33 +++-- src/mpp/Enc.hpp | 21 ++- test/EncDecGPerfTest.cpp | 14 +- test/EncDecTest.cpp | 34 +++++ 5 files changed, 343 insertions(+), 32 deletions(-) create mode 100644 src/mpp/ContAdapter.hpp diff --git a/src/mpp/ContAdapter.hpp b/src/mpp/ContAdapter.hpp new file mode 100644 index 000000000..c869058de --- /dev/null +++ b/src/mpp/ContAdapter.hpp @@ -0,0 +1,273 @@ +#pragma once +/* + * Copyright 2010-2024 Tarantool AUTHORS: please see AUTHORS file. + * + * Redistribution and use in source and binary forms, with or + * without modification, are permitted provided that the following + * conditions are met: + * + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the + * following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials + * provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY AUTHORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL + * AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR + * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + * THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include +#include +#include + +#include "../Utils/CStr.hpp" +#include "../Utils/Traits.hpp" + +namespace mpp { + +namespace encode_details { + +/** Common data+size pair that is used for writing of variable-length data. */ +struct WData { + const char *data; + size_t size; +}; + +/** Random struct; used to check whether container has template write method. */ +struct TestWriteStruct { + uint32_t a; + uint16_t b; +}; + +/** Test that container if Buffer-like: has several needed write methods. */ +template +struct is_write_callable_h : std::false_type {}; + +template +struct is_write_callable_h().write(uint8_t{})), + decltype(std::declval().write(uint64_t{})), + decltype(std::declval().write(TestWriteStruct{})), + decltype(std::declval().write({(const char *)0, 1})), + decltype(std::declval().write(tnt::CStr<'a', 'b'>{})) + >> : std::true_type {}; + +template +constexpr bool is_write_callable_v = is_write_callable_h::value; + +template +class BufferWriter { +public: + explicit BufferWriter(CONT& cont_) : cont{cont_} {} + void write(WData data) { cont.write({data.data, data.size}); } + template + void write(tnt::CStr str) + { + cont.write(std::move(str)); + } + template + void write(T&& t) + { + cont.write(std::forward(t)); + } + +private: + CONT& cont; +}; + +template +class StdContWriter { +public: + static_assert(sizeof(*std::declval().data()) == 1); + explicit StdContWriter(CONT& cont_) : cont{cont_} {} + void write(WData data) + { + size_t old_size = std::size(cont); + cont.resize(old_size + data.size); + std::memcpy(std::data(cont) + old_size, data.data, data.size); + } + template + void write(tnt::CStr data) + { + size_t old_size = std::size(cont); + cont.resize(old_size + data.size); + std::memcpy(std::data(cont) + old_size, data.data, data.size); + } + + template + void write(T&& t) + { + static_assert(std::is_standard_layout_v>); + size_t old_size = std::size(cont); + cont.resize(old_size + sizeof(T)); + std::memcpy(std::data(cont) + old_size, &t, sizeof(T)); + } + +private: + CONT& cont; +}; + +template +class PtrWriter { +public: + static_assert(sizeof(C) == 1); + static_assert(!std::is_const_v); + explicit PtrWriter(C *& ptr_) : ptr{ptr_} {} + void write(WData data) + { + std::memcpy(ptr, data.data, data.size); + ptr += data.size; + } + template + void write(tnt::CStr data) + { + std::memcpy(ptr, data.data, data.size); + ptr += data.size; + } + + template + void write(T&& t) + { + static_assert(std::is_standard_layout_v>); + std::memcpy(ptr, &t, sizeof(t)); + ptr += sizeof(t); + } + +private: + C *& ptr; +}; + +template +auto +wr(CONT& cont) +{ + if constexpr (is_write_callable_v) + return BufferWriter{cont}; + else if constexpr (tnt::is_resizable_v && tnt::is_contiguous_v) + return StdContWriter{cont}; + else if constexpr (std::is_pointer_v) + return PtrWriter>{cont}; + else + static_assert(tnt::always_false_v); +} + +} // namespace encode_details + +namespace decode_details { + +/** Common data+size pair that is used for reading of variable-length data. */ +struct RData { + char *data; + size_t size; +}; + +/** Struct that when used as read argument, means skipping of 'size' data. */ +struct Skip { + size_t size; +}; + +/** Random struct; used to check whether container has template read method. */ +struct TestReadStruct { + uint8_t a; + uint64_t b; +}; + +/** Test that container if Buffer-like: has several needed read methods. */ +template +struct is_read_callable_h : std::false_type {}; + +template +struct is_read_callable_h().read(*(uint8_t*)0)), + decltype(std::declval().read(*(uint64_t*)0)), + decltype(std::declval().read(*(TestReadStruct*)0)), + decltype(std::declval().read({(char *)0, 1})), + decltype(std::declval().template get()), + decltype(std::declval().read({1})) + >> : std::true_type {}; + +template +constexpr bool is_read_callable_v = is_read_callable_h::value; + +template +class BufferReader { +public: + explicit BufferReader(CONT& cont_) : cont{cont_} {} + void read(RData data) { cont.read({data.data, data.size}); } + void read(Skip data) { cont.read({data.size}); } + template + void read(T&& t) + { + cont.read(std::forward(t)); + } + template + T get() + { + return cont.template get(); + } + +private: + CONT& cont; +}; + +template +class PtrReader { +public: + static_assert(sizeof(C) == 1); + explicit PtrReader(C *& ptr_) : ptr{ptr_} {} + void read(RData data) + { + std::memcpy(data.data, ptr, data.size); + ptr += data.size; + } + void read(Skip data) { ptr += data.size; } + + template + void read(T& t) + { + static_assert(std::is_standard_layout_v>); + std::memcpy(&t, ptr, sizeof(t)); + ptr += sizeof(t); + } + template + T get() + { + static_assert(std::is_standard_layout_v>); + T t; + std::memcpy(&t, ptr, sizeof(t)); + return t; + } + +private: + C *& ptr; +}; + +template +auto +rd(CONT& cont) +{ + if constexpr (is_read_callable_v) + return BufferReader{cont}; + else if constexpr (std::is_pointer_v) + return PtrReader>{cont}; + else + static_assert(tnt::always_false_v); +} + +} // namespace decode_details + +} // namespace mpp diff --git a/src/mpp/Dec.hpp b/src/mpp/Dec.hpp index 9bc4ecc94..fd0074a96 100644 --- a/src/mpp/Dec.hpp +++ b/src/mpp/Dec.hpp @@ -33,12 +33,19 @@ #include #include #include +#include +#include #include +#include +#include "BSwap.hpp" #include "ClassRule.hpp" +#include "ContAdapter.hpp" #include "Constants.hpp" #include "Rules.hpp" #include "Spec.hpp" +#include "../Utils/CStr.hpp" +#include "../Utils/Traits.hpp" namespace mpp { @@ -60,7 +67,7 @@ constexpr bool is_any_putable_v = /** * If it is true, the object of type T will not be decoded - raw data will * be saved to it. - * + * * Now it supports only a pair of iterators (probably, wrapped with * mpp::as_raw). The check implicilty implies that BUF is an iterator, not * buffer - it would be strange to pass a pair of buffer to decoder. @@ -411,7 +418,7 @@ auto read_value(BUF& buf) using RULE = rule_by_family_t; if constexpr (SUBRULE == SIMPLEX_SUBRULE) { typename RULE::simplex_value_t tag; - buf.read(tag); + rd(buf).read(tag); assert(tag >= rule_simplex_tag_range_v.first); assert(tag <= rule_simplex_tag_range_v.last); [[maybe_unused]] typename RULE::simplex_value_t val = @@ -427,12 +434,12 @@ auto read_value(BUF& buf) return val; } else { uint8_t tag; - buf.read(tag); + rd(buf).read(tag); assert(tag == RULE::complex_tag + SUBRULE); using TYPES = typename RULE::complex_types; using V = std::tuple_element_t; under_uint_t u; - buf.read(u); + rd(buf).read(u); V val = bswap(u); return val; } @@ -445,7 +452,7 @@ auto read_item(BUF& buf, ITEM& item) auto val = read_value(buf); if constexpr (RULE::has_ext) { int8_t ext_type; - buf.read(ext_type); + rd(buf).read(ext_type); item.ext_type = ext_type; } if constexpr (RULE::has_data) { @@ -468,11 +475,11 @@ auto read_item(BUF& buf, ITEM& item) if (size > std::size(item)) size = std::size(item); } - buf.read({std::data(item), size}); + rd(buf).read({std::data(item), size}); if constexpr (tnt::is_limited_v || !tnt::is_resizable_v) { if (size < size_t(val)) - buf.read({size_t(val) - size}); + rd(buf).read({size_t(val) - size}); } } else if constexpr (RULE::has_children) { if constexpr (tnt::is_clearable_v) @@ -743,7 +750,7 @@ decode_jump(BUF& buf, T... t) { static_assert(path_item_type(PATH::last()) != PIT_BAD); static constexpr auto jumps = JumpsBuilder::build(); - uint8_t tag = buf.template get(); + uint8_t tag = rd(buf).template get(); return jumps.data[tag](buf, t...); } @@ -922,10 +929,10 @@ bool jump_skip(BUF& buf, T... t) if constexpr (RULE::has_ext) { int8_t ext_type; - buf.read(ext_type); + rd(buf).read(ext_type); } if constexpr (RULE::has_data) { - buf.read({size_t(val)}); + rd(buf).read({size_t(val)}); } if constexpr (RULE::has_children) { auto& arg = std::get(std::tie(t...)); @@ -1116,7 +1123,7 @@ bool jump_find_key([[maybe_unused]] K k, tnt::iseq<>, BUF& buf, T... t) static_assert(path_item_type(PATH::last()) == PIT_DYN_KEY); using NEXT_PATH = path_push_t; if constexpr (FAMILY == MP_STR) - buf.read({k}); + rd(buf).read({k}); return decode_impl(buf, t..., size_t(1)); } @@ -1143,7 +1150,7 @@ bool jump_find_key(K k, tnt::iseq, BUF& buf, T... t) if (compare_key(k, key, buf)) { if constexpr (FAMILY == MP_STR) - buf.read({k}); + rd(buf).read({k}); return decode_impl(buf, t...); } @@ -1274,7 +1281,7 @@ bool broken_msgpack_jump(BUF&, T...) template bool -decode(BUF& buf, T&&... t) +decode(BUF&& buf, T&&... t) { // TODO: Guard bool res = decode_details::decode(buf, std::forward(t)...); diff --git a/src/mpp/Enc.hpp b/src/mpp/Enc.hpp index d5ea2623b..846dc3330 100644 --- a/src/mpp/Enc.hpp +++ b/src/mpp/Enc.hpp @@ -38,6 +38,7 @@ #include "BSwap.hpp" #include "ClassRule.hpp" +#include "ContAdapter.hpp" #include "Constants.hpp" #include "Rules.hpp" #include "Spec.hpp" @@ -50,10 +51,6 @@ //TODO : error handling + names //TODO : min + max (detect max for string literal and static capacity) //TODO : avoid reinterpret_cast -//TODO : add raw(N) -//TODO : add std::variant -//TODO : add std::optional -//TODO : universal buffer namespace mpp { @@ -503,7 +500,7 @@ bool encode(CONT &cont, tnt::CStr prefix, tnt::iseq) { static_assert(sizeof...(I) == 0); - cont.write(prefix); + wr(cont).write(prefix); return true; } @@ -642,13 +639,13 @@ encode(CONT &cont, tnt::CStr prefix, } else if constexpr(tnt::is_string_constant_v) { return encode(cont, prefix.join(u), ais, more...); } else if constexpr(tnt::is_contiguous_v) { - cont.write(prefix); - cont.write({std::data(u), std::size(u)}); + wr(cont).write(prefix); + wr(cont).write({std::data(u), std::size(u)}); return encode(cont, tnt::CStr<>{}, ais, more...); } else { static_assert(std::is_standard_layout_v); - cont.write(prefix); - cont.write(u); + wr(cont).write(prefix); + wr(cont).write(u); return encode(cont, tnt::CStr<>{}, ais, more...); } } else { @@ -677,8 +674,8 @@ encode(CONT &cont, tnt::CStr prefix, size_t soff = find_simplex_offset(value); if (soff < rule_t::simplex_value_range.count) { uint8_t tag = rule_t::simplex_tag + soff; - cont.write(prefix); - cont.write(tag); + wr(cont).write(prefix); + wr(cont).write(tag); return encode(cont, tnt::CStr<>{}, is, as_raw(ext), as_raw(data), more...); @@ -706,7 +703,7 @@ encode(CONT &cont, tnt::CStr prefix, template bool -encode(CONT &cont, const T&... t) +encode(CONT &&cont, const T&... t) { // TODO: Guard tnt::iseq<> is; diff --git a/test/EncDecGPerfTest.cpp b/test/EncDecGPerfTest.cpp index 09c0db1e4..257d5120c 100644 --- a/test/EncDecGPerfTest.cpp +++ b/test/EncDecGPerfTest.cpp @@ -103,6 +103,10 @@ struct SimpleRData { size_t size; }; +struct SimpleSkip { + size_t size; +}; + struct SimpleReader { const char *pos; @@ -111,19 +115,15 @@ struct SimpleReader { memcpy(data.data, pos, data.size); pos += data.size; } - template - void read(T &t) + void read(SimpleSkip data) { - memcpy(&t, pos, sizeof(t)); - pos += sizeof(t); + pos += data.size; } template - T read() + void read(T &t) { - T t; memcpy(&t, pos, sizeof(t)); pos += sizeof(t); - return t; } template T get() diff --git a/test/EncDecTest.cpp b/test/EncDecTest.cpp index c76976926..8c8ad2e60 100644 --- a/test/EncDecTest.cpp +++ b/test/EncDecTest.cpp @@ -1323,6 +1323,39 @@ test_variant() fail_unless(monostate_wr == monostate_rd); } +void +test_cont_adapter() +{ + { + std::vector vec; + mpp::encode(vec, 10, "abc", std::forward_as_tuple(false, 1.)); + int res1 = 0; + std::string res2; + bool res3 = true; + double res4 = 2.; + mpp::decode(vec.data(), res1, res2, std::tie(res3, res4)); + fail_unless(res1 == 10); + fail_unless(res2 == "abc"); + fail_unless(res3 == false); + fail_unless(res4 == 1.); + } + { + char buf[16]; + char *p = buf; + mpp::encode(p, 10, "abc", std::forward_as_tuple(false, 1.)); + int res1 = 0; + std::string res2; + bool res3 = true; + double res4 = 2.; + p = buf; + mpp::decode(p, res1, res2, std::tie(res3, res4)); + fail_unless(res1 == 10); + fail_unless(res2 == "abc"); + fail_unless(res3 == false); + fail_unless(res4 == 1.); + } +} + int main() { test_under_ints(); @@ -1334,4 +1367,5 @@ int main() test_optional(); test_raw(); test_variant(); + test_cont_adapter(); }