From 967fce0a98d0aab72e23fafd5ed4612d0e48c182 Mon Sep 17 00:00:00 2001 From: Alex Shabalin Date: Wed, 7 Aug 2024 18:16:00 +0200 Subject: [PATCH] Rephrase the pool explanation. More implementation docs. --- immer/extra/persist/README.rst | 16 +++++-- immer/extra/persist/json/json_immer.hpp | 22 +++++++++ immer/extra/persist/json/names.hpp | 6 +++ immer/extra/persist/json/persistable.hpp | 58 ++++++++++-------------- immer/extra/persist/json/policy.hpp | 8 ++-- immer/extra/persist/json/pools.hpp | 2 +- 6 files changed, 69 insertions(+), 43 deletions(-) diff --git a/immer/extra/persist/README.rst b/immer/extra/persist/README.rst index 1d17192f..bb87719c 100644 --- a/immer/extra/persist/README.rst +++ b/immer/extra/persist/README.rst @@ -79,10 +79,18 @@ Which generates some JSON like this: As you can see, the value is serialized with every ``immer`` container replaced by an identifier. This identifier is a key into a pool, which is serialized just after. -A pool is a data structure that ``immer-persist`` uses to work with the underlying structure of the ``immer`` containers. -For example, ``vector`` and ``flex_vector`` use Radix Balanced Trees internally that have inner nodes and leaf nodes. These -nodes can be seen in the JSON above. In other words, containers can be put into a pool and later can be retrieved from it, all while preserving -the structural sharing. One pool can only work with one container type. +A pool represents a *set* of ``immer`` containers of a given type. For example, we may have a pool that contains all +``immer::vector`` of our document. You can think of it as a little database of ``immer`` containers. When +serializing the pool, the internal structure of all those ``immer`` containers is written, preserving the structural +sharing between those containers. The nodes of the trees that implement the ``immer`` containers are represented +directly in the JSON and, because we are representing all the containers as a whole, those nodes that are referenced in +multiple trees can be stored only once. That same structure is preserved when reading the pool back from disk and +reconstructing the vectors (and other containers) from it, thus allowing us to preserve the structural sharing across +sessions. + +.. note:: + Currently, ``immer-persist`` makes a distiction between pools used for saving containers (*output* pools) and for loading containers (*input* pools), + similar to ``cereal`` with its ``InputArchive`` and ``OutputArchive`` distiction. Currently, ``immer-persist`` focuses on JSON as the serialization format and uses the ``cereal`` library internally. In principle, other formats and serialization libraries could be supported in the future. diff --git a/immer/extra/persist/json/json_immer.hpp b/immer/extra/persist/json/json_immer.hpp index 272866e0..55f96d62 100644 --- a/immer/extra/persist/json/json_immer.hpp +++ b/immer/extra/persist/json/json_immer.hpp @@ -13,6 +13,13 @@ namespace immer::persist { namespace detail { +/** + * @brief This struct has an interface of an output archive but does + * nothing when requested to serialize data. It is used when we only care about + * the pools and have no need to serialize the actual document. + * + * @ingroup persist-api + */ struct blackhole_output_archive { void setNextName(const char* name) {} @@ -70,6 +77,21 @@ constexpr bool is_pool_empty() * Adapted from cereal/archives/adapters.hpp */ +/** + * @brief An output archive wrapper that provides access to the ``Pools`` stored + * inside. And serializes the ``pools`` object alongside the user document. + * + * @tparam Previous Type of the ``cereal`` output archive into which all the + * serialization calls are forwarded. + * @tparam Pools The wrapped pools. See ``output_pools``. + * @tparam WrapFn A function that is called on each serialized value in order to + * wrap the desired ``immer`` containers into ``persistable``. + * @tparam PoolNameFn A function that is used to determine the name for each + * individual pool. It's called with an empty ``immer`` container and must + * return the name for the corresponding pool. + * + * @ingroup persist-api + */ template class json_immer_output_archive : public cereal::OutputArchive< diff --git a/immer/extra/persist/json/names.hpp b/immer/extra/persist/json/names.hpp index a9eb0f82..0c27be37 100644 --- a/immer/extra/persist/json/names.hpp +++ b/immer/extra/persist/json/names.hpp @@ -14,6 +14,12 @@ auto get_demangled_name(const T&) template class error_duplicate_pool_name_found; +/** + * @brief This function ensures that all the names are unique for the given map + * of types to names. Otherwise, it triggers a compile-time error. + * + * @ingroup persist-api + */ inline auto are_type_names_unique(auto type_names) { namespace hana = boost::hana; diff --git a/immer/extra/persist/json/persistable.hpp b/immer/extra/persist/json/persistable.hpp index be31fcdb..5645cf04 100644 --- a/immer/extra/persist/json/persistable.hpp +++ b/immer/extra/persist/json/persistable.hpp @@ -11,28 +11,29 @@ namespace immer::persist { -namespace detail { -template -constexpr bool is_iterable{}; - -template -constexpr bool is_iterable().begin()), - decltype(std::declval().end())>> = - true; - -template -auto get_iterator_type() -{ - if constexpr (is_iterable) { - return T{}.begin(); - } else { - return; - } -} - -} // namespace detail - +/** + * @brief A wrapper that allows the library to serialize the wrapped container + * using a corresponding pool. + * + * When saving, it saves the container into the pool by performing the following + * steps: + * - request the output pool corresponding to the type of the ``Container`` + * - save the container to the pool by calling ``add_to_pool`` + * - acquire the ID from the pool for the just saved container + * - save the ID into the output archive. + * + * Similarly, the steps for loading are: + * - container ID is loaded from the input archive by ``cereal`` + * - request the input pool corresponding to the type of the ``Container`` + * - load the container with the required ID from the input pool. + * + * Consequently, instead of the container's actual data, ``persistable`` would + * serialize only the ID of the wrapped container. + * + * @tparam Container ``immer`` container that should be serialized using a pool. + * + * @ingroup persist-api + */ template struct persistable { @@ -52,19 +53,6 @@ struct persistable friend bool operator==(const persistable& left, const persistable& right) = default; - - // template , bool> = true> - // friend auto begin(const persistable& value) - // { - // return value.container.begin(); - // } - - // friend std::enable_if_t, - // decltype(std::declval().end())> - // end(const persistable& value) - // { - // return value.container.end(); - // } }; template cereal-inline, for + * example. * - Types of `immer` containers that will be serialized using pools. One * pool contains nodes of only one `immer` container type. * - Names for each per-type pool. @@ -46,7 +47,8 @@ auto get_pools_types(const T&) * * Other possible way would be to use a third-party library to serialize the * value inline (without the `value0` node) by taking a dependency on - * https://github.com/LowCostCustoms/cereal-inline, for example. + * cereal-inline, + * for example. * * @ingroup persist-policy */ diff --git a/immer/extra/persist/json/pools.hpp b/immer/extra/persist/json/pools.hpp index 993c958c..6b6f0056 100644 --- a/immer/extra/persist/json/pools.hpp +++ b/immer/extra/persist/json/pools.hpp @@ -64,7 +64,7 @@ struct output_pools { Storage storage_; - // To aling the interface with input_pools + // To align the interface with input_pools Storage& storage() { return storage_; } const Storage& storage() const { return storage_; }