More compact maps format: pairs are lists of 2 elements

arximboldi · Aug 26, 2024 · 6de8600 · 6de8600
1 parent ece80df
commit 6de8600
Show file tree

Hide file tree

Showing 7 changed files with 356 additions and 579 deletions.
diff --git a/doc/persist.rst b/doc/persist.rst
@@ -222,7 +222,7 @@ the ``new_value`` and inspect the JSON:
    :start-after: start-save-new_value
    :end-before:  end-save-new_value
 
-And indeed, we can see in the JSON that the node ``{"key": 2, "value": [10, 20]}`` is reused in both vectors.
+And indeed, we can see in the JSON that the node ``[2, [10, 20]]`` is reused in both vectors.
 
 
 Transformation into a different type
@@ -270,7 +270,7 @@ And serialize it with pools:
    :start-after: start-save-new_value-str
    :end-before:  end-save-new_value-str
 
-In the resulting JSON we can confirm that the node ``{"key": 2, "value": ["_1_", "_2_"]}`` is reused for both vectors.
+In the resulting JSON we can confirm that the node ``[2, ["_1_", "_2_"]]`` is reused for both vectors.
 
 .. _transforming-hash-based-containers:
 
@@ -459,7 +459,7 @@ The resulting JSON looks like:
    :start-after: start-nested-value-json
    :end-before:  end-nested-value-json
 
-Looking at the JSON we can confirm that the node ``{"key": 2, "value": [1, 2]}`` is reused.
+Looking at the JSON we can confirm that the node ``[2, [1, 2]]`` is reused.
 Let's define a ``conversion_map`` like this:
 
 .. literalinclude:: ../test/extra/persist/test_for_docs.cpp
@@ -497,7 +497,7 @@ And we can serialize it again to confirm that the structural sharing of the nest
    :start-after: start-verify-structural-sharing-of-nested
    :end-before:  end-verify-structural-sharing-of-nested
 
-We can see that the ``{"key": 2, "value": ["_1_", "_2_"]}`` node is still being reused in the two vectors.
+We can see that the ``[2, ["_1_", "_2_"]]`` node is still being reused in the two vectors.
 
 
 Policy

diff --git a/immer/extra/persist/detail/cereal/compact_map.hpp b/immer/extra/persist/detail/cereal/compact_map.hpp
@@ -0,0 +1,83 @@
+#pragma once
+
+#include <cereal/cereal.hpp>
+
+#include <immer/map.hpp>
+
+namespace immer::persist::detail {
+
+template <class K, class T>
+struct compact_pair
+{
+    K key;
+    T value;
+};
+
+template <class K, class T>
+struct compact_map
+{
+    immer::map<K, T>& map;
+};
+
+template <class K, class T>
+compact_map<K, T> make_compact_map(immer::map<K, T>& map)
+{
+    return compact_map<K, T>{map};
+}
+
+} // namespace immer::persist::detail
+
+namespace cereal {
+
+template <typename Archive, typename K, typename T>
+void CEREAL_LOAD_FUNCTION_NAME(Archive& ar,
+                               immer::persist::detail::compact_pair<K, T>& m)
+{
+    size_type size;
+    ar(make_size_tag(size));
+    if (size != 2) {
+        throw Exception{"A pair must be a list of 2 elements"};
+    }
+
+    ar(m.key);
+    ar(m.value);
+}
+
+template <typename Archive, typename K, typename T>
+void CEREAL_SAVE_FUNCTION_NAME(
+    Archive& ar, const immer::persist::detail::compact_pair<K, T>& m)
+{
+    ar(make_size_tag(static_cast<size_type>(2)));
+    ar(m.key);
+    ar(m.value);
+}
+
+template <typename Archive, typename K, typename T>
+void CEREAL_LOAD_FUNCTION_NAME(Archive& ar,
+                               immer::persist::detail::compact_map<K, T>& m)
+{
+    size_type size;
+    ar(make_size_tag(size));
+
+    for (auto i = size_type{}; i < size; ++i) {
+        auto pair = immer::persist::detail::compact_pair<K, T>{};
+        ar(pair);
+
+        m.map =
+            std::move(m.map).set(std::move(pair.key), std::move(pair.value));
+    }
+    if (size != m.map.size())
+        throw Exception{"duplicate ids?"};
+}
+
+template <typename Archive, typename K, typename T>
+void CEREAL_SAVE_FUNCTION_NAME(
+    Archive& ar, const immer::persist::detail::compact_map<K, T>& m)
+{
+    ar(make_size_tag(static_cast<size_type>(m.map.size())));
+    for (auto&& v : m.map) {
+        ar(immer::persist::detail::compact_pair<K, T>{v.first, v.second});
+    }
+}
+
+} // namespace cereal
diff --git a/immer/extra/persist/detail/rbts/pool.hpp b/immer/extra/persist/detail/rbts/pool.hpp
@@ -3,6 +3,7 @@
 #include <immer/extra/cereal/immer_map.hpp>
 #include <immer/extra/cereal/immer_vector.hpp>
 #include <immer/extra/persist/detail/alias.hpp>
+#include <immer/extra/persist/detail/cereal/compact_map.hpp>
 #include <immer/extra/persist/detail/common/pool.hpp>
 
 #include <immer/array.hpp>
@@ -72,10 +73,12 @@ struct output_pool
     template <class Archive>
     void save(Archive& ar) const
     {
+        auto leaves_copy = leaves;
+        auto inners_copy = inners;
         ar(CEREAL_NVP(B),
            CEREAL_NVP(BL),
-           CEREAL_NVP(leaves),
-           CEREAL_NVP(inners),
+           cereal::make_nvp("leaves", detail::make_compact_map(leaves_copy)),
+           cereal::make_nvp("inners", detail::make_compact_map(inners_copy)),
            CEREAL_NVP(vectors));
     }
 };
@@ -118,8 +121,8 @@ struct input_pool
         auto& BL = bits_leaf;
         ar(CEREAL_NVP(B),
            CEREAL_NVP(BL),
-           CEREAL_NVP(leaves),
-           CEREAL_NVP(inners),
+           cereal::make_nvp("leaves", detail::make_compact_map(leaves)),
+           cereal::make_nvp("inners", detail::make_compact_map(inners)),
            CEREAL_NVP(vectors));
     }
 

diff --git a/test/extra/persist/test_for_docs.cpp b/test/extra/persist/test_for_docs.cpp
@@ -118,20 +118,15 @@ TEST_CASE("Docs save with immer-persist", "[docs]")
       "B": 5,
       "BL": 1,
       "inners": [
-        {"key": 0, "value": {"children": [2], "relaxed": false}},
-        {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
-      ],
-      "leaves": [
-        {"key": 1, "value": [3]},
-        {"key": 2, "value": [1, 2]},
-        {"key": 4, "value": [5, 6]},
-        {"key": 5, "value": [3, 4]}
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
       ],
+      "leaves": [[1, [3]], [2, [1, 2]], [4, [5, 6]], [5, [3, 4]]],
       "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
     }
   }
 }
-    )");
+        )");
         // include:intro/end-persist-json
         REQUIRE(json_t::parse(str) == expected_json);
 
@@ -227,29 +222,20 @@ TEST_CASE("Custom policy", "[docs]")
     "vector_of_ints": {
       "B": 5,
       "BL": 1,
-      "leaves": [
-        {"key": 1, "value": [3]},
-        {"key": 2, "value": [1, 2]},
-        {"key": 4, "value": [5, 6]},
-        {"key": 5, "value": [3, 4]}
-      ],
+      "leaves": [[1, [3]], [2, [1, 2]], [4, [5, 6]], [5, [3, 4]]],
       "inners": [
-        {"key": 0, "value": {"children": [2], "relaxed": false}},
-        {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
       ],
       "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
     },
     "vector_of_strings": {
       "B": 5,
       "BL": 1,
-      "leaves": [
-        {"key": 1, "value": ["one", "two"]},
-        {"key": 3, "value": ["five"]},
-        {"key": 4, "value": ["three", "four"]}
-      ],
+      "leaves": [[1, ["one", "two"]], [3, ["five"]], [4, ["three", "four"]]],
       "inners": [
-        {"key": 0, "value": {"children": [], "relaxed": false}},
-        {"key": 2, "value": {"children": [1, 4], "relaxed": false}}
+        [0, {"children": [], "relaxed": false}],
+        [2, {"children": [1, 4], "relaxed": false}]
       ],
       "vectors": [{"root": 0, "tail": 1}, {"root": 2, "tail": 3}]
     }
@@ -330,27 +316,22 @@ TEST_CASE("Transformations", "[docs]")
         const auto str =
             immer::persist::cereal_save_with_pools(new_value, policy);
         const auto expected_json = json_t::parse(R"(
-          {
-            "pools": {
-              "ints": {
-                "B": 5,
-                "BL": 1,
-                "inners": [
-                  {"key": 0, "value": {"children": [2], "relaxed": false}},
-                  {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
-                ],
-                "leaves": [
-                  {"key": 1, "value": [30]},
-                  {"key": 2, "value": [10, 20]},
-                  {"key": 4, "value": [50, 60]},
-                  {"key": 5, "value": [30, 40]}
-                ],
-                "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
-              }
-            },
-            "value0": {"ints": 0, "ints2": 1}
-          }
-    )");
+{
+  "pools": {
+    "ints": {
+      "B": 5,
+      "BL": 1,
+      "inners": [
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
+      ],
+      "leaves": [[1, [30]], [2, [10, 20]], [4, [50, 60]], [5, [30, 40]]],
+      "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
+    }
+  },
+  "value0": {"ints": 0, "ints2": 1}
+}
+        )");
         REQUIRE(json_t::parse(str) == expected_json);
         // include:end-save-new_value
     }
@@ -386,27 +367,27 @@ TEST_CASE("Transformations", "[docs]")
         const auto str =
             immer::persist::cereal_save_with_pools(new_value, policy);
         const auto expected_json = json_t::parse(R"(
-          {
-            "pools": {
-              "str": {
-                "B": 5,
-                "BL": 1,
-                "inners": [
-                  {"key": 0, "value": {"children": [2], "relaxed": false}},
-                  {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
-                ],
-                "leaves": [
-                  {"key": 1, "value": ["_3_"]},
-                  {"key": 2, "value": ["_1_", "_2_"]},
-                  {"key": 4, "value": ["_5_", "_6_"]},
-                  {"key": 5, "value": ["_3_", "_4_"]}
-                ],
-                "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
-              }
-            },
-            "value0": {"str": 0, "str2": 1}
-          }
-            )");
+{
+  "pools": {
+    "str": {
+      "B": 5,
+      "BL": 1,
+      "inners": [
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
+      ],
+      "leaves": [
+        [1, ["_3_"]],
+        [2, ["_1_", "_2_"]],
+        [4, ["_5_", "_6_"]],
+        [5, ["_3_", "_4_"]]
+      ],
+      "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
+    }
+  },
+  "value0": {"str": 0, "str2": 1}
+}
+        )");
         REQUIRE(json_t::parse(str) == expected_json);
         // include:end-save-new_value-str
     }
@@ -698,22 +679,17 @@ TEST_CASE("Transform nested containers", "[docs]")
       "B": 5,
       "BL": 1,
       "inners": [
-        {"key": 0, "value": {"children": [2], "relaxed": false}},
-        {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
-      ],
-      "leaves": [
-        {"key": 1, "value": [3]},
-        {"key": 2, "value": [1, 2]},
-        {"key": 4, "value": [5, 6]},
-        {"key": 5, "value": [3, 4]}
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
       ],
+      "leaves": [[1, [3]], [2, [1, 2]], [4, [5, 6]], [5, [3, 4]]],
       "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
     },
     "nested": {
       "B": 5,
       "BL": 3,
-      "inners": [{"key": 0, "value": {"children": [], "relaxed": false}}],
-      "leaves": [{"key": 1, "value": [{"ints": 0}, {"ints": 1}]}],
+      "inners": [[0, {"children": [], "relaxed": false}]],
+      "leaves": [[1, [{"ints": 0}, {"ints": 1}]]],
       "vectors": [{"root": 0, "tail": 1}]
     }
   },
@@ -773,22 +749,22 @@ TEST_CASE("Transform nested containers", "[docs]")
     "nested": {
       "B": 5,
       "BL": 3,
-      "inners": [{"key": 0, "value": {"children": [], "relaxed": false}}],
-      "leaves": [{"key": 1, "value": [{"str": 0}, {"str": 1}]}],
+      "inners": [[0, {"children": [], "relaxed": false}]],
+      "leaves": [[1, [{"str": 0}, {"str": 1}]]],
       "vectors": [{"root": 0, "tail": 1}]
     },
     "str": {
       "B": 5,
       "BL": 1,
       "inners": [
-        {"key": 0, "value": {"children": [2], "relaxed": false}},
-        {"key": 3, "value": {"children": [2, 5], "relaxed": false}}
+        [0, {"children": [2], "relaxed": false}],
+        [3, {"children": [2, 5], "relaxed": false}]
       ],
       "leaves": [
-        {"key": 1, "value": ["_3_"]},
-        {"key": 2, "value": ["_1_", "_2_"]},
-        {"key": 4, "value": ["_5_", "_6_"]},
-        {"key": 5, "value": ["_3_", "_4_"]}
+        [1, ["_3_"]],
+        [2, ["_1_", "_2_"]],
+        [4, ["_5_", "_6_"]],
+        [5, ["_3_", "_4_"]]
       ],
       "vectors": [{"root": 0, "tail": 1}, {"root": 3, "tail": 4}]
     }