feat(ffi): Add support for auto/user-generated KV-pairs in `KeyValueP…

…airLogEvent`; Detect and invalidate duplicate keys among non-leaf nodes when constructing a `KeyValuePairLogEvent`. (y-scope#558)

Co-authored-by: kirkrodrigues <[email protected]>

components/core/src/clp/ffi/KeyValuePairLogEvent.hpp

@@ -17,10 +17,13 @@
 namespace clp::ffi {
 /**
  * A log event containing key-value pairs. Each event contains:
- * - A collection of node-ID & value pairs, where each pair represents a leaf `SchemaTreeNode` in
- *   the `SchemaTree`.
- * - A reference to the `SchemaTree`
- * - The UTC offset of the current log event
+ * - A reference to the schema tree for auto-generated keys.
+ * - A reference to the schema tree for user-generated keys.
+ * - A collection of auto-generated node-ID & value pairs, where each pair represents a leaf
+ *   `SchemaTree::Node` in the schema tree for auto-generated keys.
+ * - A collection of user-generated node-ID & value pairs, where each pair represents a leaf
+ *   `SchemaTree::Node` in the schema tree for user-generated keys.
+ * - The UTC offset of the current log event.
  */
 class KeyValuePairLogEvent {
 public:
@@ -29,15 +32,21 @@ class KeyValuePairLogEvent {
 
     // Factory functions
     /**
-     * @param schema_tree
-     * @param node_id_value_pairs
+     * @param auto_gen_keys_schema_tree
+     * @param user_gen_keys_schema_tree
+     * @param auto_gen_node_id_value_pairs
+     * @param user_gen_node_id_value_pairs
      * @param utc_offset
      * @return A result containing the key-value pair log event or an error code indicating the
-     * failure. See `validate_node_id_value_pairs` for the possible error codes.
+     * failure:
+     * - std::errc::invalid_argument if any of the given schema tree pointers are null.
+     * - Forwards `validate_node_id_value_pairs`'s return values.
      */
     [[nodiscard]] static auto create(
-            std::shared_ptr<SchemaTree const> schema_tree,
-            NodeIdValuePairs node_id_value_pairs,
+            std::shared_ptr<SchemaTree const> auto_gen_keys_schema_tree,
+            std::shared_ptr<SchemaTree const> user_gen_keys_schema_tree,
+            NodeIdValuePairs auto_gen_node_id_value_pairs,
+            NodeIdValuePairs user_gen_node_id_value_pairs,
             UtcOffset utc_offset
     ) -> OUTCOME_V2_NAMESPACE::std_result<KeyValuePairLogEvent>;
 
@@ -53,51 +62,77 @@ class KeyValuePairLogEvent {
     ~KeyValuePairLogEvent() = default;
 
     // Methods
-    [[nodiscard]] auto get_schema_tree() const -> SchemaTree const& { return *m_schema_tree; }
+    [[nodiscard]] auto get_auto_gen_keys_schema_tree() const -> SchemaTree const& {
+        return *m_auto_gen_keys_schema_tree;
+    }
 
-    [[nodiscard]] auto get_node_id_value_pairs() const -> NodeIdValuePairs const& {
-        return m_node_id_value_pairs;
+    [[nodiscard]] auto get_user_gen_keys_schema_tree() const -> SchemaTree const& {
+        return *m_user_gen_keys_schema_tree;
     }
 
-    [[nodiscard]] auto get_utc_offset() const -> UtcOffset { return m_utc_offset; }
+    [[nodiscard]] auto get_auto_gen_node_id_value_pairs() const -> NodeIdValuePairs const& {
+        return m_auto_gen_node_id_value_pairs;
+    }
+
+    [[nodiscard]] auto get_user_gen_node_id_value_pairs() const -> NodeIdValuePairs const& {
+        return m_user_gen_node_id_value_pairs;
+    }
 
     /**
      * @return A result containing a bitmap where every bit corresponds to the ID of a node in the
-     * schema tree, and the set bits correspond to the nodes in the subtree defined by all paths
-     * from the root node to the nodes in `node_id_value_pairs`; or an error code indicating a
-     * failure:
-     * - std::errc::result_out_of_range if a node ID in `node_id_value_pairs` doesn't exist in the
-     *   schema tree.
+     * schema tree for auto-generated keys, and the set bits correspond to the nodes in the subtree
+     * defined by all paths from the root node to the nodes in `m_auto_gen_node_id_value_pairs`; or
+     * an error code indicating a failure:
+     * - Forwards `get_schema_subtree_bitmap`'s return values.
      */
-    [[nodiscard]] auto get_schema_subtree_bitmap(
+    [[nodiscard]] auto get_auto_gen_keys_schema_subtree_bitmap(
     ) const -> OUTCOME_V2_NAMESPACE::std_result<std::vector<bool>>;
 
     /**
-     * Serializes the log event into a `nlohmann::json` object.
-     * @return A result containing the serialized JSON object or an error code indicating the
-     * failure:
-     * - std::errc::protocol_error if a value in the log event couldn't be decoded or it couldn't be
-     *   inserted into a JSON object.
-     * - std::errc::result_out_of_range if a node ID in the log event doesn't exist in the schema
-     *   tree.
+     * @return A result containing a bitmap where every bit corresponds to the ID of a node in the
+     * schema tree for user-generated keys, and the set bits correspond to the nodes in the subtree
+     * defined by all paths from the root node to the nodes in `m_user_gen_node_id_value_pairs`; or
+     * an error code indicating a failure:
+     * - Forwards `get_schema_subtree_bitmap`'s return values.
+     */
+    [[nodiscard]] auto get_user_gen_keys_schema_subtree_bitmap(
+    ) const -> OUTCOME_V2_NAMESPACE::std_result<std::vector<bool>>;
+
+    [[nodiscard]] auto get_utc_offset() const -> UtcOffset { return m_utc_offset; }
+
+    /**
+     * Serializes the log event into `nlohmann::json` objects.
+     * @return A result containing a pair or an error code indicating the failure:
+     * - The pair:
+     *   - Serialized auto-generated key-value pairs as a JSON object
+     *   - Serialized user-generated key-value pairs as a JSON object
+     * - The possible error codes:
+     *   - Forwards `get_auto_gen_keys_schema_subtree_bitmap`'s return values on failure.
+     *   - Forwards `serialize_node_id_value_pairs_to_json`'s return values on failure.
      */
     [[nodiscard]] auto serialize_to_json(
-    ) const -> OUTCOME_V2_NAMESPACE::std_result<nlohmann::json>;
+    ) const -> OUTCOME_V2_NAMESPACE::std_result<std::pair<nlohmann::json, nlohmann::json>>;
 
 private:
     // Constructor
     KeyValuePairLogEvent(
-            std::shared_ptr<SchemaTree const> schema_tree,
-            NodeIdValuePairs node_id_value_pairs,
+            std::shared_ptr<SchemaTree const> auto_gen_keys_schema_tree,
+            std::shared_ptr<SchemaTree const> user_gen_keys_schema_tree,
+            NodeIdValuePairs auto_gen_node_id_value_pairs,
+            NodeIdValuePairs user_gen_node_id_value_pairs,
             UtcOffset utc_offset
     )
-            : m_schema_tree{std::move(schema_tree)},
-              m_node_id_value_pairs{std::move(node_id_value_pairs)},
+            : m_auto_gen_keys_schema_tree{std::move(auto_gen_keys_schema_tree)},
+              m_user_gen_keys_schema_tree{std::move(user_gen_keys_schema_tree)},
+              m_auto_gen_node_id_value_pairs{std::move(auto_gen_node_id_value_pairs)},
+              m_user_gen_node_id_value_pairs{std::move(user_gen_node_id_value_pairs)},
               m_utc_offset{utc_offset} {}
 
     // Variables
-    std::shared_ptr<SchemaTree const> m_schema_tree;
-    NodeIdValuePairs m_node_id_value_pairs;
+    std::shared_ptr<SchemaTree const> m_auto_gen_keys_schema_tree;
+    std::shared_ptr<SchemaTree const> m_user_gen_keys_schema_tree;
+    NodeIdValuePairs m_auto_gen_node_id_value_pairs;
+    NodeIdValuePairs m_user_gen_node_id_value_pairs;
     UtcOffset m_utc_offset{0};
 };
 }  // namespace clp::ffi

components/core/src/clp/ffi/SchemaTree.hpp

@@ -128,6 +128,8 @@ class SchemaTree {
         ~Node() = default;
 
         // Methods
+        [[nodiscard]] auto operator==(Node const& rhs) const -> bool = default;
+
         [[nodiscard]] auto get_id() const -> id_t { return m_id; }
 
         [[nodiscard]] auto is_root() const -> bool { return false == m_parent_id.has_value(); }
@@ -249,6 +251,10 @@ class SchemaTree {
     ~SchemaTree() = default;
 
     // Methods
+    [[nodiscard]] auto operator==(SchemaTree const& rhs) const -> bool {
+        return m_tree_nodes == rhs.m_tree_nodes;
+    }
+
     [[nodiscard]] auto get_size() const -> size_t { return m_tree_nodes.size(); }
 
     [[nodiscard]] auto get_root() const -> Node const& { return m_tree_nodes[cRootId]; }

components/core/src/clp/ffi/ir_stream/Deserializer.hpp

@@ -115,7 +115,8 @@ class Deserializer {
     Deserializer(IrUnitHandler ir_unit_handler) : m_ir_unit_handler{std::move(ir_unit_handler)} {}
 
     // Variables
-    std::shared_ptr<SchemaTree> m_schema_tree{std::make_shared<SchemaTree>()};
+    std::shared_ptr<SchemaTree> m_auto_gen_keys_schema_tree{std::make_shared<SchemaTree>()};
+    std::shared_ptr<SchemaTree> m_user_gen_keys_schema_tree{std::make_shared<SchemaTree>()};
     UtcOffset m_utc_offset{0};
     IrUnitHandler m_ir_unit_handler;
     bool m_is_complete{false};
@@ -183,9 +184,13 @@ auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& read
     auto const ir_unit_type{optional_ir_unit_type.value()};
     switch (ir_unit_type) {
         case IrUnitType::LogEvent: {
-            auto result{
-                    deserialize_ir_unit_kv_pair_log_event(reader, tag, m_schema_tree, m_utc_offset)
-            };
+            auto result{deserialize_ir_unit_kv_pair_log_event(
+                    reader,
+                    tag,
+                    m_auto_gen_keys_schema_tree,
+                    m_user_gen_keys_schema_tree,
+                    m_utc_offset
+            )};
             if (result.has_error()) {
                 return result.error();
             }
@@ -207,7 +212,7 @@ auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& read
             }
 
             auto const node_locator{result.value()};
-            if (m_schema_tree->has_node(node_locator)) {
+            if (m_user_gen_keys_schema_tree->has_node(node_locator)) {
                 return std::errc::protocol_error;
             }
 
@@ -217,7 +222,7 @@ auto Deserializer<IrUnitHandler>::deserialize_next_ir_unit(ReaderInterface& read
                 return ir_error_code_to_errc(err);
             }
 
-            std::ignore = m_schema_tree->insert_node(node_locator);
+            std::ignore = m_user_gen_keys_schema_tree->insert_node(node_locator);
             break;
         }
 

components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.cpp

@@ -551,7 +551,8 @@ auto deserialize_ir_unit_utc_offset_change(ReaderInterface& reader
 auto deserialize_ir_unit_kv_pair_log_event(
         ReaderInterface& reader,
         encoded_tag_t tag,
-        std::shared_ptr<SchemaTree> schema_tree,
+        std::shared_ptr<SchemaTree> auto_gen_keys_schema_tree,
+        std::shared_ptr<SchemaTree> user_gen_keys_schema_tree,
         UtcOffset utc_offset
 ) -> OUTCOME_V2_NAMESPACE::std_result<KeyValuePairLogEvent> {
     auto const schema_result{deserialize_schema(reader, tag)};
@@ -579,7 +580,9 @@ auto deserialize_ir_unit_kv_pair_log_event(
     }
 
     return KeyValuePairLogEvent::create(
-            std::move(schema_tree),
+            std::move(auto_gen_keys_schema_tree),
+            std::move(user_gen_keys_schema_tree),
+            {},
             std::move(node_id_value_pairs),
             utc_offset
     );

components/core/src/clp/ffi/ir_stream/ir_unit_deserialization_methods.hpp

@@ -57,10 +57,12 @@ namespace clp::ffi::ir_stream {
  * Deserializes a key-value pair log event IR unit.
  * @param reader
  * @param tag
- * @param schema_tree Schema tree used to construct the KV-pair log event.
+ * @param auto_gen_keys_schema_tree Schema tree for auto-generated keys, used to construct the
+ * KV-pair log event.
+ * @param user_gen_keys_schema_tree Schema tree for user-generated keys, used to construct the
+ * KV-pair log event.
  * @param utc_offset UTC offset used to construct the KV-pair log event.
- * @return A result containing the deserialized log event or an error code indicating the
- * failure:
+ * @return A result containing the deserialized log event or an error code indicating the failure:
  * - std::errc::result_out_of_range if the IR stream is truncated.
  * - std::errc::protocol_error if the IR stream is corrupted.
  * - std::errc::protocol_not_supported if the IR stream contains an unsupported metadata format
@@ -72,7 +74,8 @@ namespace clp::ffi::ir_stream {
 [[nodiscard]] auto deserialize_ir_unit_kv_pair_log_event(
         ReaderInterface& reader,
         encoded_tag_t tag,
-        std::shared_ptr<SchemaTree> schema_tree,
+        std::shared_ptr<SchemaTree> auto_gen_keys_schema_tree,
+        std::shared_ptr<SchemaTree> user_gen_keys_schema_tree,
         UtcOffset utc_offset
 ) -> OUTCOME_V2_NAMESPACE::std_result<KeyValuePairLogEvent>;
 }  // namespace clp::ffi::ir_stream

components/core/tests/test-ffi_IrUnitHandlerInterface.cpp

@@ -87,9 +87,13 @@ auto test_ir_unit_handler_interface(clp::ffi::ir_stream::IrUnitHandlerInterface
 
 auto test_ir_unit_handler_interface(clp::ffi::ir_stream::IrUnitHandlerInterface auto& handler
 ) -> void {
-    auto test_log_event_result{
-            KeyValuePairLogEvent::create(std::make_shared<SchemaTree>(), {}, cTestUtcOffset)
-    };
+    auto test_log_event_result{KeyValuePairLogEvent::create(
+            std::make_shared<SchemaTree>(),
+            std::make_shared<SchemaTree>(),
+            {},
+            {},
+            cTestUtcOffset
+    )};
     REQUIRE(
             (false == test_log_event_result.has_error()
              && IRErrorCode::IRErrorCode_Success
@@ -127,7 +131,7 @@ TEMPLATE_TEST_CASE(
     REQUIRE(
             (optional_log_event.has_value()
              && optional_log_event.value().get_utc_offset() == cTestUtcOffset
-             && optional_log_event.value().get_node_id_value_pairs().empty())
+             && optional_log_event.value().get_user_gen_node_id_value_pairs().empty())
     );
     auto const& optional_schema_tree_locator{handler.get_schema_tree_node_locator()};
     REQUIRE(