clp-s: Add support for serializing structured arrays. (y-scope#413)

Co-authored-by: wraymo <[email protected]>
Bill-hbrhbr · Jun 7, 2024 · 2725b9a · 2725b9a
1 parent 2c37cc6
commit 2725b9a
Show file tree

Hide file tree

Showing 5 changed files with 343 additions and 22 deletions.
diff --git a/components/core/src/clp_s/ArchiveReader.cpp b/components/core/src/clp_s/ArchiveReader.cpp
@@ -143,25 +143,17 @@ BaseColumnReader* ArchiveReader::append_reader_column(SchemaReader& reader, int3
 
 void ArchiveReader::append_unordered_reader_columns(
         SchemaReader& reader,
-        NodeType unordered_object_type,
+        int32_t mst_subtree_root_node_id,
         std::span<int32_t> schema_ids,
         bool should_marshal_records
 ) {
-    int32_t mst_subtree_root_node_id = INT32_MAX;
     size_t object_begin_pos = reader.get_column_size();
     for (int32_t column_id : schema_ids) {
         if (Schema::schema_entry_is_unordered_object(column_id)) {
             continue;
         }
         BaseColumnReader* column_reader = nullptr;
         auto const& node = m_schema_tree->get_node(column_id);
-        if (INT32_MAX == mst_subtree_root_node_id) {
-            mst_subtree_root_node_id = m_schema_tree->find_matching_subtree_root_in_subtree(
-                    -1,
-                    column_id,
-                    unordered_object_type
-            );
-        }
         switch (node.get_type()) {
             case NodeType::Integer:
                 column_reader = new Int64ColumnReader(column_id);
@@ -214,20 +206,38 @@ SchemaReader& ArchiveReader::create_schema_reader(
             should_marshal_records
     );
     auto timestamp_column_ids = m_timestamp_dict->get_authoritative_timestamp_column_ids();
-
     for (size_t i = 0; i < schema.size(); ++i) {
         int32_t column_id = schema[i];
         if (Schema::schema_entry_is_unordered_object(column_id)) {
             size_t length = Schema::get_unordered_object_length(column_id);
+
+            auto sub_schema = schema.get_view(i + 1, length);
+            auto mst_subtree_root_node_id = m_schema_tree->find_matching_subtree_root_in_subtree(
+                    -1,
+                    SchemaReader::get_first_column_in_span(sub_schema),
+                    Schema::get_unordered_object_type(column_id)
+            );
             append_unordered_reader_columns(
                     m_schema_reader,
-                    Schema::get_unordered_object_type(column_id),
-                    schema.get_view(i + 1, length),
+                    mst_subtree_root_node_id,
+                    sub_schema,
                     should_marshal_records
             );
             i += length;
             continue;
         }
+        if (i >= schema.get_num_ordered()) {
+            // Length one unordered object that doesn't have a tag. This is only allowed when the
+            // column id is the root of the unordered object, so we can pass it directly to
+            // append_unordered_reader_columns.
+            append_unordered_reader_columns(
+                    m_schema_reader,
+                    column_id,
+                    std::span<int32_t>(),
+                    should_marshal_records
+            );
+            continue;
+        }
         BaseColumnReader* column_reader = append_reader_column(m_schema_reader, column_id);
 
         if (should_extract_timestamp && column_reader && timestamp_column_ids.count(column_id) > 0)

diff --git a/components/core/src/clp_s/ArchiveReader.hpp b/components/core/src/clp_s/ArchiveReader.hpp
@@ -149,13 +149,13 @@ class ArchiveReader {
     /**
      * Appends columns for the entire schema of an unordered object.
      * @param reader
-     * @param unordered_object_type
+     * @param mst_subtree_root_node_id
      * @param schema_ids
      * @param should_marshal_records
      */
     void append_unordered_reader_columns(
             SchemaReader& reader,
-            NodeType unordered_object_type,
+            int32_t mst_subtree_root_node_id,
             std::span<int32_t> schema_ids,
             bool should_marshal_records
     );

diff --git a/components/core/src/clp_s/JsonSerializer.hpp b/components/core/src/clp_s/JsonSerializer.hpp
@@ -23,7 +23,9 @@ class JsonSerializer {
         AddStringValue,
         AddNullValue,
         BeginArray,
-        EndArray
+        EndArray,
+        BeginUnnamedObject,
+        BeginUnnamedArray,
     };
 
     static int64_t const cReservedLength = 4096;
@@ -76,19 +78,25 @@ class JsonSerializer {
     void end_document() { m_json_string[m_json_string.size() - 1] = '}'; }
 
     void end_object() {
-        if (m_op_list[m_op_list_index - 2] != BeginObject) {
+        if (m_op_list[m_op_list_index - 2] != BeginObject
+            && m_op_list[m_op_list_index - 2] != BeginUnnamedObject)
+        {
             m_json_string.pop_back();
         }
         m_json_string += "},";
     }
 
+    void begin_array_document() { m_json_string += "["; }
+
     void begin_array() {
         append_key();
         m_json_string += "[";
     }
 
     void end_array() {
-        if (m_op_list[m_op_list_index - 2] != BeginArray) {
+        if (m_op_list[m_op_list_index - 2] != BeginArray
+            && m_op_list[m_op_list_index - 2] != BeginUnnamedArray)
+        {
             m_json_string.pop_back();
         }
         m_json_string += "],";