modify

Signed-off-by: jinjiabao.jjb <[email protected]>

src/index/pyramid.cpp

@@ -295,6 +295,17 @@ Pyramid::Deserialize(const BinarySet& binary_set) {
 
 tl::expected<void, Error>
 Pyramid::Deserialize(const ReaderSet& reader_set) {
+    auto keys = reader_set.GetKeys();
+    for (const auto& path : keys) {
+        const auto& reader = reader_set.Get(path);
+        auto path_slices = split(path, PART_OCTOTHORPE);
+        std::shared_ptr<IndexNode> node = try_get_node_with_init(indexes_, path_slices[0]);
+        for (int j = 1; j < path_slices.size(); ++j) {
+            node = try_get_node_with_init(node->children, path_slices[j]);
+        }
+        node->CreateIndex(pyramid_param_.index_builder);
+        node->index->Deserialize(reader_to_readerset(reader));
+    }
     return {};
 }
 

tests/fixtures/test_reader.h

@@ -0,0 +1,50 @@
+
+// Copyright 2024-present the vsag project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <string.h>
+
+#include "vsag/binaryset.h"
+#include "vsag/readerset.h"
+
+namespace fixtures {
+
+class TestReader : public vsag::Reader {
+public:
+    TestReader(vsag::Binary binary) : binary_(binary) {
+    }
+
+    void
+    Read(uint64_t offset, uint64_t len, void* dest) override {
+        memcpy((char*)dest, binary_.data.get() + offset, len);
+    }
+
+    void
+    AsyncRead(uint64_t offset, uint64_t len, void* dest, vsag::CallBack callback) override {
+        Read(offset, len, dest);
+        callback(vsag::IOErrorCode::IO_SUCCESS, "success");
+    }
+
+    uint64_t
+    Size() const override {
+        return binary_.size;
+    }
+
+private:
+    vsag::Binary binary_;
+};
+
+}  // namespace fixtures

tests/test_index.cpp

@@ -16,6 +16,7 @@
 #include "test_index.h"
 
 #include "fixtures/test_logger.h"
+#include "fixtures/test_reader.h"
 #include "fixtures/thread_pool.h"
 #include "simd/fp32_simd.h"
 
@@ -413,6 +414,7 @@ TestIndex::TestSerializeBinarySet(const IndexPtr& index_from,
         auto query = vsag::Dataset::Make();
         query->NumElements(1)
             ->Dim(dim)
+            ->Paths(queries->GetPaths() + i)
             ->Float32Vectors(queries->GetFloat32Vectors() + i * dim)
             ->Owner(false);
         auto res_from = index_from->KnnSearch(query, topk, search_param);
@@ -433,16 +435,13 @@ TestIndex::TestSerializeReaderSet(const IndexPtr& index_from,
                                   const std::string& search_param,
                                   const std::string& index_name,
                                   bool expected_success) {
-    auto dir = fixtures::TempDir("serialize");
-    auto path = dir.GenerateRandomFile();
-    std::ofstream outfile(path, std::ios::out | std::ios::binary);
-    auto serialize_index = index_from->Serialize(outfile);
-    REQUIRE(serialize_index.has_value() == expected_success);
-    outfile.close();
-
     vsag::ReaderSet rs;
-    auto reader = vsag::Factory::CreateLocalFileReader(path, 0, 0);
-    rs.Set(index_name, reader);
+    auto serialize_binary = index_from->Serialize();
+    REQUIRE(serialize_binary.has_value() == expected_success);
+    auto binary_set = serialize_binary.value();
+    for (const auto& key : binary_set.GetKeys()) {
+        rs.Set(key, std::make_shared<TestReader>(binary_set.Get(key)));
+    }
     auto deserialize_index = index_to->Deserialize(rs);
     REQUIRE(deserialize_index.has_value() == expected_success);
 
@@ -454,6 +453,7 @@ TestIndex::TestSerializeReaderSet(const IndexPtr& index_from,
         auto query = vsag::Dataset::Make();
         query->NumElements(1)
             ->Dim(dim)
+            ->Paths(queries->GetPaths() + i)
             ->Float32Vectors(queries->GetFloat32Vectors() + i * dim)
             ->Owner(false);
         auto res_from = index_from->KnnSearch(query, topk, search_param);

tests/test_pyramid.cpp

@@ -87,3 +87,30 @@ TEST_CASE_PERSISTENT_FIXTURE(fixtures::PyramidTestIndex,
         TestRangeSearch(index, dataset, search_param, 0.49, 5, true);
     }
 }
+
+TEST_CASE_PERSISTENT_FIXTURE(fixtures::PyramidTestIndex,
+                             "Pyramid Serialize File",
+                             "[ft][pyramid]") {
+    auto origin_size = vsag::Options::Instance().block_size_limit();
+    auto size = GENERATE(1024 * 1024 * 2);
+    auto metric_type = GENERATE("l2", "ip", "cosine");
+    const std::string name = "pyramid";
+    auto search_param = fmt::format(search_param_tmp, 200);
+
+    for (auto& dim : dims) {
+        vsag::Options::Instance().set_block_size_limit(size);
+        auto param = GeneratePyramidBuildParametersString(metric_type, dim);
+        auto index = TestFactory(name, param, true);
+        auto dataset = pool.GetDatasetAndCreate(dim, base_count, metric_type, /*with_path=*/true);
+        TestBuildIndex(index, dataset, true);
+        SECTION("serialize/deserialize by binary") {
+            auto index2 = TestFactory(name, param, true);
+            TestSerializeBinarySet(index, index2, dataset, search_param, true);
+        }
+        SECTION("serialize/deserialize by binary") {
+            auto index2 = TestFactory(name, param, true);
+            TestSerializeReaderSet(index, index2, dataset, search_param, name, true);
+        }
+    }
+    vsag::Options::Instance().set_block_size_limit(origin_size);
+}