From 05e38e4df3cf763a9fd4c0fb4a470ea144ce4cbb Mon Sep 17 00:00:00 2001 From: Jin Hai Date: Sun, 20 Oct 2024 11:56:05 +0800 Subject: [PATCH] Fix UT Signed-off-by: Jin Hai --- .../storage/buffer/buffer_handle.cpp | 9 ++- .../storage/buffer/buffer_manager.cpp | 26 ++++++--- src/unit_test/storage/buffer/buffer_obj.cpp | 8 ++- .../persistence/persistence_manager.cpp | 16 +++--- src/unit_test/storage/wal/wal_entry.cpp | 56 ++++++++++--------- 5 files changed, 70 insertions(+), 45 deletions(-) diff --git a/src/unit_test/storage/buffer/buffer_handle.cpp b/src/unit_test/storage/buffer/buffer_handle.cpp index ef722225b4..c59a577b9c 100644 --- a/src/unit_test/storage/buffer/buffer_handle.cpp +++ b/src/unit_test/storage/buffer/buffer_handle.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "gtest/gtest.h" + import base_test; import stl; import buffer_manager; @@ -23,6 +24,8 @@ import global_resource_usage; import infinity_context; import persistence_manager; import default_values; +import virtual_store; +import wal_manager; using namespace infinity; @@ -41,7 +44,11 @@ TEST_P(BufferHandleTest, test1) { auto base_dir = MakeShared(GetFullDataDir()); auto persistence_dir = MakeShared(GetFullPersistDir()); - UniquePtr persistence_manager = MakeUnique(*persistence_dir, *base_dir, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT); + UniquePtr persistence_manager = MakeUnique(*persistence_dir, + *base_dir, + DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT, + StorageType::kLocal, + StorageMode::kWritable); BufferManager buffer_manager(memory_limit, base_dir, temp_dir, persistence_manager.get()); SizeT test_size1 = 512; diff --git a/src/unit_test/storage/buffer/buffer_manager.cpp b/src/unit_test/storage/buffer/buffer_manager.cpp index 0809454274..bcc2f3db11 100644 --- a/src/unit_test/storage/buffer/buffer_manager.cpp +++ b/src/unit_test/storage/buffer/buffer_manager.cpp @@ -32,6 +32,7 @@ import config; import infinity_exception; import persistence_manager; import default_values; +import wal_manager; using namespace infinity; @@ -114,7 +115,8 @@ TEST_F(BufferManagerTest, cleanup_test) { for (SizeT i = 0; i < file_num; ++i) { auto file_name = MakeShared(fmt::format("file_{}", i)); - auto file_worker = MakeUnique(data_dir_, temp_dir_, MakeShared(""), file_name, file_size, buffer_mgr.persistence_manager()); + auto file_worker = + MakeUnique(data_dir_, temp_dir_, MakeShared(""), file_name, file_size, buffer_mgr.persistence_manager()); auto *buffer_obj = buffer_mgr.AllocateBufferObject(std::move(file_worker)); buffer_objs.push_back(buffer_obj); { @@ -195,7 +197,11 @@ TEST_F(BufferManagerTest, varfile_test) { SizeT buffer_size = 100; SizeT file_num = 10; - SharedPtr persistence_manager_ = MakeShared(*persistence_dir_, *data_dir_, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT); + SharedPtr persistence_manager_ = MakeShared(*persistence_dir_, + *data_dir_, + DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT, + StorageType::kLocal, + StorageMode::kWritable); BufferManager buffer_mgr(buffer_size, data_dir_, temp_dir_, persistence_manager_.get()); Vector buffer_objs; for (SizeT i = 0; i < file_num; ++i) { @@ -351,7 +357,7 @@ class BufferManagerParallelTest : public BufferManagerTest { test_obj->Check(file_info); } } -// LOG_INFO(fmt::format("Test {} thread {} finished", test_i, thread_i)); + // LOG_INFO(fmt::format("Test {} thread {} finished", test_i, thread_i)); } }; @@ -401,7 +407,11 @@ class Test1Obj : public TestObj { TEST_F(BufferManagerParallelTest, parallel_test1) { for (int i = 0; i < 1; ++i) { - SharedPtr persistence_manager_ = MakeShared(*persistence_dir_, *data_dir_, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT); + SharedPtr persistence_manager_ = MakeShared(*persistence_dir_, + *data_dir_, + DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT, + StorageType::kLocal, + StorageMode::kWritable); auto buffer_mgr = MakeUnique(buffer_size, data_dir_, temp_dir_, persistence_manager_.get()); auto test1_obj = MakeUnique(avg_file_size, buffer_mgr.get(), data_dir_, temp_dir_); @@ -409,7 +419,7 @@ TEST_F(BufferManagerParallelTest, parallel_test1) { for (SizeT i = 0; i < file_n; ++i) { file_infos.emplace_back(i); } -// LOG_INFO(fmt::format("Start parallel test1 {}", i)); + // LOG_INFO(fmt::format("Start parallel test1 {}", i)); for (SizeT test_i = 0; test_i < test_n_; test_i++) { Atomic finished_n = 0; for (auto &file_info : file_infos) { @@ -427,7 +437,7 @@ TEST_F(BufferManagerParallelTest, parallel_test1) { EXPECT_EQ(buffer_mgr->memory_usage(), 0); buffer_mgr->RemoveClean(); -// LOG_INFO(fmt::format("Finished parallel test1 {}", i)); + // LOG_INFO(fmt::format("Finished parallel test1 {}", i)); ResetDir(); } } @@ -490,7 +500,7 @@ TEST_F(BufferManagerParallelTest, parallel_test2) { for (SizeT i = 0; i < file_n; ++i) { file_infos.emplace_back(i); } -// LOG_INFO(fmt::format("Start parallel test2 {}", i)); + // LOG_INFO(fmt::format("Start parallel test2 {}", i)); for (SizeT test_i = 0; test_i < test_n_; test_i++) { Atomic finished_n = 0; for (auto &file_info : file_infos) { @@ -533,7 +543,7 @@ TEST_F(BufferManagerParallelTest, parallel_test2) { ASSERT_EQ(buffer_mgr->memory_usage(), 0); buffer_mgr->RemoveClean(); -// LOG_INFO(fmt::format("Finished parallel test2 {}", i)); + // LOG_INFO(fmt::format("Finished parallel test2 {}", i)); ResetDir(); } } diff --git a/src/unit_test/storage/buffer/buffer_obj.cpp b/src/unit_test/storage/buffer/buffer_obj.cpp index 7122c61467..7b1baa3acf 100644 --- a/src/unit_test/storage/buffer/buffer_obj.cpp +++ b/src/unit_test/storage/buffer/buffer_obj.cpp @@ -55,6 +55,7 @@ import wal_manager; import internal_types; import persistence_manager; import default_values; +import virtual_store; using namespace infinity; @@ -96,8 +97,11 @@ TEST_F(BufferObjTest, test1) { auto base_dir = MakeShared(GetFullDataDir()); auto persistence_dir = MakeShared(data_dir + "/persistence"); - UniquePtr persistence_manager = - MakeUnique(*persistence_dir, *base_dir, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT); + UniquePtr persistence_manager = MakeUnique(*persistence_dir, + *base_dir, + DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT, + StorageType::kLocal, + StorageMode::kWritable); BufferManager buffer_manager(memory_limit, base_dir, temp_dir, persistence_manager.get()); SizeT test_size1 = 1024; diff --git a/src/unit_test/storage/persistence/persistence_manager.cpp b/src/unit_test/storage/persistence/persistence_manager.cpp index f1350b85c2..894158bca2 100644 --- a/src/unit_test/storage/persistence/persistence_manager.cpp +++ b/src/unit_test/storage/persistence/persistence_manager.cpp @@ -1,4 +1,5 @@ #include "gtest/gtest.h" + import base_test; import stl; import persistence_manager; @@ -6,6 +7,7 @@ import virtual_store; import third_party; import persist_result_handler; import local_file_handle; +import wal_manager; using namespace infinity; namespace fs = std::filesystem; @@ -18,10 +20,10 @@ class PersistenceManagerTest : public BaseTest { file_dir_ = String(GetFullTmpDir()) + "/persistence_src"; system(("mkdir -p " + workspace_).c_str()); system(("mkdir -p " + file_dir_).c_str()); - pm_ = MakeUnique(workspace_, file_dir_, ObjSizeLimit); + pm_ = MakeUnique(workspace_, file_dir_, ObjSizeLimit, StorageType::kLocal, StorageMode::kWritable); handler_ = MakeUnique(pm_.get()); } - void CheckObjData(const String& obj_addr, const String& data); + void CheckObjData(const String &obj_addr, const String &data); protected: String workspace_{}; @@ -31,7 +33,7 @@ class PersistenceManagerTest : public BaseTest { UniquePtr handler_; }; -void PersistenceManagerTest::CheckObjData(const String& local_file_path, const String& data) { +void PersistenceManagerTest::CheckObjData(const String &local_file_path, const String &data) { PersistReadResult result = pm_->GetObjCache(local_file_path); const ObjAddr &obj_addr = handler_->HandleReadResult(result); String obj_path = pm_->GetObjPath(obj_addr.obj_key_); @@ -130,7 +132,7 @@ TEST_F(PersistenceManagerTest, PersistFileMultiThread) { obj_addrs[file_path] = obj_addr; }); } - for (auto& thread : threads) { + for (auto &thread : threads) { thread.join(); } ASSERT_EQ(file_paths.size(), persist_strs.size()); @@ -176,7 +178,7 @@ TEST_F(PersistenceManagerTest, CleanupBasic) { CheckObjData(file_paths[i], persist_strs[i]); } - for (const auto& obj_path : obj_paths) { + for (const auto &obj_path : obj_paths) { ASSERT_TRUE(fs::exists(obj_path)); } @@ -184,11 +186,11 @@ TEST_F(PersistenceManagerTest, CleanupBasic) { std::mt19937 g(rd()); std::shuffle(file_paths.begin(), file_paths.end(), g); - for (auto& file_path : file_paths) { + for (auto &file_path : file_paths) { PersistWriteResult result = pm_->Cleanup(file_path); handler_->HandleWriteResult(result); } - for (const auto& obj_path : obj_paths) { + for (const auto &obj_path : obj_paths) { ASSERT_FALSE(fs::exists(obj_path)); } } \ No newline at end of file diff --git a/src/unit_test/storage/wal/wal_entry.cpp b/src/unit_test/storage/wal/wal_entry.cpp index 08d0fdc82d..074edb73f9 100644 --- a/src/unit_test/storage/wal/wal_entry.cpp +++ b/src/unit_test/storage/wal/wal_entry.cpp @@ -37,6 +37,8 @@ import statement_common; import data_type; import persistence_manager; import embedding_info; +import wal_manager; +import virtual_store; using namespace infinity; @@ -299,7 +301,7 @@ TEST_F(WalEntryTest, ReadWriteVFS) { String workspace = GetFullPersistDir(); String data_dir = GetFullDataDir(); SizeT object_size_limit = 100; - PersistenceManager pm(workspace, data_dir, object_size_limit); + PersistenceManager pm(workspace, data_dir, object_size_limit, StorageType::kLocal, StorageMode::kWritable); ObjAddr obj_addr0{.obj_key_ = "key1", .part_offset_ = 0, .part_size_ = 10}; ObjAddr obj_addr1{.obj_key_ = "key1", .part_offset_ = 10, .part_size_ = 20}; pm.SaveLocalPath(paths[0], obj_addr0); @@ -340,10 +342,10 @@ TEST_F(WalEntryTest, WalEntryIterator) { if (wal_entry == nullptr) { break; } -// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_)); -// for (const auto &cmd : wal_entry->cmds_) { -// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); -// } + // Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_)); + // for (const auto &cmd : wal_entry->cmds_) { + // Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); + // } } } @@ -366,8 +368,8 @@ TEST_F(WalEntryTest, WalEntryIterator) { max_commit_ts = checkpoint_cmd->max_commit_ts_; catalog_path = checkpoint_cmd->catalog_path_; -// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts)); -// Println("Catalog Path: {}", catalog_path); + // Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts)); + // Println("Catalog Path: {}", catalog_path); break; } } @@ -385,13 +387,13 @@ TEST_F(WalEntryTest, WalEntryIterator) { } // phase 3: replay the entries -// Println("Start to replay the entries", ""); -// for (const auto &entry : replay_entries) { -// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_)); -// for (const auto &cmd : entry->cmds_) { -// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); -// } -// } + // Println("Start to replay the entries", ""); + // for (const auto &entry : replay_entries) { + // Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_)); + // for (const auto &cmd : entry->cmds_) { + // Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); + // } + // } EXPECT_EQ(max_commit_ts, 123ul); EXPECT_EQ(catalog_path, String("catalog")); EXPECT_EQ(replay_entries.size(), 1u); @@ -415,10 +417,10 @@ TEST_F(WalEntryTest, WalListIterator) { if (wal_entry.get() == nullptr) { break; } -// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_)); -// for (const auto &cmd : wal_entry->cmds_) { -// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); -// } + // Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_)); + // for (const auto &cmd : wal_entry->cmds_) { + // Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); + // } } Vector> replay_entries; @@ -440,8 +442,8 @@ TEST_F(WalEntryTest, WalListIterator) { max_commit_ts = checkpoint_cmd->max_commit_ts_; catalog_path = checkpoint_cmd->catalog_path_; -// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts)); -// Println("Catalog Path: {}", catalog_path); + // Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts)); + // Println("Catalog Path: {}", catalog_path); break; } } @@ -459,13 +461,13 @@ TEST_F(WalEntryTest, WalListIterator) { } // phase 3: replay the entries -// Println("Start to replay the entries", ""); -// for (const auto &entry : replay_entries) { -// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_)); -// for (const auto &cmd : entry->cmds_) { -// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); -// } -// } + // Println("Start to replay the entries", ""); + // for (const auto &entry : replay_entries) { + // Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_)); + // for (const auto &cmd : entry->cmds_) { + // Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType())); + // } + // } EXPECT_EQ(max_commit_ts, 123ul); EXPECT_EQ(catalog_path, ckp_file_path); EXPECT_EQ(replay_entries.size(), 1u);