Skip to content

Commit

Permalink
Fix UT
Browse files Browse the repository at this point in the history 
Signed-off-by: Jin Hai <[email protected]>
  • Loading branch information
@JinHai-CN
JinHai-CN committed Oct 20, 2024
1 parent e890557 commit 05e38e4
Show file tree
Hide file tree
Showing 5 changed files with 70 additions and 45 deletions.
9 changes: 8 additions & 1 deletion src/unit_test/storage/buffer/buffer_handle.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
// limitations under the License.

#include "gtest/gtest.h"

import base_test;
import stl;
import buffer_manager;
Expand All @@ -23,6 +24,8 @@ import global_resource_usage;
import infinity_context;
import persistence_manager;
import default_values;
import virtual_store;
import wal_manager;

using namespace infinity;

Expand All @@ -41,7 +44,11 @@ TEST_P(BufferHandleTest, test1) {
auto base_dir = MakeShared<String>(GetFullDataDir());
auto persistence_dir = MakeShared<String>(GetFullPersistDir());

UniquePtr<PersistenceManager> persistence_manager = MakeUnique<PersistenceManager>(*persistence_dir, *base_dir, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT);
UniquePtr<PersistenceManager> persistence_manager = MakeUnique<PersistenceManager>(*persistence_dir,
*base_dir,
DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT,
StorageType::kLocal,
StorageMode::kWritable);
BufferManager buffer_manager(memory_limit, base_dir, temp_dir, persistence_manager.get());

SizeT test_size1 = 512;
Expand Down
26 changes: 18 additions & 8 deletions src/unit_test/storage/buffer/buffer_manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import config;
import infinity_exception;
import persistence_manager;
import default_values;
import wal_manager;

using namespace infinity;

Expand Down Expand Up @@ -114,7 +115,8 @@ TEST_F(BufferManagerTest, cleanup_test) {

for (SizeT i = 0; i < file_num; ++i) {
auto file_name = MakeShared<String>(fmt::format("file_{}", i));
auto file_worker = MakeUnique<DataFileWorker>(data_dir_, temp_dir_, MakeShared<String>(""), file_name, file_size, buffer_mgr.persistence_manager());
auto file_worker =
MakeUnique<DataFileWorker>(data_dir_, temp_dir_, MakeShared<String>(""), file_name, file_size, buffer_mgr.persistence_manager());
auto *buffer_obj = buffer_mgr.AllocateBufferObject(std::move(file_worker));
buffer_objs.push_back(buffer_obj);
{
Expand Down Expand Up @@ -195,7 +197,11 @@ TEST_F(BufferManagerTest, varfile_test) {
SizeT buffer_size = 100;
SizeT file_num = 10;

SharedPtr<PersistenceManager> persistence_manager_ = MakeShared<PersistenceManager>(*persistence_dir_, *data_dir_, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT);
SharedPtr<PersistenceManager> persistence_manager_ = MakeShared<PersistenceManager>(*persistence_dir_,
*data_dir_,
DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT,
StorageType::kLocal,
StorageMode::kWritable);
BufferManager buffer_mgr(buffer_size, data_dir_, temp_dir_, persistence_manager_.get());
Vector<BufferObj *> buffer_objs;
for (SizeT i = 0; i < file_num; ++i) {
Expand Down Expand Up @@ -351,7 +357,7 @@ class BufferManagerParallelTest : public BufferManagerTest {
test_obj->Check(file_info);
}
}
// LOG_INFO(fmt::format("Test {} thread {} finished", test_i, thread_i));
// LOG_INFO(fmt::format("Test {} thread {} finished", test_i, thread_i));
}
};

Expand Down Expand Up @@ -401,15 +407,19 @@ class Test1Obj : public TestObj {

TEST_F(BufferManagerParallelTest, parallel_test1) {
for (int i = 0; i < 1; ++i) {
SharedPtr<PersistenceManager> persistence_manager_ = MakeShared<PersistenceManager>(*persistence_dir_, *data_dir_, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT);
SharedPtr<PersistenceManager> persistence_manager_ = MakeShared<PersistenceManager>(*persistence_dir_,
*data_dir_,
DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT,
StorageType::kLocal,
StorageMode::kWritable);
auto buffer_mgr = MakeUnique<BufferManager>(buffer_size, data_dir_, temp_dir_, persistence_manager_.get());
auto test1_obj = MakeUnique<Test1Obj>(avg_file_size, buffer_mgr.get(), data_dir_, temp_dir_);

Vector<FileInfo> file_infos;
for (SizeT i = 0; i < file_n; ++i) {
file_infos.emplace_back(i);
}
// LOG_INFO(fmt::format("Start parallel test1 {}", i));
// LOG_INFO(fmt::format("Start parallel test1 {}", i));
for (SizeT test_i = 0; test_i < test_n_; test_i++) {
Atomic<SizeT> finished_n = 0;
for (auto &file_info : file_infos) {
Expand All @@ -427,7 +437,7 @@ TEST_F(BufferManagerParallelTest, parallel_test1) {
EXPECT_EQ(buffer_mgr->memory_usage(), 0);
buffer_mgr->RemoveClean();

// LOG_INFO(fmt::format("Finished parallel test1 {}", i));
// LOG_INFO(fmt::format("Finished parallel test1 {}", i));
ResetDir();
}
}
Expand Down Expand Up @@ -490,7 +500,7 @@ TEST_F(BufferManagerParallelTest, parallel_test2) {
for (SizeT i = 0; i < file_n; ++i) {
file_infos.emplace_back(i);
}
// LOG_INFO(fmt::format("Start parallel test2 {}", i));
// LOG_INFO(fmt::format("Start parallel test2 {}", i));
for (SizeT test_i = 0; test_i < test_n_; test_i++) {
Atomic<SizeT> finished_n = 0;
for (auto &file_info : file_infos) {
Expand Down Expand Up @@ -533,7 +543,7 @@ TEST_F(BufferManagerParallelTest, parallel_test2) {
ASSERT_EQ(buffer_mgr->memory_usage(), 0);
buffer_mgr->RemoveClean();

// LOG_INFO(fmt::format("Finished parallel test2 {}", i));
// LOG_INFO(fmt::format("Finished parallel test2 {}", i));
ResetDir();
}
}
8 changes: 6 additions & 2 deletions src/unit_test/storage/buffer/buffer_obj.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@ import wal_manager;
import internal_types;
import persistence_manager;
import default_values;
import virtual_store;

using namespace infinity;

Expand Down Expand Up @@ -96,8 +97,11 @@ TEST_F(BufferObjTest, test1) {
auto base_dir = MakeShared<String>(GetFullDataDir());
auto persistence_dir = MakeShared<String>(data_dir + "/persistence");

UniquePtr<PersistenceManager> persistence_manager =
MakeUnique<PersistenceManager>(*persistence_dir, *base_dir, DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT);
UniquePtr<PersistenceManager> persistence_manager = MakeUnique<PersistenceManager>(*persistence_dir,
*base_dir,
DEFAULT_PERSISTENCE_OBJECT_SIZE_LIMIT,
StorageType::kLocal,
StorageMode::kWritable);
BufferManager buffer_manager(memory_limit, base_dir, temp_dir, persistence_manager.get());

SizeT test_size1 = 1024;
Expand Down
16 changes: 9 additions & 7 deletions src/unit_test/storage/persistence/persistence_manager.cpp
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
#include "gtest/gtest.h"

import base_test;
import stl;
import persistence_manager;
import virtual_store;
import third_party;
import persist_result_handler;
import local_file_handle;
import wal_manager;

using namespace infinity;
namespace fs = std::filesystem;
Expand All @@ -18,10 +20,10 @@ class PersistenceManagerTest : public BaseTest {
file_dir_ = String(GetFullTmpDir()) + "/persistence_src";
system(("mkdir -p " + workspace_).c_str());
system(("mkdir -p " + file_dir_).c_str());
pm_ = MakeUnique<PersistenceManager>(workspace_, file_dir_, ObjSizeLimit);
pm_ = MakeUnique<PersistenceManager>(workspace_, file_dir_, ObjSizeLimit, StorageType::kLocal, StorageMode::kWritable);
handler_ = MakeUnique<PersistResultHandler>(pm_.get());
}
void CheckObjData(const String& obj_addr, const String& data);
void CheckObjData(const String &obj_addr, const String &data);

protected:
String workspace_{};
Expand All @@ -31,7 +33,7 @@ class PersistenceManagerTest : public BaseTest {
UniquePtr<PersistResultHandler> handler_;
};

void PersistenceManagerTest::CheckObjData(const String& local_file_path, const String& data) {
void PersistenceManagerTest::CheckObjData(const String &local_file_path, const String &data) {
PersistReadResult result = pm_->GetObjCache(local_file_path);
const ObjAddr &obj_addr = handler_->HandleReadResult(result);
String obj_path = pm_->GetObjPath(obj_addr.obj_key_);
Expand Down Expand Up @@ -130,7 +132,7 @@ TEST_F(PersistenceManagerTest, PersistFileMultiThread) {
obj_addrs[file_path] = obj_addr;
});
}
for (auto& thread : threads) {
for (auto &thread : threads) {
thread.join();
}
ASSERT_EQ(file_paths.size(), persist_strs.size());
Expand Down Expand Up @@ -176,19 +178,19 @@ TEST_F(PersistenceManagerTest, CleanupBasic) {
CheckObjData(file_paths[i], persist_strs[i]);
}

for (const auto& obj_path : obj_paths) {
for (const auto &obj_path : obj_paths) {
ASSERT_TRUE(fs::exists(obj_path));
}

std::random_device rd;
std::mt19937 g(rd());
std::shuffle(file_paths.begin(), file_paths.end(), g);

for (auto& file_path : file_paths) {
for (auto &file_path : file_paths) {
PersistWriteResult result = pm_->Cleanup(file_path);
handler_->HandleWriteResult(result);
}
for (const auto& obj_path : obj_paths) {
for (const auto &obj_path : obj_paths) {
ASSERT_FALSE(fs::exists(obj_path));
}
}
56 changes: 29 additions & 27 deletions src/unit_test/storage/wal/wal_entry.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,8 @@ import statement_common;
import data_type;
import persistence_manager;
import embedding_info;
import wal_manager;
import virtual_store;

using namespace infinity;

Expand Down Expand Up @@ -299,7 +301,7 @@ TEST_F(WalEntryTest, ReadWriteVFS) {
String workspace = GetFullPersistDir();
String data_dir = GetFullDataDir();
SizeT object_size_limit = 100;
PersistenceManager pm(workspace, data_dir, object_size_limit);
PersistenceManager pm(workspace, data_dir, object_size_limit, StorageType::kLocal, StorageMode::kWritable);
ObjAddr obj_addr0{.obj_key_ = "key1", .part_offset_ = 0, .part_size_ = 10};
ObjAddr obj_addr1{.obj_key_ = "key1", .part_offset_ = 10, .part_size_ = 20};
pm.SaveLocalPath(paths[0], obj_addr0);
Expand Down Expand Up @@ -340,10 +342,10 @@ TEST_F(WalEntryTest, WalEntryIterator) {
if (wal_entry == nullptr) {
break;
}
// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_));
// for (const auto &cmd : wal_entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_));
// for (const auto &cmd : wal_entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
}
}

Expand All @@ -366,8 +368,8 @@ TEST_F(WalEntryTest, WalEntryIterator) {
max_commit_ts = checkpoint_cmd->max_commit_ts_;
catalog_path = checkpoint_cmd->catalog_path_;

// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts));
// Println("Catalog Path: {}", catalog_path);
// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts));
// Println("Catalog Path: {}", catalog_path);
break;
}
}
Expand All @@ -385,13 +387,13 @@ TEST_F(WalEntryTest, WalEntryIterator) {
}

// phase 3: replay the entries
// Println("Start to replay the entries", "");
// for (const auto &entry : replay_entries) {
// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_));
// for (const auto &cmd : entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// }
// Println("Start to replay the entries", "");
// for (const auto &entry : replay_entries) {
// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_));
// for (const auto &cmd : entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// }
EXPECT_EQ(max_commit_ts, 123ul);
EXPECT_EQ(catalog_path, String("catalog"));
EXPECT_EQ(replay_entries.size(), 1u);
Expand All @@ -415,10 +417,10 @@ TEST_F(WalEntryTest, WalListIterator) {
if (wal_entry.get() == nullptr) {
break;
}
// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_));
// for (const auto &cmd : wal_entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// Println("WAL ENTRY COMMIT TS:", std::to_string(wal_entry->commit_ts_));
// for (const auto &cmd : wal_entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
}

Vector<SharedPtr<WalEntry>> replay_entries;
Expand All @@ -440,8 +442,8 @@ TEST_F(WalEntryTest, WalListIterator) {
max_commit_ts = checkpoint_cmd->max_commit_ts_;
catalog_path = checkpoint_cmd->catalog_path_;

// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts));
// Println("Catalog Path: {}", catalog_path);
// Println("Checkpoint Max Commit Ts: {}", std::to_string(max_commit_ts));
// Println("Catalog Path: {}", catalog_path);
break;
}
}
Expand All @@ -459,13 +461,13 @@ TEST_F(WalEntryTest, WalListIterator) {
}

// phase 3: replay the entries
// Println("Start to replay the entries", "");
// for (const auto &entry : replay_entries) {
// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_));
// for (const auto &cmd : entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// }
// Println("Start to replay the entries", "");
// for (const auto &entry : replay_entries) {
// Println("WAL ENTRY COMMIT TS:", std::to_string(entry->commit_ts_));
// for (const auto &cmd : entry->cmds_) {
// Println(" WAL CMD: ", WalCmd::WalCommandTypeToString(cmd->GetType()));
// }
// }
EXPECT_EQ(max_commit_ts, 123ul);
EXPECT_EQ(catalog_path, ckp_file_path);
EXPECT_EQ(replay_entries.size(), 1u);
Expand Down

0 comments on commit 05e38e4

Please sign in to comment.