[BugFix] Fix error l0 memory usage for cloud native pindex (StarRocks…

…#55255) Signed-off-by: sevev <[email protected]>
vivo · Jan 23, 2025 · 9870b05 · 9870b05
1 parent a3566ca
commit 9870b05
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 2 deletions.
diff --git a/be/src/storage/lake/lake_persistent_index.cpp b/be/src/storage/lake/lake_persistent_index.cpp
@@ -172,8 +172,7 @@ void LakePersistentIndex::set_difference(KeyIndexSet* key_indexes, const KeyInde
 
 bool LakePersistentIndex::is_memtable_full() const {
     const auto memtable_mem_size = _memtable->memory_usage();
-    // We have two memtable in index, so memtable memory limit means half of `l0_max_mem_usage`.
-    const bool mem_size_exceed = memtable_mem_size >= config::l0_max_mem_usage / 2;
+    const bool mem_size_exceed = memtable_mem_size >= config::l0_max_mem_usage;
     // When update memory is urgent, using a lower limit (`l0_min_mem_usage`).
     const bool mem_tracker_exceed =
             _tablet_mgr->update_mgr()->mem_tracker()->limit_exceeded_by_ratio(config::memory_urgent_level) &&

diff --git a/be/test/storage/lake/lake_persistent_index_test.cpp b/be/test/storage/lake/lake_persistent_index_test.cpp
@@ -356,4 +356,33 @@ TEST_F(LakePersistentIndexTest, test_insert_delete) {
     config::l0_max_mem_usage = l0_max_mem_usage;
 }
 
+TEST_F(LakePersistentIndexTest, test_memtable_full) {
+    auto tablet_id = _tablet_metadata->id();
+    auto index = std::make_unique<LakePersistentIndex>(_tablet_mgr.get(), tablet_id);
+    ASSERT_OK(index->init(_tablet_metadata->sstable_meta()));
+
+    size_t old_l0_max_mem_usage = config::l0_max_mem_usage;
+    config::l0_max_mem_usage = 1073741824;
+    using Key = uint64_t;
+    vector<Key> keys;
+    vector<Slice> key_slices;
+    vector<IndexValue> values;
+    const int N = 10000;
+    keys.reserve(N);
+    key_slices.reserve(N);
+    for (int i = 0; i < N; i++) {
+        keys.emplace_back(i);
+        key_slices.emplace_back((uint8_t*)(&keys[i]), sizeof(Key));
+        values.emplace_back(i * 2);
+    }
+    ASSERT_OK(index->insert(N, key_slices.data(), values.data(), 0));
+
+    ASSERT_FALSE(index->is_memtable_full());
+    config::l0_max_mem_usage = index->memory_usage() + 1;
+    ASSERT_FALSE(index->is_memtable_full());
+    config::l0_max_mem_usage = index->memory_usage();
+    ASSERT_TRUE(index->is_memtable_full());
+    config::l0_max_mem_usage = old_l0_max_mem_usage;
+}
+
 } // namespace starrocks::lake