Skip to content

Commit

Permalink
Fix sharded
Browse files Browse the repository at this point in the history
  • Loading branch information
@liyunfan1223
liyunfan1223 committed Feb 26, 2024
1 parent f544651 commit 3bddecc
Show file tree
Hide file tree
Showing 6 changed files with 114 additions and 120 deletions.
22 changes: 12 additions & 10 deletions cache/hill.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,7 @@ class HillSubReplacer {

if (top_lru_.size() >= lru_size_) {
std::string& _key = top_lru_.back();
auto &rm_entry = real_map_[_key];
auto& rm_entry = real_map_[_key];
HillHandle* oldH = rm_entry.h_;
int lvl = rm_entry.insert_level;
real_lru_[lvl].push_front(_key);
Expand Down Expand Up @@ -429,9 +429,10 @@ class HillSubReplacer {
continue;
}
// MRU
for (auto iter = real_lru_[i].begin(); iter != real_lru_[i].end() && attempts; iter++, attempts--) {
std::string &key = *iter;
auto &entry = real_map_[key];
for (auto iter = real_lru_[i].begin();
iter != real_lru_[i].end() && attempts; iter++, attempts--) {
std::string& key = *iter;
auto& entry = real_map_[key];
if (entry.h_->HasRefs()) {
continue;
}
Expand Down Expand Up @@ -487,13 +488,14 @@ class HillSubReplacer {
} else {
// evict item in top list
int attempts = 15;
std::string evict_key; // = top_lru_.back();
int evict_level{-1}; // = real_map_[evict_key].insert_level;
std::string evict_key; // = top_lru_.back();
int evict_level{-1}; // = real_map_[evict_key].insert_level;
std::list<std::string>::iterator evicted_iter;
assert(top_lru_.size());
for (auto iter = top_lru_.rbegin(); iter != top_lru_.rend() && attempts; iter++, attempts--) {
std::string &key = *iter;
auto &entry = real_map_[key];
for (auto iter = top_lru_.rbegin(); iter != top_lru_.rend() && attempts;
iter++, attempts--) {
std::string& key = *iter;
auto& entry = real_map_[key];
if (entry.h_->HasRefs()) {
continue;
}
Expand Down Expand Up @@ -771,7 +773,7 @@ class HillReplacer : public Replacer {
//// assert(replacer_r_.Get(key) == nullptr);
//// assert(replacer_s_.Get(key) == nullptr);
}
void Touch(const std::string& key, HillHandle *e) {
void Touch(const std::string& key, HillHandle* e) {
// std::cout << "touch";
// replacer_r_.Touch(key);
replacer_r_.Access(key, e);
Expand Down
71 changes: 31 additions & 40 deletions cache/hill_test.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
#include <vector>

#include "cache/lru_cache.h"
#include "cache/sharded_hill_cache.h"
#include "cache/typed_cache.h"
#include "port/stack_trace.h"
#include "rocksdb/cache.h"
Expand All @@ -24,7 +25,6 @@
#include "util/hash_containers.h"
#include "util/string_util.h"
#include "util/thread_operation.h"
#include "cache/sharded_hill_cache.h"

namespace ROCKSDB_NAMESPACE {

Expand Down Expand Up @@ -884,7 +884,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG2) {
std::to_string(i) + randomString);
}
db->Flush(FlushOptions());

uint64_t block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
uint64_t block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
Expand All @@ -895,16 +895,15 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG2) {
(block_cache_hits + block_cache_misses)))
<< " hit: " << block_cache_hits << " miss: " << block_cache_misses
<< "\n";
std::cout << "Start reading" << "\n";
std::cout << "Start reading"
<< "\n";
stats->Reset();
for (size_t i = 0; i < access_num; i++) {
int idx = ord[i];
s = db->Get(ReadOptions(), std::to_string(idx), &value);
if (i % 10000 == 0) {
block_cache_hits =
stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand All @@ -915,8 +914,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG2) {
}
}
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand Down Expand Up @@ -980,16 +978,15 @@ TEST_P(CacheTest, InRocksDBZipfDistributeLRUBIG2) {
(block_cache_hits + block_cache_misses)))
<< " hit: " << block_cache_hits << " miss: " << block_cache_misses
<< "\n";
std::cout << "Start reading" << "\n";
std::cout << "Start reading"
<< "\n";
stats->Reset();
for (size_t i = 0; i < access_num; i++) {
int idx = ord[i];
s = db->Get(ReadOptions(), std::to_string(idx), &value);
if (i % 10000 == 0) {
block_cache_hits =
stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "LRU Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand All @@ -1000,8 +997,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeLRUBIG2) {
}
}
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "LRU Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand Down Expand Up @@ -1056,7 +1052,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG3) {
std::to_string(i) + randomString);
}
db->Flush(FlushOptions());

uint64_t block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
uint64_t block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
Expand All @@ -1067,16 +1063,15 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG3) {
(block_cache_hits + block_cache_misses)))
<< " hit: " << block_cache_hits << " miss: " << block_cache_misses
<< "\n";
std::cout << "Start reading" << "\n";
std::cout << "Start reading"
<< "\n";
stats->Reset();
for (size_t i = 0; i < access_num; i++) {
int idx = ord[i];
s = db->Get(ReadOptions(), std::to_string(idx), &value);
if (i % 10000 == 0) {
block_cache_hits =
stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand All @@ -1087,8 +1082,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeHillBIG3) {
}
}
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand Down Expand Up @@ -1152,16 +1146,15 @@ TEST_P(CacheTest, InRocksDBZipfDistributeLRUBIG3) {
(block_cache_hits + block_cache_misses)))
<< " hit: " << block_cache_hits << " miss: " << block_cache_misses
<< "\n";
std::cout << "Start reading" << "\n";
std::cout << "Start reading"
<< "\n";
stats->Reset();
for (size_t i = 0; i < access_num; i++) {
int idx = ord[i];
s = db->Get(ReadOptions(), std::to_string(idx), &value);
if (i % 10000 == 0) {
block_cache_hits =
stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "LRU Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand All @@ -1172,8 +1165,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeLRUBIG3) {
}
}
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "LRU Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand Down Expand Up @@ -1225,9 +1217,10 @@ TEST_P(CacheTest, InRocksDBZipfDistributeShardHill) {
s = db->Put(WriteOptions(), std::to_string(i),
std::to_string(i) + randomString);
}
std::cout << "Start flushing" << "\n";
std::cout << "Start flushing"
<< "\n";
db->Flush(FlushOptions());

uint64_t block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
uint64_t block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
Expand All @@ -1238,16 +1231,15 @@ TEST_P(CacheTest, InRocksDBZipfDistributeShardHill) {
(block_cache_hits + block_cache_misses)))
<< " hit: " << block_cache_hits << " miss: " << block_cache_misses
<< "\n";
std::cout << "Start reading" << "\n";
std::cout << "Start reading"
<< "\n";
stats->Reset();
for (size_t i = 0; i < access_num; i++) {
int idx = ord[i];
s = db->Get(ReadOptions(), std::to_string(idx), &value);
if (i % 10000 == 0) {
block_cache_hits =
stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand All @@ -1258,8 +1250,7 @@ TEST_P(CacheTest, InRocksDBZipfDistributeShardHill) {
}
}
block_cache_hits = stats->getTickerCount(rocksdb::BLOCK_CACHE_HIT);
block_cache_misses =
stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
block_cache_misses = stats->getTickerCount(rocksdb::BLOCK_CACHE_MISS);
std::cout << "Hill Block Cache Hit rate: "
<< ((block_cache_hits + block_cache_misses == 0)
? 0
Expand Down
1 change: 0 additions & 1 deletion cache/sharded_cache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@

namespace ROCKSDB_NAMESPACE {


// Optional base class for classes implementing the CacheShard concept
class CacheShardBase {
public:
Expand Down
6 changes: 2 additions & 4 deletions cache/sharded_hill_cache.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
// //

#include "cache/sharded_hill_cache.h"
#include "cache/hill.h"

#include "cache/hill.h"
#include "rocksdb/cache.h"

namespace ROCKSDB_NAMESPACE {
Expand All @@ -14,9 +14,7 @@ std::shared_ptr<Cache> ShardedHillCacheOptions::MakeHillCache() const {
if (opt.num_shard_bits < 0) {
opt.num_shard_bits = 2;
}
auto hill =
std::make_shared<sharded_hill_cache::ShardedHillCache>(
opt);
auto hill = std::make_shared<sharded_hill_cache::ShardedHillCache>(opt);
return hill;
}

Expand Down
Loading

0 comments on commit 3bddecc

Please sign in to comment.