Skip to content

Commit

Permalink
Fixes for #168, #167, #166, #165. (#169)
Browse files Browse the repository at this point in the history
  • Loading branch information
@tonyastolfi
tonyastolfi authored Sep 13, 2024
1 parent 0df6f5f commit d7aa4b1
Show file tree
Hide file tree
Showing 16 changed files with 211 additions and 218 deletions.
6 changes: 3 additions & 3 deletions conanfile.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,10 +69,10 @@ def configure(self):


def requirements(self):
self.requires("batteries/0.55.1", **VISIBLE)
self.requires("boost/1.83.0", **VISIBLE)
self.requires("batteries/0.56.7", **VISIBLE)
self.requires("boost/1.85.0", **VISIBLE, **OVERRIDE)
self.requires("cli11/2.3.2", **VISIBLE)
self.requires("glog/0.6.0", **VISIBLE)
self.requires("glog/0.7.0", **VISIBLE, **OVERRIDE)
self.requires("gtest/1.14.0", **VISIBLE)
self.requires("libbacktrace/cci.20210118", **VISIBLE)
self.requires("openssl/3.2.0", **VISIBLE)
Expand Down
2 changes: 2 additions & 0 deletions src/llfs/logging.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ struct NullStream {
#define LLFS_LOG_WARNING() LLFS_LOG_NO_OUTPUT()
#define LLFS_LOG_INFO() LLFS_LOG_NO_OUTPUT()
#define LLFS_LOG_INFO_EVERY_N(n) LLFS_LOG_NO_OUTPUT()
#define LLFS_LOG_INFO_EVERY_T(t) LLFS_LOG_NO_OUTPUT()
#define LLFS_VLOG(verbosity) LLFS_LOG_NO_OUTPUT()
#define LLFS_PLOG_ERROR() LLFS_LOG_NO_OUTPUT()
#define LLFS_PLOG_WARNING() LLFS_LOG_NO_OUTPUT()
Expand All @@ -103,6 +104,7 @@ struct NullStream {
#define LLFS_LOG_WARNING() LOG(WARNING)
#define LLFS_LOG_INFO() LOG(INFO)
#define LLFS_LOG_INFO_EVERY_N(n) LOG_EVERY_N(INFO, (n))
#define LLFS_LOG_INFO_EVERY_T(t) LOG_EVERY_T(INFO, (t))
#define LLFS_VLOG(verbosity) VLOG((verbosity))
#define LLFS_PLOG_ERROR() PLOG(ERROR)
#define LLFS_PLOG_WARNING() PLOG(WARNING)
Expand Down
106 changes: 0 additions & 106 deletions src/llfs/mem_fuse.test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,119 +34,13 @@ namespace {

using namespace llfs::int_types;

namespace termxx {

namespace color {

constexpr auto Black = "\033[30m";
constexpr auto Red = "\033[31m";
constexpr auto Green = "\033[32m";
constexpr auto Yellow = "\033[33m";
constexpr auto Blue = "\033[34m";
constexpr auto Magenta = "\033[35m";
constexpr auto Cyan = "\033[36m";
constexpr auto White = "\033[37m";

} //namespace color

namespace fill {

constexpr auto Black = "\033[40m";
constexpr auto Red = "\033[41m";
constexpr auto Green = "\033[42m";
constexpr auto Yellow = "\033[43m";
constexpr auto Blue = "\033[44m";
constexpr auto Magenta = "\033[45m";
constexpr auto Cyan = "\033[46m";
constexpr auto White = "\033[47m";

} //namespace fill

namespace style {

constexpr auto Bold = "\033[1m";
constexpr auto BoldOff = "\033[21m";

constexpr auto Italic = "\033[3m";
constexpr auto ItalicOff = "\033[23m";

constexpr auto Underline = "\033[4m";
constexpr auto UnderlineOff = "\033[24m";

constexpr auto Inverse = "\033[7m";
constexpr auto InverseOff = "\033[27m";

} //namespace style

constexpr auto Reset = "\033[0m";

} //namespace termxx

void CustomPrefix(std::ostream& s, const google::LogMessageInfo& l, void*)
{
switch (l.severity[0]) {
case 'E':
s << termxx::color::Red;
break;

case 'W':
s << termxx::color::Yellow;
break;

case 'I':
s << termxx::color::Blue;
break;
}

s << l.severity //
<< termxx::Reset //
<< " [" //

//----- --- -- - - - -
<< termxx::style::Italic //
<< termxx::color::Magenta //
<< std::setw(4) << 1900 + l.time.year() //
<< "/" << std::setw(2) << 1 + l.time.month() //
<< "/" << std::setw(2) << l.time.day() //
<< termxx::style::ItalicOff

//----- --- -- - - - -
<< termxx::color::White << termxx::style::Bold //
<< ' ' << std::setw(2) << l.time.hour() //
<< ':' << std::setw(2) << l.time.min() //
<< ':' << std::setw(2) << l.time.sec() //
<< "." << std::setw(6) << l.time.usec() //
<< termxx::Reset //

//----- --- -- - - - -
<< termxx::color::Cyan //
<< ' ' << std::setfill(' ') << std::setw(5) << l.thread_id << std::setfill('0')
<< ' '

//----- --- -- - - - -
<< termxx::color::Blue << termxx::style::Underline //
<< l.filename << ':' << l.line_number //
<< termxx::Reset //
<< "]" //

//----- --- -- - - - -
<< termxx::Reset;
}

//=#=#==#==#===============+=+=+=+=++=++++++++++++++-++-+--+-+----+---------------
//
class MemFuseTest : public ::testing::Test
{
public:
void SetUp() override
{
// Initialize and configure logging.
//
if (false) {
google::InitGoogleLogging("llfs_Test", google::CustomPrefixCallback{CustomPrefix},
/*prefix_callback_data=*/nullptr);
}

// Enable task dumping signal handler.
//
batt::enable_dump_tasks();
Expand Down
2 changes: 1 addition & 1 deletion src/llfs/memory_page_arena.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ PageArena make_memory_page_arena(batt::TaskScheduler& scheduler, PageCount n_pag

return PageArena{
std::make_unique<MemoryPageDevice>(device_id, n_pages, page_size),
PageAllocator::recover_or_die(PageAllocatorRuntimeOptions{scheduler, name},
PageAllocator::recover_or_die(PageAllocatorRuntimeOptions{scheduler, name}, page_size,
PageIdFactory{n_pages, device_id},
*std::make_unique<MemoryLogDeviceFactory>(log_size))};
}
Expand Down
11 changes: 6 additions & 5 deletions src/llfs/page_allocator.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ u64 PageAllocator::calculate_log_size(u64 physical_page_count, u64 max_attachmen
//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - -
//
StatusOr<std::unique_ptr<PageAllocator>> PageAllocator::recover(
const PageAllocatorRuntimeOptions& runtime_options, const PageIdFactory& page_ids,
LogDeviceFactory& log_device_factory)
const PageAllocatorRuntimeOptions& runtime_options, PageSize page_size,
const PageIdFactory& page_ids, LogDeviceFactory& log_device_factory)
{
initialize_status_codes();

Expand Down Expand Up @@ -98,8 +98,8 @@ StatusOr<std::unique_ptr<PageAllocator>> PageAllocator::recover(
// Now we can create the PageAllocator.
//
auto page_allocator = std::unique_ptr<PageAllocator>{
new PageAllocator{runtime_options.scheduler, runtime_options.name, std::move(*recovered_log),
std::move(recovered_state)}};
new PageAllocator{runtime_options.scheduler, runtime_options.name, page_size,
std::move(*recovered_log), std::move(recovered_state)}};

// Set the recovering_users state.
//
Expand All @@ -122,10 +122,11 @@ StatusOr<std::unique_ptr<PageAllocator>> PageAllocator::recover(
//==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - -
//
PageAllocator::PageAllocator(batt::TaskScheduler& scheduler, std::string_view name,
std::unique_ptr<LogDevice> log_device,
PageSize page_size, std::unique_ptr<LogDevice> log_device,
std::unique_ptr<PageAllocator::State> recovered_state) noexcept
: name_{name}
, log_device_{std::move(log_device)}
, page_size_{page_size}
, state_{std::move(recovered_state)}
, checkpoint_task_{scheduler.schedule_task(),
[this] {
Expand Down
35 changes: 30 additions & 5 deletions src/llfs/page_allocator.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,14 +64,15 @@ class PageAllocator
static u64 calculate_log_size(u64 physical_page_count, u64 max_attachments);

static StatusOr<std::unique_ptr<PageAllocator>> recover(
const PageAllocatorRuntimeOptions& options, const PageIdFactory& page_ids,
const PageAllocatorRuntimeOptions& options, PageSize page_size, const PageIdFactory& page_ids,
LogDeviceFactory& log_device_factory);

static std::unique_ptr<PageAllocator> recover_or_die(const PageAllocatorRuntimeOptions& options,
PageSize page_size,
const PageIdFactory& page_ids,
LogDeviceFactory& log_device_factory)
{
auto result = PageAllocator::recover(options, page_ids, log_device_factory);
auto result = PageAllocator::recover(options, page_size, page_ids, log_device_factory);
BATT_CHECK(result.ok()) << result.status();
return std::move(*result);
}
Expand Down Expand Up @@ -166,16 +167,26 @@ class PageAllocator
auto debug_info()
{
return [this](std::ostream& out) {
out << "PageAllocator{.free=" << this->state_.no_lock().free_pool_size() << "/"
<< this->state_.no_lock().page_device_capacity() << ",}";
out << "PageAllocator{.page_size=" << batt::dump_size(this->page_size())
<< ", .free=" << this->free_pool_size() << "/" << this->page_device_capacity() << ",}";
};
}

u64 page_size() const
{
return this->page_size_;
}

u64 free_pool_size() const
{
return this->state_.no_lock().free_pool_size();
}

u64 page_device_capacity() const
{
return this->state_.no_lock().page_device_capacity();
}

page_device_id_int get_device_id() const
{
return this->state_.no_lock().page_ids().get_device_id();
Expand Down Expand Up @@ -207,7 +218,7 @@ class PageAllocator
}

private: //==#==========+==+=+=++=+++++++++++-+-+--+----- --- -- - - - -
explicit PageAllocator(batt::TaskScheduler& scheduler, std::string_view name,
explicit PageAllocator(batt::TaskScheduler& scheduler, std::string_view name, PageSize page_size,
std::unique_ptr<LogDevice> log_device,
std::unique_ptr<State> recovered_state) noexcept;

Expand All @@ -233,6 +244,10 @@ class PageAllocator
//
TypedSlotWriter<PackedPageAllocatorEvent> slot_writer_{*this->log_device_};

// The page size for the associated PageDevice.
//
const PageSize page_size_;

// This is the target state for the PageAllocator; it is used to validate incoming proposals
// (events) and determine which ones are invalid, valid but already processed, and valid in need
// of processing.
Expand Down Expand Up @@ -374,6 +389,16 @@ inline StatusOr<slot_offset_type> PageAllocator::update_page_ref_counts(

LLFS_VLOG(2) << "updating ref counts: " << batt::dump_range(txn->ref_counts, batt::Pretty::True);

static std::atomic<usize> sample_count{0};
static std::atomic<usize> prc_count{0};

sample_count.fetch_add(1);
prc_count.fetch_add(txn->ref_counts.size());

LLFS_LOG_INFO_EVERY_T(5.0 /*seconds*/)
<< "Average pages per allocator update: "
<< ((double)prc_count.load() / (double)sample_count.load());

StatusOr<slot_offset_type> update_status = this->update(*txn);
BATT_REQUIRE_OK(update_status);

Expand Down
34 changes: 21 additions & 13 deletions src/llfs/page_allocator.test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,8 @@ using llfs::testing::FakeLogDeviceFactory;
using llfs::testing::FakeLogDeviceReader;
using llfs::testing::FakeLogDeviceWriter;

const PageSize kTestPageSize{4096};

TEST(PageAllocatorTest, UpdateRefCounts)
{
constexpr u64 kNumPages = 100;
Expand All @@ -73,7 +75,7 @@ TEST(PageAllocatorTest, UpdateRefCounts)

StatusOr<std::unique_ptr<PageAllocator>> page_allocator = PageAllocator::recover(
PageAllocatorRuntimeOptions{batt::Runtime::instance().default_scheduler(), "Test"},
PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
kTestPageSize, PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
*std::make_unique<MemoryLogDeviceFactory>(
PageAllocator::calculate_log_size(kNumPages, kMaxAttachments)));

Expand Down Expand Up @@ -150,6 +152,8 @@ TEST(PageAllocatorTest, LogCrashRecovery)
constexpr u64 kMaxAttachments = 64;
constexpr u64 kNumSeeds = 1000;

const auto page_size = PageSize{4096};

std::uniform_int_distribution<usize> pick_page_count(1u, kNumPages);
std::uniform_int_distribution<usize> pick_client(0u, kMaxAttachments - 1u);
std::uniform_int_distribution<usize> pick_physical_page(0u, kNumPages - 1u);
Expand Down Expand Up @@ -194,7 +198,7 @@ TEST(PageAllocatorTest, LogCrashRecovery)
PageAllocatorRuntimeOptions{
/*TODO [tastolfi 2022-01-21] use fake*/ batt::Runtime::instance().default_scheduler(),
"Test"},
PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
page_size, PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
fake_log_factory);

ASSERT_TRUE(status_or_page_allocator.ok());
Expand Down Expand Up @@ -326,6 +330,7 @@ TEST(PageAllocatorTest, LogCrashRecovery)
/*TODO [tastolfi 2022-01-21] use fake*/ batt::Runtime::instance()
.default_scheduler(),
"Test"},
page_size,
PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
fake_recovered_log_factory);

Expand Down Expand Up @@ -382,6 +387,7 @@ TEST(PageAllocatorTest, LogCrashRecovery)
constexpr usize kNumPages = 8;
constexpr usize kMaxAttachments = 4;
constexpr i32 kMaxCrashCount = 3;

const usize kSimLogSize =
/*size=*/PageAllocator::calculate_log_size(/*physical_page_count=*/kNumPages,
/*max_attachments=*/kMaxAttachments);
Expand Down Expand Up @@ -616,7 +622,7 @@ class PageAllocatorModel
LLFS_VLOG(2) << "entered recover() " << BATT_INSPECT(this->context_.work_count().get_value());

StatusOr<std::unique_ptr<PageAllocator>> status_or_page_allocator = PageAllocator::recover(
PageAllocatorRuntimeOptions{this->fake_scheduler_, "Test"},
PageAllocatorRuntimeOptions{this->fake_scheduler_, "Test"}, kTestPageSize,
PageIdFactory{/*device_capacity=*/PageCount{kNumPages}, /*page_device_id=*/0},
*this->fake_log_factory_);

Expand Down Expand Up @@ -747,12 +753,13 @@ TEST(PageAllocatorTest, RefCountDeltaCheckpointSliceTrim)
llfs::MemoryLogDevice* p_mem_log = nullptr;

batt::StatusOr<std::unique_ptr<llfs::PageAllocator>> page_allocator_status =
llfs::PageAllocator::recover(
options, id_factory, *std::make_unique<llfs::BasicLogDeviceFactory>([&p_mem_log] {
auto mem_log = std::make_unique<llfs::MemoryLogDevice>(kLogSize);
p_mem_log = mem_log.get();
return mem_log;
}));
llfs::PageAllocator::recover(options, kTestPageSize, id_factory,
*std::make_unique<llfs::BasicLogDeviceFactory>([&p_mem_log] {
auto mem_log =
std::make_unique<llfs::MemoryLogDevice>(kLogSize);
p_mem_log = mem_log.get();
return mem_log;
}));

// Verify initial state.
//
Expand Down Expand Up @@ -831,7 +838,8 @@ TEST(PageAllocatorTest, RefCountDeltaCheckpointSliceTrim)
//
batt::StatusOr<std::unique_ptr<llfs::PageAllocator>> page_allocator2_status =
llfs::PageAllocator::recover(
options, id_factory, *std::make_unique<llfs::BasicLogDeviceFactory>([&snapshot] {
options, kTestPageSize, id_factory,
*std::make_unique<llfs::BasicLogDeviceFactory>([&snapshot] {
auto mem_log2 = std::make_unique<llfs::MemoryLogDevice>(kLogSize);
mem_log2->restore_snapshot(snapshot, llfs::LogReadMode::kDurable);
return mem_log2;
Expand Down Expand Up @@ -952,7 +960,7 @@ TEST(PageAllocatorTest, DeterministicDeadPage)
const boost::uuids::uuid user_B_id = llfs::random_uuid();

batt::StatusOr<std::unique_ptr<llfs::PageAllocator>> page_allocator_status =
llfs::PageAllocator::recover(options, id_factory,
llfs::PageAllocator::recover(options, kTestPageSize, id_factory,
*std::make_unique<llfs::MemoryLogDeviceFactory>(kLogSize));

// Verify initial state.
Expand Down Expand Up @@ -1163,7 +1171,7 @@ TEST(PageAllocatorTest, TooManyAttachments)
const llfs::PageIdFactory id_factory{llfs::PageCount{kNumPages}, /*device_id=*/0};

batt::StatusOr<std::unique_ptr<llfs::PageAllocator>> page_allocator_status =
llfs::PageAllocator::recover(options, id_factory,
llfs::PageAllocator::recover(options, kTestPageSize, id_factory,
*std::make_unique<llfs::MemoryLogDeviceFactory>(kLogSize));

// Verify initial state.
Expand Down Expand Up @@ -1262,7 +1270,7 @@ TEST(PageAllocatorTest, CancelAllocate)

for (usize seed = 0; seed < kNumRandomSeeds; ++seed) {
batt::StatusOr<std::unique_ptr<llfs::PageAllocator>> page_allocator_status =
llfs::PageAllocator::recover(options, id_factory,
llfs::PageAllocator::recover(options, kTestPageSize, id_factory,
*std::make_unique<llfs::MemoryLogDeviceFactory>(kLogSize));

ASSERT_TRUE(page_allocator_status.ok()) << BATT_INSPECT(page_allocator_status.status());
Expand Down
Loading

0 comments on commit d7aa4b1

Please sign in to comment.