Skip to content

Commit

Permalink
node: fix corner case errors in snapshot decompressor (#1793)
Browse files Browse the repository at this point in the history 
node: add unit tests
  • Loading branch information
@canepat
canepat authored Feb 1, 2024
1 parent 401c608 commit 2ecfc88
Show file tree
Hide file tree
Showing 2 changed files with 43 additions and 10 deletions.
23 changes: 16 additions & 7 deletions silkworm/node/snapshots/seg/decompressor.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,9 @@ std::size_t PatternTable::build_condensed(std::span<Pattern> patterns, uint64_t
const auto last_cw = insert_word(codewords_list_.back().get());
return last_cw->table()->build_condensed(patterns, highest_depth, 0, 0, depth);
}
if (highest_depth == 0) {
throw std::runtime_error("cannot build pattern tree: highest_depth reached zero");
}
const auto b0 = build_condensed(patterns, highest_depth - 1, code, bits + 1, depth + 1);
return b0 + build_condensed(patterns.subspan(b0), highest_depth - 1, static_cast<uint16_t>((1 << bits) | code), bits + 1, depth + 1);
}
Expand Down Expand Up @@ -273,6 +276,9 @@ int PositionTable::build_tree(std::span<Position> positions, uint64_t highest_de
children_[code] = std::move(child_table);
return children_[code]->build_tree(positions, highest_depth, 0, 0, depth);
}
if (highest_depth == 0) {
throw std::runtime_error("cannot build position tree: highest_depth reached zero");
}
const int b0 = build_tree(positions, highest_depth - 1, code, bits + 1, depth + 1);
return b0 + build_tree(positions.subspan(static_cast<std::size_t>(b0)), highest_depth - 1, static_cast<uint16_t>((1 << bits) | code), bits + 1, depth + 1);
}
Expand Down Expand Up @@ -409,7 +415,7 @@ void Decompressor::read_patterns(ByteView dict) {
SILK_TRACE << "Pattern count: " << patterns.size() << " highest depth: " << pattern_highest_depth;

pattern_dict_ = std::make_unique<PatternTable>(pattern_highest_depth);
if (dict.length() > 0) {
if (!dict.empty()) {
pattern_dict_->build_condensed({patterns.data(), patterns.size()});
}

Expand Down Expand Up @@ -465,7 +471,7 @@ void Decompressor::read_positions(ByteView dict) {
SILK_TRACE << "Position count: " << positions.size() << " highest depth: " << position_highest_depth;

position_dict_ = std::make_unique<PositionTable>(position_highest_depth);
if (dict.length() > 0) {
if (!dict.empty()) {
position_dict_->build({positions.data(), positions.size()});
}

Expand Down Expand Up @@ -732,6 +738,12 @@ ByteView Decompressor::Iterator::next_pattern() {
const PatternTable* table = decoder_->pattern_dict_.get();
if (table->bit_length() == 0) {
return table->codeword(0)->pattern();
const auto* codeword{table->codeword(0)};
if (codeword == nullptr) {
throw std::runtime_error{
"Unexpected missing codeword for code: 0 in snapshot: " + decoder_->compressed_path().string()};
}
return codeword->pattern();
}
uint8_t length{0};
ByteView pattern{};
Expand All @@ -740,11 +752,8 @@ ByteView Decompressor::Iterator::next_pattern() {

const auto* codeword{table->search_condensed(code)};
if (codeword == nullptr) {
const auto error_msg =
"Unexpected missing codeword for code: " + std::to_string(code) +
" in snapshot: " + decoder_->compressed_path().string();
SILK_ERROR << error_msg;
throw std::runtime_error{error_msg};
throw std::runtime_error{
"Unexpected missing codeword for code: " + std::to_string(code) + " in snapshot: " + decoder_->compressed_path().string()};
}
length = codeword->code_length();
if (length == 0) {
Expand Down
30 changes: 27 additions & 3 deletions silkworm/node/snapshots/seg/decompressor_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

#include "decompressor.hpp"

#include <algorithm>
#include <filesystem>
#include <map>
#include <span>
Expand Down Expand Up @@ -147,8 +148,6 @@ TEST_CASE("PatternTable::PatternTable", "[silkworm][node][seg][decompressor]") {
}

TEST_CASE("PatternTable::build_condensed", "[silkworm][node][seg][decompressor]") {
PatternTable table1{0};

std::span<Pattern> patterns0{};
Bytes v1{0x00, 0x11};
std::vector<Pattern> patterns1{{0, v1}};
Expand All @@ -160,9 +159,10 @@ TEST_CASE("PatternTable::build_condensed", "[silkworm][node][seg][decompressor]"
{"two patterns", std::span<Pattern>{patterns2.data(), patterns2.size()}},
};

PatternTable table{2}; // max_depth in all patterns
for (const auto& [test_name, pattern_span] : test_spans) {
SECTION(test_name) {
CHECK(table1.build_condensed(pattern_span) == pattern_span.size());
CHECK(table.build_condensed(pattern_span) == pattern_span.size());
}
}
}
Expand Down Expand Up @@ -256,12 +256,36 @@ TEST_CASE("Decompressor::open invalid files", "[silkworm][node][seg][decompresso
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("compressed file is too short: 31"));
}
SECTION("cannot build pattern tree: highest_depth reached zero") {
TemporaryFile tmp_file;
tmp_file.write(*silkworm::from_hex("0x000000000000000C000000000000000400000000000000150309000000000000"));
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("cannot build pattern tree: highest_depth reached zero"));
}
SECTION("pattern dict is invalid: data skip failed at 22") {
TemporaryFile tmp_file;
tmp_file.write(*silkworm::from_hex("0x000000000000000C00000000000000040000000000000016000000000000000003ff"));
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("pattern dict is invalid: data skip failed at 22"));
}
SECTION("pattern dict is invalid: length read failed at 1") {
TemporaryFile tmp_file;
tmp_file.write(*silkworm::from_hex("0x0000000000000000000000000000000000000000000000010000000000000000"));
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("pattern dict is invalid: length read failed at 1"));
}
SECTION("cannot build position tree: highest_depth reached zero") {
TemporaryFile tmp_file;
tmp_file.write(*silkworm::from_hex("0x000000000000000C0000000000000004000000000000000000000000000000160309"));
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("cannot build position tree: highest_depth reached zero"));
}
SECTION("position dict is invalid: position read failed at 22") {
TemporaryFile tmp_file;
tmp_file.write(*silkworm::from_hex("0x000000000000000C00000000000000040000000000000000000000000000001603ff"));
Decompressor decoder{tmp_file.path()};
CHECK_THROWS_MATCHES(decoder.open(), std::runtime_error, Message("position dict is invalid: position read failed at 22"));
}
}

TEST_CASE("Decompressor::open valid files", "[silkworm][node][seg][decompressor]") {
Expand Down

0 comments on commit 2ecfc88

Please sign in to comment.