From e7e4acc7ebadb0cb924e62dae183afd8e37cbcb0 Mon Sep 17 00:00:00 2001 From: evoskuil Date: Sun, 25 Feb 2024 11:53:55 -0500 Subject: [PATCH 1/3] Add hash to context query, add set/set_link(txs) query. --- .../bitcoin/database/impl/query/archive.ipp | 33 ++++ .../bitcoin/database/impl/query/validate.ipp | 7 +- include/bitcoin/database/query.hpp | 5 +- .../database/tables/archives/header.hpp | 5 +- test/query/archive.cpp | 151 +++++++++++++++++- test/tables/archives/header.cpp | 9 +- 6 files changed, 200 insertions(+), 10 deletions(-) diff --git a/include/bitcoin/database/impl/query/archive.ipp b/include/bitcoin/database/impl/query/archive.ipp index b7b6f8e4..f8df2b12 100644 --- a/include/bitcoin/database/impl/query/archive.ipp +++ b/include/bitcoin/database/impl/query/archive.ipp @@ -115,6 +115,12 @@ inline bool CLASS::set(const hash_digest& point_hash) NOEXCEPT return !set_link(point_hash).is_terminal(); } +TEMPLATE +inline bool CLASS::set(const block& block) NOEXCEPT +{ + return !set_link(block).is_terminal(); +} + TEMPLATE inline bool CLASS::set(const transaction& tx) NOEXCEPT { @@ -710,6 +716,33 @@ header_link CLASS::set_link(const block& block, const context& ctx) NOEXCEPT // ======================================================================== } +TEMPLATE +header_link CLASS::set_link(const block& block) NOEXCEPT +{ + // This sets only the txs of a block with header/context already archived. + const auto header_fk = to_header(block.hash()); + if (header_fk.is_terminal()) + return {}; + + // GUARDED (block (txs) redundancy) + // This guard is only effective if there is a single database thread. + if (is_associated(header_fk)) + return header_fk; + + tx_links links{}; + links.reserve(block.transactions_ptr()->size()); + for (const auto& tx: *block.transactions_ptr()) + if (!push_link_value(links, set_link(*tx))) + return {}; + + // ======================================================================== + const auto scope = store_.get_transactor(); + + return store_.txs.put(header_fk, table::txs::slab{ {}, links }) ? + header_fk : table::header::link{}; + // ======================================================================== +} + } // namespace database } // namespace libbitcoin diff --git a/include/bitcoin/database/impl/query/validate.ipp b/include/bitcoin/database/impl/query/validate.ipp index 28b20aa6..6565c6c8 100644 --- a/include/bitcoin/database/impl/query/validate.ipp +++ b/include/bitcoin/database/impl/query/validate.ipp @@ -125,13 +125,14 @@ bool CLASS::get_context(context& ctx, } TEMPLATE -bool CLASS::get_context_and_timestamp(context& ctx, uint32_t& timestamp, - const header_link& link) const NOEXCEPT +bool CLASS::get_check_context(context& ctx, hash_digest& hash, + uint32_t& timestamp, const header_link& link) const NOEXCEPT { - table::header::get_context_and_timestamp header{}; + table::header::get_check_context header{}; if (!store_.header.get(link, header)) return false; + hash = std::move(header.key); ctx = std::move(header.ctx); timestamp = header.timestamp; return true; diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index 33b61945..d06ea00c 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -60,6 +60,7 @@ class query using output = system::chain::output; using header = system::chain::header; using transaction = system::chain::transaction; + using transactions = system::chain::transaction_cptrs; using inputs_ptr = system::chain::inputs_ptr; using outputs_ptr = system::chain::outputs_ptr; using transactions_ptr = system::chain::transactions_ptr; @@ -198,6 +199,7 @@ class query inline bool set(const header& header, const context& ctx) NOEXCEPT; inline bool set(const block& block, const chain_context& ctx) NOEXCEPT; inline bool set(const block& block, const context& ctx) NOEXCEPT; + inline bool set(const block& block) NOEXCEPT; inline bool set(const hash_digest& point_hash) NOEXCEPT; inline bool set(const transaction& tx) NOEXCEPT; @@ -245,6 +247,7 @@ class query header_link set_link(const header& header, const context& ctx) NOEXCEPT; header_link set_link(const block& block, const chain_context& ctx) NOEXCEPT; header_link set_link(const block& block, const context& ctx) NOEXCEPT; + header_link set_link(const block& block) NOEXCEPT; point_link set_link(const hash_digest& point_hash) NOEXCEPT; tx_link set_link(const transaction& tx) NOEXCEPT; @@ -278,7 +281,7 @@ class query bool get_version(uint32_t& version, const header_link& link) const NOEXCEPT; bool get_bits(uint32_t& bits, const header_link& link) const NOEXCEPT; bool get_context(context& ctx, const header_link& link) const NOEXCEPT; - bool get_context_and_timestamp(context& ctx, uint32_t& timestamp, + bool get_check_context(context& ctx, hash_digest& hash, uint32_t& timestamp, const header_link& link) const NOEXCEPT; bool set_block_preconfirmable(const header_link& link) NOEXCEPT; diff --git a/include/bitcoin/database/tables/archives/header.hpp b/include/bitcoin/database/tables/archives/header.hpp index 225091c6..dce055fc 100644 --- a/include/bitcoin/database/tables/archives/header.hpp +++ b/include/bitcoin/database/tables/archives/header.hpp @@ -256,17 +256,20 @@ struct header uint32_t mtp{}; }; - struct get_context_and_timestamp + struct get_check_context : public schema::header { inline bool from_data(reader& source) NOEXCEPT { + source.rewind_bytes(sk); + key = source.read_hash(); context::from_data(source, ctx); source.skip_bytes(link::size + sizeof(uint32_t)); timestamp = source.read_little_endian(); return source; } + search_key key{}; context ctx{}; uint32_t timestamp{}; }; diff --git a/test/query/archive.cpp b/test/query/archive.cpp index 92ee6d89..10a91ae0 100644 --- a/test/query/archive.cpp +++ b/test/query/archive.cpp @@ -565,7 +565,7 @@ BOOST_AUTO_TEST_CASE(query_archive__set_block__get_block__expected) BOOST_REQUIRE_EQUAL(store.create(events), error::success); BOOST_REQUIRE_EQUAL(store.open(events), error::success); - // Set header/tx/association. + // Set block (header/txs). BOOST_REQUIRE(!query.is_block(test::genesis.hash())); BOOST_REQUIRE(query.set(test::genesis, test::context)); BOOST_REQUIRE(query.is_block(test::genesis.hash())); @@ -606,6 +606,155 @@ BOOST_AUTO_TEST_CASE(query_archive__set_block__get_block__expected) BOOST_REQUIRE_EQUAL(hashes, test::genesis.transaction_hashes(false)); } +BOOST_AUTO_TEST_CASE(query_archive__set_block_txs__get_block__expected) +{ + const auto genesis_header_head = system::base16_chunk( + "010000" // record count + "ffffff" // bucket[0]... + "ffffff" + "ffffff" + "000000" // pk-> + "ffffff"); + const auto genesis_header_body = system::base16_chunk( + "ffffff" // next-> + "6fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000" // sk (block.hash) + "04030201" // flags + "141312" // height + "24232221" // mtp + "ffffff" // previous_block_hash (header_fk - not found) + "01000000" // version + "29ab5f49" // timestamp + "ffff001d" // bits + "1dac2b7c" // nonce + "3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a"); // merkle_root + const auto genesis_tx_head = system::base16_chunk( + "01000000" // record count + "ffffffff" // bucket[0]... + "ffffffff" + "ffffffff" + "00000000" // pk-> + "ffffffff"); + const auto genesis_tx_body = system::base16_chunk( + "ffffffff" // next-> + "3ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a" // sk (tx.hash(false)) + "01" // coinbase + "cc0000" // witless + "cc0000" // witness + "00000000" // locktime + "01000000" // version + "010000" // ins_count + "010000" // outs_count + "0000000000"); // puts_fk-> + const auto genesis_puts_head = system::base16_chunk("0900000000"); + const auto genesis_puts_body = system::base16_chunk( + "00000000" // spend0_fk-> + "0000000000"); // output0_fk-> + + const auto genesis_output_head = system::base16_chunk("5100000000"); + const auto genesis_output_body = system::base16_chunk( + "00000000" // parent_fk-> + "ff00f2052a01000000" // value + "434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac"); // script + const auto genesis_point_head = system::base16_chunk( + "00000000" // record count (null point, empty) + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff"); + const auto genesis_point_body = system::base16_chunk(""); + const auto genesis_spend_head = system::base16_chunk( + "01000000" // record count + "00000000" // spend0_fk-> + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff"); + + // ffffffffffffffffffffff00000000ffffffff0000000000 + const auto genesis_spend_body = system::base16_chunk( + "ffffffff" // terminal-> + "ffffffff" // fp: point_fk-> + "ffffff" // fp: point_index (null) + "00000000" // parent_fk-> + "ffffffff" // sequence + "0000000000"); // input_fk-> (coinbase) + const auto genesis_input_head = system::base16_chunk("4f00000000"); + const auto genesis_input_body = system::base16_chunk( + "4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73" // script + "00"); // witness + const auto genesis_txs_head = system::base16_chunk( + "0f000000" // slabs size + "00000000" // pk-> + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff" + "ffffffff"); + const auto genesis_txs_body = system::base16_chunk( + "ffffffff" // next-> + "000000" // header_fk + "01000000" // txs count (1) + "00000000"); // transaction[0] + + settings settings{}; + settings.header_buckets = 5; + settings.tx_buckets = 5; + settings.point_buckets = 5; + settings.spend_buckets = 5; + settings.txs_buckets = 10; + settings.path = TEST_DIRECTORY; + test::chunk_store store{ settings }; + test::query_accessor query{ store }; + BOOST_REQUIRE_EQUAL(store.create(events), error::success); + BOOST_REQUIRE_EQUAL(store.open(events), error::success); + + // Set header and then txs. + BOOST_REQUIRE(!query.is_block(test::genesis.hash())); + BOOST_REQUIRE(query.set(test::genesis.header(), test::context)); + BOOST_REQUIRE(query.set(test::genesis)); + BOOST_REQUIRE(query.is_block(test::genesis.hash())); + + // Verify idempotentcy (these do not change store state). + ////BOOST_REQUIRE(query.set(test::genesis.header(), test::context)); + ////BOOST_REQUIRE(query.set(test::genesis.header(), test::context)); + ////BOOST_REQUIRE(query.set(test::genesis, test::context)); + ////BOOST_REQUIRE(query.set(test::genesis, test::context)); + + table::header::record element1{}; + BOOST_REQUIRE(store.header.get(query.to_header(test::genesis.hash()), element1)); + BOOST_REQUIRE_EQUAL(store.close(events), error::success); + + BOOST_REQUIRE_EQUAL(store.header_head(), genesis_header_head); + BOOST_REQUIRE_EQUAL(store.tx_head(), genesis_tx_head); + BOOST_REQUIRE_EQUAL(store.point_head(), genesis_point_head); + BOOST_REQUIRE_EQUAL(store.input_head(), genesis_input_head); + BOOST_REQUIRE_EQUAL(store.output_head(), genesis_output_head); + BOOST_REQUIRE_EQUAL(store.puts_head(), genesis_puts_head); + BOOST_REQUIRE_EQUAL(store.spend_head(), genesis_spend_head); + BOOST_REQUIRE_EQUAL(store.txs_head(), genesis_txs_head); + + BOOST_REQUIRE_EQUAL(store.header_body(), genesis_header_body); + BOOST_REQUIRE_EQUAL(store.tx_body(), genesis_tx_body); + BOOST_REQUIRE_EQUAL(store.point_body(), genesis_point_body); + BOOST_REQUIRE_EQUAL(store.input_body(), genesis_input_body); + BOOST_REQUIRE_EQUAL(store.output_body(), genesis_output_body); + BOOST_REQUIRE_EQUAL(store.spend_body(), genesis_spend_body); + BOOST_REQUIRE_EQUAL(store.txs_body(), genesis_txs_body); + + const auto pointer1 = query.get_block(query.to_header(test::genesis.hash())); + BOOST_REQUIRE(pointer1); + BOOST_REQUIRE(*pointer1 == test::genesis); + + const auto hashes = query.get_tx_keys(query.to_header(test::genesis.hash())); + BOOST_REQUIRE_EQUAL(hashes.size(), 1u); + BOOST_REQUIRE_EQUAL(hashes, test::genesis.transaction_hashes(false)); +} + // Moved to protected, set_link(block) covers. ////BOOST_AUTO_TEST_CASE(query_archive__set_links__get_block__expected) ////{ diff --git a/test/tables/archives/header.cpp b/test/tables/archives/header.cpp index 1e263a88..8b80bb51 100644 --- a/test/tables/archives/header.cpp +++ b/test/tables/archives/header.cpp @@ -226,10 +226,11 @@ BOOST_AUTO_TEST_CASE(header__put__get_context__expected) BOOST_REQUIRE(instance.get(1, context)); BOOST_REQUIRE(context.ctx == expected.ctx); - table::header::get_context_and_timestamp context_and_timestamp{}; - BOOST_REQUIRE(instance.get(1, context_and_timestamp)); - BOOST_REQUIRE(context_and_timestamp.ctx == expected.ctx); - BOOST_REQUIRE_EQUAL(context_and_timestamp.timestamp, expected.timestamp); + table::header::get_check_context check_context{}; + BOOST_REQUIRE(instance.get(1, check_context)); + BOOST_REQUIRE(check_context.ctx == expected.ctx); + BOOST_REQUIRE_EQUAL(check_context.timestamp, expected.timestamp); + BOOST_REQUIRE_EQUAL(check_context.key, key); } BOOST_AUTO_TEST_CASE(header__it__pk__expected) From 8ad22b3eeff365c00d3f3f2a4aaba6ceb2091b46 Mon Sep 17 00:00:00 2001 From: evoskuil Date: Sun, 25 Feb 2024 13:45:10 -0500 Subject: [PATCH 2/3] Change get_all_unassociated_above() to deliver map with context. --- .../database/impl/query/initialize.ipp | 24 +++++++- include/bitcoin/database/query.hpp | 5 +- test/query/initialize.cpp | 55 ++++++++++++++++--- 3 files changed, 73 insertions(+), 11 deletions(-) diff --git a/include/bitcoin/database/impl/query/initialize.ipp b/include/bitcoin/database/impl/query/initialize.ipp index 68bb508d..7b3698cd 100644 --- a/include/bitcoin/database/impl/query/initialize.ipp +++ b/include/bitcoin/database/impl/query/initialize.ipp @@ -70,15 +70,33 @@ size_t CLASS::get_last_associated_from(size_t height) const NOEXCEPT } TEMPLATE -hashes CLASS::get_all_unassociated_above(size_t height) const NOEXCEPT +context_map CLASS::get_all_unassociated_above(size_t height) const NOEXCEPT { - hashes out{}; + context_map out{}; const auto top = get_top_candidate(); + table::header::get_check_context context{}; while (height < top) { const auto header_fk = to_candidate(++height); if (!is_associated(header_fk)) - out.push_back(get_header_key(header_fk)); + { + if (!store_.header.get(header_fk, context)) + return {}; + + out.emplace(context.key, system::chain::context + { + context.ctx.flags, + context.timestamp, + context.ctx.mtp, + system::possible_wide_cast(context.ctx.height), + + // HACK: overloading minimum_block_version (unused). + // HACK: do not cast since change to fk size should break. + header_fk + ////uint32_t minimum_block_version; + ////uint32_t work_required; + }); + } } return out; diff --git a/include/bitcoin/database/query.hpp b/include/bitcoin/database/query.hpp index d06ea00c..c5b4dd1a 100644 --- a/include/bitcoin/database/query.hpp +++ b/include/bitcoin/database/query.hpp @@ -20,6 +20,7 @@ #define LIBBITCOIN_DATABASE_QUERY_HPP #include +#include #include #include #include @@ -45,6 +46,8 @@ using input_links = std_vector; using output_links = std_vector; using foreign_point = table::spend::search_key; using two_counts = std::pair; +using context_map = std::unordered_map; template class query @@ -82,7 +85,7 @@ class query inline size_t get_top_confirmed() const NOEXCEPT; size_t get_fork() const NOEXCEPT; size_t get_last_associated_from(size_t height) const NOEXCEPT; - hashes get_all_unassociated_above(size_t height) const NOEXCEPT; + context_map get_all_unassociated_above(size_t height) const NOEXCEPT; hashes get_candidate_hashes(const heights& heights) const NOEXCEPT; hashes get_confirmed_hashes(const heights& heights) const NOEXCEPT; diff --git a/test/query/initialize.cpp b/test/query/initialize.cpp index e31a3d0f..99fa96a9 100644 --- a/test/query/initialize.cpp +++ b/test/query/initialize.cpp @@ -341,24 +341,65 @@ BOOST_AUTO_TEST_CASE(query_initialize__get_all_unassociated_above__gapped_candid test::chunk_store store{ settings }; test::query_accessor query{ store }; BOOST_REQUIRE_EQUAL(store.create(events), error::success); + + constexpr database::context context2 + { + 0x01020302, // flags + 0x00121312, // height (3 bytes) + 0x21222322 // mtp + }; + constexpr database::context context3 + { + 0x01020303, // flags + 0x00121313, // height (3 bytes) + 0x21222323 // mtp + }; BOOST_REQUIRE(query.initialize(test::genesis)); BOOST_REQUIRE(query.set(test::block1, test::context)); - BOOST_REQUIRE(query.set(test::block2.header(), test::context)); // header only - BOOST_REQUIRE(query.set(test::block3.header(), test::context)); // header only + BOOST_REQUIRE(query.set(test::block2.header(), context2)); // header only + BOOST_REQUIRE(query.set(test::block3.header(), context3)); // header only BOOST_REQUIRE(query.push_candidate(query.to_header(test::block1.hash()))); BOOST_REQUIRE(query.push_candidate(query.to_header(test::block2.hash()))); BOOST_REQUIRE(query.push_candidate(query.to_header(test::block3.hash()))); auto unassociated = query.get_all_unassociated_above(0); BOOST_REQUIRE_EQUAL(unassociated.size(), 2u); - BOOST_REQUIRE_EQUAL(unassociated.front(), test::block2.hash()); - BOOST_REQUIRE_EQUAL(unassociated.back(), test::block3.hash()); + + auto result2 = unassociated[test::block2.hash()]; + BOOST_REQUIRE_EQUAL(result2.forks, context2.flags); + BOOST_REQUIRE_EQUAL(result2.timestamp, test::block2.header().timestamp()); + BOOST_REQUIRE_EQUAL(result2.median_time_past, context2.mtp); + BOOST_REQUIRE_EQUAL(result2.height, context2.height); + + auto result3 = unassociated[test::block3.hash()]; + BOOST_REQUIRE_EQUAL(result3.forks, context3.flags); + BOOST_REQUIRE_EQUAL(result3.timestamp, test::block3.header().timestamp()); + BOOST_REQUIRE_EQUAL(result3.median_time_past, context3.mtp); + BOOST_REQUIRE_EQUAL(result3.height, context3.height); + unassociated = query.get_all_unassociated_above(1); BOOST_REQUIRE_EQUAL(unassociated.size(), 2u); - BOOST_REQUIRE_EQUAL(unassociated.front(), test::block2.hash()); - BOOST_REQUIRE_EQUAL(unassociated.back(), test::block3.hash()); + + result2 = unassociated[test::block2.hash()]; + BOOST_REQUIRE_EQUAL(result2.forks, context2.flags); + BOOST_REQUIRE_EQUAL(result2.timestamp, test::block2.header().timestamp()); + BOOST_REQUIRE_EQUAL(result2.median_time_past, context2.mtp); + BOOST_REQUIRE_EQUAL(result2.height, context2.height); + + result3 = unassociated[test::block3.hash()]; + BOOST_REQUIRE_EQUAL(result3.forks, context3.flags); + BOOST_REQUIRE_EQUAL(result3.timestamp, test::block3.header().timestamp()); + BOOST_REQUIRE_EQUAL(result3.median_time_past, context3.mtp); + BOOST_REQUIRE_EQUAL(result3.height, context3.height); + unassociated = query.get_all_unassociated_above(2); BOOST_REQUIRE_EQUAL(unassociated.size(), 1u); - BOOST_REQUIRE_EQUAL(unassociated.back(), test::block3.hash()); + + result3 = unassociated[test::block3.hash()]; + BOOST_REQUIRE_EQUAL(result3.forks, context3.flags); + BOOST_REQUIRE_EQUAL(result3.timestamp, test::block3.header().timestamp()); + BOOST_REQUIRE_EQUAL(result3.median_time_past, context3.mtp); + BOOST_REQUIRE_EQUAL(result3.height, context3.height); + unassociated = query.get_all_unassociated_above(3); BOOST_REQUIRE_EQUAL(unassociated.size(), 0u); } From 21371251abc0b7ad6e6ac53276478e4dce9af44e Mon Sep 17 00:00:00 2001 From: evoskuil Date: Sun, 25 Feb 2024 17:27:55 -0500 Subject: [PATCH 3/3] Don't hack in header_fk for get_all_associated... query. --- .../database/impl/query/initialize.ipp | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/include/bitcoin/database/impl/query/initialize.ipp b/include/bitcoin/database/impl/query/initialize.ipp index 7b3698cd..c07afc63 100644 --- a/include/bitcoin/database/impl/query/initialize.ipp +++ b/include/bitcoin/database/impl/query/initialize.ipp @@ -80,22 +80,19 @@ context_map CLASS::get_all_unassociated_above(size_t height) const NOEXCEPT const auto header_fk = to_candidate(++height); if (!is_associated(header_fk)) { - if (!store_.header.get(header_fk, context)) - return {}; - - out.emplace(context.key, system::chain::context + if (store_.header.get(header_fk, context)) { - context.ctx.flags, - context.timestamp, - context.ctx.mtp, - system::possible_wide_cast(context.ctx.height), - - // HACK: overloading minimum_block_version (unused). - // HACK: do not cast since change to fk size should break. - header_fk - ////uint32_t minimum_block_version; - ////uint32_t work_required; - }); + out.emplace(context.key, system::chain::context + { + context.ctx.flags, + context.timestamp, + context.ctx.mtp, + system::possible_wide_cast(context.ctx.height), + + ////// HACK: overloading minimum_block_version (unused). + ////header_fk + }); + } } }