Merge pull request #550 from evoskuil/master

Parallel block download work in progress.

include/bitcoin/node/chasers/chaser.hpp

@@ -57,17 +57,23 @@ class BCN_API chaser
         /// The block chaser works with the header-first protocol.
         header,
 
+        /// New unassociated blocks exist in the strong branch.
+        unassociated,
+
         /// A block has been downloaded, checked and stored (header_t).
         /// Issued by 'check' and handled by 'connect'.
         checked,
+        unchecked,
 
         /// A branch has been connected (header_t|height_t).
         /// Issued by 'connect' and handled by 'confirm'.
         connected,
+        unconnected,
 
         /// A branch has been confirmed (fork header_t|height_t).
         /// Issued by 'confirm' and handled by 'transaction'.
         confirmed,
+        unconfirmed,
 
         /// A new transaction has been added to the pool (transaction_t).
         /// Issued by 'transaction' and handled by 'candidate'.

include/bitcoin/node/chasers/chaser_check.hpp

@@ -20,6 +20,7 @@
 #define LIBBITCOIN_NODE_CHASERS_CHASER_CHECK_HPP
 
 #include <functional>
+#include <memory>
 #include <bitcoin/network.hpp>
 #include <bitcoin/node/define.hpp>
 #include <bitcoin/node/chasers/chaser.hpp>
@@ -34,9 +35,8 @@ class BCN_API chaser_check
   : public chaser
 {
 public:
-    // context_map casts header_fk into context.minimum_block_version.
-    using map = database::context_map;
-    typedef std::function<void(const code&, const map&)> handler;
+    typedef std::shared_ptr<database::context_map> map_ptr;
+    typedef std::function<void(const code&, const map_ptr&)> handler;
 
     DELETE_COPY_MOVE(chaser_check);
 
@@ -46,7 +46,7 @@ class BCN_API chaser_check
     virtual code start() NOEXCEPT;
 
     virtual void get_hashes(handler&& handler) NOEXCEPT;
-    virtual void put_hashes(const map& map,
+    virtual void put_hashes(const map_ptr& map,
         network::result_handler&& handler) NOEXCEPT;
 
 protected:
@@ -55,13 +55,14 @@ class BCN_API chaser_check
         link value) NOEXCEPT;
 
     virtual void do_get_hashes(const handler& handler) NOEXCEPT;
-    virtual void do_put_hashes(const map& map,
+    virtual void do_put_hashes(const map_ptr& map,
         const network::result_handler& handler) NOEXCEPT;
 
 private:
     void do_handle_event(const code& ec, chase event_, link value) NOEXCEPT;
 
-    map map_{};
+    // This is protected by strand.
+    map_ptr map_{};
 };
 
 } // namespace node

include/bitcoin/node/chasers/chaser_header.hpp

@@ -64,6 +64,9 @@ class BCN_API chaser_header
     virtual void handle_event(const code& ec, chase event_,
         link value) NOEXCEPT;
 
+    // Handle events.
+    virtual void handle_unchecked(height_t height) NOEXCEPT;
+
     /// Sum of work from header to branch point (excluded).
     virtual bool get_branch_work(uint256_t& work, size_t& point,
         system::hashes& tree_branch, header_links& store_branch,

include/bitcoin/node/full_node.hpp

@@ -71,9 +71,13 @@ class BCN_API full_node
 
     /// Manage download queue.
     virtual void get_hashes(chaser_check::handler&& handler) NOEXCEPT;
-    virtual void put_hashes(const chaser_check::map& map,
+    virtual void put_hashes(const chaser_check::map_ptr& map,
         network::result_handler&& handler) NOEXCEPT;
 
+    /// Set chaser event (does not require network strand).
+    virtual void notify(const code& ec, chaser::chase event_,
+        chaser::link value) NOEXCEPT;
+
     /// Properties.
     /// -----------------------------------------------------------------------
 
@@ -101,6 +105,8 @@ class BCN_API full_node
     void do_start(const network::result_handler& handler) NOEXCEPT override;
     void do_run(const network::result_handler& handler) NOEXCEPT override;
     void do_close() NOEXCEPT override;
+    virtual void do_notify(const code& ec, chaser::chase event_,
+        chaser::link value) NOEXCEPT;
 
 private:
     // These are thread safe.

include/bitcoin/node/protocols/protocol.hpp

@@ -64,9 +64,13 @@ class BCN_API protocol
 
     /// Manage download queue.
     virtual void get_hashes(chaser_check::handler&& handler) NOEXCEPT;
-    virtual void put_hashes(const chaser_check::map& map,
+    virtual void put_hashes(const chaser_check::map_ptr& map,
         network::result_handler&& handler) NOEXCEPT;
 
+    /// Set chaser event (does not require network strand).
+    virtual void notify(const code& ec, chaser::chase event_,
+        chaser::link value) NOEXCEPT;
+
     /// Configuration settings for all libraries.
     const configuration& config() const NOEXCEPT;
 

include/bitcoin/node/protocols/protocol_block_in.hpp

@@ -88,11 +88,8 @@ class BCN_API protocol_block_in
     network::messages::get_data create_get_data(
         const network::messages::inventory& message) const NOEXCEPT;
 
-    // Thread safe.
+    // This is thread safe.
     const network::messages::inventory::type_id block_type_;
-
-    // Protected by strand.
-    system::chain::checkpoint top_{};
 };
 
 } // namespace node

include/bitcoin/node/protocols/protocol_block_in_31800.hpp

@@ -37,6 +37,7 @@ class BCN_API protocol_block_in_31800
 public:
     typedef std::shared_ptr<protocol_block_in_31800> ptr;
     using type_id = network::messages::inventory::type_id;
+    using map_ptr = chaser_check::map_ptr;
 
     BC_PUSH_WARNING(NO_THROW_IN_NOEXCEPT)
     template <typename Session>
@@ -49,7 +50,8 @@ class BCN_API protocol_block_in_31800
         block_type_(session.config().network.witness_node() ?
             type_id::witness_block : type_id::block),
         performance_timer_(std::make_shared<network::deadline>(session.log,
-            channel->strand(), session.config().node.sample_period()))
+            channel->strand(), session.config().node.sample_period())),
+        map_(std::make_shared<database::context_map>())
     {
     }
     BC_POP_WARNING()
@@ -59,6 +61,9 @@ class BCN_API protocol_block_in_31800
     void stopping(const code& ec) NOEXCEPT override;
 
 protected:
+    /// Request blocks from peer.
+    virtual void send_get_data(const map_ptr& map) NOEXCEPT;
+
     /// Recieved incoming block message.
     virtual bool handle_receive_block(const code& ec,
         const network::messages::block::cptr& message) NOEXCEPT;
@@ -72,25 +77,22 @@ class BCN_API protocol_block_in_31800
     /// Manage download queue.
     virtual void handle_put_hashes(const code& ec) NOEXCEPT;
     virtual void handle_get_hashes(const code& ec,
-        const chaser_check::map& map) NOEXCEPT;
+        const map_ptr& map) NOEXCEPT;
 
 private:
-    network::messages::get_data create_get_data(
-        const chaser_check::map& map) const NOEXCEPT;
-
     void do_handle_performance(const code& ec) NOEXCEPT;
-    void do_handle_get_hashes(const code& ec,
-        const chaser_check::map& map) NOEXCEPT;
+    network::messages::get_data create_get_data(
+        const map_ptr& map) const NOEXCEPT;
 
     // These are thread safe.
     const bool report_performance_;
     const network::messages::inventory::type_id block_type_;
 
     // These are protected by strand.
     uint64_t bytes_{ zero };
-    chaser_check::map map_{};
     network::steady_clock::time_point start_{};
     network::deadline::ptr performance_timer_;
+    map_ptr map_;
 };
 
 } // namespace node

include/bitcoin/node/protocols/protocol_header_in_31800.hpp

@@ -62,9 +62,6 @@ class BCN_API protocol_header_in_31800
         const system::hash_digest& last) const NOEXCEPT;
     network::messages::get_headers create_get_headers(
         system::hashes&& start_hashes) const NOEXCEPT;
-
-    // Protected by strand.
-    system::chain::checkpoint top_{};
 };
 
 } // namespace node

include/bitcoin/node/sessions/session.hpp

@@ -47,9 +47,13 @@ class BCN_API session
 
     /// Manage download queue.
     virtual void get_hashes(chaser_check::handler&& handler) NOEXCEPT;
-    virtual void put_hashes(const chaser_check::map& map,
+    virtual void put_hashes(const chaser_check::map_ptr& map,
         network::result_handler&& handler) NOEXCEPT;
 
+    /// Set chaser event (does not require network strand).
+    virtual void notify(const code& ec, chaser::chase event_,
+        chaser::link value) NOEXCEPT;
+
     /// Configuration settings for all libraries.
     const configuration& config() const NOEXCEPT;
 

src/chasers/chaser_block.cpp

@@ -74,12 +74,9 @@ void chaser_block::handle_event(const code& ec, chase event_,
 }
 
 // private
-void chaser_block::do_handle_event(const code&, chase event_, link) NOEXCEPT
+void chaser_block::do_handle_event(const code&, chase, link) NOEXCEPT
 {
     BC_ASSERT_MSG(stranded(), "chaser_block");
-
-    if (event_ == chase::stop)
-        tree_.clear();
 }
 
 void chaser_block::organize(const block::cptr& block,
@@ -113,12 +110,30 @@ void chaser_block::do_organize(const block::cptr& block_ptr,
         return;
     }
 
-    if (tree_.contains(hash) || query.is_block(hash))
+    if (tree_.contains(hash))
     {
         handler(error::duplicate_block, {});
         return;
     }
 
+    // If header exists test for prior invalidity as a block.
+    const auto key = query.to_header(hash);
+    if (!key.is_terminal())
+    {
+        const auto ec = query.get_block_state(key);
+        if (ec == database::error::block_unconfirmable)
+        {
+            handler(ec, {});
+            return;
+        }
+
+        if (ec != database::error::unassociated)
+        {
+            handler(error::duplicate_block, {});
+            return;
+        }
+    }
+
     // Results from running headers-first and then blocks-first.
     auto state = get_state(header.previous_block_hash());
     if (!state)

src/chasers/chaser_check.cpp

@@ -19,6 +19,7 @@
 #include <bitcoin/node/chasers/chaser_check.hpp>
 
 #include <functional>
+#include <memory>
 #include <variant>
 #include <bitcoin/network.hpp>
 #include <bitcoin/node/error.hpp>
@@ -46,8 +47,9 @@ chaser_check::~chaser_check() NOEXCEPT
 
 code chaser_check::start() NOEXCEPT
 {
-    // Initialize from genesis block.
-    handle_header(zero);
+    // Initialize map to all unassociated blocks starting at genesis.
+    map_ = std::make_shared<database::context_map>(
+        archive().get_all_unassociated_above(zero));
 
     BC_ASSERT_MSG(node_stranded(), "chaser_check");
     return subscribe(
@@ -84,7 +86,7 @@ void chaser_check::get_hashes(handler&& handler) NOEXCEPT
             this, std::move(handler)));
 }
 
-void chaser_check::put_hashes(const chaser_check::map& map,
+void chaser_check::put_hashes(const map_ptr& map,
     network::result_handler&& handler) NOEXCEPT
 {
     boost::asio::post(strand(),
@@ -95,25 +97,40 @@ void chaser_check::put_hashes(const chaser_check::map& map,
 // protected
 // ----------------------------------------------------------------------------
 
-void chaser_check::do_get_hashes(const handler&) NOEXCEPT
+void chaser_check::do_get_hashes(const handler& handler) NOEXCEPT
 {
     BC_ASSERT_MSG(stranded(), "chaser_check");
+    BC_ASSERT_MSG(map_, "chaser_check not started");
+
+    ///////////////////////////////////////////////////////////////////////////
+    // TODO: get them!
+    const auto hashes = std::make_shared<database::context_map>();
+    ///////////////////////////////////////////////////////////////////////////
+
+    handler(error::success, hashes);
 }
 
-void chaser_check::do_put_hashes(const chaser_check::map&,
-    const network::result_handler&) NOEXCEPT
+void chaser_check::do_put_hashes(const map_ptr& map,
+    const network::result_handler& handler) NOEXCEPT
 {
     BC_ASSERT_MSG(stranded(), "chaser_check");
+    BC_ASSERT_MSG(map_, "chaser_check not started");
+
+    map_->merge(*map);
+    handler(error::success);
 }
 
+// Stale branches are just be allowed to complete (still downloaded).
 void chaser_check::handle_header(height_t branch_point) NOEXCEPT
 {
-    // Map and peer maps may have newly stale blocks.
-    // All stale branches can just be allowed to complete.
-    // The connect chaser will verify proper advancement.
-    const auto& query = archive();
-    map_.merge(query.get_all_unassociated_above(
-        query.get_last_associated_from(branch_point)));
+    BC_ASSERT_MSG(stranded(), "chaser_check");
+    BC_ASSERT_MSG(map_, "chaser_check not started");
+
+    const auto before = map_->size();
+    map_->merge(archive().get_all_unassociated_above(branch_point));
+    const auto after = map_->size();
+    if (after > before)
+    notify(error::success, chaser::chase::unassociated, {});
 }
 
 BC_POP_WARNING()

src/chasers/chaser_connect.cpp

@@ -82,6 +82,10 @@ void chaser_connect::do_handle_event(const code& ec, chase event_,
 void chaser_connect::handle_checked(header_t) NOEXCEPT
 {
     BC_ASSERT_MSG(stranded(), "chaser_connect");
+
+    // TODO: if on this block, advance position until encountering a block that
+    // is not checked. It may not be possible to do this on height alone, since
+    // an asynchronous reorganization may render the height ambiguous.
 }
 
 BC_POP_WARNING()