validator generated POCs over grpc

include/blockchain.hrl

@@ -55,7 +55,7 @@
          time :: non_neg_integer(),
          height :: non_neg_integer(),
          hash :: blockchain_block:hash(),
-         pocs :: map(),
+         pocs :: [{integer(), binary()}] | map(),
          hbbft_round :: non_neg_integer(),
          election_info :: {non_neg_integer(), non_neg_integer()},
          penalties :: {binary(), [{pos_integer(), binary()}]}

include/blockchain_vars.hrl

@@ -135,9 +135,24 @@
 %% Number of blocks to wait before a hotspot can submit a poc challenge request
 -define(poc_challenge_interval, poc_challenge_interval).
 
+%% Number of challenges per block: integer
+-define(poc_challenge_rate, poc_challenge_rate).
+
+%% Actor type of the challenger: not set or 'validator'
+-define(poc_challenger_type, poc_challenger_type).
+
 %% Allow to switch POC version
 -define(poc_version, poc_version).
 
+%% Number of blocks after a POC is started at which point it will timeout/expire: integer
+-define(poc_timeout, poc_timeout).
+
+%% Number of blocks after poc_timeout at which point the poc public data will be deleted from the ledger: integer
+%% NOTE: the public poc data is required as part of the receipt_v2 txn validations
+%%       and so this value must be sufficient as to give time for absorb to occur
+-define(poc_receipts_absorb_timeout, poc_receipts_absorb_timeout).
+
+
 %% Number of blocks to wait before a hotspot can be eligible to participate in a poc
 %% challenge. This would avoid new hotspots getting challenged before they sync to an
 %% acceptable height.

rebar.config

@@ -40,7 +40,7 @@
     {erlang_stats, ".*", {git, "https://github.com/helium/erlang-stats.git", {branch, "master"}}},
     {e2qc, ".*", {git, "https://github.com/helium/e2qc", {branch, "master"}}},
     {vincenty, ".*", {git, "https://github.com/helium/vincenty", {branch, "master"}}},
-    {helium_proto, {git, "https://github.com/helium/proto.git", {branch, "master"}}},
+    {helium_proto, {git, "https://github.com/helium/proto.git", {branch, "andymck/poc-grpc"}}},
     {merkerl, ".*", {git, "https://github.com/helium/merkerl.git", {branch, "master"}}},
     {xxhash, {git, "https://github.com/pierreis/erlang-xxhash", {branch, "master"}}},
     {exor_filter, ".*", {git, "https://github.com/mpope9/exor_filter", {branch, "master"}}},

rebar.lock

@@ -71,7 +71,7 @@
  {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},0},
  {<<"helium_proto">>,
   {git,"https://github.com/helium/proto.git",
-       {ref,"30f17c5d1a7942297923f4e743c681c46f917fc3"}},
+       {ref,"f743a80e534bdc78805e3c5438cb466bec3c0b6f"}},
   0},
  {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2},
  {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},1},

src/blockchain.erl

@@ -796,24 +796,43 @@ get_block_info(Height, Chain = #blockchain{db=DB, info=InfoCF}) ->
             Error
     end.
 
-
--spec mk_block_info(blockchain_block:hash(), blockchain_block:block()) -> #block_info_v2{}.
 mk_block_info(Hash, Block) ->
-    PoCs = lists:flatmap(
-             fun(Txn) ->
-                     case blockchain_txn:type(Txn) of
-                         blockchain_txn_poc_request_v1 ->
-                             [{blockchain_txn_poc_request_v1:onion_key_hash(Txn),
-                               blockchain_txn_poc_request_v1:block_hash(Txn)}];
-                         _ -> []
-                     end
-             end,
-             blockchain_block:transactions(Block)),
+    %% POCs in the block will either be the poc request txns
+    %% or the poc keys in the block metadata
+    %% which is dependant upon chain var poc_challenger_type
+    %% given mk_block_info is called when loading blocks from a snapshot
+    %% and the ledger at this point seems to be empty, its not
+    %% possible to consult the ledger to get the chain var value
+    %% in this snapshot load scenario
+    %% as such check for the presence of either
+    %% and if the poc request txns are present, go with those
+    %% if not default to poc keys
+    %% TODO: REVIEW THIS APPROACH
+    PoCKeys = blockchain_block_v1:poc_keys(Block),
+    PoCRequests = maps:from_list(
+                    lists:flatmap(
+                        fun(Txn) ->
+                                 case blockchain_txn:type(Txn) of
+                                     blockchain_txn_poc_request_v1 ->
+                                         [{blockchain_txn_poc_request_v1:onion_key_hash(Txn),
+                                           blockchain_txn_poc_request_v1:block_hash(Txn)}];
+                                     _ -> []
+                                 end
+                         end,
+                     blockchain_block:transactions(Block))
+                ),
+    PoCs =
+        case PoCRequests of
+            M when map_size(M) == 0 ->
+                PoCKeys;
+            _ ->
+                PoCRequests
+        end,
 
     #block_info_v2{time = blockchain_block:time(Block),
                    hash = Hash,
                    height = blockchain_block:height(Block),
-                   pocs = maps:from_list(PoCs),
+                   pocs = PoCs,
                    hbbft_round = blockchain_block:hbbft_round(Block),
                    election_info = blockchain_block_v1:election_info(Block),
                    penalties = {blockchain_block_v1:bba_completion(Block), blockchain_block_v1:seen_votes(Block)}}.
@@ -1155,6 +1174,7 @@ add_block_(Block, Blockchain, Syncing) ->
                                              Ledger, Height, Blockchain);
                         _ -> ok
                     end,
+
                     case blockchain_txn:Fun(Block, Blockchain, BeforeCommit, IsRescue) of
                         {error, Reason}=Error ->
                             lager:error("Error absorbing transaction, Ignoring Hash: ~p, Reason: ~p", [blockchain_block:hash_block(Block), Reason]),
@@ -3178,7 +3198,8 @@ blocks_test_() ->
                                                election_epoch => 1,
                                                epoch_start => 0,
                                                seen_votes => [],
-                                               bba_completion => <<>>
+                                               bba_completion => <<>>,
+                                               poc_keys => []
                                               }),
              Hash = blockchain_block:hash_block(Block),
              ok = add_block(Block, Chain),
@@ -3256,7 +3277,8 @@ get_block_test_() ->
                                                election_epoch => 1,
                                                epoch_start => 0,
                                                seen_votes => [],
-                                               bba_completion => <<>>
+                                               bba_completion => <<>>,
+                                               poc_keys => []
                                               }),
              Hash = blockchain_block:hash_block(Block),
              ok = add_block(Block, Chain),
@@ -3300,7 +3322,9 @@ block_info_upgrade_test() ->
                                       election_epoch => 1,
                                       epoch_start => 0,
                                       seen_votes => [],
-                                      bba_completion => <<>>
+                                      bba_completion => <<>>,
+                                      poc_keys => []
+
                                       }),
     V1BlockInfo = #block_info{  height = 1,
                                 time = 1,
@@ -3309,11 +3333,11 @@ block_info_upgrade_test() ->
     ExpV2BlockInfo = #block_info_v2{height = 1,
                                     time = 1,
                                     hash = <<"blockhash">>,
-                                    pocs = #{},
+                                    pocs = [],
                                     hbbft_round = 1,
                                     election_info = {1, 0},
                                     penalties = {<<>>, []}},
     V2BlockInfo = upgrade_block_info(V1BlockInfo, Block, Chain),
-    ?assertMatch(V2BlockInfo, ExpV2BlockInfo).
+    ?assertMatch(ExpV2BlockInfo, V2BlockInfo).
 
 -endif.

src/blockchain_block_v1.erl

@@ -31,6 +31,7 @@
     seen_votes/1,
     bba_completion/1,
     snapshot_hash/1,
+    poc_keys/1,
     verify_signatures/4, verify_signatures/5,
     is_rescue_block/1,
     is_election_block/1,
@@ -58,7 +59,8 @@
                        rescue_signature => binary(),
                        seen_votes => [{pos_integer(), binary()}],
                        bba_completion => binary(),
-                       snapshot_hash => binary()
+                       snapshot_hash => binary(),
+                       poc_keys => [any()]
                       }.
 
 -export_type([block/0, block_map/0]).
@@ -77,7 +79,9 @@ new(#{prev_hash := PrevHash,
       election_epoch := ElectionEpoch,
       epoch_start := EpochStart,
       seen_votes := Votes,
-      bba_completion := Completion} = Map) ->
+      bba_completion := Completion,
+      poc_keys := PocKeys } = Map) ->
+    lager:info("*** new block with poc keys ~p",[PocKeys]),
     #blockchain_block_v1_pb{
        prev_hash = PrevHash,
        height = Height,
@@ -89,7 +93,8 @@ new(#{prev_hash := PrevHash,
        epoch_start = EpochStart,
        seen_votes = [wrap_vote(V) || V <- lists:sort(Votes)],
        bba_completion = Completion,
-       snapshot_hash = maps:get(snapshot_hash, Map, <<>>)
+       snapshot_hash = maps:get(snapshot_hash, Map, <<>>),
+       poc_keys = [wrap_poc_key(V) || V <- lists:sort(PocKeys)]
       }.
 
 -spec rescue(block_map())-> block().
@@ -191,6 +196,9 @@ bba_completion(Block) ->
 snapshot_hash(Block) ->
     Block#blockchain_block_v1_pb.snapshot_hash.
 
+-spec poc_keys(block()) -> [any()].
+poc_keys(Block) ->
+    [unwrap_poc_key(V) || V <- Block#blockchain_block_v1_pb.poc_keys].
 %%--------------------------------------------------------------------
 %% @doc
 %% @end
@@ -233,6 +241,7 @@ new_genesis_block(Transactions) ->
                   election_epoch => 1,
                   epoch_start => 0,
                   seen_votes => [],
+                  poc_keys => [],
                   bba_completion => <<>>}).
 
 %%--------------------------------------------------------------------
@@ -388,8 +397,8 @@ to_json(Block, _Opts) ->
       hash => ?BIN_TO_B64(hash_block(Block)),
       prev_hash => ?BIN_TO_B64(prev_hash(Block)),
       transactions => [
-        #{ 
-            hash => ?BIN_TO_B64(blockchain_txn:hash(T)), 
+        #{
+            hash => ?BIN_TO_B64(blockchain_txn:hash(T)),
             type => blockchain_txn:json_type(T)
         } || T <- transactions(Block)]
      }.
@@ -437,6 +446,13 @@ wrap_vote({Idx, Vector}) ->
 unwrap_vote(#blockchain_seen_vote_v1_pb{index = Idx, vector = Vector}) ->
     {Idx, Vector}.
 
+-spec wrap_poc_key({integer(), binary()}) -> #blockchain_poc_key_pb{}.
+wrap_poc_key({PosInCG, Key}) ->
+    #blockchain_poc_key_pb{pos = PosInCG, key = Key}.
+
+-spec unwrap_poc_key(#blockchain_poc_key_pb{}) -> {integer(), binary()}.
+unwrap_poc_key(#blockchain_poc_key_pb{pos = PosInCG, key = Key}) ->
+    {PosInCG, Key}.
 
 %% ------------------------------------------------------------------
 %% EUNIT Tests
@@ -454,7 +470,8 @@ new_merge(Overrides) ->
              election_epoch => 0,
              epoch_start => 0,
              seen_votes => [],
-             bba_completion => <<>>
+             bba_completion => <<>>,
+             poc_keys => []
            },
           Overrides)).
 
@@ -571,7 +588,8 @@ remove_var_txns_test() ->
                 election_epoch => 0,
                 epoch_start    => 0,
                 seen_votes     => [],
-                bba_completion => <<>>
+                bba_completion => <<>>,
+                poc_keys       => []
              }
         ),
     ?assertMatch(

src/blockchain_utils.erl

@@ -196,7 +196,7 @@ pfind(F, ToDos, Timeout) ->
             Parent = self(),
             Workers = lists:foldl(
                 fun(Args, Acc) ->
-                    {Pid, _Ref} = 
+                    {Pid, _Ref} =
                         erlang:spawn_opt(
                             fun() ->
                                 Result = erlang:apply(F, Args),
@@ -221,7 +221,7 @@ pfind(F, ToDos, Timeout) ->
     after Timeout ->
         false
     end.
- 
+
 pfind_rcv(_Ref, Result, 0) ->
     Result;
 pfind_rcv(Ref, Result, Left) ->