From 1d2bd57b95923b939d51cf044a8472f912ca14ac Mon Sep 17 00:00:00 2001 From: Andrew McKenzie Date: Mon, 21 Feb 2022 11:32:17 +0000 Subject: [PATCH] validator generated POCs over grpc --- include/blockchain.hrl | 2 +- include/blockchain_vars.hrl | 15 + rebar.config | 2 +- rebar.lock | 2 +- src/blockchain.erl | 60 +- src/blockchain_block_v1.erl | 32 +- src/blockchain_utils.erl | 4 +- src/ledger/v1/blockchain_ledger_poc_v3.erl | 163 ++ .../v1/blockchain_ledger_snapshot_v1.erl | 25 +- src/ledger/v1/blockchain_ledger_v1.erl | 365 +++- src/poc/blockchain_poc_packet_v2.erl | 375 ++++ src/poc/blockchain_poc_target_v5.erl | 201 +++ src/transactions/blockchain_txn.erl | 26 +- .../v1/blockchain_txn_vars_v1.erl | 28 + .../v2/blockchain_txn_poc_receipts_v2.erl | 1572 +++++++++++++++++ .../v2/blockchain_txn_rewards_v2.erl | 84 +- ...blockchain_grpc_sc_client_test_handler.erl | 6 +- test/blockchain_simple_SUITE.erl | 238 ++- ...custom.erl => grpc_client_stream_test.erl} | 2 +- test/t_chain.erl | 3 +- test/test_utils.erl | 3 +- 21 files changed, 3066 insertions(+), 142 deletions(-) create mode 100644 src/ledger/v1/blockchain_ledger_poc_v3.erl create mode 100644 src/poc/blockchain_poc_packet_v2.erl create mode 100644 src/poc/blockchain_poc_target_v5.erl create mode 100644 src/transactions/v2/blockchain_txn_poc_receipts_v2.erl rename test/{grpc_client_stream_custom.erl => grpc_client_stream_test.erl} (99%) diff --git a/include/blockchain.hrl b/include/blockchain.hrl index dedd42ee83..26053c8076 100644 --- a/include/blockchain.hrl +++ b/include/blockchain.hrl @@ -55,7 +55,7 @@ time :: non_neg_integer(), height :: non_neg_integer(), hash :: blockchain_block:hash(), - pocs :: map(), + pocs :: [{integer(), binary()}] | map(), hbbft_round :: non_neg_integer(), election_info :: {non_neg_integer(), non_neg_integer()}, penalties :: {binary(), [{pos_integer(), binary()}]} diff --git a/include/blockchain_vars.hrl b/include/blockchain_vars.hrl index b29c0a094f..4efc8dbd1d 100644 --- a/include/blockchain_vars.hrl +++ b/include/blockchain_vars.hrl @@ -135,9 +135,24 @@ %% Number of blocks to wait before a hotspot can submit a poc challenge request -define(poc_challenge_interval, poc_challenge_interval). +%% Number of challenges per block: integer +-define(poc_challenge_rate, poc_challenge_rate). + +%% Actor type of the challenger: not set or 'validator' +-define(poc_challenger_type, poc_challenger_type). + %% Allow to switch POC version -define(poc_version, poc_version). +%% Number of blocks after a POC is started at which point it will timeout/expire: integer +-define(poc_timeout, poc_timeout). + +%% Number of blocks after poc_timeout at which point the poc public data will be deleted from the ledger: integer +%% NOTE: the public poc data is required as part of the receipt_v2 txn validations +%% and so this value must be sufficient as to give time for absorb to occur +-define(poc_receipts_absorb_timeout, poc_receipts_absorb_timeout). + + %% Number of blocks to wait before a hotspot can be eligible to participate in a poc %% challenge. This would avoid new hotspots getting challenged before they sync to an %% acceptable height. diff --git a/rebar.config b/rebar.config index 218cfb4a99..a9307ed429 100644 --- a/rebar.config +++ b/rebar.config @@ -40,7 +40,7 @@ {erlang_stats, ".*", {git, "https://github.com/helium/erlang-stats.git", {branch, "master"}}}, {e2qc, ".*", {git, "https://github.com/helium/e2qc", {branch, "master"}}}, {vincenty, ".*", {git, "https://github.com/helium/vincenty", {branch, "master"}}}, - {helium_proto, {git, "https://github.com/helium/proto.git", {branch, "master"}}}, + {helium_proto, {git, "https://github.com/helium/proto.git", {branch, "andymck/poc-grpc"}}}, {merkerl, ".*", {git, "https://github.com/helium/merkerl.git", {branch, "master"}}}, {xxhash, {git, "https://github.com/pierreis/erlang-xxhash", {branch, "master"}}}, {exor_filter, ".*", {git, "https://github.com/mpope9/exor_filter", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index 8478eb81d5..0e6abb3ee7 100644 --- a/rebar.lock +++ b/rebar.lock @@ -71,7 +71,7 @@ {<<"hackney">>,{pkg,<<"hackney">>,<<"1.18.0">>},0}, {<<"helium_proto">>, {git,"https://github.com/helium/proto.git", - {ref,"30f17c5d1a7942297923f4e743c681c46f917fc3"}}, + {ref,"f743a80e534bdc78805e3c5438cb466bec3c0b6f"}}, 0}, {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},2}, {<<"idna">>,{pkg,<<"idna">>,<<"6.1.1">>},1}, diff --git a/src/blockchain.erl b/src/blockchain.erl index 2211ce2428..1557616663 100644 --- a/src/blockchain.erl +++ b/src/blockchain.erl @@ -796,24 +796,43 @@ get_block_info(Height, Chain = #blockchain{db=DB, info=InfoCF}) -> Error end. - --spec mk_block_info(blockchain_block:hash(), blockchain_block:block()) -> #block_info_v2{}. mk_block_info(Hash, Block) -> - PoCs = lists:flatmap( - fun(Txn) -> - case blockchain_txn:type(Txn) of - blockchain_txn_poc_request_v1 -> - [{blockchain_txn_poc_request_v1:onion_key_hash(Txn), - blockchain_txn_poc_request_v1:block_hash(Txn)}]; - _ -> [] - end - end, - blockchain_block:transactions(Block)), + %% POCs in the block will either be the poc request txns + %% or the poc keys in the block metadata + %% which is dependant upon chain var poc_challenger_type + %% given mk_block_info is called when loading blocks from a snapshot + %% and the ledger at this point seems to be empty, its not + %% possible to consult the ledger to get the chain var value + %% in this snapshot load scenario + %% as such check for the presence of either + %% and if the poc request txns are present, go with those + %% if not default to poc keys + %% TODO: REVIEW THIS APPROACH + PoCKeys = blockchain_block_v1:poc_keys(Block), + PoCRequests = maps:from_list( + lists:flatmap( + fun(Txn) -> + case blockchain_txn:type(Txn) of + blockchain_txn_poc_request_v1 -> + [{blockchain_txn_poc_request_v1:onion_key_hash(Txn), + blockchain_txn_poc_request_v1:block_hash(Txn)}]; + _ -> [] + end + end, + blockchain_block:transactions(Block)) + ), + PoCs = + case PoCRequests of + M when map_size(M) == 0 -> + PoCKeys; + _ -> + PoCRequests + end, #block_info_v2{time = blockchain_block:time(Block), hash = Hash, height = blockchain_block:height(Block), - pocs = maps:from_list(PoCs), + pocs = PoCs, hbbft_round = blockchain_block:hbbft_round(Block), election_info = blockchain_block_v1:election_info(Block), penalties = {blockchain_block_v1:bba_completion(Block), blockchain_block_v1:seen_votes(Block)}}. @@ -1155,6 +1174,7 @@ add_block_(Block, Blockchain, Syncing) -> Ledger, Height, Blockchain); _ -> ok end, + case blockchain_txn:Fun(Block, Blockchain, BeforeCommit, IsRescue) of {error, Reason}=Error -> lager:error("Error absorbing transaction, Ignoring Hash: ~p, Reason: ~p", [blockchain_block:hash_block(Block), Reason]), @@ -3178,7 +3198,8 @@ blocks_test_() -> election_epoch => 1, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] }), Hash = blockchain_block:hash_block(Block), ok = add_block(Block, Chain), @@ -3256,7 +3277,8 @@ get_block_test_() -> election_epoch => 1, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] }), Hash = blockchain_block:hash_block(Block), ok = add_block(Block, Chain), @@ -3300,7 +3322,9 @@ block_info_upgrade_test() -> election_epoch => 1, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] + }), V1BlockInfo = #block_info{ height = 1, time = 1, @@ -3309,11 +3333,11 @@ block_info_upgrade_test() -> ExpV2BlockInfo = #block_info_v2{height = 1, time = 1, hash = <<"blockhash">>, - pocs = #{}, + pocs = [], hbbft_round = 1, election_info = {1, 0}, penalties = {<<>>, []}}, V2BlockInfo = upgrade_block_info(V1BlockInfo, Block, Chain), - ?assertMatch(V2BlockInfo, ExpV2BlockInfo). + ?assertMatch(ExpV2BlockInfo, V2BlockInfo). -endif. diff --git a/src/blockchain_block_v1.erl b/src/blockchain_block_v1.erl index ead7eb5273..5d45d5c1b5 100644 --- a/src/blockchain_block_v1.erl +++ b/src/blockchain_block_v1.erl @@ -31,6 +31,7 @@ seen_votes/1, bba_completion/1, snapshot_hash/1, + poc_keys/1, verify_signatures/4, verify_signatures/5, is_rescue_block/1, is_election_block/1, @@ -58,7 +59,8 @@ rescue_signature => binary(), seen_votes => [{pos_integer(), binary()}], bba_completion => binary(), - snapshot_hash => binary() + snapshot_hash => binary(), + poc_keys => [any()] }. -export_type([block/0, block_map/0]). @@ -77,7 +79,9 @@ new(#{prev_hash := PrevHash, election_epoch := ElectionEpoch, epoch_start := EpochStart, seen_votes := Votes, - bba_completion := Completion} = Map) -> + bba_completion := Completion, + poc_keys := PocKeys } = Map) -> + lager:info("*** new block with poc keys ~p",[PocKeys]), #blockchain_block_v1_pb{ prev_hash = PrevHash, height = Height, @@ -89,7 +93,8 @@ new(#{prev_hash := PrevHash, epoch_start = EpochStart, seen_votes = [wrap_vote(V) || V <- lists:sort(Votes)], bba_completion = Completion, - snapshot_hash = maps:get(snapshot_hash, Map, <<>>) + snapshot_hash = maps:get(snapshot_hash, Map, <<>>), + poc_keys = [wrap_poc_key(V) || V <- lists:sort(PocKeys)] }. -spec rescue(block_map())-> block(). @@ -191,6 +196,9 @@ bba_completion(Block) -> snapshot_hash(Block) -> Block#blockchain_block_v1_pb.snapshot_hash. +-spec poc_keys(block()) -> [any()]. +poc_keys(Block) -> + [unwrap_poc_key(V) || V <- Block#blockchain_block_v1_pb.poc_keys]. %%-------------------------------------------------------------------- %% @doc %% @end @@ -233,6 +241,7 @@ new_genesis_block(Transactions) -> election_epoch => 1, epoch_start => 0, seen_votes => [], + poc_keys => [], bba_completion => <<>>}). %%-------------------------------------------------------------------- @@ -388,8 +397,8 @@ to_json(Block, _Opts) -> hash => ?BIN_TO_B64(hash_block(Block)), prev_hash => ?BIN_TO_B64(prev_hash(Block)), transactions => [ - #{ - hash => ?BIN_TO_B64(blockchain_txn:hash(T)), + #{ + hash => ?BIN_TO_B64(blockchain_txn:hash(T)), type => blockchain_txn:json_type(T) } || T <- transactions(Block)] }. @@ -437,6 +446,13 @@ wrap_vote({Idx, Vector}) -> unwrap_vote(#blockchain_seen_vote_v1_pb{index = Idx, vector = Vector}) -> {Idx, Vector}. +-spec wrap_poc_key({integer(), binary()}) -> #blockchain_poc_key_pb{}. +wrap_poc_key({PosInCG, Key}) -> + #blockchain_poc_key_pb{pos = PosInCG, key = Key}. + +-spec unwrap_poc_key(#blockchain_poc_key_pb{}) -> {integer(), binary()}. +unwrap_poc_key(#blockchain_poc_key_pb{pos = PosInCG, key = Key}) -> + {PosInCG, Key}. %% ------------------------------------------------------------------ %% EUNIT Tests @@ -454,7 +470,8 @@ new_merge(Overrides) -> election_epoch => 0, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] }, Overrides)). @@ -571,7 +588,8 @@ remove_var_txns_test() -> election_epoch => 0, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] } ), ?assertMatch( diff --git a/src/blockchain_utils.erl b/src/blockchain_utils.erl index 4beb70d40f..2177efab24 100644 --- a/src/blockchain_utils.erl +++ b/src/blockchain_utils.erl @@ -196,7 +196,7 @@ pfind(F, ToDos, Timeout) -> Parent = self(), Workers = lists:foldl( fun(Args, Acc) -> - {Pid, _Ref} = + {Pid, _Ref} = erlang:spawn_opt( fun() -> Result = erlang:apply(F, Args), @@ -221,7 +221,7 @@ pfind(F, ToDos, Timeout) -> after Timeout -> false end. - + pfind_rcv(_Ref, Result, 0) -> Result; pfind_rcv(Ref, Result, Left) -> diff --git a/src/ledger/v1/blockchain_ledger_poc_v3.erl b/src/ledger/v1/blockchain_ledger_poc_v3.erl new file mode 100644 index 0000000000..d5d9506dd8 --- /dev/null +++ b/src/ledger/v1/blockchain_ledger_poc_v3.erl @@ -0,0 +1,163 @@ +%%%------------------------------------------------------------------- +%% @doc +%% == Blockchain Ledger PoC V3 == +%% @end +%%%------------------------------------------------------------------- +-module(blockchain_ledger_poc_v3). + +-export([ + new/4, + onion_key_hash/1, onion_key_hash/2, + challenger/1, challenger/2, + block_hash/1, block_hash/2, + start_height/1, start_height/2, + verify/3, + serialize/1, deserialize/1, + rxtx/0, rx/0, tx/0, fail/0 +]). + +-include("blockchain.hrl"). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. +-record(poc_v3, { + onion_key_hash :: binary(), + challenger :: libp2p_crypto:pubkey_bin(), + block_hash :: binary(), + start_height :: pos_integer(), + unused = 3 :: 3 +}). + +-define(RXTX, rxtx). +-define(RX, rx). +-define(TX, tx). +-define(FAIL, fail). + +-type poc_result_type() :: rxtx | rx | tx | fail. +-type poc_result_types() :: [poc_result_type()]. +-type poc() :: #poc_v3{}. +-type pocs() :: [poc()]. +-export_type([poc/0, pocs/0, poc_result_type/0, poc_result_types/0]). + +-spec new(binary(), libp2p_crypto:pubkey_bin(), binary(), pos_integer()) -> poc(). +new(OnionKeyHash, Challenger, BlockHash, StartHeight) -> + #poc_v3{ + onion_key_hash=OnionKeyHash, + challenger=Challenger, + block_hash=BlockHash, + start_height=StartHeight + }. + +-spec onion_key_hash(poc()) -> binary(). +onion_key_hash(PoC) -> + PoC#poc_v3.onion_key_hash. + +-spec onion_key_hash(binary(), poc()) -> poc(). +onion_key_hash(OnionKeyHash, PoC) -> + PoC#poc_v3{onion_key_hash=OnionKeyHash}. + +-spec challenger(poc()) -> libp2p_crypto:pubkey_bin(). +challenger(PoC) -> + PoC#poc_v3.challenger. + +-spec challenger(libp2p_crypto:pubkey_bin(), poc()) -> poc(). +challenger(Challenger, PoC) -> + PoC#poc_v3{challenger=Challenger}. + +-spec block_hash(poc()) -> binary(). +block_hash(PoC) -> + PoC#poc_v3.block_hash. + +-spec block_hash(binary(), poc()) -> poc(). +block_hash(Challenger, PoC) -> + PoC#poc_v3{block_hash=Challenger}. + +-spec start_height(poc()) -> pos_integer(). +start_height(PoC) -> + PoC#poc_v3.start_height. + +-spec start_height(pos_integer(), poc()) -> poc(). +start_height(Height, PoC) -> + PoC#poc_v3{start_height=Height}. + +-spec verify(poc(), libp2p_crypto:pubkey_bin(), binary()) -> boolean(). +verify(PoC, Challenger, BlockHash) -> + ?MODULE:challenger(PoC) =:= Challenger andalso ?MODULE:block_hash(PoC) =:= BlockHash. + +-spec serialize(poc()) -> binary(). +serialize(PoC) -> + BinPoC = erlang:term_to_binary(PoC), + <<3, BinPoC/binary>>. + +deserialize(<<3, Bin/binary>>) -> + erlang:binary_to_term(Bin). + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec rxtx() -> poc_result_type(). +rxtx() -> + ?RXTX. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec rx() -> poc_result_type(). +rx() -> + ?RX. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec tx() -> poc_result_type(). +tx() -> + ?TX. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec fail() -> poc_result_type(). +fail() -> + ?FAIL. + +%% ------------------------------------------------------------------ +%% EUNIT Tests +%% ------------------------------------------------------------------ +-ifdef(TEST). + +new_test() -> + PoC = #poc_v3{ + onion_key_hash= <<"some key bin">>, + challenger = <<"address">>, + block_hash = <<"block_hash">>, + start_height = 120000, + unused = 3 + }, + ?assertEqual(PoC, new(<<"some key bin">>, <<"address">>, <<"block_hash">>, 120000)). + +onion_key_hash_test() -> + PoC = new(<<"some key bin">>, <<"address">>, <<"block_hash">>, 120000), + ?assertEqual(<<"some key bin">>, onion_key_hash(PoC)), + ?assertEqual(<<"some key bin 2">>, onion_key_hash(onion_key_hash(<<"some key bin 2">>, PoC))). + +challenger_test() -> + PoC = new(<<"some key bin">>, <<"address">>, <<"block_hash">>, 120000), + ?assertEqual(<<"address">>, challenger(PoC)), + ?assertEqual(<<"address 2">>, challenger(challenger(<<"address 2">>, PoC))). + +block_hash_test() -> + PoC = new(<<"some key bin">>, <<"address">>, <<"block_hash">>, 120000), + ?assertEqual(<<"block_hash">>, block_hash(PoC)), + ?assertEqual(<<"block_hash 2">>, block_hash(block_hash(<<"block_hash 2">>, PoC))). + +start_height_test() -> + PoC = new(<<"some key bin">>, <<"address">>, <<"block_hash">>, 120000), + ?assertEqual(120000, start_height(PoC)), + ?assertEqual(200000, start_height(start_height(200000, PoC))). + +-endif. diff --git a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl index a7b6a642ff..e39db0e443 100644 --- a/src/ledger/v1/blockchain_ledger_snapshot_v1.erl +++ b/src/ledger/v1/blockchain_ledger_snapshot_v1.erl @@ -607,9 +607,14 @@ load_into_ledger(Snapshot, L0, Mode) -> case blockchain_ledger_v1:check_key(<<"poc_upgrade">>, L) of true -> ok; _ -> - %% have to do this here, otherwise it'll break block loads - blockchain_ledger_v1:upgrade_pocs(L), - blockchain_ledger_v1:mark_key(<<"poc_upgrade">>, L) + case blockchain_ledger_v1:config(?poc_challenger_type, L) of + {ok, _validator} -> + ok; + {error, not_found} -> + %% have to do this here, otherwise it'll break block loads + blockchain_ledger_v1:upgrade_pocs(L), + blockchain_ledger_v1:mark_key(<<"poc_upgrade">>, L) + end end, blockchain_ledger_v1:commit_context(L). @@ -700,15 +705,17 @@ load_blocks(Ledger0, Chain, Snapshot) -> Ht = blockchain_block:height(Block), %% since hash and block are written at the same time, just getting the %% hash from the height is an acceptable presence check, and much cheaper - case blockchain:get_block_hash(Ht, Chain, false) of - {ok, _Hash} -> + BlockHash = + case blockchain:get_block_hash(Ht, Chain, false) of + {ok, Hash} -> lager:info("skipping block ~p", [Ht]), %% already have it, don't need to store it again. - ok; + Hash; _ -> lager:info("saving block ~p", [Ht]), - ok = blockchain:save_block(Block, Chain) - end, + ok = blockchain:save_block(Block, Chain), + blockchain_block_v1:hash_block(Block) + end, print_memory(), case Ht > Curr2 of %% we need some blocks before for history, only absorb if they're @@ -719,7 +726,7 @@ load_blocks(Ledger0, Chain, Snapshot) -> Chain1 = blockchain:ledger(Ledger2, Chain), Rescue = blockchain_block:is_rescue_block(Block), {ok, _Chain} = blockchain_txn:absorb_block(Block, Rescue, Chain1), - %% Hash = blockchain_block:hash_block(Block), + ok = blockchain_ledger_v1:process_poc_keys(Block, Ht, BlockHash, Ledger2), ok = blockchain_ledger_v1:maybe_gc_pocs(Chain1, Ledger2), ok = blockchain_ledger_v1:maybe_gc_scs(Chain1, Ledger2), %% ok = blockchain_ledger_v1:refresh_gateway_witnesses(Hash, Ledger2), diff --git a/src/ledger/v1/blockchain_ledger_v1.erl b/src/ledger/v1/blockchain_ledger_v1.erl index b9473251d2..1cb88d98ab 100644 --- a/src/ledger/v1/blockchain_ledger_v1.erl +++ b/src/ledger/v1/blockchain_ledger_v1.erl @@ -82,7 +82,14 @@ find_pocs/2, find_poc/3, request_poc/6, + process_poc_keys/4, + save_public_poc/5, + find_public_poc/2, + delete_public_poc/2, + update_public_poc/2, + active_public_pocs/1, delete_poc/3, + purge_pocs/1, maybe_gc_pocs/2, maybe_gc_scs/2, maybe_gc_h3dex/1, @@ -957,7 +964,12 @@ raw_fingerprint(Ledger, Extended) -> {entries_cf(Ledger), blockchain_ledger_entry_v1}, {dc_entries_cf(Ledger), blockchain_ledger_data_credits_entry_v1}, {htlcs_cf(Ledger), blockchain_ledger_htlc_v1}, - {pocs_cf(Ledger), blockchain_ledger_poc_v2}, + {pocs_cf(Ledger), case config(?poc_challenger_type, Ledger) of + {ok, validator} -> + blockchain_ledger_poc_v3; + _ -> + blockchain_ledger_poc_v2 + end}, {securities_cf(Ledger), blockchain_ledger_security_entry_v1}, {routing_cf(Ledger), blockchain_ledger_routing_v1}, {state_channels_cf(Ledger), state_channel}, @@ -2068,7 +2080,190 @@ delete_poc(OnionKeyHash, Challenger, Ledger) -> PoCsCF = pocs_cf(Ledger), cache_delete(Ledger, PoCsCF, <>). +-spec process_poc_keys( + Block :: blockchain_block:block(), + BlockHeight :: pos_integer(), + BlockHash :: binary(), + Ledger :: ledger() +) -> ok. +process_poc_keys(Block, BlockHeight, BlockHash, Ledger) -> + %% we need to update the ledger with public poc data + %% based on the blocks poc ephemeral keys + %% these will be a prop with tuples: {MemberPosInCG :: Integer, PocKeyHash :: binary()} + case blockchain:config(?poc_challenger_type, Ledger) of + {ok, validator} -> + BlockPocEphemeralKeys = blockchain_block_v1:poc_keys(Block), + {ok, CGMembers} = blockchain_ledger_v1:consensus_members(Ledger), + lists:foreach( + fun({CGPos, OnionKeyHash}) -> + %% the published poc key is a hash of the public key, aka the onion key hash + ChallengerAddr = lists:nth(CGPos, CGMembers), + lager:info("saving public poc data for poc key ~p and challenger ~p at blockheight ~p", [OnionKeyHash, ChallengerAddr, BlockHeight]), + catch ok = blockchain_ledger_v1:save_public_poc(OnionKeyHash, ChallengerAddr, BlockHash, BlockHeight, Ledger) + end, + BlockPocEphemeralKeys); + _ -> + ok + end, + ok. + +-spec save_public_poc( OnionKeyHash :: binary(), + Challenger :: libp2p_crypto:pubkey_bin(), + BlockHash :: binary(), + BlockHeight :: pos_integer(), + Ledger :: ledger()) -> ok | {error, any()}. +save_public_poc(OnionKeyHash, Challenger, BlockHash, BlockHeight, Ledger) -> + case blockchain_ledger_v1:get_validator(Challenger, Ledger) of + {error, _Reason}=Error -> + lager:warning("failed to save poc for key ~p, validator not found for challenger ~p",[Challenger]), + Error; + {ok, _ChallengerInfo} -> + case ?MODULE:find_public_poc(OnionKeyHash, Ledger) of + {error, not_found} -> + save_public_poc_(OnionKeyHash, Challenger, BlockHash, BlockHeight, Ledger); + {error, _Reason} = Error -> + Error + end + end. + +save_public_poc_(OnionKeyHash, Challenger, BlockHash, BlockHeight, Ledger) -> + PoC = blockchain_ledger_poc_v3:new(OnionKeyHash, Challenger, BlockHash, BlockHeight), + PoCBin = blockchain_ledger_poc_v3:serialize(PoC), + PoCsCF = pocs_cf(Ledger), + cache_put(Ledger, PoCsCF, OnionKeyHash, PoCBin). + +-spec find_public_poc(binary(), ledger()) -> {ok, blockchain_ledger_poc_v3:poc()} | {error, any()}. +find_public_poc(OnionKeyHash, Ledger) -> + PoCsCF = pocs_cf(Ledger), + case cache_get(Ledger, PoCsCF, OnionKeyHash, []) of + {ok, BinPoC} -> + {ok, blockchain_ledger_poc_v3:deserialize(BinPoC)}; + not_found -> + {error, not_found}; + Error -> + Error + end. + +-spec delete_public_poc(binary(), ledger()) -> ok | {error, any()}. +delete_public_poc(OnionKeyHash, Ledger) -> + case ?MODULE:find_public_poc(OnionKeyHash, Ledger) of + {error, not_found} -> + ok; + {error, _}=Error -> + Error; + {ok, _PoC} -> + PoCsCF = pocs_cf(Ledger), + cache_delete(Ledger, PoCsCF, OnionKeyHash) + end. + +-spec update_public_poc(POC :: blockchain_ledger_poc_v3:poc(), + Ledger :: ledger()) -> ok | {error, _}. +update_public_poc(POC, Ledger) -> + POCAddr = blockchain_ledger_poc_v3:onion_key_hash(POC), + Bin = blockchain_ledger_poc_v3:serialize(POC), + POCsCF = pocs_cf(Ledger), + cache_put(Ledger, POCsCF, POCAddr, Bin). + + +-spec active_public_pocs(ledger()) -> [blockchain_ledger_poc_v3:poc()]. +active_public_pocs(Ledger) -> + POCsCF = pocs_cf(Ledger), + cache_fold( + Ledger, + POCsCF, + fun + ({_KeyHash, <<3, _Bin/binary>> = PoCBin}, Acc) -> + try + POC = blockchain_ledger_poc_v3:deserialize(PoCBin), + [POC | Acc] + catch _:_ -> + lager:info("could not decode poc, possible wrong version, ignoring: ~p", [PoCBin]), + Acc + end; + ({_KeyHash, _NonV3PoCBin}, Acc) -> + lager:info("could not decode poc, possible wrong version, ignoring: ~p", [_NonV3PoCBin]), + Acc + end, + [] + ). + +-spec purge_pocs(ledger()) -> ok | {error, any()}. +purge_pocs(Ledger) -> + PoCsCF = pocs_cf(Ledger), + _ = cache_fold( + Ledger, + PoCsCF, + fun({KeyHash, _}, _Acc) -> + cache_delete(Ledger, PoCsCF, KeyHash) + end, + [] + ). + maybe_gc_pocs(Chain, Ledger) -> + case blockchain:config(?poc_challenger_type, Ledger) of + {ok, validator} -> + maybe_gc_pocs(Chain, Ledger, validator); + _ -> + maybe_gc_pocs(Chain, Ledger, undefined) + end. +maybe_gc_pocs(_Chain, Ledger, validator) -> + %% iterate over the public POCs, + %% delete any which are beyond the lifespan + %% of when the active POC would have ended + %% and any receipts txn would have been + %% 'expected' to be absorbed + %% a public poc is a representation of the data + %% included in a block as part of poc metadata + %% we need to keep it available on the ledger + %% until after the associated receipt v2 txn has + %% has been absorbed or would have been expected + %% to be absorbed + {ok, CurHeight} = current_height(Ledger), + %% set GC point off by one so it doesnt align with so many other gc processes + case CurHeight rem 101 == 0 of + true -> + POCsCF = pocs_cf(Ledger), + {ok, POCTimeout} = get_config(?poc_timeout, Ledger, 10), + {ok, POCReceiptsAbsorbTimeout} = get_config(?poc_receipts_absorb_timeout, Ledger, 50), + + %% allow for the possibility there may be a mix of POC versions in the POC CF + %% this can happen when transitioning from hotspot generated POCs -> validator generated POCs + %% or the reverse + %% anything other than V3s we will want to GC no matter what + %% V3s we will GC if lifespan is up + cache_fold( + Ledger, + POCsCF, + fun + ({_KeyHash, <<3, _Bin/binary>> = PoCBin}, Acc) -> + V3PoC = blockchain_ledger_poc_v3:deserialize(PoCBin), + POCStartHeight = blockchain_ledger_poc_v3:start_height(V3PoC), + OnionKeyHash = blockchain_ledger_poc_v3:onion_key_hash(V3PoC), + %% the public poc data is required by the receipts v2 txn absorb + %% the public poc will be GCed as part of that absorb + %% but in case that fails we will GC it here after giving + %% the txn N blocks to be absorbed + case (CurHeight - POCStartHeight) > (POCTimeout + POCReceiptsAbsorbTimeout) of + true -> + %% the lifespan of the POC for this key has passed, we can GC + ok = delete_public_poc(OnionKeyHash, Ledger); + _ -> + ok + end, + Acc; + + ({KeyHash, _NonV3PoCBin} = _PoC, Acc) -> + lager:info("non v3 poc, deleting: ~p", [_PoC]), + cache_delete(Ledger, POCsCF, KeyHash), + Acc + end, + [] + ), + ok; + _ -> + ok + end; +maybe_gc_pocs(Chain, Ledger, _) -> {ok, Height} = current_height(Ledger), Version = case ?MODULE:config(?poc_version, Ledger) of {ok, V} -> V; @@ -2094,43 +2289,56 @@ maybe_gc_pocs(Chain, Ledger) -> Acc end end, #{}, lists:seq(Height - ((PoCInterval * 2) + 1), Height)), + + %% allow for the possibility there may be a mix of POC versions in the POC CF + %% this can happen when transitioning from hotspot generated POCs -> validator generated POCs + %% or the reverse + %% anything other than V2s we will want to GC no matter what + %% V2s we will GC if lifespan is up cache_fold( - Ledger, - PoCsCF, - fun({KeyHash, BinPoC}, Acc) -> - %% this CF contains all the poc request state that needs to be retained - %% between request and receipt validation. however, it's possible that - %% both requests stop and a receipt never comes, which leads to stale (and - %% in some cases differing) data in the ledger. here, we pull that data - %% out and delete anything that's too old, as determined by being older - %% than twice the request interval, which controls receipt validity. - PoC = blockchain_ledger_poc_v2:deserialize(BinPoC), - H = blockchain_ledger_poc_v2:block_hash(PoC), - case H of - <<>> -> - %% pre-upgrade pocs are ancient - cache_delete(Ledger, PoCsCF, KeyHash); - _ -> - case maps:find(H, Hashes) of - {ok, BH} -> - %% not sure this is even needed, it might - %% always be true? but just in case - case (Height - BH) < PoCInterval * 2 of - false -> - cache_delete(Ledger, PoCsCF, KeyHash); - true -> - ok - end; - error -> - %% if it's not in the hashes map, it's too - %% old by construction - cache_delete(Ledger, PoCsCF, KeyHash) - end - end, - Acc - end, - [] - ), + Ledger, + PoCsCF, + fun + ({KeyHash, <<3, _Rest/binary>>}, Acc) -> + %% we have a v3 POC, must be some contagion from a revert from validator generated POCs + %% to hotspot generated POCs + %% can be deleted + ok = delete_public_poc(KeyHash, Ledger), + Acc; + ({KeyHash, BinPoC}, Acc) -> + %% this CF contains all the poc request state that needs to be retained + %% between request and receipt validation. however, it's possible that + %% both requests stop and a receipt never comes, which leads to stale (and + %% in some cases differing) data in the ledger. here, we pull that data + %% out and delete anything that's too old, as determined by being older + %% than twice the request interval, which controls receipt validity. + PoC = blockchain_ledger_poc_v2:deserialize(BinPoC), + H = blockchain_ledger_poc_v2:block_hash(PoC), + case H of + <<>> -> + %% pre-upgrade pocs are ancient + cache_delete(Ledger, PoCsCF, KeyHash); + _ -> + case maps:find(H, Hashes) of + {ok, BH} -> + %% not sure this is even needed, it might + %% always be true? but just in case + case (Height - BH) < PoCInterval * 2 of + false -> + cache_delete(Ledger, PoCsCF, KeyHash); + true -> + ok + end; + error -> + %% if it's not in the hashes map, it's too + %% old by construction + cache_delete(Ledger, PoCsCF, KeyHash) + end + end, + Acc + end, + [] + ), ok; _ -> ok @@ -2138,23 +2346,29 @@ maybe_gc_pocs(Chain, Ledger) -> upgrade_pocs(Ledger) -> PoCsCF = pocs_cf(Ledger), - ToStore = cache_fold( - Ledger, - PoCsCF, - fun({KeyHash, BinPoCs}, Acc) -> - SPoCs = erlang:binary_to_term(BinPoCs), - cache_delete(Ledger, PoCsCF, KeyHash), - lists:foldl( - fun(SPoC, A) -> - PoC = blockchain_ledger_poc_v2:deserialize(SPoC), - Challenger = blockchain_ledger_poc_v2:challenger(PoC), - [{<>, SPoC} | A] - end, Acc, SPoCs) - end, []), - lists:foreach(fun({K, V}) -> - cache_put(Ledger, PoCsCF, K, V) - end, ToStore), - ok. + %% don't need to do this in the validator challenge world + case blockchain_ledger_v1:config(?poc_challenger_type, Ledger) of + {ok, validator} -> + ok; + {error, not_found} -> + ToStore = cache_fold( + Ledger, + PoCsCF, + fun({KeyHash, BinPoCs}, Acc) -> + SPoCs = erlang:binary_to_term(BinPoCs), + cache_delete(Ledger, PoCsCF, KeyHash), + lists:foldl( + fun(SPoC, A) -> + PoC = blockchain_ledger_poc_v2:deserialize(SPoC), + Challenger = blockchain_ledger_poc_v2:challenger(PoC), + [{<>, SPoC} | A] + end, Acc, SPoCs) + end, []), + lists:foreach(fun({K, V}) -> + cache_put(Ledger, PoCsCF, K, V) + end, ToStore), + ok + end. -spec zone_list_to_pubkey_bins(ZoneList :: [h3:h3_index()], Ledger :: ledger()) -> [libp2p_crypto:pubkey_bin()]. @@ -5020,6 +5234,27 @@ load_threshold_txns(Txns, Ledger) -> -spec snapshot_pocs(ledger()) -> [{binary(), binary()}]. snapshot_pocs(Ledger) -> + case blockchain:config(?poc_challenger_type, Ledger) of + {ok, Type} -> + snapshot_pocs(Type, Ledger); + _ -> + snapshot_pocs(undefined, Ledger) + end. + +-spec snapshot_pocs(atom(), ledger()) -> [{binary(), binary()}]. +snapshot_pocs(validator, Ledger) -> + PoCsCF = pocs_cf(Ledger), + lists:sort( + maps:to_list( + cache_fold( + Ledger, PoCsCF, + fun({OnionKeyHash, BValue}, Acc) -> + PoC = binary_to_term(BValue), + Value = blockchain_ledger_poc_v3:deserialize(PoC), + maps:put(OnionKeyHash, Value, Acc) + end, #{}, + []))); +snapshot_pocs(_, Ledger) -> PoCsCF = pocs_cf(Ledger), lists:sort( maps:to_list( @@ -5033,6 +5268,23 @@ snapshot_pocs(Ledger) -> []))). load_pocs(PoCs, Ledger) -> + case blockchain:config(?poc_challenger_type, Ledger) of + {ok, Type} -> + load_pocs(Type, PoCs, Ledger); + _ -> + load_pocs(undefined, PoCs, Ledger) + end. + +load_pocs(validator, PoCs, Ledger) -> + PoCsCF = pocs_cf(Ledger), + maps:map( + fun(OnionHash, P) -> + BPoC = term_to_binary(blockchain_ledger_poc_v3:serialize(P)), + cache_put(Ledger, PoCsCF, OnionHash, BPoC) + end, + maps:from_list(PoCs)), + ok; +load_pocs(_, PoCs, Ledger) -> PoCsCF = pocs_cf(Ledger), maps:map( fun(OnionHash, P) -> @@ -5333,6 +5585,11 @@ get_sc_mod(Channel, Ledger) -> _ -> blockchain_ledger_state_channel_v1 end. +get_config(Var, Ledger, Default) -> + case blockchain:config(Var, Ledger) of + {ok, V} -> {ok, V}; + _ -> {ok, Default} + end. %%-------------------------------------------------------------------- %% @doc %% This function allows us to get a lower sc_max_actors for testing diff --git a/src/poc/blockchain_poc_packet_v2.erl b/src/poc/blockchain_poc_packet_v2.erl new file mode 100644 index 0000000000..250cd14c94 --- /dev/null +++ b/src/poc/blockchain_poc_packet_v2.erl @@ -0,0 +1,375 @@ +%%%------------------------------------------------------------------- +%% @doc +%% == Blockchain PoC Packet V2 == +%% @end +%%%------------------------------------------------------------------- +-module(blockchain_poc_packet_v2). + +-export([build/3, decrypt/2]). + +-include("blockchain_vars.hrl"). + +-if(?OTP_RELEASE > 22). +%% Ericsson why do you hate us so? +-define(ENCRYPT(Key, IV, AAD, PlainText, TagLength), crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, PlainText, AAD, TagLength, true)). +-define(DECRYPT(Key, IV, AAD, CipherText, Tag), crypto:crypto_one_time_aead(aes_256_gcm, Key, IV, CipherText, AAD, Tag, false)). +-else. +-define(ENCRYPT(Key, IV, AAD, PlainText, TagLength), crypto:block_encrypt(aes_gcm, Key, IV, {AAD, PlainText, TagLength})). +-define(DECRYPT(Key, IV, AAD, CipherText, Tag), crypto:block_decrypt(aes_gcm, Key, IV, {AAD, CipherText, Tag})). +-endif. + +%% @doc A module for constructing a v2 onion packet. +%% +%% Onion packets are nested encrypted packets that have 4 important properties: +%% +%% * All layers are the same size +%% * No decrypter knows how many layers remain +%% * The padding added at each layer is deterministic +%% * No decryptor knows the target of the next layer +%% +%% The outermost packet looks like this: +%% <> +%% +%% The authenticated data is the IV and the public key. The tag is the AES-GCM message +%% authentication code. +%% +%% After decryption the plaintext looks like this: +%% <> +%% +%% The decryptor then appends the first Length+1 bytes of the SHA512 +%% of the Data field. The new packet thus looks like this: +%% +%% <> +%% +%% Thus the next decryptor sees an identical length packet which it can decrypt in the same way. +%% +%% At the end of the packet, the final layer is entirely padding and cannot be decrypted. + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +%% @doc Attempt to decrypt the outer layer of a PoC onion packet. +%% If the decryption was sucessfull, return the per-layer data and the next layer of the onion packet, with padding applied. +%% If the decryption fails, return `error'. +-spec decrypt(Packet :: binary(), ECDHFun :: libp2p_crypto:ecdh_fun()) -> error | {Payload :: binary(), NextLayer :: binary()}. +decrypt(<>, ECDHFun) -> + try libp2p_crypto:bin_to_pubkey(OnionCompactKey) of + PubKey -> + SecretKey = ECDHFun(PubKey), %% note secret key will be used as block key + IV = <<0:80/integer, IV0:16/integer-unsigned-little>>, + case ?DECRYPT(SecretKey, IV, <>, + CipherText, Tag) of + <> -> + PaddingSize = DataSize +5, + <> = crypto:hash(sha512, Data), + NextIV = <<((IV0 bxor Xor) band 16#ffff):16/integer-unsigned-little>>, + {<>, <>}; + _ -> + error + end + catch error:enotsup -> + %% corrupted or invalid key + error + end. + +%% @doc Construct a PoC onion packet. +%% The packets are encrypted for each layer's public key using an ECDH exchange with the private key of the ephemeral onion key. +%% All the layer data should be the same size. The general overhead of the packet is 33+2 + (5 * LayerCount) in addition to the size of all the +%% layer data fields. The IV should be a random 16 bit number. The IV will change for each layer (although this is not strictly necessary). +-spec build(OnionKey :: libp2p_crypto:key_map(), + IV :: non_neg_integer(), + KeysAndData :: [{libp2p_crypto:pubkey(), binary()}, ...]) -> {OuterLayer :: binary(), Layers :: [binary()]}. +build(#{secret := OnionPrivKey, public := OnionPubKey}, IV, PubKeysAndData) -> + ECDHFun = libp2p_crypto:mk_ecdh_fun(OnionPrivKey), + OnionCompactKey = libp2p_crypto:pubkey_to_bin(OnionPubKey), + N = length(PubKeysAndData), + + {Keys, Data} = lists:unzip(PubKeysAndData), + + ECDHKeys = lists:map(fun(Key) -> ECDHFun(Key) end, Keys), + + IVs = compute_ivs(IV, PubKeysAndData), + + MatrixLength = N*(N+1), + EntryMatrix = list_to_tuple([undefined || _ <- lists:seq(1, MatrixLength)]), + + %% TODO document the packet construction + + %% fill in the data cells + DataMatrix = lists:foldl(fun({Row, Col=1}, Acc) -> + %% For column 1, the value is the Payload for that row + CellData = lists:nth(Row, Data), + DataSize = byte_size(CellData), + setelement(((Row-1)*N)+Col, Acc, <>); + ({Row, Col}, Acc) -> + %% For other columns, the value is (Row+1, Column -1) ^ Key(Row+1) + setelement(((Row-1)*N)+Col, Acc, encrypt_cell(Row+1, Col-1, N, Acc, OnionCompactKey, ECDHKeys, IVs)) + end, EntryMatrix, lists:reverse(lists:sort([ {X, Y} || X <- lists:seq(1, N+1), Y <- lists:seq(1, N), X+Y =< N+1])) ), + + %% fill in the padding cells + PaddingMatrix = lists:foldl(fun({Row, Col}, Acc) when Col == N -> + %% For column N, the value is Hash(Row-1, 1) ^ Key(Row) + CellData = lists:nth(Row-1, Data), + DataSize = byte_size(CellData) + 1 + 4, + <> = crypto:hash(sha512, CellData), + case Row > N of + false -> + ExtraTagBytes = ((N-Row)*4), + TempMatrix = setelement(((Row-1)*N)+Col, Acc, <<0:(ExtraTagBytes*8)/integer, Hash/binary>>), + <<_:ExtraTagBytes/binary, Cell/binary>> = encrypt_cell(Row, Col, N, TempMatrix, OnionCompactKey, ECDHKeys, IVs), + setelement(((Row-1)*N)+Col, Acc, Cell); + true -> + setelement(((Row-1)*N)+Col, Acc, Hash) + end; + ({Row, Col}, Acc) -> + %% For column < N, the value is (Row-1, Column +1) ^ Key(Row) + CellData = element(((Row-2)*N) + (Col+1), Acc), + case Row > N of + false -> + ExtraTagBytes = ((N-Row)*4), + TempMatrix = setelement(((Row-1)*N)+Col, Acc, <<0:(ExtraTagBytes*8)/integer, CellData/binary>>), + <<_:ExtraTagBytes/binary, Cell/binary>>= encrypt_cell(Row, Col, N, TempMatrix, OnionCompactKey, ECDHKeys, IVs), + setelement(((Row-1)*N)+Col, Acc, Cell); + true -> + setelement(((Row-1)*N)+Col, Acc, CellData) + end + end, DataMatrix, lists:sort([ {X, Y} || X <- lists:seq(1, N+1), Y <- lists:seq(1, N), X+Y > N+1])), + + %% now we need to re-encrypt the data cells now we have the padding in place, row by row, removing the padding bytes from the previous row + %% and propogating the tags upwards + EncryptedMatrix = case N of + 1 -> + %% for a depth of 1 this step is unnecessary + PaddingMatrix; + _ -> + lists:foldl(fun(R, Acc) -> + PaddingSize = byte_size(lists:nth(R, Data)) + 5, + Row = encrypt_row(R, N, Acc, OnionCompactKey, ECDHKeys, IVs), + TAcc = setelement(((R-2)*N)+2, Acc, binary:part(Row, 0, byte_size(Row) - PaddingSize)), + lists:foldl(fun(E, Acc2) -> + %% zero out all the other columns but 1 and 2 for this row + setelement(((R-2)*N)+E, Acc2, <<>>) + end, TAcc, lists:seq(3, N)) + end, setelement(N, PaddingMatrix, <<>>), lists:reverse(lists:seq(2,N))) + end, + + [FirstRow|_] = PacketRows = lists:map(fun(RowNumber) -> + case RowNumber > N of + true -> + %% the last row is all padding + Bins = lists:sublist(tuple_to_list(EncryptedMatrix), ((RowNumber-1)*N)+1, N), + list_to_binary(Bins); + false -> + encrypt_row(RowNumber, N, EncryptedMatrix, OnionCompactKey, ECDHKeys, IVs) + end + end, lists:seq(1, N+1)), + {<<(hd(IVs)):16/integer-unsigned-little, OnionCompactKey/binary, FirstRow/binary>>, PacketRows}. + + +%% ------------------------------------------------------------------ +%% Internal Function Definitions +%% ------------------------------------------------------------------ + +encrypt_cell(Row, Column, N, Matrix, OnionCompactKey, ECDHKeys, IVs) -> + SecretKey = lists:nth(Row, ECDHKeys), + Bins = lists:sublist(tuple_to_list(Matrix), ((Row-1)*N)+1, Column), + Offset = lists:sum([byte_size(X) || X <- Bins]) - byte_size(lists:last(Bins)), + IV0 = lists:nth(Row, IVs), + IV = <<0:80/integer, IV0:16/integer-unsigned-little>>, + {CipherText, _Tag} = ?ENCRYPT(SecretKey, + IV, <>, + list_to_binary(Bins), 4), + << _:Offset/binary, Cell/binary>> = CipherText, + Cell. + +encrypt_row(Row, N, Matrix, OnionCompactKey, ECDHKeys, IVs) -> + SecretKey = lists:nth(Row, ECDHKeys), + Bins = lists:sublist(tuple_to_list(Matrix), ((Row-1)*N)+1, N), + IV0 = lists:nth(Row, IVs), + IV = <<0:80/integer, IV0:16/integer-unsigned-little>>, + {CipherText, Tag} = ?ENCRYPT(SecretKey, + IV, <>, + list_to_binary(Bins), 4), + <>. + +compute_ivs(InitialIV, KeysAndData) -> + lists:foldl(fun({_, Data}, [H|_]=Acc) -> + PaddingSize = byte_size(Data) + 5, + <<_:PaddingSize/binary, Xor:16/integer-unsigned-little, _/binary>> = crypto:hash(sha512, Data), + [(H bxor Xor) band 16#ffff | Acc] + end, [InitialIV], lists:reverse(KeysAndData)). + +-ifdef(TEST). + +encrypt_decrypt_multi_layer_poc_v4_test_() -> + [{"no blockhash entropy", fun() -> + encrypt_decrypt() + end}, + {"added blockhash entropy", fun() -> + encrypt_decrypt() + end}]. + +encrypt_decrypt_single_layer_poc_v4_test_() -> + [{"no blockhash entropy", fun() -> + encrypt_decrypt_single_layer() + end}, + {"added blockhash entropy", fun() -> + encrypt_decrypt_single_layer() + end}]. + +encrypt_decrypt_double_layer_poc_v4_test_() -> + [{"no blockhash entropy", fun() -> + encrypt_decrypt_double_layer() + end}, + {"added blockhash entropy", fun() -> + encrypt_decrypt_double_layer() + end}]. + +encrypt_decrypt_test_() -> + [{"no blockhash entropy", fun() -> + encrypt_decrypt() + end}, + {"added blockhash entropy", fun() -> + encrypt_decrypt() + end}]. + +encrypt_decrypt_single_layer() -> + #{secret := PrivKey1, public := PubKey1} = libp2p_crypto:generate_keys(ecc_compact), + + OnionKey = libp2p_crypto:generate_keys(ecc_compact), + + PubKeys = [PubKey1], + PrivKeys = [PrivKey1], + + LayerData = [<<"abc">>], + + KeysAndData = lists:zip(PubKeys, LayerData), + + IV = rand:uniform(16384), + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + %% make sure it's deterministic + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + + #{secret := PrivOnionKey, public := PubOnionKey} = OnionKey, + + ECDHFun1 = libp2p_crypto:mk_ecdh_fun(PrivKey1), + ECDHFun2 = libp2p_crypto:mk_ecdh_fun(PrivOnionKey), + SecretKey1 = ECDHFun1(PubOnionKey), + SecretKey2 = ECDHFun2(PubKey1), + ?assertEqual(SecretKey1, SecretKey2), + {<<"abc">>, Remainder1} = decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PrivKey1)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey1]])), + OnionCompactKey = libp2p_crypto:pubkey_to_bin(PubOnionKey), + %ExpectedIV = IV+1, + <<_IV:16/integer-unsigned-little, OnionCompactKey:33/binary, _Rest/binary>> = Remainder1, + %% check all packets are the same length + ?assertEqual(1, length(lists:usort([ byte_size(B) || B <- [OuterPacket, Remainder1]]))), + %% check all the packets at each decryption layer are as expected, and have the right IV + {IVs, Layers} = lists:unzip([{ThisIV, Layer} || <> + <- [OuterPacket, Remainder1], ThisKey == OnionCompactKey]), + ?assertEqual(Layers, Rows), + ?assertEqual(IVs, compute_ivs(IV, KeysAndData)), + ok. + +encrypt_decrypt_double_layer() -> + #{secret := PrivKey1, public := PubKey1} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey2, public := PubKey2} = libp2p_crypto:generate_keys(ecc_compact), + + OnionKey = libp2p_crypto:generate_keys(ecc_compact), + + PubKeys = [PubKey1, PubKey2], + PrivKeys = [PrivKey1, PrivKey2], + + LayerData = [<<"abc">>, <<"def">>], + + KeysAndData = lists:zip(PubKeys, LayerData), + + IV = rand:uniform(16384), + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + %% make sure it's deterministic + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + + #{secret := PrivOnionKey, public := PubOnionKey} = OnionKey, + + ECDHFun1 = libp2p_crypto:mk_ecdh_fun(PrivKey1), + ECDHFun2 = libp2p_crypto:mk_ecdh_fun(PrivOnionKey), + SecretKey1 = ECDHFun1(PubOnionKey), + SecretKey2 = ECDHFun2(PubKey1), + ?assertEqual(SecretKey1, SecretKey2), + {<<"abc">>, Remainder1} = decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PrivKey1)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey1]])), + OnionCompactKey = libp2p_crypto:pubkey_to_bin(PubOnionKey), + %ExpectedIV = IV+1, + <<_IV:16/integer-unsigned-little, OnionCompactKey:33/binary, _Rest/binary>> = Remainder1, + {<<"def">>, Remainder2} = decrypt(Remainder1, libp2p_crypto:mk_ecdh_fun(PrivKey2)), + %% check all packets are the same length + ?assertEqual(1, length(lists:usort([ byte_size(B) || B <- [OuterPacket, Remainder1]]))), + %% check all the packets at each decryption layer are as expected, and have the right IV + {IVs, Layers} = lists:unzip([{ThisIV, Layer} || <> + <- [OuterPacket, Remainder1, Remainder2], ThisKey == OnionCompactKey]), + ?assertEqual(Layers, Rows), + ?assertEqual(IVs, compute_ivs(IV, KeysAndData)), + ok. + + +encrypt_decrypt() -> + #{secret := PrivKey1, public := PubKey1} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey2, public := PubKey2} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey3, public := PubKey3} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey4, public := PubKey4} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey5, public := PubKey5} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey6, public := PubKey6} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey7, public := PubKey7} = libp2p_crypto:generate_keys(ecc_compact), + #{secret := PrivKey8, public := PubKey8} = libp2p_crypto:generate_keys(ecc_compact), + + OnionKey = libp2p_crypto:generate_keys(ecc_compact), + + PubKeys = [PubKey1, PubKey2, PubKey3, PubKey4, PubKey5, PubKey6, PubKey7, PubKey8], + PrivKeys = [PrivKey1, PrivKey2, PrivKey3, PrivKey4, PrivKey5, PrivKey6, PrivKey7, PrivKey8], + + LayerData = [<<"abc">>, <<"def">>, <<"ghi">>, <<"jhk">>, <<"lmn">>, <<"opq">>, <<"rst">>, <<"uvw">>], + + KeysAndData = lists:zip(PubKeys, LayerData), + + IV = rand:uniform(16384), + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + %% make sure it's deterministic + {OuterPacket, Rows} = build(OnionKey, IV, KeysAndData), + + #{secret := PrivOnionKey, public := PubOnionKey} = OnionKey, + + ECDHFun1 = libp2p_crypto:mk_ecdh_fun(PrivKey1), + ECDHFun2 = libp2p_crypto:mk_ecdh_fun(PrivOnionKey), + SecretKey1 = ECDHFun1(PubOnionKey), + SecretKey2 = ECDHFun2(PubKey1), + ?assertEqual(SecretKey1, SecretKey2), + {<<"abc">>, Remainder1} = decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PrivKey1)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(OuterPacket, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey1]])), + OnionCompactKey = libp2p_crypto:pubkey_to_bin(PubOnionKey), + %ExpectedIV = IV+1, + <<_IV:16/integer-unsigned-little, OnionCompactKey:33/binary, _Rest/binary>> = Remainder1, + {<<"def">>, Remainder2} = decrypt(Remainder1, libp2p_crypto:mk_ecdh_fun(PrivKey2)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder1, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey2]])), + {<<"ghi">>, Remainder3} = decrypt(Remainder2, libp2p_crypto:mk_ecdh_fun(PrivKey3)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder2, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey3]])), + {<<"jhk">>, Remainder4} = decrypt(Remainder3, libp2p_crypto:mk_ecdh_fun(PrivKey4)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder3, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey4]])), + {<<"lmn">>, Remainder5} = decrypt(Remainder4, libp2p_crypto:mk_ecdh_fun(PrivKey5)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder4, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey5]])), + {<<"opq">>, Remainder6} = decrypt(Remainder5, libp2p_crypto:mk_ecdh_fun(PrivKey6)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder5, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey6]])), + {<<"rst">>, Remainder7} = decrypt(Remainder6, libp2p_crypto:mk_ecdh_fun(PrivKey7)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder6, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey7]])), + {<<"uvw">>, Remainder8} = decrypt(Remainder7, libp2p_crypto:mk_ecdh_fun(PrivKey8)), + ?assert(lists:all(fun(E) -> E == error end, [ decrypt(Remainder7, libp2p_crypto:mk_ecdh_fun(PK)) || PK <- PrivKeys -- [PrivKey8]])), + %% check all packets are the same length + ?assertEqual(1, length(lists:usort([ byte_size(B) || B <- [OuterPacket, Remainder1, Remainder2, Remainder3, Remainder4, Remainder5, Remainder6, Remainder7, Remainder8]]))), + %% check all the packets at each decryption layer are as expected, and have the right IV + {IVs, Layers} = lists:unzip([{ThisIV, Layer} || <> + <- [OuterPacket, Remainder1, Remainder2, Remainder3, Remainder4, Remainder5, Remainder6, Remainder7, Remainder8], ThisKey == OnionCompactKey]), + ?assertEqual(Layers, Rows), + ?assertEqual(IVs, compute_ivs(IV, KeysAndData)), + ok. + +-endif. diff --git a/src/poc/blockchain_poc_target_v5.erl b/src/poc/blockchain_poc_target_v5.erl new file mode 100644 index 0000000000..ce232ed128 --- /dev/null +++ b/src/poc/blockchain_poc_target_v5.erl @@ -0,0 +1,201 @@ +%%%----------------------------------------------------------------------------- +%%% @doc blockchain_poc_target_v5 implementation. +%%% +%%% The targeting mechanism is based on the following conditions: +%%% - Deterministically i dentify a target region based on public key +%%% - Deterministically select a challengee from target region based on private key +%%% +%%%----------------------------------------------------------------------------- +-module(blockchain_poc_target_v5). + +-include_lib("blockchain/include/blockchain_utils.hrl"). +-include_lib("blockchain/include/blockchain_vars.hrl"). +-include_lib("blockchain/include/blockchain_caps.hrl"). + +-export([ + target_zone/2, + gateways_for_zone/5, + target/5 +]). + +-spec target_zone( + RandState :: rand:state(), + Ledger :: blockchain_ledger_v1:ledger() +) -> {ok, {[h3:h3_index()], h3:h3_index(), rand:state()}} | {error, any()}. +target_zone(RandState, Ledger) -> + %% Get all hexes once + HexList = sorted_hex_list(Ledger), + %% Initialize seed with Hash once + %% Initial zone to begin targeting into + case choose_zone(RandState, HexList) of + {error, _} = ErrorResp -> ErrorResp; + {ok, {Hex, HexRandState}} -> {ok, {HexList, Hex, HexRandState}} + end. + + +%% @doc Finds all valid gateways in specified zone +-spec gateways_for_zone( + ChallengerPubkeyBin :: libp2p_crypto:pubkey_bin(), + Ledger :: blockchain_ledger_v1:ledger(), + Vars :: map(), + HexList :: [h3:h3_index()], + Attempted :: [{h3:h3_index(), rand:state()}] +) -> {ok, [libp2p_crypto:pubkey_bin()]} | {error, any()}. +gateways_for_zone( + ChallengerPubkeyBin, + Ledger, + Vars, + HexList, + [{Hex, HexRandState0} | Tail] = _Attempted +) -> + {ok, Height} = blockchain_ledger_v1:current_height(Ledger), + %% Get a list of gateway pubkeys within this hex + {ok, AddrList0} = blockchain_ledger_v1:get_hex(Hex, Ledger), + lager:info("gateways for hex ~p: ~p", [Hex, AddrList0]), + %% Limit max number of potential targets in the zone + {HexRandState, AddrList} = limit_addrs(Vars, HexRandState0, AddrList0), + case filter(AddrList, Ledger, Height, Vars) of + FilteredList when length(FilteredList) >= 1 -> + lager:info("*** found gateways for hex ~p: ~p", [Hex, FilteredList]), + {ok, FilteredList}; + _ -> + lager:info("*** failed to find gateways for zone ~p, trying again", [Hex]), + %% no eligible target in this zone + %% find a new zone + case choose_zone(HexRandState, HexList) of + {error, _} = ErrorResp -> ErrorResp; + {ok, New} -> + %% remove Hex from attemped, add New to attempted and retry + gateways_for_zone(ChallengerPubkeyBin, Ledger, Vars, HexList, [New | Tail]) + end + + + end. + +-spec target( + ChallengerPubkeyBin :: libp2p_crypto:pubkey_bin(), + InitTargetRandState :: rand:state(), + ZoneRandState :: rand:state(), + Ledger :: blockchain_ledger_v1:ledger(), + Vars :: map() +) -> {ok, {libp2p_crypto:pubkey_bin(), rand:state()}} | {error, any()}. +target(ChallengerPubkeyBin, InitTargetRandState, ZoneRandState, Ledger, Vars) -> + %% Get all hexes once + HexList = sorted_hex_list(Ledger), + lager:info("*** HexList ~p", [HexList]), + %% Initial zone to begin targeting into + case choose_zone(ZoneRandState, HexList) of + {error, _} = ErrorResp -> + ErrorResp; + {ok, {InitHex, InitHexRandState}} -> + lager:info("*** target got InitHex ~p and InitHexRandState ~p", [InitHex, InitHexRandState]), + target_( + ChallengerPubkeyBin, + InitTargetRandState, + Ledger, + Vars, + HexList, + InitHex, + InitHexRandState + ) + end. + +%% @doc Finds a potential target to start the path from. +-spec target_( + ChallengerPubkeyBin :: libp2p_crypto:pubkey_bin(), + InitTargetRandState :: rand:state(), + Ledger :: blockchain_ledger_v1:ledger(), + Vars :: map(), + HexList :: [h3:h3_index()], + InitHex :: h3:h3_index(), + InitHexRandState :: rand:state() +) -> {ok, {libp2p_crypto:pubkey_bin(), rand:state()}}. +target_( + ChallengerPubkeyBin, + InitTargetRandState, + Ledger, + Vars, + HexList, + InitHex, + InitHexRandState +) -> + {ok, ZoneGWs} = gateways_for_zone(ChallengerPubkeyBin, Ledger, Vars, HexList, [ + {InitHex, InitHexRandState} + ]), + %% Assign probabilities to each of these gateways + ProbTargetMap = lists:foldl( + fun(A, Acc) -> + Prob = blockchain_utils:normalize_float(prob_randomness_wt(Vars) * 1.0), + maps:put(A, Prob, Acc) + end, + #{}, + ZoneGWs + ), + lager:info("*** ProbTargetMap ~p", [ProbTargetMap]), + %% Sort the scaled probabilities in default order by gateway pubkey_bin + %% make sure that we carry the rand_state through for determinism + + {RandVal, TargetRandState} = rand:uniform_s(InitTargetRandState), + {ok, TargetPubkeybin} = blockchain_utils:icdf_select( + lists:keysort(1, maps:to_list(ProbTargetMap)), + RandVal + ), + {ok, {TargetPubkeybin, TargetRandState}}. + +%% @doc Filter gateways based on these conditions: +%% - gateways which do not have the relevant capability +-spec filter( + AddrList :: [libp2p_crypto:pubkey_bin()], + Ledger :: blockchain_ledger_v1:ledger(), + Height :: non_neg_integer(), + Vars :: map() +) -> [libp2p_crypto:pubkey_bin()]. +filter(AddrList, Ledger, _Height, _Vars) -> + lists:filter( + fun(A) -> + {ok, Gateway} = blockchain_ledger_v1:find_gateway_info(A, Ledger), + Mode = blockchain_ledger_gateway_v2:mode(Gateway), + blockchain_ledger_gateway_v2:is_valid_capability( + Mode, + ?GW_CAPABILITY_POC_CHALLENGEE, + Ledger + ) + end, + AddrList + ). + +%%%------------------------------------------------------------------- +%% Helpers +%%%------------------------------------------------------------------- +-spec prob_randomness_wt(Vars :: map()) -> float(). +prob_randomness_wt(Vars) -> + maps:get(poc_v5_target_prob_randomness_wt, Vars). + +-spec sorted_hex_list(Ledger :: blockchain_ledger_v1:ledger()) -> [h3:h3_index()]. +sorted_hex_list(Ledger) -> + %% Grab the list of parent hexes + {ok, Hexes} = blockchain_ledger_v1:get_hexes(Ledger), + lists:keysort(1, maps:to_list(Hexes)). + +-spec choose_zone( + RandState :: rand:state(), + HexList :: [h3:h3_index()] +) -> {ok, {h3:h3_index(), rand:state()}} | {error, empty_hex_list}. +choose_zone(_RandState, [] = _HexList) -> + {error, empty_hex_list}; +choose_zone(RandState, HexList) -> + {HexVal, HexRandState} = rand:uniform_s(RandState), + case blockchain_utils:icdf_select(HexList, HexVal) of + {error, zero_weight} -> + lager:info("choose zone error with zero weight, trying again", []), + %% retry + choose_zone(HexRandState, HexList); + {ok, Hex} -> + lager:info("choose zone success, found hex ~p", [Hex]), + {ok, {Hex, HexRandState}} + end. + +limit_addrs(#{?poc_witness_consideration_limit := Limit}, RandState, Witnesses) -> + blockchain_utils:deterministic_subset(Limit, RandState, Witnesses); +limit_addrs(_Vars, RandState, Witnesses) -> + {RandState, Witnesses}. diff --git a/src/transactions/blockchain_txn.erl b/src/transactions/blockchain_txn.erl index 94e49fc57a..be33fb5840 100644 --- a/src/transactions/blockchain_txn.erl +++ b/src/transactions/blockchain_txn.erl @@ -46,7 +46,8 @@ | blockchain_txn_transfer_validator_stake_v1:txn_transfer_validator_stake() | blockchain_txn_unstake_validator_v1:txn_unstake_validator() | blockchain_txn_unstake_validator_v1:txn_validator_heartbeat() - | blockchain_txn_transfer_hotspot_v2:txn_transfer_hotspot_v2(). + | blockchain_txn_transfer_hotspot_v2:txn_transfer_hotspot_v2() + | blockchain_txn_poc_receipts_v2:txn_poc_receipts_v2(). -type before_commit_callback() :: fun((blockchain:blockchain(), blockchain_block:hash()) -> ok | {error, any()}). -type txns() :: [txn()]. @@ -137,7 +138,8 @@ {blockchain_txn_validator_heartbeat_v1, 33}, {blockchain_txn_gen_price_oracle_v1, 34}, {blockchain_txn_consensus_group_failure_v1, 35}, - {blockchain_txn_transfer_hotspot_v2, 36} + {blockchain_txn_transfer_hotspot_v2, 36}, + {blockchain_txn_poc_receipts_v2, 37} ]). block_delay() -> @@ -246,7 +248,10 @@ wrap_txn(#blockchain_txn_validator_heartbeat_v1_pb{} = Txn) -> wrap_txn(#blockchain_txn_consensus_group_failure_v1_pb{} = Txn) -> #blockchain_txn_pb{txn={consensus_group_failure, Txn}}; wrap_txn(#blockchain_txn_transfer_hotspot_v2_pb{}=Txn) -> - #blockchain_txn_pb{txn={transfer_hotspot_v2, Txn}}. + #blockchain_txn_pb{txn={transfer_hotspot_v2, Txn}}; +wrap_txn(#blockchain_txn_poc_receipts_v2_pb{}=Txn) -> + #blockchain_txn_pb{txn={poc_receipts_v2, Txn}}. + -spec unwrap_txn(#blockchain_txn_pb{}) -> blockchain_txn:txn(). unwrap_txn(#blockchain_txn_pb{txn={bundle, #blockchain_txn_bundle_v1_pb{transactions=Txns} = Bundle}}) -> @@ -300,6 +305,8 @@ validate([Txn | Tail] = Txns, Valid, Invalid, PType, PBuf, Chain) -> validate(Tail, Valid, Invalid, Type, [Txn | PBuf], Chain); blockchain_txn_poc_receipts_v1 when PType == undefined orelse PType == Type -> validate(Tail, Valid, Invalid, Type, [Txn | PBuf], Chain); + blockchain_txn_poc_receipts_v2 when PType == undefined orelse PType == Type -> + validate(Tail, Valid, Invalid, Type, [Txn | PBuf], Chain); _Else when PType == undefined -> Start = erlang:monotonic_time(millisecond), case catch Type:is_valid(Txn, Chain) of @@ -533,10 +540,12 @@ absorb_block(Block, Rescue, Chain) -> Transactions0 = blockchain_block:transactions(Block), Transactions = lists:sort(fun sort/2, (Transactions0)), Height = blockchain_block:height(Block), + Hash = blockchain_block:hash_block(Block), case absorb_txns(Transactions, Rescue, Chain) of ok -> ok = blockchain_ledger_v1:increment_height(Block, Ledger), ok = blockchain_ledger_v1:process_delayed_actions(Height, Ledger, Chain), + ok = blockchain_ledger_v1:process_poc_keys(Block, Height, Hash, Ledger), {ok, Chain}; Error -> Error @@ -694,7 +703,9 @@ type(#blockchain_txn_transfer_validator_stake_v1_pb{}) -> type(#blockchain_txn_validator_heartbeat_v1_pb{}) -> blockchain_txn_validator_heartbeat_v1; type(#blockchain_txn_transfer_hotspot_v2_pb{}) -> - blockchain_txn_transfer_hotspot_v2. + blockchain_txn_transfer_hotspot_v2; +type(#blockchain_txn_poc_receipts_v2_pb{}) -> + blockchain_txn_poc_receipts_v2. -spec validate_fields([{{atom(), iodata() | undefined}, {binary, pos_integer()} | @@ -910,8 +921,10 @@ absorb_aux(Block0, Chain0) -> plain_absorb_(Block, Chain0) -> case ?MODULE:absorb_block(Block, Chain0) of {ok, _} -> - %% Hash = blockchain_block:hash_block(Block), + Hash = blockchain_block:hash_block(Block), + Height = blockchain_block:height(Block), Ledger0 = blockchain:ledger(Chain0), + ok = blockchain_ledger_v1:process_poc_keys(Block, Height, Hash, Ledger0), ok = blockchain_ledger_v1:maybe_gc_pocs(Chain0, Ledger0), ok = blockchain_ledger_v1:maybe_gc_scs(Chain0, Ledger0), ok = blockchain_ledger_v1:maybe_gc_h3dex(Ledger0), @@ -1004,6 +1017,9 @@ actor(Txn) -> blockchain_txn_state_channel_close_v1:state_channel_owner(Txn); blockchain_txn_assert_location_v2 -> blockchain_txn_assert_location_v2:gateway(Txn); + blockchain_txn_poc_receipts_v2 -> + blockchain_txn_poc_receipts_v2:challenger(Txn); + _ -> <<>> end. diff --git a/src/transactions/v1/blockchain_txn_vars_v1.erl b/src/transactions/v1/blockchain_txn_vars_v1.erl index 34da3db8af..d14b44c36c 100644 --- a/src/transactions/v1/blockchain_txn_vars_v1.erl +++ b/src/transactions/v1/blockchain_txn_vars_v1.erl @@ -709,6 +709,14 @@ var_hook(?poc_hexing_type, hex_h3dex, Ledger) -> %% rebuild hexes since we're back to updating them blockchain:bootstrap_hexes(Ledger), ok; +%% poc challenger type has been modified +%% we want to clear out the pocs CF +%% we dont care about its value, if its been +%% updated then we wipe all POCs +var_hook(?poc_challenger_type, _, Ledger) -> + lager:info("poc_challenger_type changed, purging pocs", []), + purge_pocs(Ledger), + ok; var_hook(_Var, _Value, _Ledger) -> ok. @@ -721,6 +729,10 @@ unset_hook(?poc_hexing_type, Ledger) -> %% rebuild hexes since we're back to updating them blockchain:bootstrap_hexes(Ledger), ok; +unset_hook(?poc_challenger_type, Ledger) -> + lager:info("poc_challenger_type unset, purging pocs", []), + purge_pocs(Ledger), + ok; unset_hook(_Var, _Ledger) -> ok. @@ -950,6 +962,20 @@ validate_var(?min_assert_h3_res, Value) -> validate_int(Value, "min_assert_h3_res", 0, 15, false); validate_var(?poc_challenge_interval, Value) -> validate_int(Value, "poc_challenge_interval", 10, 1440, false); +validate_var(?poc_challenge_rate, Value) -> + validate_int(Value, "poc_challenge_rate", 1, 10000, false); +validate_var(?poc_timeout, Value) -> + validate_int(Value, "poc_timeout", 1, 50, false); +validate_var(?poc_receipts_absorb_timeout, Value) -> + validate_int(Value, "poc_receipts_absorb_timeout", 10, 100, false); + +validate_var(?poc_challenger_type, Value) -> + case Value of + validator -> + ok; + _ -> + throw({error, {poc_challenger_type, Value}}) + end; validate_var(?poc_version, Value) -> case Value of N when is_integer(N), N >= 1, N =< 11 -> @@ -1532,6 +1558,8 @@ validate_region_params(Var, Value) when is_binary(Value) -> validate_region_params(Var, Value) -> throw({error, {invalid_region_param_not_binary, Var, Value}}). +purge_pocs(Ledger) -> + blockchain_ledger_v1:purge_pocs(Ledger). %% ------------------------------------------------------------------ %% EUNIT Tests diff --git a/src/transactions/v2/blockchain_txn_poc_receipts_v2.erl b/src/transactions/v2/blockchain_txn_poc_receipts_v2.erl new file mode 100644 index 0000000000..08afb82855 --- /dev/null +++ b/src/transactions/v2/blockchain_txn_poc_receipts_v2.erl @@ -0,0 +1,1572 @@ +%%%------------------------------------------------------------------- +%% @doc +%% == Blockchain Transaction Proof of Coverage Receipts == +%%%------------------------------------------------------------------- +-module(blockchain_txn_poc_receipts_v2). + +-behavior(blockchain_txn). +-behavior(blockchain_json). + +-include("blockchain.hrl"). +-include("blockchain_json.hrl"). +-include("blockchain_caps.hrl"). +-include("blockchain_vars.hrl"). +-include("blockchain_utils.hrl"). +-include_lib("helium_proto/include/blockchain_txn_poc_receipts_v2_pb.hrl"). +-include_lib("public_key/include/public_key.hrl"). + +-export([ + new/5, + hash/1, + onion_key_hash/1, + challenger/1, + secret/1, + path/1, + fee/1, + fee_payer/2, + block_hash/1, + signature/1, + sign/2, + is_valid/2, + absorb/2, + create_secret_hash/2, + connections/1, + deltas/1, deltas/2, + check_path_continuation/1, + print/1, + json_type/0, + to_json/2, + poc_id/1, + good_quality_witnesses/2, + valid_witnesses/3, valid_witnesses/4, + tagged_witnesses/3, + get_channels/2, get_channels/4 +]). + +-ifdef(TEST). +-include_lib("eunit/include/eunit.hrl"). +-endif. + +-type txn_poc_receipts() :: #blockchain_txn_poc_receipts_v2_pb{}. +-type deltas() :: [{libp2p_crypto:pubkey_bin(), {float(), float()}}]. +-type tagged_witnesses() :: [{IsValid :: boolean(), InvalidReason :: binary(), Witness :: blockchain_poc_witness_v1:witness()}]. + +-export_type([txn_poc_receipts/0]). + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec new(libp2p_crypto:pubkey_bin(), binary(), binary(), + blockchain_poc_path_element_v1:path(), binary()) -> txn_poc_receipts(). +new(Challenger, Secret, OnionKeyHash, Path, BlockHash) -> + #blockchain_txn_poc_receipts_v2_pb{ + challenger=Challenger, + secret=Secret, + onion_key_hash=OnionKeyHash, + path=Path, + fee=0, + signature = <<>>, + block_hash = BlockHash + }. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec hash(txn_poc_receipts()) -> blockchain_txn:hash(). +hash(Txn) -> + BaseTxn = Txn#blockchain_txn_poc_receipts_v2_pb{signature = <<>>}, + EncodedTxn = blockchain_txn_poc_receipts_v2_pb:encode_msg(BaseTxn), + crypto:hash(sha256, EncodedTxn). + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec challenger(txn_poc_receipts()) -> libp2p_crypto:pubkey_bin(). +challenger(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.challenger. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec secret(txn_poc_receipts()) -> binary(). +secret(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.secret. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec onion_key_hash(txn_poc_receipts()) -> binary(). +onion_key_hash(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.onion_key_hash. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec path(txn_poc_receipts()) -> blockchain_poc_path_element_v1:path(). +path(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.path. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec fee(txn_poc_receipts()) -> 0. +fee(_Txn) -> + 0. + +-spec fee_payer(txn_poc_receipts(), blockchain_ledger_v1:ledger()) -> libp2p_crypto:pubkey_bin() | undefined. +fee_payer(_Txn, _Ledger) -> + undefined. + +block_hash(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.block_hash. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec signature(txn_poc_receipts()) -> binary(). +signature(Txn) -> + Txn#blockchain_txn_poc_receipts_v2_pb.signature. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec sign(txn_poc_receipts(), libp2p_crypto:sig_fun()) -> txn_poc_receipts(). +sign(Txn, SigFun) -> + BaseTxn = Txn#blockchain_txn_poc_receipts_v2_pb{signature = <<>>}, + EncodedTxn = blockchain_txn_poc_receipts_v2_pb:encode_msg(BaseTxn), + Txn#blockchain_txn_poc_receipts_v2_pb{signature=SigFun(EncodedTxn)}. + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec is_valid(txn_poc_receipts(), blockchain:blockchain()) -> ok | {error, atom()} | {error, {atom(), any()}}. +is_valid(Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + Challenger = ?MODULE:challenger(Txn), + Signature = ?MODULE:signature(Txn), + PubKey = libp2p_crypto:bin_to_pubkey(Challenger), + BaseTxn = Txn#blockchain_txn_poc_receipts_v2_pb{signature = <<>>}, + EncodedTxn = blockchain_txn_poc_receipts_v2_pb:encode_msg(BaseTxn), + {ok, POCVersion} = blockchain:config(?poc_version, Ledger), + case libp2p_crypto:verify(EncodedTxn, Signature, PubKey) of + false -> + {error, bad_signature}; + true -> + %% check the challenger is actually a validator and it exists + case blockchain_ledger_v1:get_validator(Challenger, Ledger) of + {error, _Reason}=Error -> + lager:info("poc_receipts: validator not found for challenger ~p",[Challenger]), + Error; + {ok, _ChallengerInfo} -> + case ?MODULE:path(Txn) =:= [] of + true -> + {error, empty_path}; + false -> + case check_is_valid_poc(POCVersion, Txn, Chain) of + {ok, Channels} -> + lager:debug("POCID: ~p, validated ok with reported channels: ~p", + [poc_id(Txn), Channels]), + ok; + Error -> Error + end + end + end + end. + + +-spec check_is_valid_poc(POCVersion :: pos_integer(), + Txn :: txn_poc_receipts(), + Chain :: blockchain:blockchain()) -> {ok, [non_neg_integer(), ...]} | {error, any()}. +check_is_valid_poc(POCVersion, Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + Challenger = ?MODULE:challenger(Txn), + POCOnionKeyHash = ?MODULE:onion_key_hash(Txn), + BlockHash = ?MODULE:block_hash(Txn), + POCID = ?MODULE:poc_id(Txn), + StartPre = maybe_start_duration(), + case blockchain_ledger_v1:find_public_poc(POCOnionKeyHash, Ledger) of + {error, Reason}=Error -> + lager:warning([{poc_id, POCID}], + "poc_receipts error find_public_poc, poc_onion_key_hash: ~p, reason: ~p", + [POCOnionKeyHash, Reason]), + Error; + {ok, PoC} -> + Secret = ?MODULE:secret(Txn), + case blockchain_ledger_poc_v3:verify(PoC, Challenger, BlockHash) of + false -> + {error, invalid_poc}; + true -> + PrePocBlockHeight = blockchain_ledger_poc_v3:start_height(PoC), + case blockchain:get_block_info(PrePocBlockHeight, Chain) of + {error, Reason}=Error -> + lager:warning([{poc_id, POCID}], + "poc_receipts error get_block, last_challenge: ~p, reason: ~p", + [PrePocBlockHeight, Reason]), + Error; + {ok, #block_info_v2{height = BlockHeight, + time = BlockTime, + pocs = BlockPoCs}} -> + %% check the onion key is in the block + case lists:keyfind(POCOnionKeyHash, 2, BlockPoCs) of + false -> + {error, onion_key_hash_mismatch}; + _ -> + %% Note there are 2 block hashes here; one is the block hash encoded into the original + %% PoC request used to establish a lower bound on when that PoC request was made, + %% and one is the block hash at which the PoC was absorbed onto the chain. + %% + %% The first, mediated via a chain var, is mixed with the ECDH derived key for each layer + %% of the onion to ensure that nodes cannot decrypt the onion layer if they are not synced + %% with the chain. + %% + %% The second of these is combined with the PoC secret to produce the combined entropy + %% from both the chain and from the PoC requester. + %% + %% Keeping these distinct and using them for their intended purpose is important. + PrePoCBlockHash = blockchain_ledger_poc_v3:block_hash(PoC), + StartLA = maybe_log_duration(prelude, StartPre), + {ok, OldLedger} = blockchain:ledger_at(BlockHeight, Chain), + StartFT = maybe_log_duration(ledger_at, StartLA), + Vars = vars(OldLedger), + Keys = libp2p_crypto:keys_from_bin(Secret), + Entropy = <>, + {Path, StartP} = get_path(POCVersion, Challenger, BlockTime, Entropy, Keys, Vars, OldLedger, Ledger, StartFT), + N = erlang:length(Path), + [<> | LayerData] = blockchain_txn_poc_receipts_v2:create_secret_hash(Entropy, N+1), + OnionList = lists:zip([libp2p_crypto:bin_to_pubkey(P) || P <- Path], LayerData), + {_Onion, Layers} = case blockchain:config(?poc_typo_fixes, Ledger) of + {ok, true} -> + blockchain_poc_packet_v2:build(Keys, IV, OnionList); + _ -> + blockchain_poc_packet_v2:build(Keys, IV, OnionList) + end, + %% no witness will exist with the first layer hash + [_|LayerHashes] = [crypto:hash(sha256, L) || L <- Layers], + StartV = maybe_log_duration(packet_construction, StartP), + Channels = get_channels_(POCVersion, OldLedger, Path, LayerData, no_prefetch), + %% %% run validations + Ret = validate(POCVersion, Txn, Path, LayerData, LayerHashes, OldLedger), + maybe_log_duration(receipt_validation, StartV), + case Ret of + ok -> + {ok, Channels}; + {error, _}=E -> E + end + end + end + end + end. + +get_path(_POCVersion, Challenger, BlockTime, Entropy, Keys, Vars, OldLedger, Ledger, StartT) -> + %% Targeting phase + %% Find the original target + #{public := _OnionCompactKey, secret := {ecc_compact, POCPrivKey}} = Keys, + #'ECPrivateKey'{privateKey = PrivKeyBin} = POCPrivKey, + POCPrivKeyHash = crypto:hash(sha256, PrivKeyBin), + ZoneRandState = blockchain_utils:rand_state(Entropy), + InitTargetRandState = blockchain_utils:rand_state(POCPrivKeyHash), + {ok, {Target, TargetRandState}} = blockchain_poc_target_v4:target(Challenger, InitTargetRandState, ZoneRandState, Ledger, Vars), + %% Path building phase + StartB = maybe_log_duration(target, StartT), + RetB = blockchain_poc_path_v4:build(Target, TargetRandState, OldLedger, BlockTime, Vars), + EndT = maybe_log_duration(build, StartB), + {RetB, EndT}. + +%% TODO: I'm not sure that this is actually faster than checking the time, but I suspect that it'll +%% be more lock-friendly? +maybe_start_duration() -> + case application:get_env(blockchain, log_validation_times, false) of + true -> + erlang:monotonic_time(microsecond); + _ -> 0 + end. + +maybe_log_duration(Type, Start) -> + case application:get_env(blockchain, log_validation_times, false) of + true -> + End = erlang:monotonic_time(microsecond), + lager:info("~p took ~p usec", [Type, End - Start]), + End; + _ -> ok + end. + +-spec connections(Txn :: txn_poc_receipts()) -> [{Transmitter :: libp2p_crypto:pubkey_bin(), Receiver :: libp2p_crypto:pubkey_bin(), + RSSI :: integer(), Timestamp :: non_neg_integer()}]. +connections(Txn) -> + Paths = ?MODULE:path(Txn), + TaggedPaths = lists:zip(lists:seq(1, length(Paths)), Paths), + lists:foldl(fun({PathPos, PathElement}, Acc) -> + case blockchain_poc_path_element_v1:receipt(PathElement) of + undefined -> []; + Receipt -> + case blockchain_poc_receipt_v1:origin(Receipt) of + radio -> + {_, PrevElem} = lists:keyfind(PathPos - 1, 1, TaggedPaths), + [{blockchain_poc_path_element_v1:challengee(PrevElem), blockchain_poc_receipt_v1:gateway(Receipt), + blockchain_poc_receipt_v1:signal(Receipt), blockchain_poc_receipt_v1:timestamp(Receipt)}]; + _ -> + [] + end + end ++ + lists:map(fun(Witness) -> + {blockchain_poc_path_element_v1:challengee(PathElement), + blockchain_poc_witness_v1:gateway(Witness), + blockchain_poc_witness_v1:signal(Witness), + blockchain_poc_witness_v1:timestamp(Witness)} + end, blockchain_poc_path_element_v1:witnesses(PathElement)) ++ Acc + end, [], TaggedPaths). + +%%-------------------------------------------------------------------- +%% @doc Return a list of {gateway, {alpha, beta}} two tuples after +%% looking at a single poc rx txn. Alpha and Beta are the shaping parameters for +%% the beta distribution curve. +%% +%% An increment in alpha implies that we have gained more confidence in a hotspot +%% being active and has succesffully either been a witness or sent a receipt. The +%% actual increment values are debatable and have been put here for testing, although +%% they do seem to behave well. +%% +%% An increment in beta implies we gain confidence in a hotspot not doing it's job correctly. +%% This should be rare and only happen when we are certain that a particular hotspot in the path +%% has not done it's job. +%% +%% @end +%%-------------------------------------------------------------------- +-spec deltas(Txn :: txn_poc_receipts()) -> deltas(). +deltas(Txn) -> + Path = blockchain_txn_poc_receipts_v2:path(Txn), + Length = length(Path), + lists:reverse(element(1, lists:foldl(fun({N, Element}, {Acc, true}) -> + Challengee = blockchain_poc_path_element_v1:challengee(Element), + Receipt = blockchain_poc_path_element_v1:receipt(Element), + Witnesses = blockchain_poc_path_element_v1:witnesses(Element), + NextElements = lists:sublist(Path, N+1, Length), + HasContinued = check_path_continuation(NextElements), + {Val, Continue} = assign_alpha_beta(HasContinued, Receipt, Witnesses), + {set_deltas(Challengee, Val, Acc), Continue}; + (_, Acc) -> + Acc + end, + {[], true}, + lists:zip(lists:seq(1, Length), Path)))). + +-spec deltas(Txn :: txn_poc_receipts(), + Chain :: blockchain:blockchain()) -> deltas(). +deltas(Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + {ok, POCVersion} = blockchain:config(?poc_version, Ledger), + deltas(POCVersion, Txn, Chain). + +-spec deltas(POCVersion :: pos_integer(), + Txn :: txn_poc_receipts(), + Chain :: blockchain:blockchain()) -> deltas(). +deltas(_POCVersion, Txn, Chain) -> + calculate_delta(Txn, Chain). + +-spec calculate_delta(Txn :: txn_poc_receipts(), + Chain :: blockchain:blockchain()) -> deltas(). +calculate_delta(Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + Path = blockchain_txn_poc_receipts_v2:path(Txn), + Length = length(Path), + + try get_channels(Txn, Chain) of + {ok, Channels} -> + + lists:reverse(element(1, lists:foldl(fun({ElementPos, Element}, {Acc, true}) -> + Challengee = blockchain_poc_path_element_v1:challengee(Element), + NextElements = lists:sublist(Path, ElementPos+1, Length), + HasContinued = check_path_continuation(NextElements), + + {PreviousElement, ReceiptChannel, WitnessChannel} = + case ElementPos of + 1 -> + {undefined, 0, hd(Channels)}; + _ -> + {lists:nth(ElementPos - 1, Path), lists:nth(ElementPos - 1, Channels), lists:nth(ElementPos, Channels)} + end, + + {Val, Continue} = calculate_alpha_beta(HasContinued, Element, PreviousElement, ReceiptChannel, WitnessChannel, Ledger), + {set_deltas(Challengee, Val, Acc), Continue}; + (_, Acc) -> + Acc + end, + {[], true}, + lists:zip(lists:seq(1, Length), Path)))) + catch + _:_ -> + [] + end. + +-spec check_path_continuation(Elements :: [blockchain_poc_path_element_v1:poc_element()]) -> boolean(). +check_path_continuation(Elements) -> + lists:any(fun(E) -> + blockchain_poc_path_element_v1:receipt(E) /= undefined orelse + blockchain_poc_path_element_v1:witnesses(E) /= [] + end, + Elements). + +-spec assign_alpha_beta(HasContinued :: boolean(), + Receipt :: undefined | blockchain_poc_receipt_v1:poc_receipt(), + Witnesses :: [blockchain_poc_witness_v1:poc_witness()]) -> {{float(), 0 | 1}, boolean()}. +assign_alpha_beta(HasContinued, Receipt, Witnesses) -> + case {HasContinued, Receipt, Witnesses} of + {true, undefined, _} -> + %% path continued, no receipt, don't care about witnesses + {{0.8, 0}, true}; + {true, Receipt, _} when Receipt /= undefined -> + %% path continued, receipt, don't care about witnesses + {{1, 0}, true}; + {false, undefined, Wxs} when length(Wxs) > 0 -> + %% path broke, no receipt, witnesses + {{0.9, 0}, true}; + {false, Receipt, []} when Receipt /= undefined -> + %% path broke, receipt, no witnesses + %% likely the next hop broke the path + case blockchain_poc_receipt_v1:origin(Receipt) of + p2p -> + %% you really did nothing here other than be online + {{0, 0}, false}; + radio -> + %% not enough information to decide who screwed up + %% but you did receive a packet over the radio, so you + %% get partial credit + {{0.2, 0}, false} + end; + {false, Receipt, Wxs} when Receipt /= undefined andalso length(Wxs) > 0 -> + %% path broke, receipt, witnesses + {{0.9, 0}, true}; + {false, _, _} -> + %% path broke, you killed it + {{0, 1}, false} + end. + +-spec calculate_alpha_beta(HasContinued :: boolean(), + Element :: blockchain_poc_path_element_v1:poc_element(), + PreviousElement :: blockchain_poc_path_element_v1:poc_element(), + ReceiptChannel :: non_neg_integer(), + WitnessChannel :: non_neg_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> {{float(), 0 | 1}, boolean()}. +calculate_alpha_beta(HasContinued, Element, PreviousElement, ReceiptChannel, WitnessChannel, Ledger) -> + Receipt = valid_receipt(PreviousElement, Element, ReceiptChannel, Ledger), + Witnesses = valid_witnesses(Element, WitnessChannel, Ledger), + allocate_alpha_beta(HasContinued, Element, Receipt, Witnesses, Ledger). + +-spec allocate_alpha_beta(HasContinued :: boolean(), + Element :: blockchain_poc_path_element_v1:poc_element(), + Receipt :: undefined | blockchain_poc_receipt_v1:poc_receipt(), + Witnesses :: [blockchain_poc_witness_v1:poc_witness()], + Ledger :: blockchain_ledger_v1:ledger()) -> {{float(), 0 | 1}, boolean()}. +allocate_alpha_beta(HasContinued, Element, Receipt, Witnesses, Ledger) -> + case {HasContinued, Receipt, Witnesses} of + {true, undefined, _} -> + %% path continued, no receipt, don't care about witnesses + {{0.8, 0}, true}; + {true, Receipt, _} when Receipt /= undefined -> + %% path continued, receipt, don't care about witnesses + {{1, 0}, true}; + {false, undefined, Wxs} when length(Wxs) > 0 -> + %% path broke, no receipt, witnesses + {calculate_witness_quality(Element, Ledger), true}; + {false, Receipt, []} when Receipt /= undefined -> + %% path broke, receipt, no witnesses + case blockchain_poc_receipt_v1:origin(Receipt) of + p2p -> + %% you really did nothing here other than be online + {{0, 0}, false}; + radio -> + %% not enough information to decide who screwed up + %% but you did receive a packet over the radio, so you + %% get partial credit + {{0.2, 0}, false} + end; + {false, Receipt, Wxs} when Receipt /= undefined andalso length(Wxs) > 0 -> + %% path broke, receipt, witnesses + {calculate_witness_quality(Element, Ledger), true}; + {false, _, _} -> + %% path broke, you killed it + {{0, 1}, false} + end. + +-spec calculate_witness_quality(Element :: blockchain_poc_path_element_v1:poc_element(), + Ledger :: blockchain_ledger_v1:ledger()) -> {float(), 0}. +calculate_witness_quality(Element, Ledger) -> + case blockchain_poc_path_element_v1:receipt(Element) of + undefined -> + %% no poc receipt + case good_quality_witnesses(Element, Ledger) of + [] -> + %% Either the witnesses are too close or the RSSIs are too high + %% no alpha bump + {0, 0}; + _ -> + %% high alpha bump, but not as high as when there is a receipt + {0.7, 0} + end; + _Receipt -> + %% element has a receipt + case good_quality_witnesses(Element, Ledger) of + [] -> + %% Either the witnesses are too close or the RSSIs are too high + %% no alpha bump + {0, 0}; + _ -> + %% high alpha bump + {0.9, 0} + end + end. + +-spec set_deltas(Challengee :: libp2p_crypto:pubkey_bin(), + {A :: float(), B :: 0 | 1}, + Deltas :: deltas()) -> deltas(). +set_deltas(Challengee, {A, B}, Deltas) -> + [{Challengee, {A, B}} | Deltas]. + +-spec good_quality_witnesses(Element :: blockchain_poc_path_element_v1:poc_element(), + Ledger :: blockchain_ledger_v1:ledger()) -> [blockchain_poc_witness_v1:poc_witness()]. +good_quality_witnesses(Element, Ledger) -> + {ok, POCVersion} = blockchain:config(?poc_version, Ledger), + good_quality_witnesses(POCVersion, Element, Ledger). + +-spec good_quality_witnesses(POCVersion :: pos_integer(), + Element :: blockchain_poc_path_element_v1:poc_element(), + Ledger :: blockchain_ledger_v1:ledger()) -> [blockchain_poc_witness_v1:poc_witness()]. +good_quality_witnesses(_POCVersion, Element, _Ledger) -> + Witnesses = blockchain_poc_path_element_v1:witnesses(Element), + Witnesses. + +%% Iterate over all poc_path elements and for each path element calls a given +%% callback function with reason tagged witnesses and valid receipt. +tagged_path_elements_fold(Fun, Acc0, Txn, Ledger, Chain) -> + try get_channels(Txn, Chain) of + {ok, Channels} -> + Path = ?MODULE:path(Txn), + lists:foldl(fun({ElementPos, Element}, Acc) -> + {PreviousElement, ReceiptChannel, WitnessChannel} = + case ElementPos of + 1 -> + {undefined, 0, hd(Channels)}; + _ -> + {lists:nth(ElementPos - 1, Path), lists:nth(ElementPos - 1, Channels), lists:nth(ElementPos, Channels)} + end, + + %% if either crashes, the whole thing is invalid from a rewards perspective + {FilteredReceipt, TaggedWitnesses} = + try {valid_receipt(PreviousElement, Element, ReceiptChannel, Ledger), + tagged_witnesses(Element, WitnessChannel, Ledger)} of + Res -> Res + catch _:_ -> + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + {undefined, [{false, <<"tagged_witnesses_crashed">>, Witness} || Witness <- Witnesses]} + end, + + Fun(Element, {TaggedWitnesses, FilteredReceipt}, Acc) + end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)); + {error, request_block_hash_not_found} -> [] + catch + throw:{error,{region_var_not_set,Region}} -> + Path = ?MODULE:path(Txn), + lists:foldl(fun({_ElementPos, Element}, Acc) -> + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + Fun(Element, {[{false, list_to_binary(io_lib:format("missing_region_parameters_for_~p", [Region])), Witness} || Witness <- Witnesses], undefined}, Acc) + end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)); + throw:{error,{unknown_region, UnknownH3}} -> + Path = ?MODULE:path(Txn), + lists:foldl(fun({_ElementPos, Element}, Acc) -> + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + Fun(Element, {lists:map(fun(Witness) -> {false, list_to_binary(io_lib:format("challengee_region_unknown_~p", [UnknownH3])), Witness} end, Witnesses) , undefined}, Acc) + end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)); + error:{badmatch, {error, {not_set, Region}}} -> + Path = ?MODULE:path(Txn), + lists:foldl(fun({_ElementPos, Element}, Acc) -> + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + Fun(Element, {[{false, list_to_binary(io_lib:format("missing_region_parameters_for_~p", [Region])), Witness} || Witness <- Witnesses], undefined}, Acc) + end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)); + error:{badmatch, {error, regulatory_regions_not_set}} -> + Path = ?MODULE:path(Txn), + lists:foldl(fun({_ElementPos, Element}, Acc) -> + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + Fun(Element, {[{false, <<"regulatory_regions_not_set">>, Witness} || Witness <- Witnesses], undefined}, Acc) + end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)) + end. + + +%% again this is broken because of the current witness situation + +%% Iterate over all poc_path elements and for each path element calls a given +%% callback function with the valid witnesses and valid receipt. +%% valid_path_elements_fold(Fun, Acc0, Txn, Ledger, Chain) -> +%% Path = ?MODULE:path(Txn), +%% try get_channels(Txn, Chain) of +%% {ok, Channels} -> +%% lists:foldl(fun({ElementPos, Element}, Acc) -> +%% {PreviousElement, ReceiptChannel, WitnessChannel} = +%% case ElementPos of +%% 1 -> +%% {undefined, 0, hd(Channels)}; +%% _ -> +%% {lists:nth(ElementPos - 1, Path), lists:nth(ElementPos - 1, Channels), lists:nth(ElementPos, Channels)} +%% end, + +%% FilteredReceipt = valid_receipt(PreviousElement, Element, ReceiptChannel, Ledger), +%% FilteredWitnesses = valid_witnesses(Element, WitnessChannel, Ledger), + +%% Fun(Element, {FilteredWitnesses, FilteredReceipt}, Acc) +%% end, Acc0, lists:zip(lists:seq(1, length(Path)), Path)) +%% catch _:_ -> +%% Acc0 +%% end. + + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- + -spec absorb(txn_poc_receipts(), blockchain:blockchain()) -> ok | {error, atom()} | {error, {atom(), any()}}. +absorb(Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + {ok, POCVersion} = blockchain:config(?poc_version, Ledger), + absorb(POCVersion, Txn, Chain). + + -spec absorb(pos_integer(), txn_poc_receipts(), blockchain:blockchain()) -> ok | {error, atom()} | {error, {atom(), any()}}. +absorb(_POCVersion, Txn, Chain) -> + OnionKeyHash = ?MODULE:onion_key_hash(Txn), + Challenger = ?MODULE:challenger(Txn), + BlockHash = ?MODULE:block_hash(Txn), + Ledger = blockchain:ledger(Chain), + POCID = ?MODULE:poc_id(Txn), + try + %% get these to make sure we're not replaying. + PoC = case blockchain_ledger_v1:find_public_poc(OnionKeyHash, Ledger) of + {ok, Ps} -> + Ps; + {error, not_found} -> + lager:warning("potential replay: ~p not found", [OnionKeyHash]), + throw(replay) + end, + lager:info("POC POC POC ~p", [PoC]), + case blockchain_ledger_poc_v3:verify(PoC, Challenger, BlockHash) of + false -> + {error, invalid_poc}; + true -> + %% get rid of this for the time being, we will need to restore it later when + %% the clean witness restore thing lands + + %% %% Add filtered witnesses with poc-v9 + %% ok = valid_path_elements_fold(fun(Element, {FilteredWitnesses, FilteredReceipt}, _) -> + %% Challengee = blockchain_poc_path_element_v1:challengee(Element), + %% case FilteredReceipt of + %% undefined -> + %% ok = blockchain_ledger_v1:insert_witnesses(Challengee, FilteredWitnesses, Ledger); + %% FR -> + %% ok = blockchain_ledger_v1:insert_witnesses(Challengee, FilteredWitnesses ++ [FR], Ledger) + %% end + %% end, ok, Txn, Ledger, Chain); + + %% This isn't ideal, but we need to do delta calculation _before_ we delete the poc + %% as new calculate_delta calls back into check_is_valid_poc + case blockchain:config(?election_version, Ledger) of + %% election v4 removed score from consideration + {ok, EV} when EV >= 4 -> + ok; + _ -> + lists:foreach(fun({Gateway, Delta}) -> + blockchain_ledger_v1:update_gateway_score(Gateway, Delta, Ledger) + end, + ?MODULE:deltas(Txn, Chain)) + end, + ok = blockchain_ledger_v1:delete_public_poc(OnionKeyHash, Ledger) + end + catch throw:Reason -> + {error, Reason}; + What:Why:Stacktrace -> + lager:error([{poc_id, POCID}], "poc receipt calculation failed: ~p ~p ~p", + [What, Why, Stacktrace]), + {error, state_missing} + end. + +%%-------------------------------------------------------------------- +%% @doc +%% Insert witnesses for gateways in the path into the ledger +%% @end +%%-------------------------------------------------------------------- +%%-spec insert_witnesses(Path :: blockchain_poc_path_element_v1:path(), +%% LowerTimeBound :: non_neg_integer(), +%% UpperTimeBound :: non_neg_integer(), +%% Ledger :: blockchain_ledger_v1:ledger()) -> ok. +%%insert_witnesses(Path, LowerTimeBound, UpperTimeBound, Ledger) -> +%% Length = length(Path), +%% lists:foreach(fun({N, Element}) -> +%% Challengee = blockchain_poc_path_element_v1:challengee(Element), +%% Witnesses = blockchain_poc_path_element_v1:witnesses(Element), +%% %% TODO check these witnesses have valid RSSI +%% WitnessInfo0 = lists:foldl(fun(Witness, Acc) -> +%% TS = case blockchain_poc_witness_v1:timestamp(Witness) of +%% T when T < LowerTimeBound -> +%% LowerTimeBound; +%% T when T > UpperTimeBound -> +%% UpperTimeBound; +%% T -> +%% T +%% end, +%% [{blockchain_poc_witness_v1:signal(Witness), TS, blockchain_poc_witness_v1:gateway(Witness)} | Acc] +%% end, +%% [], +%% Witnesses), +%% NextElements = lists:sublist(Path, N+1, Length), +%% WitnessInfo = case check_path_continuation(NextElements) of +%% true -> +%% %% the next hop is also a witness for this +%% NextHopElement = hd(NextElements), +%% NextHopAddr = blockchain_poc_path_element_v1:challengee(NextHopElement), +%% case blockchain_poc_path_element_v1:receipt(NextHopElement) of +%% undefined -> +%% %% There is no receipt from the next hop +%% %% We clamp to LowerTimeBound as best-effort +%% [{undefined, LowerTimeBound, NextHopAddr} | WitnessInfo0]; +%% NextHopReceipt -> +%% [{blockchain_poc_receipt_v1:signal(NextHopReceipt), +%% blockchain_poc_receipt_v1:timestamp(NextHopReceipt), +%% NextHopAddr} | WitnessInfo0] +%% end; +%% false -> +%% WitnessInfo0 +%% end, +%% blockchain_ledger_v1:add_gateway_witnesses(Challengee, WitnessInfo, Ledger) +%% end, lists:zip(lists:seq(1, Length), Path)). + +%%-------------------------------------------------------------------- +%% @doc +%% @end +%%-------------------------------------------------------------------- +-spec create_secret_hash(binary(), non_neg_integer()) -> [binary()]. +create_secret_hash(Secret, X) when X > 0 -> + create_secret_hash(Secret, X, []). + +-spec create_secret_hash(binary(), non_neg_integer(), [binary()]) -> [binary()]. +create_secret_hash(_Secret, 0, Acc) -> + Acc; +create_secret_hash(Secret, X, []) -> + Bin = crypto:hash(sha256, Secret), + <> = Bin, + create_secret_hash(Bin, X-1, [Hash]); +create_secret_hash(Secret, X, Acc) -> + Bin = <> = crypto:hash(sha256, Secret), + create_secret_hash(Bin, X-1, [Hash|Acc]). + +%% @doc Validate the proof of coverage receipt path. +%% +%% The proof of coverage receipt path consists of a list of `poc path elements', +%% each of which corresponds to a layer of the onion-encrypted PoC packet. This +%% path element records who the layer was intended for, if we got a receipt from +%% them and any other peers that witnessed this packet but were unable to decrypt +%% it. This function verifies that all the receipts and witnesses appear in the +%% expected order, and satisfy all the validity checks. +%% +%% Because the first packet is delivered over p2p, it cannot have witnesses and +%% the receipt must indicate the origin was `p2p'. All subsequent packets must +%% have the receipt origin as `radio'. The final layer has the packet composed +%% entirely of padding, so there cannot be a valid receipt, but there can be +%% witnesses (as evidence that the final recipient transmitted it). +-spec validate(pos_integer(), txn_poc_receipts(), list(), + [binary(), ...], [binary(), ...], blockchain_ledger_v1:ledger()) -> ok | {error, atom()}. +validate(_POCVersion, Txn, Path, LayerData, LayerHashes, OldLedger) -> + TxnPath = ?MODULE:path(Txn), + TxnPathLength = length(TxnPath), + RebuiltPathLength = length(Path), + ZippedLayers = lists:zip(LayerData, LayerHashes), + ZippedLayersLength = length(ZippedLayers), + POCID = ?MODULE:poc_id(Txn), + lager:debug([{poc_id, POCID}], "starting poc receipt validation..."), + + case TxnPathLength == RebuiltPathLength of + false -> + HumanTxnPath = [element(2, erl_angry_purple_tiger:animal_name(libp2p_crypto:bin_to_b58(blockchain_poc_path_element_v1:challengee(E)))) || E <- TxnPath], + HumanRebuiltPath = [element(2, erl_angry_purple_tiger:animal_name(libp2p_crypto:bin_to_b58(A))) || A <- Path], + lager:warning([{poc_id, POCID}], "TxnPathLength: ~p, RebuiltPathLength: ~p", + [TxnPathLength, RebuiltPathLength]), + lager:warning([{poc_id, POCID}], "TxnPath: ~p", [HumanTxnPath]), + lager:warning([{poc_id, POCID}], "RebuiltPath: ~p", [HumanRebuiltPath]), + blockchain_ledger_v1:delete_context(OldLedger), + {error, path_length_mismatch}; + true -> + %% Now check whether layers are of equal length + case TxnPathLength == ZippedLayersLength of + false -> + lager:error([{poc_id, POCID}], "TxnPathLength: ~p, ZippedLayersLength: ~p", + [TxnPathLength, ZippedLayersLength]), + blockchain_ledger_v1:delete_context(OldLedger), + {error, zip_layer_length_mismatch}; + true -> + PerHopMaxWitnesses = blockchain_utils:poc_per_hop_max_witnesses(OldLedger), + Result = lists:foldl( + fun(_, {error, _} = Error) -> + Error; + ({Elem, Gateway, {LayerDatum, LayerHash}}, _Acc) -> + case blockchain_poc_path_element_v1:challengee(Elem) == Gateway of + true -> + IsFirst = Elem == hd(?MODULE:path(Txn)), + Receipt = blockchain_poc_path_element_v1:receipt(Elem), + ExpectedOrigin = case IsFirst of + true -> p2p; + false -> radio + end, + %% check the receipt + case + Receipt == undefined orelse + (blockchain_poc_receipt_v1:is_valid(Receipt, OldLedger) andalso + blockchain_poc_receipt_v1:gateway(Receipt) == Gateway andalso + blockchain_poc_receipt_v1:data(Receipt) == LayerDatum andalso + blockchain_poc_receipt_v1:origin(Receipt) == ExpectedOrigin) + of + true -> + %% ok the receipt looks good, check the witnesses + Witnesses = blockchain_poc_path_element_v1:witnesses(Elem), + case erlang:length(Witnesses) > PerHopMaxWitnesses of + true -> + {error, too_many_witnesses}; + false -> + %% check there are no duplicates in witnesses list + WitnessGateways = [blockchain_poc_witness_v1:gateway(W) || W <- Witnesses], + case length(WitnessGateways) == length(lists:usort(WitnessGateways)) of + false -> + {error, duplicate_witnesses}; + true -> + check_witness_layerhash(Witnesses, Gateway, LayerHash, OldLedger) + end + end; + false -> + case Receipt == undefined of + true -> + lager:warning([{poc_id, POCID}], + "Receipt undefined, ExpectedOrigin: ~p, LayerDatum: ~p, Gateway: ~p", + [Receipt, ExpectedOrigin, LayerDatum, Gateway]); + false -> + lager:warning([{poc_id, POCID}], + "Origin: ~p, ExpectedOrigin: ~p, Data: ~p, LayerDatum: ~p, ReceiptGateway: ~p, Gateway: ~p", + [blockchain_poc_receipt_v1:origin(Receipt), + ExpectedOrigin, + blockchain_poc_receipt_v1:data(Receipt), + LayerDatum, + blockchain_poc_receipt_v1:gateway(Receipt), + Gateway]) + end, + {error, invalid_receipt} + end; + _ -> + lager:error([{poc_id, POCID}], "receipt not in order"), + {error, receipt_not_in_order} + end + end, + ok, + lists:zip3(TxnPath, Path, ZippedLayers) + ), + %% clean up ledger context + blockchain_ledger_v1:delete_context(OldLedger), + Result + + end + end. + +print(#blockchain_txn_poc_receipts_v2_pb{ + challenger=Challenger, + onion_key_hash=OnionKeyHash, + path=Path + }=Txn) -> + io_lib:format("type=poc_receipts_v2 hash=~p challenger=~p onion=~p path:\n\t~s", + [?TO_B58(?MODULE:hash(Txn)), + ?TO_ANIMAL_NAME(Challenger), + ?TO_B58(OnionKeyHash), + print_path(Path)]). + +print_path(Path) -> + string:join(lists:map(fun(Element) -> + blockchain_poc_path_element_v1:print(Element) + end, + Path), "\n\t"). + +json_type() -> + <<"poc_receipts_v2">>. + +-spec to_json(Txn :: txn_poc_receipts(), + Opts :: blockchain_json:opts()) -> blockchain_json:json_object(). +to_json(Txn, Opts) -> + PathElems = + case {lists:keyfind(ledger, 1, Opts), lists:keyfind(chain, 1, Opts)} of + {{ledger, Ledger}, {chain, Chain}} -> + FoldedPath = + tagged_path_elements_fold(fun(Elem, {TaggedWitnesses, ValidReceipt}, Acc) -> + ElemOpts = [{tagged_witnesses, TaggedWitnesses}, + {valid_receipt, ValidReceipt}], + [{Elem, ElemOpts} | Acc] + end, [], Txn, Ledger, Chain), + lists:reverse(FoldedPath); + {_, _} -> + [{Elem, []} || Elem <- path(Txn)] + end, + #{ + type => ?MODULE:json_type(), + hash => ?BIN_TO_B64(hash(Txn)), + secret => ?BIN_TO_B64(secret(Txn)), + onion_key_hash => ?BIN_TO_B64(onion_key_hash(Txn)), + path => [blockchain_poc_path_element_v1:to_json(Elem, ElemOpts) || {Elem, ElemOpts} <- PathElems], + fee => fee(Txn), + challenger => ?BIN_TO_B58(challenger(Txn)), + block_hash => ?BIN_TO_B64(block_hash(Txn)) + }. + + +check_witness_layerhash(Witnesses, Gateway, LayerHash, OldLedger) -> + %% all the witnesses should have the right LayerHash + %% and be valid + case + lists:all( + fun(Witness) -> + %% the witnesses should have an asserted location + %% at the point when the request was mined! + WitnessGateway = blockchain_poc_witness_v1:gateway(Witness), + case blockchain_ledger_v1:find_gateway_location(WitnessGateway, OldLedger) of + {error, _} -> + false; + {ok, _} when Gateway == WitnessGateway -> + false; + {ok, GWLoc} -> + GWLoc /= undefined andalso + blockchain_poc_witness_v1:is_valid(Witness, OldLedger) andalso + blockchain_poc_witness_v1:packet_hash(Witness) == LayerHash + end + end, + Witnesses + ) + of + true -> ok; + false -> {error, invalid_witness} + end. + +-spec poc_id(txn_poc_receipts()) -> binary(). +poc_id(Txn) -> + ?BIN_TO_B64(?MODULE:onion_key_hash(Txn)). + +vars(Ledger) -> + blockchain_utils:vars_binary_keys_to_atoms( + maps:from_list(blockchain_ledger_v1:snapshot_vars(Ledger))). + +-spec valid_receipt(PreviousElement :: undefined | blockchain_poc_path_element_v1:poc_element(), + Element :: blockchain_poc_path_element_v1:poc_element(), + Channel :: non_neg_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> undefined | blockchain_poc_receipt_v1:poc_receipt(). +valid_receipt(undefined, _Element, _Channel, _Ledger) -> + %% first hop in the path, cannot be validated. + undefined; +valid_receipt(PreviousElement, Element, Channel, Ledger) -> + case blockchain_poc_path_element_v1:receipt(Element) of + undefined -> + %% nothing to validate + undefined; + Receipt -> + Version = poc_version(Ledger), + DstPubkeyBin = blockchain_poc_path_element_v1:challengee(Element), + SrcPubkeyBin = blockchain_poc_path_element_v1:challengee(PreviousElement), + {ok, SourceLoc} = blockchain_ledger_v1:find_gateway_location(SrcPubkeyBin, Ledger), + {ok, DestinationLoc} = blockchain_ledger_v1:find_gateway_location(DstPubkeyBin, Ledger), + SourceRegion = blockchain_region_v1:h3_to_region(SourceLoc, Ledger), + DestinationRegion = blockchain_region_v1:h3_to_region(DestinationLoc, Ledger), + {ok, ExclusionCells} = blockchain_ledger_v1:config(?poc_v4_exclusion_cells, Ledger), + {ok, ParentRes} = blockchain_ledger_v1:config(?poc_v4_parent_res, Ledger), + SourceParentIndex = h3:parent(SourceLoc, ParentRes), + DestinationParentIndex = h3:parent(DestinationLoc, ParentRes), + + case is_same_region(Version, SourceRegion, DestinationRegion) of + false -> + lager:debug("Not in the same region!~nSrcPubkeyBin: ~p, DstPubkeyBin: ~p, SourceLoc: ~p, DestinationLoc: ~p", + [blockchain_utils:addr2name(SrcPubkeyBin), + blockchain_utils:addr2name(DstPubkeyBin), + SourceRegion, DestinationRegion]), + undefined; + true -> + Limit = blockchain:config(?poc_distance_limit, Ledger), + case is_too_far(Limit, SourceLoc, DestinationLoc) of + {true, Distance} -> + lager:debug("Src too far from destination!~nSrcPubkeyBin: ~p, DstPubkeyBin: ~p, SourceLoc: ~p, DestinationLoc: ~p, Distance: ~p", + [blockchain_utils:addr2name(SrcPubkeyBin), + blockchain_utils:addr2name(DstPubkeyBin), + SourceLoc, DestinationLoc, Distance]), + undefined; + false -> + try h3:grid_distance(SourceParentIndex, DestinationParentIndex) of + Dist when Dist >= ExclusionCells -> + RSSI = blockchain_poc_receipt_v1:signal(Receipt), + SNR = blockchain_poc_receipt_v1:snr(Receipt), + Freq = blockchain_poc_receipt_v1:frequency(Receipt), + MinRcvSig = min_rcv_sig(Receipt, Ledger, SourceLoc, SourceRegion, DstPubkeyBin, DestinationLoc, Freq, Version), + case RSSI < MinRcvSig of + false -> + %% RSSI is impossibly high discard this receipt + lager:debug("receipt ~p -> ~p rejected at height ~p for RSSI ~p above FSPL ~p with SNR ~p", + [?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(PreviousElement)), + ?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(Element)), + element(2, blockchain_ledger_v1:current_height(Ledger)), + RSSI, MinRcvSig, SNR]), + undefined; + true -> + case check_valid_frequency(SourceRegion, Freq, Ledger, Version) of + true -> + case blockchain:config(?data_aggregation_version, Ledger) of + {ok, DataAggVsn} when DataAggVsn > 1 -> + case blockchain_poc_receipt_v1:channel(Receipt) == Channel of + true -> + lager:debug("receipt ok"), + Receipt; + false -> + lager:debug("receipt ~p -> ~p rejected at height ~p for channel ~p /= ~p RSSI ~p SNR ~p", + [?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(PreviousElement)), + ?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(Element)), + element(2, blockchain_ledger_v1:current_height(Ledger)), + blockchain_poc_receipt_v1:channel(Receipt), Channel, + RSSI, SNR]), + undefined + end; + _ -> + %% SNR+Freq+Channels not collected, nothing else we can check + Receipt + end; + _ -> + undefined + end + end; + _ -> + %% too close + undefined + catch + _:_ -> + %% pentagonal distortion + undefined + end + end + end + + end. + +-spec valid_witnesses(Element :: blockchain_poc_path_element_v1:poc_element(), + Channel :: non_neg_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> blockchain_poc_witness_v1:poc_witnesses(). +valid_witnesses(Element, Channel, Ledger) -> + TaggedWitnesses = tagged_witnesses(Element, Channel, Ledger), + [ W || {true, _, W} <- TaggedWitnesses ]. + +valid_witnesses(Element, Channel, RegionVars, Ledger) -> + TaggedWitnesses = tagged_witnesses(Element, Channel, RegionVars, Ledger), + [ W || {true, _, W} <- TaggedWitnesses ]. + +-spec is_too_far(Limit :: any(), + SrcLoc :: h3:h3_index(), + DstLoc :: h3:h3_index()) -> {true, float()} | false. +is_too_far(Limit, SrcLoc, DstLoc) -> + case Limit of + {ok, L} -> + Distance = blockchain_utils:distance(SrcLoc, DstLoc), + case Distance > L of + true -> + {true, Distance}; + false -> + false + end; + _ -> + %% var not set, it's not too far (don't consider it) + false + end. + +-spec check_valid_frequency(Region0 :: {error, any()} | {ok, atom()}, + Frequency :: float(), + Ledger :: blockchain_ledger_v1:ledger(), + Version :: non_neg_integer()) -> boolean(). +check_valid_frequency(Region0, Frequency, Ledger, _Version) -> + {ok, Region} = Region0, + {ok, Params} = blockchain_region_params_v1:for_region(Region, Ledger), + ChannelFreqs = [blockchain_region_param_v1:channel_frequency(I) || I <- Params], + lists:any(fun(E) -> abs(E - Frequency*?MHzToHzMultiplier) =< 1000 end, ChannelFreqs). + +-spec is_same_region( + Version :: non_neg_integer(), + SourceRegion :: {error, any()} | {ok, atom()}, + DstRegion :: {error, any()} | {ok, atom()} +) -> boolean(). +is_same_region(_Version, SourceRegion0, DstRegion0) -> + {ok, SourceRegion} = SourceRegion0, + case DstRegion0 of + {ok, DstRegion} -> + SourceRegion == DstRegion; + {error, _} -> + false + end. + + +%% This function adds a tag to each witness specifying a reason why a witness was considered invalid, +%% Further, this same function is used to check witness validity in valid_witnesses fun. +-spec tagged_witnesses(Element :: blockchain_poc_path_element_v1:poc_element(), + Channel :: non_neg_integer(), + Ledger :: blockchain_ledger_v1:ledger()) -> tagged_witnesses(). +tagged_witnesses(Element, Channel, Ledger) -> + {ok, RegionVars} = blockchain_region_v1:get_all_region_bins(Ledger), + tagged_witnesses(Element, Channel, RegionVars, Ledger). + +%%-spec tagged_witnesses(Element :: blockchain_poc_path_element_v1:poc_element(), +%% Channel :: non_neg_integer(), +%% RegionVars0 :: no_prefetch | [{atom(), binary() | {error, any()}}] | {ok, [{atom(), binary() | {error, any()}}]}, +%% Ledger :: blockchain_ledger_v1:ledger()) -> tagged_witnesses(). +tagged_witnesses(Element, Channel, RegionVars0, Ledger) -> + SrcPubkeyBin = blockchain_poc_path_element_v1:challengee(Element), + {ok, SourceLoc} = blockchain_ledger_v1:find_gateway_location(SrcPubkeyBin, Ledger), + RegionVars = + case RegionVars0 of + {ok, RV} -> RV; + RV when is_list(RV) -> RV; + {error, _Reason} -> no_prefetch + end, + SourceRegion = blockchain_region_v1:h3_to_region(SourceLoc, Ledger, RegionVars), + {ok, ParentRes} = blockchain_ledger_v1:config(?poc_v4_parent_res, Ledger), + SourceParentIndex = h3:parent(SourceLoc, ParentRes), + + %% foldl will re-reverse + Witnesses = lists:reverse(blockchain_poc_path_element_v1:witnesses(Element)), + + DiscardZeroFreq = blockchain_ledger_v1:config(?discard_zero_freq_witness, Ledger), + {ok, ExclusionCells} = blockchain_ledger_v1:config(?poc_v4_exclusion_cells, Ledger), + %% intentionally do not require + DAV = blockchain:config(?data_aggregation_version, Ledger), + Limit = blockchain:config(?poc_distance_limit, Ledger), + Version = poc_version(Ledger), + + lists:foldl(fun(Witness, Acc) -> + DstPubkeyBin = blockchain_poc_witness_v1:gateway(Witness), + {ok, DestinationLoc} = blockchain_ledger_v1:find_gateway_location(DstPubkeyBin, Ledger), + DestinationRegion = blockchain_region_v1:h3_to_region(DestinationLoc, Ledger, RegionVars), +%% case blockchain_region_v1:h3_to_region(DestinationLoc, Ledger, RegionVars) of +%% {error, {unknown_region, _Loc}} when Version >= 11 -> +%% lager:warning("saw unknown region for ~p loc ~p", +%% [DstPubkeyBin, DestinationLoc]), +%% unknown; +%% {error, _} -> unknown; +%% {ok, DR} -> {ok, DR} +%% end, + DestinationParentIndex = h3:parent(DestinationLoc, ParentRes), + Freq = blockchain_poc_witness_v1:frequency(Witness), + + case {DiscardZeroFreq, Freq} of + {{ok, true}, 0.0} -> + [{false, <<"witness_zero_freq">>, Witness} | Acc]; + _ -> + case is_same_region(Version, SourceRegion, DestinationRegion) of + false -> + lager:debug("Not in the same region!~nSrcPubkeyBin: ~p, DstPubkeyBin: ~p, SourceLoc: ~p, DestinationLoc: ~p", + [blockchain_utils:addr2name(SrcPubkeyBin), + blockchain_utils:addr2name(DstPubkeyBin), + SourceLoc, DestinationLoc]), + [{false, <<"witness_not_same_region">>, Witness} | Acc]; + true -> + case is_too_far(Limit, SourceLoc, DestinationLoc) of + {true, Distance} -> + lager:debug("Src too far from destination!~nSrcPubkeyBin: ~p, DstPubkeyBin: ~p, SourceLoc: ~p, DestinationLoc: ~p, Distance: ~p", + [blockchain_utils:addr2name(SrcPubkeyBin), + blockchain_utils:addr2name(DstPubkeyBin), + SourceLoc, DestinationLoc, Distance]), + [{false, <<"witness_too_far">>, Witness} | Acc]; + false -> + try h3:grid_distance(SourceParentIndex, DestinationParentIndex) of + Dist when Dist >= ExclusionCells -> + RSSI = blockchain_poc_witness_v1:signal(Witness), + SNR = blockchain_poc_witness_v1:snr(Witness), + MinRcvSig = min_rcv_sig(blockchain_poc_path_element_v1:receipt(Element), + Ledger, + SourceLoc, + SourceRegion, + DstPubkeyBin, + DestinationLoc, + Freq, + Version), + + case RSSI < MinRcvSig of + false -> + %% RSSI is impossibly high discard this witness + lager:debug("witness ~p -> ~p rejected at height ~p for RSSI ~p above FSPL ~p with SNR ~p", + [?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(Element)), + ?TO_ANIMAL_NAME(blockchain_poc_witness_v1:gateway(Witness)), + element(2, blockchain_ledger_v1:current_height(Ledger)), + RSSI, MinRcvSig, SNR]), + [{false, <<"witness_rssi_too_high">>, Witness} | Acc]; + true -> + case check_valid_frequency(SourceRegion, Freq, Ledger, Version) of + true -> + case DAV of + {ok, DataAggVsn} when DataAggVsn > 1 -> + case blockchain_poc_witness_v1:channel(Witness) == Channel of + true -> + lager:debug("witness ok"), + [{true, <<"ok">>, Witness} | Acc]; + false -> + lager:debug("witness ~p -> ~p rejected at height ~p for channel ~p /= ~p RSSI ~p SNR ~p", + [?TO_ANIMAL_NAME(blockchain_poc_path_element_v1:challengee(Element)), + ?TO_ANIMAL_NAME(blockchain_poc_witness_v1:gateway(Witness)), + element(2, blockchain_ledger_v1:current_height(Ledger)), + blockchain_poc_witness_v1:channel(Witness), Channel, + RSSI, SNR]), + [{false, <<"witness_on_incorrect_channel">>, Witness} | Acc] + end; + _ -> + %% SNR+Freq+Channels not collected, nothing else we can check + [{true, <<"insufficient_data">>, Witness} | Acc] + end; + _ -> + [{false, <<"incorrect_frequency">>, Witness} | Acc] + end + end; + _ -> + %% too close + [{false, <<"witness_too_close">>, Witness} | Acc] + catch _:_ -> + %% pentagonal distortion + [{false, <<"pentagonal_distortion">>, Witness} | Acc] + end + + end + end + end + end, [], Witnesses). + +-spec get_channels(Txn :: txn_poc_receipts(), + Chain :: blockchain:blockchain()) -> {ok, [non_neg_integer()]} | {error, any()}. +get_channels(Txn, Chain) -> + Ledger = blockchain:ledger(Chain), + Version = poc_version(Ledger), + {ok, RegionVars} = blockchain_region_v1:get_all_region_bins(Ledger), + get_channels(Txn, Version, RegionVars, Chain). + +-spec get_channels(Txn :: txn_poc_receipts(), + POCVersion :: pos_integer(), + RegionVars :: no_prefetch | [{atom(), binary() | {error, any()}}] | {ok, [{atom(), binary() | {error, any()}}]}, + Chain :: blockchain:blockchain()) -> {ok, [non_neg_integer()]} | {error, any()}. +get_channels(Txn, POCVersion, RegionVars, Chain) -> + Path0 = ?MODULE:path(Txn), + PathLength = length(Path0), + BlockHash = ?MODULE:block_hash(Txn), + OnionKeyHash = ?MODULE:onion_key_hash(Txn), + Ledger = blockchain:ledger(Chain), + case BlockHash of + <<>> -> + {error, request_block_hash_not_found}; + undefined -> + {error, request_block_hash_not_found}; + _BH -> + Entropy1 = <>, + [_ | LayerData] = blockchain_txn_poc_receipts_v2:create_secret_hash(Entropy1, PathLength+1), + Path = [blockchain_poc_path_element_v1:challengee(Element) || Element <- Path0], + Channels = get_channels_(POCVersion, Ledger, Path, LayerData, RegionVars), + {ok, Channels} + end. + +-spec get_channels_(POCVersion :: pos_integer(), + Ledger :: blockchain_ledger_v1:ledger(), + Path :: [libp2p_crypto:pubkey_bin()], + LayerData :: [binary()], + RegionVars :: no_prefetch | [{atom(), binary() | {error, any()}}] | {ok, [{atom(), binary() | {error, any()}}]} | {error, any()}) -> [non_neg_integer()]. +get_channels_(_POCVersion, Ledger, Path, LayerData, RegionVars0) -> + Challengee = hd(Path), + RegionVars = + case RegionVars0 of + {ok, RV} -> RV; + RV when is_list(RV) -> RV; + no_prefetch -> no_prefetch; + {error, Reason} -> error({get_channels_region, Reason}) + end, + + ChannelCount = + %% Get from region vars + %% Just get the channels using the challengee's region from head of the path + %% We assert that all path members (which is only 1 member, beacon right now) + %% will be in the same region + case blockchain_ledger_v1:find_gateway_location(Challengee, Ledger) of + {error, _}=E -> + throw(E); + {ok, ChallengeeLoc} -> + case blockchain_region_v1:h3_to_region(ChallengeeLoc, Ledger, RegionVars) of + {error, _}=E -> + throw(E); + {ok, Region} -> + {ok, Params} = blockchain_region_params_v1:for_region(Region, Ledger), + length(Params) + end + end, + lists:map(fun(Layer) -> + <> = Layer, + IntData rem ChannelCount + end, LayerData). + +-spec min_rcv_sig(Receipt :: undefined | blockchain_poc_receipt_v1:receipt(), + Ledger :: blockchain_ledger_v1:ledger(), + SourceLoc :: h3:h3_index(), + SourceRegion0 :: {ok, atom()} | {error, any()}, + DstPubkeyBin :: libp2p_crypto:pubkey_bin(), + DestinationLoc :: h3:h3_index(), + Freq :: float(), + POCVersion :: non_neg_integer()) -> float(). +min_rcv_sig(undefined, Ledger, SourceLoc, SourceRegion0, DstPubkeyBin, DestinationLoc, Freq, _POCVersion) -> + %% Receipt can be undefined + %% Estimate tx power because there is no receipt with attached tx_power + lager:debug("SourceLoc: ~p, Freq: ~p", [SourceLoc, Freq]), + {ok, SourceRegion} = SourceRegion0, + {ok, TxPower} = estimated_tx_power(SourceRegion, Freq, Ledger), + FSPL = calc_fspl(DstPubkeyBin, SourceLoc, DestinationLoc, Freq, Ledger), + case blockchain:config(?fspl_loss, Ledger) of + {ok, Loss} -> blockchain_utils:min_rcv_sig(FSPL, TxPower) * Loss; + _ -> blockchain_utils:min_rcv_sig(FSPL, TxPower) + end; +min_rcv_sig(Receipt, Ledger, SourceLoc, SourceRegion0, DstPubkeyBin, DestinationLoc, Freq, POCVersion) -> + %% We do have a receipt + %% Get tx_power from attached receipt and use it to calculate min_rcv_sig + case blockchain_poc_receipt_v1:tx_power(Receipt) of + %% Missing protobuf fields have default value as 0 + TxPower when TxPower == undefined; TxPower == 0 -> + min_rcv_sig(undefined, Ledger, SourceLoc, SourceRegion0, + DstPubkeyBin, DestinationLoc, Freq, POCVersion); + TxPower -> + FSPL = calc_fspl(DstPubkeyBin, SourceLoc, DestinationLoc, Freq, Ledger), + case blockchain:config(?fspl_loss, Ledger) of + {ok, Loss} -> blockchain_utils:min_rcv_sig(FSPL, TxPower) * Loss; + _ -> blockchain_utils:min_rcv_sig(FSPL, TxPower) + end + end. + +calc_fspl(DstPubkeyBin, SourceLoc, DestinationLoc, Freq, Ledger) -> + {ok, DstGR} = blockchain_ledger_v1:find_gateway_gain(DstPubkeyBin, Ledger), + %% NOTE: Transmit gain is set to 0 when calculating free_space_path_loss + %% This is because the packet forwarder will be configured to subtract the antenna + %% gain and miner will always transmit at region EIRP. + GT = 0, + GR = DstGR / 10, + blockchain_utils:free_space_path_loss(SourceLoc, DestinationLoc, Freq, GT, GR). + +estimated_tx_power(Region, Freq, Ledger) -> + {ok, Params} = blockchain_region_params_v1:for_region(Region, Ledger), + FreqEirps = [{blockchain_region_param_v1:channel_frequency(I), + blockchain_region_param_v1:max_eirp(I)} || I <- Params], + %% NOTE: Convert src frequency to Hz before checking freq match for EIRP value + EIRP = eirp_from_closest_freq(Freq * ?MHzToHzMultiplier, FreqEirps), + {ok, EIRP / 10}. + +eirp_from_closest_freq(Freq, [Head | Tail]) -> + eirp_from_closest_freq(Freq, Tail, Head). + +eirp_from_closest_freq(_Freq, [], {_BestFreq, BestEIRP}) -> BestEIRP; +eirp_from_closest_freq(Freq, [ {NFreq, NEirp} | Rest ], {BestFreq, BestEIRP}) -> + case abs(Freq - NFreq) =< abs(Freq - BestFreq) of + true -> + eirp_from_closest_freq(Freq, Rest, {NFreq, NEirp}); + false -> + eirp_from_closest_freq(Freq, Rest, {BestFreq, BestEIRP}) + end. + +-spec poc_version(blockchain_ledger_v1:ledger()) -> non_neg_integer(). +poc_version(Ledger) -> + case blockchain:config(?poc_version, Ledger) of + {error, not_found} -> 0; + {ok, V} -> V + end. + +%% ------------------------------------------------------------------ +%% EUNIT Tests +%% ------------------------------------------------------------------ +-ifdef(TEST). + +new_test() -> + Tx = #blockchain_txn_poc_receipts_v2_pb{ + challenger = <<"challenger">>, + secret = <<"secret">>, + onion_key_hash = <<"onion">>, + path=[], + fee = 0, + signature = <<>>, + block_hash = <<"blockhash">> + }, + ?assertEqual(Tx, new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>)). + +onion_key_hash_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(<<"onion">>, onion_key_hash(Tx)). + +challenger_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(<<"challenger">>, challenger(Tx)). + +blockhash_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(<<"blockhash">>, block_hash(Tx)). + +secret_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(<<"secret">>, secret(Tx)). + +path_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual([], path(Tx)). + +fee_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(0, fee(Tx)). + +signature_test() -> + Tx = new(<<"challenger">>, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + ?assertEqual(<<>>, signature(Tx)). + +sign_test() -> + #{public := PubKey, secret := PrivKey} = libp2p_crypto:generate_keys(ecc_compact), + Challenger = libp2p_crypto:pubkey_to_bin(PubKey), + SigFun = libp2p_crypto:mk_sig_fun(PrivKey), + Tx0 = new(Challenger, <<"secret">>, <<"onion">>, [], <<"blockhash">>), + Tx1 = sign(Tx0, SigFun), + Sig = signature(Tx1), + EncodedTx1 = blockchain_txn_poc_receipts_v2_pb:encode_msg(Tx1#blockchain_txn_poc_receipts_v2_pb{signature = <<>>}), + ?assert(libp2p_crypto:verify(EncodedTx1, Sig, PubKey)). + +create_secret_hash_test() -> + Secret = crypto:strong_rand_bytes(8), + Members = create_secret_hash(Secret, 10), + ?assertEqual(10, erlang:length(Members)), + + Members2 = create_secret_hash(Secret, 10), + ?assertEqual(10, erlang:length(Members2)), + + lists:foreach( + fun(M) -> + ?assert(lists:member(M, Members2)) + end, + Members + ), + ok. + +ensure_unique_layer_test() -> + Secret = crypto:strong_rand_bytes(8), + Members = create_secret_hash(Secret, 10), + ?assertEqual(10, erlang:length(Members)), + ?assertEqual(10, sets:size(sets:from_list(Members))), + ok. + +delta_test() -> + Challenger = <<"challenger">>, + Secret = <<"secret">>, + OnionKeyHash = <<"onion_key_hash">>, + BlockHash = <<"blockhash">>, + + Receipt = blockchain_poc_receipt_v1:new(<<"r">>, 10, 10, <<"data">>, p2p, 1.2, 915.2, 2, <<"dr">>), + + W1 = blockchain_poc_witness_v1:new(<<"w1">>, 10, 10, <<"ph">>, 1.2, 915.2, 2, <<"dr">>), + W2 = blockchain_poc_witness_v1:new(<<"w2">>, 10, 10, <<"ph">>, 1.2, 915.2, 2, <<"dr">>), + Witnesses = [W1, W2], + + P1 = blockchain_poc_path_element_v1:new(<<"c1">>, Receipt, Witnesses), + P2 = blockchain_poc_path_element_v1:new(<<"c2">>, undefined, []), + P3 = blockchain_poc_path_element_v1:new(<<"c3">>, undefined, []), + P4 = blockchain_poc_path_element_v1:new(<<"c4">>, undefined, []), + P5 = blockchain_poc_path_element_v1:new(<<"c5">>, undefined, []), + Path1 = [P1, P2, P3, P4, P5], + Txn1 = new(Challenger, Secret, OnionKeyHash, Path1, BlockHash), + + Deltas1 = deltas(Txn1), + ?assertEqual(2, length(Deltas1)), + ?assertEqual({0.9, 0}, proplists:get_value(<<"c1">>, Deltas1)), + ?assertEqual({0, 1}, proplists:get_value(<<"c2">>, Deltas1)), + + P1Prime = blockchain_poc_path_element_v1:new(<<"c1">>, Receipt, []), + P2Prime = blockchain_poc_path_element_v1:new(<<"c2">>, undefined, []), + P3Prime = blockchain_poc_path_element_v1:new(<<"c3">>, undefined, []), + P4Prime = blockchain_poc_path_element_v1:new(<<"c3">>, undefined, []), + P5Prime = blockchain_poc_path_element_v1:new(<<"c3">>, undefined, []), + Path2 = [P1Prime, P2Prime, P3Prime, P4Prime, P5Prime], + Txn2 = new(Challenger, Secret, OnionKeyHash, Path2, BlockHash), + + Deltas2 = deltas(Txn2), + ?assertEqual(1, length(Deltas2)), + ?assertEqual({0, 0}, proplists:get_value(<<"c1">>, Deltas2)), + ok. + +duplicate_delta_test() -> + Txn = {blockchain_txn_poc_receipts_v2_pb,<<"foo">>, + <<"bar">>, + <<"baz">>, + [{blockchain_poc_path_element_v1_pb,<<"first">>, + {blockchain_poc_receipt_v1_pb,<<"a">>, + 1559953989978238892,0,<<"§Úi½">>,p2p, + <<"b">>, 10.1, 912.4}, + []}, + {blockchain_poc_path_element_v1_pb,<<"second">>, + undefined, + [{blockchain_poc_witness_v1_pb,<<"fourth">>, + 1559953991034558678,-100, + <<>>, + <<>>, 10.1, 912.4}, + {blockchain_poc_witness_v1_pb,<<"first">>, + 1559953991035078007,-72, + <<>>, + <<>>, 10.1, 912.4}]}, + {blockchain_poc_path_element_v1_pb,<<"third">>, + undefined,[]}, + {blockchain_poc_path_element_v1_pb,<<"second">>, + undefined, + [{blockchain_poc_witness_v1_pb,<<"fourth">>, + 1559953992074400943,-100, + <<>>, + <<>>, 10.1, 912.4}, + {blockchain_poc_witness_v1_pb,<<"first">>, + 1559953992075156868,-84, + <<>>, + <<>>, 10.1, 912.4}]}], + 0, + <<"gg">>, <<"blockhash">>}, + + Deltas = deltas(Txn), + ?assertEqual(4, length(Deltas)), + SecondDeltas = proplists:get_all_values(<<"second">>, Deltas), + ?assertEqual(2, length(SecondDeltas)), + {SecondAlphas, _} = lists:unzip(SecondDeltas), + ?assert(lists:sum(SecondAlphas) > 1), + ok. + +to_json_test() -> + Challenger = <<"challenger">>, + Secret = <<"secret">>, + OnionKeyHash = <<"onion_key_hash">>, + BlockHash = <<"blockhash">>, + + Receipt = blockchain_poc_receipt_v1:new(<<"r">>, 10, 10, <<"data">>, p2p, 1.2, 915.2, 2, <<"dr">>), + + W1 = blockchain_poc_witness_v1:new(<<"w1">>, 10, 10, <<"ph">>, 1.2, 915.2, 2, <<"dr">>), + W2 = blockchain_poc_witness_v1:new(<<"w2">>, 10, 10, <<"ph">>, 1.2, 915.2, 2, <<"dr">>), + Witnesses = [W1, W2], + + P1 = blockchain_poc_path_element_v1:new(<<"c1">>, Receipt, Witnesses), + P2 = blockchain_poc_path_element_v1:new(<<"c2">>, Receipt, Witnesses), + P3 = blockchain_poc_path_element_v1:new(<<"c3">>, Receipt, Witnesses), + Path = [P1, P2, P3], + + Txn = new(Challenger, Secret, OnionKeyHash, Path, BlockHash), + Json = to_json(Txn, []), + + ?assert(lists:all(fun(K) -> maps:is_key(K, Json) end, + [type, hash, secret, onion_key_hash, block_hash, path, fee, challenger])). + + +eirp_from_closest_freq_test() -> + FreqEirps = [{915.8, 10}, {915.3, 20}, {914.9, 30}, {915.2, 15}, {915.7, 12}, {916.9, 100}], + EIRP = eirp_from_closest_freq(915.1, FreqEirps), + ?assertEqual(15, EIRP). + +-endif. diff --git a/src/transactions/v2/blockchain_txn_rewards_v2.erl b/src/transactions/v2/blockchain_txn_rewards_v2.erl index dbd662162b..0684a8fa08 100644 --- a/src/transactions/v2/blockchain_txn_rewards_v2.erl +++ b/src/transactions/v2/blockchain_txn_rewards_v2.erl @@ -514,6 +514,12 @@ calculate_reward_for_txn(blockchain_txn_poc_receipts_v1 = T, Txn, _End, WitnessTime = erlang:monotonic_time(microsecond) - Start2, perf({T, witnesses}, WitnessTime), Acc2; +calculate_reward_for_txn(blockchain_txn_poc_receipts_v2, Txn, _End, + #{ poc_challenger := Challenger } = Acc, Chain, Ledger, Vars) -> + Acc0 = poc_challenger_reward(Txn, Challenger, Vars), + Acc1 = calculate_poc_challengee_rewards(Txn, Acc#{ poc_challenger => Acc0 }, Chain, Ledger, Vars), + calculate_poc_witness_rewards(Txn, Acc1, Chain, Ledger, Vars); + calculate_reward_for_txn(blockchain_txn_state_channel_close_v1, Txn, End, Acc, Chain, Ledger, Vars) -> calculate_dc_rewards(Txn, End, Acc, Chain, Ledger, Vars); calculate_reward_for_txn(Type, Txn, _End, Acc, _Chain, Ledger, _Vars) -> @@ -544,7 +550,8 @@ consider_overage(Type, Txn, Acc, Ledger) -> Vars :: reward_vars() ) -> rewards_share_metadata(). calculate_poc_challengee_rewards(Txn, #{ poc_challengee := ChallengeeMap } = Acc, Chain, Ledger, #{ var_map := VarMap } = Vars) -> - Path = blockchain_txn_poc_receipts_v1:path(Txn), + TxnType = blockchain_txn:type(Txn), + Path = TxnType:path(Txn), NewCM = poc_challengees_rewards_(Vars, Path, Path, Txn, Chain, Ledger, true, VarMap, ChallengeeMap), Acc#{ poc_challengee => NewCM }. @@ -917,10 +924,11 @@ securities_rewards(Ledger, #{epoch_reward := EpochReward, Acc :: rewards_share_map(), Vars :: reward_vars() ) -> rewards_share_map(). poc_challenger_reward(Txn, ChallengerRewards, #{poc_version := Version}) -> - Challenger = blockchain_txn_poc_receipts_v1:challenger(Txn), + TxnType = blockchain_txn:type(Txn), + Challenger = TxnType:challenger(Txn), I = maps:get(Challenger, ChallengerRewards, 0), - case blockchain_txn_poc_receipts_v1:check_path_continuation( - blockchain_txn_poc_receipts_v1:path(Txn)) of + case TxnType:check_path_continuation( + TxnType:path(Txn)) of true when is_integer(Version) andalso Version > 4 -> maps:put(Challenger, I+2, ChallengerRewards); _ -> @@ -983,6 +991,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, VarMap, Acc0) when Version >= 2 -> RegionVars = maps:get(region_vars, Vars), % explode on purpose + TxnType = blockchain_txn:type(Txn), WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), @@ -1004,7 +1013,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, undefined -> Acc1 = case Witnesses /= [] orelse - blockchain_txn_poc_receipts_v1:check_path_continuation(Path) + TxnType:check_path_continuation(Path) of true when is_integer(Version), Version > 4, IsFirst == true -> %% while we don't have a receipt for this node, we do know @@ -1049,7 +1058,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, radio -> Acc1 = case Witnesses /= [] orelse - blockchain_txn_poc_receipts_v1:check_path_continuation(Path) + TxnType:check_path_continuation(Path) of true when is_integer(Version), Version > 4 -> %% this challengee both rx'd and tx'd over radio @@ -1091,7 +1100,7 @@ poc_challengees_rewards_(#{poc_version := Version}=Vars, %% the challengee did their job Acc1 = case Witnesses /= [] orelse - blockchain_txn_poc_receipts_v1:check_path_continuation(Path) + TxnType:check_path_continuation(Path) of false -> %% path did not continue, this is an 'all gray' path @@ -1152,7 +1161,8 @@ normalize_reward_unit(_TxRewardUnitCap, Unit) -> Unit. normalize_reward_unit(Unit) when Unit > 1.0 -> 1.0; normalize_reward_unit(Unit) -> Unit. --spec poc_witness_reward( Txn :: blockchain_txn_poc_receipts_v1:txn_poc_receipts(), +-spec poc_witness_reward( Txn :: blockchain_txn_poc_receipts_v1:txn_poc_receipts() | + blockchain_txn_poc_receipts_v2:txn_poc_receipts(), AccIn :: rewards_share_map(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), @@ -1162,17 +1172,17 @@ poc_witness_reward(Txn, AccIn, #{ poc_version := POCVersion, var_map := VarMap } = Vars) when is_integer(POCVersion) andalso POCVersion >= 9 -> - + TxnType = blockchain_txn:type(Txn), WitnessRedundancy = maps:get(witness_redundancy, Vars, undefined), DecayRate = maps:get(poc_reward_decay_rate, Vars, undefined), DensityTgtRes = maps:get(density_tgt_res, Vars, undefined), RegionVars = maps:get(region_vars, Vars), % explode on purpose - KeyHash = blockchain_txn_poc_receipts_v1:onion_key_hash(Txn), + KeyHash = TxnType:onion_key_hash(Txn), try %% Get channels without validation - {ok, Channels} = blockchain_txn_poc_receipts_v1:get_channels(Txn, POCVersion, RegionVars, Chain), - Path = blockchain_txn_poc_receipts_v1:path(Txn), + {ok, Channels} = TxnType:get_channels(Txn, POCVersion, RegionVars, Chain), + Path = TxnType:path(Txn), %% Do the new thing for witness filtering lists:foldl( @@ -1184,7 +1194,7 @@ poc_witness_reward(Txn, AccIn, ValidWitnesses = case get({KeyHash, ElemHash}) of undefined -> - VW = blockchain_txn_poc_receipts_v1:valid_witnesses(Elem, WitnessChannel, + VW = TxnType:valid_witnesses(Elem, WitnessChannel, RegionVars, Ledger), put({KeyHash, ElemHash}, VW), VW; @@ -1266,9 +1276,10 @@ poc_witness_reward(Txn, AccIn, poc_witness_reward(Txn, AccIn, _Chain, Ledger, #{ poc_version := POCVersion } = Vars) when is_integer(POCVersion) andalso POCVersion > 4 -> + TxnType = blockchain_txn:type(Txn), lists:foldl( fun(Elem, A) -> - case blockchain_txn_poc_receipts_v1:good_quality_witnesses(Elem, Ledger) of + case TxnType:good_quality_witnesses(Elem, Ledger) of [] -> A; GoodQualityWitnesses -> @@ -1283,9 +1294,10 @@ poc_witness_reward(Txn, AccIn, _Chain, Ledger, end end, AccIn, - blockchain_txn_poc_receipts_v1:path(Txn) + TxnType:path(Txn) ); poc_witness_reward(Txn, AccIn, _Chain, _Ledger, Vars) -> + TxnType = blockchain_txn:type(Txn), lists:foldl( fun(Elem, A) -> lists:foldl( @@ -1298,7 +1310,7 @@ poc_witness_reward(Txn, AccIn, _Chain, _Ledger, Vars) -> blockchain_poc_path_element_v1:witnesses(Elem)) end, AccIn, - blockchain_txn_poc_receipts_v1:path(Txn)). + TxnType:path(Txn)). -spec normalize_witness_rewards( WitnessRewards :: rewards_share_map(), Vars :: reward_vars() ) -> rewards_map(). @@ -1504,7 +1516,8 @@ poc_witness_reward_unit(R, W, N) -> %% the value does not asympotically tend to 2.0, instead it tends to 0.0 normalize_reward_unit(blockchain_utils:normalize_float((N - (1 - math:pow(R, (W - N))))/W)). --spec legit_witnesses( Txn :: blockchain_txn_poc_receipts_v1:txn_poc_receipts(), +-spec legit_witnesses( Txn :: blockchain_txn_poc_receipts_v1:txn_poc_receipts() | + blockchain_txn_poc_receipts_v2:txn_poc_receipts(), Chain :: blockchain:blockchain(), Ledger :: blockchain_ledger_v1:ledger(), Elem :: blockchain_poc_path_element_v1:poc_element(), @@ -1513,19 +1526,20 @@ poc_witness_reward_unit(R, W, N) -> Version :: pos_integer() ) -> [blockchain_txn_poc_witnesses_v1:poc_witness()]. legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, RegionVars, Version) -> + TxnType = blockchain_txn:type(Txn), case Version of V when is_integer(V), V >= 9 -> try %% Get channels without validation - {ok, Channels} = blockchain_txn_poc_receipts_v1:get_channels(Txn, Version, RegionVars, Chain), + {ok, Channels} = TxnType:get_channels(Txn, Version, RegionVars, Chain), ElemPos = blockchain_utils:index_of(Elem, StaticPath), WitnessChannel = lists:nth(ElemPos, Channels), - KeyHash = blockchain_txn_poc_receipts_v1:onion_key_hash(Txn), + KeyHash = TxnType:onion_key_hash(Txn), ElemHash = erlang:phash2(Elem), ValidWitnesses = case get({KeyHash, ElemHash}) of undefined -> - VW = blockchain_txn_poc_receipts_v1:valid_witnesses(Elem, WitnessChannel, RegionVars, Ledger), + VW = TxnType:valid_witnesses(Elem, WitnessChannel, RegionVars, Ledger), put({KeyHash, ElemHash}, VW), VW; VW -> VW @@ -1540,7 +1554,7 @@ legit_witnesses(Txn, Chain, Ledger, Elem, StaticPath, RegionVars, Version) -> [] end; V when is_integer(V), V > 4 -> - blockchain_txn_poc_receipts_v1:good_quality_witnesses(Elem, Ledger); + TxnType:good_quality_witnesses(Elem, Ledger); _ -> blockchain_poc_path_element_v1:witnesses(Elem) end. @@ -1633,9 +1647,9 @@ poc_challengers_rewards_2_test() -> ElemForA = blockchain_poc_path_element_v1:new(<<"a">>, ReceiptForA, []), Txns = [ - blockchain_txn_poc_receipts_v1:new(<<"a">>, <<"Secret">>, <<"OnionKeyHash">>, []), - blockchain_txn_poc_receipts_v1:new(<<"b">>, <<"Secret">>, <<"OnionKeyHash">>, []), - blockchain_txn_poc_receipts_v1:new(<<"c">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA]) + blockchain_txn_poc_receipts_v2:new(<<"a">>, <<"Secret">>, <<"OnionKeyHash">>, [], <<"BlockHash">>), + blockchain_txn_poc_receipts_v2:new(<<"b">>, <<"Secret">>, <<"OnionKeyHash">>, [], <<"BlockHash">>), + blockchain_txn_poc_receipts_v2:new(<<"c">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA], <<"BlockHash">>) ], Vars = #{ epoch_reward => 1000, @@ -1703,13 +1717,13 @@ poc_challengees_rewards_3_test() -> Txns = [ %% No rewards here, Only receipt with no witness or subsequent receipt - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForB, ElemForA]), %% 1, 2 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForB, ElemForA], <<"BlockHash">>), %% 1, 2 %% Reward because of witness - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForAWithWitness]), %% 3 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForAWithWitness], <<"BlockHash">>), %% 3 %% Reward because of next elem has receipt - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA, ElemForB, ElemForC]), %% 3, 2, 2 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA, ElemForB, ElemForC], <<"BlockHash">>), %% 3, 2, 2 %% Reward because of witness (adding to make reward 50/50) - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForBWithWitness]) %% 3 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForBWithWitness], <<"BlockHash">>) %% 3 ], Rewards = #{ %% a gets 8 shares @@ -1720,7 +1734,7 @@ poc_challengees_rewards_3_test() -> {gateway, poc_challengees, <<"c">>} => 44 }, ChallengeeShares = lists:foldl(fun(T, Acc) -> - Path = blockchain_txn_poc_receipts_v1:path(T), + Path = blockchain_txn_poc_receipts_v2:path(T), poc_challengees_rewards_(Vars, Path, Path, T, Chain, Ledger, true, #{}, Acc) end, #{}, @@ -1774,8 +1788,8 @@ poc_witnesses_rewards_test() -> Witness2 = blockchain_poc_witness_v1:new(<<"b">>, 1, -80, <<>>), Elem = blockchain_poc_path_element_v1:new(<<"c">>, <<"Receipt not undefined">>, [Witness1, Witness2]), Txns = [ - blockchain_txn_poc_receipts_v1:new(<<"d">>, <<"Secret">>, <<"OnionKeyHash">>, [Elem, Elem]), - blockchain_txn_poc_receipts_v1:new(<<"e">>, <<"Secret">>, <<"OnionKeyHash">>, [Elem, Elem]) + blockchain_txn_poc_receipts_v2:new(<<"d">>, <<"Secret">>, <<"OnionKeyHash">>, [Elem, Elem], <<"BlockHash">>), + blockchain_txn_poc_receipts_v2:new(<<"e">>, <<"Secret">>, <<"OnionKeyHash">>, [Elem, Elem], <<"BlockHash">>) ], Rewards = #{{gateway,poc_witnesses,<<"a">>} => 25, @@ -1939,13 +1953,13 @@ dc_rewards_v3_spillover_test() -> Txns = [ %% No rewards here, Only receipt with no witness or subsequent receipt - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForB, ElemForA]), %% 1, 2 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForB, ElemForA], <<"BlockHash">>), %% 1, 2 %% Reward because of witness - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForAWithWitness]), %% 3 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForAWithWitness], <<"BlockHash">>), %% 3 %% Reward because of next elem has receipt - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA, ElemForB, ElemForC]), %% 3, 2, 2 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForA, ElemForB, ElemForC], <<"BlockHash">>), %% 3, 2, 2 %% Reward because of witness (adding to make reward 50/50) - blockchain_txn_poc_receipts_v1:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForBWithWitness]) %% 3 + blockchain_txn_poc_receipts_v2:new(<<"X">>, <<"Secret">>, <<"OnionKeyHash">>, [ElemForBWithWitness], <<"BlockHash">>) %% 3 ], diff --git a/test/blockchain_grpc_sc_client_test_handler.erl b/test/blockchain_grpc_sc_client_test_handler.erl index b8668cbea6..3da9e96079 100644 --- a/test/blockchain_grpc_sc_client_test_handler.erl +++ b/test/blockchain_grpc_sc_client_test_handler.erl @@ -38,7 +38,7 @@ dial(_SwarmTID, Peer, _Opts) -> {ok, PeerGrpcPort} = p2p_port_to_grpc_port(Peer), lager:info("connecting over grpc to peer ~p on port ~p", [Peer, PeerGrpcPort]), {ok, Connection} = grpc_client:connect(tcp, "127.0.0.1", PeerGrpcPort), - grpc_client_stream_custom:new( + grpc_client_stream_test:new( Connection, 'helium.state_channel', msg, @@ -68,12 +68,12 @@ handle_msg({data, #blockchain_state_channel_message_v1_pb{msg = Msg}}, StreamSta handle_info({send_offer, Offer}, StreamState) -> lager:info("sending offer over grpc: ~p", [Offer]), Msg = blockchain_state_channel_message_v1:wrap_msg(Offer), - NewStreamState = grpc_client_stream_custom:send_msg(StreamState, Msg, false), + NewStreamState = grpc_client_stream_test:send_msg(StreamState, Msg, false), NewStreamState; handle_info({send_packet, Packet}, StreamState) -> lager:info("sending packet over grpc: ~p", [Packet]), Msg = blockchain_state_channel_message_v1:wrap_msg(Packet), - NewStreamState = grpc_client_stream_custom:send_msg(StreamState, Msg, false), + NewStreamState = grpc_client_stream_test:send_msg(StreamState, Msg, false), NewStreamState; handle_info(_Msg, StreamState) -> lager:warning("grpc client unhandled msg: ~p", [_Msg]), diff --git a/test/blockchain_simple_SUITE.erl b/test/blockchain_simple_SUITE.erl index a616fce6ec..8f4af473f5 100644 --- a/test/blockchain_simple_SUITE.erl +++ b/test/blockchain_simple_SUITE.erl @@ -39,6 +39,8 @@ election_v6_test/1, chain_vars_test/1, chain_vars_set_unset_test/1, + poc_v2_set_challenger_type_chain_var_test/1, + poc_v2_unset_challenger_type_chain_var_test/1, token_burn_test/1, payer_test/1, poc_sync_interval_test/1, @@ -93,6 +95,8 @@ all() -> light_gw_election_v4_test, chain_vars_test, chain_vars_set_unset_test, + poc_v2_set_challenger_type_chain_var_test, + poc_v2_unset_challenger_type_chain_var_test, token_burn_test, payer_test, poc_sync_interval_test, @@ -114,10 +118,29 @@ all() -> init_per_testcase(TestCase, Config) -> Config0 = blockchain_ct_utils:init_base_dir_config(?MODULE, TestCase, Config), - Balance = 5000, + Balance = + case TestCase of + poc_v2_unset_challenger_type_chain_var_test -> ?bones(15000); + _ -> 5000 + end, {ok, Sup, {PrivKey, PubKey}, Opts} = test_utils:init(?config(base_dir, Config0)), %% two tests rely on the swarm not being in the consensus group, so exclude them here ExtraVars = case TestCase of + poc_v2_unset_challenger_type_chain_var_test -> + #{ + ?election_version => 5, + ?validator_version => 3, + ?validator_minimum_stake => ?bones(10000), + ?validator_liveness_grace_period => 10, + ?validator_liveness_interval => 5, + ?validator_key_check => true, + ?stake_withdrawal_cooldown => 10, + ?stake_withdrawal_max => 500, + ?dkg_penalty => 1.0, + ?penalty_history_limit => 100, + ?election_bba_penalty => 0.01, + ?election_seen_penalty => 0.03 + }; election_v3_test -> #{election_version => 3, election_bba_penalty => 0.01, @@ -2433,8 +2456,8 @@ chain_vars_test(Config) -> ok; {ok, 2} when Height >= (Delay + 1) -> ok; - Res -> - throw({error, {chain_var_wrong_height, Res, Height}}) + _Res -> + throw({error, {chain_var_wrong_height, _Res, Height}}) end end, lists:seq(1, 15) @@ -2511,6 +2534,212 @@ chain_vars_set_unset_test(Config) -> ok. +poc_v2_set_challenger_type_chain_var_test(Config) -> + %% fake a hotspot generated POC + %% confirm the ledger is populated + %% turn on validators as poc challengers + %% confirm the commit hooks wipe the pocs from the ledger + + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + {Priv, _} = ?config(master_key, Config), + + Ledger = blockchain:ledger(Chain), + + %% add a gw to act as our challenger + [_, {Payer, {_, PayerPrivKey, _}}, {Owner, {_, OwnerPrivKey, _}}|_] = ConsensusMembers, + PayerSigFun = libp2p_crypto:mk_sig_fun(PayerPrivKey), + OwnerSigFun = libp2p_crypto:mk_sig_fun(OwnerPrivKey), + + #{public := GatewayPubKey, secret := GatewayPrivKey} = libp2p_crypto:generate_keys(ecc_compact), + Gateway = libp2p_crypto:pubkey_to_bin(GatewayPubKey), + GatewaySigFun = libp2p_crypto:mk_sig_fun(GatewayPrivKey), + + AddGatewayTx = blockchain_txn_add_gateway_v1:new(Owner, Gateway, Payer), + SignedAddGatewayTx0 = blockchain_txn_add_gateway_v1:sign(AddGatewayTx, OwnerSigFun), + SignedAddGatewayTx1 = blockchain_txn_add_gateway_v1:sign_request(SignedAddGatewayTx0, GatewaySigFun), + SignedAddGatewayTx2 = blockchain_txn_add_gateway_v1:sign_payer(SignedAddGatewayTx1, PayerSigFun), + + AssertLocationRequestTx = blockchain_txn_assert_location_v1:new(Gateway, Owner, Payer, ?TEST_LOCATION, 1), + SignedAssertLocationTx0 = blockchain_txn_assert_location_v1:sign_request(AssertLocationRequestTx, GatewaySigFun), + SignedAssertLocationTx1 = blockchain_txn_assert_location_v1:sign(SignedAssertLocationTx0, OwnerSigFun), + SignedAssertLocationTx2 = blockchain_txn_assert_location_v1:sign_payer(SignedAssertLocationTx1, PayerSigFun), + + {ok, AddGWBlock} = test_utils:create_block(ConsensusMembers, [SignedAddGatewayTx2, SignedAssertLocationTx2]), + _ = blockchain_gossip_handler:add_block(AddGWBlock, Chain, self(), blockchain_swarm:swarm()), + + ct:pal("height: ~p", [blockchain:height(Chain)]), + + ok = test_utils:wait_until(fun() -> {ok, 4} >= blockchain:height(Chain) end), + + %% populate the ledger with POC data + %% before the chain var is modified the poc_challenger_type + %% chain var will be unset, thus in this test it will modify + %% from unset/undefined to validator + {ok, BlockHash} = blockchain:head_hash(Chain), + POCReqTxn = fake_poc_request(Gateway, GatewaySigFun, BlockHash), + OnionKeyHash = blockchain_txn_poc_request_v1:onion_key_hash(POCReqTxn), + {ok, POCBlock} = test_utils:create_block(ConsensusMembers, [POCReqTxn]), + _ = blockchain_gossip_handler:add_block(POCBlock, Chain, self(), blockchain_swarm:swarm()), + %% confirm the poc is on the ledger + {ok, [_POC1]} = blockchain_ledger_v1:find_poc(OnionKeyHash, Ledger), + + %% create the var txn to enable validators as poc challengers + Vars = #{poc_challenger_type => validator}, + ct:pal("priv_key ~p", [Priv]), + + VarTxn = blockchain_txn_vars_v1:new(Vars, 3), + Proof = blockchain_txn_vars_v1:create_proof(Priv, VarTxn), + VarTxn1 = blockchain_txn_vars_v1:proof(VarTxn, Proof), + {ok, VarBlock} = test_utils:create_block(ConsensusMembers, [VarTxn1]), + _ = blockchain_gossip_handler:add_block(VarBlock, Chain, self(), blockchain_swarm:swarm()), + + %% wait for the chain var to take effect + {ok, Delay} = blockchain:config(?vars_commit_delay, Ledger), + {ok, Height} = blockchain:height(Chain), + CommitHeight = Height + Delay, + + ct:pal("commit height ~p", [CommitHeight]), + %% Add some blocks up unil the commit height + lists:foreach( + fun(_) -> + {ok, Block} = test_utils:create_block(ConsensusMembers, []), + _ = blockchain_gossip_handler:add_block(Block, Chain, self(), blockchain_swarm:swarm()), + {ok, CurHeight} = blockchain:height(Chain), + case blockchain:config(poc_challenger_type, Ledger) of % ignore "?" + {error, not_found} when CurHeight < CommitHeight -> + ok; + {ok, validator} -> + ok; + Res -> + throw({error, {chain_var_wrong_height, Res, CurHeight}}) + end + end, + lists:seq(1, CommitHeight) + ), + %% confirm the poc no longer exists on the ledger + {error,not_found} = blockchain_ledger_v1:find_poc(OnionKeyHash, Ledger), + ok. + +poc_v2_unset_challenger_type_chain_var_test(Config) -> + %% state a validator, then issue a chain var to + %% turn on validators as poc challengers + %% confirm the ledger is populated + %% then unset the above chain var and confirm + %% the commit hooks wipe the pocs from the ledger + + ConsensusMembers = ?config(consensus_members, Config), + Chain = ?config(chain, Config), + {Priv, _} = ?config(master_key, Config), + Ledger = blockchain:ledger(Chain), + + %% + %% stake a validator + %% + + %% owner of the validator + [{OwnerPubkeyBin, {_OwnerPub, _OwnerPriv, OwnerSigFun}} | _] = ?config(genesis_members, Config), + + %% make a validator + [{StakePubkeyBin, {_StakePub, _StakePriv, _StakeSigFun}}] = test_utils:generate_keys(1), + ct:pal("StakePubkeyBin: ~p~nOwnerPubkeyBin: ~p", [StakePubkeyBin, OwnerPubkeyBin]), + + ValTxn = blockchain_txn_stake_validator_v1:new( + StakePubkeyBin, + OwnerPubkeyBin, + ?bones(10000), + ?bones(5) + ), + SignedValTxn = blockchain_txn_stake_validator_v1:sign(ValTxn, OwnerSigFun), + ct:pal("SignedStakeTxn: ~p", [SignedValTxn]), + + {ok, AddGWBlock} = test_utils:create_block(ConsensusMembers, [SignedValTxn]), + _ = blockchain_gossip_handler:add_block(AddGWBlock, Chain, self(), blockchain_swarm:swarm()), + + %% + %% create a var txn to set enable validator challengers + %$ + Vars = #{poc_challenger_type => validator}, + ct:pal("priv_key ~p", [Priv]), + + VarTxn = blockchain_txn_vars_v1:new(Vars, 3), + Proof = blockchain_txn_vars_v1:create_proof(Priv, VarTxn), + VarTxn1 = blockchain_txn_vars_v1:proof(VarTxn, Proof), + {ok, VarBlock} = test_utils:create_block(ConsensusMembers, [VarTxn1]), + _ = blockchain_gossip_handler:add_block(VarBlock, Chain, self(), blockchain_swarm:swarm()), + + %% wait for the chain var to take effect + {ok, Height} = blockchain:height(Chain), + {ok, Delay} = blockchain:config(?vars_commit_delay, Ledger), + CommitHeight = Height + Delay, + + ct:pal("commit height ~p", [CommitHeight]), + %% Add some blocks up unil the commit height + lists:foreach( + fun(_) -> + {ok, Block} = test_utils:create_block(ConsensusMembers, []), + _ = blockchain_gossip_handler:add_block(Block, Chain, self(), blockchain_swarm:swarm()), + {ok, CurHeight} = blockchain:height(Chain), + case blockchain:config(poc_challenger_type, Ledger) of % ignore "?" + {error, not_found} when CurHeight < CommitHeight -> + ok; + {ok, validator} -> + ok; + Res -> + throw({error, {chain_var_wrong_height, Res, CurHeight}}) + end + end, + lists:seq(1, CommitHeight - Height) + ), + + %% fake a validator poc by manually populating the ledger + Ledger1 = blockchain_ledger_v1:new_context(Ledger), + fake_public_poc(<<"onion_key_hash_1">>, OwnerPubkeyBin, <<"blockhash">>, Height, Ledger1), + blockchain_ledger_v1:commit_context(Ledger1), + + %% confirm the poc data exists + [_ValidatorPOC1 | _] = blockchain_ledger_v1:active_public_pocs(Ledger), + + + %% + %% now unset the poc_challenger_type chain var + %% and confirm the ledger is wiped of pocs + %% + + %% create a var txn to unset enable validator challengers + UnsetVarTxn = blockchain_txn_vars_v1:new(#{}, 4, #{unsets => [poc_challenger_type]}), + UnsetProof = blockchain_txn_vars_v1:create_proof(Priv, UnsetVarTxn), + UnsetVarTxn1 = blockchain_txn_vars_v1:proof(UnsetVarTxn, UnsetProof), + + {ok, UnsetBlock} = test_utils:create_block(ConsensusMembers, [UnsetVarTxn1]), + _ = blockchain_gossip_handler:add_block(UnsetBlock, Chain, self(), blockchain_swarm:swarm()), + + %% wait for the unset to take effect + {ok, Height3} = blockchain:height(Chain), + UnsetCommitHeight = Height3 + Delay, + lists:foreach( + fun(_) -> + {ok, Block1} = test_utils:create_block(ConsensusMembers, []), + _ = blockchain_gossip_handler:add_block(Block1, Chain, self(), blockchain_swarm:swarm()), + {ok, CurHeight1} = blockchain:height(Chain), + ct:pal("Height1 ~p", [CurHeight1]), + case blockchain:config(poc_challenger_type, Ledger) of % ignore "?" + {ok, _} when CurHeight1 < UnsetCommitHeight -> + ok; + {error, not_found} -> + ok; + Res -> + throw({error, {chain_var_wrong_height, Res, CurHeight1, UnsetCommitHeight}}) + end + end, + lists:seq(1, UnsetCommitHeight) + ), + + %% confirm the public pocs have been wiped from the ledger + [] = blockchain_ledger_v1:active_public_pocs(Ledger), + + ok. + token_burn_test(Config) -> ConsensusMembers = ?config(consensus_members, Config), Balance = ?config(balance, Config), @@ -3421,3 +3650,6 @@ fake_poc_request(Gateway, GatewaySigFun, BlockHash) -> OnionKeyHash0 = crypto:hash(sha256, libp2p_crypto:pubkey_to_bin(OnionCompactKey0)), PoCReqTxn0 = blockchain_txn_poc_request_v1:new(Gateway, SecretHash0, OnionKeyHash0, BlockHash, 1), blockchain_txn_poc_request_v1:sign(PoCReqTxn0, GatewaySigFun). + +fake_public_poc(OnionKeyHash, Challenger, BlockHash, BlockHeight, Ledger) -> + ok = blockchain_ledger_v1:save_public_poc(OnionKeyHash, Challenger, BlockHash, BlockHeight, Ledger). diff --git a/test/grpc_client_stream_custom.erl b/test/grpc_client_stream_test.erl similarity index 99% rename from test/grpc_client_stream_custom.erl rename to test/grpc_client_stream_test.erl index e68faf89ce..636f01c112 100644 --- a/test/grpc_client_stream_custom.erl +++ b/test/grpc_client_stream_test.erl @@ -17,7 +17,7 @@ %% @private An a-synchronous client with a queue-like interface. %% A gen_server is started for each stream, this keeps track %% of the status of the http2 stream and it buffers responses in a queue. --module(grpc_client_stream_custom). +-module(grpc_client_stream_test). -behaviour(gen_server). diff --git a/test/t_chain.erl b/test/t_chain.erl index 9c8673829e..6e03fd5e47 100644 --- a/test/t_chain.erl +++ b/test/t_chain.erl @@ -275,7 +275,8 @@ txns_to_block(Chain, ConsensusMembers, Txns) -> election_epoch => 1, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] }, Block0 = blockchain_block_v1:new(BlockParams), BinBlock = blockchain_block:serialize(Block0), diff --git a/test/test_utils.erl b/test/test_utils.erl index b6a6400d80..067aa27486 100644 --- a/test/test_utils.erl +++ b/test/test_utils.erl @@ -278,7 +278,8 @@ make_block(Blockchain, ConsensusMembers, STxs, BlockParamsOverride) -> election_epoch => 1, epoch_start => 0, seen_votes => [], - bba_completion => <<>> + bba_completion => <<>>, + poc_keys => [] }, BlockParams = maps:merge(BlockParamsDefault, BlockParamsOverride), Block0 = blockchain_block_v1:new(BlockParams),