From be52835d3399e6e0712f9db8a1e8489c63946a07 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Wed, 10 Jun 2020 23:22:23 +0700 Subject: [PATCH 01/37] feat: skeleton for EthGasStationStrategy --- .../lib/omg_child_chain/block_queue.ex | 9 +- .../block_queue/block_submission.ex | 192 ++++++++++ .../lib/omg_child_chain/block_queue/core.ex | 352 ++---------------- .../block_queue/gas_analyzer.ex | 7 +- .../block_queue/gas_price_adjustment.ex | 42 --- .../lib/omg_child_chain/configuration.ex | 5 + .../lib/omg_child_chain/gas_price.ex | 51 +++ .../gas_price/eth_gas_station_strategy.ex | 136 +++++++ .../gas_price/legacy_gas_strategy.ex | 212 +++++++++++ .../lib/omg_child_chain/supervisor.ex | 7 +- .../lib/omg_child_chain/sync_supervisor.ex | 2 - config/config.exs | 1 + 12 files changed, 636 insertions(+), 380 deletions(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_price_adjustment.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex index 6f8a2dfa09..553cc5dc0e 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex @@ -46,7 +46,6 @@ defmodule OMG.ChildChain.BlockQueue do alias OMG.ChildChain.BlockQueue.Core alias OMG.ChildChain.BlockQueue.Core.BlockSubmission alias OMG.ChildChain.BlockQueue.GasAnalyzer - alias OMG.ChildChain.BlockQueue.GasPriceAdjustment alias OMG.Eth alias OMG.Eth.Client alias OMG.Eth.Encoding @@ -104,8 +103,6 @@ defmodule OMG.ChildChain.BlockQueue do _ = Logger.info("Starting BlockQueue, top_mined_hash: #{inspect(Encoding.to_hex(top_mined_hash))}") block_submit_every_nth = Keyword.fetch!(args, :block_submit_every_nth) - block_submit_max_gas_price = Keyword.fetch!(args, :block_submit_max_gas_price) - gas_price_adj_params = %GasPriceAdjustment{max_gas_price: block_submit_max_gas_price} core = Core.new( @@ -115,8 +112,7 @@ defmodule OMG.ChildChain.BlockQueue do parent_height: parent_height, child_block_interval: child_block_interval, block_submit_every_nth: block_submit_every_nth, - finality_threshold: finality_threshold, - gas_price_adj_params: gas_price_adj_params + finality_threshold: finality_threshold ) {:ok, state} = @@ -208,8 +204,7 @@ defmodule OMG.ChildChain.BlockQueue do submit_result = Eth.submit_block(submission.hash, submission.nonce, submission.gas_price) newest_mined_blknum = RootChain.get_mined_child_block() - - final_result = Core.process_submit_result(submission, submit_result, newest_mined_blknum) + final_result = BlockSubmission.process_result(submission, submit_result, newest_mined_blknum) final_result = case final_result do diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex new file mode 100644 index 0000000000..11444edd85 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex @@ -0,0 +1,192 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.BlockQueue.BlockSubmission do + @moduledoc """ + Handles block submission. + """ + require Logger + + @type hash() :: <<_::256>> + @type plasma_block_num() :: non_neg_integer() + + @type t() :: %__MODULE__{ + num: plasma_block_num(), + hash: hash(), + nonce: non_neg_integer(), + gas_price: pos_integer() + } + defstruct [:num, :hash, :nonce, :gas_price] + + @type submit_result_t() :: {:ok, <<_::256>>} | {:error, map} + + @doc """ + Based on the result of a submission transaction returned from the Ethereum node, figures out what to do (namely: + crash on or ignore an error response that is expected). + + It caters for the differences of those responses between Ethereum node RPC implementations. + + In general terms, those are the responses handled: + - **known transaction** - this is common and expected to occur, since we are tracking submissions ourselves and + liberally resubmitting same transactions; this is ignored + - **low replacement price** - due to the gas price selection mechanism, there are cases where transaction will get + resubmitted with a lower gas price; this is ignored + - **account locked** - Ethereum node reports the authority account is locked; this causes a crash + - **nonce too low** - there is an inherent race condition - when we're resubmitting a block, we do it with the same + nonce, meanwhile it might happen that Ethereum mines this submission in this very instant; this is ignored if we + indeed have just mined that submission, causes a crash otherwise + """ + @spec process_result(t(), submit_result_t(), plasma_block_num()) :: + {:ok, binary()} | :ok | {:error, atom} + def process_result(submission, submit_result, newest_mined_blknum) + + def process_result(submission, {:ok, txhash}, _newest_mined_blknum) do + log_success(submission, txhash) + {:ok, txhash} + end + + # https://github.com/ethereum/go-ethereum/commit/9938d954c8391682947682543cf9b52196507a88#diff-8fecce9bb4c486ebc22226cf681416e2 + def process_result( + submission, + {:error, %{"code" => -32_000, "message" => "already known"}}, + _newest_mined_blknum + ) do + log_known_tx(submission) + :ok + end + + # maybe this will get deprecated soon once the network migrates to 1.9.11. + # Look at the previous function header for commit reference. + # `fmt.Errorf("known transaction: %x", hash)` has been removed + def process_result( + submission, + {:error, %{"code" => -32_000, "message" => "known transaction" <> _}}, + _newest_mined_blknum + ) do + log_known_tx(submission) + :ok + end + + # parity error code for duplicated tx + def process_result( + submission, + {:error, %{"code" => -32_010, "message" => "Transaction with the same hash was already imported."}}, + _newest_mined_blknum + ) do + log_known_tx(submission) + :ok + end + + def process_result( + submission, + {:error, %{"code" => -32_000, "message" => "replacement transaction underpriced"}}, + _newest_mined_blknum + ) do + log_low_replacement_price(submission) + :ok + end + + # parity version + def process_result( + submission, + {:error, %{"code" => -32_010, "message" => "Transaction gas price is too low. There is another" <> _}}, + _newest_mined_blknum + ) do + log_low_replacement_price(submission) + :ok + end + + def process_result( + submission, + {:error, %{"code" => -32_000, "message" => "authentication needed: password or unlock"}}, + newest_mined_blknum + ) do + diagnostic = prepare_diagnostic(submission, newest_mined_blknum) + log_locked(diagnostic) + {:error, :account_locked} + end + + def process_result( + submission, + {:error, %{"code" => -32_000, "message" => "nonce too low"}}, + newest_mined_blknum + ) do + process_nonce_too_low(submission, newest_mined_blknum) + end + + # parity specific error for nonce-too-low + def process_result( + submission, + {:error, %{"code" => -32_010, "message" => "Transaction nonce is too low." <> _}}, + newest_mined_blknum + ) do + process_nonce_too_low(submission, newest_mined_blknum) + end + + # ganache has this error, but these are valid nonce_too_low errors, that just don't make any sense + # `process_nonce_too_low/2` would mark it as a genuine failure and crash the BlockQueue :( + # however, everything seems to just work regardless, things get retried and mined eventually + # NOTE: we decide to degrade the severity to warn and continue, considering it's just `ganache` + def process_result( + _submission, + {:error, %{"code" => -32_000, "data" => %{"stack" => "n: the tx doesn't have the correct nonce" <> _}}} = error, + _newest_mined_blknum + ) do + log_ganache_nonce_too_low(error) + :ok + end + + defp log_ganache_nonce_too_low(error) do + # runtime sanity check if we're actually running `ganache`, if we aren't and we're here, we must crash + :ganache = Application.fetch_env!(:omg_eth, :eth_node) + _ = Logger.warn(inspect(error)) + :ok + end + + defp log_success(submission, txhash) do + _ = Logger.info("Submitted #{inspect(submission)} at: #{inspect(txhash)}") + :ok + end + + defp log_known_tx(submission) do + _ = Logger.debug("Submission #{inspect(submission)} is known transaction - ignored") + :ok + end + + defp log_low_replacement_price(submission) do + _ = Logger.debug("Submission #{inspect(submission)} is known, but with higher price - ignored") + :ok + end + + defp log_locked(diagnostic) do + _ = Logger.error("It seems that authority account is locked: #{inspect(diagnostic)}. Check README.md") + :ok + end + + defp process_nonce_too_low(%__MODULE__{num: blknum} = submission, newest_mined_blknum) do + if blknum <= newest_mined_blknum do + # apparently the `nonce too low` error is related to the submission having been mined while it was prepared + :ok + else + diagnostic = prepare_diagnostic(submission, newest_mined_blknum) + _ = Logger.error("Submission unexpectedly failed with nonce too low: #{inspect(diagnostic)}") + {:error, :nonce_too_low} + end + end + + defp prepare_diagnostic(submission, newest_mined_blknum) do + config = Application.get_all_env(:omg_eth) |> Keyword.take([:contract_addr, :authority_address, :txhash_contract]) + %{submission: submission, newest_mined_blknum: newest_mined_blknum, config: config} + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index 854b6f0c16..27873474aa 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -19,14 +19,10 @@ defmodule OMG.ChildChain.BlockQueue.Core do Responsible for - keeping a queue of blocks lined up for submission to Ethereum. - determining the cadence of forming/submitting blocks to Ethereum. - - determining correct gas price and ensuring submissions get mined eventually Relies on RootChain contract's 'authority' account not being used to send any other transactions, beginning from the nonce=1 transaction. - Calculates gas price and resubmits using a higher gas price, if submissions are not being mined. - See [section](#gas-price-selection) - ### Form block deciding Orders to form a new block when: @@ -42,57 +38,14 @@ defmodule OMG.ChildChain.BlockQueue.Core do Respects the [nonces restriction](https://github.com/omisego/elixir-omg/blob/master/docs/details.md#nonces-restriction) mechanism, i.e. the submission nonce is derived from the child chain block number to submit. Currently it is: nonce=1 blknum=1000, nonce=2 blknum=2000 etc. - - ### Gas price selection - - The mechanism employed is minimalistic, aiming at: - - pushing formed block submissions as reliably as possible, avoiding delayed mining of submissions as much as possible - - saving Ether only when certain that we're overpaying - - being simple and avoiding any external factors driving the mechanism - - The mechanics goes as follows: - - If: - - we've got a new child block formed, whose submission isn't yet mined and - - it's been more than 2 (`OMG.ChildChain.BlockQueue.GasPriceAdjustment.eth_gap_without_child_blocks`) root chain blocks - since a submission has last been seen mined - - the gas price is raised by a factor of 2 (`OMG.ChildChain.BlockQueue.GasPriceAdjustment.gas_price_raising_factor`) - - **NOTE** there's also an upper limit for the gas price (`OMG.ChildChain.BlockQueue.GasPriceAdjustment.max_gas_price`) - - If: - - we've got a new child block formed, whose submission isn't yet mined and - - it's been no more than 2 (`OMG.ChildChain.BlockQueue.GasPriceAdjustment.eth_gap_without_child_blocks`) root chain blocks - since a submission has last been seen mined - - the gas price is lowered by a factor of 0.9 ('OMG.ChildChain.BlockQueue.GasPriceAdjustment.gas_price_lowering_factor') """ - alias OMG.ChildChain.BlockQueue alias OMG.ChildChain.BlockQueue.Core - alias OMG.ChildChain.BlockQueue.GasPriceAdjustment + alias OMG.ChildChain.BlockQueue.BlockSubmission + alias OMG.Eth.GasPrice use OMG.Utils.LoggerExt - defmodule BlockSubmission do - @moduledoc """ - Represents all the parts of a `submitBlock` transaction to be done on behalf of the authority address, that is - determined by the `OMG.ChildChain.BlockQueue` - """ - - @type hash() :: <<_::256>> - @type plasma_block_num() :: non_neg_integer() - - @type t() :: %__MODULE__{ - num: plasma_block_num(), - hash: hash(), - nonce: non_neg_integer(), - gas_price: pos_integer() - } - defstruct [:num, :hash, :nonce, :gas_price] - end - @zero_bytes32 <<0::size(256)>> defstruct [ @@ -101,14 +54,11 @@ defmodule OMG.ChildChain.BlockQueue.Core do :mined_child_block_num, :last_enqueued_block_at_height, :wait_for_enqueue, - last_parent_height: 0, formed_child_block_num: 0, - gas_price_to_use: 20_000_000_000, # config: child_block_interval: nil, block_submit_every_nth: 1, - finality_threshold: 12, - gas_price_adj_params: %GasPriceAdjustment{} + finality_threshold: 12 ] @type t() :: %__MODULE__{ @@ -121,8 +71,6 @@ defmodule OMG.ChildChain.BlockQueue.Core do parent_height: BlockQueue.eth_height(), # whether we're pending an enqueue signal with a new block wait_for_enqueue: boolean(), - # gas price to use when (re)submitting transactions - gas_price_to_use: pos_integer(), last_enqueued_block_at_height: pos_integer(), # CONFIG CONSTANTS below # spacing of child blocks in RootChain contract, being the amount of deposit decimals per child block @@ -130,14 +78,9 @@ defmodule OMG.ChildChain.BlockQueue.Core do # configure to trigger forming a child chain block every this many Ethereum blocks are mined since enqueueing block_submit_every_nth: pos_integer(), # depth of max reorg we take into account - finality_threshold: pos_integer(), - # the gas price adjustment strategy parameters - gas_price_adj_params: GasPriceAdjustment.t(), - last_parent_height: non_neg_integer() + finality_threshold: pos_integer() } - @type submit_result_t() :: {:ok, <<_::256>>} | {:error, map} - @doc """ Initializes the state of the `OMG.ChildChain.BlockQueue` based on data from `OMG.DB` and configuration """ @@ -182,15 +125,29 @@ defmodule OMG.ChildChain.BlockQueue.Core do @spec set_ethereum_status(Core.t(), BlockQueue.eth_height(), BlockQueue.plasma_block_num(), boolean()) :: {:do_form_block, Core.t()} | {:dont_form_block, Core.t()} def set_ethereum_status(state, parent_height, mined_child_block_num, is_empty_block) do - new_state = - %{state | parent_height: parent_height} + state = + state + |> Map.put(:parent_height, parent_height) |> set_mined(mined_child_block_num) - |> adjust_gas_price() - if should_form_block?(new_state, is_empty_block) do - {:do_form_block, %{new_state | wait_for_enqueue: true}} - else - {:dont_form_block, new_state} + :ok = + GasPrice.recalculate_all( + state.blocks, + state.parent_height, + state.mined_child_block_num, + state.formed_child_block_num, + state.child_block_interval + ) + + {:ok, gas_price_to_use} = GasPrice.suggest() + state = Map.update!(state, :gas_price_to_use, gas_price_to_use) + + case should_form_block?(state, is_empty_block) do + true -> + {:do_form_block, %{state | wait_for_enqueue: true}} + + false -> + {:dont_form_block, state} end end @@ -223,123 +180,6 @@ defmodule OMG.ChildChain.BlockQueue.Core do |> Enum.map(&Map.put(&1, :gas_price, state.gas_price_to_use)) end - # TODO: consider moving this logic to separate module - @doc """ - Based on the result of a submission transaction returned from the Ethereum node, figures out what to do (namely: - crash on or ignore an error response that is expected). - - It caters for the differences of those responses between Ethereum node RPC implementations. - - In general terms, those are the responses handled: - - **known transaction** - this is common and expected to occur, since we are tracking submissions ourselves and - liberally resubmitting same transactions; this is ignored - - **low replacement price** - due to the gas price selection mechanism, there are cases where transaction will get - resubmitted with a lower gas price; this is ignored - - **account locked** - Ethereum node reports the authority account is locked; this causes a crash - - **nonce too low** - there is an inherent race condition - when we're resubmitting a block, we do it with the same - nonce, meanwhile it might happen that Ethereum mines this submission in this very instant; this is ignored if we - indeed have just mined that submission, causes a crash otherwise - """ - @spec process_submit_result(BlockSubmission.t(), submit_result_t(), BlockSubmission.plasma_block_num()) :: - {:ok, binary()} | :ok | {:error, atom} - def process_submit_result(submission, submit_result, newest_mined_blknum) - - def process_submit_result(submission, {:ok, txhash}, _newest_mined_blknum) do - log_success(submission, txhash) - {:ok, txhash} - end - - # https://github.com/ethereum/go-ethereum/commit/9938d954c8391682947682543cf9b52196507a88#diff-8fecce9bb4c486ebc22226cf681416e2 - def process_submit_result( - submission, - {:error, %{"code" => -32_000, "message" => "already known"}}, - _newest_mined_blknum - ) do - log_known_tx(submission) - :ok - end - - # maybe this will get deprecated soon once the network migrates to 1.9.11. - # Look at the previous function header for commit reference. - # `fmt.Errorf("known transaction: %x", hash)` has been removed - def process_submit_result( - submission, - {:error, %{"code" => -32_000, "message" => "known transaction" <> _}}, - _newest_mined_blknum - ) do - log_known_tx(submission) - :ok - end - - # parity error code for duplicated tx - def process_submit_result( - submission, - {:error, %{"code" => -32_010, "message" => "Transaction with the same hash was already imported."}}, - _newest_mined_blknum - ) do - log_known_tx(submission) - :ok - end - - def process_submit_result( - submission, - {:error, %{"code" => -32_000, "message" => "replacement transaction underpriced"}}, - _newest_mined_blknum - ) do - log_low_replacement_price(submission) - :ok - end - - # parity version - def process_submit_result( - submission, - {:error, %{"code" => -32_010, "message" => "Transaction gas price is too low. There is another" <> _}}, - _newest_mined_blknum - ) do - log_low_replacement_price(submission) - :ok - end - - def process_submit_result( - submission, - {:error, %{"code" => -32_000, "message" => "authentication needed: password or unlock"}}, - newest_mined_blknum - ) do - diagnostic = prepare_diagnostic(submission, newest_mined_blknum) - log_locked(diagnostic) - {:error, :account_locked} - end - - def process_submit_result( - submission, - {:error, %{"code" => -32_000, "message" => "nonce too low"}}, - newest_mined_blknum - ) do - process_nonce_too_low(submission, newest_mined_blknum) - end - - # parity specific error for nonce-too-low - def process_submit_result( - submission, - {:error, %{"code" => -32_010, "message" => "Transaction nonce is too low." <> _}}, - newest_mined_blknum - ) do - process_nonce_too_low(submission, newest_mined_blknum) - end - - # ganache has this error, but these are valid nonce_too_low errors, that just don't make any sense - # `process_nonce_too_low/2` would mark it as a genuine failure and crash the BlockQueue :( - # however, everything seems to just work regardless, things get retried and mined eventually - # NOTE: we decide to degrade the severity to warn and continue, considering it's just `ganache` - def process_submit_result( - _submission, - {:error, %{"code" => -32_000, "data" => %{"stack" => "n: the tx doesn't have the correct nonce" <> _}}} = error, - _newest_mined_blknum - ) do - log_ganache_nonce_too_low(error) - :ok - end - # # Private functions # @@ -382,105 +222,6 @@ defmodule OMG.ChildChain.BlockQueue.Core do %{state | formed_child_block_num: top_known_block, mined_child_block_num: mined_child_block_num, blocks: blocks} end - # Updates gas price to use basing on :calculate_gas_price function, updates current parent height - # and last mined child block number in the state which used by gas price calculations - @spec adjust_gas_price(Core.t()) :: Core.t() - defp adjust_gas_price(%Core{gas_price_adj_params: %GasPriceAdjustment{last_block_mined: nil} = gas_params} = state) do - # initializes last block mined - %{ - state - | gas_price_adj_params: GasPriceAdjustment.with(gas_params, state.parent_height, state.mined_child_block_num) - } - end - - defp adjust_gas_price( - %Core{blocks: blocks, parent_height: parent_height, last_parent_height: last_parent_height} = state - ) do - if parent_height <= last_parent_height or - !Enum.find(blocks, to_mined_block_filter(state)) do - state - else - new_gas_price = calculate_gas_price(state) - _ = Logger.debug("using new gas price '#{inspect(new_gas_price)}'") - - new_state = - state - |> set_gas_price(new_gas_price) - |> update_last_checked_mined_block_num() - - %{new_state | last_parent_height: parent_height} - end - end - - # Calculates the gas price basing on simple strategy to raise the gas price by gas_price_raising_factor - # when gap of mined parent blocks is growing and droping the price by gas_price_lowering_factor otherwise - @spec calculate_gas_price(Core.t()) :: pos_integer() - defp calculate_gas_price(%Core{ - formed_child_block_num: formed_child_block_num, - mined_child_block_num: mined_child_block_num, - gas_price_to_use: gas_price_to_use, - parent_height: parent_height, - gas_price_adj_params: %GasPriceAdjustment{ - gas_price_lowering_factor: gas_price_lowering_factor, - gas_price_raising_factor: gas_price_raising_factor, - eth_gap_without_child_blocks: eth_gap_without_child_blocks, - max_gas_price: max_gas_price, - last_block_mined: {lastchecked_parent_height, lastchecked_mined_block_num} - } - }) do - multiplier = - with true <- blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num), - true <- eth_blocks_gap_filled?(parent_height, lastchecked_parent_height, eth_gap_without_child_blocks), - false <- new_blocks_mined?(mined_child_block_num, lastchecked_mined_block_num) do - gas_price_raising_factor - else - _ -> gas_price_lowering_factor - end - - Kernel.min( - max_gas_price, - Kernel.round(multiplier * gas_price_to_use) - ) - end - - # Updates the state with information about last parent height and mined child block number - @spec update_last_checked_mined_block_num(Core.t()) :: Core.t() - defp update_last_checked_mined_block_num( - %Core{ - parent_height: parent_height, - mined_child_block_num: mined_child_block_num, - gas_price_adj_params: %GasPriceAdjustment{ - last_block_mined: {_lastechecked_parent_height, lastchecked_mined_block_num} - } - } = state - ) do - if lastchecked_mined_block_num < mined_child_block_num do - %Core{ - state - | gas_price_adj_params: - GasPriceAdjustment.with(state.gas_price_adj_params, parent_height, mined_child_block_num) - } - else - state - end - end - - defp blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) do - formed_child_block_num > mined_child_block_num - end - - defp eth_blocks_gap_filled?(parent_height, last_height, eth_gap_without_child_blocks) do - parent_height - last_height >= eth_gap_without_child_blocks - end - - defp new_blocks_mined?(mined_child_block_num, last_mined_block_num) do - mined_child_block_num > last_mined_block_num - end - - defp set_gas_price(state, price) do - %{state | gas_price_to_use: price} - end - @spec next_blknum_to_mine(Core.t()) :: pos_integer() defp next_blknum_to_mine(%{mined_child_block_num: mined, child_block_interval: interval}), do: mined + interval @@ -598,47 +339,4 @@ defmodule OMG.ChildChain.BlockQueue.Core do defp validate_block_hash(expected, {_blknum, blkhash}) when expected == blkhash, do: :ok defp validate_block_hash(_, nil), do: {:error, :mined_blknum_not_found_in_db} defp validate_block_hash(_, _), do: {:error, :hashes_dont_match} - - defp log_ganache_nonce_too_low(error) do - # runtime sanity check if we're actually running `ganache`, if we aren't and we're here, we must crash - :ganache = Application.fetch_env!(:omg_eth, :eth_node) - _ = Logger.warn(inspect(error)) - :ok - end - - defp log_success(submission, txhash) do - _ = Logger.info("Submitted #{inspect(submission)} at: #{inspect(txhash)}") - :ok - end - - defp log_known_tx(submission) do - _ = Logger.debug("Submission #{inspect(submission)} is known transaction - ignored") - :ok - end - - defp log_low_replacement_price(submission) do - _ = Logger.debug("Submission #{inspect(submission)} is known, but with higher price - ignored") - :ok - end - - defp log_locked(diagnostic) do - _ = Logger.error("It seems that authority account is locked: #{inspect(diagnostic)}. Check README.md") - :ok - end - - defp process_nonce_too_low(%BlockSubmission{num: blknum} = submission, newest_mined_blknum) do - if blknum <= newest_mined_blknum do - # apparently the `nonce too low` error is related to the submission having been mined while it was prepared - :ok - else - diagnostic = prepare_diagnostic(submission, newest_mined_blknum) - _ = Logger.error("Submission unexpectedly failed with nonce too low: #{inspect(diagnostic)}") - {:error, :nonce_too_low} - end - end - - defp prepare_diagnostic(submission, newest_mined_blknum) do - config = Application.get_all_env(:omg_eth) |> Keyword.take([:contract_addr, :authority_address, :txhash_contract]) - %{submission: submission, newest_mined_blknum: newest_mined_blknum, config: config} - end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_analyzer.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_analyzer.ex index fea391b202..b2a82ba18c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_analyzer.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_analyzer.ex @@ -101,7 +101,12 @@ defmodule OMG.ChildChain.BlockQueue.GasAnalyzer do gas_price_value = parse_gas(gas_price) gas_used_value = parse_gas(gas_used) gas_used = gas_price_value * gas_used_value - _ = Logger.info("Block submitted with receipt hash #{txhash} and gas used #{gas_used} wei") + + _ = + Logger.info( + "Block submitted with receipt hash #{txhash} at gas price #{gas_price_value} wei and gas used #{gas_used} wei" + ) + gas_used {eth_get_transaction_receipt, eth_get_transaction_by_hash} -> diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_price_adjustment.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_price_adjustment.ex deleted file mode 100644 index 8126dc12bd..0000000000 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/gas_price_adjustment.ex +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright 2019-2020 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.BlockQueue.GasPriceAdjustment do - @moduledoc """ - Encapsulates the Eth gas price adjustment strategy parameters into its own structure - """ - - defstruct eth_gap_without_child_blocks: 2, - gas_price_lowering_factor: 0.9, - gas_price_raising_factor: 2.0, - max_gas_price: 20_000_000_000, - last_block_mined: nil - - @type t() :: %__MODULE__{ - # minimum blocks count where child blocks are not mined therefore gas price needs to be increased - eth_gap_without_child_blocks: pos_integer(), - # the factor the gas price will be decreased by - gas_price_lowering_factor: float(), - # the factor the gas price will be increased by - gas_price_raising_factor: float(), - # maximum gas price above which raising has no effect, limits the gas price calculation - max_gas_price: pos_integer(), - # remembers ethereum height and last child block mined, used for the gas price calculation - last_block_mined: tuple() | nil - } - - def with(state, last_checked_parent_height, last_checked_mined_child_block_num) do - %{state | last_block_mined: {last_checked_parent_height, last_checked_mined_child_block_num}} - end -end diff --git a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex index 6f9a78d611..131d986808 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex @@ -43,6 +43,11 @@ defmodule OMG.ChildChain.Configuration do Application.fetch_env!(@app, :block_submit_max_gas_price) end + @spec block_submit_gas_price_strategy() :: module() | no_return() + def block_submit_gas_price_strategy() do + Application.fetch_env!(@app, :block_submit_gas_price_strategy) + end + @doc """ Prepares options Keyword for the FeeServer process """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex new file mode 100644 index 0000000000..a9b7c9d0a0 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -0,0 +1,51 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice do + @moduledoc """ + Suggests gas prices based on different strategies. + """ + alias OMG.ChildChain.Configuration + alias OMG.ChildChain.GasPrice.EthGasStationStrategy + alias OMG.ChildChain.GasPrice.LegacyGasStrategy + + @type price() :: pos_integer() + + @doc """ + Trigger gas price recalculations for all strategies. + """ + @spec recalculate_all(map(), pos_integer(), pos_integer(), pos_integer(), pos_integer()) :: :ok + def recalculate_all(blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval) do + _ = EthGasStationStrategy.recalculate() + + _ = + LegacyGasStrategy.recalculate( + blocks, + parent_height, + mined_child_block_num, + formed_child_block_num, + child_block_interval + ) + + :ok + end + + @doc """ + Suggests the optimal gas price using the configured strategy. + """ + @spec suggest() :: {:ok, price()} + def suggest() do + Configuration.block_submit_gas_price_strategy().get_price() + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex new file mode 100644 index 0000000000..a6fa43adff --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex @@ -0,0 +1,136 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.EthGasStationStrategy do + @moduledoc """ + Suggests gas prices based on EthGasStation's algorithm. + + Ported from https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py + """ + use GenServer + require Logger + + alias OMG.ChildChain.GasPrice + + @recalculate_interval_ms 60_000 + + @type t() :: %__MODULE__{ + gas_price_to_use: pos_integer(), + max_gas_price: pos_integer() + } + + defstruct gas_price_to_use: 20_000_000_000, + max_gas_price: 20_000_000_000 + + @doc """ + Starts the EthGasStation strategy. + """ + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: __MODULE__) + end + + @doc false + def init(args) do + state = %__MODULE__{ + max_gas_price: Keyword.fetch!(args, :max_gas_price) + } + + {:ok, state, {:continue, :start_recalculate}} + end + + @doc false + def handle_continue(:start_recalculate, state) do + _ = send(self(), :recalculate) + {:ok, _} = :timer.send_interval(@recalculate_interval_ms, self(), :recalculate) + + _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + {:noreply, state} + end + + @doc """ + Suggests the optimal gas price. + """ + @spec get_price() :: GasPrice.price() + def get_price() do + GenServer.call(__MODULE__, :get_price) + end + + @doc """ + Triggers gas price recalculation. + + This function does not return the price. To get the price, use `get_price/0` instead. + """ + @spec recalculate() :: :ok + def recalculate() do + :ok + end + + @doc false + def handle_call(:get_price, state) do + {:reply, {:ok, state.gas_price_to_use}, state} + end + + @doc false + def handle_info(:recalculate, state) do + # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py + # + # def make_predictTable(block, alltx, hashpower, avg_timemined): + # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) + # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) + # predictTable = predictTable.append(ptable2).reset_index(drop=True) + # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) + # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) + # return(predictTable) + # + # def get_hpa(gasprice, hashpower): + # """gets the hash power accpeting the gas price over last 200 blocks""" + # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] + # if gasprice > hashpower.index.max(): + # hpa = 100 + # elif gasprice < hashpower.index.min(): + # hpa = 0 + # else: + # hpa = hpa.max() + # return int(hpa) + # + # def analyze_last200blocks(block, blockdata): + # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] + # #create hashpower accepting dataframe based on mingasprice accepted in block + # hashpower = recent_blocks.groupby('mingasprice').count() + # hashpower = hashpower.rename(columns={'block_number': 'count'}) + # hashpower['cum_blocks'] = hashpower['count'].cumsum() + # totalblocks = hashpower['count'].sum() + # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 + # #get avg blockinterval time + # blockinterval = recent_blocks.sort_values('block_number').diff() + # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan + # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan + # avg_timemined = blockinterval['time_mined'].mean() + # if np.isnan(avg_timemined): + # avg_timemined = 15 + # return(hashpower, avg_timemined) + # + # def get_fast(): + # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] + # fastest = series.min() + # return float(fastest) + # + # def get_fastest(): + # hpmax = prediction_table['hashpower_accepting'].max() + # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] + # return float(fastest) + + {:noreply, state} + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex new file mode 100644 index 0000000000..42fc349c67 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex @@ -0,0 +1,212 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do + @moduledoc """ + Determines the optimal gas price based on the childchain's legacy strategy. + + ### Gas price selection + + The mechanism employed is minimalistic, aiming at: + - pushing formed block submissions as reliably as possible, avoiding delayed mining of submissions as much as possible + - saving Ether only when certain that we're overpaying + - being simple and avoiding any external factors driving the mechanism + + The mechanics goes as follows: + + If: + - we've got a new child block formed, whose submission isn't yet mined and + - it's been more than 2 (`eth_gap_without_child_blocks`) root chain blocks + since a submission has last been seen mined + + the gas price is raised by a factor of 2 (`gas_price_raising_factor`) + + **NOTE** there's also an upper limit for the gas price (`max_gas_price`) + + If: + - we've got a new child block formed, whose submission isn't yet mined and + - it's been no more than 2 (`eth_gap_without_child_blocks`) root chain blocks + since a submission has last been seen mined + + the gas price is lowered by a factor of 0.9 ('gas_price_lowering_factor') + """ + use GenServer + require Logger + + @type t() :: %__MODULE__{ + # minimum blocks count where child blocks are not mined therefore gas price needs to be increased + eth_gap_without_child_blocks: pos_integer(), + # the factor the gas price will be decreased by + gas_price_lowering_factor: float(), + # the factor the gas price will be increased by + gas_price_raising_factor: float(), + # last gas price calculated + gas_price_to_use: pos_integer(), + # maximum gas price above which raising has no effect, limits the gas price calculation + max_gas_price: pos_integer(), + # last parent height successfully evaluated for gas price + last_parent_height: pos_integer(), + # last child block mined + last_mined_child_block_num: pos_integer() + } + + defstruct eth_gap_without_child_blocks: 2, + gas_price_lowering_factor: 0.9, + gas_price_raising_factor: 2.0, + gas_price_to_use: 20_000_000_000, + max_gas_price: 20_000_000_000, + last_block_mined: 0, + last_parent_height: 0, + last_mined_child_block_num: 0 + + @doc """ + Starts the legacy gas price strategy. + """ + def start_link(args) do + GenServer.start_link(__MODULE__, args, name: __MODULE__) + end + + @doc false + def init(args) do + state = %__MODULE__{ + max_gas_price: Keyword.fetch!(args, :max_gas_price) + } + + {:ok, state} + end + + @doc """ + Suggests the optimal gas price. + """ + def get_price() do + GenServer.call(__MODULE__, :get_price) + end + + @doc """ + Triggers gas price recalculation. + + This function does not return the price. To get the price, use `get_price/0` instead. + """ + @spec recalculate(map(), pos_integer(), pos_integer(), pos_integer(), pos_integer()) :: :ok + def recalculate(blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval) do + # Unfortunately the legacy algorithm requires a blocking operation as its result needs to be applied to the upcoming block immediately. + GenServer.call( + __MODULE__, + {:recalculate, blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval} + ) + end + + @doc false + def handle_call(:get_price, state) do + {:reply, {:ok, state.gas_price_to_use}, state} + end + + @doc false + def handle_call( + {:recalculate, blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval}, + state + ) do + latest = %{ + blocks: blocks, + parent_height: parent_height, + mined_child_block_num: mined_child_block_num, + formed_child_block_num: formed_child_block_num, + child_block_interval: child_block_interval + } + + {:reply, :ok, do_recalculate(latest, state)} + end + + defp do_recalculate(latest, state) do + cond do + latest.parent_height - state.last_parent_height < state.eth_gap_without_child_blocks -> + state + + !blocks_to_mine(latest.blocks, latest.mined_child_block_num, latest.formed_child_block_num, latest.child_block_interval) -> + state + + true -> + gas_price_to_use = + calculate_gas_price( + latest.mined_child_block_num, + latest.formed_child_block_num, + state.last_mined_child_block_num, + state.gas_price_to_use, + state.gas_price_raising_factor, + state.gas_price_lowering_factor, + state.max_gas_price + ) + + state = Map.update!(state, :gas_price_to_use, gas_price_to_use) + _ = Logger.debug("using new gas price '#{inspect(state.gas_price_to_use)}'") + + case state.last_mined_child_block_num < latest.mined_child_block_num do + true -> + state + |> Map.update!(:last_parent_height, latest.parent_height) + |> Map.update!(:last_mined_child_block_num, latest.mined_child_block_num) + + false -> + state + end + end + end + + # Calculates the gas price basing on simple strategy to raise the gas price by gas_price_raising_factor + # when gap of mined parent blocks is growing and droping the price by gas_price_lowering_factor otherwise + @spec calculate_gas_price( + non_neg_integer(), + non_neg_integer(), + non_neg_integer(), + pos_integer(), + float(), + float(), + pos_integer() + ) :: pos_integer() + defp calculate_gas_price( + mined_child_block_num, + formed_child_block_num, + last_mined_child_block_num, + gas_price_to_use, + raising_factor, + lowering_factor, + max_gas_price + ) do + blocks_needs_be_mined? = blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) + new_blocks_mined? = new_blocks_mined?(mined_child_block_num, last_mined_child_block_num) + + multiplier = + case {blocks_needs_be_mined?, new_blocks_mined?} do + {false, _} -> 1.0 + {true, false} -> raising_factor + {_, true} -> lowering_factor + end + + Kernel.min(max_gas_price, Kernel.round(multiplier * gas_price_to_use)) + end + + defp blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) do + formed_child_block_num > mined_child_block_num + end + + defp new_blocks_mined?(mined_child_block_num, last_mined_block_num) do + mined_child_block_num > last_mined_block_num + end + + defp blocks_to_mine(blocks, mined_child_block_num, child_block_interval, formed_child_block_num) do + Enum.find(blocks, fn {blknum, _} -> + mined_child_block_num + child_block_interval <= blknum and blknum <= formed_child_block_num + end) + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex index 4cbc97ec40..c27c200e3e 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex @@ -23,6 +23,8 @@ defmodule OMG.ChildChain.Supervisor do alias OMG.ChildChain.Configuration alias OMG.ChildChain.DatadogEvent.ContractEventConsumer alias OMG.ChildChain.FeeServer + alias OMG.ChildChain.GasPrice.EthGasStationStrategy + alias OMG.ChildChain.GasPrice.LegacyStrategy alias OMG.ChildChain.Monitor alias OMG.ChildChain.SyncSupervisor alias OMG.ChildChain.Tracer @@ -46,6 +48,7 @@ defmodule OMG.ChildChain.Supervisor do :ok = Storage.ensure_ets_init(blocks_cache()) {:ok, contract_deployment_height} = RootChain.get_root_deployment_height() metrics_collection_interval = Configuration.metrics_collection_interval() + block_submit_max_gas_price = Configuration.block_submit_max_gas_price() fee_server_opts = Configuration.fee_server_opts() fee_claimer_address = OMG.Configuration.fee_claimer_address() child_block_interval = OMG.Eth.Configuration.child_block_interval() @@ -68,7 +71,9 @@ defmodule OMG.ChildChain.Supervisor do restart: :permanent, type: :supervisor } - ]} + ]}, + {EthGasStationStrategy, [max_gas_price: block_submit_max_gas_price]}, + {LegacyGasStrategy, [max_gas_price: block_submit_max_gas_price]} ] is_datadog_disabled = is_disabled?() diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex index f9615993cc..2974da3504 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex @@ -51,7 +51,6 @@ defmodule OMG.ChildChain.SyncSupervisor do block_queue_eth_height_check_interval_ms = Configuration.block_queue_eth_height_check_interval_ms() submission_finality_margin = Configuration.submission_finality_margin() block_submit_every_nth = Configuration.block_submit_every_nth() - block_submit_max_gas_price = Configuration.block_submit_max_gas_price() ethereum_events_check_interval_ms = OMG.Configuration.ethereum_events_check_interval_ms() coordinator_eth_height_check_interval_ms = OMG.Configuration.coordinator_eth_height_check_interval_ms() deposit_finality_margin = OMG.Configuration.deposit_finality_margin() @@ -69,7 +68,6 @@ defmodule OMG.ChildChain.SyncSupervisor do block_queue_eth_height_check_interval_ms: block_queue_eth_height_check_interval_ms, submission_finality_margin: submission_finality_margin, block_submit_every_nth: block_submit_every_nth, - block_submit_max_gas_price: block_submit_max_gas_price, child_block_interval: child_block_interval ]}, {RootChainCoordinator, diff --git a/config/config.exs b/config/config.exs index 75c683c92f..bcf7d930fb 100644 --- a/config/config.exs +++ b/config/config.exs @@ -46,6 +46,7 @@ config :omg_child_chain, block_queue_eth_height_check_interval_ms: 6_000, block_submit_every_nth: 1, block_submit_max_gas_price: 20_000_000_000, + block_submit_gas_price_strategy: OMG.ChildChain.GasPrice.LegacyGasStrategy, metrics_collection_interval: 60_000, fee_adapter_check_interval_ms: 10_000, fee_buffer_duration_ms: 30_000, From 5aabd1bc78f1bcf729e23b8ae26599c9815a7df5 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Wed, 10 Jun 2020 23:39:57 +0700 Subject: [PATCH 02/37] feat: add BLOCK_SUBMIT_GAS_PRICE_STRATEGY flag --- .../lib/omg_child_chain/block_queue.ex | 2 +- .../lib/omg_child_chain/block_queue/core.ex | 2 +- .../lib/omg_child_chain/gas_price.ex | 4 +- ...on_strategy.ex => poisson_gas_strategy.ex} | 4 +- .../set_block_submit_gas_price_strategy.ex | 59 +++++++++++++++++++ .../lib/omg_child_chain/supervisor.ex | 6 +- 6 files changed, 68 insertions(+), 9 deletions(-) rename apps/omg_child_chain/lib/omg_child_chain/gas_price/{eth_gas_station_strategy.ex => poisson_gas_strategy.ex} (97%) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex index 553cc5dc0e..708548f081 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex @@ -43,8 +43,8 @@ defmodule OMG.ChildChain.BlockQueue do alias OMG.Block alias OMG.ChildChain.BlockQueue.Balance + alias OMG.ChildChain.BlockQueue.BlockSubmission alias OMG.ChildChain.BlockQueue.Core - alias OMG.ChildChain.BlockQueue.Core.BlockSubmission alias OMG.ChildChain.BlockQueue.GasAnalyzer alias OMG.Eth alias OMG.Eth.Client diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index 27873474aa..2629787bda 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -42,7 +42,7 @@ defmodule OMG.ChildChain.BlockQueue.Core do alias OMG.ChildChain.BlockQueue alias OMG.ChildChain.BlockQueue.Core alias OMG.ChildChain.BlockQueue.BlockSubmission - alias OMG.Eth.GasPrice + alias OMG.ChildChain.GasPrice use OMG.Utils.LoggerExt diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index a9b7c9d0a0..894d414799 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -17,7 +17,7 @@ defmodule OMG.ChildChain.GasPrice do Suggests gas prices based on different strategies. """ alias OMG.ChildChain.Configuration - alias OMG.ChildChain.GasPrice.EthGasStationStrategy + alias OMG.ChildChain.GasPrice.PoissonGasStrategy alias OMG.ChildChain.GasPrice.LegacyGasStrategy @type price() :: pos_integer() @@ -27,7 +27,7 @@ defmodule OMG.ChildChain.GasPrice do """ @spec recalculate_all(map(), pos_integer(), pos_integer(), pos_integer(), pos_integer()) :: :ok def recalculate_all(blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval) do - _ = EthGasStationStrategy.recalculate() + _ = PoissonGasStrategy.recalculate() _ = LegacyGasStrategy.recalculate( diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex similarity index 97% rename from apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex rename to apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex index a6fa43adff..a06f53483b 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/eth_gas_station_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex @@ -12,9 +12,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -defmodule OMG.ChildChain.GasPrice.EthGasStationStrategy do +defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do @moduledoc """ - Suggests gas prices based on EthGasStation's algorithm. + Suggests gas prices based on Poisson regression model (also used by EthGasStation). Ported from https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex new file mode 100644 index 0000000000..f4be96217c --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex @@ -0,0 +1,59 @@ +# Copyright 2019-2019 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitGasPriceStrategy do + @moduledoc false + alias OMG.ChildChain.GasPrice.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.PoissonGasStrategy + require Logger + + @behaviour Config.Provider + + @app :omg_child_chain + @config_key :block_submit_gas_price_strategy + @env_var_name "BLOCK_SUBMIT_GAS_PRICE_STRATEGY" + + def init(args) do + args + end + + def load(config, _args) do + _ = on_load() + strategy = strategy() + Config.Reader.merge(config, omg_child_chain: [block_submit_gas_price_strategy: strategy]) + end + + defp strategy() do + default = Application.get_env(@app, @config_key) + + strategy = + @env_var_name + |> System.get_env() + |> get_strategy(default) + + _ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@config_key} Value: #{inspect(strategy)}.") + + strategy + end + + defp get_strategy("LEGACY", _), do: LegacyGasStrategy + defp get_strategy("POISSON", _), do: PoissonGasStrategy + defp get_strategy(nil, default), do: default + defp get_strategy(input, _), do: exit("#{@env_var_name} must be either LEGACY or POISSON. Got #{inspect(input)}.") + + defp on_load() do + _ = Application.ensure_all_started(:logger) + _ = Application.load(@app) + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex index c27c200e3e..fb4acf92c0 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex @@ -23,8 +23,8 @@ defmodule OMG.ChildChain.Supervisor do alias OMG.ChildChain.Configuration alias OMG.ChildChain.DatadogEvent.ContractEventConsumer alias OMG.ChildChain.FeeServer - alias OMG.ChildChain.GasPrice.EthGasStationStrategy - alias OMG.ChildChain.GasPrice.LegacyStrategy + alias OMG.ChildChain.GasPrice.PoissonGasStrategy + alias OMG.ChildChain.GasPrice.LegacyGasStrategy alias OMG.ChildChain.Monitor alias OMG.ChildChain.SyncSupervisor alias OMG.ChildChain.Tracer @@ -72,7 +72,7 @@ defmodule OMG.ChildChain.Supervisor do type: :supervisor } ]}, - {EthGasStationStrategy, [max_gas_price: block_submit_max_gas_price]}, + {PoissonGasStrategy, [max_gas_price: block_submit_max_gas_price]}, {LegacyGasStrategy, [max_gas_price: block_submit_max_gas_price]} ] From 93bb53d861e6de49cc4ce43de1ce670d4bfeba8d Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Thu, 11 Jun 2020 00:11:34 +0700 Subject: [PATCH 03/37] docs: add comment why PoissonGasStrategy.recalculate/0 doesn't do anything --- .../lib/omg_child_chain/gas_price/poisson_gas_strategy.ex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex index a06f53483b..c9561ddef9 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex @@ -73,6 +73,8 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do """ @spec recalculate() :: :ok def recalculate() do + # Poisson regression strategy recalculates based on its interval, not a recalculation trigger + # by the BlockQueue. So it immediately returns :ok :ok end From 8f29e86cad2c60ae56aad5f31e196e5c60307d5e Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Thu, 11 Jun 2020 12:41:02 +0700 Subject: [PATCH 04/37] refactor: use @behaviour --- .../lib/omg_child_chain/block_queue/core.ex | 20 ++++---- .../lib/omg_child_chain/gas_price.ex | 27 ++++------- .../gas_price/legacy_gas_strategy.ex | 47 +++++++++++-------- .../gas_price/poisson_gas_strategy.ex | 26 ++++++---- .../lib/omg_child_chain/gas_price/strategy.ex | 20 ++++++++ 5 files changed, 84 insertions(+), 56 deletions(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index 2629787bda..d12fe116af 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -130,16 +130,16 @@ defmodule OMG.ChildChain.BlockQueue.Core do |> Map.put(:parent_height, parent_height) |> set_mined(mined_child_block_num) - :ok = - GasPrice.recalculate_all( - state.blocks, - state.parent_height, - state.mined_child_block_num, - state.formed_child_block_num, - state.child_block_interval - ) - - {:ok, gas_price_to_use} = GasPrice.suggest() + recalculate_params = [ + blocks: state.blocks, + parent_height: state.parent_height, + mined_child_block_num: state.mined_child_block_num, + formed_child_block_num: state.formed_child_block_num, + child_block_interval: state.child_block_interval + ] + + :ok = GasPrice.recalculate_all(recalculate_params) + {:ok, gas_price_to_use} = GasPrice.get_price() state = Map.update!(state, :gas_price_to_use, gas_price_to_use) case should_form_block?(state, is_empty_block) do diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index 894d414799..9feac0c2a1 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -17,35 +17,26 @@ defmodule OMG.ChildChain.GasPrice do Suggests gas prices based on different strategies. """ alias OMG.ChildChain.Configuration - alias OMG.ChildChain.GasPrice.PoissonGasStrategy alias OMG.ChildChain.GasPrice.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.PoissonGasStrategy + + @type t() :: pos_integer() - @type price() :: pos_integer() + @strategies [LegacyGasStrategy, PoissonGasStrategy] @doc """ Trigger gas price recalculations for all strategies. """ - @spec recalculate_all(map(), pos_integer(), pos_integer(), pos_integer(), pos_integer()) :: :ok - def recalculate_all(blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval) do - _ = PoissonGasStrategy.recalculate() - - _ = - LegacyGasStrategy.recalculate( - blocks, - parent_height, - mined_child_block_num, - formed_child_block_num, - child_block_interval - ) - - :ok + @spec recalculate_all(Keyword.t()) :: :ok + def recalculate_all(params) do + Enum.each(@strategies, fn strategy -> :ok = strategy.recalculate(params) end) end @doc """ Suggests the optimal gas price using the configured strategy. """ - @spec suggest() :: {:ok, price()} - def suggest() do + @spec get_price() :: {:ok, t()} + def get_price() do Configuration.block_submit_gas_price_strategy().get_price() end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex index 42fc349c67..917ac46f9a 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex @@ -44,6 +44,10 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do use GenServer require Logger + alias OMG.ChildChain.GasPrice.Strategy + + @behaviour Strategy + @type t() :: %__MODULE__{ # minimum blocks count where child blocks are not mined therefore gas price needs to be increased eth_gap_without_child_blocks: pos_integer(), @@ -78,6 +82,7 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do end @doc false + @impl GenServer def init(args) do state = %__MODULE__{ max_gas_price: Keyword.fetch!(args, :max_gas_price) @@ -89,6 +94,8 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do @doc """ Suggests the optimal gas price. """ + @impl Strategy + @spec get_price() :: {:ok, GasPrice.t()} def get_price() do GenServer.call(__MODULE__, :get_price) end @@ -96,35 +103,35 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do @doc """ Triggers gas price recalculation. + Returns `:ok` on success. Raises an error if a required param is missing. + This function does not return the price. To get the price, use `get_price/0` instead. """ - @spec recalculate(map(), pos_integer(), pos_integer(), pos_integer(), pos_integer()) :: :ok - def recalculate(blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval) do - # Unfortunately the legacy algorithm requires a blocking operation as its result needs to be applied to the upcoming block immediately. - GenServer.call( - __MODULE__, - {:recalculate, blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval} - ) + @impl Strategy + @spec recalculate(Keyword.t()) :: :ok | no_return() + def recalculate(params) do + latest = %{ + blocks: Keyword.fetch!(params, :blocks), + parent_height: Keyword.fetch!(params, :parent_height), + mined_child_block_num: Keyword.fetch!(params, :mined_child_block_num), + formed_child_block_num: Keyword.fetch!(params, :formed_child_block_num), + child_block_interval: Keyword.fetch!(params, :child_block_interval) + } + + # Using `call()` here as the legacy algorithm requires a blocking operation. + # Its result needs to be applied to the upcoming block immediately. + GenServer.call(__MODULE__, {:recalculate, latest}) end @doc false - def handle_call(:get_price, state) do + @impl GenServer + def handle_call(:get_price, _, state) do {:reply, {:ok, state.gas_price_to_use}, state} end @doc false - def handle_call( - {:recalculate, blocks, parent_height, mined_child_block_num, formed_child_block_num, child_block_interval}, - state - ) do - latest = %{ - blocks: blocks, - parent_height: parent_height, - mined_child_block_num: mined_child_block_num, - formed_child_block_num: formed_child_block_num, - child_block_interval: child_block_interval - } - + @impl GenServer + def handle_call({:recalculate, latest}, _, state) do {:reply, :ok, do_recalculate(latest, state)} end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex index c9561ddef9..ab11f60ddd 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex @@ -22,6 +22,9 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do require Logger alias OMG.ChildChain.GasPrice + alias OMG.ChildChain.GasPrice.Strategy + + @behaviour Strategy @recalculate_interval_ms 60_000 @@ -41,6 +44,7 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do end @doc false + @impl GenServer def init(args) do state = %__MODULE__{ max_gas_price: Keyword.fetch!(args, :max_gas_price) @@ -50,6 +54,7 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do end @doc false + @impl GenServer def handle_continue(:start_recalculate, state) do _ = send(self(), :recalculate) {:ok, _} = :timer.send_interval(@recalculate_interval_ms, self(), :recalculate) @@ -61,29 +66,34 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do @doc """ Suggests the optimal gas price. """ - @spec get_price() :: GasPrice.price() + @impl Strategy + @spec get_price() :: GasPrice.t() def get_price() do GenServer.call(__MODULE__, :get_price) end @doc """ - Triggers gas price recalculation. + A stub that handles the recalculation trigger. + + Since Poisson regression strategy recalculates based on its interval, not a recalculation trigger + # by the BlockQueue. This function simply returns `:ok` without any computation. - This function does not return the price. To get the price, use `get_price/0` instead. + To get the price, use `get_price/0` instead. """ - @spec recalculate() :: :ok - def recalculate() do - # Poisson regression strategy recalculates based on its interval, not a recalculation trigger - # by the BlockQueue. So it immediately returns :ok + @impl Strategy + @spec recalculate(Keyword.t()) :: :ok + def recalculate(_params) do :ok end @doc false - def handle_call(:get_price, state) do + @impl GenServer + def handle_call(:get_price, _, state) do {:reply, {:ok, state.gas_price_to_use}, state} end @doc false + @impl GenServer def handle_info(:recalculate, state) do # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py # diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex new file mode 100644 index 0000000000..14fcead68d --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex @@ -0,0 +1,20 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.Strategy do + alias OMG.ChildChain.GasPrice + + @callback recalculate(params :: Keyword.t()) :: :ok | no_return() + @callback get_price() :: {:ok, GasPrice.t()} +end From 0243fde97c84fb91034bc3844f0ed758ffd123b3 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 12 Jun 2020 03:43:22 +0700 Subject: [PATCH 05/37] feat: fetch gas price history --- .../gas_price/legacy_gas_strategy.ex | 8 ++ .../gas_price/poisson_gas_strategy.ex | 106 ++------------- .../poisson_gas_strategy/analyzer.ex | 123 +++++++++++++++++ .../gas_price/poisson_gas_strategy/fetcher.ex | 69 ++++++++++ .../gas_price/poisson_gas_strategy/history.ex | 127 ++++++++++++++++++ 5 files changed, 335 insertions(+), 98 deletions(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex index 917ac46f9a..36b2d673ff 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex @@ -123,6 +123,10 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do GenServer.call(__MODULE__, {:recalculate, latest}) end + # + # GenServer callbacks + # + @doc false @impl GenServer def handle_call(:get_price, _, state) do @@ -135,6 +139,10 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do {:reply, :ok, do_recalculate(latest, state)} end + # + # Internal implementations + # + defp do_recalculate(latest, state) do cond do latest.parent_height - state.last_parent_height < state.eth_gap_without_child_blocks -> diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex index ab11f60ddd..2705bb80ce 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex @@ -18,49 +18,19 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do Ported from https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py """ - use GenServer require Logger - alias OMG.ChildChain.GasPrice alias OMG.ChildChain.GasPrice.Strategy + alias OMG.ChildChain.GasPrice.PoissonGasStrategy.Analyzer @behaviour Strategy - @recalculate_interval_ms 60_000 - - @type t() :: %__MODULE__{ - gas_price_to_use: pos_integer(), - max_gas_price: pos_integer() - } - - defstruct gas_price_to_use: 20_000_000_000, - max_gas_price: 20_000_000_000 - @doc """ - Starts the EthGasStation strategy. + Starts the Poisson regression strategy. """ - def start_link(args) do - GenServer.start_link(__MODULE__, args, name: __MODULE__) - end - - @doc false - @impl GenServer - def init(args) do - state = %__MODULE__{ - max_gas_price: Keyword.fetch!(args, :max_gas_price) - } - - {:ok, state, {:continue, :start_recalculate}} - end - - @doc false - @impl GenServer - def handle_continue(:start_recalculate, state) do - _ = send(self(), :recalculate) - {:ok, _} = :timer.send_interval(@recalculate_interval_ms, self(), :recalculate) - - _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") - {:noreply, state} + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + Analyzer.start_link(opts) end @doc """ @@ -69,14 +39,14 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do @impl Strategy @spec get_price() :: GasPrice.t() def get_price() do - GenServer.call(__MODULE__, :get_price) + GenServer.call(Analyzer, :get_price) end @doc """ A stub that handles the recalculation trigger. - Since Poisson regression strategy recalculates based on its interval, not a recalculation trigger - # by the BlockQueue. This function simply returns `:ok` without any computation. + Since Poisson regression strategy recalculates based on its interval and not a recalculation + triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. To get the price, use `get_price/0` instead. """ @@ -85,64 +55,4 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do def recalculate(_params) do :ok end - - @doc false - @impl GenServer - def handle_call(:get_price, _, state) do - {:reply, {:ok, state.gas_price_to_use}, state} - end - - @doc false - @impl GenServer - def handle_info(:recalculate, state) do - # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py - # - # def make_predictTable(block, alltx, hashpower, avg_timemined): - # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) - # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) - # predictTable = predictTable.append(ptable2).reset_index(drop=True) - # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) - # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) - # return(predictTable) - # - # def get_hpa(gasprice, hashpower): - # """gets the hash power accpeting the gas price over last 200 blocks""" - # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] - # if gasprice > hashpower.index.max(): - # hpa = 100 - # elif gasprice < hashpower.index.min(): - # hpa = 0 - # else: - # hpa = hpa.max() - # return int(hpa) - # - # def analyze_last200blocks(block, blockdata): - # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] - # #create hashpower accepting dataframe based on mingasprice accepted in block - # hashpower = recent_blocks.groupby('mingasprice').count() - # hashpower = hashpower.rename(columns={'block_number': 'count'}) - # hashpower['cum_blocks'] = hashpower['count'].cumsum() - # totalblocks = hashpower['count'].sum() - # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 - # #get avg blockinterval time - # blockinterval = recent_blocks.sort_values('block_number').diff() - # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan - # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan - # avg_timemined = blockinterval['time_mined'].mean() - # if np.isnan(avg_timemined): - # avg_timemined = 15 - # return(hashpower, avg_timemined) - # - # def get_fast(): - # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] - # fastest = series.min() - # return float(fastest) - # - # def get_fastest(): - # hpmax = prediction_table['hashpower_accepting'].max() - # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] - # return float(fastest) - - {:noreply, state} - end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex new file mode 100644 index 0000000000..51c071e4a7 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex @@ -0,0 +1,123 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.Analyzer do + @moduledoc """ + Responsible for starting the gas price history and analyzing its records + in order to recommend an optimal gas price. + """ + use GenServer + require Logger + + alias OMG.ChildChain.GasPrice.PoissonGasStrategy.History + + @type t() :: %__MODULE__{ + recommended_gas_price: pos_integer(), + max_gas_price: pos_integer(), + analyzed_height: pos_integer() + } + + defstruct recommended_gas_price: 20_000_000_000, + max_gas_price: 20_000_000_000, + analyzed_height: 0 + + @doc false + @spec start_link([max_gas_price: pos_integer(), event_bus: module()]) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + # + # GenServer initialization + # + + @doc false + @impl GenServer + def init(opts) do + # Starts the history process where it will obtain gas price records from + history_opts = [event_bus: Keyword.fetch!(opts, :event_bus)] + {:ok, _pid} = History.start_link(history_opts) + + # Prepares its own state + state = %__MODULE__{ + max_gas_price: Keyword.fetch!(opts, :max_gas_price) + } + + _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + {:ok, state} + end + + # + # GenServer callbacks + # + + @doc false + @impl GenServer + def handle_call(:get_price, _, state) do + {:reply, {:ok, state.recommended_gas_price}, state} + end + + # + # Internal implementations + # + + + # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py + # + # def make_predictTable(block, alltx, hashpower, avg_timemined): + # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) + # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) + # predictTable = predictTable.append(ptable2).reset_index(drop=True) + # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) + # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) + # return(predictTable) + # + # def get_hpa(gasprice, hashpower): + # """gets the hash power accpeting the gas price over last 200 blocks""" + # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] + # if gasprice > hashpower.index.max(): + # hpa = 100 + # elif gasprice < hashpower.index.min(): + # hpa = 0 + # else: + # hpa = hpa.max() + # return int(hpa) + # + # def analyze_last200blocks(block, blockdata): + # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] + # #create hashpower accepting dataframe based on mingasprice accepted in block + # hashpower = recent_blocks.groupby('mingasprice').count() + # hashpower = hashpower.rename(columns={'block_number': 'count'}) + # hashpower['cum_blocks'] = hashpower['count'].cumsum() + # totalblocks = hashpower['count'].sum() + # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 + # #get avg blockinterval time + # blockinterval = recent_blocks.sort_values('block_number').diff() + # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan + # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan + # avg_timemined = blockinterval['time_mined'].mean() + # if np.isnan(avg_timemined): + # avg_timemined = 15 + # return(hashpower, avg_timemined) + # + # def get_fast(): + # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] + # fastest = series.min() + # return float(fastest) + # + # def get_fastest(): + # hpmax = prediction_table['hashpower_accepting'].max() + # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] + # return float(fastest) +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex new file mode 100644 index 0000000000..62468369b8 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex @@ -0,0 +1,69 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.Fetcher do + require Logger + + alias Ethereumex.HttpClient + alias OMG.Eth.Encoding + + @per_batch 20 + @retries 5 + @retry_ms 10_000 + + @doc """ + Prepares a stream that fetches the gas prices within the given heights. + """ + @spec stream(Range.t()) :: Enumerable.t() + def stream(heights) do + heights + |> Stream.chunk_every(@per_batch) + |> Stream.flat_map(fn heights -> + _ = Logger.info("Fetching heights #{inspect(hd(heights))} - #{inspect(Enum.at(heights, -1))}...") + + {:ok, results} = + heights + |> Enum.map(fn height -> {:eth_get_block_by_number, [Encoding.to_hex(height), true]} end) + |> batch_request() + + results + end) + end + + defp batch_request(requests, retries \\ @retries) + + defp batch_request(requests, 1) do + _ = Logger.warn("Last attempt to batch request: #{inspect(requests)}") + HttpClient.batch_request(requests) + end + + defp batch_request(requests, retries) do + case HttpClient.batch_request(requests) do + {:error, _} = response -> + _ = Logger.warn(""" + Batch request failed. Retrying in #{inspect(@retry_ms)}ms with #{inspect(retries - 1)} retries left. + + Request: #{inspect(requests)} + + Response: #{inspect(response)} + """) + + _ = Process.sleep(@retry_ms) + batch_request(requests, retries - 1) + + response -> + response + end + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex new file mode 100644 index 0000000000..e8a5eff9fa --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex @@ -0,0 +1,127 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do + @moduledoc """ + Responsible for creating the gas price history store and managing its records, + including fetching, transforming, inserting and pruning the gas price records. + """ + use GenServer + require Logger + + alias OMG.ChildChain.GasPrice.PoissonGasStrategy.Fetcher + alias OMG.Eth.Encoding + alias OMG.Eth.EthereumHeight + + @num_blocks 200 + @history_table :gas_price_history + + @type state() :: %__MODULE__{ + earliest_stored_height: non_neg_integer(), + latest_stored_height: non_neg_integer() + } + + defstruct earliest_stored_height: 0, + latest_stored_height: 0 + + @type record() :: {height :: non_neg_integer(), prices :: [float()], timestamp :: non_neg_integer()} + + @doc false + @spec start_link([event_bus: module()]) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @spec all() :: [record()] + def all() do + :ets.match(@history_table, "$1") + end + + # + # GenServer initialization + # + + @doc false + @impl GenServer + def init(opts) do + state = %__MODULE__{} + event_bus = Keyword.fetch!(opts, :event_bus) + + _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + {:ok, state, {:continue, {:initialize, event_bus}}} + end + + @doc false + @impl GenServer + def handle_continue({:initialize, event_bus}, state) do + :prices = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) + :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) + + _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") + {:noreply, state, {:continue, :populate_prices}} + end + + @doc false + @impl GenServer + def handle_continue(:populate_prices, state) do + {:ok, to_height} = EthereumHeight.get() + from_height = to_height - @num_blocks + :ok = do_populate_prices(from_height, to_height, state) + + {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: to_height}} + end + + # + # GenServer callbacks + # + + @doc false + @impl GenServer + def handle_info({:internal_event_bus, :ethereum_new_height, height}, state) do + from_height = max(height - @num_blocks, state.earliest_stored_height) + :ok = do_populate_prices(from_height, height, state) + + {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: height}} + end + + # + # Internal implementations + # + + defp do_populate_prices(from_height, to_height, state) do + :ok = + from_height..to_height + |> Fetcher.stream() + |> stream_insert() + |> Stream.run() + + :ok = prune_heights(state.earliest_stored_height, from_height - 1) + _ = Logger.info("#{__MODULE__} removed gas prices from Eth heights: #{state.earliest_stored_height} - #{from_height - 1}.") + _ = Logger.info("#{__MODULE__} populated gas prices from Eth heights: #{from_height} - #{to_height}.") + end + + defp stream_insert(stream_blocks) do + Stream.each(stream_blocks, fn block -> + height = Encoding.int_from_hex(block["number"]) + prices = Enum.map(block["transactions"], fn tx -> Encoding.int_from_hex(tx["gasPrice"]) end) + timestamp = Encoding.int_from_hex(block["timestamp"]) + + true = :ets.insert(@history_table, {height, prices, timestamp}) + end) + end + + defp prune_heights(from, to) do + Enum.each(from..to, fn height -> :ets.delete(@history_table, height) end) + end +end From 51f8a05528601f9fb46142a9d0dc1df32441507a Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 02:08:42 +0700 Subject: [PATCH 06/37] refactor: gas history --- .../lib/omg_child_chain/gas_price/history.ex | 55 ++++++++ .../fetcher.ex | 5 +- .../history.ex => history/server.ex} | 61 ++++++--- .../poisson_gas_strategy/analyzer.ex | 123 ------------------ 4 files changed, 98 insertions(+), 146 deletions(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex rename apps/omg_child_chain/lib/omg_child_chain/gas_price/{poisson_gas_strategy => history}/fetcher.ex (95%) rename apps/omg_child_chain/lib/omg_child_chain/gas_price/{poisson_gas_strategy/history.ex => history/server.ex} (69%) delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex new file mode 100644 index 0000000000..fe332424ee --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex @@ -0,0 +1,55 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.History do + @moduledoc """ + Starts the gas price history service and provides gas price records. + """ + alias OMG.ChildChain.GasPrice.History.Server + + @type t() :: [record()] + @type record() :: {height :: non_neg_integer(), prices :: [float()], timestamp :: non_neg_integer()} + + @doc """ + Start the gas price history service. + """ + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(Server, opts, name: Server) + end + + @doc """ + Subscribes a process to gas price history changes. + + On each history change, the subscriber can detect the change by handling `{History, :updated}`: + + ## Examples + + def handle_info({History, :updated}, state) do + # Do your operations on history update here + end + """ + @spec subscribe(function()) :: :ok + def subscribe(pid) do + GenServer.cast(Server, {:subscribe, pid}) + end + + @doc """ + Get all existing gas price records. + """ + @spec all() :: t() + def all() do + Server.all() + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex similarity index 95% rename from apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex rename to apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex index 62468369b8..aecba3c1ab 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/fetcher.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.Fetcher do +defmodule OMG.ChildChain.GasPrice.History.Fetcher do require Logger alias Ethereumex.HttpClient @@ -51,7 +51,8 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.Fetcher do defp batch_request(requests, retries) do case HttpClient.batch_request(requests) do {:error, _} = response -> - _ = Logger.warn(""" + _ = + Logger.warn(""" Batch request failed. Retrying in #{inspect(@retry_ms)}ms with #{inspect(retries - 1)} retries left. Request: #{inspect(requests)} diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex similarity index 69% rename from apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex rename to apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index e8a5eff9fa..f26831d7b8 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/history.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -12,39 +12,39 @@ # See the License for the specific language governing permissions and # limitations under the License. -defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do +defmodule OMG.ChildChain.GasPrice.History.Server do @moduledoc """ Responsible for creating the gas price history store and managing its records, including fetching, transforming, inserting and pruning the gas price records. + + Also provides subscription that triggers on history changes. """ use GenServer require Logger - alias OMG.ChildChain.GasPrice.PoissonGasStrategy.Fetcher + alias OMG.ChildChain.GasPrice.History + alias OMG.ChildChain.GasPrice.History.Fetcher alias OMG.Eth.Encoding alias OMG.Eth.EthereumHeight - @num_blocks 200 @history_table :gas_price_history @type state() :: %__MODULE__{ + num_blocks: non_neg_integer(), earliest_stored_height: non_neg_integer(), - latest_stored_height: non_neg_integer() + latest_stored_height: non_neg_integer(), + subscribers: [pid()] } - defstruct earliest_stored_height: 0, - latest_stored_height: 0 - - @type record() :: {height :: non_neg_integer(), prices :: [float()], timestamp :: non_neg_integer()} + defstruct num_blocks: 200, + earliest_stored_height: 0, + latest_stored_height: 0, + subscribers: [] @doc false - @spec start_link([event_bus: module()]) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - - @spec all() :: [record()] + @spec all() :: History.t() def all() do + # No need to go through the GenSever to fetch from ets. :ets.match(@history_table, "$1") end @@ -55,10 +55,14 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do @doc false @impl GenServer def init(opts) do - state = %__MODULE__{} event_bus = Keyword.fetch!(opts, :event_bus) + num_blocks = Keyword.fetch!(opts, :num_blocks) - _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + state = %__MODULE__{ + num_blocks: num_blocks + } + + _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") {:ok, state, {:continue, {:initialize, event_bus}}} end @@ -68,7 +72,6 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do :prices = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) - _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") {:noreply, state, {:continue, :populate_prices}} end @@ -76,7 +79,7 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do @impl GenServer def handle_continue(:populate_prices, state) do {:ok, to_height} = EthereumHeight.get() - from_height = to_height - @num_blocks + from_height = to_height - state.num_blocks :ok = do_populate_prices(from_height, to_height, state) {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: to_height}} @@ -89,12 +92,19 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do @doc false @impl GenServer def handle_info({:internal_event_bus, :ethereum_new_height, height}, state) do - from_height = max(height - @num_blocks, state.earliest_stored_height) + from_height = max(height - state.num_blocks, state.earliest_stored_height) :ok = do_populate_prices(from_height, height, state) {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: height}} end + @doc false + @impl GenServer + def handle_cast({:subscribe, subscriber}, state) do + subscribers = Enum.uniq([subscriber | state.subscribers]) + {:noreply, %{state | subscribers: subscribers}} + end + # # Internal implementations # @@ -107,8 +117,17 @@ defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.History do |> Stream.run() :ok = prune_heights(state.earliest_stored_height, from_height - 1) - _ = Logger.info("#{__MODULE__} removed gas prices from Eth heights: #{state.earliest_stored_height} - #{from_height - 1}.") - _ = Logger.info("#{__MODULE__} populated gas prices from Eth heights: #{from_height} - #{to_height}.") + + _ = + Logger.info( + "#{__MODULE__} removed gas prices from Eth heights: #{state.earliest_stored_height} - #{from_height - 1}." + ) + + _ = Logger.info("#{__MODULE__} available gas prices from Eth heights: #{from_height} - #{to_height}.") + + # Inform all subscribers that the history has been updated. + _ = Enum.each(state.subscribers, fn subscriber -> send(subscriber, {History, :updated}) end) + :ok end defp stream_insert(stream_blocks) do diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex deleted file mode 100644 index 51c071e4a7..0000000000 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy/analyzer.ex +++ /dev/null @@ -1,123 +0,0 @@ -# Copyright 2020 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy.Analyzer do - @moduledoc """ - Responsible for starting the gas price history and analyzing its records - in order to recommend an optimal gas price. - """ - use GenServer - require Logger - - alias OMG.ChildChain.GasPrice.PoissonGasStrategy.History - - @type t() :: %__MODULE__{ - recommended_gas_price: pos_integer(), - max_gas_price: pos_integer(), - analyzed_height: pos_integer() - } - - defstruct recommended_gas_price: 20_000_000_000, - max_gas_price: 20_000_000_000, - analyzed_height: 0 - - @doc false - @spec start_link([max_gas_price: pos_integer(), event_bus: module()]) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) - end - - # - # GenServer initialization - # - - @doc false - @impl GenServer - def init(opts) do - # Starts the history process where it will obtain gas price records from - history_opts = [event_bus: Keyword.fetch!(opts, :event_bus)] - {:ok, _pid} = History.start_link(history_opts) - - # Prepares its own state - state = %__MODULE__{ - max_gas_price: Keyword.fetch!(opts, :max_gas_price) - } - - _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") - {:ok, state} - end - - # - # GenServer callbacks - # - - @doc false - @impl GenServer - def handle_call(:get_price, _, state) do - {:reply, {:ok, state.recommended_gas_price}, state} - end - - # - # Internal implementations - # - - - # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py - # - # def make_predictTable(block, alltx, hashpower, avg_timemined): - # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) - # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) - # predictTable = predictTable.append(ptable2).reset_index(drop=True) - # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) - # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) - # return(predictTable) - # - # def get_hpa(gasprice, hashpower): - # """gets the hash power accpeting the gas price over last 200 blocks""" - # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] - # if gasprice > hashpower.index.max(): - # hpa = 100 - # elif gasprice < hashpower.index.min(): - # hpa = 0 - # else: - # hpa = hpa.max() - # return int(hpa) - # - # def analyze_last200blocks(block, blockdata): - # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] - # #create hashpower accepting dataframe based on mingasprice accepted in block - # hashpower = recent_blocks.groupby('mingasprice').count() - # hashpower = hashpower.rename(columns={'block_number': 'count'}) - # hashpower['cum_blocks'] = hashpower['count'].cumsum() - # totalblocks = hashpower['count'].sum() - # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 - # #get avg blockinterval time - # blockinterval = recent_blocks.sort_values('block_number').diff() - # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan - # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan - # avg_timemined = blockinterval['time_mined'].mean() - # if np.isnan(avg_timemined): - # avg_timemined = 15 - # return(hashpower, avg_timemined) - # - # def get_fast(): - # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] - # fastest = series.min() - # return float(fastest) - # - # def get_fastest(): - # hpmax = prediction_table['hashpower_accepting'].max() - # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] - # return float(fastest) -end From a849d76cba60f5674eca11cae5a4fd0fe03620a8 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 02:15:04 +0700 Subject: [PATCH 07/37] refactor: gas strategies --- .../gas_price/poisson_gas_strategy.ex | 58 ------ .../lib/omg_child_chain/gas_price/strategy.ex | 5 + .../{ => strategy}/legacy_gas_strategy.ex | 9 +- .../strategy/poisson_gas_strategy.ex | 176 ++++++++++++++++++ .../set_block_submit_gas_price_strategy.ex | 4 +- 5 files changed, 190 insertions(+), 62 deletions(-) delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex rename apps/omg_child_chain/lib/omg_child_chain/gas_price/{ => strategy}/legacy_gas_strategy.ex (96%) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex deleted file mode 100644 index 2705bb80ce..0000000000 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/poisson_gas_strategy.ex +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright 2020 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.GasPrice.PoissonGasStrategy do - @moduledoc """ - Suggests gas prices based on Poisson regression model (also used by EthGasStation). - - Ported from https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py - """ - require Logger - alias OMG.ChildChain.GasPrice - alias OMG.ChildChain.GasPrice.Strategy - alias OMG.ChildChain.GasPrice.PoissonGasStrategy.Analyzer - - @behaviour Strategy - - @doc """ - Starts the Poisson regression strategy. - """ - @spec start_link(Keyword.t()) :: GenServer.on_start() - def start_link(opts) do - Analyzer.start_link(opts) - end - - @doc """ - Suggests the optimal gas price. - """ - @impl Strategy - @spec get_price() :: GasPrice.t() - def get_price() do - GenServer.call(Analyzer, :get_price) - end - - @doc """ - A stub that handles the recalculation trigger. - - Since Poisson regression strategy recalculates based on its interval and not a recalculation - triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. - - To get the price, use `get_price/0` instead. - """ - @impl Strategy - @spec recalculate(Keyword.t()) :: :ok - def recalculate(_params) do - :ok - end -end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex index 14fcead68d..e7ad6ce85e 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex @@ -13,6 +13,11 @@ # limitations under the License. defmodule OMG.ChildChain.GasPrice.Strategy do + @moduledoc """ + The behaviour shared by all gas price strategies. + + To see what strategies are available, look into `OMG.ChildChain.GasPrice.Strategy.*` modules. + """ alias OMG.ChildChain.GasPrice @callback recalculate(params :: Keyword.t()) :: :ok | no_return() diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex similarity index 96% rename from apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex rename to apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex index 36b2d673ff..6249fcf725 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do +defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do @moduledoc """ Determines the optimal gas price based on the childchain's legacy strategy. @@ -148,7 +148,12 @@ defmodule OMG.ChildChain.GasPrice.LegacyGasStrategy do latest.parent_height - state.last_parent_height < state.eth_gap_without_child_blocks -> state - !blocks_to_mine(latest.blocks, latest.mined_child_block_num, latest.formed_child_block_num, latest.child_block_interval) -> + !blocks_to_mine( + latest.blocks, + latest.mined_child_block_num, + latest.formed_child_block_num, + latest.child_block_interval + ) -> state true -> diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex new file mode 100644 index 0000000000..a204d5c60a --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -0,0 +1,176 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do + @moduledoc """ + Suggests gas prices based on Poisson regression model (also used by EthGasStation). + + Requires `OMG.ChildChain.GasPrice.Strategy.History` running. + + Ported from https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py + """ + use GenServer + require Logger + alias OMG.ChildChain.GasPrice + alias OMG.ChildChain.GasPrice.History + alias OMG.ChildChain.GasPrice.Strategy + + @type t() :: %__MODULE__{ + recommendations: pos_integer() + } + + defstruct recommendations: 20_000_000_000 + + @thresholds %{ + safe_low: 35, + standard: 60, + fast: 90, + fastest: 100 + } + + @behaviour Strategy + + @doc """ + Starts the Poisson regression strategy. + """ + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Suggests the optimal gas price. + """ + @impl Strategy + @spec get_price() :: GasPrice.t() + def get_price() do + GenServer.call(__MODULE__, :get_price) + end + + @doc """ + A stub that handles the recalculation trigger. + + Since Poisson regression strategy recalculates based on its interval and not a recalculation + triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. + + To get the price, use `get_price/0` instead. + """ + @impl Strategy + @spec recalculate(Keyword.t()) :: :ok + def recalculate(_params) do + :ok + end + + # + # GenServer initialization + # + + @doc false + @impl GenServer + def init(_init_arg) do + _ = History.subscribe(self()) + state = %__MODULE__{} + + _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + {:ok, state} + end + + # + # GenServer callbacks + # + + @doc false + @impl GenServer + def handle_call(:get_price, _, state) do + {:reply, {:ok, state.recommended_gas_price}, state} + end + + @doc false + @impl GenServer + def handle_info({History, :updated}, state) do + recommendations = do_recalculate() + + {:noreply, %{state | recommendations: recommendations}} + end + + # + # Internal implementations + # + + defp do_recalculate() do + sorted_min_prices = + History.all() + |> Enum.map(fn {_height, prices, _timestamp} -> Enum.min(prices) end) + |> Enum.sort() + + block_count = length(sorted_min_prices) + + Enum.map(@thresholds, fn value -> + position = floor(block_count * value / 100) - 1 + Enum.at(sorted_min_prices, position) + end) + end + + # + # Internal implementations + # + + # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py + # + # def make_predictTable(block, alltx, hashpower, avg_timemined): + # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) + # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) + # predictTable = predictTable.append(ptable2).reset_index(drop=True) + # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) + # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) + # return(predictTable) + # + # def get_hpa(gasprice, hashpower): + # """gets the hash power accpeting the gas price over last 200 blocks""" + # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] + # if gasprice > hashpower.index.max(): + # hpa = 100 + # elif gasprice < hashpower.index.min(): + # hpa = 0 + # else: + # hpa = hpa.max() + # return int(hpa) + # + # def analyze_last200blocks(block, blockdata): + # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] + # #create hashpower accepting dataframe based on mingasprice accepted in block + # hashpower = recent_blocks.groupby('mingasprice').count() + # hashpower = hashpower.rename(columns={'block_number': 'count'}) + # hashpower['cum_blocks'] = hashpower['count'].cumsum() + # totalblocks = hashpower['count'].sum() + # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 + # #get avg blockinterval time + # blockinterval = recent_blocks.sort_values('block_number').diff() + # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan + # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan + # avg_timemined = blockinterval['time_mined'].mean() + # if np.isnan(avg_timemined): + # avg_timemined = 15 + # return(hashpower, avg_timemined) + # + # def get_fast(): + # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] + # fastest = series.min() + # return float(fastest) + # + # def get_fastest(): + # hpmax = prediction_table['hashpower_accepting'].max() + # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] + # return float(fastest) +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex index f4be96217c..b36f13345d 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex @@ -14,8 +14,8 @@ defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitGasPriceStrategy do @moduledoc false - alias OMG.ChildChain.GasPrice.LegacyGasStrategy - alias OMG.ChildChain.GasPrice.PoissonGasStrategy + alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy require Logger @behaviour Config.Provider From 4d3e45d8f3f6d96ad4a707de44371312423937c8 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 02:15:20 +0700 Subject: [PATCH 08/37] feat: add block percentile gas strategy --- .../strategy/block_percentile_gas_strategy.ex | 132 ++++++++++++++++++ .../omg_child_chain/gas_price/supervisor.ex | 50 +++++++ .../set_block_submit_gas_price_strategy.ex | 7 +- 3 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex new file mode 100644 index 0000000000..07225112a3 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -0,0 +1,132 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do + @moduledoc """ + Suggests gas prices based on the percentile of blocks that accepted as the minimum gas price. + + Requires `OMG.ChildChain.GasPrice.Strategy.History` running. + """ + use GenServer + require Logger + alias OMG.ChildChain.GasPrice + alias OMG.ChildChain.GasPrice.History + alias OMG.ChildChain.GasPrice.Strategy + + @type t() :: %__MODULE__{ + recommendations: %{ + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } + } + + defstruct recommendations: %{ + safe_low: 20_000_000_000, + standard: 20_000_000_000, + fast: 20_000_000_000, + fastest: 20_000_000_000 + } + + @thresholds %{ + safe_low: 35, + standard: 60, + fast: 90, + fastest: 100 + } + + @behaviour Strategy + + @doc """ + Starts the Poisson regression strategy. + """ + @spec start_link(Keyword.t()) :: GenServer.on_start() + def start_link(opts) do + GenServer.start_link(__MODULE__, opts, name: __MODULE__) + end + + @doc """ + Suggests the optimal gas price. + """ + @impl Strategy + @spec get_price() :: GasPrice.t() + def get_price() do + GenServer.call(__MODULE__, :get_price) + end + + @doc """ + A stub that handles the recalculation trigger. + + Since Poisson regression strategy recalculates based on its interval and not a recalculation + triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. + + To get the price, use `get_price/0` instead. + """ + @impl Strategy + @spec recalculate(Keyword.t()) :: :ok + def recalculate(_params) do + :ok + end + + # + # GenServer initialization + # + + @doc false + @impl GenServer + def init(_init_arg) do + _ = History.subscribe(self()) + state = %__MODULE__{} + + _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + {:ok, state} + end + + # + # GenServer callbacks + # + + @doc false + @impl GenServer + def handle_call(:get_price, _, state) do + {:reply, {:ok, state.recommended_gas_price}, state} + end + + @doc false + @impl GenServer + def handle_info({History, :updated}, state) do + recommendations = do_recalculate() + + {:noreply, %{state | recommendations: recommendations}} + end + + # + # Internal implementations + # + + defp do_recalculate() do + sorted_min_prices = + History.all() + |> Enum.map(fn {_height, prices, _timestamp} -> Enum.min(prices) end) + |> Enum.sort() + + block_count = length(sorted_min_prices) + + Enum.map(@thresholds, fn value -> + position = floor(block_count * value / 100) - 1 + Enum.at(sorted_min_prices, position) + end) + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex new file mode 100644 index 0000000000..7b699ed2d0 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex @@ -0,0 +1,50 @@ +# Copyright 2019-2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.Supervisor do + @moduledoc """ + Supervises services related to gas price. + """ + use Supervisor + require Logger + + alias OMG.ChildChain.GasPrice.History + alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy + alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy + + def start_link(args) do + Supervisor.start_link(__MODULE__, args, name: __MODULE__) + end + + def init(init_arg) do + args = children(init_arg) + opts = [strategy: :one_for_one] + + _ = Logger.info("Starting #{__MODULE__}") + Supervisor.init(args, opts) + end + + defp children(args) do + event_bus = Keyword.fetch!(args, :event_bus) + num_blocks = Keyword.fetch!(args, :num_blocks) + + [ + {History, [event_bus: event_bus, num_blocks: num_blocks]}, + {BlockPercentileGasStrategy, []}, + {LegacyGasStrategy, []}, + {PoissonGasStrategy, []} + ] + end +end diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex index b36f13345d..9682d59c29 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex @@ -14,6 +14,7 @@ defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitGasPriceStrategy do @moduledoc false + alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy require Logger @@ -48,9 +49,13 @@ defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitGasPriceStrategy do end defp get_strategy("LEGACY", _), do: LegacyGasStrategy + defp get_strategy("BLOCK_PERCENTILE", _), do: BlockPercentileGasStrategy defp get_strategy("POISSON", _), do: PoissonGasStrategy defp get_strategy(nil, default), do: default - defp get_strategy(input, _), do: exit("#{@env_var_name} must be either LEGACY or POISSON. Got #{inspect(input)}.") + + defp get_strategy(input, _) do + exit("#{@env_var_name} must be either LEGACY, BLOCK_PERCENTILE or POISSON. Got #{inspect(input)}.") + end defp on_load() do _ = Application.ensure_all_started(:logger) From 9dff3f0155b7464fec534ee97f0aa29bf07d24cf Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 02:31:03 +0700 Subject: [PATCH 09/37] style: minor codestyle refactor --- .../lib/omg_child_chain/configuration.ex | 7 +++++- .../lib/omg_child_chain/gas_price/history.ex | 2 +- .../gas_price/history/server.ex | 6 ----- .../strategy/block_percentile_gas_strategy.ex | 8 +++---- .../gas_price/strategy/legacy_gas_strategy.ex | 8 +++---- .../strategy/poisson_gas_strategy.ex | 22 ++++++++++++++----- .../omg_child_chain/gas_price/supervisor.ex | 15 +++++++------ config/config.exs | 1 + 8 files changed, 40 insertions(+), 29 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex index 131d986808..1d4acdaf80 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/configuration.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/configuration.ex @@ -43,11 +43,16 @@ defmodule OMG.ChildChain.Configuration do Application.fetch_env!(@app, :block_submit_max_gas_price) end - @spec block_submit_gas_price_strategy() :: module() | no_return() + @spec block_submit_gas_price_strategy() :: no_return | module() def block_submit_gas_price_strategy() do Application.fetch_env!(@app, :block_submit_gas_price_strategy) end + @spec block_submit_gas_price_history_blocks() :: no_return | pos_integer() + def block_submit_gas_price_history_blocks() do + Application.fetch_env!(@app, :block_submit_gas_price_history_blocks) + end + @doc """ Prepares options Keyword for the FeeServer process """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex index fe332424ee..d052d503bf 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex @@ -32,7 +32,7 @@ defmodule OMG.ChildChain.GasPrice.History do @doc """ Subscribes a process to gas price history changes. - On each history change, the subscriber can detect the change by handling `{History, :updated}`: + On each history change, the subscriber can detect the change by handling `{History, :updated}` messages. ## Examples diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index f26831d7b8..d3b54348a9 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -72,12 +72,6 @@ defmodule OMG.ChildChain.GasPrice.History.Server do :prices = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) - {:noreply, state, {:continue, :populate_prices}} - end - - @doc false - @impl GenServer - def handle_continue(:populate_prices, state) do {:ok, to_height} = EthereumHeight.get() from_height = to_height - state.num_blocks :ok = do_populate_prices(from_height, to_height, state) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 07225112a3..ae29be231f 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -53,8 +53,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do Starts the Poisson regression strategy. """ @spec start_link(Keyword.t()) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) + def start_link(init_arg) do + GenServer.start_link(__MODULE__, init_arg, name: __MODULE__) end @doc """ @@ -69,7 +69,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc """ A stub that handles the recalculation trigger. - Since Poisson regression strategy recalculates based on its interval and not a recalculation + Since this strategy recalculates based on its interval and not a recalculation triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. To get the price, use `get_price/0` instead. @@ -90,7 +90,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do _ = History.subscribe(self()) state = %__MODULE__{} - _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") {:ok, state} end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex index 6249fcf725..0cd908d70b 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex @@ -77,15 +77,15 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do @doc """ Starts the legacy gas price strategy. """ - def start_link(args) do - GenServer.start_link(__MODULE__, args, name: __MODULE__) + def start_link(init_arg) do + GenServer.start_link(__MODULE__, init_arg, name: __MODULE__) end @doc false @impl GenServer - def init(args) do + def init(init_arg) do state = %__MODULE__{ - max_gas_price: Keyword.fetch!(args, :max_gas_price) + max_gas_price: Keyword.fetch!(init_arg, :max_gas_price) } {:ok, state} diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index a204d5c60a..7f76424c24 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -27,10 +27,20 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do alias OMG.ChildChain.GasPrice.Strategy @type t() :: %__MODULE__{ - recommendations: pos_integer() + recommendations: %{ + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } } - defstruct recommendations: 20_000_000_000 + defstruct recommendations: %{ + safe_low: 20_000_000_000, + standard: 20_000_000_000, + fast: 20_000_000_000, + fastest: 20_000_000_000 + } @thresholds %{ safe_low: 35, @@ -45,8 +55,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do Starts the Poisson regression strategy. """ @spec start_link(Keyword.t()) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(__MODULE__, opts, name: __MODULE__) + def start_link(init_arg) do + GenServer.start_link(__MODULE__, init_arg, name: __MODULE__) end @doc """ @@ -61,7 +71,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc """ A stub that handles the recalculation trigger. - Since Poisson regression strategy recalculates based on its interval and not a recalculation + Since this strategy recalculates based on its interval and not a recalculation triggered by the `recalculate/1`'s caller, this function simply returns `:ok` without any computation. To get the price, use `get_price/0` instead. @@ -82,7 +92,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do _ = History.subscribe(self()) state = %__MODULE__{} - _ = Logger.info("Started #{inspect(__MODULE__)}: #{inspect(state)}") + _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") {:ok, state} end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex index 7b699ed2d0..42e7c195c0 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex @@ -19,6 +19,8 @@ defmodule OMG.ChildChain.GasPrice.Supervisor do use Supervisor require Logger + alias OMG.Bus + alias OMG.ChildChain.Configuration alias OMG.ChildChain.GasPrice.History alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy @@ -29,21 +31,20 @@ defmodule OMG.ChildChain.GasPrice.Supervisor do end def init(init_arg) do - args = children(init_arg) - opts = [strategy: :one_for_one] + children = children(init_arg) _ = Logger.info("Starting #{__MODULE__}") - Supervisor.init(args, opts) + Supervisor.init(children, [strategy: :one_for_one]) end defp children(args) do - event_bus = Keyword.fetch!(args, :event_bus) - num_blocks = Keyword.fetch!(args, :num_blocks) + num_blocks = Configuration.get(:block_submit_gas_price_history_blocks) + max_gas_price = Configuration.get(:block_submit_max_gas_price) [ - {History, [event_bus: event_bus, num_blocks: num_blocks]}, + {History, [event_bus: OMG.Bus, num_blocks: num_blocks]}, + {LegacyGasStrategy, [max_gas_price: max_gas_price]}, {BlockPercentileGasStrategy, []}, - {LegacyGasStrategy, []}, {PoissonGasStrategy, []} ] end diff --git a/config/config.exs b/config/config.exs index 71f389ee86..6865fec004 100644 --- a/config/config.exs +++ b/config/config.exs @@ -47,6 +47,7 @@ config :omg_child_chain, block_submit_every_nth: 1, block_submit_max_gas_price: 20_000_000_000, block_submit_gas_price_strategy: OMG.ChildChain.GasPrice.LegacyGasStrategy, + block_submit_gas_price_history_blocks: 200, metrics_collection_interval: 60_000, fee_adapter_check_interval_ms: 10_000, fee_buffer_duration_ms: 30_000, From 72f24256de8c96f19de21cef4eb936971b3698fa Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 11:24:11 +0700 Subject: [PATCH 10/37] fix: pick one price suggestion --- .../gas_price/strategy/block_percentile_gas_strategy.ex | 5 ++++- .../gas_price/strategy/poisson_gas_strategy.ex | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index ae29be231f..8e7027830f 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -47,6 +47,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do fastest: 100 } + @target_threshold :fast + @behaviour Strategy @doc """ @@ -101,7 +103,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc false @impl GenServer def handle_call(:get_price, _, state) do - {:reply, {:ok, state.recommended_gas_price}, state} + price = state.recommendations[@target_threshold] + {:reply, {:ok, price}, state} end @doc false diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index 7f76424c24..e078ac7566 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -49,6 +49,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do fastest: 100 } + @target_threshold :fast + @behaviour Strategy @doc """ @@ -103,7 +105,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc false @impl GenServer def handle_call(:get_price, _, state) do - {:reply, {:ok, state.recommended_gas_price}, state} + price = state.recommendations[@target_threshold] + {:reply, {:ok, price}, state} end @doc false From ecfbf1e224314bb9f433c8390225eb91c091a3a7 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 19 Jun 2020 17:50:58 +0700 Subject: [PATCH 11/37] docs: add comment on ets --- .../lib/omg_child_chain/gas_price/history/server.ex | 11 +++++++++++ docs/deployment_configuration.md | 1 + 2 files changed, 12 insertions(+) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index d3b54348a9..15d7ea0662 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -18,6 +18,15 @@ defmodule OMG.ChildChain.GasPrice.History.Server do including fetching, transforming, inserting and pruning the gas price records. Also provides subscription that triggers on history changes. + + This module utilizes [`ets`](https://elixir-lang.org/getting-started/mix-otp/ets.html) + instead of GenServer for history storage for the following reasons: + + 1. Multiple pricing strategies will be accessing the history. ets allows those strategies to + access the records concurrently. + + 2. Fetching the history is RPC-intensive and unnecessary between GenServer crashes. + Using ets allows the records to persist across crashes. """ use GenServer require Logger @@ -69,6 +78,8 @@ defmodule OMG.ChildChain.GasPrice.History.Server do @doc false @impl GenServer def handle_continue({:initialize, event_bus}, state) do + # The ets table is not initialized with `:read_concurrency` because we are expecting interleaving + # reads and writes. See http://erlang.org/doc/man/ets.html :prices = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) diff --git a/docs/deployment_configuration.md b/docs/deployment_configuration.md index 71b055b0e3..91a7ed26dd 100644 --- a/docs/deployment_configuration.md +++ b/docs/deployment_configuration.md @@ -23,6 +23,7 @@ ***Child Chain only*** - "BLOCK_SUBMIT_MAX_GAS_PRICE" - The maximum gas price to use for block submission. The first block submission after application boot will use the max price. The gas price gradually adjusts on subsequent blocks to reach the current optimum price . Defaults to `20000000000` (20 Gwei). +- "BLOCK_SUBMIT_GAS_PRICE_STRATEGY" - The gas price strategy to use for block submission. Note that all strategies will still run, but only the one configured here will be used for actual submission. Defaults to `LEGACY`. - "FEE_ADAPTER" - The adapter to use to populate the fee specs. Either `file` or `feed` (case-insensitive). Defaults to `file` with an empty fee specs. - "FEE_CLAIMER_ADDRESS" - 20-bytes HEX-encoded string of Ethereum address of Fee Claimer. - "FEE_BUFFER_DURATION_MS" - Buffer period during which a fee is still valid after being updated. From 16cd9afb4fb6af2f5a0fb928afcc897c5226dd1e Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Fri, 26 Jun 2020 02:09:09 +0700 Subject: [PATCH 12/37] feat: make it run --- .../lib/omg_child_chain/block_queue/core.ex | 16 ++++- .../lib/omg_child_chain/gas_price.ex | 8 +-- ...{supervisor.ex => gas_price_supervisor.ex} | 25 +++----- .../lib/omg_child_chain/gas_price/history.ex | 15 +++++ .../gas_price/history/server.ex | 55 ++++++++++------- .../lib/omg_child_chain/gas_price/strategy.ex | 3 +- .../strategy/block_percentile_gas_strategy.ex | 61 +++++++++++++------ .../gas_price/strategy/legacy_gas_strategy.ex | 11 +++- .../strategy/poisson_gas_strategy.ex | 61 +++++++++++++------ .../lib/omg_child_chain/supervisor.ex | 9 ++- config/config.exs | 2 +- 11 files changed, 169 insertions(+), 97 deletions(-) rename apps/omg_child_chain/lib/omg_child_chain/gas_price/{supervisor.ex => gas_price_supervisor.ex} (69%) diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index d12fe116af..5f7b176c70 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -47,6 +47,7 @@ defmodule OMG.ChildChain.BlockQueue.Core do use OMG.Utils.LoggerExt @zero_bytes32 <<0::size(256)>> + @default_gas_price 20_000_000_000 defstruct [ :blocks, @@ -55,6 +56,7 @@ defmodule OMG.ChildChain.BlockQueue.Core do :last_enqueued_block_at_height, :wait_for_enqueue, formed_child_block_num: 0, + gas_price: @default_gas_price, # config: child_block_interval: nil, block_submit_every_nth: 1, @@ -67,6 +69,8 @@ defmodule OMG.ChildChain.BlockQueue.Core do mined_child_block_num: BlockQueue.plasma_block_num(), # newest formed block num formed_child_block_num: BlockQueue.plasma_block_num(), + # gas price to use when submitting transactions + gas_price: pos_integer(), # current Ethereum block height parent_height: BlockQueue.eth_height(), # whether we're pending an enqueue signal with a new block @@ -139,8 +143,14 @@ defmodule OMG.ChildChain.BlockQueue.Core do ] :ok = GasPrice.recalculate_all(recalculate_params) - {:ok, gas_price_to_use} = GasPrice.get_price() - state = Map.update!(state, :gas_price_to_use, gas_price_to_use) + + gas_price = + case GasPrice.get_price() do + {:ok, price} -> price + {:error, _} -> @default_gas_price + end + + state = %{state | gas_price: gas_price} case should_form_block?(state, is_empty_block) do true -> @@ -177,7 +187,7 @@ defmodule OMG.ChildChain.BlockQueue.Core do |> Enum.filter(to_mined_block_filter(state)) |> Enum.map(fn {_blknum, block} -> block end) |> Enum.sort_by(& &1.num) - |> Enum.map(&Map.put(&1, :gas_price, state.gas_price_to_use)) + |> Enum.map(&Map.put(&1, :gas_price, state.gas_price)) end # diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index 9feac0c2a1..6da11b0360 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -17,10 +17,8 @@ defmodule OMG.ChildChain.GasPrice do Suggests gas prices based on different strategies. """ alias OMG.ChildChain.Configuration - alias OMG.ChildChain.GasPrice.LegacyGasStrategy - alias OMG.ChildChain.GasPrice.PoissonGasStrategy - - @type t() :: pos_integer() + alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy @strategies [LegacyGasStrategy, PoissonGasStrategy] @@ -35,7 +33,7 @@ defmodule OMG.ChildChain.GasPrice do @doc """ Suggests the optimal gas price using the configured strategy. """ - @spec get_price() :: {:ok, t()} + @spec get_price() :: {:ok, pos_integer()} | {:error, atom()} def get_price() do Configuration.block_submit_gas_price_strategy().get_price() end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex similarity index 69% rename from apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex rename to apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex index 42e7c195c0..e3c045ecd4 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -defmodule OMG.ChildChain.GasPrice.Supervisor do +defmodule OMG.ChildChain.GasPrice.GasPriceSupervisor do @moduledoc """ Supervises services related to gas price. """ @@ -20,32 +20,27 @@ defmodule OMG.ChildChain.GasPrice.Supervisor do require Logger alias OMG.Bus - alias OMG.ChildChain.Configuration alias OMG.ChildChain.GasPrice.History alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy - def start_link(args) do - Supervisor.start_link(__MODULE__, args, name: __MODULE__) + def start_link(init_arg) do + Supervisor.start_link(__MODULE__, init_arg, name: __MODULE__) end def init(init_arg) do - children = children(init_arg) + num_blocks = Keyword.fetch!(init_arg, :num_blocks) + max_gas_price = Keyword.fetch!(init_arg, :max_gas_price) - _ = Logger.info("Starting #{__MODULE__}") - Supervisor.init(children, [strategy: :one_for_one]) - end - - defp children(args) do - num_blocks = Configuration.get(:block_submit_gas_price_history_blocks) - max_gas_price = Configuration.get(:block_submit_max_gas_price) - - [ - {History, [event_bus: OMG.Bus, num_blocks: num_blocks]}, + children = [ + {History, [event_bus: Bus, num_blocks: num_blocks]}, {LegacyGasStrategy, [max_gas_price: max_gas_price]}, {BlockPercentileGasStrategy, []}, {PoissonGasStrategy, []} ] + + _ = Logger.info("Starting #{__MODULE__}") + Supervisor.init(children, strategy: :one_for_one) end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex index d052d503bf..5ba587fca3 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex @@ -21,6 +21,21 @@ defmodule OMG.ChildChain.GasPrice.History do @type t() :: [record()] @type record() :: {height :: non_neg_integer(), prices :: [float()], timestamp :: non_neg_integer()} + @doc """ + Defines the child specification for this module, so that it can be started conveniently by a supervisor. + + ## Examples + + Supervisor.init([{History, []}], strategy: :one_for_one) + """ + def child_spec(opts) do + %{ + id: __MODULE__, + start: {__MODULE__, :start_link, [opts]}, + type: :worker + } + end + @doc """ Start the gas price history service. """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index 15d7ea0662..1b125eb498 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -42,19 +42,21 @@ defmodule OMG.ChildChain.GasPrice.History.Server do num_blocks: non_neg_integer(), earliest_stored_height: non_neg_integer(), latest_stored_height: non_neg_integer(), + history_ets: :ets.tid(), subscribers: [pid()] } defstruct num_blocks: 200, earliest_stored_height: 0, latest_stored_height: 0, + history_ets: nil, subscribers: [] @doc false @spec all() :: History.t() def all() do # No need to go through the GenSever to fetch from ets. - :ets.match(@history_table, "$1") + :ets.tab2list(@history_table) end # @@ -64,30 +66,33 @@ defmodule OMG.ChildChain.GasPrice.History.Server do @doc false @impl GenServer def init(opts) do - event_bus = Keyword.fetch!(opts, :event_bus) num_blocks = Keyword.fetch!(opts, :num_blocks) + event_bus = Keyword.fetch!(opts, :event_bus) + :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) + + # The ets table is not initialized with `:read_concurrency` because we are expecting interleaving + # reads and writes. See http://erlang.org/doc/man/ets.html + history_ets = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) state = %__MODULE__{ - num_blocks: num_blocks + num_blocks: num_blocks, + history_ets: history_ets } _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") - {:ok, state, {:continue, {:initialize, event_bus}}} + {:ok, state, {:continue, :first_fetch}} end @doc false @impl GenServer - def handle_continue({:initialize, event_bus}, state) do - # The ets table is not initialized with `:read_concurrency` because we are expecting interleaving - # reads and writes. See http://erlang.org/doc/man/ets.html - :prices = :ets.new(@history_table, [:ordered_set, :protected, :named_table]) - :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) - + def handle_continue(:first_fetch, state) do {:ok, to_height} = EthereumHeight.get() from_height = to_height - state.num_blocks - :ok = do_populate_prices(from_height, to_height, state) + {:ok, earliest_height, latest_height} = do_populate_prices(from_height, to_height, state) + + state = %{state | earliest_stored_height: earliest_height, latest_stored_height: latest_height} - {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: to_height}} + {:noreply, state} end # @@ -97,10 +102,10 @@ defmodule OMG.ChildChain.GasPrice.History.Server do @doc false @impl GenServer def handle_info({:internal_event_bus, :ethereum_new_height, height}, state) do - from_height = max(height - state.num_blocks, state.earliest_stored_height) - :ok = do_populate_prices(from_height, height, state) + from_height = height - state.num_blocks + {:ok, earliest_height, latest_height} = do_populate_prices(from_height, height, state) - {:noreply, %{state | earliest_stored_height: from_height, latest_stored_height: height}} + {:noreply, %{state | earliest_stored_height: earliest_height, latest_stored_height: latest_height}} end @doc false @@ -115,13 +120,17 @@ defmodule OMG.ChildChain.GasPrice.History.Server do # defp do_populate_prices(from_height, to_height, state) do + fetch_from_height = max(from_height, state.latest_stored_height) + + # Fetch and insert new heights, leaving obsolete heights intact. :ok = - from_height..to_height + fetch_from_height..to_height |> Fetcher.stream() - |> stream_insert() + |> stream_insert(state.history_ets) |> Stream.run() - :ok = prune_heights(state.earliest_stored_height, from_height - 1) + # Prune obsolete heights. + :ok = prune_heights(state.history_ets, state.earliest_stored_height, from_height - 1) _ = Logger.info( @@ -132,20 +141,20 @@ defmodule OMG.ChildChain.GasPrice.History.Server do # Inform all subscribers that the history has been updated. _ = Enum.each(state.subscribers, fn subscriber -> send(subscriber, {History, :updated}) end) - :ok + {:ok, from_height, to_height} end - defp stream_insert(stream_blocks) do + defp stream_insert(stream_blocks, history_ets) do Stream.each(stream_blocks, fn block -> height = Encoding.int_from_hex(block["number"]) prices = Enum.map(block["transactions"], fn tx -> Encoding.int_from_hex(tx["gasPrice"]) end) timestamp = Encoding.int_from_hex(block["timestamp"]) - true = :ets.insert(@history_table, {height, prices, timestamp}) + true = :ets.insert(history_ets, {height, prices, timestamp}) end) end - defp prune_heights(from, to) do - Enum.each(from..to, fn height -> :ets.delete(@history_table, height) end) + defp prune_heights(history_ets, from, to) do + Enum.each(from..to, fn height -> :ets.delete(history_ets, height) end) end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex index e7ad6ce85e..b09ee170cb 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy.ex @@ -18,8 +18,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy do To see what strategies are available, look into `OMG.ChildChain.GasPrice.Strategy.*` modules. """ - alias OMG.ChildChain.GasPrice @callback recalculate(params :: Keyword.t()) :: :ok | no_return() - @callback get_price() :: {:ok, GasPrice.t()} + @callback get_price() :: {:ok, pos_integer()} | {:error, atom()} end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 8e7027830f..70c5d84c34 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -20,20 +20,23 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do """ use GenServer require Logger - alias OMG.ChildChain.GasPrice alias OMG.ChildChain.GasPrice.History alias OMG.ChildChain.GasPrice.Strategy @type t() :: %__MODULE__{ - recommendations: %{ - safe_low: float(), - standard: float(), - fast: float(), - fastest: float() - } + prices: + %{ + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } + | error() } - defstruct recommendations: %{ + @type error() :: {:error, :all_empty_blocks} + + defstruct prices: %{ safe_low: 20_000_000_000, standard: 20_000_000_000, fast: 20_000_000_000, @@ -63,7 +66,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do Suggests the optimal gas price. """ @impl Strategy - @spec get_price() :: GasPrice.t() + @spec get_price() :: {:ok, pos_integer()} | error() def get_price() do GenServer.call(__MODULE__, :get_price) end @@ -103,16 +106,25 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc false @impl GenServer def handle_call(:get_price, _, state) do - price = state.recommendations[@target_threshold] - {:reply, {:ok, price}, state} + price = + case state.prices do + {:error, _} = error -> + error + + prices -> + {:ok, prices[@target_threshold]} + end + + {:reply, price, state} end @doc false @impl GenServer def handle_info({History, :updated}, state) do - recommendations = do_recalculate() + prices = do_recalculate() - {:noreply, %{state | recommendations: recommendations}} + _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") + {:noreply, %{state | prices: prices}} end # @@ -122,14 +134,23 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do defp do_recalculate() do sorted_min_prices = History.all() - |> Enum.map(fn {_height, prices, _timestamp} -> Enum.min(prices) end) + |> Enum.reduce([], fn + # Skips empty blocks (possible in local chain and low traffic testnets) + {_height, [], _timestamp}, acc -> acc + {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] + end) |> Enum.sort() - block_count = length(sorted_min_prices) - - Enum.map(@thresholds, fn value -> - position = floor(block_count * value / 100) - 1 - Enum.at(sorted_min_prices, position) - end) + # Handles when all blocks are empty (possible on local chain and low traffic testnets) + case length(sorted_min_prices) do + 0 -> + {:error, :all_empty_blocks} + + block_count -> + Enum.map(@thresholds, fn {_name, value} -> + position = floor(block_count * value / 100) - 1 + Enum.at(sorted_min_prices, position) + end) + end end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex index 0cd908d70b..3f77f14676 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex @@ -95,7 +95,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do Suggests the optimal gas price. """ @impl Strategy - @spec get_price() :: {:ok, GasPrice.t()} + @spec get_price() :: {:ok, pos_integer()} def get_price() do GenServer.call(__MODULE__, :get_price) end @@ -136,14 +136,19 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do @doc false @impl GenServer def handle_call({:recalculate, latest}, _, state) do - {:reply, :ok, do_recalculate(latest, state)} + state = recalculate_state(latest, state) + + _ = + Logger.info("#{__MODULE__}: Gas calculation triggered. New price suggestion: #{inspect(state.gas_price_to_use)}") + + {:reply, :ok, state} end # # Internal implementations # - defp do_recalculate(latest, state) do + defp recalculate_state(latest, state) do cond do latest.parent_height - state.last_parent_height < state.eth_gap_without_child_blocks -> state diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index e078ac7566..c8046815ba 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -22,20 +22,23 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do """ use GenServer require Logger - alias OMG.ChildChain.GasPrice alias OMG.ChildChain.GasPrice.History alias OMG.ChildChain.GasPrice.Strategy @type t() :: %__MODULE__{ - recommendations: %{ - safe_low: float(), - standard: float(), - fast: float(), - fastest: float() - } + prices: + %{ + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } + | error() } - defstruct recommendations: %{ + @type error() :: {:error, :all_empty_blocks} + + defstruct prices: %{ safe_low: 20_000_000_000, standard: 20_000_000_000, fast: 20_000_000_000, @@ -65,7 +68,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do Suggests the optimal gas price. """ @impl Strategy - @spec get_price() :: GasPrice.t() + @spec get_price() :: {:ok, pos_integer()} | error() def get_price() do GenServer.call(__MODULE__, :get_price) end @@ -105,16 +108,25 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc false @impl GenServer def handle_call(:get_price, _, state) do - price = state.recommendations[@target_threshold] - {:reply, {:ok, price}, state} + price = + case state.prices do + {:error, _} = error -> + error + + prices -> + {:ok, prices[@target_threshold]} + end + + {:reply, price, state} end @doc false @impl GenServer def handle_info({History, :updated}, state) do - recommendations = do_recalculate() + prices = do_recalculate() - {:noreply, %{state | recommendations: recommendations}} + _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") + {:noreply, %{state | prices: prices}} end # @@ -124,15 +136,24 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do defp do_recalculate() do sorted_min_prices = History.all() - |> Enum.map(fn {_height, prices, _timestamp} -> Enum.min(prices) end) + |> Enum.reduce([], fn + # Skips empty blocks (possible in local chain and low traffic testnets) + {_height, [], _timestamp}, acc -> acc + {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] + end) |> Enum.sort() - block_count = length(sorted_min_prices) - - Enum.map(@thresholds, fn value -> - position = floor(block_count * value / 100) - 1 - Enum.at(sorted_min_prices, position) - end) + # Handles when all blocks are empty (possible on local chain and low traffic testnets) + case length(sorted_min_prices) do + 0 -> + {:error, :all_empty_blocks} + + block_count -> + Enum.map(@thresholds, fn {_name, value} -> + position = floor(block_count * value / 100) - 1 + Enum.at(sorted_min_prices, position) + end) + end end # diff --git a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex index fb4acf92c0..e087a21246 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex @@ -23,8 +23,7 @@ defmodule OMG.ChildChain.Supervisor do alias OMG.ChildChain.Configuration alias OMG.ChildChain.DatadogEvent.ContractEventConsumer alias OMG.ChildChain.FeeServer - alias OMG.ChildChain.GasPrice.PoissonGasStrategy - alias OMG.ChildChain.GasPrice.LegacyGasStrategy + alias OMG.ChildChain.GasPrice.GasPriceSupervisor alias OMG.ChildChain.Monitor alias OMG.ChildChain.SyncSupervisor alias OMG.ChildChain.Tracer @@ -48,7 +47,8 @@ defmodule OMG.ChildChain.Supervisor do :ok = Storage.ensure_ets_init(blocks_cache()) {:ok, contract_deployment_height} = RootChain.get_root_deployment_height() metrics_collection_interval = Configuration.metrics_collection_interval() - block_submit_max_gas_price = Configuration.block_submit_max_gas_price() + gas_price_history_blocks = Configuration.block_submit_gas_price_history_blocks() + max_gas_price = Configuration.block_submit_max_gas_price() fee_server_opts = Configuration.fee_server_opts() fee_claimer_address = OMG.Configuration.fee_claimer_address() child_block_interval = OMG.Eth.Configuration.child_block_interval() @@ -72,8 +72,7 @@ defmodule OMG.ChildChain.Supervisor do type: :supervisor } ]}, - {PoissonGasStrategy, [max_gas_price: block_submit_max_gas_price]}, - {LegacyGasStrategy, [max_gas_price: block_submit_max_gas_price]} + {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price]} ] is_datadog_disabled = is_disabled?() diff --git a/config/config.exs b/config/config.exs index 550c2a256e..e56d96cd70 100644 --- a/config/config.exs +++ b/config/config.exs @@ -46,7 +46,7 @@ config :omg_child_chain, block_queue_eth_height_check_interval_ms: 6_000, block_submit_every_nth: 1, block_submit_max_gas_price: 20_000_000_000, - block_submit_gas_price_strategy: OMG.ChildChain.GasPrice.LegacyGasStrategy, + block_submit_gas_price_strategy: OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy, block_submit_gas_price_history_blocks: 200, metrics_collection_interval: 60_000, fee_adapter_check_interval_ms: 10_000, From 6e03b5e7cce8e5bb33ae2a46f2a051763fcb2eb0 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sat, 27 Jun 2020 12:24:13 +0700 Subject: [PATCH 13/37] docs: gas price strategies --- .../lib/omg_child_chain/gas_price.ex | 12 ++++++++++ .../gas_price/gas_price_supervisor.ex | 5 ++++ .../gas_price/history/fetcher.ex | 6 +++++ .../gas_price/strategy/legacy_gas_strategy.ex | 24 ++++++++++--------- 4 files changed, 36 insertions(+), 11 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index 6da11b0360..d973e96409 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -15,6 +15,18 @@ defmodule OMG.ChildChain.GasPrice do @moduledoc """ Suggests gas prices based on different strategies. + + ## Usage + + Include `OMG.ChildChain.GasPrice.GasPriceSupervisor` in the supervision tree: + + children = [ + {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price]} + ] + + Supervisor.init(children, strategy: :one_for_one) + + Then, call `OMG.ChildChain.GasPrice.get_price()` to get the gas price suggestion. """ alias OMG.ChildChain.Configuration alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex index e3c045ecd4..5ecb3bc3b8 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex @@ -35,7 +35,12 @@ defmodule OMG.ChildChain.GasPrice.GasPriceSupervisor do children = [ {History, [event_bus: Bus, num_blocks: num_blocks]}, + # Unfortunately LegacyGasStrategy cannot rely on its caller to enforce `max_gas_price` + # because the strategy can keep doubling the price through the roof and take forever + # to get back down below `max_gas_price` without its own ceiling. {LegacyGasStrategy, [max_gas_price: max_gas_price]}, + # Other strategies don't need `max_gas_price` at the algorithm level because they depend + # on historical statistics and not an uncapped multiplier. {BlockPercentileGasStrategy, []}, {PoissonGasStrategy, []} ] diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex index aecba3c1ab..b7943a51e6 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex @@ -18,12 +18,18 @@ defmodule OMG.ChildChain.GasPrice.History.Fetcher do alias Ethereumex.HttpClient alias OMG.Eth.Encoding + # Too large a batch and the Eth client will time out, especially because we are + # requesting for all transaction info in each block. Use a number that we are certain + # the client is able to serve or within `@retries`. @per_batch 20 + @retries 5 @retry_ms 10_000 @doc """ Prepares a stream that fetches the gas prices within the given heights. + + Internally this function fetches the input `heights` in batches of #{@per_batch} blocks. """ @spec stream(Range.t()) :: Enumerable.t() def stream(heights) do diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex index 3f77f14676..ecef2380e0 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex @@ -56,7 +56,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do # the factor the gas price will be increased by gas_price_raising_factor: float(), # last gas price calculated - gas_price_to_use: pos_integer(), + gas_price: pos_integer(), # maximum gas price above which raising has no effect, limits the gas price calculation max_gas_price: pos_integer(), # last parent height successfully evaluated for gas price @@ -68,7 +68,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do defstruct eth_gap_without_child_blocks: 2, gas_price_lowering_factor: 0.9, gas_price_raising_factor: 2.0, - gas_price_to_use: 20_000_000_000, + gas_price: 20_000_000_000, max_gas_price: 20_000_000_000, last_block_mined: 0, last_parent_height: 0, @@ -84,6 +84,9 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do @doc false @impl GenServer def init(init_arg) do + # Unfortunately LegacyGasStrategy cannot rely on its caller to enforce `max_gas_price` + # because the strategy can keep doubling the price through the roof and take forever + # to get back down below `max_gas_price` without its own ceiling. state = %__MODULE__{ max_gas_price: Keyword.fetch!(init_arg, :max_gas_price) } @@ -130,7 +133,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do @doc false @impl GenServer def handle_call(:get_price, _, state) do - {:reply, {:ok, state.gas_price_to_use}, state} + {:reply, {:ok, state.gas_price}, state} end @doc false @@ -138,8 +141,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do def handle_call({:recalculate, latest}, _, state) do state = recalculate_state(latest, state) - _ = - Logger.info("#{__MODULE__}: Gas calculation triggered. New price suggestion: #{inspect(state.gas_price_to_use)}") + _ = Logger.info("#{__MODULE__}: Gas price recalculated. New price: #{inspect(state.gas_price)}") {:reply, :ok, state} end @@ -162,19 +164,19 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do state true -> - gas_price_to_use = + gas_price = calculate_gas_price( latest.mined_child_block_num, latest.formed_child_block_num, state.last_mined_child_block_num, - state.gas_price_to_use, + state.gas_price, state.gas_price_raising_factor, state.gas_price_lowering_factor, state.max_gas_price ) - state = Map.update!(state, :gas_price_to_use, gas_price_to_use) - _ = Logger.debug("using new gas price '#{inspect(state.gas_price_to_use)}'") + state = Map.update!(state, :gas_price, gas_price) + _ = Logger.debug("using new gas price '#{inspect(state.gas_price)}'") case state.last_mined_child_block_num < latest.mined_child_block_num do true -> @@ -203,7 +205,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do mined_child_block_num, formed_child_block_num, last_mined_child_block_num, - gas_price_to_use, + gas_price, raising_factor, lowering_factor, max_gas_price @@ -218,7 +220,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do {_, true} -> lowering_factor end - Kernel.min(max_gas_price, Kernel.round(multiplier * gas_price_to_use)) + Kernel.min(max_gas_price, Kernel.round(multiplier * gas_price)) end defp blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) do From d0fb11050144d94db41c310542c3966fba815749 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sat, 27 Jun 2020 12:24:32 +0700 Subject: [PATCH 14/37] refactor: cleaner gas price recalculation --- .../strategy/block_percentile_gas_strategy.ex | 17 +++++++++-------- .../gas_price/strategy/poisson_gas_strategy.ex | 17 +++++++++-------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 70c5d84c34..0c8ff9141c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -132,14 +132,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do # defp do_recalculate() do - sorted_min_prices = - History.all() - |> Enum.reduce([], fn - # Skips empty blocks (possible in local chain and low traffic testnets) - {_height, [], _timestamp}, acc -> acc - {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] - end) - |> Enum.sort() + sorted_min_prices = History.all() |> filter_min() |> Enum.sort() # Handles when all blocks are empty (possible on local chain and low traffic testnets) case length(sorted_min_prices) do @@ -153,4 +146,12 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do end) end end + + defp filter_min(prices) do + Enum.reduce(prices, [], fn + # Skips empty blocks (possible in local chain and low traffic testnets) + {_height, [], _timestamp}, acc -> acc + {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] + end) + end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index c8046815ba..dca3972fa0 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -134,14 +134,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do # defp do_recalculate() do - sorted_min_prices = - History.all() - |> Enum.reduce([], fn - # Skips empty blocks (possible in local chain and low traffic testnets) - {_height, [], _timestamp}, acc -> acc - {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] - end) - |> Enum.sort() + sorted_min_prices = History.all() |> filter_min() |> Enum.sort() # Handles when all blocks are empty (possible on local chain and low traffic testnets) case length(sorted_min_prices) do @@ -156,6 +149,14 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do end end + defp filter_min(prices) do + Enum.reduce(prices, [], fn + # Skips empty blocks (possible in local chain and low traffic testnets) + {_height, [], _timestamp}, acc -> acc + {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] + end) + end + # # Internal implementations # From 0b5caeaa12c3a3e90c8606ba4f2dca5d34585cbe Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 02:20:09 +0700 Subject: [PATCH 15/37] feat: ethgasstation algo that compiles --- .../lib/omg_child_chain/block_queue/core.ex | 2 +- .../gas_price/history/fetcher.ex | 3 + .../strategy/poisson_gas_strategy.ex | 77 +----------- .../poisson_gas_strategy/algorithm.ex | 114 ++++++++++++++++++ 4 files changed, 123 insertions(+), 73 deletions(-) create mode 100644 apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index 5f7b176c70..11aac232d2 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -40,8 +40,8 @@ defmodule OMG.ChildChain.BlockQueue.Core do nonce=1 blknum=1000, nonce=2 blknum=2000 etc. """ alias OMG.ChildChain.BlockQueue - alias OMG.ChildChain.BlockQueue.Core alias OMG.ChildChain.BlockQueue.BlockSubmission + alias OMG.ChildChain.BlockQueue.Core alias OMG.ChildChain.GasPrice use OMG.Utils.LoggerExt diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex index b7943a51e6..6bdd541d3c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex @@ -13,6 +13,9 @@ # limitations under the License. defmodule OMG.ChildChain.GasPrice.History.Fetcher do + @moduledoc """ + Provides functions to stream-fetch block history from an Ethereum client. + """ require Logger alias Ethereumex.HttpClient diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index dca3972fa0..169c9a5315 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -24,6 +24,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do require Logger alias OMG.ChildChain.GasPrice.History alias OMG.ChildChain.GasPrice.Strategy + alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm @type t() :: %__MODULE__{ prices: @@ -134,78 +135,10 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do # defp do_recalculate() do - sorted_min_prices = History.all() |> filter_min() |> Enum.sort() - - # Handles when all blocks are empty (possible on local chain and low traffic testnets) - case length(sorted_min_prices) do - 0 -> - {:error, :all_empty_blocks} - - block_count -> - Enum.map(@thresholds, fn {_name, value} -> - position = floor(block_count * value / 100) - 1 - Enum.at(sorted_min_prices, position) - end) - end - end + price_history = History.all() - defp filter_min(prices) do - Enum.reduce(prices, [], fn - # Skips empty blocks (possible in local chain and low traffic testnets) - {_height, [], _timestamp}, acc -> acc - {_height, prices, _timestamp}, acc -> [Enum.min(prices) | acc] - end) + {hash_percentages, lowest_min_price, highest_min_price} = Algorithm.analyze_blocks(price_history) + prediction_table = Algorithm.make_prediction_table(hash_percentages, lowest_min_price, highest_min_price) + Algorithm.get_recommendations(@thresholds, prediction_table) end - - # - # Internal implementations - # - - # Port over https://github.com/ethgasstation/gasstation-express-oracle/blob/master/gasExpress.py - # - # def make_predictTable(block, alltx, hashpower, avg_timemined): - # predictTable = pd.DataFrame({'gasprice' : range(10, 1010, 10)}) - # ptable2 = pd.DataFrame({'gasprice' : range(0, 10, 1)}) - # predictTable = predictTable.append(ptable2).reset_index(drop=True) - # predictTable = predictTable.sort_values('gasprice').reset_index(drop=True) - # predictTable['hashpower_accepting'] = predictTable['gasprice'].apply(get_hpa, args=(hashpower,)) - # return(predictTable) - # - # def get_hpa(gasprice, hashpower): - # """gets the hash power accpeting the gas price over last 200 blocks""" - # hpa = hashpower.loc[gasprice >= hashpower.index, 'hashp_pct'] - # if gasprice > hashpower.index.max(): - # hpa = 100 - # elif gasprice < hashpower.index.min(): - # hpa = 0 - # else: - # hpa = hpa.max() - # return int(hpa) - # - # def analyze_last200blocks(block, blockdata): - # recent_blocks = blockdata.loc[blockdata['block_number'] > (block-200), ['mingasprice', 'block_number']] - # #create hashpower accepting dataframe based on mingasprice accepted in block - # hashpower = recent_blocks.groupby('mingasprice').count() - # hashpower = hashpower.rename(columns={'block_number': 'count'}) - # hashpower['cum_blocks'] = hashpower['count'].cumsum() - # totalblocks = hashpower['count'].sum() - # hashpower['hashp_pct'] = hashpower['cum_blocks']/totalblocks*100 - # #get avg blockinterval time - # blockinterval = recent_blocks.sort_values('block_number').diff() - # blockinterval.loc[blockinterval['block_number'] > 1, 'time_mined'] = np.nan - # blockinterval.loc[blockinterval['time_mined']< 0, 'time_mined'] = np.nan - # avg_timemined = blockinterval['time_mined'].mean() - # if np.isnan(avg_timemined): - # avg_timemined = 15 - # return(hashpower, avg_timemined) - # - # def get_fast(): - # series = prediction_table.loc[prediction_table['hashpower_accepting'] >= FAST, 'gasprice'] - # fastest = series.min() - # return float(fastest) - # - # def get_fastest(): - # hpmax = prediction_table['hashpower_accepting'].max() - # fastest = prediction_table.loc[prediction_table['hashpower_accepting'] == hpmax, 'gasprice'].values[0] - # return float(fastest) end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex new file mode 100644 index 0000000000..21fe34f031 --- /dev/null +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex @@ -0,0 +1,114 @@ +# Copyright 2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do + @moduledoc """ + The algorithmic functions for PoissonGasStrategy. + """ + @wei_to_gwei 1_000_000_000 + @prediction_table_ranges :lists.seq(0, 9, 1) ++ :lists.seq(10, 1010, 10) + + @doc """ + Analyzes historical blocks. Returns the lowest minimum gas price found in a block, + the highest minimum gas price found in a block, and the percentage of blocks accepted + per each minimum gas price seen. + + Equivalent to [gasExpress.py's `analyze_last200blocks()`](https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L119-L134) + """ + def analyze_blocks(price_history) do + # [min_price1, min_price2, min_price3, ...] + sorted_min_prices = + price_history + |> remove_empty() + |> aggregate_min_prices() + |> Enum.sort() + + num_blocks = length(sorted_min_prices) + lowest_min_price = Enum.at(sorted_min_prices, 0) + highest_min_price = Enum.at(sorted_min_prices, -1) + + # %{min_price => num_blocks_accepted} + num_blocks_by_min_prices = Enum.frequencies(sorted_min_prices) + + # %{min_price => cumulative_num_blocks_accepted} + cumulative_sum = cumulative_sum(num_blocks_by_min_prices) + + # %{min_price => hash_percentage} + hash_percentages = Enum.map(cumulative_sum, fn {_min_price, cumsum} -> cumsum / num_blocks * 100 end) + + {hash_percentages, lowest_min_price, highest_min_price} + end + + @doc """ + Generates the gas price prediction table based on the historical hash percentages. + + Equivalent to [gasExpress.py's `make_predictTable()`](https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L137-L145) + """ + def make_prediction_table(hash_percentages, lowest_min_price, highest_min_price) do + Enum.into(@prediction_table_ranges, %{}, fn gas_price -> + hpa = + cond do + gas_price > highest_min_price -> 100 + gas_price < lowest_min_price -> 0 + true -> get_hpa(hash_percentages, gas_price) + end + + {gas_price, hpa} + end) + end + + @doc """ + Converts the gas price prediction table into easily digestible thresholds. + + Equivalent to [gasExpress.py's `get_gasprice_recs()`](https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L147-L176) + """ + def get_recommendations(thresholds, prediction_table) do + Enum.map(thresholds, fn {threshold_name, threshold_value} -> + suggested_price = + prediction_table + |> Enum.find(fn {_gas_price, hpa} -> hpa >= threshold_value end) + |> elem(0) + + {threshold_name, suggested_price} + end) + end + + defp remove_empty(history) do + Enum.reject(history, fn {_height, prices, _timestamp} -> Enum.empty?(prices) end) + end + + defp aggregate_min_prices(history) do + Enum.map(history, fn {_height, prices, _timestamp} -> min_gwei(prices) end) + end + + defp min_gwei(prices) do + min_price = Enum.min(prices) + Integer.floor_div(min_price, @wei_to_gwei) * @wei_to_gwei + end + + defp cumulative_sum(values) do + values + |> Enum.scan(fn {min_price, num_blocks}, {_previous_min_price, previous_cumsum} -> + {min_price, num_blocks + previous_cumsum} + end) + |> Enum.into(%{}) + end + + defp get_hpa(hash_percentages, gas_price) do + hash_percentages + |> Enum.filter(fn {min_price, _} -> gas_price >= min_price end) + |> Enum.map(&elem(&1, 1)) + |> Enum.max() + end +end From d3785390c4ab027aebbb0b995f590ae49f2b13a0 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 02:39:14 +0700 Subject: [PATCH 16/37] feat: log all price suggestions --- .../lib/omg_child_chain/gas_price.ex | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index d973e96409..ecf395d47b 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -28,11 +28,13 @@ defmodule OMG.ChildChain.GasPrice do Then, call `OMG.ChildChain.GasPrice.get_price()` to get the gas price suggestion. """ + require Logger alias OMG.ChildChain.Configuration + alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy - @strategies [LegacyGasStrategy, PoissonGasStrategy] + @strategies [BlockPercentileGasStrategy, LegacyGasStrategy, PoissonGasStrategy] @doc """ Trigger gas price recalculations for all strategies. @@ -47,6 +49,16 @@ defmodule OMG.ChildChain.GasPrice do """ @spec get_price() :: {:ok, pos_integer()} | {:error, atom()} def get_price() do - Configuration.block_submit_gas_price_strategy().get_price() + price_suggestions = + Enum.reduce(@strategies, %{}, fn strategy, suggestions -> + Map.put(suggestions, strategy, strategy.get_price()) + end) + + _ = Logger.info("#{__MODULE__}: All price suggestions: #{inspect(price_suggestions)}") + + gas_price = price_suggestions[Configuration.block_submit_gas_price_strategy()] + + _ = Logger.info("#{__MODULE__}: Suggesting gas price: #{gas_price / 1_000_000_000} gwei.") + gas_price end end From 19cf8208285a1cf439bec985a1111bc2eab5ab94 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:01:32 +0700 Subject: [PATCH 17/37] fix: wrong price suggestion return format --- .../gas_price/strategy/block_percentile_gas_strategy.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 0c8ff9141c..71b9ff0790 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -140,9 +140,9 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do {:error, :all_empty_blocks} block_count -> - Enum.map(@thresholds, fn {_name, value} -> + Enum.map(@thresholds, fn {threshold_name, value} -> position = floor(block_count * value / 100) - 1 - Enum.at(sorted_min_prices, position) + {threshold_name, Enum.at(sorted_min_prices, position)} end) end end From 27f602793d84733b43bb51399c61d6b1fb2c6c5e Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:01:57 +0700 Subject: [PATCH 18/37] fix: poisson strategy division unit --- .../poisson_gas_strategy/algorithm.ex | 50 ++++++++++++++----- 1 file changed, 38 insertions(+), 12 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex index 21fe34f031..917137618f 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex @@ -15,8 +15,19 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do @moduledoc """ The algorithmic functions for PoissonGasStrategy. + + Note that the internal unit used in this strategy is 10 Gwei, and therefore internal conversions + from wei are based on 1e8 (`100_000_000`) instead of the usual 1e9 (`1_000_000_000`). + + The return of public functions, however, are already converted back to wei. """ - @wei_to_gwei 1_000_000_000 + + # Note that the division by 1e8 instead of 1e9 here is intentional. + # The unit is per 10 Gwei, which aligns with `@prediction_table_ranges`. + # See: https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L48 + @wei_to_10gwei 100_000_000 + + # 1, 2, 3, .., 10, 20, 30, .., 1010 @prediction_table_ranges :lists.seq(0, 9, 1) ++ :lists.seq(10, 1010, 10) @doc """ @@ -31,7 +42,8 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do sorted_min_prices = price_history |> remove_empty() - |> aggregate_min_prices() + |> extract_min_prices() + |> round_10gwei() |> Enum.sort() num_blocks = length(sorted_min_prices) @@ -45,7 +57,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do cumulative_sum = cumulative_sum(num_blocks_by_min_prices) # %{min_price => hash_percentage} - hash_percentages = Enum.map(cumulative_sum, fn {_min_price, cumsum} -> cumsum / num_blocks * 100 end) + hash_percentages = Enum.map(cumulative_sum, fn {min_price, cumsum} -> {min_price, cumsum / num_blocks * 100} end) {hash_percentages, lowest_min_price, highest_min_price} end @@ -56,7 +68,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do Equivalent to [gasExpress.py's `make_predictTable()`](https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L137-L145) """ def make_prediction_table(hash_percentages, lowest_min_price, highest_min_price) do - Enum.into(@prediction_table_ranges, %{}, fn gas_price -> + Enum.into(@prediction_table_ranges, [], fn gas_price -> hpa = cond do gas_price > highest_min_price -> 100 @@ -76,9 +88,10 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do def get_recommendations(thresholds, prediction_table) do Enum.map(thresholds, fn {threshold_name, threshold_value} -> suggested_price = - prediction_table - |> Enum.find(fn {_gas_price, hpa} -> hpa >= threshold_value end) - |> elem(0) + case Enum.find(prediction_table, fn {_gas_price, hpa} -> hpa >= threshold_value end) do + nil -> nil + {gas_price, _hpa} -> gas_price / 10 + end {threshold_name, suggested_price} end) @@ -88,13 +101,26 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do Enum.reject(history, fn {_height, prices, _timestamp} -> Enum.empty?(prices) end) end - defp aggregate_min_prices(history) do - Enum.map(history, fn {_height, prices, _timestamp} -> min_gwei(prices) end) + defp extract_min_prices(history) do + Enum.map(history, fn {_height, prices, _timestamp} -> Enum.min(prices) end) end - defp min_gwei(prices) do - min_price = Enum.min(prices) - Integer.floor_div(min_price, @wei_to_gwei) * @wei_to_gwei + defp round_10gwei(prices) when is_list(prices) do + Enum.map(prices, &round_10gwei/1) + end + + # https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L46-L57 + defp round_10gwei(price) do + case price / @wei_to_10gwei do + gp when gp >= 1 and gp < 10 -> + floor(gp) + + gp when gp >= 10 -> + floor(gp / 10) * 10 + + _ -> + 0 + end end defp cumulative_sum(values) do From 9e416c11ccb15717824e505fffec6a9b3b53ba32 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:02:24 +0700 Subject: [PATCH 19/37] feat: detailed gas price logging --- .../lib/omg_child_chain/gas_price.ex | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index ecf395d47b..894c43960c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -18,6 +18,14 @@ defmodule OMG.ChildChain.GasPrice do ## Usage + Prepare the gas price configurations: + + config :omg_child_chain, + # ... + block_submit_gas_price_strategy: OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy, + block_submit_max_gas_price: 20_000_000_000, + block_submit_gas_price_history_blocks: 200 + Include `OMG.ChildChain.GasPrice.GasPriceSupervisor` in the supervision tree: children = [ @@ -47,7 +55,7 @@ defmodule OMG.ChildChain.GasPrice do @doc """ Suggests the optimal gas price using the configured strategy. """ - @spec get_price() :: {:ok, pos_integer()} | {:error, atom()} + @spec get_price() :: {:ok, pos_integer()} | {:error, :no_gas_price_suggestion} def get_price() do price_suggestions = Enum.reduce(@strategies, %{}, fn strategy, suggestions -> @@ -56,9 +64,16 @@ defmodule OMG.ChildChain.GasPrice do _ = Logger.info("#{__MODULE__}: All price suggestions: #{inspect(price_suggestions)}") - gas_price = price_suggestions[Configuration.block_submit_gas_price_strategy()] + strategy = Configuration.block_submit_gas_price_strategy() + + case price_suggestions[strategy] do + {:ok, gas_price} -> + _ = Logger.info("#{__MODULE__}: Suggesting gas price: #{gas_price / 1_000_000_000} gwei.") + {:ok, gas_price} - _ = Logger.info("#{__MODULE__}: Suggesting gas price: #{gas_price / 1_000_000_000} gwei.") - gas_price + error -> + _ = Logger.info("#{__MODULE__}: Gas price suggestion for #{strategy} cannot be determined. Got: #{error}.") + error + end end end From 6a72751e83ebcf5f805637f8082ee386e5ddb2a6 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:05:40 +0700 Subject: [PATCH 20/37] feat: add random geth-random-gas-price-tx-factory to docker-compose.dev.yml --- docker-compose.dev.yml | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 25adc122a7..dfc7ea401a 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -44,3 +44,40 @@ services: - "2023-2024:2023-2024" - "8125:8125/udp" - "8126:8126/tcp" + + # Submits an Ethereum transaction with a random gas price between 1 - 100 Gwei + # every $SLEEP_SECONDS interval + geth-random-gas-price-tx-factory: + image: curlimages/curl:7.71.0 + command: > + sh -c ' + while true + do + TX_AMOUNT_GWEI=$$((1 + RANDOM % 10)) + TX_AMOUNT_WEI=$$(($$TX_AMOUNT_GWEI*1000000000)) + TX_AMOUNT_HEX="0x$$(printf "%x" '$$TX_AMOUNT_WEI')" + + GAS_PRICE_GWEI=$$((1 + RANDOM % 100)) + GAS_PRICE_WEI=$$(($$GAS_PRICE_GWEI*1000000000)) + GAS_PRICE_HEX="0x$$(printf "%x" '$$GAS_PRICE_WEI')" + echo Submitting $$TX_AMOUNT_GWEI Gwei tx with gas price of $$GAS_PRICE_GWEI Gwei + + curl -s -X POST http://geth:8545 \ + -H "Content-Type: application/json" \ + --data "{ + \"jsonrpc\": \"2.0\", + \"method\": \"eth_sendTransaction\", + \"params\": [{ + \"from\": \"0x6de4b3b9c28e9c3e84c2b2d3a875c947a84de68d\", + \"to\": \"0x6de4b3b9c28e9c3e84c2b2d3a875c947a84de68d\", + \"gasPrice\": \"'$$GAS_PRICE_HEX'\", + \"value\": \"'$$TX_AMOUNT_HEX'\" + }], + \"id\":1 + }" + + sleep $$SLEEP_SECONDS + done + ' + environment: + - SLEEP_SECONDS=10 From f0a3565bc5c11db7efac4817398a4836c91103ae Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:08:18 +0700 Subject: [PATCH 21/37] docs: list available gas price strategies --- docs/deployment_configuration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deployment_configuration.md b/docs/deployment_configuration.md index 91a7ed26dd..b6e4742e88 100644 --- a/docs/deployment_configuration.md +++ b/docs/deployment_configuration.md @@ -23,7 +23,7 @@ ***Child Chain only*** - "BLOCK_SUBMIT_MAX_GAS_PRICE" - The maximum gas price to use for block submission. The first block submission after application boot will use the max price. The gas price gradually adjusts on subsequent blocks to reach the current optimum price . Defaults to `20000000000` (20 Gwei). -- "BLOCK_SUBMIT_GAS_PRICE_STRATEGY" - The gas price strategy to use for block submission. Note that all strategies will still run, but only the one configured here will be used for actual submission. Defaults to `LEGACY`. +- "BLOCK_SUBMIT_GAS_PRICE_STRATEGY" - The gas price strategy to use for block submission. Note that all strategies will still run, but only the one configured here will be used for actual submission. Suppots `LEGACY`, `BLOCK_PERCENTILE` and `POISSON`. Defaults to `LEGACY`. - "FEE_ADAPTER" - The adapter to use to populate the fee specs. Either `file` or `feed` (case-insensitive). Defaults to `file` with an empty fee specs. - "FEE_CLAIMER_ADDRESS" - 20-bytes HEX-encoded string of Ethereum address of Fee Claimer. - "FEE_BUFFER_DURATION_MS" - Buffer period during which a fee is still valid after being updated. From ff091b0c6a03e9f173fb9faa415fd19b42671c47 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:16:43 +0700 Subject: [PATCH 22/37] feat: make geth-random-gas-price-tx-factory's max gas price configurable --- docker-compose.dev.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index dfc7ea401a..0717a2486f 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -45,7 +45,7 @@ services: - "8125:8125/udp" - "8126:8126/tcp" - # Submits an Ethereum transaction with a random gas price between 1 - 100 Gwei + # Submits an Ethereum transaction with a random gas price between 1 - $MAX_GAS_PRICE_GWEI Gwei # every $SLEEP_SECONDS interval geth-random-gas-price-tx-factory: image: curlimages/curl:7.71.0 @@ -57,7 +57,7 @@ services: TX_AMOUNT_WEI=$$(($$TX_AMOUNT_GWEI*1000000000)) TX_AMOUNT_HEX="0x$$(printf "%x" '$$TX_AMOUNT_WEI')" - GAS_PRICE_GWEI=$$((1 + RANDOM % 100)) + GAS_PRICE_GWEI=$$((1 + RANDOM % $$MAX_GAS_PRICE_GWEI)) GAS_PRICE_WEI=$$(($$GAS_PRICE_GWEI*1000000000)) GAS_PRICE_HEX="0x$$(printf "%x" '$$GAS_PRICE_WEI')" echo Submitting $$TX_AMOUNT_GWEI Gwei tx with gas price of $$GAS_PRICE_GWEI Gwei @@ -80,4 +80,5 @@ services: done ' environment: + - MAX_GAS_PRICE_GWEI=100 - SLEEP_SECONDS=10 From 9a2b48b7cb87feb5b9ebcaa1e228c74572173468 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Sun, 28 Jun 2020 16:26:04 +0700 Subject: [PATCH 23/37] refactor: use the more natural 100mwei unit --- .../poisson_gas_strategy/algorithm.ex | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex index 917137618f..c223d904f5 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex @@ -16,16 +16,17 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do @moduledoc """ The algorithmic functions for PoissonGasStrategy. - Note that the internal unit used in this strategy is 10 Gwei, and therefore internal conversions - from wei are based on 1e8 (`100_000_000`) instead of the usual 1e9 (`1_000_000_000`). + Note that to align with the original EthGasStation's implementation, the internal unit + used in this strategy is 0.1 Gwei (100 Mwei), and therefore internal conversions from wei + are based on 1e8 (`100_000_000`) instead of the usual 1e9 (`1_000_000_000`). - The return of public functions, however, are already converted back to wei. + The return of public functions, however, are already converted back to wei to remain + consistent and compatible with other strategies. + + See https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py """ - # Note that the division by 1e8 instead of 1e9 here is intentional. - # The unit is per 10 Gwei, which aligns with `@prediction_table_ranges`. - # See: https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L48 - @wei_to_10gwei 100_000_000 + @amount100mwei 100_000_000 # 1, 2, 3, .., 10, 20, 30, .., 1010 @prediction_table_ranges :lists.seq(0, 9, 1) ++ :lists.seq(10, 1010, 10) @@ -90,7 +91,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do suggested_price = case Enum.find(prediction_table, fn {_gas_price, hpa} -> hpa >= threshold_value end) do nil -> nil - {gas_price, _hpa} -> gas_price / 10 + {gas_price, _hpa} -> gas_price * @amount100mwei end {threshold_name, suggested_price} @@ -111,7 +112,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do # https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L46-L57 defp round_10gwei(price) do - case price / @wei_to_10gwei do + case price / @amount100mwei do gp when gp >= 1 and gp < 10 -> floor(gp) From ed1e1322993d59eeb860d8953873ba9d5f476de2 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Mon, 29 Jun 2020 18:39:02 +0700 Subject: [PATCH 24/37] refactor: move configurations to the supervisor --- .../lib/omg_child_chain/block_queue.ex | 4 +++ .../lib/omg_child_chain/block_queue/core.ex | 27 ++++++++++++++++--- .../lib/omg_child_chain/gas_price.ex | 20 +++----------- .../lib/omg_child_chain/gas_price/history.ex | 9 ++++--- .../strategy/block_percentile_gas_strategy.ex | 4 +-- .../strategy/poisson_gas_strategy.ex | 19 +++++++++---- .../poisson_gas_strategy/algorithm.ex | 5 ---- .../lib/omg_child_chain/sync_supervisor.ex | 6 ++++- .../omg_child_chain/gas_price/history_test.ex | 23 ++++++++++++++++ 9 files changed, 81 insertions(+), 36 deletions(-) create mode 100644 apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex index 708548f081..c3baf887fb 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue.ex @@ -103,6 +103,8 @@ defmodule OMG.ChildChain.BlockQueue do _ = Logger.info("Starting BlockQueue, top_mined_hash: #{inspect(Encoding.to_hex(top_mined_hash))}") block_submit_every_nth = Keyword.fetch!(args, :block_submit_every_nth) + block_submit_max_gas_price = Keyword.fetch!(args, :block_submit_max_gas_price) + block_submit_gas_price_strategy = Keyword.fetch!(args, :block_submit_gas_price_strategy) core = Core.new( @@ -112,6 +114,8 @@ defmodule OMG.ChildChain.BlockQueue do parent_height: parent_height, child_block_interval: child_block_interval, block_submit_every_nth: block_submit_every_nth, + block_submit_max_gas_price: block_submit_max_gas_price, + block_submit_gas_price_strategy: block_submit_gas_price_strategy, finality_threshold: finality_threshold ) diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index 11aac232d2..d43acb1e75 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -48,6 +48,7 @@ defmodule OMG.ChildChain.BlockQueue.Core do @zero_bytes32 <<0::size(256)>> @default_gas_price 20_000_000_000 + @gwei 1_000_000_000 defstruct [ :blocks, @@ -60,6 +61,8 @@ defmodule OMG.ChildChain.BlockQueue.Core do # config: child_block_interval: nil, block_submit_every_nth: 1, + block_submit_gas_price_strategy: nil, + block_submit_max_gas_price: @default_gas_price * 2, finality_threshold: 12 ] @@ -81,6 +84,10 @@ defmodule OMG.ChildChain.BlockQueue.Core do child_block_interval: pos_integer(), # configure to trigger forming a child chain block every this many Ethereum blocks are mined since enqueueing block_submit_every_nth: pos_integer(), + # the gas price strategy module to determine the gas price + block_submit_gas_price_strategy: module(), + # the maximum gas price to use + block_submit_max_gas_price: pos_integer(), # depth of max reorg we take into account finality_threshold: pos_integer() } @@ -144,10 +151,24 @@ defmodule OMG.ChildChain.BlockQueue.Core do :ok = GasPrice.recalculate_all(recalculate_params) + strategy = state.block_submit_gas_price_strategy + max_gas_price = state.block_submit_max_gas_price + gas_price = - case GasPrice.get_price() do - {:ok, price} -> price - {:error, _} -> @default_gas_price + case GasPrice.get_price(strategy) do + {:ok, price} when price > max_gas_price -> + _ = Logger.info("#{__MODULE__}: Gas price from #{strategy} exceeded max_gas_price: #{price / @gwei} gwei. " + <> "Lowering down to #{state.block_submit_max_gas_price / @gwei} gwei.") + state.block_submit_max_gas_price + + {:ok, price} -> + _ = Logger.info("#{__MODULE__}: Gas price from #{strategy} applied: #{price / @gwei} gwei.") + price + + {:error, :no_gas_price_history} = error -> + _ = Logger.info("#{__MODULE__}: Gas price from #{strategy} failed: #{inspect(error)}. " + <> "Using the existing price: #{state.gas_price / @gwei} gwei.") + state.gas_price end state = %{state | gas_price: gas_price} diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index 894c43960c..389073f3d3 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -37,7 +37,6 @@ defmodule OMG.ChildChain.GasPrice do Then, call `OMG.ChildChain.GasPrice.get_price()` to get the gas price suggestion. """ require Logger - alias OMG.ChildChain.Configuration alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy @@ -53,27 +52,16 @@ defmodule OMG.ChildChain.GasPrice do end @doc """ - Suggests the optimal gas price using the configured strategy. + Suggests the optimal gas price using the provided target strategy. """ - @spec get_price() :: {:ok, pos_integer()} | {:error, :no_gas_price_suggestion} - def get_price() do + @spec get_price(module()) :: {:ok, pos_integer()} | {:error, :no_gas_price_history} + def get_price(target_strategy) do price_suggestions = Enum.reduce(@strategies, %{}, fn strategy, suggestions -> Map.put(suggestions, strategy, strategy.get_price()) end) _ = Logger.info("#{__MODULE__}: All price suggestions: #{inspect(price_suggestions)}") - - strategy = Configuration.block_submit_gas_price_strategy() - - case price_suggestions[strategy] do - {:ok, gas_price} -> - _ = Logger.info("#{__MODULE__}: Suggesting gas price: #{gas_price / 1_000_000_000} gwei.") - {:ok, gas_price} - - error -> - _ = Logger.info("#{__MODULE__}: Gas price suggestion for #{strategy} cannot be determined. Got: #{error}.") - error - end + price_suggestions[target_strategy] end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex index 5ba587fca3..d47027f0bb 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex @@ -28,10 +28,10 @@ defmodule OMG.ChildChain.GasPrice.History do Supervisor.init([{History, []}], strategy: :one_for_one) """ - def child_spec(opts) do + def child_spec(start_arg) do %{ id: __MODULE__, - start: {__MODULE__, :start_link, [opts]}, + start: {__MODULE__, :start_link, [start_arg]}, type: :worker } end @@ -40,8 +40,9 @@ defmodule OMG.ChildChain.GasPrice.History do Start the gas price history service. """ @spec start_link(Keyword.t()) :: GenServer.on_start() - def start_link(opts) do - GenServer.start_link(Server, opts, name: Server) + def start_link(start_arg) do + {name, start_arg} = Keyword.pop(start_arg, :name, Server) + GenServer.start_link(Server, start_arg, name: name) end @doc """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 71b9ff0790..5c96b125b3 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -34,7 +34,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do | error() } - @type error() :: {:error, :all_empty_blocks} + @type error() :: {:error, :no_gas_price_history} defstruct prices: %{ safe_low: 20_000_000_000, @@ -137,7 +137,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do # Handles when all blocks are empty (possible on local chain and low traffic testnets) case length(sorted_min_prices) do 0 -> - {:error, :all_empty_blocks} + {:error, :no_gas_price_history} block_count -> Enum.map(@thresholds, fn {threshold_name, value} -> diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index 169c9a5315..4d055c7471 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -37,7 +37,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do | error() } - @type error() :: {:error, :all_empty_blocks} + @type error() :: {:error, :no_gas_price_history} defstruct prices: %{ safe_low: 20_000_000_000, @@ -134,11 +134,20 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do # Internal implementations # + # An equivalent of https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L213-L229 defp do_recalculate() do - price_history = History.all() + case exclude_empty(History.all()) do + [] -> + {:error, :no_gas_price_history} + + price_history -> + {hash_percentages, lowest_min_price, highest_min_price} = Algorithm.analyze_blocks(price_history) + prediction_table = Algorithm.make_prediction_table(hash_percentages, lowest_min_price, highest_min_price) + Algorithm.get_recommendations(@thresholds, prediction_table) + end + end - {hash_percentages, lowest_min_price, highest_min_price} = Algorithm.analyze_blocks(price_history) - prediction_table = Algorithm.make_prediction_table(hash_percentages, lowest_min_price, highest_min_price) - Algorithm.get_recommendations(@thresholds, prediction_table) + defp exclude_empty(history) do + Enum.reject(history, fn {_height, prices, _timestamp} -> Enum.empty?(prices) end) end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex index c223d904f5..4ec499d0ff 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex @@ -42,7 +42,6 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do # [min_price1, min_price2, min_price3, ...] sorted_min_prices = price_history - |> remove_empty() |> extract_min_prices() |> round_10gwei() |> Enum.sort() @@ -98,10 +97,6 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do end) end - defp remove_empty(history) do - Enum.reject(history, fn {_height, prices, _timestamp} -> Enum.empty?(prices) end) - end - defp extract_min_prices(history) do Enum.map(history, fn {_height, prices, _timestamp} -> Enum.min(prices) end) end diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex index e3f5ac7d8d..0033501c13 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex @@ -63,6 +63,8 @@ defmodule OMG.ChildChain.SyncSupervisor do block_queue_eth_height_check_interval_ms = Configuration.block_queue_eth_height_check_interval_ms() submission_finality_margin = Configuration.submission_finality_margin() block_submit_every_nth = Configuration.block_submit_every_nth() + block_submit_max_gas_price = Configuration.block_submit_max_gas_price() + block_submit_gas_price_strategy = Configuration.block_submit_gas_price_strategy() ethereum_events_check_interval_ms = OMG.Configuration.ethereum_events_check_interval_ms() coordinator_eth_height_check_interval_ms = OMG.Configuration.coordinator_eth_height_check_interval_ms() deposit_finality_margin = OMG.Configuration.deposit_finality_margin() @@ -80,7 +82,9 @@ defmodule OMG.ChildChain.SyncSupervisor do block_queue_eth_height_check_interval_ms: block_queue_eth_height_check_interval_ms, submission_finality_margin: submission_finality_margin, block_submit_every_nth: block_submit_every_nth, - child_block_interval: child_block_interval + block_submit_max_gas_price: block_submit_max_gas_price, + block_submit_gas_price_strategy: block_submit_gas_price_strategy, + child_block_interval: child_block_interval, ]}, {RootChainCoordinator, CoordinatorSetup.coordinator_setup( diff --git a/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex b/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex new file mode 100644 index 0000000000..9442899465 --- /dev/null +++ b/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex @@ -0,0 +1,23 @@ +# Copyright 2019-2020 OmiseGO Pte Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +defmodule OMG.ChildChain.GasPrice.HistoryTest do + use ExUnit.Case, async: true + alias OMG.ChildChain.GasPrice.History + + describe "start_link/1" do + test "starts the history server successfully" do + + end + end +end From 791ab228bee6f821782dd5a43714b9762e9cff4a Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Mon, 29 Jun 2020 18:40:26 +0700 Subject: [PATCH 25/37] fix: upgrade block submission debug logs to info --- .../lib/omg_child_chain/block_queue/block_submission.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex index 11444edd85..e4558522d8 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/block_submission.ex @@ -160,12 +160,12 @@ defmodule OMG.ChildChain.BlockQueue.BlockSubmission do end defp log_known_tx(submission) do - _ = Logger.debug("Submission #{inspect(submission)} is known transaction - ignored") + _ = Logger.info("Submission #{inspect(submission)} is known transaction - ignored") :ok end defp log_low_replacement_price(submission) do - _ = Logger.debug("Submission #{inspect(submission)} is known, but with higher price - ignored") + _ = Logger.info("Submission #{inspect(submission)} is known, but with higher price - ignored") :ok end From 44cc807016a44794e8ce40c3838fb8b0fb20aa5a Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Mon, 29 Jun 2020 18:44:30 +0700 Subject: [PATCH 26/37] style: mix format --- .../lib/omg_child_chain/block_queue/core.ex | 16 ++++++++++++---- .../lib/omg_child_chain/sync_supervisor.ex | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex index d43acb1e75..894dcd84c4 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/block_queue/core.ex @@ -157,8 +157,12 @@ defmodule OMG.ChildChain.BlockQueue.Core do gas_price = case GasPrice.get_price(strategy) do {:ok, price} when price > max_gas_price -> - _ = Logger.info("#{__MODULE__}: Gas price from #{strategy} exceeded max_gas_price: #{price / @gwei} gwei. " - <> "Lowering down to #{state.block_submit_max_gas_price / @gwei} gwei.") + _ = + Logger.info( + "#{__MODULE__}: Gas price from #{strategy} exceeded max_gas_price: #{price / @gwei} gwei. " <> + "Lowering down to #{state.block_submit_max_gas_price / @gwei} gwei." + ) + state.block_submit_max_gas_price {:ok, price} -> @@ -166,8 +170,12 @@ defmodule OMG.ChildChain.BlockQueue.Core do price {:error, :no_gas_price_history} = error -> - _ = Logger.info("#{__MODULE__}: Gas price from #{strategy} failed: #{inspect(error)}. " - <> "Using the existing price: #{state.gas_price / @gwei} gwei.") + _ = + Logger.info( + "#{__MODULE__}: Gas price from #{strategy} failed: #{inspect(error)}. " <> + "Using the existing price: #{state.gas_price / @gwei} gwei." + ) + state.gas_price end diff --git a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex index 0033501c13..665f6c2b37 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/sync_supervisor.ex @@ -84,7 +84,7 @@ defmodule OMG.ChildChain.SyncSupervisor do block_submit_every_nth: block_submit_every_nth, block_submit_max_gas_price: block_submit_max_gas_price, block_submit_gas_price_strategy: block_submit_gas_price_strategy, - child_block_interval: child_block_interval, + child_block_interval: child_block_interval ]}, {RootChainCoordinator, CoordinatorSetup.coordinator_setup( From cdae33bca2f9a11d782df296851e6e6c39f9a768 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Mon, 29 Jun 2020 18:49:37 +0700 Subject: [PATCH 27/37] fix: test modules should be .exs --- .../gas_price/{history_test.ex => history_test.exs} | 1 - 1 file changed, 1 deletion(-) rename apps/omg_child_chain/test/omg_child_chain/gas_price/{history_test.ex => history_test.exs} (99%) diff --git a/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex b/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.exs similarity index 99% rename from apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex rename to apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.exs index 9442899465..8e60e5ddde 100644 --- a/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.ex +++ b/apps/omg_child_chain/test/omg_child_chain/gas_price/history_test.exs @@ -17,7 +17,6 @@ defmodule OMG.ChildChain.GasPrice.HistoryTest do describe "start_link/1" do test "starts the history server successfully" do - end end end From 41b6eab3a27041fb8d9a7b678cd43fa4b58292f3 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Mon, 29 Jun 2020 18:49:52 +0700 Subject: [PATCH 28/37] style: fix credo complaining function is too nested --- .../strategy/poisson_gas_strategy/algorithm.ex | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex index 4ec499d0ff..69fd483feb 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy/algorithm.ex @@ -87,16 +87,18 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy.Algorithm do """ def get_recommendations(thresholds, prediction_table) do Enum.map(thresholds, fn {threshold_name, threshold_value} -> - suggested_price = - case Enum.find(prediction_table, fn {_gas_price, hpa} -> hpa >= threshold_value end) do - nil -> nil - {gas_price, _hpa} -> gas_price * @amount100mwei - end - - {threshold_name, suggested_price} + price = find_threshold_crossing(prediction_table, threshold_value) + {threshold_name, price} end) end + defp find_threshold_crossing(prediction_table, threshold_value) do + case Enum.find(prediction_table, fn {_gas_price, hpa} -> hpa >= threshold_value end) do + nil -> nil + {gas_price, _hpa} -> gas_price * @amount100mwei + end + end + defp extract_min_prices(history) do Enum.map(history, fn {_height, prices, _timestamp} -> Enum.min(prices) end) end From 5f5c132a5f40402a393a17d6dd8738994e485727 Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Tue, 30 Jun 2020 14:31:13 +0700 Subject: [PATCH 29/37] fix: block percentile not poisson --- .../gas_price/strategy/block_percentile_gas_strategy.ex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 5c96b125b3..90da14937f 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -55,7 +55,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @behaviour Strategy @doc """ - Starts the Poisson regression strategy. + Starts the block percentile strategy. """ @spec start_link(Keyword.t()) :: GenServer.on_start() def start_link(init_arg) do From 0401a8faf69d58a3606390304d12ff1c67f7ee9f Mon Sep 17 00:00:00 2001 From: Unnawut Leepaisalsuwanna Date: Tue, 30 Jun 2020 14:31:37 +0700 Subject: [PATCH 30/37] refactor: more readable multiplier conditions --- .../gas_price/strategy/legacy_gas_strategy.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex index ecef2380e0..8ac1647ef0 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/legacy_gas_strategy.ex @@ -210,24 +210,24 @@ defmodule OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy do lowering_factor, max_gas_price ) do - blocks_needs_be_mined? = blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) - new_blocks_mined? = new_blocks_mined?(mined_child_block_num, last_mined_child_block_num) + more_blocks_to_mine? = more_blocks_to_mine?(formed_child_block_num, mined_child_block_num) + new_blocks_recently_mined? = new_blocks_recently_mined?(mined_child_block_num, last_mined_child_block_num) multiplier = - case {blocks_needs_be_mined?, new_blocks_mined?} do + case {more_blocks_to_mine?, new_blocks_recently_mined?} do {false, _} -> 1.0 {true, false} -> raising_factor - {_, true} -> lowering_factor + {true, true} -> lowering_factor end Kernel.min(max_gas_price, Kernel.round(multiplier * gas_price)) end - defp blocks_needs_be_mined?(formed_child_block_num, mined_child_block_num) do + defp more_blocks_to_mine?(formed_child_block_num, mined_child_block_num) do formed_child_block_num > mined_child_block_num end - defp new_blocks_mined?(mined_child_block_num, last_mined_block_num) do + defp new_blocks_recently_mined?(mined_child_block_num, last_mined_block_num) do mined_child_block_num > last_mined_block_num end From e77ed9a256073a6e7a13d2ce7f6063b18f356cb2 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 21 Aug 2020 14:41:38 +0700 Subject: [PATCH 31/37] refactor: move max gas price and gas price strategy to releases.exs --- .../set_block_submit_gas_price_strategy.ex | 64 ------------------- .../set_block_submit_max_gas_price.ex | 52 --------------- .../set_block_submit_max_gas_price_test.exs | 44 ------------- config/releases.exs | 11 ++++ 4 files changed, 11 insertions(+), 160 deletions(-) delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex delete mode 100644 apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_max_gas_price.ex delete mode 100644 apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_max_gas_price_test.exs diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex deleted file mode 100644 index 9682d59c29..0000000000 --- a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_gas_price_strategy.ex +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2019-2019 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitGasPriceStrategy do - @moduledoc false - alias OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy - alias OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy - alias OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy - require Logger - - @behaviour Config.Provider - - @app :omg_child_chain - @config_key :block_submit_gas_price_strategy - @env_var_name "BLOCK_SUBMIT_GAS_PRICE_STRATEGY" - - def init(args) do - args - end - - def load(config, _args) do - _ = on_load() - strategy = strategy() - Config.Reader.merge(config, omg_child_chain: [block_submit_gas_price_strategy: strategy]) - end - - defp strategy() do - default = Application.get_env(@app, @config_key) - - strategy = - @env_var_name - |> System.get_env() - |> get_strategy(default) - - _ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@config_key} Value: #{inspect(strategy)}.") - - strategy - end - - defp get_strategy("LEGACY", _), do: LegacyGasStrategy - defp get_strategy("BLOCK_PERCENTILE", _), do: BlockPercentileGasStrategy - defp get_strategy("POISSON", _), do: PoissonGasStrategy - defp get_strategy(nil, default), do: default - - defp get_strategy(input, _) do - exit("#{@env_var_name} must be either LEGACY, BLOCK_PERCENTILE or POISSON. Got #{inspect(input)}.") - end - - defp on_load() do - _ = Application.ensure_all_started(:logger) - _ = Application.load(@app) - end -end diff --git a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_max_gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_max_gas_price.ex deleted file mode 100644 index c55a2225f0..0000000000 --- a/apps/omg_child_chain/lib/omg_child_chain/release_tasks/set_block_submit_max_gas_price.ex +++ /dev/null @@ -1,52 +0,0 @@ -# Copyright 2019-2019 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitMaxGasPrice do - @moduledoc false - @behaviour Config.Provider - require Logger - - @app :omg_child_chain - @config_key :block_submit_max_gas_price - @env_var_name "BLOCK_SUBMIT_MAX_GAS_PRICE" - - def init(args) do - args - end - - def load(config, _args) do - _ = on_load() - max_gas_price = max_gas_price() - Config.Reader.merge(config, omg_child_chain: [block_submit_max_gas_price: max_gas_price]) - end - - defp max_gas_price() do - max_gas_price = - @env_var_name - |> System.get_env() - |> validate_integer(Application.get_env(@app, @config_key)) - - _ = Logger.info("CONFIGURATION: App: #{@app} Key: #{@config_key} Value: #{inspect(max_gas_price)}.") - - max_gas_price - end - - defp validate_integer(value, _default) when is_binary(value), do: String.to_integer(value) - defp validate_integer(_, default), do: default - - defp on_load() do - _ = Application.ensure_all_started(:logger) - _ = Application.load(@app) - end -end diff --git a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_max_gas_price_test.exs b/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_max_gas_price_test.exs deleted file mode 100644 index 8c41ef0dee..0000000000 --- a/apps/omg_child_chain/test/omg_child_chain/release_tasks/set_block_submit_max_gas_price_test.exs +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2019-2020 OmiseGO Pte Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -defmodule OMG.ChildChain.ReleaseTasks.SetBlockSubmitMaxGasPriceTest do - use ExUnit.Case, async: true - alias OMG.ChildChain.ReleaseTasks.SetBlockSubmitMaxGasPrice - - @app :omg_child_chain - @config_key :block_submit_max_gas_price - @env_var_name "BLOCK_SUBMIT_MAX_GAS_PRICE" - - test "sets the block_submit_max_gas_price when the system's env var is present" do - :ok = System.put_env(@env_var_name, "1000000000") - config = SetBlockSubmitMaxGasPrice.load([], []) - configured_value = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key) - assert configured_value == 1_000_000_000 - :ok = System.delete_env(@env_var_name) - end - - test "uses the default app env value if not defined in system's env var" do - :ok = System.delete_env(@env_var_name) - default_value = Application.get_env(@app, @config_key) - config = SetBlockSubmitMaxGasPrice.load([], []) - configured_value = config |> Keyword.fetch!(@app) |> Keyword.fetch!(@config_key) - assert configured_value == default_value - end - - test "fails if the system's env value is not a valid stringified integer" do - :ok = System.put_env(@env_var_name, "invalid") - assert catch_error(SetBlockSubmitMaxGasPrice.load([], [])) == :badarg - :ok = System.delete_env(@env_var_name) - end -end diff --git a/config/releases.exs b/config/releases.exs index 412baea707..8462eaf8be 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -5,6 +5,17 @@ import Config # # See https://hexdocs.pm/mix/1.9.0/Mix.Tasks.Release.html#module-runtime-configuration +config :omg_child_chain, + block_submit_max_gas_price: String.to_integer(System.get_env("BLOCK_SUBMIT_MAX_GAS_PRICE") || "20000000000"), + block_submit_gas_price_strategy: + case System.get_env("BLOCK_SUBMIT_GAS_PRICE_STRATEGY") do + "POISSON" -> OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy + "BLOCK_PERCENTILE" -> OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy + "LEGACY" -> OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy + nil -> OMG.ChildChain.GasPrice.Strategy.LegacyGasStrategy + invalid -> raise("Invalid gas strategy. Got: #{invalid}") + end + config :omg_watcher_info, OMG.WatcherInfo.DB.Repo, # Have at most `:pool_size` DB connections on standby and serving DB queries. pool_size: String.to_integer(System.get_env("WATCHER_INFO_DB_POOL_SIZE") || "10"), From 13b089814b6128f4313f1755f5952bb134d57426 Mon Sep 17 00:00:00 2001 From: unnawut Date: Fri, 21 Aug 2020 18:37:26 +0700 Subject: [PATCH 32/37] refactor: clarify gas strategy calculation --- .../strategy/block_percentile_gas_strategy.ex | 27 ++++++++++++++++--- .../strategy/poisson_gas_strategy.ex | 4 +-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 90da14937f..892d10a6a7 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -43,6 +43,13 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do fastest: 20_000_000_000 } + @typep thresholds() :: %{ + safe_low: non_neg_integer(), + standard: non_neg_integer(), + fast: non_neg_integer(), + fastest: non_neg_integer() + } + @thresholds %{ safe_low: 35, standard: 60, @@ -52,6 +59,13 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @target_threshold :fast + @typep recommendations() :: %{ + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } + @behaviour Strategy @doc """ @@ -121,7 +135,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc false @impl GenServer def handle_info({History, :updated}, state) do - prices = do_recalculate() + prices = calculate(History.all(), @thresholds) _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") {:noreply, %{state | prices: prices}} @@ -131,8 +145,13 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do # Internal implementations # - defp do_recalculate() do - sorted_min_prices = History.all() |> filter_min() |> Enum.sort() + # Returns the recommended gas prices for each of the provided `@thresholds`. It does the following: + # 1. For each historical block, take the minimum gas price accepted by the block + # 2. Sort the minimum gas prices from lowest to highest prices + # 3. Extract the gas prices at the given thresholds (in other word, the percentile) + @spec calculate(History.t(), thresholds()) :: recommendations() | {:error, :no_gas_price_history} + defp calculate(history, thresholds) do + sorted_min_prices = history |> filter_min() |> Enum.sort() # Handles when all blocks are empty (possible on local chain and low traffic testnets) case length(sorted_min_prices) do @@ -140,7 +159,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do {:error, :no_gas_price_history} block_count -> - Enum.map(@thresholds, fn {threshold_name, value} -> + Enum.map(thresholds, fn {threshold_name, value} -> position = floor(block_count * value / 100) - 1 {threshold_name, Enum.at(sorted_min_prices, position)} end) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index 4d055c7471..6942a1606c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -124,7 +124,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc false @impl GenServer def handle_info({History, :updated}, state) do - prices = do_recalculate() + prices = calculate() _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") {:noreply, %{state | prices: prices}} @@ -135,7 +135,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do # # An equivalent of https://github.com/ethgasstation/gasstation-express-oracle/blob/3cfb354/gasExpress.py#L213-L229 - defp do_recalculate() do + defp calculate() do case exclude_empty(History.all()) do [] -> {:error, :no_gas_price_history} From 07cb6292766fb5fd21006e289f92bfc7d65db689 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 25 Aug 2020 18:40:13 +0700 Subject: [PATCH 33/37] docs: more gas calculation edge case explanation --- .../gas_price/strategy/block_percentile_gas_strategy.ex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index 892d10a6a7..daf68babbf 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -149,6 +149,11 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do # 1. For each historical block, take the minimum gas price accepted by the block # 2. Sort the minimum gas prices from lowest to highest prices # 3. Extract the gas prices at the given thresholds (in other word, the percentile) + # + # Edge case behaviours: + # 1. No historical gas prices available. Returns `{:error, :no_gas_price_history}`. + # Prices continue to be fetched and recalculated on the same regular basis. + # 2. Too few historical gas prices. Calculation will be done on as much as the price data is available. @spec calculate(History.t(), thresholds()) :: recommendations() | {:error, :no_gas_price_history} defp calculate(history, thresholds) do sorted_min_prices = history |> filter_min() |> Enum.sort() From c08c9b3a802b18c51929ca4c557007c91aac2b43 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 1 Sep 2020 14:43:14 +0700 Subject: [PATCH 34/37] feat: allow configuring ethereum url to gas price logic --- .../lib/omg_child_chain/gas_price.ex | 2 +- .../gas_price/gas_price_supervisor.ex | 3 ++- .../omg_child_chain/gas_price/history/fetcher.ex | 16 ++++++++-------- .../omg_child_chain/gas_price/history/server.ex | 8 ++++++-- .../lib/omg_child_chain/supervisor.ex | 3 ++- apps/omg_eth/lib/omg_eth/configuration.ex | 7 +++++++ mix.lock | 4 ++-- 7 files changed, 28 insertions(+), 15 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex index 389073f3d3..a9399d2d51 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price.ex @@ -29,7 +29,7 @@ defmodule OMG.ChildChain.GasPrice do Include `OMG.ChildChain.GasPrice.GasPriceSupervisor` in the supervision tree: children = [ - {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price]} + {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price, ethereum_url: ethereum_url]} ] Supervisor.init(children, strategy: :one_for_one) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex index 5ecb3bc3b8..9b2d7b5133 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/gas_price_supervisor.ex @@ -32,9 +32,10 @@ defmodule OMG.ChildChain.GasPrice.GasPriceSupervisor do def init(init_arg) do num_blocks = Keyword.fetch!(init_arg, :num_blocks) max_gas_price = Keyword.fetch!(init_arg, :max_gas_price) + ethereum_url = Keyword.fetch!(init_arg, :ethereum_url) children = [ - {History, [event_bus: Bus, num_blocks: num_blocks]}, + {History, [event_bus: Bus, num_blocks: num_blocks, ethereum_url: ethereum_url]}, # Unfortunately LegacyGasStrategy cannot rely on its caller to enforce `max_gas_price` # because the strategy can keep doubling the price through the roof and take forever # to get back down below `max_gas_price` without its own ceiling. diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex index 6bdd541d3c..f1dc86f95d 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/fetcher.ex @@ -34,8 +34,8 @@ defmodule OMG.ChildChain.GasPrice.History.Fetcher do Internally this function fetches the input `heights` in batches of #{@per_batch} blocks. """ - @spec stream(Range.t()) :: Enumerable.t() - def stream(heights) do + @spec stream(Range.t(), String.t()) :: Enumerable.t() + def stream(heights, ethereum_url) do heights |> Stream.chunk_every(@per_batch) |> Stream.flat_map(fn heights -> @@ -44,21 +44,21 @@ defmodule OMG.ChildChain.GasPrice.History.Fetcher do {:ok, results} = heights |> Enum.map(fn height -> {:eth_get_block_by_number, [Encoding.to_hex(height), true]} end) - |> batch_request() + |> batch_request(ethereum_url) results end) end - defp batch_request(requests, retries \\ @retries) + defp batch_request(requests, ethereum_url, retries \\ @retries) - defp batch_request(requests, 1) do + defp batch_request(requests, ethereum_url, 1) do _ = Logger.warn("Last attempt to batch request: #{inspect(requests)}") - HttpClient.batch_request(requests) + HttpClient.batch_request(requests, url: ethereum_url) end - defp batch_request(requests, retries) do - case HttpClient.batch_request(requests) do + defp batch_request(requests, ethereum_url, retries) do + case HttpClient.batch_request(requests, url: ethereum_url) do {:error, _} = response -> _ = Logger.warn(""" diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index 1b125eb498..6806798140 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -43,6 +43,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do earliest_stored_height: non_neg_integer(), latest_stored_height: non_neg_integer(), history_ets: :ets.tid(), + ethereum_url: String.t(), subscribers: [pid()] } @@ -50,6 +51,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do earliest_stored_height: 0, latest_stored_height: 0, history_ets: nil, + ethereum_url: nil, subscribers: [] @doc false @@ -68,6 +70,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do def init(opts) do num_blocks = Keyword.fetch!(opts, :num_blocks) event_bus = Keyword.fetch!(opts, :event_bus) + ethereum_url = Keyword.fetch!(opts, :ethereum_url) :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) # The ets table is not initialized with `:read_concurrency` because we are expecting interleaving @@ -76,7 +79,8 @@ defmodule OMG.ChildChain.GasPrice.History.Server do state = %__MODULE__{ num_blocks: num_blocks, - history_ets: history_ets + history_ets: history_ets, + ethereum_url: ethereum_url } _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") @@ -125,7 +129,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do # Fetch and insert new heights, leaving obsolete heights intact. :ok = fetch_from_height..to_height - |> Fetcher.stream() + |> Fetcher.stream(state.ethereum_url) |> stream_insert(state.history_ets) |> Stream.run() diff --git a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex index e087a21246..1884112160 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex @@ -49,6 +49,7 @@ defmodule OMG.ChildChain.Supervisor do metrics_collection_interval = Configuration.metrics_collection_interval() gas_price_history_blocks = Configuration.block_submit_gas_price_history_blocks() max_gas_price = Configuration.block_submit_max_gas_price() + ethereum_url = OMG.Eth.Configuration.node_url() fee_server_opts = Configuration.fee_server_opts() fee_claimer_address = OMG.Configuration.fee_claimer_address() child_block_interval = OMG.Eth.Configuration.child_block_interval() @@ -72,7 +73,7 @@ defmodule OMG.ChildChain.Supervisor do type: :supervisor } ]}, - {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price]} + {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price, ethereum_url: ethereum_url]} ] is_datadog_disabled = is_disabled?() diff --git a/apps/omg_eth/lib/omg_eth/configuration.ex b/apps/omg_eth/lib/omg_eth/configuration.ex index 84c30d5a95..bac1d3db5a 100644 --- a/apps/omg_eth/lib/omg_eth/configuration.ex +++ b/apps/omg_eth/lib/omg_eth/configuration.ex @@ -66,6 +66,13 @@ defmodule OMG.Eth.Configuration do Application.fetch_env!(@app, :eth_node) end + @spec node_url() :: String.t() | no_return + def node_url() do + # Intentionally fetching from `:ethereumex` app. This config can be moved into @app context once all + # `Ethereumex.HttpClient.*` calls are passed a url instead of relying on `ethereumex.url` global config. + Application.fetch_env!(:ethereumex, :url) + end + @spec ethereum_events_check_interval_ms() :: pos_integer | no_return def ethereum_events_check_interval_ms() do Application.fetch_env!(@app, :ethereum_events_check_interval_ms) diff --git a/mix.lock b/mix.lock index ad608dad70..7d18382f24 100644 --- a/mix.lock +++ b/mix.lock @@ -17,7 +17,7 @@ "ecto_sql": {:hex, :ecto_sql, "3.4.4", "d28bac2d420f708993baed522054870086fd45016a9d09bb2cd521b9c48d32ea", [:mix], [{:db_connection, "~> 2.2", [hex: :db_connection, repo: "hexpm", optional: false]}, {:ecto, "~> 3.4.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:myxql, "~> 0.3.0 or ~> 0.4.0", [hex: :myxql, repo: "hexpm", optional: true]}, {:postgrex, "~> 0.15.0", [hex: :postgrex, repo: "hexpm", optional: true]}, {:tds, "~> 2.1.0", [hex: :tds, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "edb49af715dd72f213b66adfd0f668a43c17ed510b5d9ac7528569b23af57fe8"}, "erlex": {:hex, :erlex, "0.2.6", "c7987d15e899c7a2f34f5420d2a2ea0d659682c06ac607572df55a43753aa12e", [:mix], [], "hexpm", "2ed2e25711feb44d52b17d2780eabf998452f6efda104877a3881c2f8c0c0c75"}, "erlexec": {:hex, :erlexec, "1.10.9", "3cbb3476f942bfb8b68b85721c21c1835061cf6dd35f5285c2362e85b100ddc7", [:rebar3], [], "hexpm", "271e5b5f2d91cdb9887efe74d89026c199bfc69f074cade0d08dab60993fa14e"}, - "ethereumex": {:hex, :ethereumex, "0.6.2", "310954c27e7f7b0397795525f7eaae60169fbe5082a5935bde4a65a121fad8e7", [:mix], [{:httpoison, "~> 1.6", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1", [hex: :poolboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "76d2c65cbbdac3abf13cbc9615767c90678a8d56643cd34d6ce114782769e57c"}, + "ethereumex": {:hex, :ethereumex, "0.6.4", "58e998acb13b45a2b76b954b1d503f2f5f0e8118c0a14769c59264ef3ee4c301", [:mix], [{:httpoison, "~> 1.6", [hex: :httpoison, repo: "hexpm", optional: false]}, {:jason, "~> 1.2", [hex: :jason, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5.1", [hex: :poolboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "abc0bed1ba691645700f55bc843be7d08a23284e20ad889cabe6279e13debb32"}, "ex_abi": {:hex, :ex_abi, "0.4.0", "ff7e7f5b56c228b117e1f54e80c668a8f0424c275f233a50373548b70d99bd5c", [:mix], [{:exth_crypto, "~> 0.1.6", [hex: :exth_crypto, repo: "hexpm", optional: false]}], "hexpm", "2d33499de38c54531103e58530d0453863fb6149106327f691001873b0556e68"}, "ex_doc": {:hex, :ex_doc, "0.20.2", "1bd0dfb0304bade58beb77f20f21ee3558cc3c753743ae0ddbb0fd7ba2912331", [:mix], [{:earmark, "~> 1.3", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "8e24fc8ff9a50b9f557ff020d6c91a03cded7e59ac3e0eec8a27e771430c7d27"}, "ex_machina": {:hex, :ex_machina, "2.4.0", "09a34c5d371bfb5f78399029194a8ff67aff340ebe8ba19040181af35315eabb", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm", "a20bc9ddc721b33ea913b93666c5d0bdca5cbad7a67540784ae277228832d72c"}, @@ -29,7 +29,7 @@ "exth_crypto": {:hex, :exth_crypto, "0.1.6", "8e636a9bcb75d8e32451be96e547a495121ed2178d078db294edb0f81f7cf2e8", [:mix], [{:binary, "~> 0.0.4", [hex: :binary, repo: "hexpm", optional: false]}, {:keccakf1600, "~> 2.0.0", [hex: :keccakf1600_orig, repo: "hexpm", optional: false]}, {:libsecp256k1, "~> 0.1.9", [hex: :libsecp256k1, repo: "hexpm", optional: false]}], "hexpm", "45d6faf4b889f8fc526deba059e0c7947423784ab1e7fa85be8db4c46cf4416b"}, "fake_server": {:hex, :fake_server, "2.1.0", "aefed08a587e2498fdb39ac9de6f9eabbe7bd83da9801d08d3574d61b7eb03d5", [:mix], [{:cowboy, "~> 2.5", [hex: :cowboy, repo: "hexpm", optional: false]}, {:poison, ">= 1.0.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm", "3200d57a523b27d2c8ebfc1a80b76697b3c8a06bf9d678d82114f5f98d350c75"}, "hackney": {:hex, :hackney, "1.16.0", "5096ac8e823e3a441477b2d187e30dd3fff1a82991a806b2003845ce72ce2d84", [:rebar3], [{:certifi, "2.5.2", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.1", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.0", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.6", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "3bf0bebbd5d3092a3543b783bf065165fa5d3ad4b899b836810e513064134e18"}, - "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, + "httpoison": {:hex, :httpoison, "1.7.0", "abba7d086233c2d8574726227b6c2c4f6e53c4deae7fe5f6de531162ce9929a0", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "975cc87c845a103d3d1ea1ccfd68a2700c211a434d8428b10c323dc95dc5b980"}, "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, "ink": {:hex, :ink, "1.1.1", "0fea8b923a2f9fe27f7a8ab77dc389b3cb12f860d80e680805be1ed653066cff", [:mix], [{:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "bba3c21115fa2bc13a6ab3d602f9859020ca19e6b0b58fff384fb1bd273b3ac9"}, "jason": {:hex, :jason, "1.2.1", "12b22825e22f468c02eb3e4b9985f3d0cb8dc40b9bd704730efa11abd2708c44", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b659b8571deedf60f79c5a608e15414085fa141344e2716fbd6988a084b5f993"}, From fb580e7ca6d325003bd06ff5bdecd00b60d05534 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 1 Sep 2020 15:24:07 +0700 Subject: [PATCH 35/37] refactor: use :ets.select_delete/2 to prune gas price history --- .../lib/omg_child_chain/gas_price/history/server.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index 6806798140..97ac276fec 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -159,6 +159,8 @@ defmodule OMG.ChildChain.GasPrice.History.Server do end defp prune_heights(history_ets, from, to) do - Enum.each(from..to, fn height -> :ets.delete(history_ets, height) end) + :ets.select_delete(history_ets, :ets.fun2ms(fn + {height, _prices, _timestamp} when height >= from and height <= to -> true + end)) end end From cc35c6f6fbabe3fc2725025be8ae090b61490180 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 1 Sep 2020 17:40:02 +0700 Subject: [PATCH 36/37] refactor: replace custom pubsub with OMG.Bus --- .../lib/omg_child_chain/gas_price/history.ex | 16 ------------- .../gas_price/history/server.ex | 23 ++++++++----------- .../strategy/block_percentile_gas_strategy.ex | 7 +++--- .../strategy/poisson_gas_strategy.ex | 7 +++--- 4 files changed, 18 insertions(+), 35 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex index d47027f0bb..dd68a2d731 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history.ex @@ -45,22 +45,6 @@ defmodule OMG.ChildChain.GasPrice.History do GenServer.start_link(Server, start_arg, name: name) end - @doc """ - Subscribes a process to gas price history changes. - - On each history change, the subscriber can detect the change by handling `{History, :updated}` messages. - - ## Examples - - def handle_info({History, :updated}, state) do - # Do your operations on history update here - end - """ - @spec subscribe(function()) :: :ok - def subscribe(pid) do - GenServer.cast(Server, {:subscribe, pid}) - end - @doc """ Get all existing gas price records. """ diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index 97ac276fec..b98b745a1f 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -44,7 +44,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do latest_stored_height: non_neg_integer(), history_ets: :ets.tid(), ethereum_url: String.t(), - subscribers: [pid()] + event_bus: module() } defstruct num_blocks: 200, @@ -52,7 +52,7 @@ defmodule OMG.ChildChain.GasPrice.History.Server do latest_stored_height: 0, history_ets: nil, ethereum_url: nil, - subscribers: [] + event_bus: nil @doc false @spec all() :: History.t() @@ -69,8 +69,8 @@ defmodule OMG.ChildChain.GasPrice.History.Server do @impl GenServer def init(opts) do num_blocks = Keyword.fetch!(opts, :num_blocks) - event_bus = Keyword.fetch!(opts, :event_bus) ethereum_url = Keyword.fetch!(opts, :ethereum_url) + event_bus = Keyword.fetch!(opts, :event_bus) :ok = event_bus.subscribe({:root_chain, "ethereum_new_height"}, link: true) # The ets table is not initialized with `:read_concurrency` because we are expecting interleaving @@ -80,7 +80,8 @@ defmodule OMG.ChildChain.GasPrice.History.Server do state = %__MODULE__{ num_blocks: num_blocks, history_ets: history_ets, - ethereum_url: ethereum_url + ethereum_url: ethereum_url, + event_bus: event_bus } _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") @@ -112,13 +113,6 @@ defmodule OMG.ChildChain.GasPrice.History.Server do {:noreply, %{state | earliest_stored_height: earliest_height, latest_stored_height: latest_height}} end - @doc false - @impl GenServer - def handle_cast({:subscribe, subscriber}, state) do - subscribers = Enum.uniq([subscriber | state.subscribers]) - {:noreply, %{state | subscribers: subscribers}} - end - # # Internal implementations # @@ -143,8 +137,11 @@ defmodule OMG.ChildChain.GasPrice.History.Server do _ = Logger.info("#{__MODULE__} available gas prices from Eth heights: #{from_height} - #{to_height}.") - # Inform all subscribers that the history has been updated. - _ = Enum.each(state.subscribers, fn subscriber -> send(subscriber, {History, :updated}) end) + # Publish `:history_updated` event through the OMG.Bus + {:child_chain, "gas_price_history"} + |> state.event_bus.new(:history_updated, to_height) + |> state.event_bus.direct_local_broadcast() + {:ok, from_height, to_height} end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index daf68babbf..d3f0add76c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -105,8 +105,9 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc false @impl GenServer - def init(_init_arg) do - _ = History.subscribe(self()) + def init(init_arg) do + event_bus = Keyword.fetch!(init_arg, :event_bus) + :ok = event_bus.subscribe({:child_chain, "gas_price_history"}, link: true) state = %__MODULE__{} _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") @@ -134,7 +135,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @doc false @impl GenServer - def handle_info({History, :updated}, state) do + def handle_info({:internal_event_bus, :history_updated, _height}, state) do prices = calculate(History.all(), @thresholds) _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex index 6942a1606c..4420f3954a 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/poisson_gas_strategy.ex @@ -94,8 +94,9 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc false @impl GenServer - def init(_init_arg) do - _ = History.subscribe(self()) + def init(init_arg) do + event_bus = Keyword.fetch!(init_arg, :event_bus) + :ok = event_bus.subscribe({:child_chain, "gas_price_history"}, link: true) state = %__MODULE__{} _ = Logger.info("Started #{__MODULE__}: #{inspect(state)}") @@ -123,7 +124,7 @@ defmodule OMG.ChildChain.GasPrice.Strategy.PoissonGasStrategy do @doc false @impl GenServer - def handle_info({History, :updated}, state) do + def handle_info({:internal_event_bus, :history_updated, _height}, state) do prices = calculate() _ = Logger.info("#{__MODULE__}: History updated. Prices recalculated to: #{inspect(prices)}") From 25e4839417512cc0aed38f561e942316d0436368 Mon Sep 17 00:00:00 2001 From: unnawut Date: Tue, 1 Sep 2020 17:41:28 +0700 Subject: [PATCH 37/37] style: mix format --- .../gas_price/history/server.ex | 9 ++++++--- .../strategy/block_percentile_gas_strategy.ex | 20 +++++++++---------- .../lib/omg_child_chain/supervisor.ex | 3 ++- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex index b98b745a1f..f035b6fb7d 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/history/server.ex @@ -156,8 +156,11 @@ defmodule OMG.ChildChain.GasPrice.History.Server do end defp prune_heights(history_ets, from, to) do - :ets.select_delete(history_ets, :ets.fun2ms(fn - {height, _prices, _timestamp} when height >= from and height <= to -> true - end)) + :ets.select_delete( + history_ets, + :ets.fun2ms(fn + {height, _prices, _timestamp} when height >= from and height <= to -> true + end) + ) end end diff --git a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex index d3f0add76c..a94f6bea27 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/gas_price/strategy/block_percentile_gas_strategy.ex @@ -44,11 +44,11 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do } @typep thresholds() :: %{ - safe_low: non_neg_integer(), - standard: non_neg_integer(), - fast: non_neg_integer(), - fastest: non_neg_integer() - } + safe_low: non_neg_integer(), + standard: non_neg_integer(), + fast: non_neg_integer(), + fastest: non_neg_integer() + } @thresholds %{ safe_low: 35, @@ -60,11 +60,11 @@ defmodule OMG.ChildChain.GasPrice.Strategy.BlockPercentileGasStrategy do @target_threshold :fast @typep recommendations() :: %{ - safe_low: float(), - standard: float(), - fast: float(), - fastest: float() - } + safe_low: float(), + standard: float(), + fast: float(), + fastest: float() + } @behaviour Strategy diff --git a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex index 1884112160..2a650c3d9c 100644 --- a/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex +++ b/apps/omg_child_chain/lib/omg_child_chain/supervisor.ex @@ -73,7 +73,8 @@ defmodule OMG.ChildChain.Supervisor do type: :supervisor } ]}, - {GasPriceSupervisor, [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price, ethereum_url: ethereum_url]} + {GasPriceSupervisor, + [num_blocks: gas_price_history_blocks, max_gas_price: max_gas_price, ethereum_url: ethereum_url]} ] is_datadog_disabled = is_disabled?()