From 40740d651af449ccc898f182b877288f66ec1f28 Mon Sep 17 00:00:00 2001 From: Tadej Date: Mon, 29 May 2023 00:19:28 +0200 Subject: [PATCH] feat: add pagination support for blockscout --- config/config.exs | 3 +++ lib/adapters/api/blockscout.ex | 33 +++++++++++++++++++------- lib/address.ex | 6 +++-- lib/ethcule_poirot/address_explorer.ex | 28 +++++++++++++++++----- lib/ethcule_poirot/network_explorer.ex | 1 + 5 files changed, 55 insertions(+), 16 deletions(-) diff --git a/config/config.exs b/config/config.exs index 9f2c3c9..5bdff6f 100644 --- a/config/config.exs +++ b/config/config.exs @@ -18,3 +18,6 @@ config :ethcule_poirot, Adapters.Api.Blockscout, config :ethcule_poirot, Adapters.Api.DissrupTheGraph, api_url: System.get_env("DISSRUP_THE_GRAPH_API_URL"), api_timeout: 120_000 + +config :logger, :console, + level: :info diff --git a/lib/adapters/api/blockscout.ex b/lib/adapters/api/blockscout.ex index 7d96149..d00ba53 100644 --- a/lib/adapters/api/blockscout.ex +++ b/lib/adapters/api/blockscout.ex @@ -35,13 +35,15 @@ defmodule Adapters.Api.Blockscout do end @impl true - def transactions_for_address(eth_address) do + def transactions_for_address(eth_address, search_after) do + search_first = 15 + Logger.debug("Query with: {\"eth_address\": \"#{eth_address}\", \"search_first\": #{search_first}, \"search_after\": \"#{search_after}\"}") Neuron.query( """ - query($eth_address: AddressHash!) { + query($eth_address: AddressHash!, $search_first: Int, $search_after: String) { address(hash: $eth_address) { contractCode - transactions(last: 23, count: 23) { + transactions(first: $search_first, after: $search_after) { edges { node { hash @@ -51,11 +53,15 @@ defmodule Adapters.Api.Blockscout do status } } + pageInfo { + hasNextPage + endCursor + } } } } """, - %{eth_address: eth_address} + %{eth_address: eth_address, search_first: search_first, search_after: search_after} ) |> parse_transactions(eth_address) end @@ -84,6 +90,8 @@ defmodule Adapters.Api.Blockscout do defp parse_transactions({:ok, %Neuron.Response{body: body}}, eth_address) do transactions = body["data"]["address"]["transactions"]["edges"] || [] contract_code = !!body["data"]["address"]["contractCode"] + end_cursor = body["data"]["address"]["transactions"]["pageInfo"]["endCursor"] + has_next_page = body["data"]["address"]["transactions"]["pageInfo"]["hasNextPage"] transactions_struct = Enum.map( @@ -100,18 +108,27 @@ defmodule Adapters.Api.Blockscout do %Address{ eth_address: eth_address, contract: contract_code, - transactions: transactions_struct + transactions: transactions_struct, + end_cursor: end_cursor, + has_next_page: has_next_page } end @spec parse_transactions({:error, any()}, String.t()) :: Address.t() - defp parse_transactions(_request_result, eth_address) do - Logger.warn("Failed ETH transactions for #{eth_address}") + defp parse_transactions(request_result, eth_address) do + case request_result do + {:error, error} -> + Logger.warn("Failed ETH transactions for #{eth_address}: #{inspect(error)}") + + _ -> + Logger.debug("Successful ETH transactions for #{eth_address}") + end %Address{ eth_address: eth_address, contract: false, - transactions: [] + transactions: [], + has_next_page: false } end end diff --git a/lib/address.ex b/lib/address.ex index 95dcd60..e4883dc 100644 --- a/lib/address.ex +++ b/lib/address.ex @@ -1,11 +1,13 @@ defmodule Address do @moduledoc false - defstruct [:eth_address, :contract, :transactions] + defstruct [:eth_address, :contract, :transactions, :end_cursor, :has_next_page] @type t :: %__MODULE__{ eth_address: String.t() | nil, contract: boolean() | nil, - transactions: list(Transaction.t()) | nil + transactions: list(Transaction.t()) | nil, + end_cursor: String.t() | nil, + has_next_page: boolean() | nil, } end diff --git a/lib/ethcule_poirot/address_explorer.ex b/lib/ethcule_poirot/address_explorer.ex index 3210690..ed59522 100644 --- a/lib/ethcule_poirot/address_explorer.ex +++ b/lib/ethcule_poirot/address_explorer.ex @@ -36,19 +36,34 @@ defmodule EthculePoirot.AddressExplorer do end @impl true - def handle_info(:start, %{depth: depth} = state) do + def handle_info(:start, %{depth: depth} = state) do + search_after = nil + handle_info_loop(state, depth, search_after) + NetworkExplorer.node_visited(state.eth_address) + {:stop, :normal, state} + end + + defp handle_info_loop(state, depth, search_after) do state.eth_address - |> state.api_handler.transactions_for_address() + |> state.api_handler.transactions_for_address(search_after) |> handle_transactions(depth) + |> handle_next_page(state, depth, search_after) + end - NetworkExplorer.node_visited(state.eth_address) - - {:stop, :normal, state} + @spec handle_next_page(Address.t(), any(), pos_integer(), String.t()) :: any() + defp handle_next_page(address_info, state, depth, search_after) do + if address_info.has_next_page do + state + |> handle_info_loop(depth, address_info.end_cursor) + else + Logger.debug("Last page for #{address_info.eth_address} reached (after #{search_after})") + end end - @spec handle_transactions(Address.t(), pos_integer()) :: any() + @spec handle_transactions(Address.t(), pos_integer()) :: Address.t() defp handle_transactions(%{transactions: []} = address_info, _depth) do update_node_label(address_info.contract, address_info.eth_address) + address_info end defp handle_transactions(address_info, depth) do @@ -61,6 +76,7 @@ defmodule EthculePoirot.AddressExplorer do NetworkExplorer.visit_node(next_address, depth - 1) end) + address_info end @spec update_node_label(true, String.t()) :: any() diff --git a/lib/ethcule_poirot/network_explorer.ex b/lib/ethcule_poirot/network_explorer.ex index 8a15684..eb51336 100644 --- a/lib/ethcule_poirot/network_explorer.ex +++ b/lib/ethcule_poirot/network_explorer.ex @@ -49,6 +49,7 @@ defmodule EthculePoirot.NetworkExplorer do Logger.info("Already explored #{eth_address}") {:noreply, state} else + Logger.debug("Queued #{eth_address}") new_known = MapSet.put(state.known, eth_address) {processes_count, new_remaining, new_exploring} =