Merge pull request #1 from NethermindEth/feat/add_erc20_tokens

Adding ERC20 Transfer Dataclasses & Starknet Event to Transfer Parsing

nethermind/idealis/logging.py

@@ -81,7 +81,7 @@ def __init__(
         self.logger_labels = logger_labels or {}
         self.write_delay = write_delay
 
-        self._buffer = queue.Queue()
+        self._buffer: queue.Queue = queue.Queue()
         self._request_session = requests.session()
         self._flush_thread = threading.Thread(target=self._flush, daemon=True)
 

nethermind/idealis/parse/ethereum/trace.py

@@ -18,11 +18,11 @@ def unpack_debug_trace_block_response(
     return_create_traces = []
     return_events = []
 
-    for tx_index, tx_trace_dict in enumerate(block_traces):
+    for transaction_index, tx_trace_dict in enumerate(block_traces):
         call_traces, create_traces, events = unpack_debug_trace_transaction_response(
             trace_dict=tx_trace_dict,
             block_number=block_number,
-            transaction_index=tx_index,
+            transaction_index=transaction_index,
         )
         return_call_traces += call_traces
         return_create_traces += create_traces

nethermind/idealis/parse/shared/erc_20_tokens.py

@@ -0,0 +1,139 @@
+import warnings
+
+from nethermind.idealis.types.base import ERC20Transfer
+from nethermind.idealis.utils import to_bytes
+
+NULL_ADDRESS = [
+    to_bytes("0x00", pad=32),
+    to_bytes("0x00", pad=20),
+]
+
+
+def _apply_credits(
+    balance_state: dict[bytes, int],
+    transfer: ERC20Transfer,
+):
+    warnings.warn(
+        "This function is un-tested & may not work as expected",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+
+    if transfer.to_address in NULL_ADDRESS:
+        if balance_state[b"total_supply"] < transfer.value:
+            raise ValueError(f"Cannot Burn more tokens than the total supply")
+
+        balance_state[b"total_supply"] -= transfer.value
+
+    elif transfer.to_address not in balance_state:
+        balance_state.update({transfer.to_address: transfer.value})
+    else:
+        balance_state[transfer.to_address] += transfer.value
+
+
+def _apply_debits(
+    balance_state: dict[bytes, int],
+    transfer: ERC20Transfer,
+):
+    warnings.warn("This function is un-tested & may not work as expected", DeprecationWarning, stacklevel=2)
+
+    if transfer.from_address not in NULL_ADDRESS and transfer.from_address not in balance_state:
+        raise ValueError(f"Address {transfer.from_address.hex()} has 0 tokens.  Cannot transfer from this address")
+
+    if transfer.from_address in NULL_ADDRESS:
+        balance_state[b"total_supply"] += transfer.value
+    else:
+        if transfer.value > balance_state[transfer.from_address]:
+            raise ValueError(f"Address {transfer.from_address.hex()} has insufficient tokens to transfer")
+
+        balance_state[transfer.from_address] -= transfer.value
+
+
+def generate_balance_state(transfers: list[ERC20Transfer]) -> dict[bytes, int]:
+    warnings.warn("This function is un-tested & may not work as expected", DeprecationWarning, stacklevel=2)
+    balance_state = {b"total_supply": 0}
+
+    for transfer in transfers:
+        if transfer.value <= 0:
+            raise ValueError(f"Cannot parse negative token transfers")
+
+        _apply_credits(balance_state, transfer)
+        _apply_debits(balance_state, transfer)
+
+    return balance_state
+
+
+def generate_balance_state_history(
+    transfers: list[ERC20Transfer],
+    snapshot_frequency: int = 100_000,
+) -> dict[int, dict[bytes, int]]:
+    """
+    Given a list of **SORTED** ERC20 transfers, parse the state of the balances for each address at
+    block_number snapshots.
+
+    Requires the ERC20 token to emit a Transfer events to & from the null address when minting/burning.  If the
+    ERC20 token does not do this, generate ERC20 transfer events from that tokens mint/burn events & pass them to
+    this function
+
+    :param transfers: ordered list of ERC20Transfers
+    :param snapshot_frequency: Capture the balance state every snapshot_frequency blocks
+    :return: balance_state
+
+    {
+        800,000: {
+            b"total_supply": 300,
+            b"\x12\x34": 100,
+            b"\x56\x78": 200
+        },
+        900,000: {
+            b"total_supply": 800,
+            b"\x12\x34": 200,
+            b"\x56\x78": 600
+        }
+    }
+    """
+
+    warnings.warn("This function is un-tested & may not work as expected", DeprecationWarning, stacklevel=2)
+
+    last_snapshot = ((transfers[0].block_number // snapshot_frequency) + 1) * snapshot_frequency
+    balance_state: dict[bytes, int] = {b"total_supply": 0}
+    histories: dict[int, dict[bytes, int]] = {}
+
+    for transfer in transfers:
+        if transfer.block_number > last_snapshot:
+            histories.update({last_snapshot: balance_state.copy()})
+            last_snapshot += snapshot_frequency
+
+        if transfer.value <= 0:
+            raise ValueError(f"Cannot parse negative token transfers")
+
+        _apply_credits(balance_state, transfer)  # Credit balances based on to_address
+
+        _apply_debits(balance_state, transfer)  # Debit balances based on from_address
+
+    return histories
+
+
+def apply_transfers_to_balance_state(
+    balance_state: dict[bytes, int], transfers: list[ERC20Transfer]
+) -> dict[bytes, int]:
+    """
+    Given a balance state history, a block number, and a list of transfers for the token from the latest snapshot
+    to the block_number, calculate the balance state at the block_number
+
+    :param balance_state: *THIS STATE IS MUTATED*
+    :param transfers:
+    :return:
+    """
+
+    warnings.warn("This function is un-tested & may not work as expected", DeprecationWarning, stacklevel=2)
+
+    for transfer in transfers:
+        if transfer.value <= 0:
+            raise ValueError(f"Cannot parse negative token transfers")
+
+        _apply_credits(balance_state, transfer)  # Credit balances based on to_address
+
+        _apply_debits(balance_state, transfer)  # Debit balances based on from_address
+
+    return balance_state

nethermind/idealis/parse/starknet/event.py

@@ -1,7 +1,7 @@
 from typing import Any
 
+from nethermind.idealis.types.base import ERC20Transfer, ERC721Transfer
 from nethermind.idealis.types.starknet.core import Event
-from nethermind.idealis.types.starknet.tokens import ERC20Transfer, ERC721Transfer
 from nethermind.idealis.utils import to_bytes
 from nethermind.starknet_abi.utils import starknet_keccak
 
@@ -25,51 +25,103 @@ def parse_event_response(rpc_response: dict[str, Any]) -> list[Event]:
     ]
 
 
-def filter_erc_20_transfers(events: list[Event]) -> list[ERC20Transfer]:
+# -------------------------------------
+# Unhandled starknet transfer events...
+# -------------------------------------
+# {to,from,amountOrId}  -- Not handled.  WTF? :face_palm:
+# {to,ext,token,amount}  -- Not handled  What is ext?
+# {id,amount,caller,sender,receiver}  -- Not handled.  1155
+# {to,from,asset,amount}  -- Not handled.  Multi ERC Contract?
+# {id,to,amount,exp_time}  -- Not handled.  WTF?  Where is the from_address?
+# {to,from,tick,value}  -- Not handled.  WTF?
+
+
+def filter_transfers(events: list[Event]) -> tuple[list[ERC20Transfer], list[ERC721Transfer]]:
     """
     Filter out ERC20 Transfer events from a list of Starknet Events
     """
+    erc_20_transfers, erc_721_transfers = [], []
 
-    return [
-        ERC20Transfer(
-            block_number=event.block_number,
-            transaction_index=event.transaction_index,
-            event_index=event.event_index,
-            token_address=event.contract_address,
-            from_address=event.decoded_params["from_"],
-            to_address=event.decoded_params["to"],
-            value=event.decoded_params["value"],
-        )
-        for event in events
-        if len(event.keys) > 0
-        and event.keys[0] == TRANSFER_SIGNATURE
-        and event.decoded_params is not None
-        and "value" in event.decoded_params
-        and "from_" in event.decoded_params
-        and "to" in event.decoded_params
-    ]
+    for event in events:
+        if not event.keys or not event.decoded_params or event.keys[0] != TRANSFER_SIGNATURE:
+            continue
 
+        shared_params = {
+            "block_number": event.block_number,
+            "transaction_index": event.transaction_index,
+            "event_index": event.event_index,
+            "token_address": event.contract_address,
+        }
+        sorted_keys = tuple(sorted(event.decoded_params.keys()))
+        match sorted_keys:
+            # -----------------------------------------
+            # ERC20 Transfer Cases
+            # -----------------------------------------
 
-def filter_erc_721_transfers(events: list[Event]) -> list[ERC721Transfer]:
-    """
-    Filter out ERC721 Transfer events from a list of Starknet Events
-    """
+            case ("from", "to", "value") | ("from_", "to", "value") | ("amount", "from", "to"):
+                # Standard ERC20 transfers
+                erc_20_transfers.append(
+                    ERC20Transfer(
+                        from_address=event.decoded_params.get("from") or event.decoded_params["from_"],
+                        to_address=event.decoded_params["to"],
+                        value=event.decoded_params.get("value") or event.decoded_params["amount"],
+                        **shared_params,  # type: ignore
+                    )
+                )
 
-    return [
-        ERC721Transfer(
-            block_number=event.block_number,
-            transaction_index=event.transaction_index,
-            event_index=event.event_index,
-            token_address=event.contract_address,
-            from_address=event.decoded_params["from_"],
-            to_address=event.decoded_params["to"],
-            token_id=event.decoded_params["tokenId"],
-        )
-        for event in events
-        if len(event.keys) > 0
-        and event.keys[0] == TRANSFER_SIGNATURE
-        and event.decoded_params is not None
-        and "tokenId" in event.decoded_params
-        and "from_" in event.decoded_params
-        and "to" in event.decoded_params
-    ]
+            case ("counter", "from", "to", "value"):
+                erc_20_transfers.append(
+                    ERC20Transfer(
+                        from_address=event.decoded_params["from"],
+                        to_address=event.decoded_params["to"],
+                        value=event.decoded_params["value"],
+                        **shared_params,  # type: ignore
+                    )
+                )
+
+            case ("amount", "from_address", "to_address"):
+                erc_20_transfers.append(
+                    ERC20Transfer(
+                        from_address=event.decoded_params["from_address"],
+                        to_address=event.decoded_params["to_address"],
+                        value=event.decoded_params["amount"],
+                        **shared_params,  # type: ignore
+                    )
+                )
+
+            case ("recipient", "sender", "value"):
+                erc_20_transfers.append(
+                    ERC20Transfer(
+                        from_address=event.decoded_params["sender"],
+                        to_address=event.decoded_params["recipient"],
+                        value=event.decoded_params["value"],
+                        **shared_params,  # type: ignore
+                    )
+                )
+
+            # -----------------------------------------
+            # ERC721 Transfer Cases
+            # -----------------------------------------
+
+            case ("from", "to", "token_id"):  # Standard ERC721 transfers
+                erc_721_transfers.append(
+                    ERC721Transfer(
+                        from_address=event.decoded_params["from"],
+                        to_address=event.decoded_params["to"],
+                        token_id=event.decoded_params["token_id"],
+                        **shared_params,  # type: ignore
+                    )
+                )
+            case ("_from", "_to", "_tokenId") | ("_from", "to", "tokenId") | ("from_", "to", "tokenId"):
+                erc_721_transfers.append(
+                    ERC721Transfer(
+                        from_address=event.decoded_params.get("_from") or event.decoded_params["from_"],
+                        to_address=event.decoded_params.get("to") or event.decoded_params["_to"],
+                        token_id=event.decoded_params.get("tokenId") or event.decoded_params["_tokenId"],
+                        **shared_params,  # type: ignore
+                    )
+                )
+            case _:
+                continue
+
+    return erc_20_transfers, erc_721_transfers