Dependency Update

.env.sample

@@ -3,11 +3,12 @@ DUNE_USER=
 DUNE_PASSWORD=
 DUNE_QUERY_ID=
 
+ETH_RPC=https://rpc.ankr.com/eth
+GNOSIS_RPC=https://rpc.ankr.com/gnosis
+
 # Official Dune API
 DUNE_API_KEY=
 
-INFURA_KEY=
-
 ORDERBOOK_HOST=
 ORDERBOOK_USER=
 ORDERBOOK_PORT=

.github/workflows/pull-request.yaml

@@ -8,10 +8,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
-      - name: Setup Python 3.10
+      - name: Setup Python 3.12
         uses: actions/setup-python@v2
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Install Requirements
         run:
           pip install -r requirements.txt
@@ -26,5 +26,3 @@ jobs:
       - name: Unit Tests
         run:
           python -m pytest tests
-        env:
-          INFURA_KEY: ${{ secrets.INFURA_KEY }}

README.md

@@ -27,13 +27,12 @@ New updated Docker instructions:
 ```shell
 docker run \                                                   
     -e SPELLBOOK_PATH=$SPELLBOOK_PATH \
-    -e INFURA_KEY=$INFURA_KEY \
     -e DUNE_API_KEY=$DUNE_API_KEY \
     -v $SPELLBOOK_PATH:$SPELLBOOK_PATH \
     ghcr.io/cowprotocol/data-misc-missing-tokens:main
 ```
 
-Note that this will require `SPELLBOOK_PATH`, `DUNE_API_KEY` and `INFURA_KEY` variables set.
+Note that this will require `SPELLBOOK_PATH`, `DUNE_API_KEY` variables set.
 
 Step-by-step instructions:
 
@@ -47,7 +46,7 @@ clone it to your local machine, and create a new branch with
     ```sh 
     python -m src.missing_tokens
     ```
-   Note that this will require a `DUNE_API_KEY` and `INFURA_KEY`. 
+   Note that this will require a `DUNE_API_KEY` 
 This script will print the contents to be inserted in the console.
 4. Results should be inserted into:
    - V1 - `deprecated-dune-v1-abstractions/ethereum/erc20/tokens.sql` 

requirements.txt

@@ -1,20 +1,19 @@
 # Dev
-black>=22.6.0
-mypy>=0.961
-pylint>=2.14.4
-pytest>=7.1.2
+black>=24.4.0
+mypy>=1.9.0
+pylint>=3.1.0
+pytest>=8.1.1
 
 # Project
 duneapi==6.0.0
-dune-client>=0.0.6
-python-dotenv==0.21.0
-PyYAML==6.0
-requests==2.28.1
-web3==5.30.0
-psycopg2-binary>=2.9.3
-SQLAlchemy==1.4.41
-pandas==1.5.0
-click==8.1.3
-marshmallow==3.18.0
-xlsxwriter==3.0.3
-tqdm==4.64.1
+dune_client>=1.7.3
+python-dotenv>=0.21.1
+requests>=2.31.0
+web3>=6.17.2
+psycopg2-binary>=2.9.9
+SQLAlchemy>=2.0.29
+pandas>=2.2.2
+click>=8.1.7
+marshmallow>=3.14.1
+xlsxwriter>=3.1.4
+tqdm>=4.65.2

src/cip3_eth_spent.py

@@ -3,7 +3,7 @@
 from duneapi.util import open_query
 
 
-def fetch_eth_spent(dune: DuneAPI):
+def fetch_eth_spent(dune: DuneAPI) -> None:
     """
     Fetches ETH spent on CIP-9 Fee subsidies
     https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e

src/constants.py

@@ -1,3 +1,4 @@
+import os
 from pathlib import Path
 
 
@@ -17,3 +18,6 @@
         "type": "function",
     }
 ]
+
+ETH_RPC = os.environ.get("ETH_RPC", "https://rpc.ankr.com/eth")
+GNOSIS_RPC = os.environ.get("GNOSIS_RPC", "https://rpc.gnosischain.com")

src/db/pg_client.py

@@ -3,7 +3,7 @@
 import psycopg2
 from dotenv import load_dotenv
 from psycopg2._psycopg import connection
-from sqlalchemy import create_engine, engine
+from sqlalchemy import create_engine, Engine
 
 load_dotenv()
 host = os.environ["ORDERBOOK_HOST"]
@@ -31,7 +31,7 @@ def pg_connect() -> connection:
     )
 
 
-def pg_engine() -> engine:
+def pg_engine() -> Engine:
     return create_engine(db_string())
 
 

src/dune_2_excel.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
 from dune_client.models import ResultsResponse
-from dune_client.query import Query
+from dune_client.query import QueryBase as Query
 from dune_client.types import QueryParameter
 import pandas as pd
 from tqdm import tqdm

src/gas_saved.py

@@ -1,12 +1,11 @@
-import os
-
 import click
 import pandas as pd
 from dotenv import load_dotenv
 from eth_typing.encoding import HexStr
 from web3 import Web3
 from web3.types import TxReceipt
 
+from src.constants import ETH_RPC
 from src.db.pg_client import pg_engine
 
 
@@ -36,9 +35,7 @@ def main(batch_tx_hash: str) -> int:
     # Ref: https://github.com/cowprotocol/services/blob/fd5f7cf47a6afdff89b310b60b869dfc577ac7a7/crates/shared/src/price_estimation/gas.rs#L37
     df_quotes["gas_amount"] = df_quotes["gas_amount"].apply(lambda x: x - 106391)
     load_dotenv()
-    w3 = Web3(
-        Web3.HTTPProvider(f"https://mainnet.infura.io/v3/{os.environ['INFURA_KEY']}")
-    )
+    w3 = Web3(Web3.HTTPProvider(ETH_RPC))
     tx: TxReceipt = w3.eth.get_transaction_receipt(HexStr(batch_tx_hash))
     gas_used = tx["gasUsed"]
     print(

src/missing_prices.py

@@ -7,7 +7,7 @@
 import requests
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
-from dune_client.query import Query
+from dune_client.query import QueryBase as Query
 from dune_client.types import Address
 from marshmallow import fields
 

src/missing_tokens.py

@@ -7,11 +7,13 @@
 import web3.exceptions
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
-from dune_client.query import Query as DuneQuery
+from dune_client.query import QueryBase as DuneQuery
 from dune_client.types import QueryParameter, Address
 
 from web3 import Web3
 
+from src.constants import ETH_RPC, GNOSIS_RPC
+
 
 class Network(Enum):
     """
@@ -25,11 +27,11 @@ def as_dune_v2_repr(self) -> str:
         """Returns Dune V1 Network String (as compatible with Dune V2 Engine)"""
         return {Network.MAINNET: "ethereum", Network.GNOSIS: "gnosis"}[self]
 
-    def node_url(self, api_key: str) -> str:
+    def node_url(self) -> str:
         """Returns URL to Node for Network"""
         return {
-            Network.MAINNET: f"https://mainnet.infura.io/v3/{api_key}",
-            Network.GNOSIS: "https://rpc.gnosischain.com",
+            Network.MAINNET: ETH_RPC,
+            Network.GNOSIS: GNOSIS_RPC,
         }[self]
 
     @property
@@ -67,8 +69,8 @@ class TokenDetails:  # pylint:disable=too-few-public-methods
     """EVM token Details (including address, symbol, decimals)"""
 
     def __init__(self, address: Address, w3: Web3):
-        self.address = Web3.toChecksumAddress(address.address)
-        if self.address == Web3.toChecksumAddress(
+        self.address = Web3.to_checksum_address(address.address)
+        if self.address == Web3.to_checksum_address(
             "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"
         ):
             self.symbol = "ETH"
@@ -83,29 +85,29 @@ def as_dune_string(self) -> str:
         Returns Dune Representation of an ERC20 Token:
         https://github.com/duneanalytics/spellbook/blob/main/models/tokens/ethereum/tokens_ethereum_erc20.sql
         Example:
-          ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18),
+          , ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18)
         """
-        return f",({str(self.address).lower()}, '{self.symbol}', {self.decimals})"
+        return f", ({str(self.address).lower()}, '{self.symbol}', {self.decimals})"
 
 
 def fetch_missing_tokens(dune: DuneClient, network: Network) -> list[Address]:
     """Uses Official DuneAPI and to fetch Missing Tokens"""
     query = DuneQuery(
         name="V3: Missing Tokens on {{Blockchain}}",
-        query_id=2444707,
+        query_id=3645484,
         params=[
             QueryParameter.enum_type("Blockchain", network.as_dune_v2_repr()),
             QueryParameter.date_type("DateFrom", "2023-01-01 00:00:00"),
             QueryParameter.number_type("Popularity", 250),
         ],
     )
     print(f"Fetching missing tokens for {network} from {query.url()}")
-    v2_missing = dune.refresh(query, ping_frequency=10)
+    v2_missing = dune.run_query(query, ping_frequency=10)
 
     return [Address(row["token"]) for row in v2_missing.get_rows()]
 
 
-def replace_line(old_line: str, new_line: str, file_loc: str):
+def replace_line(old_line: str, new_line: str, file_loc: str) -> None:
     """Overwrites old_line with new_line in file at file_loc"""
     for line in fileinput.input(file_loc, inplace=True):
         if line == old_line:
@@ -116,7 +118,7 @@ def replace_line(old_line: str, new_line: str, file_loc: str):
 
 def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None:
     """Script's main entry point, runs for given network."""
-    w3 = Web3(Web3.HTTPProvider(chain.node_url(os.environ["INFURA_KEY"])))
+    w3 = Web3(Web3.HTTPProvider(chain.node_url()))
     client = DuneClient(os.environ["DUNE_API_KEY"])
     missing_tokens = fetch_missing_tokens(client, chain)
 
@@ -135,15 +137,15 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None
                 ignored.add(token)
                 print(f"ContractLogicError on {token} - skipping.")
 
-        results = "\n        ".join(
+        results = "\n    ".join(
             token_details[t].as_dune_string()
             for t in missing_tokens
             if t not in ignored
         )
         if insert_loc:
             print(f"Writing Tokens to File {insert_loc}")
-            old_line = "     ) AS temp_table (contract_address, symbol, decimals)\n"
-            new_line = "        " + results + "\n" + old_line
+            old_line = ") AS temp_table (contract_address, symbol, decimals)\n"
+            new_line = "    " + results + "\n" + old_line
             replace_line(old_line, new_line, insert_loc)
         else:
             print(f"Missing Tokens:\n\n{results}\n")
@@ -157,7 +159,7 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None
     for blockchain in list(Network):
         chain_name = blockchain.as_dune_v2_repr()
         print(f"Execute on {chain_name}")
-        token_file = f"models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql"
+        token_file = f"tokens/models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql"
         spellbook_file = (
             os.path.join(spellbook_path, token_file) if spellbook_path else None
         )

src/orderbook.py

@@ -17,8 +17,7 @@
     case,
 )
 
-# TODO - I find it strange that LegacyCursor is still being returned...
-from sqlalchemy.engine import LegacyCursorResult
+from sqlalchemy.engine import CursorResult
 
 from src.db.pg_client import pg_engine
 
@@ -68,7 +67,7 @@ def sql_alchemy_basic(db: engine):
 
     with db.connect() as conn:
         select_statement = invalidation_table.select()
-        result_set: LegacyCursorResult = conn.execute(select_statement)
+        result_set: CursorResult = conn.execute(select_statement)
         for r in result_set:
             print(bin_str(r.order_uid))
 
@@ -117,7 +116,7 @@ def sql_alchemy_advanced(db: engine):
     )
     with db.connect() as conn:
         print("Querying for spam traders having more failed orders than successful")
-        result_set: LegacyCursorResult = conn.execute(select_statement)
+        result_set: CursorResult = conn.execute(select_statement)
         results = result_set.all()
         print(f"Found {len(results)} traders with more failed orders than trades")
         print(list(result_set.keys()))
@@ -143,7 +142,7 @@ def sql_alchemy_advanced(db: engine):
 
     print("Now querying with raw sql")
     with db.connect() as conn:
-        result_set: LegacyCursorResult = conn.execute(raw_query)
+        result_set: CursorResult = conn.execute(raw_query)
         print(f"Found {len(result_set.all())} with raw query")
 
 

src/retention/get_relevant_ens.py

@@ -6,7 +6,7 @@
 from duneapi.api import DuneAPI
 from duneapi.types import DuneQuery, Network, QueryParameter
 from duneapi.util import open_query
-from src.subgraph.ens_data import get_wallet_ens_data
+from src.subgraph.ens_data import get_wallet_ens_data, WalletNameMap
 from src.utils import write_to_json, valid_date
 
 SUBGRAPH_URL = "https://api.thegraph.com/subgraphs/name/ensdomains/ens"
@@ -26,7 +26,7 @@ def __str__(self) -> str:
 
 def fetch_retained_users(
     dune: DuneAPI, category: RetentionCategory, day: datetime.datetime
-):
+) -> WalletNameMap:
     """
     Fetches ETH spent on CIP-9 Fee subsidies
     https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e

src/subgraph/ens_data.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 from web3 import Web3
 
-from src.constants import PUBLIC_RESOLVER_ABI
+from src.constants import PUBLIC_RESOLVER_ABI, ETH_RPC
 from src.subgraph.fetch import execute_subgraph_query
 from src.utils import partition_array
 
@@ -23,12 +23,12 @@
 }
 
 load_dotenv()
-w3 = Web3(Web3.HTTPProvider(f"https://mainnet.infura.io/v3/{os.environ['INFURA_KEY']}"))
+w3 = Web3(Web3.HTTPProvider(ETH_RPC))
 
 
 def read_ens_text(resolver: str, node: str, key: str) -> str:
     resolver_contract = w3.eth.contract(
-        address=Web3.toChecksumAddress(resolver), abi=PUBLIC_RESOLVER_ABI
+        address=Web3.to_checksum_address(resolver), abi=PUBLIC_RESOLVER_ABI
     )
 
     text: str = resolver_contract.caller.text(node, key)
@@ -84,7 +84,6 @@ def get_result_page(wallets: list[str], skip: int, block: Optional[int] = None)
         subgraph_url="https://api.thegraph.com/subgraphs/name/ensdomains/ens",
         query=resolve_query(list(wallets), skip, block),
     )
-    print(result_json)
     return result_json["data"]["domains"]
 
 

tests/test_subgraph.py

@@ -28,7 +28,7 @@ def test_fetch_ens_data_small(self):
                 }
             ],
         }
-        results = get_wallet_ens_data(set(expected_ens_records.keys()), 15687500)
+        results = get_wallet_ens_data(set(expected_ens_records.keys()))
         for wallet, data in expected_ens_records.items():
             self.assertEqual(results[wallet], data, f"failed for wallet {wallet}")