update packages and missing_tokens query

requirements.txt

@@ -1,20 +1,19 @@
 # Dev
-black>=22.6.0
-mypy>=0.961
-pylint>=2.14.4
-pytest>=7.1.2
+black>=24.4.0
+mypy>=1.9.0
+pylint>=3.1.0
+pytest>=8.1.1
 
 # Project
 duneapi==6.0.0
-dune-client>=0.0.6
-python-dotenv==0.21.0
-PyYAML==6.0
-requests==2.28.1
-web3==5.30.0
-psycopg2-binary>=2.9.3
-SQLAlchemy==1.4.41
-pandas==1.5.0
-click==8.1.3
-marshmallow==3.18.0
-xlsxwriter==3.0.3
-tqdm==4.64.1
+dune_client>=1.7.0
+python-dotenv>=0.21.1
+requests>=2.31.0
+web3>=6.17.2
+psycopg2-binary>=2.9.9
+SQLAlchemy>=2.0.29
+pandas>=2.2.2
+click>=8.1.7
+marshmallow>=3.14.1
+xlsxwriter>=3.1.4
+tqdm>=4.65.2

src/cip3_eth_spent.py

@@ -3,7 +3,7 @@
 from duneapi.util import open_query
 
 
-def fetch_eth_spent(dune: DuneAPI):
+def fetch_eth_spent(dune: DuneAPI) -> None:
     """
     Fetches ETH spent on CIP-9 Fee subsidies
     https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e

src/db/pg_client.py

@@ -3,7 +3,7 @@
 import psycopg2
 from dotenv import load_dotenv
 from psycopg2._psycopg import connection
-from sqlalchemy import create_engine, engine
+from sqlalchemy import create_engine, Engine
 
 load_dotenv()
 host = os.environ["ORDERBOOK_HOST"]
@@ -31,7 +31,7 @@ def pg_connect() -> connection:
     )
 
 
-def pg_engine() -> engine:
+def pg_engine() -> Engine:
     return create_engine(db_string())
 
 

src/dune_2_excel.py

@@ -6,7 +6,7 @@
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
 from dune_client.models import ResultsResponse
-from dune_client.query import Query
+from dune_client.query import QueryBase as Query
 from dune_client.types import QueryParameter
 import pandas as pd
 from tqdm import tqdm

src/missing_prices.py

@@ -7,7 +7,7 @@
 import requests
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
-from dune_client.query import Query
+from dune_client.query import QueryBase as Query
 from dune_client.types import Address
 from marshmallow import fields
 

src/missing_tokens.py

@@ -7,7 +7,7 @@
 import web3.exceptions
 from dotenv import load_dotenv
 from dune_client.client import DuneClient
-from dune_client.query import Query as DuneQuery
+from dune_client.query import QueryBase as DuneQuery
 from dune_client.types import QueryParameter, Address
 
 from web3 import Web3
@@ -67,8 +67,8 @@ class TokenDetails:  # pylint:disable=too-few-public-methods
     """EVM token Details (including address, symbol, decimals)"""
 
     def __init__(self, address: Address, w3: Web3):
-        self.address = Web3.toChecksumAddress(address.address)
-        if self.address == Web3.toChecksumAddress(
+        self.address = Web3.to_checksum_address(address.address)
+        if self.address == Web3.to_checksum_address(
             "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE"
         ):
             self.symbol = "ETH"
@@ -83,29 +83,29 @@ def as_dune_string(self) -> str:
         Returns Dune Representation of an ERC20 Token:
         https://github.com/duneanalytics/spellbook/blob/main/models/tokens/ethereum/tokens_ethereum_erc20.sql
         Example:
-          ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18),
+          , ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18)
         """
-        return f",({str(self.address).lower()}, '{self.symbol}', {self.decimals})"
+        return f", ({str(self.address).lower()}, '{self.symbol}', {self.decimals})"
 
 
 def fetch_missing_tokens(dune: DuneClient, network: Network) -> list[Address]:
     """Uses Official DuneAPI and to fetch Missing Tokens"""
     query = DuneQuery(
         name="V3: Missing Tokens on {{Blockchain}}",
-        query_id=2444707,
+        query_id=3645484,
         params=[
             QueryParameter.enum_type("Blockchain", network.as_dune_v2_repr()),
             QueryParameter.date_type("DateFrom", "2023-01-01 00:00:00"),
-            QueryParameter.number_type("Popularity", 250),
+            QueryParameter.number_type("Popularity", 100),
         ],
     )
     print(f"Fetching missing tokens for {network} from {query.url()}")
-    v2_missing = dune.refresh(query, ping_frequency=10)
+    v2_missing = dune.run_query(query, ping_frequency=10)
 
     return [Address(row["token"]) for row in v2_missing.get_rows()]
 
 
-def replace_line(old_line: str, new_line: str, file_loc: str):
+def replace_line(old_line: str, new_line: str, file_loc: str) -> None:
     """Overwrites old_line with new_line in file at file_loc"""
     for line in fileinput.input(file_loc, inplace=True):
         if line == old_line:
@@ -135,15 +135,15 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None
                 ignored.add(token)
                 print(f"ContractLogicError on {token} - skipping.")
 
-        results = "\n        ".join(
+        results = "\n    ".join(
             token_details[t].as_dune_string()
             for t in missing_tokens
             if t not in ignored
         )
         if insert_loc:
             print(f"Writing Tokens to File {insert_loc}")
-            old_line = "     ) AS temp_table (contract_address, symbol, decimals)\n"
-            new_line = "        " + results + "\n" + old_line
+            old_line = ") AS temp_table (contract_address, symbol, decimals)\n"
+            new_line = "    " + results + "\n" + old_line
             replace_line(old_line, new_line, insert_loc)
         else:
             print(f"Missing Tokens:\n\n{results}\n")
@@ -157,12 +157,12 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None
     for blockchain in list(Network):
         chain_name = blockchain.as_dune_v2_repr()
         print(f"Execute on {chain_name}")
-        token_file = f"models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql"
+        token_file = f"tokens/models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql"
         spellbook_file = (
             os.path.join(spellbook_path, token_file) if spellbook_path else None
         )
 
         run_missing_tokens(
             chain=blockchain,
-            insert_loc=spellbook_file,
+            # insert_loc=spellbook_file,
         )

src/orderbook.py

@@ -17,8 +17,7 @@
     case,
 )
 
-# TODO - I find it strange that LegacyCursor is still being returned...
-from sqlalchemy.engine import LegacyCursorResult
+from sqlalchemy.engine import CursorResult
 
 from src.db.pg_client import pg_engine
 
@@ -68,7 +67,7 @@ def sql_alchemy_basic(db: engine):
 
     with db.connect() as conn:
         select_statement = invalidation_table.select()
-        result_set: LegacyCursorResult = conn.execute(select_statement)
+        result_set: CursorResult = conn.execute(select_statement)
         for r in result_set:
             print(bin_str(r.order_uid))
 
@@ -117,7 +116,7 @@ def sql_alchemy_advanced(db: engine):
     )
     with db.connect() as conn:
         print("Querying for spam traders having more failed orders than successful")
-        result_set: LegacyCursorResult = conn.execute(select_statement)
+        result_set: CursorResult = conn.execute(select_statement)
         results = result_set.all()
         print(f"Found {len(results)} traders with more failed orders than trades")
         print(list(result_set.keys()))
@@ -143,7 +142,7 @@ def sql_alchemy_advanced(db: engine):
 
     print("Now querying with raw sql")
     with db.connect() as conn:
-        result_set: LegacyCursorResult = conn.execute(raw_query)
+        result_set: CursorResult = conn.execute(raw_query)
         print(f"Found {len(result_set.all())} with raw query")
 
 

src/retention/get_relevant_ens.py

@@ -6,7 +6,7 @@
 from duneapi.api import DuneAPI
 from duneapi.types import DuneQuery, Network, QueryParameter
 from duneapi.util import open_query
-from src.subgraph.ens_data import get_wallet_ens_data
+from src.subgraph.ens_data import get_wallet_ens_data, WalletNameMap
 from src.utils import write_to_json, valid_date
 
 SUBGRAPH_URL = "https://api.thegraph.com/subgraphs/name/ensdomains/ens"
@@ -26,7 +26,7 @@ def __str__(self) -> str:
 
 def fetch_retained_users(
     dune: DuneAPI, category: RetentionCategory, day: datetime.datetime
-):
+) -> WalletNameMap:
     """
     Fetches ETH spent on CIP-9 Fee subsidies
     https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e

src/subgraph/ens_data.py

@@ -28,7 +28,7 @@
 
 def read_ens_text(resolver: str, node: str, key: str) -> str:
     resolver_contract = w3.eth.contract(
-        address=Web3.toChecksumAddress(resolver), abi=PUBLIC_RESOLVER_ABI
+        address=Web3.to_checksum_address(resolver), abi=PUBLIC_RESOLVER_ABI
     )
 
     text: str = resolver_contract.caller.text(node, key)
@@ -84,7 +84,6 @@ def get_result_page(wallets: list[str], skip: int, block: Optional[int] = None)
         subgraph_url="https://api.thegraph.com/subgraphs/name/ensdomains/ens",
         query=resolve_query(list(wallets), skip, block),
     )
-    print(result_json)
     return result_json["data"]["domains"]
 
 

tests/test_subgraph.py

@@ -28,7 +28,7 @@ def test_fetch_ens_data_small(self):
                 }
             ],
         }
-        results = get_wallet_ens_data(set(expected_ens_records.keys()), 15687500)
+        results = get_wallet_ens_data(set(expected_ens_records.keys()))
         for wallet, data in expected_ens_records.items():
             self.assertEqual(results[wallet], data, f"failed for wallet {wallet}")
 

tests/test_utils.py

@@ -5,8 +5,6 @@
 import shutil
 from datetime import datetime
 
-from duneapi.types import Network as LegacyDuneNetwork
-
 from src.missing_tokens import Network
 from src.utils import partition_array, write_to_json, valid_date