diff --git a/.env.sample b/.env.sample index ba42409..6020279 100644 --- a/.env.sample +++ b/.env.sample @@ -3,11 +3,12 @@ DUNE_USER= DUNE_PASSWORD= DUNE_QUERY_ID= +ETH_RPC=https://rpc.ankr.com/eth +GNOSIS_RPC=https://rpc.ankr.com/gnosis + # Official Dune API DUNE_API_KEY= -INFURA_KEY= - ORDERBOOK_HOST= ORDERBOOK_USER= ORDERBOOK_PORT= diff --git a/.github/workflows/pull-request.yaml b/.github/workflows/pull-request.yaml index 3232388..e4d6a8e 100644 --- a/.github/workflows/pull-request.yaml +++ b/.github/workflows/pull-request.yaml @@ -8,10 +8,10 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: Setup Python 3.10 + - name: Setup Python 3.12 uses: actions/setup-python@v2 with: - python-version: '3.10' + python-version: '3.12' - name: Install Requirements run: pip install -r requirements.txt @@ -26,5 +26,3 @@ jobs: - name: Unit Tests run: python -m pytest tests - env: - INFURA_KEY: ${{ secrets.INFURA_KEY }} diff --git a/README.md b/README.md index 472fbf3..3cfb11a 100644 --- a/README.md +++ b/README.md @@ -27,13 +27,12 @@ New updated Docker instructions: ```shell docker run \ -e SPELLBOOK_PATH=$SPELLBOOK_PATH \ - -e INFURA_KEY=$INFURA_KEY \ -e DUNE_API_KEY=$DUNE_API_KEY \ -v $SPELLBOOK_PATH:$SPELLBOOK_PATH \ ghcr.io/cowprotocol/data-misc-missing-tokens:main ``` -Note that this will require `SPELLBOOK_PATH`, `DUNE_API_KEY` and `INFURA_KEY` variables set. +Note that this will require `SPELLBOOK_PATH`, `DUNE_API_KEY` variables set. Step-by-step instructions: @@ -47,7 +46,7 @@ clone it to your local machine, and create a new branch with ```sh python -m src.missing_tokens ``` - Note that this will require a `DUNE_API_KEY` and `INFURA_KEY`. + Note that this will require a `DUNE_API_KEY` This script will print the contents to be inserted in the console. 4. Results should be inserted into: - V1 - `deprecated-dune-v1-abstractions/ethereum/erc20/tokens.sql` diff --git a/requirements.txt b/requirements.txt index f49eac7..0681193 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,20 +1,19 @@ # Dev -black>=22.6.0 -mypy>=0.961 -pylint>=2.14.4 -pytest>=7.1.2 +black>=24.4.0 +mypy>=1.9.0 +pylint>=3.1.0 +pytest>=8.1.1 # Project duneapi==6.0.0 -dune-client>=0.0.6 -python-dotenv==0.21.0 -PyYAML==6.0 -requests==2.28.1 -web3==5.30.0 -psycopg2-binary>=2.9.3 -SQLAlchemy==1.4.41 -pandas==1.5.0 -click==8.1.3 -marshmallow==3.18.0 -xlsxwriter==3.0.3 -tqdm==4.64.1 +dune_client>=1.7.3 +python-dotenv>=0.21.1 +requests>=2.31.0 +web3>=6.17.2 +psycopg2-binary>=2.9.9 +SQLAlchemy>=2.0.29 +pandas>=2.2.2 +click>=8.1.7 +marshmallow>=3.14.1 +xlsxwriter>=3.1.4 +tqdm>=4.65.2 diff --git a/src/cip3_eth_spent.py b/src/cip3_eth_spent.py index ae51dc2..e3c9c47 100644 --- a/src/cip3_eth_spent.py +++ b/src/cip3_eth_spent.py @@ -3,7 +3,7 @@ from duneapi.util import open_query -def fetch_eth_spent(dune: DuneAPI): +def fetch_eth_spent(dune: DuneAPI) -> None: """ Fetches ETH spent on CIP-9 Fee subsidies https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e diff --git a/src/constants.py b/src/constants.py index 36b46a9..19a722b 100644 --- a/src/constants.py +++ b/src/constants.py @@ -1,3 +1,4 @@ +import os from pathlib import Path @@ -17,3 +18,6 @@ "type": "function", } ] + +ETH_RPC = os.environ.get("ETH_RPC", "https://rpc.ankr.com/eth") +GNOSIS_RPC = os.environ.get("GNOSIS_RPC", "https://rpc.gnosischain.com") diff --git a/src/db/pg_client.py b/src/db/pg_client.py index 2b42c34..d5bbf1f 100644 --- a/src/db/pg_client.py +++ b/src/db/pg_client.py @@ -3,7 +3,7 @@ import psycopg2 from dotenv import load_dotenv from psycopg2._psycopg import connection -from sqlalchemy import create_engine, engine +from sqlalchemy import create_engine, Engine load_dotenv() host = os.environ["ORDERBOOK_HOST"] @@ -31,7 +31,7 @@ def pg_connect() -> connection: ) -def pg_engine() -> engine: +def pg_engine() -> Engine: return create_engine(db_string()) diff --git a/src/dune_2_excel.py b/src/dune_2_excel.py index 69c39a6..f13f068 100644 --- a/src/dune_2_excel.py +++ b/src/dune_2_excel.py @@ -6,7 +6,7 @@ from dotenv import load_dotenv from dune_client.client import DuneClient from dune_client.models import ResultsResponse -from dune_client.query import Query +from dune_client.query import QueryBase as Query from dune_client.types import QueryParameter import pandas as pd from tqdm import tqdm diff --git a/src/gas_saved.py b/src/gas_saved.py index 5541e95..50428ee 100644 --- a/src/gas_saved.py +++ b/src/gas_saved.py @@ -1,5 +1,3 @@ -import os - import click import pandas as pd from dotenv import load_dotenv @@ -7,6 +5,7 @@ from web3 import Web3 from web3.types import TxReceipt +from src.constants import ETH_RPC from src.db.pg_client import pg_engine @@ -36,9 +35,7 @@ def main(batch_tx_hash: str) -> int: # Ref: https://github.com/cowprotocol/services/blob/fd5f7cf47a6afdff89b310b60b869dfc577ac7a7/crates/shared/src/price_estimation/gas.rs#L37 df_quotes["gas_amount"] = df_quotes["gas_amount"].apply(lambda x: x - 106391) load_dotenv() - w3 = Web3( - Web3.HTTPProvider(f"https://mainnet.infura.io/v3/{os.environ['INFURA_KEY']}") - ) + w3 = Web3(Web3.HTTPProvider(ETH_RPC)) tx: TxReceipt = w3.eth.get_transaction_receipt(HexStr(batch_tx_hash)) gas_used = tx["gasUsed"] print( diff --git a/src/missing_prices.py b/src/missing_prices.py index 9826a87..2cfb055 100644 --- a/src/missing_prices.py +++ b/src/missing_prices.py @@ -7,7 +7,7 @@ import requests from dotenv import load_dotenv from dune_client.client import DuneClient -from dune_client.query import Query +from dune_client.query import QueryBase as Query from dune_client.types import Address from marshmallow import fields diff --git a/src/missing_tokens.py b/src/missing_tokens.py index c6bfbdc..78dae7f 100644 --- a/src/missing_tokens.py +++ b/src/missing_tokens.py @@ -7,11 +7,13 @@ import web3.exceptions from dotenv import load_dotenv from dune_client.client import DuneClient -from dune_client.query import Query as DuneQuery +from dune_client.query import QueryBase as DuneQuery from dune_client.types import QueryParameter, Address from web3 import Web3 +from src.constants import ETH_RPC, GNOSIS_RPC + class Network(Enum): """ @@ -25,11 +27,11 @@ def as_dune_v2_repr(self) -> str: """Returns Dune V1 Network String (as compatible with Dune V2 Engine)""" return {Network.MAINNET: "ethereum", Network.GNOSIS: "gnosis"}[self] - def node_url(self, api_key: str) -> str: + def node_url(self) -> str: """Returns URL to Node for Network""" return { - Network.MAINNET: f"https://mainnet.infura.io/v3/{api_key}", - Network.GNOSIS: "https://rpc.gnosischain.com", + Network.MAINNET: ETH_RPC, + Network.GNOSIS: GNOSIS_RPC, }[self] @property @@ -67,8 +69,8 @@ class TokenDetails: # pylint:disable=too-few-public-methods """EVM token Details (including address, symbol, decimals)""" def __init__(self, address: Address, w3: Web3): - self.address = Web3.toChecksumAddress(address.address) - if self.address == Web3.toChecksumAddress( + self.address = Web3.to_checksum_address(address.address) + if self.address == Web3.to_checksum_address( "0xEeeeeEeeeEeEeeEeEeEeeEEEeeeeEeeeeeeeEEeE" ): self.symbol = "ETH" @@ -83,16 +85,16 @@ def as_dune_string(self) -> str: Returns Dune Representation of an ERC20 Token: https://github.com/duneanalytics/spellbook/blob/main/models/tokens/ethereum/tokens_ethereum_erc20.sql Example: - ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18), + , ('0xfcc5c47be19d06bf83eb04298b026f81069ff65b', 'yCRV', 18) """ - return f",({str(self.address).lower()}, '{self.symbol}', {self.decimals})" + return f", ({str(self.address).lower()}, '{self.symbol}', {self.decimals})" def fetch_missing_tokens(dune: DuneClient, network: Network) -> list[Address]: """Uses Official DuneAPI and to fetch Missing Tokens""" query = DuneQuery( name="V3: Missing Tokens on {{Blockchain}}", - query_id=2444707, + query_id=3645484, params=[ QueryParameter.enum_type("Blockchain", network.as_dune_v2_repr()), QueryParameter.date_type("DateFrom", "2023-01-01 00:00:00"), @@ -100,12 +102,12 @@ def fetch_missing_tokens(dune: DuneClient, network: Network) -> list[Address]: ], ) print(f"Fetching missing tokens for {network} from {query.url()}") - v2_missing = dune.refresh(query, ping_frequency=10) + v2_missing = dune.run_query(query, ping_frequency=10) return [Address(row["token"]) for row in v2_missing.get_rows()] -def replace_line(old_line: str, new_line: str, file_loc: str): +def replace_line(old_line: str, new_line: str, file_loc: str) -> None: """Overwrites old_line with new_line in file at file_loc""" for line in fileinput.input(file_loc, inplace=True): if line == old_line: @@ -116,7 +118,7 @@ def replace_line(old_line: str, new_line: str, file_loc: str): def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None: """Script's main entry point, runs for given network.""" - w3 = Web3(Web3.HTTPProvider(chain.node_url(os.environ["INFURA_KEY"]))) + w3 = Web3(Web3.HTTPProvider(chain.node_url())) client = DuneClient(os.environ["DUNE_API_KEY"]) missing_tokens = fetch_missing_tokens(client, chain) @@ -135,15 +137,15 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None ignored.add(token) print(f"ContractLogicError on {token} - skipping.") - results = "\n ".join( + results = "\n ".join( token_details[t].as_dune_string() for t in missing_tokens if t not in ignored ) if insert_loc: print(f"Writing Tokens to File {insert_loc}") - old_line = " ) AS temp_table (contract_address, symbol, decimals)\n" - new_line = " " + results + "\n" + old_line + old_line = ") AS temp_table (contract_address, symbol, decimals)\n" + new_line = " " + results + "\n" + old_line replace_line(old_line, new_line, insert_loc) else: print(f"Missing Tokens:\n\n{results}\n") @@ -157,7 +159,7 @@ def run_missing_tokens(chain: Network, insert_loc: Optional[str] = None) -> None for blockchain in list(Network): chain_name = blockchain.as_dune_v2_repr() print(f"Execute on {chain_name}") - token_file = f"models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql" + token_file = f"tokens/models/tokens/{chain_name}/tokens_{chain_name}_erc20.sql" spellbook_file = ( os.path.join(spellbook_path, token_file) if spellbook_path else None ) diff --git a/src/orderbook.py b/src/orderbook.py index bef44ff..b7846ca 100644 --- a/src/orderbook.py +++ b/src/orderbook.py @@ -17,8 +17,7 @@ case, ) -# TODO - I find it strange that LegacyCursor is still being returned... -from sqlalchemy.engine import LegacyCursorResult +from sqlalchemy.engine import CursorResult from src.db.pg_client import pg_engine @@ -68,7 +67,7 @@ def sql_alchemy_basic(db: engine): with db.connect() as conn: select_statement = invalidation_table.select() - result_set: LegacyCursorResult = conn.execute(select_statement) + result_set: CursorResult = conn.execute(select_statement) for r in result_set: print(bin_str(r.order_uid)) @@ -117,7 +116,7 @@ def sql_alchemy_advanced(db: engine): ) with db.connect() as conn: print("Querying for spam traders having more failed orders than successful") - result_set: LegacyCursorResult = conn.execute(select_statement) + result_set: CursorResult = conn.execute(select_statement) results = result_set.all() print(f"Found {len(results)} traders with more failed orders than trades") print(list(result_set.keys())) @@ -143,7 +142,7 @@ def sql_alchemy_advanced(db: engine): print("Now querying with raw sql") with db.connect() as conn: - result_set: LegacyCursorResult = conn.execute(raw_query) + result_set: CursorResult = conn.execute(raw_query) print(f"Found {len(result_set.all())} with raw query") diff --git a/src/retention/get_relevant_ens.py b/src/retention/get_relevant_ens.py index b163727..5c45e89 100644 --- a/src/retention/get_relevant_ens.py +++ b/src/retention/get_relevant_ens.py @@ -6,7 +6,7 @@ from duneapi.api import DuneAPI from duneapi.types import DuneQuery, Network, QueryParameter from duneapi.util import open_query -from src.subgraph.ens_data import get_wallet_ens_data +from src.subgraph.ens_data import get_wallet_ens_data, WalletNameMap from src.utils import write_to_json, valid_date SUBGRAPH_URL = "https://api.thegraph.com/subgraphs/name/ensdomains/ens" @@ -26,7 +26,7 @@ def __str__(self) -> str: def fetch_retained_users( dune: DuneAPI, category: RetentionCategory, day: datetime.datetime -): +) -> WalletNameMap: """ Fetches ETH spent on CIP-9 Fee subsidies https://snapshot.org/#/cow.eth/proposal/0x4bb9b614bdc4354856c4d0002ad0845b73b5290e5799013192cbc6491e6eea0e diff --git a/src/subgraph/ens_data.py b/src/subgraph/ens_data.py index 343e9be..ed77e58 100644 --- a/src/subgraph/ens_data.py +++ b/src/subgraph/ens_data.py @@ -6,7 +6,7 @@ from dotenv import load_dotenv from web3 import Web3 -from src.constants import PUBLIC_RESOLVER_ABI +from src.constants import PUBLIC_RESOLVER_ABI, ETH_RPC from src.subgraph.fetch import execute_subgraph_query from src.utils import partition_array @@ -23,12 +23,12 @@ } load_dotenv() -w3 = Web3(Web3.HTTPProvider(f"https://mainnet.infura.io/v3/{os.environ['INFURA_KEY']}")) +w3 = Web3(Web3.HTTPProvider(ETH_RPC)) def read_ens_text(resolver: str, node: str, key: str) -> str: resolver_contract = w3.eth.contract( - address=Web3.toChecksumAddress(resolver), abi=PUBLIC_RESOLVER_ABI + address=Web3.to_checksum_address(resolver), abi=PUBLIC_RESOLVER_ABI ) text: str = resolver_contract.caller.text(node, key) @@ -84,7 +84,6 @@ def get_result_page(wallets: list[str], skip: int, block: Optional[int] = None) subgraph_url="https://api.thegraph.com/subgraphs/name/ensdomains/ens", query=resolve_query(list(wallets), skip, block), ) - print(result_json) return result_json["data"]["domains"] diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_subgraph.py b/tests/test_subgraph.py index 71689a5..556ec98 100644 --- a/tests/test_subgraph.py +++ b/tests/test_subgraph.py @@ -28,7 +28,7 @@ def test_fetch_ens_data_small(self): } ], } - results = get_wallet_ens_data(set(expected_ens_records.keys()), 15687500) + results = get_wallet_ens_data(set(expected_ens_records.keys())) for wallet, data in expected_ens_records.items(): self.assertEqual(results[wallet], data, f"failed for wallet {wallet}") diff --git a/tests/test_utils.py b/tests/test_utils.py index afa0126..ffe1efd 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -5,8 +5,6 @@ import shutil from datetime import datetime -from duneapi.types import Network as LegacyDuneNetwork - from src.missing_tokens import Network from src.utils import partition_array, write_to_json, valid_date @@ -48,11 +46,6 @@ def test_network_class(self): self.assertEqual(gnosis.chain_id, 100) self.assertEqual(mainnet.chain_id, 1) - self.assertEqual(gnosis.node_url("FakeKey"), "https://rpc.gnosischain.com") - self.assertEqual( - mainnet.node_url("FakeKey"), "https://mainnet.infura.io/v3/FakeKey" - ) - self.assertEqual(gnosis.as_dune_v2_repr(), "gnosis") self.assertEqual(mainnet.as_dune_v2_repr(), "ethereum")