From 4b5a2c95e9ee553a2178a8a7de6dbbab343d5df7 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 8 Nov 2024 02:09:20 +0200 Subject: [PATCH 01/10] Add curve usdx pool events --- docker-compose.yml | 21 + .../arbitrum_mainnet/abi/CurveUSDx.json | 687 ++++++++++++++++++ .../arbitrum_mainnet/network_config.yaml | 13 +- .../@subsquid+squid-gen-evm+1.2.1.patch | 21 +- 4 files changed, 725 insertions(+), 17 deletions(-) create mode 100644 indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json diff --git a/docker-compose.yml b/docker-compose.yml index dc4e0c48d..58b6e0963 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,27 @@ services: volumes: - ./parquet-data:/parquet-data + arbitrum-mainnet-curve-usdx-indexer: + build: + context: ./indexers-v2 + dockerfile: Dockerfile + networks: + - data + depends_on: + - db + restart: always + environment: + DB_HOST: db + DB_PORT: 5432 + DB_NAME: arbitrum_mainnet + DB_PASS: $PG_PASSWORD + GQL_PORT: 4350 + RPC_ENDPOINT: wss://arbitrum-one-rpc.publicnode.com + NETWORK_NAME: arbitrum_mainnet + CONFIG_NAME: curve_usdx + volumes: + - ./parquet-data:/parquet-data + base-mainnet-indexer: build: context: ./indexers-v2 diff --git a/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json b/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json new file mode 100644 index 000000000..74233ccaf --- /dev/null +++ b/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json @@ -0,0 +1,687 @@ +{ + "address": "0x096a8865367686290639bc50bf8d85c0110d9fea", + "abi": [ + { + "name": "Transfer", + "inputs": [ + { "name": "sender", "type": "address", "indexed": true }, + { "name": "receiver", "type": "address", "indexed": true }, + { "name": "value", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "Approval", + "inputs": [ + { "name": "owner", "type": "address", "indexed": true }, + { "name": "spender", "type": "address", "indexed": true }, + { "name": "value", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "TokenExchange", + "inputs": [ + { "name": "buyer", "type": "address", "indexed": true }, + { "name": "sold_id", "type": "int128", "indexed": false }, + { "name": "tokens_sold", "type": "uint256", "indexed": false }, + { "name": "bought_id", "type": "int128", "indexed": false }, + { "name": "tokens_bought", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "TokenExchangeUnderlying", + "inputs": [ + { "name": "buyer", "type": "address", "indexed": true }, + { "name": "sold_id", "type": "int128", "indexed": false }, + { "name": "tokens_sold", "type": "uint256", "indexed": false }, + { "name": "bought_id", "type": "int128", "indexed": false }, + { "name": "tokens_bought", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "AddLiquidity", + "inputs": [ + { "name": "provider", "type": "address", "indexed": true }, + { "name": "token_amounts", "type": "uint256[]", "indexed": false }, + { "name": "fees", "type": "uint256[]", "indexed": false }, + { "name": "invariant", "type": "uint256", "indexed": false }, + { "name": "token_supply", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "RemoveLiquidity", + "inputs": [ + { "name": "provider", "type": "address", "indexed": true }, + { "name": "token_amounts", "type": "uint256[]", "indexed": false }, + { "name": "fees", "type": "uint256[]", "indexed": false }, + { "name": "token_supply", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "RemoveLiquidityOne", + "inputs": [ + { "name": "provider", "type": "address", "indexed": true }, + { "name": "token_id", "type": "int128", "indexed": false }, + { "name": "token_amount", "type": "uint256", "indexed": false }, + { "name": "coin_amount", "type": "uint256", "indexed": false }, + { "name": "token_supply", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "RemoveLiquidityImbalance", + "inputs": [ + { "name": "provider", "type": "address", "indexed": true }, + { "name": "token_amounts", "type": "uint256[]", "indexed": false }, + { "name": "fees", "type": "uint256[]", "indexed": false }, + { "name": "invariant", "type": "uint256", "indexed": false }, + { "name": "token_supply", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "RampA", + "inputs": [ + { "name": "old_A", "type": "uint256", "indexed": false }, + { "name": "new_A", "type": "uint256", "indexed": false }, + { "name": "initial_time", "type": "uint256", "indexed": false }, + { "name": "future_time", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "StopRampA", + "inputs": [ + { "name": "A", "type": "uint256", "indexed": false }, + { "name": "t", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "ApplyNewFee", + "inputs": [ + { "name": "fee", "type": "uint256", "indexed": false }, + { + "name": "offpeg_fee_multiplier", + "type": "uint256", + "indexed": false + } + ], + "anonymous": false, + "type": "event" + }, + { + "name": "SetNewMATime", + "inputs": [ + { "name": "ma_exp_time", "type": "uint256", "indexed": false }, + { "name": "D_ma_time", "type": "uint256", "indexed": false } + ], + "anonymous": false, + "type": "event" + }, + { + "stateMutability": "nonpayable", + "type": "constructor", + "inputs": [ + { "name": "_name", "type": "string" }, + { "name": "_symbol", "type": "string" }, + { "name": "_A", "type": "uint256" }, + { "name": "_fee", "type": "uint256" }, + { "name": "_offpeg_fee_multiplier", "type": "uint256" }, + { "name": "_ma_exp_time", "type": "uint256" }, + { "name": "_coins", "type": "address[]" }, + { "name": "_rate_multipliers", "type": "uint256[]" }, + { "name": "_asset_types", "type": "uint8[]" }, + { "name": "_method_ids", "type": "bytes4[]" }, + { "name": "_oracles", "type": "address[]" } + ], + "outputs": [] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "exchange", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "_dx", "type": "uint256" }, + { "name": "_min_dy", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "exchange", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "_dx", "type": "uint256" }, + { "name": "_min_dy", "type": "uint256" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "exchange_received", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "_dx", "type": "uint256" }, + { "name": "_min_dy", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "exchange_received", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "_dx", "type": "uint256" }, + { "name": "_min_dy", "type": "uint256" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "add_liquidity", + "inputs": [ + { "name": "_amounts", "type": "uint256[]" }, + { "name": "_min_mint_amount", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "add_liquidity", + "inputs": [ + { "name": "_amounts", "type": "uint256[]" }, + { "name": "_min_mint_amount", "type": "uint256" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity_one_coin", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "i", "type": "int128" }, + { "name": "_min_received", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity_one_coin", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "i", "type": "int128" }, + { "name": "_min_received", "type": "uint256" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity_imbalance", + "inputs": [ + { "name": "_amounts", "type": "uint256[]" }, + { "name": "_max_burn_amount", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity_imbalance", + "inputs": [ + { "name": "_amounts", "type": "uint256[]" }, + { "name": "_max_burn_amount", "type": "uint256" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "_min_amounts", "type": "uint256[]" } + ], + "outputs": [{ "name": "", "type": "uint256[]" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "_min_amounts", "type": "uint256[]" }, + { "name": "_receiver", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256[]" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "remove_liquidity", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "_min_amounts", "type": "uint256[]" }, + { "name": "_receiver", "type": "address" }, + { "name": "_claim_admin_fees", "type": "bool" } + ], + "outputs": [{ "name": "", "type": "uint256[]" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "withdraw_admin_fees", + "inputs": [], + "outputs": [] + }, + { + "stateMutability": "view", + "type": "function", + "name": "last_price", + "inputs": [{ "name": "i", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "ema_price", + "inputs": [{ "name": "i", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "get_p", + "inputs": [{ "name": "i", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "price_oracle", + "inputs": [{ "name": "i", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "D_oracle", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "transfer", + "inputs": [ + { "name": "_to", "type": "address" }, + { "name": "_value", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "bool" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "transferFrom", + "inputs": [ + { "name": "_from", "type": "address" }, + { "name": "_to", "type": "address" }, + { "name": "_value", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "bool" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "approve", + "inputs": [ + { "name": "_spender", "type": "address" }, + { "name": "_value", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "bool" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "permit", + "inputs": [ + { "name": "_owner", "type": "address" }, + { "name": "_spender", "type": "address" }, + { "name": "_value", "type": "uint256" }, + { "name": "_deadline", "type": "uint256" }, + { "name": "_v", "type": "uint8" }, + { "name": "_r", "type": "bytes32" }, + { "name": "_s", "type": "bytes32" } + ], + "outputs": [{ "name": "", "type": "bool" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "DOMAIN_SEPARATOR", + "inputs": [], + "outputs": [{ "name": "", "type": "bytes32" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "get_dx", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "dy", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "get_dy", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" }, + { "name": "dx", "type": "uint256" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "calc_withdraw_one_coin", + "inputs": [ + { "name": "_burn_amount", "type": "uint256" }, + { "name": "i", "type": "int128" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "totalSupply", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "get_virtual_price", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "calc_token_amount", + "inputs": [ + { "name": "_amounts", "type": "uint256[]" }, + { "name": "_is_deposit", "type": "bool" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "A", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "A_precise", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "balances", + "inputs": [{ "name": "i", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "get_balances", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256[]" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "stored_rates", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256[]" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "dynamic_fee", + "inputs": [ + { "name": "i", "type": "int128" }, + { "name": "j", "type": "int128" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "ramp_A", + "inputs": [ + { "name": "_future_A", "type": "uint256" }, + { "name": "_future_time", "type": "uint256" } + ], + "outputs": [] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "stop_ramp_A", + "inputs": [], + "outputs": [] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "set_new_fee", + "inputs": [ + { "name": "_new_fee", "type": "uint256" }, + { "name": "_new_offpeg_fee_multiplier", "type": "uint256" } + ], + "outputs": [] + }, + { + "stateMutability": "nonpayable", + "type": "function", + "name": "set_ma_exp_time", + "inputs": [ + { "name": "_ma_exp_time", "type": "uint256" }, + { "name": "_D_ma_time", "type": "uint256" } + ], + "outputs": [] + }, + { + "stateMutability": "view", + "type": "function", + "name": "N_COINS", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "coins", + "inputs": [{ "name": "arg0", "type": "uint256" }], + "outputs": [{ "name": "", "type": "address" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "fee", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "offpeg_fee_multiplier", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "admin_fee", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "initial_A", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "future_A", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "initial_A_time", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "future_A_time", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "admin_balances", + "inputs": [{ "name": "arg0", "type": "uint256" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "ma_exp_time", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "D_ma_time", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "ma_last_time", + "inputs": [], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "name", + "inputs": [], + "outputs": [{ "name": "", "type": "string" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "symbol", + "inputs": [], + "outputs": [{ "name": "", "type": "string" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "decimals", + "inputs": [], + "outputs": [{ "name": "", "type": "uint8" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "version", + "inputs": [], + "outputs": [{ "name": "", "type": "string" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "balanceOf", + "inputs": [{ "name": "arg0", "type": "address" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "allowance", + "inputs": [ + { "name": "arg0", "type": "address" }, + { "name": "arg1", "type": "address" } + ], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "nonces", + "inputs": [{ "name": "arg0", "type": "address" }], + "outputs": [{ "name": "", "type": "uint256" }] + }, + { + "stateMutability": "view", + "type": "function", + "name": "salt", + "inputs": [], + "outputs": [{ "name": "", "type": "bytes32" }] + } + ] +} diff --git a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml index d16149b17..32f608722 100644 --- a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml +++ b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml @@ -20,4 +20,15 @@ configs: - name: PerpsMarketProxy package: perpsFactory - name: PerpsAccountProxy - package: perpsFactory \ No newline at end of file + package: perpsFactory + + curve_usdx: + target: + type: parquet + path: "/parquet-data/arbitrum_mainnet/curve_usdx" + range: + from: 236000000 + to: latest + contracts_from_abi: + - name: CurveUSDx + abi: ./abi/CurveUSDx.json diff --git a/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch b/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch index 5dfff0e60..a9fbace85 100644 --- a/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch +++ b/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch @@ -1,27 +1,16 @@ diff --git a/node_modules/@subsquid/squid-gen-evm/lib/core.js b/node_modules/@subsquid/squid-gen-evm/lib/core.js -index 2839b6f..823fce0 100644 +index 2839b6f..902f4eb 100644 --- a/node_modules/@subsquid/squid-gen-evm/lib/core.js +++ b/node_modules/@subsquid/squid-gen-evm/lib/core.js -@@ -28,6 +28,11 @@ class CoreCodegen { - this.printImports(); - this.out.line(); - let targetPrinter = this.getTargetPrinter(); -+ this.out.line(`import { loadNetworkConfig } from './config'`); -+ this.out.line(`const networkName = process.env.NETWORK_NAME || 'arbitrum_mainnet'`); -+ this.out.line(`const configName = process.env.CONFIG_NAME || 'default'`); -+ this.out.line(`const config = loadNetworkConfig(networkName)`); -+ this.out.line(`const rangeTo = config.configs[configName].range.to`); - this.out.line(`processor.run(db, async (ctx) => {`); - this.out.indentation(() => { - targetPrinter.printPreBatch(); -@@ -38,6 +43,13 @@ class CoreCodegen { +@@ -38,6 +38,14 @@ class CoreCodegen { `new Date(block.header.timestamp)`, ]); this.out.line(); -+ this.out.block(`if (typeof rangeTo !== 'undefined' && block.header.height >= Number(rangeTo))`, () => { ++ let rangeTo = this.options.contracts[0].range.to; ++ this.out.block(`if (typeof ${rangeTo} !== 'undefined' && block.header.height >= Number(${rangeTo}))`, () => { + this.out.line(`ctx.store.setForceFlush(true)`); + }); -+ this.out.block(`else if (typeof rangeTo === 'undefined')`, () => { ++ this.out.block(`else if (typeof ${rangeTo} === 'undefined')`, () => { + this.out.line(`ctx.store.setForceFlush(true)`); + }); + this.out.line(); From b2aa34e416ceb0f51cec3059b5e91eca2ba77d0e Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Mon, 11 Nov 2024 18:49:09 +0200 Subject: [PATCH 02/10] Remove target from config --- indexers-v2/networks/arbitrum_mainnet/network_config.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml index c652b4730..a2f3f795d 100644 --- a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml +++ b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml @@ -21,9 +21,6 @@ configs: package: perpsFactory curve_usdx: - target: - type: parquet - path: "/parquet-data/arbitrum_mainnet/curve_usdx" range: from: 236000000 to: latest From 7e8a5ceea3df8265ba1197e476988508097a3beb Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Mon, 11 Nov 2024 21:15:42 +0200 Subject: [PATCH 03/10] Rename configs --- .../arbitrum_mainnet/network_config.yaml | 2 +- .../networks/base_mainnet/network_config.yaml | 31 +------------------ 2 files changed, 2 insertions(+), 31 deletions(-) diff --git a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml index 8c1fa8aae..ebd0e6a73 100644 --- a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml +++ b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml @@ -4,7 +4,7 @@ network: archive_url: "https://v2.archive.subsquid.io/network/arbitrum-one" configs: - default: + synthetix-v3: range: from: 218000000 to: latest diff --git a/indexers-v2/networks/base_mainnet/network_config.yaml b/indexers-v2/networks/base_mainnet/network_config.yaml index 2f6fde632..74ca310d6 100644 --- a/indexers-v2/networks/base_mainnet/network_config.yaml +++ b/indexers-v2/networks/base_mainnet/network_config.yaml @@ -4,7 +4,7 @@ network: archive_url: "https://v2.archive.subsquid.io/network/base-mainnet" configs: - default: + synthetix-v3: range: from: 7500000 to: latest @@ -28,35 +28,6 @@ configs: abi: ./abi/PerpsMarketProxyLegacy.json - name: BuybackSnxLegacy abi: ./abi/BuybackSnxLegacy.json - cannon_config: - package: "synthetix-omnibus" - version: "latest" - preset: "andromeda" - - base_mainnet_parquet: - range: - from: 20000000 - to: 20200000 - contracts_from_sdk: - - name: CoreProxy - package: system - - name: AccountProxy - package: system - - name: SpotMarketProxy - package: spotFactory - - name: PerpsMarketProxy - package: perpsFactory - - name: PerpsAccountProxy - package: perpsFactory - - name: buyback_snx - package: buyback_snx - contracts_from_abi: - - name: CoreProxyLegacy - abi: ./abi/CoreProxyLegacy.json - - name: PerpsMarketProxyLegacy - abi: ./abi/PerpsMarketProxyLegacy.json - - name: BuybackSnxLegacy - abi: ./abi/BuybackSnxLegacy.json cannon_config: package: "synthetix-omnibus" version: "latest" From f7f9b95e15dac5811c55038b60043b52bfb86468 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Mon, 11 Nov 2024 21:19:55 +0200 Subject: [PATCH 04/10] Remove address from abi --- indexers-v2/main.py | 10 +++++----- .../networks/base_mainnet/abi/BuybackSnxLegacy.json | 9 +++------ .../networks/base_mainnet/abi/CoreProxyLegacy.json | 9 +++------ .../base_mainnet/abi/PerpsMarketProxyLegacy.json | 9 +++------ indexers-v2/networks/base_mainnet/network_config.yaml | 3 +++ 5 files changed, 17 insertions(+), 23 deletions(-) diff --git a/indexers-v2/main.py b/indexers-v2/main.py index 4bb303c6a..067686f29 100644 --- a/indexers-v2/main.py +++ b/indexers-v2/main.py @@ -88,10 +88,9 @@ def load_network_config(path): config_file = load_network_config(path) # Get config name - if args.config_name is None: - config_name = "default" - else: - config_name = args.config_name + config_name = args.config_name + if config_name is None: + raise Exception("Config name is required") # Load shared network-level details network_params = config_file["network"] @@ -141,10 +140,11 @@ def load_network_config(path): contracts_from_abi = custom_config["contracts_from_abi"] for contract in contracts_from_abi: name = contract["name"] + address = contract["address"] with open(f"{path}/abi/{name}.json", "r") as file: contract_data = json.load(file) save_abi(contract_data["abi"], name) - contracts.append({"name": name, "address": contract_data["address"]}) + contracts.append({"name": name, "address": address}) else: message = "No contracts found in network config" raise Exception(message) diff --git a/indexers-v2/networks/base_mainnet/abi/BuybackSnxLegacy.json b/indexers-v2/networks/base_mainnet/abi/BuybackSnxLegacy.json index f974ebc06..8e27fe5da 100644 --- a/indexers-v2/networks/base_mainnet/abi/BuybackSnxLegacy.json +++ b/indexers-v2/networks/base_mainnet/abi/BuybackSnxLegacy.json @@ -1,6 +1,4 @@ -{ - "address": "0x53f1E640C058337a12D036265681bC172e6fB962", - "abi": [ +[ { "inputs": [ { @@ -179,6 +177,5 @@ ], "stateMutability": "view", "type": "function" - } - ] -} + } +] diff --git a/indexers-v2/networks/base_mainnet/abi/CoreProxyLegacy.json b/indexers-v2/networks/base_mainnet/abi/CoreProxyLegacy.json index 3b73c2eda..6b59a954a 100644 --- a/indexers-v2/networks/base_mainnet/abi/CoreProxyLegacy.json +++ b/indexers-v2/networks/base_mainnet/abi/CoreProxyLegacy.json @@ -1,6 +1,4 @@ -{ - "address": "0x32C222A9A159782aFD7529c87FA34b96CA72C696", - "abi": [ +[ { "anonymous": false, "inputs": [ @@ -110,6 +108,5 @@ ], "name": "MarketUsdWithdrawn", "type": "event" - } - ] -} + } +] diff --git a/indexers-v2/networks/base_mainnet/abi/PerpsMarketProxyLegacy.json b/indexers-v2/networks/base_mainnet/abi/PerpsMarketProxyLegacy.json index e54435031..646024862 100644 --- a/indexers-v2/networks/base_mainnet/abi/PerpsMarketProxyLegacy.json +++ b/indexers-v2/networks/base_mainnet/abi/PerpsMarketProxyLegacy.json @@ -1,6 +1,4 @@ -{ - "address": "0x0A2AF931eFFd34b81ebcc57E3d3c9B1E1dE1C9Ce", - "abi": [ +[ { "anonymous": false, "inputs": [ @@ -116,6 +114,5 @@ ], "name": "MarketUpdated", "type": "event" - } - ] -} + } +] diff --git a/indexers-v2/networks/base_mainnet/network_config.yaml b/indexers-v2/networks/base_mainnet/network_config.yaml index 74ca310d6..3514dd398 100644 --- a/indexers-v2/networks/base_mainnet/network_config.yaml +++ b/indexers-v2/networks/base_mainnet/network_config.yaml @@ -23,10 +23,13 @@ configs: package: buyback_snx contracts_from_abi: - name: CoreProxyLegacy + address: 0x32C222A9A159782aFD7529c87FA34b96CA72C696 abi: ./abi/CoreProxyLegacy.json - name: PerpsMarketProxyLegacy + address: 0x0A2AF931eFFd34b81ebcc57E3d3c9B1E1dE1C9Ce abi: ./abi/PerpsMarketProxyLegacy.json - name: BuybackSnxLegacy + address: 0x53f1E640C058337a12D036265681bC172e6fB962 abi: ./abi/BuybackSnxLegacy.json cannon_config: package: "synthetix-omnibus" From c8ad04af1433a7a283160451fd59189fd04527e8 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 00:15:16 +0200 Subject: [PATCH 05/10] Use env file --- Makefile | 4 ++++ docker-compose.yml | 34 ++++----------------------------- indexers-v2/entrypoint.sh | 2 +- indexers-v2/main.py | 37 ++++++++++++++++++++++++------------ indexers-v2/requirements.txt | 3 ++- 5 files changed, 36 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index d4cb44920..09b61b84a 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,10 @@ extract: docker compose run extractors python main.py configs/arbitrum_mainnet.yaml docker compose run extractors python main.py configs/arbitrum_sepolia.yaml +index: + docker compose run indexers-v2 --network_name base_mainnet --config_name synthetix-v3 + docker compose run indexers-v2 --network_name arbitrum_mainnet --config_name synthetix-v3 + synths: docker compose run transformer python scripts/get_synths.py diff --git a/docker-compose.yml b/docker-compose.yml index 658a491fd..f27172a0b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,44 +29,18 @@ services: tmpfs: - /dev/shm:size=4g - arbitrum-mainnet-indexer: + indexer-v2: build: context: ./indexers-v2 dockerfile: Dockerfile networks: - data - depends_on: - - db - restart: always - environment: - DB_HOST: db - DB_PORT: 5432 - DB_NAME: arbitrum_mainnet - DB_PASS: $PG_PASSWORD - GQL_PORT: 4350 - RPC_ENDPOINT: wss://arbitrum-one-rpc.publicnode.com - NETWORK_NAME: arbitrum_mainnet - volumes: - - ./parquet-data:/parquet-data - - base-mainnet-indexer: - build: - context: ./indexers-v2 - dockerfile: Dockerfile - networks: - - data - depends_on: - - db restart: always + env_file: + - .env environment: - DB_HOST: db - DB_PORT: 5432 - DB_NAME: base_mainnet - DB_PASS: $PG_PASSWORD - GQL_PORT: 4350 - RPC_ENDPOINT: https://mainnet.base.org NETWORK_NAME: base_mainnet - CONFIG_NAME: base_mainnet_parquet + CONFIG_NAME: synthetix-v3 volumes: - ./parquet-data:/parquet-data diff --git a/indexers-v2/entrypoint.sh b/indexers-v2/entrypoint.sh index d4691a5a1..63ec356c8 100644 --- a/indexers-v2/entrypoint.sh +++ b/indexers-v2/entrypoint.sh @@ -4,7 +4,7 @@ set -e # Get contract data from SDK and generate squidgen.yaml and squid.yaml -python3 main.py --network_name "$NETWORK_NAME" --rpc_endpoint "$RPC_ENDPOINT" --config_name "$CONFIG_NAME" +python3 main.py --network_name $NETWORK_NAME --config_name $CONFIG_NAME "$@" # Generate squid processor npm run generate:processor diff --git a/indexers-v2/main.py b/indexers-v2/main.py index 067686f29..7b07fbec1 100644 --- a/indexers-v2/main.py +++ b/indexers-v2/main.py @@ -1,9 +1,13 @@ import json import os import argparse +from dotenv import load_dotenv import yaml from synthetix import Synthetix +# load environment variables +load_dotenv() + def save_abi(abi, contract_name): os.makedirs("abi", exist_ok=True) @@ -73,31 +77,34 @@ def load_network_config(path): "--config_name", type=str, help="Name of the configuration to use", + required=True, + ) + parser.add_argument( + "--contract_names", + type=str, + help="Comma-separated list of contract names to index.", ) - parser.add_argument("--rpc_endpoint", type=str, help="RPC URL", required=True) args = parser.parse_args() - rpc_endpoint = args.rpc_endpoint - if rpc_endpoint is None: - message = "RPC_ENDPOINT environment variable is not set" - raise Exception(message) - - # Load config file for network network_name = args.network_name + config_name = args.config_name + contract_names = args.contract_names + + # Get contract names + if contract_names is not None: + parsed_contract_names = [name.strip() for name in contract_names.split(",")] + + # Load network config path = f"networks/{network_name}" config_file = load_network_config(path) - # Get config name - config_name = args.config_name - if config_name is None: - raise Exception("Config name is required") - # Load shared network-level details network_params = config_file["network"] if network_params is None: message = f"Network '{network_name}' not found in {path}/network_config.yaml" raise Exception(message) network_id = network_params["network_id"] + rpc_endpoint = os.getenv(f"NETWORK_{network_id}_RPC") archive_url = network_params.get("archive_url", "None") # Load custom config @@ -131,6 +138,9 @@ def load_network_config(path): if "contracts_from_sdk" in custom_config: contracts_from_sdk = custom_config["contracts_from_sdk"] for contract in contracts_from_sdk: + if contract_names is not None: + if contract["name"] not in parsed_contract_names: + continue name = contract["name"] package = contract["package"] contract_data = snx.contracts[package][name] @@ -139,6 +149,9 @@ def load_network_config(path): elif "contracts_from_abi" in custom_config: contracts_from_abi = custom_config["contracts_from_abi"] for contract in contracts_from_abi: + if contract_names is not None: + if contract["name"] not in parsed_contract_names: + continue name = contract["name"] address = contract["address"] with open(f"{path}/abi/{name}.json", "r") as file: diff --git a/indexers-v2/requirements.txt b/indexers-v2/requirements.txt index 5f446a2cc..dcd9863a9 100644 --- a/indexers-v2/requirements.txt +++ b/indexers-v2/requirements.txt @@ -1,2 +1,3 @@ synthetix==0.1.21 -PyYAML \ No newline at end of file +PyYAML +python-dotenv \ No newline at end of file From 20a11fcc21718ea885b0fd46c4bf126da5cd936c Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 00:21:53 +0200 Subject: [PATCH 06/10] Update config --- indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json | 9 +++------ .../networks/arbitrum_mainnet/network_config.yaml | 1 + 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json b/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json index 74233ccaf..47d8c1b59 100644 --- a/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json +++ b/indexers-v2/networks/arbitrum_mainnet/abi/CurveUSDx.json @@ -1,7 +1,5 @@ -{ - "address": "0x096a8865367686290639bc50bf8d85c0110d9fea", - "abi": [ - { +[ + { "name": "Transfer", "inputs": [ { "name": "sender", "type": "address", "indexed": true }, @@ -683,5 +681,4 @@ "inputs": [], "outputs": [{ "name": "", "type": "bytes32" }] } - ] -} +] diff --git a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml index 14c769d10..92a033c4b 100644 --- a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml +++ b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml @@ -26,4 +26,5 @@ configs: to: latest contracts_from_abi: - name: CurveUSDx + address: 0x096a8865367686290639bc50bf8d85c0110d9fea abi: ./abi/CurveUSDx.json From c0065f1d18e4581ff835cedd43c4a733075af1f0 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 00:26:40 +0200 Subject: [PATCH 07/10] Simplify patch --- .../@subsquid+squid-gen-evm+1.2.1.patch | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch b/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch index 5dfff0e60..a9fbace85 100644 --- a/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch +++ b/indexers-v2/patches/@subsquid+squid-gen-evm+1.2.1.patch @@ -1,27 +1,16 @@ diff --git a/node_modules/@subsquid/squid-gen-evm/lib/core.js b/node_modules/@subsquid/squid-gen-evm/lib/core.js -index 2839b6f..823fce0 100644 +index 2839b6f..902f4eb 100644 --- a/node_modules/@subsquid/squid-gen-evm/lib/core.js +++ b/node_modules/@subsquid/squid-gen-evm/lib/core.js -@@ -28,6 +28,11 @@ class CoreCodegen { - this.printImports(); - this.out.line(); - let targetPrinter = this.getTargetPrinter(); -+ this.out.line(`import { loadNetworkConfig } from './config'`); -+ this.out.line(`const networkName = process.env.NETWORK_NAME || 'arbitrum_mainnet'`); -+ this.out.line(`const configName = process.env.CONFIG_NAME || 'default'`); -+ this.out.line(`const config = loadNetworkConfig(networkName)`); -+ this.out.line(`const rangeTo = config.configs[configName].range.to`); - this.out.line(`processor.run(db, async (ctx) => {`); - this.out.indentation(() => { - targetPrinter.printPreBatch(); -@@ -38,6 +43,13 @@ class CoreCodegen { +@@ -38,6 +38,14 @@ class CoreCodegen { `new Date(block.header.timestamp)`, ]); this.out.line(); -+ this.out.block(`if (typeof rangeTo !== 'undefined' && block.header.height >= Number(rangeTo))`, () => { ++ let rangeTo = this.options.contracts[0].range.to; ++ this.out.block(`if (typeof ${rangeTo} !== 'undefined' && block.header.height >= Number(${rangeTo}))`, () => { + this.out.line(`ctx.store.setForceFlush(true)`); + }); -+ this.out.block(`else if (typeof rangeTo === 'undefined')`, () => { ++ this.out.block(`else if (typeof ${rangeTo} === 'undefined')`, () => { + this.out.line(`ctx.store.setForceFlush(true)`); + }); + this.out.line(); From 237f61f41fae28c96e6f474f890128b421be8aae Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 01:08:37 +0200 Subject: [PATCH 08/10] Stringify everything --- indexers-v2/networks/arbitrum_mainnet/network_config.yaml | 2 +- indexers-v2/networks/base_mainnet/network_config.yaml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml index 92a033c4b..6e656d3d2 100644 --- a/indexers-v2/networks/arbitrum_mainnet/network_config.yaml +++ b/indexers-v2/networks/arbitrum_mainnet/network_config.yaml @@ -26,5 +26,5 @@ configs: to: latest contracts_from_abi: - name: CurveUSDx - address: 0x096a8865367686290639bc50bf8d85c0110d9fea + address: '0x096A8865367686290639bc50bF8D85C0110d9Fea' abi: ./abi/CurveUSDx.json diff --git a/indexers-v2/networks/base_mainnet/network_config.yaml b/indexers-v2/networks/base_mainnet/network_config.yaml index 3514dd398..d6f537d7f 100644 --- a/indexers-v2/networks/base_mainnet/network_config.yaml +++ b/indexers-v2/networks/base_mainnet/network_config.yaml @@ -23,13 +23,13 @@ configs: package: buyback_snx contracts_from_abi: - name: CoreProxyLegacy - address: 0x32C222A9A159782aFD7529c87FA34b96CA72C696 + address: '0x32C222A9A159782aFD7529c87FA34b96CA72C696' abi: ./abi/CoreProxyLegacy.json - name: PerpsMarketProxyLegacy - address: 0x0A2AF931eFFd34b81ebcc57E3d3c9B1E1dE1C9Ce + address: '0x0A2AF931eFFd34b81ebcc57E3d3c9B1E1dE1C9Ce' abi: ./abi/PerpsMarketProxyLegacy.json - name: BuybackSnxLegacy - address: 0x53f1E640C058337a12D036265681bC172e6fB962 + address: '0x53f1E640C058337a12D036265681bC172e6fB962' abi: ./abi/BuybackSnxLegacy.json cannon_config: package: "synthetix-omnibus" From 2567a490295aaa1a178b2513d72dadc7aa5fa62e Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 01:09:05 +0200 Subject: [PATCH 09/10] Move address to config --- indexers-v2/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/indexers-v2/main.py b/indexers-v2/main.py index 7b07fbec1..0f74d7508 100644 --- a/indexers-v2/main.py +++ b/indexers-v2/main.py @@ -144,8 +144,10 @@ def load_network_config(path): name = contract["name"] package = contract["package"] contract_data = snx.contracts[package][name] - save_abi(contract_data["abi"], name) - contracts.append({"name": name, "address": contract_data["address"]}) + abi = contract_data["abi"] + address = contract_data["address"] + save_abi(abi, name) + contracts.append({"name": name, "address": address}) elif "contracts_from_abi" in custom_config: contracts_from_abi = custom_config["contracts_from_abi"] for contract in contracts_from_abi: @@ -153,11 +155,10 @@ def load_network_config(path): if contract["name"] not in parsed_contract_names: continue name = contract["name"] - address = contract["address"] with open(f"{path}/abi/{name}.json", "r") as file: - contract_data = json.load(file) - save_abi(contract_data["abi"], name) - contracts.append({"name": name, "address": address}) + abi = json.load(file) + save_abi(abi, name) + contracts.append({"name": name, "address": contract["address"]}) else: message = "No contracts found in network config" raise Exception(message) From af3cce6055a56a5f8100b0c378a053d902788ad9 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Tue, 12 Nov 2024 02:28:57 +0200 Subject: [PATCH 10/10] Fix destination path --- indexers-v2/main.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/indexers-v2/main.py b/indexers-v2/main.py index 0f74d7508..e874ebf22 100644 --- a/indexers-v2/main.py +++ b/indexers-v2/main.py @@ -16,13 +16,22 @@ def save_abi(abi, contract_name): def create_squidgen_config( - rpc_url, archive_url, contracts_info, block_range, config_name, rate_limit=10 + rpc_url, + archive_url, + network_name, + contracts_info, + block_range, + config_name, + rate_limit=10, ): config = { "archive": archive_url, "finalityConfirmation": 1, "chain": {"url": rpc_url, "rateLimit": rate_limit}, - "target": {"type": "parquet", "path": f"/parquet-data/{config_name}"}, + "target": { + "type": "parquet", + "path": f"/parquet-data/{network_name}/{config_name}", + }, "contracts": [], } @@ -168,6 +177,7 @@ def load_network_config(path): squidgen_config = create_squidgen_config( rpc_endpoint, archive_url, + network_name, contracts, block_range, config_name,