From d0c1eccce53bd7d9f418ff95271f7ceca1d7ea37 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 20 Dec 2024 19:38:55 +0200 Subject: [PATCH 1/6] Update makefile & docker compose config --- Makefile | 18 +++++++++----- docker-compose.indexers.yml | 47 +++++++++++++++++++++++++++++++++++++ docker-compose.yml | 27 --------------------- 3 files changed, 59 insertions(+), 33 deletions(-) create mode 100644 docker-compose.indexers.yml diff --git a/Makefile b/Makefile index 6bfc0cf5..96be62a2 100644 --- a/Makefile +++ b/Makefile @@ -14,20 +14,26 @@ extract: docker compose run extractors uv run python main.py configs/arbitrum_sepolia.yaml index: - docker compose run indexer --network_name base_mainnet --protocol_name synthetix - docker compose run indexer --network_name arbitrum_mainnet --protocol_name synthetix + docker compose -f docker-compose.indexers.yml up -d indexer-arbitrum-mainnet-synthetix + docker compose -f docker-compose.indexers.yml up -d indexer-base-mainnet-synthetix + +ingest: + docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/clean_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/clean_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/import_parquet.py + docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/import_parquet.py synths: docker compose run transformer python scripts/get_synths.py dbt: build - docker compose run transformer dbt run --target prod --profiles-dir profiles --profile synthetix + docker compose run transformer dbt run --target prod --profiles-dir profiles --profile clickhouse seed-prod: build - docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile synthetix + docker compose run transformer dbt seed --target prod --profiles-dir profiles --profile clickhouse seed-dev: build - docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile synthetix + docker compose run transformer dbt seed --target dev --profiles-dir profiles --profile clickhouse dbt-op: build - docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile synthetix + docker compose run transformer dbt run --target prod-op --profiles-dir profiles --profile clickhouse diff --git a/docker-compose.indexers.yml b/docker-compose.indexers.yml new file mode 100644 index 00000000..252d2d3b --- /dev/null +++ b/docker-compose.indexers.yml @@ -0,0 +1,47 @@ +services: + indexer-arbitrum-mainnet-synthetix: + build: + context: ./indexers + dockerfile: Dockerfile + restart: always + networks: + - data + env_file: + - .env + environment: + - NETWORK_NAME=arbitrum_mainnet + - PROTOCOL_NAME=synthetix + volumes: + - ./parquet-data:/parquet-data + + indexer-base-mainnet-synthetix: + build: + context: ./indexers + dockerfile: Dockerfile + restart: always + networks: + - data + env_file: + - .env + environment: + - NETWORK_NAME=base_mainnet + - PROTOCOL_NAME=synthetix + volumes: + - ./parquet-data:/parquet-data + + indexer-listener: + build: + context: ./indexers + dockerfile: Dockerfile.listener + networks: + - data + restart: always + environment: + PYTHONUNBUFFERED: 1 + volumes: + - ./parquet-data:/parquet-data + +networks: + data: + external: + name: data_data diff --git a/docker-compose.yml b/docker-compose.yml index 039749e4..321e4c61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,33 +42,6 @@ services: cpus: "4.0" memory: 8192M - indexer: - build: - context: ./indexers - dockerfile: Dockerfile - networks: - - data - restart: always - env_file: - - .env - environment: - NETWORK_NAME: base_mainnet - PROTOCOL_NAME: synthetix - volumes: - - ./parquet-data:/parquet-data - - indexer-listener: - build: - context: ./indexers - dockerfile: Dockerfile.listener - networks: - - data - restart: always - environment: - PYTHONUNBUFFERED: 1 - volumes: - - ./parquet-data:/parquet-data - transformer: image: ghcr.io/synthetixio/data/transformer:${VERSION} build: From b99f116ad56ecdfb9ad1b7a06a4f0e847c3b975a Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 24 Jan 2025 14:48:28 +0200 Subject: [PATCH 2/6] Update indexer composer config with logs path --- docker-compose.indexers.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.indexers.yml b/docker-compose.indexers.yml index 252d2d3b..975e0fba 100644 --- a/docker-compose.indexers.yml +++ b/docker-compose.indexers.yml @@ -13,6 +13,7 @@ services: - PROTOCOL_NAME=synthetix volumes: - ./parquet-data:/parquet-data + - ./indexers/logs/indexer-gen/arbitrum-mainnet/synthetix:/app/logs indexer-base-mainnet-synthetix: build: @@ -28,6 +29,7 @@ services: - PROTOCOL_NAME=synthetix volumes: - ./parquet-data:/parquet-data + - ./indexers/logs/indexer-gen/base-mainnet/synthetix:/app/logs indexer-listener: build: @@ -40,6 +42,7 @@ services: PYTHONUNBUFFERED: 1 volumes: - ./parquet-data:/parquet-data + - ./indexers/logs/indexer-listener:/app/logs networks: data: From 9c94fffa604206051f270b70613519e9f5345079 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 24 Jan 2025 14:48:35 +0200 Subject: [PATCH 3/6] Update Makefile --- Makefile | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 96be62a2..64bf6277 100644 --- a/Makefile +++ b/Makefile @@ -13,16 +13,13 @@ extract: docker compose run extractors uv run python main.py configs/arbitrum_mainnet.yaml docker compose run extractors uv run python main.py configs/arbitrum_sepolia.yaml +indexer-listener: + docker compose -f docker-compose.indexers.yml up -d indexer-listener + index: docker compose -f docker-compose.indexers.yml up -d indexer-arbitrum-mainnet-synthetix docker compose -f docker-compose.indexers.yml up -d indexer-base-mainnet-synthetix -ingest: - docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/clean_parquet.py - docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/clean_parquet.py - docker compose -f docker-compose.indexers.yml run indexer-arbitrum-mainnet-synthetix uv run scripts/import_parquet.py - docker compose -f docker-compose.indexers.yml run indexer-base-mainnet-synthetix uv run scripts/import_parquet.py - synths: docker compose run transformer python scripts/get_synths.py From b51726d3fb2628fea3832891514be306b36c8f48 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 24 Jan 2025 15:03:42 +0200 Subject: [PATCH 4/6] Update docekr services --- Makefile | 15 ++++++++++----- docker-compose.indexers.yml | 17 ++--------------- docker-compose.yml | 14 ++++++++++++++ 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index 64bf6277..af3488f9 100644 --- a/Makefile +++ b/Makefile @@ -13,12 +13,17 @@ extract: docker compose run extractors uv run python main.py configs/arbitrum_mainnet.yaml docker compose run extractors uv run python main.py configs/arbitrum_sepolia.yaml -indexer-listener: - docker compose -f docker-compose.indexers.yml up -d indexer-listener +listener-start: + docker compose up -d indexer-listener -index: - docker compose -f docker-compose.indexers.yml up -d indexer-arbitrum-mainnet-synthetix - docker compose -f docker-compose.indexers.yml up -d indexer-base-mainnet-synthetix +listener-stop: + docker compose down indexer-listener + +index-start: + docker compose -f docker-compose.indexers.yml up -d + +index-stop: + docker compose -f docker-compose.indexers.yml down synths: docker compose run transformer python scripts/get_synths.py diff --git a/docker-compose.indexers.yml b/docker-compose.indexers.yml index 975e0fba..05480578 100644 --- a/docker-compose.indexers.yml +++ b/docker-compose.indexers.yml @@ -31,20 +31,7 @@ services: - ./parquet-data:/parquet-data - ./indexers/logs/indexer-gen/base-mainnet/synthetix:/app/logs - indexer-listener: - build: - context: ./indexers - dockerfile: Dockerfile.listener - networks: - - data - restart: always - environment: - PYTHONUNBUFFERED: 1 - volumes: - - ./parquet-data:/parquet-data - - ./indexers/logs/indexer-listener:/app/logs - networks: data: - external: - name: data_data + name: data_data + external: true diff --git a/docker-compose.yml b/docker-compose.yml index 76a0c6ec..7094d1a2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: limits: cpus: "4.0" memory: 8192M + db: image: ghcr.io/synthetixio/data/postgres:${VERSION} restart: always @@ -42,6 +43,19 @@ services: cpus: "4.0" memory: 8192M + indexer-listener: + build: + context: ./indexers + dockerfile: Dockerfile.listener + networks: + - data + restart: always + environment: + PYTHONUNBUFFERED: 1 + volumes: + - ./parquet-data:/parquet-data + - ./indexers/logs/indexer-listener:/app/logs + transformer: image: ghcr.io/synthetixio/data/transformer:${VERSION} build: From b93a2c3d165b49ae3065464f5d07866ebfc25e37 Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 24 Jan 2025 19:39:06 +0200 Subject: [PATCH 5/6] Replace arbitrum indexer with eth --- docker-compose.indexers.yml | 6 +++--- indexers/networks/eth_mainnet/network_config.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docker-compose.indexers.yml b/docker-compose.indexers.yml index 05480578..c7efc888 100644 --- a/docker-compose.indexers.yml +++ b/docker-compose.indexers.yml @@ -1,5 +1,5 @@ services: - indexer-arbitrum-mainnet-synthetix: + indexer-eth-mainnet-synthetix: build: context: ./indexers dockerfile: Dockerfile @@ -9,11 +9,11 @@ services: env_file: - .env environment: - - NETWORK_NAME=arbitrum_mainnet + - NETWORK_NAME=eth_mainnet - PROTOCOL_NAME=synthetix volumes: - ./parquet-data:/parquet-data - - ./indexers/logs/indexer-gen/arbitrum-mainnet/synthetix:/app/logs + - ./indexers/logs/indexer-gen/eth-mainnet/synthetix:/app/logs indexer-base-mainnet-synthetix: build: diff --git a/indexers/networks/eth_mainnet/network_config.yaml b/indexers/networks/eth_mainnet/network_config.yaml index 9919abd8..207c8956 100644 --- a/indexers/networks/eth_mainnet/network_config.yaml +++ b/indexers/networks/eth_mainnet/network_config.yaml @@ -3,7 +3,7 @@ network: network_id: 1 archive_url: "https://v2.archive.subsquid.io/network/ethereum-mainnet" -configs: +protocols: synthetix: range: from: 20000000 @@ -17,7 +17,7 @@ configs: contracts_from_abi: - name: LegacyMarketProxy address: "0x3AcF163B9E6a384D539e10dAc7e11213c638b2f5" - abi: ./abi/LegacyMarketProxy.json + abi: LegacyMarketProxy.json cannon_config: package: "synthetix-omnibus" version: "latest" From db2ca82057c2ccbdf1e36edebe670a431360e38e Mon Sep 17 00:00:00 2001 From: marcus-snx Date: Fri, 24 Jan 2025 19:39:55 +0200 Subject: [PATCH 6/6] Update Makefile Co-authored-by: troy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index af3488f9..bbf4a0f4 100644 --- a/Makefile +++ b/Makefile @@ -26,7 +26,7 @@ index-stop: docker compose -f docker-compose.indexers.yml down synths: - docker compose run transformer python scripts/get_synths.py + docker compose run transformer uv run python scripts/get_synths.py dbt: build docker compose run transformer dbt run --target prod --profiles-dir profiles --profile clickhouse