diff --git a/.env b/.env index 701e23dbc..4a4230fa0 100644 --- a/.env +++ b/.env @@ -1,55 +1,14 @@ -NODE_ENV=development - -# Mainnet -NETWORK_ID=mainnet -# Testnet - Taurus -# NETWORK_ID=taurus - -LOG_LEVEL=debug - -# Mainnet -NODE_DOCKER_TAG=mainnet-2024-nov-13 -# Testnet - Taurus -# NODE_DOCKER_TAG=taurus-2024-nov-05 - -DB_USER=postgres -DB_DATABASE=postgres DB_PASSWORD=postgres -DB_PORT=5432 -DB_HOST=postgres - -# Mainnet -RPC_URLS="ws://caddy:8000" -CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd" -# Testnet - Taurus -# RPC_URLS="ws://caddy:8000 -# CHAIN_ID="0x295aeafca762a304d92ee1505548695091f6082d3f0aa4d092ac3cd6397a6c5e" - -HASURA_GRAPHQL_PORT=8080 HASURA_GRAPHQL_ADMIN_SECRET=helloworld # In local dev, we want to disable this to run the metadata first and run console in another terminal HASURA_GRAPHQL_ENABLE_CONSOLE=true HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}' -HASURA_GRAPHQL_CORS_DOMAIN="*" - -# Mainnet -DICTIONARY_DIRECTORY=autonomys -# Testnet - Taurus -# DICTIONARY_DIRECTORY=autonomys-taurus -DICTIONARY_SUBQUERY_NODE_PORT=3010 -DICTIONARY_URL="http://localhost:3010" -REDIS_PORT=6379 - -BULL_USERNAME=bull BULL_PASSWORD=board -BULL_PORT=3020 BULL_SESSION_SECRET=keyboardcat -API_PORT=3030 - NR_API_KEY="" NR_AGENT_IDENTIFIER="" diff --git a/.env.dev b/.env.dev deleted file mode 100644 index 1394a6689..000000000 --- a/.env.dev +++ /dev/null @@ -1 +0,0 @@ -# Add environment variables here to override those in the .env file. diff --git a/.env.localhost b/.env.localhost new file mode 100644 index 000000000..9d0695d11 --- /dev/null +++ b/.env.localhost @@ -0,0 +1,42 @@ +NODE_ENV=development +NETWORK_ID=dev + +LOG_LEVEL=debug + +NODE_DOCKER_TAG=mainnet-2024-nov-13 + +DB_USER=postgres +DB_DATABASE=postgres +DB_PASSWORD=postgres +DB_PORT=5432 +DB_HOST=postgres + +RPC_URLS="ws://caddy:8000" +CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd" + +HASURA_GRAPHQL_PORT=8080 +HASURA_GRAPHQL_ADMIN_SECRET=helloworld + +# In local dev, we want to disable this to run the metadata first and run console in another terminal +HASURA_GRAPHQL_ENABLE_CONSOLE=true +HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}' +HASURA_GRAPHQL_CORS_DOMAIN="*" + +DICTIONARY_DIRECTORY=autonomys +DICTIONARY_SUBQUERY_NODE_PORT=3010 +DICTIONARY_URL="http://localhost:3010" + +REDIS_PORT=6379 + +BULL_USERNAME=bull +BULL_PASSWORD=board +BULL_PORT=3020 +BULL_SESSION_SECRET=keyboardcat + +API_PORT=3030 + +NR_API_KEY="" +NR_AGENT_IDENTIFIER="" + +SLACK_TOKEN="" +SLACK_CONVERSATION_ID="" diff --git a/.env.mainnet b/.env.mainnet new file mode 100644 index 000000000..312cd619c --- /dev/null +++ b/.env.mainnet @@ -0,0 +1,42 @@ +NODE_ENV=development +NETWORK_ID=mainnet + +LOG_LEVEL=debug + +NODE_DOCKER_TAG=mainnet-2024-nov-13 + +DB_USER=postgres +DB_DATABASE=postgres +DB_PASSWORD=postgres +DB_PORT=5432 +DB_HOST=postgres + +RPC_URLS="ws://caddy:8000" +CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd" + +HASURA_GRAPHQL_PORT=8080 +HASURA_GRAPHQL_ADMIN_SECRET=helloworld + +# In local dev, we want to disable this to run the metadata first and run console in another terminal +HASURA_GRAPHQL_ENABLE_CONSOLE=true +HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}' +HASURA_GRAPHQL_CORS_DOMAIN="*" + +DICTIONARY_DIRECTORY=autonomys +DICTIONARY_SUBQUERY_NODE_PORT=3010 +DICTIONARY_URL="http://localhost:3010" + +REDIS_PORT=6379 + +BULL_USERNAME=bull +BULL_PASSWORD=board +BULL_PORT=3020 +BULL_SESSION_SECRET=keyboardcat + +API_PORT=3030 + +NR_API_KEY="" +NR_AGENT_IDENTIFIER="" + +SLACK_TOKEN="" +SLACK_CONVERSATION_ID="" diff --git a/.env.prod b/.env.prod deleted file mode 100644 index 1394a6689..000000000 --- a/.env.prod +++ /dev/null @@ -1 +0,0 @@ -# Add environment variables here to override those in the .env file. diff --git a/.env.taurus b/.env.taurus new file mode 100644 index 000000000..a20f07d53 --- /dev/null +++ b/.env.taurus @@ -0,0 +1,42 @@ +NODE_ENV=development +NETWORK_ID=taurus + +LOG_LEVEL=debug + +NODE_DOCKER_TAG=taurus-2024-nov-05 + +DB_USER=postgres +DB_DATABASE=postgres +DB_PASSWORD=postgres +DB_PORT=5432 +DB_HOST=postgres + +RPC_URLS="ws://caddy:8000" +CHAIN_ID="0x295aeafca762a304d92ee1505548695091f6082d3f0aa4d092ac3cd6397a6c5e" + +HASURA_GRAPHQL_PORT=8080 +HASURA_GRAPHQL_ADMIN_SECRET=helloworld + +# In local dev, we want to disable this to run the metadata first and run console in another terminal +HASURA_GRAPHQL_ENABLE_CONSOLE=true +HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}' +HASURA_GRAPHQL_CORS_DOMAIN="*" + +DICTIONARY_DIRECTORY=autonomys-taurus +DICTIONARY_SUBQUERY_NODE_PORT=3010 +DICTIONARY_URL="http://localhost:3010" + +REDIS_PORT=6379 + +BULL_USERNAME=bull +BULL_PASSWORD=board +BULL_PORT=3020 +BULL_SESSION_SECRET=keyboardcat + +API_PORT=3030 + +NR_API_KEY="" +NR_AGENT_IDENTIFIER="" + +SLACK_TOKEN="" +SLACK_CONVERSATION_ID="" diff --git a/docker-compose.yml b/docker-compose.yml index 8e0c0f98b..d22cea612 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,10 @@ volumes: - node-data: {} + node_data: {} postgres_db: {} redis_db: {} caddy_config: {} caddy_data: {} + api_dependencies: {} services: # Caddy Reverse Proxy with CORS enabled @@ -27,7 +28,7 @@ services: node: image: ghcr.io/autonomys/node:${NODE_DOCKER_TAG} volumes: - - node-data:/var/subspace:rw + - node_data:/var/subspace:rw ports: - "30333:30333/tcp" - "30433:30433/tcp" @@ -448,6 +449,10 @@ services: condition: service_healthy hasura: condition: service_started + volumes: + - ./indexers/api:/app + - api_dependencies:/app/node_modules + hostname: api restart: unless-stopped ports: - "${API_PORT}:3000" @@ -458,7 +463,3 @@ services: DB_USER: ${DB_USER} DB_PASSWORD: ${DB_PASSWORD} DB_DATABASE: ${DB_DATABASE} - hostname: api - volumes: - - ./indexers/api:/app - - /app/node_modules diff --git a/indexers/README.md b/indexers/README.md index 1a73885c4..889425503 100644 --- a/indexers/README.md +++ b/indexers/README.md @@ -26,10 +26,17 @@ To contribute to an indexer or modify its behavior: 1. Clone the repository if you haven't done so already. 2. Open Docker Desktop 3. Run `yarn` from this directory -4. Run `yarn dev` from this directory -5. Open a new terminal and run `yarn metadata:dev` from indexers/ directory -6. Open a new terminal and run `yarn migrate:dev` from indexers/ directory -7. Open a new terminal and run `yarn console:dev` from indexers/ directory +4. Run `bash scripts/start.sh --network taurus --env dev --task dev` from this directory + - This will start the indexers for the taurus network in dev mode + - You can change the network, env and task to fit your needs + - Tasks: + - `dev`: start the indexers stack + - `dev-bootstrap`: start the minimal indexers stack (usually used to run `apply-seeds` before running `dev`) + - `codegen`: generate the types + - `build`: build the indexers + - `metadata`: generate the metadata for Hasura + - `migrate`: start the migrations for Hasura + - `apply-seeds:mainnet`: apply the seeds for the mainnet ## Testing diff --git a/indexers/package.json b/indexers/package.json index 4f8615db8..d34720616 100644 --- a/indexers/package.json +++ b/indexers/package.json @@ -4,38 +4,23 @@ "private": true, "main": "index.js", "license": "MIT", - "workspaces": ["*"], + "workspaces": [ + "*" + ], "scripts": { "postinstall": "git submodule update --init --recursive", "bootstrap": "lerna run bootstrap", "prepack": "lerna run prepack", "graph": "nx graph", - "build": "yarn build-dictionary && lerna run build", + "build:dictionary": "./scripts/build-dictionary.sh", + "build": "yarn build:dictionary && lerna run build", "codegen": "lerna run codegen --skip-nx-cache", - "build-dictionary": "yarn build-dictionary:mainnet && yarn build-dictionary:taurus", - "build-dictionary:mainnet": "cd dictionary/autonomys && yarn && yarn codegen && yarn build && cd -", - "build-dictionary:taurus": "cd dictionary/autonomys-taurus && yarn && yarn codegen && yarn build && cd -", - "docker:compose-base": "docker compose -p", - "docker:dev": "yarn docker:compose-base dev-astral-indexers -f ../docker-compose.yml", - "docker:prod": "yarn docker:compose-base prod-astral-indexers -f ../docker-compose.yml -f ../docker-compose.prod.yml", - "docker:dev:up": "yarn docker:dev --profile dictionary --profile task --profile indexers up --remove-orphans", - "docker:prod:up": "yarn docker:prod --profile dictionary --profile task --profile indexers up --remove-orphans", - "docker:dev:bootstrap": "yarn docker:dev up --remove-orphans", - "docker:prod:bootstrap": "yarn docker:prod up --remove-orphans", - "docker:dev:down": "yarn docker:dev down", - "docker:prod:down": "yarn docker:prod down", - "dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:dev:up", - "dev-bootstrap": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:dev:bootstrap", - "console:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run console", - "metadata:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run metadata", - "migrate:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run migrate", - "apply-seeds:mainnet:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run apply-seeds:mainnet", - "prod": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:prod:up", - "prod-bootstrap": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:prod:bootstrap", - "console": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run console", - "metadata": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run metadata", - "migrate": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run migrate", - "apply-seeds:mainnet": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run apply-seeds:mainnet" + "start": "./scripts/start.sh", + "stop": "./scripts/stop.sh", + "console": "lerna run console", + "metadata": "lerna run metadata", + "migrate": "lerna run migrate", + "apply-seeds": "lerna run apply-seeds" }, "dependencies": { "lerna": "^8.1.8" diff --git a/indexers/scripts/run.sh b/indexers/scripts/run.sh new file mode 100644 index 000000000..bb0426b76 --- /dev/null +++ b/indexers/scripts/run.sh @@ -0,0 +1,53 @@ +#!/bin/bash + +# Loop through all parameters +while [[ $# -gt 0 ]]; do + case $1 in + --network) + NETWORK="$2" + shift 2 + ;; + --env) + ENV="$2" + shift 2 + ;; + --task) + TASK="$2" + shift 2 + ;; + *) + echo "Unknown parameter: $1" + exit 1 + ;; + esac +done + +# Echo the parameters +echo "Network: $NETWORK" +echo "Environment: $ENV" +echo "Task: $TASK" + +# Load network-specific env file first if it exists +if [ -f "../.env.$NETWORK" ]; then + echo "Loading .env.$NETWORK" + set -o allexport + source ../.env.$NETWORK + set +o allexport +fi + +# Load default env file if it exists +if [ -f "../.env" ]; then + echo "Loading .env" + set -o allexport + source ../.env + set +o allexport +fi + +# Execute the yarn task +if [[ "$TASK" == *"dev"* ]]; then + # Replace 'dev' with the environment value + MODIFIED_TASK="${TASK//dev/$ENV}" + yarn "$MODIFIED_TASK" +else + yarn "$TASK" +fi diff --git a/indexers/scripts/start.sh b/indexers/scripts/start.sh new file mode 100755 index 000000000..af0981eef --- /dev/null +++ b/indexers/scripts/start.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Default values +NETWORK="mainnet" +ENV="dev" +MODE="up" +BOOTSTRAP=false + +# Function to display usage +usage() { + echo "Usage: $0 [options]" + echo "Options:" + echo " --network Network to run (mainnet|taurus|localhost) [default: mainnet]" + echo " --env Environment (dev|prod) [default: dev]" + echo " --bootstrap Run in bootstrap mode" + echo " --down Stop the services" + exit 1 +} + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --network) + NETWORK="$2" + shift 2 + ;; + --env) + ENV="$2" + shift 2 + ;; + --bootstrap) + BOOTSTRAP=true + shift + ;; + --down) + MODE="down" + shift + ;; + --help) + usage + ;; + *) + echo "Unknown parameter: $1" + usage + ;; + esac +done + +# Validate network +if [[ ! "$NETWORK" =~ ^(mainnet|taurus|localhost)$ ]]; then + echo "Error: Invalid network. Must be mainnet, taurus, or localhost" + exit 1 +fi + +# Validate environment +if [[ ! "$ENV" =~ ^(dev|prod)$ ]]; then + echo "Error: Invalid environment. Must be dev or prod" + exit 1 +fi + +# Echo configuration +echo "Configuration:" +echo " Network: $NETWORK" +echo " Environment: $ENV" +echo " Mode: $MODE" +echo " Bootstrap: $BOOTSTRAP" + +# Set project name based on network and environment +PROJECT_NAME="${NETWORK}-${ENV}-astral-indexers" + +# Load environment files +if [ -f "../.env.$NETWORK" ]; then + echo "Loading .env.$NETWORK" + set -o allexport + source ../.env.$NETWORK + set +o allexport +fi + +if [ -f "../.env" ]; then + echo "Loading .env" + set -o allexport + source ../.env + set +o allexport +fi + +# Build base docker compose command +DOCKER_COMPOSE="docker compose -p $PROJECT_NAME -f ../docker-compose.yml" + +# Add production override if needed +if [ "$ENV" = "prod" ]; then + DOCKER_COMPOSE="$DOCKER_COMPOSE -f ../docker-compose.prod.yml" +fi + +# Add network-specific override if exists +if [ -f "../docker-compose.${NETWORK}.yml" ]; then + DOCKER_COMPOSE="$DOCKER_COMPOSE -f ../docker-compose.${NETWORK}.yml" +fi + +bash scripts/run.sh --network $NETWORK --env $ENV --task codegen +bash scripts/run.sh --network $NETWORK --env $ENV --task build + +# Execute docker compose command +if [ "$MODE" = "up" ]; then + if [ "$BOOTSTRAP" = true ]; then + $DOCKER_COMPOSE up --remove-orphans + else + $DOCKER_COMPOSE --profile dictionary --profile task --profile indexers up --remove-orphans + fi +else + $DOCKER_COMPOSE down +fi \ No newline at end of file