Improve indexer setup

.env

@@ -1,55 +1,14 @@
-NODE_ENV=development
-
-# Mainnet
-NETWORK_ID=mainnet
-# Testnet - Taurus
-# NETWORK_ID=taurus
-
-LOG_LEVEL=debug
-
-# Mainnet
-NODE_DOCKER_TAG=mainnet-2024-nov-13
-# Testnet - Taurus
-# NODE_DOCKER_TAG=taurus-2024-nov-05
-
-DB_USER=postgres
-DB_DATABASE=postgres
 DB_PASSWORD=postgres
-DB_PORT=5432
-DB_HOST=postgres
-
-# Mainnet
-RPC_URLS="ws://caddy:8000"
-CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd"
 
-# Testnet - Taurus
-# RPC_URLS="ws://caddy:8000
-# CHAIN_ID="0x295aeafca762a304d92ee1505548695091f6082d3f0aa4d092ac3cd6397a6c5e"
-
-HASURA_GRAPHQL_PORT=8080
 HASURA_GRAPHQL_ADMIN_SECRET=helloworld
 
 # In local dev, we want to disable this to run the metadata first and run console in another terminal
 HASURA_GRAPHQL_ENABLE_CONSOLE=true
 HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}'
-HASURA_GRAPHQL_CORS_DOMAIN="*"
-
-# Mainnet
-DICTIONARY_DIRECTORY=autonomys
-# Testnet - Taurus
-# DICTIONARY_DIRECTORY=autonomys-taurus
-DICTIONARY_SUBQUERY_NODE_PORT=3010
-DICTIONARY_URL="http://localhost:3010"
 
-REDIS_PORT=6379
-
-BULL_USERNAME=bull
 BULL_PASSWORD=board
-BULL_PORT=3020
 BULL_SESSION_SECRET=keyboardcat
 
-API_PORT=3030
-
 NR_API_KEY=""
 NR_AGENT_IDENTIFIER=""
 

.env.localhost

@@ -0,0 +1,42 @@
+NODE_ENV=development
+NETWORK_ID=dev
+
+LOG_LEVEL=debug
+
+NODE_DOCKER_TAG=mainnet-2024-nov-13
+
+DB_USER=postgres
+DB_DATABASE=postgres
+DB_PASSWORD=postgres
+DB_PORT=5432
+DB_HOST=postgres
+
+RPC_URLS="ws://caddy:8000"
+CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd"
+
+HASURA_GRAPHQL_PORT=8080
+HASURA_GRAPHQL_ADMIN_SECRET=helloworld
+
+# In local dev, we want to disable this to run the metadata first and run console in another terminal
+HASURA_GRAPHQL_ENABLE_CONSOLE=true
+HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}'
+HASURA_GRAPHQL_CORS_DOMAIN="*"
+
+DICTIONARY_DIRECTORY=autonomys
+DICTIONARY_SUBQUERY_NODE_PORT=3010
+DICTIONARY_URL="http://localhost:3010"
+
+REDIS_PORT=6379
+
+BULL_USERNAME=bull
+BULL_PASSWORD=board
+BULL_PORT=3020
+BULL_SESSION_SECRET=keyboardcat
+
+API_PORT=3030
+
+NR_API_KEY=""
+NR_AGENT_IDENTIFIER=""
+
+SLACK_TOKEN=""
+SLACK_CONVERSATION_ID=""

.env.mainnet

@@ -0,0 +1,42 @@
+NODE_ENV=development
+NETWORK_ID=mainnet
+
+LOG_LEVEL=debug
+
+NODE_DOCKER_TAG=mainnet-2024-nov-13
+
+DB_USER=postgres
+DB_DATABASE=postgres
+DB_PASSWORD=postgres
+DB_PORT=5432
+DB_HOST=postgres
+
+RPC_URLS="ws://caddy:8000"
+CHAIN_ID="0x66455a580aabff303720aa83adbe6c44502922251c03ba73686d5245da9e21bd"
+
+HASURA_GRAPHQL_PORT=8080
+HASURA_GRAPHQL_ADMIN_SECRET=helloworld
+
+# In local dev, we want to disable this to run the metadata first and run console in another terminal
+HASURA_GRAPHQL_ENABLE_CONSOLE=true
+HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}'
+HASURA_GRAPHQL_CORS_DOMAIN="*"
+
+DICTIONARY_DIRECTORY=autonomys
+DICTIONARY_SUBQUERY_NODE_PORT=3010
+DICTIONARY_URL="http://localhost:3010"
+
+REDIS_PORT=6379
+
+BULL_USERNAME=bull
+BULL_PASSWORD=board
+BULL_PORT=3020
+BULL_SESSION_SECRET=keyboardcat
+
+API_PORT=3030
+
+NR_API_KEY=""
+NR_AGENT_IDENTIFIER=""
+
+SLACK_TOKEN=""
+SLACK_CONVERSATION_ID=""

.env.taurus

@@ -0,0 +1,42 @@
+NODE_ENV=development
+NETWORK_ID=taurus
+
+LOG_LEVEL=debug
+
+NODE_DOCKER_TAG=taurus-2024-nov-05
+
+DB_USER=postgres
+DB_DATABASE=postgres
+DB_PASSWORD=postgres
+DB_PORT=5432
+DB_HOST=postgres
+
+RPC_URLS="ws://caddy:8000"
+CHAIN_ID="0x295aeafca762a304d92ee1505548695091f6082d3f0aa4d092ac3cd6397a6c5e"
+
+HASURA_GRAPHQL_PORT=8080
+HASURA_GRAPHQL_ADMIN_SECRET=helloworld
+
+# In local dev, we want to disable this to run the metadata first and run console in another terminal
+HASURA_GRAPHQL_ENABLE_CONSOLE=true
+HASURA_GRAPHQL_JWT_SECRET='{"type":"HS256","key":"this_is_a_mock_secret_with_32_chars"}'
+HASURA_GRAPHQL_CORS_DOMAIN="*"
+
+DICTIONARY_DIRECTORY=autonomys-taurus
+DICTIONARY_SUBQUERY_NODE_PORT=3010
+DICTIONARY_URL="http://localhost:3010"
+
+REDIS_PORT=6379
+
+BULL_USERNAME=bull
+BULL_PASSWORD=board
+BULL_PORT=3020
+BULL_SESSION_SECRET=keyboardcat
+
+API_PORT=3030
+
+NR_API_KEY=""
+NR_AGENT_IDENTIFIER=""
+
+SLACK_TOKEN=""
+SLACK_CONVERSATION_ID=""

docker-compose.yml

@@ -1,9 +1,10 @@
 volumes:
-  node-data: {}
+  node_data: {}
   postgres_db: {}
   redis_db: {}
   caddy_config: {}
   caddy_data: {}
+  api_dependencies: {}
 
 services:
   # Caddy Reverse Proxy with CORS enabled
@@ -27,7 +28,7 @@ services:
   node:
     image: ghcr.io/autonomys/node:${NODE_DOCKER_TAG}
     volumes:
-      - node-data:/var/subspace:rw
+      - node_data:/var/subspace:rw
     ports:
       - "30333:30333/tcp"
       - "30433:30433/tcp"
@@ -448,6 +449,10 @@ services:
         condition: service_healthy
       hasura:
         condition: service_started
+    volumes:
+      - ./indexers/api:/app
+      - api_dependencies:/app/node_modules
+    hostname: api
     restart: unless-stopped
     ports:
       - "${API_PORT}:3000"
@@ -458,7 +463,3 @@ services:
       DB_USER: ${DB_USER}
       DB_PASSWORD: ${DB_PASSWORD}
       DB_DATABASE: ${DB_DATABASE}
-    hostname: api
-    volumes:
-      - ./indexers/api:/app
-      - /app/node_modules

indexers/README.md

@@ -26,10 +26,17 @@ To contribute to an indexer or modify its behavior:
 1. Clone the repository if you haven't done so already.
 2. Open Docker Desktop
 3. Run `yarn` from this directory
-4. Run `yarn dev` from this directory
-5. Open a new terminal and run `yarn metadata:dev` from indexers/ directory
-6. Open a new terminal and run `yarn migrate:dev` from indexers/ directory
-7. Open a new terminal and run `yarn console:dev` from indexers/ directory
+4. Run `bash scripts/start.sh --network taurus --env dev --task dev` from this directory
+   - This will start the indexers for the taurus network in dev mode
+   - You can change the network, env and task to fit your needs
+   - Tasks:
+     - `dev`: start the indexers stack
+     - `dev-bootstrap`: start the minimal indexers stack (usually used to run `apply-seeds` before running `dev`)
+     - `codegen`: generate the types
+     - `build`: build the indexers
+     - `metadata`: generate the metadata for Hasura
+     - `migrate`: start the migrations for Hasura
+     - `apply-seeds:mainnet`: apply the seeds for the mainnet
 
 ## Testing
 

indexers/package.json

@@ -4,38 +4,23 @@
   "private": true,
   "main": "index.js",
   "license": "MIT",
-  "workspaces": ["*"],
+  "workspaces": [
+    "*"
+  ],
   "scripts": {
     "postinstall": "git submodule update --init --recursive",
     "bootstrap": "lerna run bootstrap",
     "prepack": "lerna run prepack",
     "graph": "nx graph",
-    "build": "yarn build-dictionary && lerna run build",
+    "build:dictionary": "./scripts/build-dictionary.sh",
+    "build": "yarn build:dictionary && lerna run build",
     "codegen": "lerna run codegen --skip-nx-cache",
-    "build-dictionary": "yarn build-dictionary:mainnet && yarn build-dictionary:taurus",
-    "build-dictionary:mainnet": "cd dictionary/autonomys && yarn && yarn codegen && yarn build && cd -",
-    "build-dictionary:taurus": "cd dictionary/autonomys-taurus && yarn && yarn codegen && yarn build && cd -",
-    "docker:compose-base": "docker compose -p",
-    "docker:dev": "yarn docker:compose-base dev-astral-indexers -f ../docker-compose.yml",
-    "docker:prod": "yarn docker:compose-base prod-astral-indexers -f ../docker-compose.yml -f ../docker-compose.prod.yml",
-    "docker:dev:up": "yarn docker:dev --profile dictionary --profile task --profile indexers up --remove-orphans",
-    "docker:prod:up": "yarn docker:prod --profile dictionary --profile task --profile indexers up --remove-orphans",
-    "docker:dev:bootstrap": "yarn docker:dev up --remove-orphans",
-    "docker:prod:bootstrap": "yarn docker:prod up --remove-orphans",
-    "docker:dev:down": "yarn docker:dev down",
-    "docker:prod:down": "yarn docker:prod down",
-    "dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:dev:up",
-    "dev-bootstrap": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:dev:bootstrap",
-    "console:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run console",
-    "metadata:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run metadata",
-    "migrate:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run migrate",
-    "apply-seeds:mainnet:dev": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.dev | xargs) && lerna run apply-seeds:mainnet",
-    "prod": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:prod:up",
-    "prod-bootstrap": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && yarn build-dictionary && lerna run codegen && lerna run build && yarn docker:prod:bootstrap",
-    "console": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run console",
-    "metadata": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run metadata",
-    "migrate": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run migrate",
-    "apply-seeds:mainnet": "export $(grep -v '^#' ../.env | xargs) && export $(grep -v '^#' ../.env.prod | xargs) && lerna run apply-seeds:mainnet"
+    "start": "./scripts/start.sh",
+    "stop": "./scripts/stop.sh",
+    "console": "lerna run console",
+    "metadata": "lerna run metadata",
+    "migrate": "lerna run migrate",
+    "apply-seeds": "lerna run apply-seeds"
   },
   "dependencies": {
     "lerna": "^8.1.8"

indexers/scripts/run.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+
+# Loop through all parameters
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        --network)
+            NETWORK="$2"
+            shift 2
+            ;;
+        --env)
+            ENV="$2"
+            shift 2
+            ;;
+        --task)
+            TASK="$2"
+            shift 2
+            ;;
+        *)
+            echo "Unknown parameter: $1"
+            exit 1
+            ;;
+    esac
+done
+
+# Echo the parameters
+echo "Network: $NETWORK"
+echo "Environment: $ENV"
+echo "Task: $TASK"
+
+# Load network-specific env file first if it exists
+if [ -f "../.env.$NETWORK" ]; then
+    echo "Loading .env.$NETWORK"
+    set -o allexport
+    source ../.env.$NETWORK
+    set +o allexport
+fi
+
+# Load default env file if it exists
+if [ -f "../.env" ]; then
+    echo "Loading .env"
+    set -o allexport
+    source ../.env
+    set +o allexport
+fi
+
+# Execute the yarn task
+if [[ "$TASK" == *"dev"* ]]; then
+    # Replace 'dev' with the environment value
+    MODIFIED_TASK="${TASK//dev/$ENV}"
+    yarn "$MODIFIED_TASK"
+else
+    yarn "$TASK"
+fi