feat(docker): Update docker-compose to use lily worker pattern (#1044)

* feat(docker): Update docker-compose to use lily worker pattern * fix: Snapshot envvar; prom host; env quotes * chore: Address comments Co-authored-by: Mike Greenberg <[email protected]>
filecoin-project · Aug 29, 2022 · da510c3 · da510c3
1 parent 593dad6
commit da510c3
Show file tree

Hide file tree

Showing 9 changed files with 162 additions and 36 deletions.
diff --git a/build/grafana/provisioning/datasources/datasource.yml b/build/grafana/provisioning/datasources/datasource.yml
@@ -18,7 +18,7 @@ datasources:
   # <int> org id. will default to orgId 1 if not specified
   orgId: 1
   # <string> url
-  url: http://localhost:9090
+  url: http://prometheus:9090
   # <string> database password, if used
   password:
   # <string> database user, if used

diff --git a/build/lily/docker_init.sh b/build/lily/docker_init.sh
@@ -4,14 +4,17 @@ set -exo pipefail
 
 mkdir -p ${LILY_REPO}/keystore
 
-if [[ ! -z "${_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then
-  # import snapshot when _LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set
+
+if [[ ! -z "${LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then
+  # set default snapshot path if not already defined
+  snapshot="${LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH:-https://snapshots.mainnet.filops.net/minimal/latest}"
+
+  # import snapshot when LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set
   if [[ -f "${LILY_REPO}/datastore/_imported" ]]; then
     echo "Skipping import, found ${LILY_REPO}/datastore/_imported file."
   else
-    echo "Importing snapshot from https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car..."
-    lily init --import-snapshot="https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car"
-
+    echo "Importing snapshot from ${snapshot}"
+    lily init --import-snapshot=${snapshot}
     status=$?
     if [ $status -eq 0 ]; then
       touch "/var/lib/lily/datastore/_imported"
@@ -22,6 +25,6 @@ else
   lily init
 fi
 
-chmod 0600 ${LILY_REPO}/keystore/*
+chmod -R 0600 ${LILY_REPO}/keystore
 
 lily $@
diff --git a/build/lily/docker.env → build/lily/notifier.env b/build/lily/docker.env → build/lily/notifier.env
@@ -5,16 +5,20 @@ LILY_REPO=/var/lib/lily
 #LILY_CONFIG=/var/lib/lily/config.toml
 
 # Postgres URL may be overridden
-#LILY_STORAGE_POSTGRESQL_DB_URL=postgres://user:pass@host:port/database?options
+LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable
 
 # Enable IMPORT_SNAPSHOT below to use snapshot on lily startup
-#_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
+#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
+#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH=
 
 # Debugging options
 #LILY_TRACING=true
-LILY_PROMETHEUS_PORT="prometheus:9090"
+LILY_PROMETHEUS_PORT=0.0.0.0:9090
 
 # Logging options
 GOLOG_LOG_FMT=json
 GOLOG_LOG_LEVEL=info
-LILY_LOG_LEVEL_NAMED="vm:error,badgerbs:error"
+LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error
+
+# Queue configuration
+LILY_REDIS_ADDR=redis-queue:6379
diff --git a/build/lily/notifier_config.toml b/build/lily/notifier_config.toml
@@ -0,0 +1,22 @@
+[API]
+  ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
+[Libp2p]
+  ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"]
+  ConnMgrLow = 400
+  ConnMgrHigh = 500
+  ConnMgrGrace = "5m0s"
+[Queue]
+  [Queue.Notifiers]
+    [Queue.Notifiers.Notifier1]
+        Network = "tcp"
+        Addr = "redis-queue:6379"
+        DB = 0
+        PoolSize = 0
+[Storage]
+  [Storage.Postgresql]
+    [Storage.Postgresql.postgres]
+      SchemaName = "lily"
+      URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL"
+      ApplicationName = "lily"
+      PoolSize = 20
+      AllowUpsert = false
diff --git a/build/lily/redis-queue.env b/build/lily/redis-queue.env
@@ -0,0 +1,2 @@
+# FOR DEV ONLY: disabled authentication
+ALLOW_EMPTY_PASSWORD=yes
diff --git a/build/lily/worker.env b/build/lily/worker.env
@@ -0,0 +1,24 @@
+# Path for lily state should be on a persisted volume
+LILY_REPO=/var/lib/lily
+
+# Config path default is below but may be customized
+#LILY_CONFIG=/var/lib/lily/config.toml
+
+# Postgres URL may be overridden
+LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable
+
+# Enable IMPORT_SNAPSHOT below to use snapshot on lily startup
+#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true
+#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH=
+
+# Debugging options
+#LILY_TRACING=true
+LILY_PROMETHEUS_PORT=0.0.0.0:9091
+
+# Logging options
+GOLOG_LOG_FMT=json
+GOLOG_LOG_LEVEL=info
+LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error
+
+# Queue configuration
+LILY_REDIS_ADDR=redis-queue:6379
diff --git a/build/lily/worker_config.toml b/build/lily/worker_config.toml
@@ -0,0 +1,32 @@
+[API]
+  ListenAddress = "/ip4/0.0.0.0/tcp/1234/http"
+[Libp2p]
+  ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"]
+  ConnMgrLow = 400
+  ConnMgrHigh = 500
+  ConnMgrGrace = "5m0s"
+[Queue]
+  [Queue.Workers]
+    [Queue.Workers.Worker1]
+      [Queue.Workers.Worker1.RedisConfig]
+        Network = "tcp"
+        Addr = "redis-queue:6379"
+        DB = 0
+        PoolSize = 0
+      [Queue.Workers.Worker1.WorkerConfig]
+        Concurrency = 1
+        LoggerLevel = "debug"
+        WatchQueuePriority = 5
+        FillQueuePriority = 3
+        IndexQueuePriority = 1
+        WalkQueuePriority = 1
+        StrictPriority = false
+        ShutdownTimeout = 30000000000
+[Storage]
+  [Storage.Postgresql]
+    [Storage.Postgresql.postgres]
+      SchemaName = "lily"
+      URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL"
+      ApplicationName = "lily"
+      PoolSize = 20
+      AllowUpsert = false
diff --git a/build/prometheus/prometheus.yml b/build/prometheus/prometheus.yml
@@ -4,4 +4,4 @@ scrape_configs:
     scrape_interval: 5s
 
     static_configs:
-      - targets: ['localhost:9991']
+      - targets: ['notifier:9090', 'worker:9091']
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -37,7 +37,7 @@ services:
       - '--web.console.templates=/usr/share/prometheus/consoles'
     ports:
       - 9090:9090
-    network_mode: "host"
+    #network_mode: "host"
     restart: always
 
   grafana:
@@ -47,38 +47,77 @@ services:
       - prometheus
     ports:
       - 3000:3000
-    network_mode: "host"
+    #network_mode: "host"
     volumes:
       - grafana_data:/var/lib/grafana
       - ./build/grafana/provisioning/:/etc/grafana/provisioning/
     env_file:
       - ./build/grafana/config.monitoring
     restart: always
 
-  #lily:
-    #container_name: lily
-    ## Select (only one) image
-    #image: filecoin/lily:v0.8.0
-    #image: filecoin/lily:v0.8.0-calibnet
-    #image: filecoin/lily:v0.8.0-calibnet-dev
-    #env_file:
-      ## Check envvars for configurable options
-      #- ./build/lily/docker.env
-    #depends_on:
-      #- prometheus
-      #- timescaledb
-      #- jaeger
-    #ports:
-      #- 1234:1234
-    #volumes:
-      #- lily_data:/var/lib/lily
-      #- ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
-    #entrypoint: /usr/bin/docker_init.sh
-    #command:
-      #- daemon
+  redis-queue:
+    image: bitnami/redis:7.0
+    env_file:
+      # Check envvars for configurable options
+      - ./build/lily/redis-queue.env
+    ports:
+      - 6379:6379
+    restart: always
+
+  notifier:
+    image: filecoin/lily:v0.11.0
+    env_file:
+      # Check envvars for configurable options
+      - ./build/lily/notifier.env
+    depends_on:
+      - prometheus
+      - timescaledb
+      - jaeger
+      - redis-queue
+    ports:
+      - 1234:1234
+    volumes:
+      # holds lily datastore repo
+      - lily_notifier_data:/var/lib/lily
+      # persist params through restarts
+      - lily_notifier_tmp:/var/tmp/filecoin-proof-parameters
+      # notifier-specific config
+      - ./build/lily/notifier_config.toml:/var/lib/lily/config.toml
+      - ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
+    entrypoint: /usr/bin/docker_init.sh
+    command:
+      - "daemon --bootstrap=false"
+
+  worker:
+    image: filecoin/lily:v0.11.0
+    env_file:
+      # Check envvars for configurable options
+      - ./build/lily/worker.env
+    depends_on:
+      - prometheus
+      - timescaledb
+      - jaeger
+      - redis-queue
+      - notifier
+    ports:
+      - 1234
+    volumes:
+      # holds lily datastore repo
+      - lily_worker_data:/var/lib/lily
+      # persist params through restarts
+      - lily_worker_tmp:/var/tmp/filecoin-proof-parameters
+      # notifier-specific config
+      - ./build/lily/worker_config.toml:/var/lib/lily/config.toml
+      - ./build/lily/docker_init.sh:/usr/bin/docker_init.sh
+    entrypoint: /usr/bin/docker_init.sh
+    command:
+      - "daemon --bootstrap=false"
 
 volumes:
   timescaledb: {}
   prometheus_data: {}
   grafana_data: {}
-  lily_data: {}
+  lily_notifier_data: {}
+  lily_worker_data: {}
+  lily_notifier_tmp: {}
+  lily_worker_tmp: {}
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		# FOR DEV ONLY: disabled authentication
		ALLOW_EMPTY_PASSWORD=yes