From da510c3c6ad04f56c3d77ec5d38d64e57803f0e5 Mon Sep 17 00:00:00 2001 From: Mike Greenberg <305332+placer14@users.noreply.github.com> Date: Mon, 29 Aug 2022 17:55:56 -0400 Subject: [PATCH] feat(docker): Update docker-compose to use lily worker pattern (#1044) * feat(docker): Update docker-compose to use lily worker pattern * fix: Snapshot envvar; prom host; env quotes * chore: Address comments Co-authored-by: Mike Greenberg --- .../provisioning/datasources/datasource.yml | 2 +- build/lily/docker_init.sh | 15 ++-- build/lily/{docker.env => notifier.env} | 12 ++- build/lily/notifier_config.toml | 22 +++++ build/lily/redis-queue.env | 2 + build/lily/worker.env | 24 +++++ build/lily/worker_config.toml | 32 +++++++ build/prometheus/prometheus.yml | 2 +- docker-compose.yml | 87 ++++++++++++++----- 9 files changed, 162 insertions(+), 36 deletions(-) rename build/lily/{docker.env => notifier.env} (53%) create mode 100644 build/lily/notifier_config.toml create mode 100644 build/lily/redis-queue.env create mode 100644 build/lily/worker.env create mode 100644 build/lily/worker_config.toml diff --git a/build/grafana/provisioning/datasources/datasource.yml b/build/grafana/provisioning/datasources/datasource.yml index bd0bcb50f..c02bb38b3 100644 --- a/build/grafana/provisioning/datasources/datasource.yml +++ b/build/grafana/provisioning/datasources/datasource.yml @@ -18,7 +18,7 @@ datasources: # org id. will default to orgId 1 if not specified orgId: 1 # url - url: http://localhost:9090 + url: http://prometheus:9090 # database password, if used password: # database user, if used diff --git a/build/lily/docker_init.sh b/build/lily/docker_init.sh index 6bf623436..fab71013a 100755 --- a/build/lily/docker_init.sh +++ b/build/lily/docker_init.sh @@ -4,14 +4,17 @@ set -exo pipefail mkdir -p ${LILY_REPO}/keystore -if [[ ! -z "${_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then - # import snapshot when _LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set + +if [[ ! -z "${LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT}" ]]; then + # set default snapshot path if not already defined + snapshot="${LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH:-https://snapshots.mainnet.filops.net/minimal/latest}" + + # import snapshot when LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT is set if [[ -f "${LILY_REPO}/datastore/_imported" ]]; then echo "Skipping import, found ${LILY_REPO}/datastore/_imported file." else - echo "Importing snapshot from https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car..." - lily init --import-snapshot="https://fil-chain-snapshots-fallback.s3.amazonaws.com/mainnet/minimal_finality_stateroots_latest.car" - + echo "Importing snapshot from ${snapshot}" + lily init --import-snapshot=${snapshot} status=$? if [ $status -eq 0 ]; then touch "/var/lib/lily/datastore/_imported" @@ -22,6 +25,6 @@ else lily init fi -chmod 0600 ${LILY_REPO}/keystore/* +chmod -R 0600 ${LILY_REPO}/keystore lily $@ diff --git a/build/lily/docker.env b/build/lily/notifier.env similarity index 53% rename from build/lily/docker.env rename to build/lily/notifier.env index f1de92d5b..d752927bc 100644 --- a/build/lily/docker.env +++ b/build/lily/notifier.env @@ -5,16 +5,20 @@ LILY_REPO=/var/lib/lily #LILY_CONFIG=/var/lib/lily/config.toml # Postgres URL may be overridden -#LILY_STORAGE_POSTGRESQL_DB_URL=postgres://user:pass@host:port/database?options +LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable # Enable IMPORT_SNAPSHOT below to use snapshot on lily startup -#_LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true +#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true +#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH= # Debugging options #LILY_TRACING=true -LILY_PROMETHEUS_PORT="prometheus:9090" +LILY_PROMETHEUS_PORT=0.0.0.0:9090 # Logging options GOLOG_LOG_FMT=json GOLOG_LOG_LEVEL=info -LILY_LOG_LEVEL_NAMED="vm:error,badgerbs:error" +LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error + +# Queue configuration +LILY_REDIS_ADDR=redis-queue:6379 diff --git a/build/lily/notifier_config.toml b/build/lily/notifier_config.toml new file mode 100644 index 000000000..ee4636012 --- /dev/null +++ b/build/lily/notifier_config.toml @@ -0,0 +1,22 @@ +[API] + ListenAddress = "/ip4/0.0.0.0/tcp/1234/http" +[Libp2p] + ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"] + ConnMgrLow = 400 + ConnMgrHigh = 500 + ConnMgrGrace = "5m0s" +[Queue] + [Queue.Notifiers] + [Queue.Notifiers.Notifier1] + Network = "tcp" + Addr = "redis-queue:6379" + DB = 0 + PoolSize = 0 +[Storage] + [Storage.Postgresql] + [Storage.Postgresql.postgres] + SchemaName = "lily" + URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL" + ApplicationName = "lily" + PoolSize = 20 + AllowUpsert = false diff --git a/build/lily/redis-queue.env b/build/lily/redis-queue.env new file mode 100644 index 000000000..f03ad9cd4 --- /dev/null +++ b/build/lily/redis-queue.env @@ -0,0 +1,2 @@ +# FOR DEV ONLY: disabled authentication +ALLOW_EMPTY_PASSWORD=yes diff --git a/build/lily/worker.env b/build/lily/worker.env new file mode 100644 index 000000000..96e937fdc --- /dev/null +++ b/build/lily/worker.env @@ -0,0 +1,24 @@ +# Path for lily state should be on a persisted volume +LILY_REPO=/var/lib/lily + +# Config path default is below but may be customized +#LILY_CONFIG=/var/lib/lily/config.toml + +# Postgres URL may be overridden +LILY_STORAGE_POSTGRESQL_DB_URL=postgres://postgres:password@timescaledb:5432/postgres?sslmode=disable + +# Enable IMPORT_SNAPSHOT below to use snapshot on lily startup +#LILY_DOCKER_INIT_IMPORT_MAINNET_SNAPSHOT=true +#LILY_DOCKER_INIT_IMPORT_SNAPSHOT_PATH= + +# Debugging options +#LILY_TRACING=true +LILY_PROMETHEUS_PORT=0.0.0.0:9091 + +# Logging options +GOLOG_LOG_FMT=json +GOLOG_LOG_LEVEL=info +LILY_LOG_LEVEL_NAMED=vm:error,badgerbs:error + +# Queue configuration +LILY_REDIS_ADDR=redis-queue:6379 diff --git a/build/lily/worker_config.toml b/build/lily/worker_config.toml new file mode 100644 index 000000000..c3c7ca5ec --- /dev/null +++ b/build/lily/worker_config.toml @@ -0,0 +1,32 @@ +[API] + ListenAddress = "/ip4/0.0.0.0/tcp/1234/http" +[Libp2p] + ListenAddresses = ["/ip4/0.0.0.0/tcp/1347"] + ConnMgrLow = 400 + ConnMgrHigh = 500 + ConnMgrGrace = "5m0s" +[Queue] + [Queue.Workers] + [Queue.Workers.Worker1] + [Queue.Workers.Worker1.RedisConfig] + Network = "tcp" + Addr = "redis-queue:6379" + DB = 0 + PoolSize = 0 + [Queue.Workers.Worker1.WorkerConfig] + Concurrency = 1 + LoggerLevel = "debug" + WatchQueuePriority = 5 + FillQueuePriority = 3 + IndexQueuePriority = 1 + WalkQueuePriority = 1 + StrictPriority = false + ShutdownTimeout = 30000000000 +[Storage] + [Storage.Postgresql] + [Storage.Postgresql.postgres] + SchemaName = "lily" + URLEnv = "LILY_STORAGE_POSTGRESQL_DB_URL" + ApplicationName = "lily" + PoolSize = 20 + AllowUpsert = false diff --git a/build/prometheus/prometheus.yml b/build/prometheus/prometheus.yml index 910a86f43..c62ecff4d 100644 --- a/build/prometheus/prometheus.yml +++ b/build/prometheus/prometheus.yml @@ -4,4 +4,4 @@ scrape_configs: scrape_interval: 5s static_configs: - - targets: ['localhost:9991'] + - targets: ['notifier:9090', 'worker:9091'] diff --git a/docker-compose.yml b/docker-compose.yml index 2a876591a..a19a675ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -37,7 +37,7 @@ services: - '--web.console.templates=/usr/share/prometheus/consoles' ports: - 9090:9090 - network_mode: "host" + #network_mode: "host" restart: always grafana: @@ -47,7 +47,7 @@ services: - prometheus ports: - 3000:3000 - network_mode: "host" + #network_mode: "host" volumes: - grafana_data:/var/lib/grafana - ./build/grafana/provisioning/:/etc/grafana/provisioning/ @@ -55,30 +55,69 @@ services: - ./build/grafana/config.monitoring restart: always - #lily: - #container_name: lily - ## Select (only one) image - #image: filecoin/lily:v0.8.0 - #image: filecoin/lily:v0.8.0-calibnet - #image: filecoin/lily:v0.8.0-calibnet-dev - #env_file: - ## Check envvars for configurable options - #- ./build/lily/docker.env - #depends_on: - #- prometheus - #- timescaledb - #- jaeger - #ports: - #- 1234:1234 - #volumes: - #- lily_data:/var/lib/lily - #- ./build/lily/docker_init.sh:/usr/bin/docker_init.sh - #entrypoint: /usr/bin/docker_init.sh - #command: - #- daemon + redis-queue: + image: bitnami/redis:7.0 + env_file: + # Check envvars for configurable options + - ./build/lily/redis-queue.env + ports: + - 6379:6379 + restart: always + + notifier: + image: filecoin/lily:v0.11.0 + env_file: + # Check envvars for configurable options + - ./build/lily/notifier.env + depends_on: + - prometheus + - timescaledb + - jaeger + - redis-queue + ports: + - 1234:1234 + volumes: + # holds lily datastore repo + - lily_notifier_data:/var/lib/lily + # persist params through restarts + - lily_notifier_tmp:/var/tmp/filecoin-proof-parameters + # notifier-specific config + - ./build/lily/notifier_config.toml:/var/lib/lily/config.toml + - ./build/lily/docker_init.sh:/usr/bin/docker_init.sh + entrypoint: /usr/bin/docker_init.sh + command: + - "daemon --bootstrap=false" + + worker: + image: filecoin/lily:v0.11.0 + env_file: + # Check envvars for configurable options + - ./build/lily/worker.env + depends_on: + - prometheus + - timescaledb + - jaeger + - redis-queue + - notifier + ports: + - 1234 + volumes: + # holds lily datastore repo + - lily_worker_data:/var/lib/lily + # persist params through restarts + - lily_worker_tmp:/var/tmp/filecoin-proof-parameters + # notifier-specific config + - ./build/lily/worker_config.toml:/var/lib/lily/config.toml + - ./build/lily/docker_init.sh:/usr/bin/docker_init.sh + entrypoint: /usr/bin/docker_init.sh + command: + - "daemon --bootstrap=false" volumes: timescaledb: {} prometheus_data: {} grafana_data: {} - lily_data: {} + lily_notifier_data: {} + lily_worker_data: {} + lily_notifier_tmp: {} + lily_worker_tmp: {}