feat(compose): create services for api workers

Signed-off-by: Tomas Pilar <[email protected]>

docker-compose.yml

@@ -5,7 +5,7 @@ services:
       redis: { condition: service_started }
       mongo: { condition: service_healthy }
       minio: { condition: service_started }
-    image: iambeeagent/bee-api:0.0.5
+    image: icr.io/i-am-bee/bee-api:0.0.7
     command: >
       /bin/sh -c '
       output=$(npx mikro-orm seeder:run 2>&1);
@@ -54,8 +54,6 @@ services:
       MILVUS_PASSWORD: password
       MILVUS_DATABASE_NAME: default
 
-      RUN_BULLMQ_WORKERS: runs,runs:cleanup,vectorStores:cleanup,vectorStores:fileProcessor,files:extraction
-
       # Used to encrypt/decrypt values
       # You can use `$ openssl rand -base64 32` to generate new one
       CRYPTO_CIPHER_KEY: random_crypto_key
@@ -81,7 +79,95 @@ services:
       - code-interpreter-storage:/storage
     ports:
       - "4000:4000"
-    profiles: [ all ]
+    profiles: [all]
+
+  bee-api-workers-node:
+    user: root
+    depends_on:
+      redis: { condition: service_started }
+      mongo: { condition: service_healthy }
+      minio: { condition: service_started }
+      bee-api: { condition: service_started }
+    image: icr.io/i-am-bee/bee-api-workers-node:0.0.7
+    environment:
+      # BACKENDS
+      LLM_BACKEND: ${LLM_BACKEND:-watsonx}
+      EMBEDDING_BACKEND: ${EMBEDDING_BACKEND:-watsonx}
+      WATSONX_PROJECT_ID: ${WATSONX_PROJECT_ID}
+      WATSONX_API_KEY: ${WATSONX_API_KEY}
+      BAM_API_KEY: ${BAM_API_KEY:-dummy}
+      OPENAI_API_KEY: ${OPENAI_API_KEY:-dummy}
+
+      MONGODB_URL: mongodb://mongo:27017?directConnection=true
+      MONGODB_DATABASE_NAME: bee-api
+      REDIS_URL: redis://redis:6379/0
+      REDIS_CACHE_URL: redis://redis:6379/8
+      HTTP_PROXY_URL: http://localhost:3128
+      BEE_CODE_INTERPRETER_URL: http://bee-code-interpreter-k3s:30051
+
+      BEE_CODE_INTERPRETER_STORAGE_BACKEND: filesystem
+      BEE_CODE_INTERPRETER_FILE_STORAGE_PATH: /storage
+      # Shutdown is not working properly anyway
+      SHUTDOWN_GRACEFUL_PERIOD: 1
+
+      S3_ENDPOINT: http://minio:9000
+      S3_ACCESS_KEY_ID: minioadmin
+      S3_SECRET_ACCESS_KEY: minioadmin
+      S3_BUCKET_FILE_STORAGE: bee-api
+
+      BEE_OBSERVE_API_URL: http://bee-observe:3000
+      BEE_OBSERVE_API_AUTH_KEY: observe-auth-key
+
+      MILVUS_HOST: milvus
+      MILVUS_PORT: 19530
+      MILVUS_USE_TLS: false
+      MILVUS_USERNAME: user
+      MILVUS_PASSWORD: password
+      MILVUS_DATABASE_NAME: default
+
+      RUN_BULLMQ_WORKERS: runs,runs:cleanup,vectorStores:cleanup,vectorStores:fileProcessor,files:extraction:postprocess
+
+      # Used to encrypt/decrypt values
+      # You can use `$ openssl rand -base64 32` to generate new one
+      CRYPTO_CIPHER_KEY: random_crypto_key
+      OTEL_SDK_DISABLED: true
+
+      # Required env variables used only in migrations
+      USER_ID_DEFAULT: dummy
+      PROJECT_ID_DEFAULT: proj_670cc04869ddffe24f4fd70f
+      ORGANIZATION_ID_DEFAULT: org_670cc04869ddffe24f4fd70d
+      ORGANIZATION_OWNER_ID_DEFAULT: dummy
+      PROJECT_ADMIN_ID_DEFAULT: dummy
+      AUTH_CLIENT_ID: dummy
+      AUTH_CLIENT_SECRET: dummy
+      AUTH_SERVER_PORT: 4001
+      AUTH_WELL_KNOWN: dummy
+      AUTH_AUDIENCE: dummy
+    volumes:
+      - code-interpreter-storage:/storage
+    profiles: [all]
+
+  bee-api-workers-python:
+    user: root
+    depends_on:
+      redis: { condition: service_started }
+      minio: { condition: service_started }
+    image: icr.io/i-am-bee/bee-api-workers-python:0.0.7
+    container_name: bee-api-worker-python
+    environment:
+      LOG_LEVEL: info
+
+      REDIS_URL: redis://redis:6379/0
+
+      EXTRACTION_BACKEND: unstructured-opensource
+
+      S3_ENDPOINT: http://minio:9000
+      S3_ACCESS_KEY_ID: minioadmin
+      S3_SECRET_ACCESS_KEY: minioadmin
+      S3_BUCKET_FILE_STORAGE: bee-api
+
+      RUN_BULLMQ_WORKERS: files:extraction
+    profiles: [all]
 
   bee-ui:
     depends_on:
@@ -92,7 +178,7 @@ services:
       NEXTAUTH_SECRET: top-secret
       API_URL: http://bee-api:4000
       DUMMY_JWT_TOKEN: "eyJhbGciOiJSUzI1NiIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJ0ZXN0IiwicHJlZmVycmVkX3VzZXJuYW1lIjoiVGVzdCBVc2VyIiwiZW1haWwiOiJ0ZXN0QGVtYWlsLmNvbSIsImlhdCI6MTUxNjIzOTAyMiwiaXNzIjoiaHR0cHM6Ly9sb2NhbGhvc3QiLCJhdWQiOiJiZWUtdGVzdCJ9.vwkGnl7lBbzJYk6BtoW3VoA3mnNJVI-nDQU8aK7zOH-rkf2pn5cn6CKwpq7enDInIXro8WtBLNZP8Nr8GQIZKahICuP3YrPRmzv7YIW8LuXKnx1hycg5OAtj0OtQi5FYwwCxTYW9pBF2it7XwQSBcW7yYsOrvgs7jVhThCOsavX0YiAROxZIhk1idZT4Pl3egfUI_dy9iBxcn7xocTnos-94wqJNt8oCVgB8ynj75yJFHJbiQ-9Tym_V3LcMHoEyv67Jzie8KugCgdpuF6EbQqcyfYJ83q5jJpR2LiuWMuGsNSbjjDY-f1vCSMo9L9-R8KFrDylT_BzLvRBswOzW7A"
-    profiles: [ all ]
+    profiles: [all]
     ports:
       - "3000:3000"
 
@@ -122,11 +208,11 @@ services:
       MLFLOW_TRACE_DELETE_IN_BATCHES_CRON_PATTERN: "0 */1 * * * *"
       MLFLOW_TRACE_DELETE_IN_BATCHES_BATCH_SIZE: 100
       NODE_ENV: production
-    profiles: [ all ]
+    profiles: [all]
 
   bee-code-interpreter-k3s:
     image: "rancher/k3s:v1.30.5-k3s1"
-    command: [ "server", "--tls-san", "bee-code-interpreter-k3s" ]
+    command: ["server", "--tls-san", "bee-code-interpreter-k3s"]
     tmpfs:
       - /run
       - /var/run
@@ -148,7 +234,7 @@ services:
       interval: 10s
       retries: 10
       start_period: 30s
-    profiles: [ all ]
+    profiles: [all]
 
   redis:
     image: redis:7
@@ -157,11 +243,11 @@ services:
     command: redis-server --save 20 1 --loglevel warning
     volumes:
       - redis:/data
-    profiles: [ all, infra ]
+    profiles: [all, infra]
 
   mongo:
     image: mongo:7
-    command: [ "--replSet", "rs0", "--bind_ip_all", "--port", "27017" ]
+    command: ["--replSet", "rs0", "--bind_ip_all", "--port", "27017"]
     ports:
       - 27017:27017
     healthcheck:
@@ -172,26 +258,31 @@ services:
       retries: 30
     volumes:
       - mongo:/data/db
-    profiles: [ all, infra ]
+    profiles: [all, infra]
 
   mlflow:
     image: bitnami/mlflow:2.16.2
     ports:
-      - '${MLFLOW_EXPOSED_PORT:-8080}:8080'
+      - "${MLFLOW_EXPOSED_PORT:-8080}:8080"
     security_opt:
       - "label=disable"
     environment:
       GUNICORN_CMD_ARGS: "--timeout=600"
-    entrypoint: [ "/bin/bash", "-c", "/entrypoint.sh && mlflow server --host 0.0.0.0 --port 8080" ]
+    entrypoint:
+      [
+        "/bin/bash",
+        "-c",
+        "/entrypoint.sh && mlflow server --host 0.0.0.0 --port 8080",
+      ]
     volumes:
       - mlflow:/app
       - ./mlflow-entrypoint.sh:/entrypoint.sh:ro
-    profiles: [ all, infra ]
+    profiles: [all, infra]
 
   # MILVUS dependencies
   milvus:
     image: milvusdb/milvus:v2.4.6
-    command: [ "milvus", "run", "standalone" ]
+    command: ["milvus", "run", "standalone"]
     security_opt:
       - seccomp:unconfined
     environment:
@@ -200,7 +291,7 @@ services:
     volumes:
       - milvus:/var/lib/milvus
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9091/healthz" ]
+      test: ["CMD", "curl", "-f", "http://localhost:9091/healthz"]
       interval: 30s
       start_period: 90s
       timeout: 20s
@@ -211,7 +302,7 @@ services:
     depends_on:
       - "etcd"
       - "minio"
-    profiles: [ all, infra ]
+    profiles: [all, infra]
   etcd:
     image: quay.io/coreos/etcd:v3.5.5
     environment:
@@ -223,11 +314,11 @@ services:
       - etcd:/etcd
     command: etcd -advertise-client-urls=http://127.0.0.1:2379 -listen-client-urls http://0.0.0.0:2379 --data-dir /etcd
     healthcheck:
-      test: [ "CMD", "etcdctl", "endpoint", "health" ]
+      test: ["CMD", "etcdctl", "endpoint", "health"]
       interval: 30s
       timeout: 20s
       retries: 3
-    profiles: [ all, infra ]
+    profiles: [all, infra]
   minio:
     image: bitnami/minio:2024.10.2
     environment:
@@ -240,18 +331,18 @@ services:
     volumes:
       - minio:/bitnami/minio/data
     healthcheck:
-      test: [ "CMD", "curl", "-f", "http://localhost:9000/minio/health/live" ]
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
       timeout: 20s
       retries: 3
-    profiles: [ all, infra ]
+    profiles: [all, infra]
   attu: # milvus UI
     image: zilliz/attu:latest
     environment:
       MILVUS_URL: milvus:19530
     ports:
       - 19531:3000
-    profiles: [ all, infra ]
+    profiles: [all, infra]
 volumes:
   etcd:
   minio: