chore: converge on docker compose up rather than docker api

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose up -d --wait
+        run: just compose-up
       - name: Build Language Plugins
         run: just build-language-plugins
       - name: Test
@@ -35,7 +35,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose up -d --wait
+        run: just compose-up
       - name: Build Language Plugins
         run: just build-language-plugins
       - name: Test README
@@ -63,7 +63,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose up -d --wait
+        run: just compose-up
       - name: Initialise database
         run: just init-db
       - name: Vet SQL
@@ -188,7 +188,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose up -d --wait
+        run: just compose-up
       - name: Init DB
         run: just init-db
       - name: Rebuild All
@@ -332,7 +332,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose --profile integration up -d --wait
+        run: just compose-up
       - name: Create DB
         run: just init-db
       - name: Download Go Modules
@@ -383,7 +383,7 @@ jobs:
       - name: Build Cache
         uses: ./.github/actions/build-cache
       - name: Docker Compose
-        run: docker compose --profile integration up -d --wait
+        run: just compose-up
       - name: Create DB
         run: just init-db
       - name: Download Go Modules

Justfile

@@ -296,6 +296,19 @@ build-docker name:
     -t ftl0/ftl-{{name}}:latest \
     -f Dockerfile.{{name}} .
 
+# Run docker compose up with all docker compose files
+compose-up:
+  #!/bin/bash
+  set -eo pipefail
+  docker_compose_files="
+  -f docker-compose.yml
+  -f internal/dev/docker-compose.grafana.yml
+  -f internal/dev/docker-compose.mysql.yml
+  -f internal/dev/docker-compose.postgres.yml
+  -f internal/dev/docker-compose.redpanda.yml
+  -f internal/dev/docker-compose.registry.yml"
+    docker compose -p "ftl" $docker_compose_files up -d --wait
+
 # Run a Just command in the Helm charts directory
 chart *args:
     @cd charts && just {{args}}

backend/provisioner/dev_provisioner.go

@@ -44,7 +44,7 @@ func provisionMysql(mysqlPort int) InMemResourceProvisionerFn {
 		dbName := strcase.ToLowerSnake(module) + "_" + strcase.ToLowerSnake(id)
 
 		// We assume that the DB hsas already been started when running in dev mode
-		mysqlDSN, err := dev.SetupMySQL(ctx, "mysql:8.4.3", mysqlPort)
+		mysqlDSN, err := dev.SetupMySQL(ctx, mysqlPort)
 		if err != nil {
 			return nil, fmt.Errorf("failed to wait for mysql to be ready: %w", err)
 		}
@@ -64,7 +64,6 @@ func provisionMysql(mysqlPort int) InMemResourceProvisionerFn {
 				}
 				return ret, nil
 			}
-
 		}
 	}
 }
@@ -84,13 +83,13 @@ func establishMySQLDB(ctx context.Context, rc *provisioner.ResourceContext, mysq
 	if res.Next() {
 		_, err = conn.ExecContext(ctx, "DROP DATABASE "+dbName)
 		if err != nil {
-			return nil, fmt.Errorf("failed to drop database: %w", err)
+			return nil, fmt.Errorf("failed to drop database %q: %w", dbName, err)
 		}
 	}
 
 	_, err = conn.ExecContext(ctx, "CREATE DATABASE "+dbName)
 	if err != nil {
-		return nil, fmt.Errorf("failed to create database: %w", err)
+		return nil, fmt.Errorf("failed to create database %q: %w", dbName, err)
 	}
 
 	if mysql.Mysql == nil {
@@ -153,10 +152,8 @@ func provisionPostgres(postgresPort int) func(ctx context.Context, rc *provision
 		dbName := strcase.ToLowerSnake(module) + "_" + strcase.ToLowerSnake(id)
 
 		// We assume that the DB has already been started when running in dev mode
-		postgresDSN, err := dev.WaitForPostgresReady(ctx, postgresPort)
-		if err != nil {
-			return nil, fmt.Errorf("failed to wait for postgres to be ready: %w", err)
-		}
+		postgresDSN := dsn.PostgresDSN("ftl", dsn.Port(postgresPort))
+
 		conn, err := otelsql.Open("pgx", postgresDSN)
 		if err != nil {
 			return nil, fmt.Errorf("failed to connect to postgres: %w", err)
@@ -180,12 +177,12 @@ func provisionPostgres(postgresPort int) func(ctx context.Context, rc *provision
 			}
 			_, err = conn.ExecContext(ctx, "DROP DATABASE "+dbName)
 			if err != nil {
-				return nil, fmt.Errorf("failed to create database: %w", err)
+				return nil, fmt.Errorf("failed to drop database %q: %w", dbName, err)
 			}
 		}
 		_, err = conn.ExecContext(ctx, "CREATE DATABASE "+dbName)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create database: %w", err)
+			return nil, fmt.Errorf("failed to create database %q: %w", dbName, err)
 		}
 
 		if pg.Postgres == nil {

docker-compose.yml

@@ -1,53 +1,4 @@
 services:
-  db:
-    image: postgres:15.10
-    command: postgres
-    user: postgres
-    # For local debugging
-    # -c logging_collector=on -c log_destination=stderr -c log_directory=/logs -c log_statement=all
-    # volumes:
-    #   - ./logs:/logs
-    restart: always
-    environment:
-      POSTGRES_PASSWORD: secret
-    ports:
-      - 15432:5432
-    healthcheck:
-      test: ["CMD-SHELL", "pg_isready"]
-      interval: 1s
-      timeout: 60s
-      retries: 60
-      start_period: 80s
-  mysql:
-    profiles:
-      - mysql
-    image: mysql:8.4.3
-    environment:
-      MYSQL_ROOT_PASSWORD: secret
-      MYSQL_USER: mysql
-      MYSQL_PASSWORD: secret
-      MYSQL_DATABASE: ftl
-    ports:
-      - "13306:3306"
-    healthcheck:
-      test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-u", "root", "--password=secret"]
-      interval: 1s
-      timeout: 60s
-      retries: 60
-      start_period: 80s
-  otel-lgtm:
-    image: grafana/otel-lgtm
-    platform: linux/amd64
-    ports:
-      - 3000:3000 # Portal Endpoint
-      - 9090:9090 # Prometheus
-      - ${OTEL_GRPC_PORT:-4317}:4317 # OTLP GRPC Collector
-      - ${OTEL_HTTP_PORT:-4317}:4318 # OTLP HTTP Collector
-    environment:
-      - ENABLE_LOGS_ALL=true
-      - GF_AUTH_ANONYMOUS_ENABLED=true
-      - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin
-      - GF_AUTH_DISABLE_LOGIN_FORM=true
   localstack:
     image: localstack/localstack
     profiles:
@@ -57,10 +8,3 @@ services:
     environment:
       SERVICES: secretsmanager
       DEBUG: 1
-  registry:
-    image: registry:2
-    ports:
-      -  "15000:5000"
-
-volumes:
-  grafana-storage: {}

frontend/cli/cmd_dev.go

@@ -73,11 +73,10 @@ func (d *devCmd) Run(
 	}
 
 	if d.InitDB {
-		dsn, err := dev.SetupPostgres(ctx, d.ServeCmd.DatabaseImage, d.ServeCmd.DBPort, true)
+		err := dev.SetupPostgres(ctx, optional.Some(d.ServeCmd.DatabaseImage), d.ServeCmd.DBPort, true)
 		if err != nil {
 			return fmt.Errorf("failed to setup database: %w", err)
 		}
-		fmt.Println(dsn)
 		return nil
 	}
 	statusManager := terminal.FromContext(ctx)

frontend/cli/cmd_serve.go

@@ -49,16 +49,16 @@ type serveCmd struct {
 
 type serveCommonConfig struct {
 	Bind                *url.URL             `help:"Starting endpoint to bind to and advertise to. Each controller, ingress, runner and language plugin will increment the port by 1" default:"http://127.0.0.1:8891"`
-	DBPort              int                  `help:"Port to use for the database." default:"15432"`
-	MysqlPort           int                  `help:"Port to use for the MySQL database, if one is required." default:"13306"`
-	RegistryPort        int                  `help:"Port to use for the registry." default:"15000"`
+	DBPort              int                  `help:"Port to use for the database." env:"FTL_DB_PORT" default:"15432"`
+	MysqlPort           int                  `help:"Port to use for the MySQL database, if one is required." env:"FTL_MYSQL_PORT" default:"13306"`
+	RegistryPort        int                  `help:"Port to use for the registry." env:"FTL_REGISTRY_PORT" default:"15000"`
 	Controllers         int                  `short:"c" help:"Number of controllers to start." default:"1"`
 	Provisioners        int                  `short:"p" help:"Number of provisioners to start." default:"1"`
 	Background          bool                 `help:"Run in the background." default:"false"`
 	Stop                bool                 `help:"Stop the running FTL instance. Can be used with --background to restart the server" default:"false"`
 	StartupTimeout      time.Duration        `help:"Timeout for the server to start up." default:"10s" env:"FTL_STARTUP_TIMEOUT"`
 	ObservabilityConfig observability.Config `embed:"" prefix:"o11y-"`
-	DatabaseImage       string               `help:"The container image to start for the database" default:"postgres:15.8" env:"FTL_DATABASE_IMAGE" hidden:""`
+	DatabaseImage       string               `help:"The container image to start for the database" default:"postgres:15.10" env:"FTL_DATABASE_IMAGE" hidden:""`
 	RegistryImage       string               `help:"The container image to start for the image registry" default:"registry:2" env:"FTL_REGISTRY_IMAGE" hidden:""`
 	GrafanaImage        string               `help:"The container image to start for the automatic Grafana instance" default:"grafana/otel-lgtm" env:"FTL_GRAFANA_IMAGE" hidden:""`
 	DisableGrafana      bool                 `help:"Disable the automatic Grafana that is started if no telemetry collector is specified." default:"false"`
@@ -163,7 +163,8 @@ func (s *serveCommonConfig) run(
 	registry.AllowInsecure = true
 	registry.Registry = fmt.Sprintf("127.0.0.1:%d/ftl", s.RegistryPort)
 	// Bring up the DB and DAL.
-	dsn, err := dev.SetupPostgres(ctx, s.DatabaseImage, s.DBPort, recreate)
+	dsn := dev.PostgresDSN(ctx, s.DBPort)
+	err = dev.SetupPostgres(ctx, optional.Some(s.DatabaseImage), s.DBPort, recreate)
 	if err != nil {
 		return err
 	}

internal/buildengine/engine.go

@@ -519,7 +519,7 @@ func (e *Engine) watchForModuleChanges(ctx context.Context, period time.Duration
 	// Build and deploy all modules first.
 	err = e.BuildAndDeploy(ctx, 1, true)
 	if err != nil {
-		logger.Errorf(err, "initial deploy failed")
+		logger.Errorf(err, "Initial deploy failed")
 	}
 
 	moduleHashes := map[string][]byte{}

internal/container/container.go

@@ -1,10 +1,12 @@
 package container
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strconv"
 	"time"
 
@@ -18,7 +20,10 @@ import (
 	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/docker/go-connections/nat"
 
+	"github.com/TBD54566975/ftl/internal/exec"
+	"github.com/TBD54566975/ftl/internal/flock"
 	"github.com/TBD54566975/ftl/internal/log"
+	"github.com/TBD54566975/ftl/internal/projectconfig"
 )
 
 var dockerClient = once.Once(func(ctx context.Context) (*client.Client, error) {
@@ -343,3 +348,40 @@ func PollContainerHealth(ctx context.Context, containerName string, timeout time
 		}
 	}
 }
+
+// ComposeUp runs docker-compose up with the given compose YAML.
+//
+// Make sure you obtain the compose yaml from a string literal or an embedded file, rather than
+// reading from disk. The project file will not be included in the release build.
+func ComposeUp(ctx context.Context, name, composeYAML string, envars ...string) error {
+	logger := log.FromContext(ctx).Scope(name)
+	ctx = log.ContextWithLogger(ctx, logger)
+
+	// A flock is used to provent Docker compose getting confused, which happens when we call `docker compose up`
+	// multiple times simultaneously for the same services.
+	projCfg, ok := projectconfig.DefaultConfigPath().Get()
+	if !ok {
+		return fmt.Errorf("failed to get project config path")
+	}
+	dir := filepath.Join(filepath.Dir(projCfg), ".ftl")
+	err := os.MkdirAll(dir, 0700)
+	if err != nil {
+		return fmt.Errorf("failed to create directory: %w", err)
+	}
+	release, err := flock.Acquire(ctx, filepath.Join(dir, fmt.Sprintf(".docker.%v.lock", name)), 1*time.Minute)
+	if err != nil {
+		return fmt.Errorf("failed to acquire lock: %w", err)
+	}
+	defer release() //nolint:errcheck
+
+	logger.Debugf("Running docker compose up")
+
+	envars = append(envars, "COMPOSE_IGNORE_ORPHANS=True")
+
+	cmd := exec.CommandWithEnv(ctx, log.Debug, ".", envars, "docker", "compose", "-f", "-", "-p", "ftl", "up", "-d", "--wait")
+	cmd.Stdin = bytes.NewReader([]byte(composeYAML))
+	if err := cmd.RunStderrError(ctx); err != nil {
+		return fmt.Errorf("failed to run docker compose up: %w", err)
+	}
+	return nil
+}