Skip to content

Commit

Permalink
feat: MySql support (#3427)
Browse files Browse the repository at this point in the history 
This adds local provisioning via docker and wires up some of the MySQL
support.

Still slightly WIP as I have run out of time to fully test it, but it
works with Java apps.
  • Loading branch information
@stuartwdouglas
stuartwdouglas authored Nov 20, 2024
1 parent ff23370 commit 91cc5b0
Show file tree
Hide file tree
Showing 178 changed files with 1,409 additions and 172 deletions.
2 changes: 2 additions & 0 deletions backend/controller/admin/testdata/go/dischema/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/admin/testdata/go/dischema/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions backend/controller/console/testdata/go/console/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/console/testdata/go/console/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 33 additions & 3 deletions backend/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ type Config struct {
}

func (c *Config) SetDefaults() {
if err := kong.ApplyDefaults(c, kong.Vars{"dsn": dsn.DSN("ftl")}); err != nil {
if err := kong.ApplyDefaults(c, kong.Vars{"dsn": dsn.PostgresDSN("ftl")}); err != nil {
panic(err)
}
if c.Advertise == nil {
Expand Down Expand Up @@ -753,6 +753,33 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl
// Initialize checksum to -1; a zero checksum does occur when the context contains no settings
lastChecksum := int64(-1)

dbTypes := map[string]modulecontext.DBType{}
deps, err := s.dal.GetActiveDeployments(ctx)
if err != nil {
return connect.NewError(connect.CodeInternal, fmt.Errorf("could not get deployments: %w", err))
}
databases := map[string]modulecontext.Database{}
for _, dep := range deps {
if dep.Module == name {
for _, decl := range dep.Schema.Decls {
if db, ok := decl.(*schema.Database); ok {
dbType, err := modulecontext.DBTypeFromString(db.Type)
if err != nil {
// Not much we can do here
continue
}
dbTypes[db.Name] = dbType
if db.Runtime != nil {
databases[db.Name] = modulecontext.Database{
DSN: db.Runtime.DSN,
DBType: dbType,
}
}
}
}
break
}
}
for {
h := sha.New()

Expand All @@ -764,10 +791,13 @@ func (s *Service) GetModuleContext(ctx context.Context, req *connect.Request[ftl
if err != nil {
return connect.NewError(connect.CodeInternal, fmt.Errorf("could not get secrets: %w", err))
}
databases, err := modulecontext.DatabasesFromSecrets(ctx, name, secrets)
secretDbs, err := modulecontext.DatabasesFromSecrets(ctx, name, secrets, dbTypes)
if err != nil {
return connect.NewError(connect.CodeInternal, fmt.Errorf("could not get databases: %w", err))
}
for k, v := range secretDbs {
databases[k] = v
}

if err := hashConfigurationMap(h, configs); err != nil {
return connect.NewError(connect.CodeInternal, fmt.Errorf("could not detect change on configs: %w", err))
Expand Down Expand Up @@ -1097,7 +1127,7 @@ func (s *Service) CreateDeployment(ctx context.Context, req *connect.Request[ftl
if err := s.sm.Set(ctx, configuration.NewRef(module.Name, key), db.Runtime.DSN); err != nil {
return nil, fmt.Errorf("could not set database secret %s: %w", key, err)
}
logger.Infof("Database declaration: %s -> %s", db.Name, db.Runtime.DSN)
logger.Infof("Database declaration: %s -> %s type %s", db.Name, db.Runtime.DSN, db.Type)
}
}

Expand Down
13 changes: 11 additions & 2 deletions backend/controller/dsn/dsn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,20 @@ func Host(host string) Option {
}
}

// DSN returns a DSN string for connecting to the FTL Controller PG database.
func DSN(dbName string, options ...Option) string {
// PostgresDSN returns a PostgresDSN string for connecting to the FTL Controller PG database.
func PostgresDSN(dbName string, options ...Option) string {
opts := &dsnOptions{port: 15432, host: "127.0.0.1"}
for _, opt := range options {
opt(opts)
}
return fmt.Sprintf("postgres://%s:%d/%s?sslmode=disable&user=postgres&password=secret", opts.host, opts.port, dbName)
}

// MySQLDSN returns a MySQLDSN string for connecting to the local MySQL database.
func MySQLDSN(dbName string, options ...Option) string {
opts := &dsnOptions{port: 13306, host: "127.0.0.1"}
for _, opt := range options {
opt(opts)
}
return fmt.Sprintf("root:secret@tcp(%s:%d)/%s?allowNativePasswords=True", opts.host, opts.port, dbName)
}
2 changes: 2 additions & 0 deletions backend/controller/encryption/testdata/go/encryption/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/encryption/testdata/go/encryption/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions backend/controller/ingress/testdata/go/httpingress/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/ingress/testdata/go/httpingress/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions backend/controller/leases/testdata/go/leases/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/BurntSushi/toml v1.4.0 // indirect
github.com/XSAM/otelsql v0.35.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
Expand All @@ -35,6 +36,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/leases/testdata/go/leases/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions backend/controller/pubsub/testdata/go/publisher/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/pubsub/testdata/go/publisher/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions backend/controller/pubsub/testdata/go/slow/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ require (
connectrpc.com/connect v1.16.2 // indirect
connectrpc.com/grpcreflect v1.2.0 // indirect
connectrpc.com/otelconnect v0.7.1 // indirect
filippo.io/edwards25519 v1.1.0 // indirect
github.com/alecthomas/atomic v0.1.0-alpha2 // indirect
github.com/alecthomas/concurrency v0.0.2 // indirect
github.com/alecthomas/participle/v2 v2.1.1 // indirect
Expand All @@ -18,6 +19,7 @@ require (
github.com/deckarep/golang-set/v2 v2.6.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-sql-driver/mysql v1.8.1 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/hashicorp/cronexpr v1.1.2 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
Expand Down
4 changes: 4 additions & 0 deletions backend/controller/pubsub/testdata/go/slow/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 91cc5b0

Please sign in to comment.