feat(*): create databases in migrator

components/ledger/libs/bun/bunconnect/flags.go

@@ -30,6 +30,7 @@ func InitFlags(flags *pflag.FlagSet) {
 
 func ConnectionOptionsFromFlags(v *viper.Viper, output io.Writer, debug bool) (*ConnectionOptions, error) {
 	var connector func(string) (driver.Connector, error)
+
 	if v.GetBool(PostgresAWSEnableIAMFlag) {
 		cfg, err := config.LoadDefaultConfig(context.Background(), iam.LoadOptionFromViper(v))
 		if err != nil {

components/ledger/libs/bun/bunmigrate/command.go

@@ -2,15 +2,8 @@ package bunmigrate
 
 import (
 	"github.com/formancehq/stack/libs/go-libs/bun/bunconnect"
-	sharedlogging "github.com/formancehq/stack/libs/go-libs/logging"
-	"github.com/pkg/errors"
-
-	"github.com/formancehq/stack/libs/go-libs/service"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 	"github.com/uptrace/bun"
-	"github.com/uptrace/bun/extra/bundebug"
-
 	// Import the postgres driver.
 	_ "github.com/lib/pq"
 )
@@ -31,26 +24,3 @@ func NewDefaultCommand(executor Executor, options ...func(command *cobra.Command
 	bunconnect.InitFlags(ret.Flags())
 	return ret
 }
-
-func Run(cmd *cobra.Command, args []string, executor Executor) error {
-	connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag))
-	if err != nil {
-		return errors.Wrap(err, "evaluating connection options")
-	}
-
-	db, err := bunconnect.OpenSQLDB(*connectionOptions)
-	if err != nil {
-		return errors.Wrap(err, "opening database")
-	}
-	defer func() {
-		err := db.Close()
-		if err != nil {
-			sharedlogging.FromContext(cmd.Context()).Errorf("Closing database: %s", err)
-		}
-	}()
-	if viper.GetBool(service.DebugFlag) {
-		db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(cmd.OutOrStdout())))
-	}
-
-	return errors.Wrap(executor(cmd, args, db), "executing migration")
-}

components/ledger/libs/bun/bunmigrate/run.go

@@ -0,0 +1,88 @@
+package bunmigrate
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"github.com/formancehq/stack/libs/go-libs/bun/bunconnect"
+	sharedlogging "github.com/formancehq/stack/libs/go-libs/logging"
+	"github.com/formancehq/stack/libs/go-libs/pointer"
+	"github.com/formancehq/stack/libs/go-libs/service"
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"github.com/uptrace/bun"
+	"github.com/uptrace/bun/extra/bundebug"
+	"github.com/xo/dburl"
+	"io"
+)
+
+func ensureDatabaseExists(ctx context.Context, connectionOptions bunconnect.ConnectionOptions) error {
+	url, err := dburl.Parse(connectionOptions.DatabaseSourceName)
+	if err != nil {
+		return err
+	}
+	originalPath := url.Path
+	url.Path = "postgres" // notes(gfyrag): default "postgres" database (most of the time?)
+	connectionOptions.DatabaseSourceName = url.String()
+
+	db, err := bunconnect.OpenSQLDB(connectionOptions)
+	if err != nil {
+		return errors.Wrap(err, "opening database")
+	}
+	defer func() {
+		err := db.Close()
+		if err != nil {
+			sharedlogging.FromContext(ctx).Errorf("Closing database: %s", err)
+		}
+	}()
+
+	row := db.QueryRowContext(ctx, `SELECT datname FROM pg_database WHERE datname = ?`, originalPath[1:])
+	if row.Err() != nil {
+		return row.Err()
+	}
+
+	if err := row.Scan(pointer.For("")); err != nil {
+		if !errors.Is(err, sql.ErrNoRows) {
+			return err
+		}
+
+		_, err = db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE "%s"`, originalPath[1:]))
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func run(ctx context.Context, output io.Writer, args []string, connectionOptions *bunconnect.ConnectionOptions,
+	executor func(args []string, db *bun.DB) error) error {
+
+	if err := ensureDatabaseExists(ctx, *connectionOptions); err != nil {
+		return err
+	}
+
+	db, err := bunconnect.OpenSQLDB(*connectionOptions)
+	if err != nil {
+		return errors.Wrap(err, "opening database")
+	}
+	defer func() {
+		_ = db.Close()
+	}()
+	if viper.GetBool(service.DebugFlag) {
+		db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(output)))
+	}
+
+	return errors.Wrap(executor(args, db), "executing migration")
+}
+
+func Run(cmd *cobra.Command, args []string, executor Executor) error {
+	connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag))
+	if err != nil {
+		return errors.Wrap(err, "evaluating connection options")
+	}
+	return run(cmd.Context(), cmd.OutOrStdout(), args, connectionOptions, func(args []string, db *bun.DB) error {
+		return executor(cmd, args, db)
+	})
+}

components/ledger/libs/bun/bunmigrate/run_test.go

@@ -0,0 +1,34 @@
+package bunmigrate
+
+import (
+	"github.com/formancehq/stack/libs/go-libs/bun/bunconnect"
+	"github.com/formancehq/stack/libs/go-libs/logging"
+	"github.com/formancehq/stack/libs/go-libs/pgtesting"
+	"github.com/stretchr/testify/require"
+	"github.com/uptrace/bun"
+	"os"
+	"testing"
+)
+
+func TestRunMigrate(t *testing.T) {
+	require.NoError(t, pgtesting.CreatePostgresServer())
+	t.Cleanup(func() {
+		require.NoError(t, pgtesting.DestroyPostgresServer())
+	})
+
+	connectionOptions := &bunconnect.ConnectionOptions{
+		DatabaseSourceName: pgtesting.Server().GetDatabaseDSN("testing"),
+		Debug:              testing.Verbose(),
+		Writer:             os.Stdout,
+	}
+	executor := func(args []string, db *bun.DB) error {
+		return nil
+	}
+
+	err := run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor)
+	require.NoError(t, err)
+
+	// Must be idempotent
+	err = run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor)
+	require.NoError(t, err)
+}

components/operator/docs/crd.md

@@ -683,7 +683,6 @@ _Appears in:_
 | `ready` _boolean_ |  |
 | `info` _string_ |  |
 | `conditions` _[Condition](#condition) array_ |  |
-| `version` _string_ |  |
 
 
 

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_auths.formance.com.yaml

@@ -167,8 +167,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml

@@ -165,8 +165,6 @@ spec:
                 items:
                   type: string
                 type: array
-              version:
-                type: string
             required:
             - authEnabled
             type: object

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_ledgers.formance.com.yaml

@@ -165,12 +165,8 @@ spec:
                 type: array
               info:
                 type: string
-              isMigratedOnV2:
-                type: boolean
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_orchestrations.formance.com.yaml

@@ -137,8 +137,6 @@ spec:
                 type: boolean
               temporalURI:
                 type: string
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_payments.formance.com.yaml

@@ -137,8 +137,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_reconciliations.formance.com.yaml

@@ -135,8 +135,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_searches.formance.com.yaml

@@ -147,8 +147,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stargates.formance.com.yaml

@@ -152,8 +152,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_wallets.formance.com.yaml

@@ -135,8 +135,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_webhooks.formance.com.yaml

@@ -135,8 +135,6 @@ spec:
                 type: string
               ready:
                 type: boolean
-              version:
-                type: string
             type: object
         type: object
     served: true

libs/go-libs/bun/bunconnect/flags.go

@@ -30,6 +30,7 @@ func InitFlags(flags *pflag.FlagSet) {
 
 func ConnectionOptionsFromFlags(v *viper.Viper, output io.Writer, debug bool) (*ConnectionOptions, error) {
 	var connector func(string) (driver.Connector, error)
+
 	if v.GetBool(PostgresAWSEnableIAMFlag) {
 		cfg, err := config.LoadDefaultConfig(context.Background(), iam.LoadOptionFromViper(v))
 		if err != nil {

libs/go-libs/bun/bunmigrate/command.go

@@ -2,15 +2,8 @@ package bunmigrate
 
 import (
 	"github.com/formancehq/stack/libs/go-libs/bun/bunconnect"
-	sharedlogging "github.com/formancehq/stack/libs/go-libs/logging"
-	"github.com/pkg/errors"
-
-	"github.com/formancehq/stack/libs/go-libs/service"
 	"github.com/spf13/cobra"
-	"github.com/spf13/viper"
 	"github.com/uptrace/bun"
-	"github.com/uptrace/bun/extra/bundebug"
-
 	// Import the postgres driver.
 	_ "github.com/lib/pq"
 )
@@ -31,26 +24,3 @@ func NewDefaultCommand(executor Executor, options ...func(command *cobra.Command
 	bunconnect.InitFlags(ret.Flags())
 	return ret
 }
-
-func Run(cmd *cobra.Command, args []string, executor Executor) error {
-	connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag))
-	if err != nil {
-		return errors.Wrap(err, "evaluating connection options")
-	}
-
-	db, err := bunconnect.OpenSQLDB(*connectionOptions)
-	if err != nil {
-		return errors.Wrap(err, "opening database")
-	}
-	defer func() {
-		err := db.Close()
-		if err != nil {
-			sharedlogging.FromContext(cmd.Context()).Errorf("Closing database: %s", err)
-		}
-	}()
-	if viper.GetBool(service.DebugFlag) {
-		db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(cmd.OutOrStdout())))
-	}
-
-	return errors.Wrap(executor(cmd, args, db), "executing migration")
-}