From ca35653dbcc7dc6b044db5907db5a1c3ad8b3d76 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Thu, 8 Feb 2024 13:54:32 +0100 Subject: [PATCH] feat: handle migrations in migrate command --- .../ledger/libs/bun/bunconnect/flags.go | 1 + .../ledger/libs/bun/bunmigrate/command.go | 30 ------- components/ledger/libs/bun/bunmigrate/run.go | 88 +++++++++++++++++++ .../ledger/libs/bun/bunmigrate/run_test.go | 34 +++++++ components/operator/docs/crd.md | 1 - ...resourcedefinition_auths.formance.com.yaml | 2 - ...ourcedefinition_gateways.formance.com.yaml | 2 - ...sourcedefinition_ledgers.formance.com.yaml | 4 - ...efinition_orchestrations.formance.com.yaml | 2 - ...ourcedefinition_payments.formance.com.yaml | 2 - ...finition_reconciliations.formance.com.yaml | 2 - ...ourcedefinition_searches.formance.com.yaml | 2 - ...urcedefinition_stargates.formance.com.yaml | 2 - ...sourcedefinition_wallets.formance.com.yaml | 2 - ...ourcedefinition_webhooks.formance.com.yaml | 2 - libs/go-libs/bun/bunconnect/flags.go | 1 + libs/go-libs/bun/bunmigrate/command.go | 30 ------- libs/go-libs/bun/bunmigrate/run.go | 88 +++++++++++++++++++ libs/go-libs/bun/bunmigrate/run_test.go | 34 +++++++ releases/sdks/go/.speakeasy/gen.lock | 2 +- 20 files changed, 247 insertions(+), 84 deletions(-) create mode 100644 components/ledger/libs/bun/bunmigrate/run.go create mode 100644 components/ledger/libs/bun/bunmigrate/run_test.go create mode 100644 libs/go-libs/bun/bunmigrate/run.go create mode 100644 libs/go-libs/bun/bunmigrate/run_test.go diff --git a/components/ledger/libs/bun/bunconnect/flags.go b/components/ledger/libs/bun/bunconnect/flags.go index 58edfdbaee..a0716e23fc 100644 --- a/components/ledger/libs/bun/bunconnect/flags.go +++ b/components/ledger/libs/bun/bunconnect/flags.go @@ -30,6 +30,7 @@ func InitFlags(flags *pflag.FlagSet) { func ConnectionOptionsFromFlags(v *viper.Viper, output io.Writer, debug bool) (*ConnectionOptions, error) { var connector func(string) (driver.Connector, error) + if v.GetBool(PostgresAWSEnableIAMFlag) { cfg, err := config.LoadDefaultConfig(context.Background(), iam.LoadOptionFromViper(v)) if err != nil { diff --git a/components/ledger/libs/bun/bunmigrate/command.go b/components/ledger/libs/bun/bunmigrate/command.go index 6549e988e5..82e6ccd5b3 100644 --- a/components/ledger/libs/bun/bunmigrate/command.go +++ b/components/ledger/libs/bun/bunmigrate/command.go @@ -2,15 +2,8 @@ package bunmigrate import ( "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" - sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" - "github.com/pkg/errors" - - "github.com/formancehq/stack/libs/go-libs/service" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/uptrace/bun" - "github.com/uptrace/bun/extra/bundebug" - // Import the postgres driver. _ "github.com/lib/pq" ) @@ -31,26 +24,3 @@ func NewDefaultCommand(executor Executor, options ...func(command *cobra.Command bunconnect.InitFlags(ret.Flags()) return ret } - -func Run(cmd *cobra.Command, args []string, executor Executor) error { - connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag)) - if err != nil { - return errors.Wrap(err, "evaluating connection options") - } - - db, err := bunconnect.OpenSQLDB(*connectionOptions) - if err != nil { - return errors.Wrap(err, "opening database") - } - defer func() { - err := db.Close() - if err != nil { - sharedlogging.FromContext(cmd.Context()).Errorf("Closing database: %s", err) - } - }() - if viper.GetBool(service.DebugFlag) { - db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(cmd.OutOrStdout()))) - } - - return errors.Wrap(executor(cmd, args, db), "executing migration") -} diff --git a/components/ledger/libs/bun/bunmigrate/run.go b/components/ledger/libs/bun/bunmigrate/run.go new file mode 100644 index 0000000000..93ab374aa8 --- /dev/null +++ b/components/ledger/libs/bun/bunmigrate/run.go @@ -0,0 +1,88 @@ +package bunmigrate + +import ( + "context" + "database/sql" + "fmt" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/pointer" + "github.com/formancehq/stack/libs/go-libs/service" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/uptrace/bun" + "github.com/uptrace/bun/extra/bundebug" + "github.com/xo/dburl" + "io" +) + +func ensureDatabaseExists(ctx context.Context, connectionOptions bunconnect.ConnectionOptions) error { + url, err := dburl.Parse(connectionOptions.DatabaseSourceName) + if err != nil { + return err + } + originalPath := url.Path + url.Path = "postgres" // notes(gfyrag): default "postgres" database (most of the time?) + connectionOptions.DatabaseSourceName = url.String() + + db, err := bunconnect.OpenSQLDB(connectionOptions) + if err != nil { + return errors.Wrap(err, "opening database") + } + defer func() { + err := db.Close() + if err != nil { + sharedlogging.FromContext(ctx).Errorf("Closing database: %s", err) + } + }() + + row := db.QueryRowContext(ctx, `SELECT datname FROM pg_database WHERE datname = ?`, originalPath[1:]) + if row.Err() != nil { + return row.Err() + } + + if err := row.Scan(pointer.For("")); err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return err + } + + _, err = db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE "%s"`, originalPath[1:])) + if err != nil { + return err + } + } + + return nil +} + +func run(ctx context.Context, output io.Writer, args []string, connectionOptions *bunconnect.ConnectionOptions, + executor func(args []string, db *bun.DB) error) error { + + if err := ensureDatabaseExists(ctx, *connectionOptions); err != nil { + return err + } + + db, err := bunconnect.OpenSQLDB(*connectionOptions) + if err != nil { + return errors.Wrap(err, "opening database") + } + defer func() { + _ = db.Close() + }() + if viper.GetBool(service.DebugFlag) { + db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(output))) + } + + return errors.Wrap(executor(args, db), "executing migration") +} + +func Run(cmd *cobra.Command, args []string, executor Executor) error { + connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag)) + if err != nil { + return errors.Wrap(err, "evaluating connection options") + } + return run(cmd.Context(), cmd.OutOrStdout(), args, connectionOptions, func(args []string, db *bun.DB) error { + return executor(cmd, args, db) + }) +} diff --git a/components/ledger/libs/bun/bunmigrate/run_test.go b/components/ledger/libs/bun/bunmigrate/run_test.go new file mode 100644 index 0000000000..afc794ddde --- /dev/null +++ b/components/ledger/libs/bun/bunmigrate/run_test.go @@ -0,0 +1,34 @@ +package bunmigrate + +import ( + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/pgtesting" + "github.com/stretchr/testify/require" + "github.com/uptrace/bun" + "os" + "testing" +) + +func TestRunMigrate(t *testing.T) { + require.NoError(t, pgtesting.CreatePostgresServer()) + t.Cleanup(func() { + require.NoError(t, pgtesting.DestroyPostgresServer()) + }) + + connectionOptions := &bunconnect.ConnectionOptions{ + DatabaseSourceName: pgtesting.Server().GetDatabaseDSN("testing"), + Debug: testing.Verbose(), + Writer: os.Stdout, + } + executor := func(args []string, db *bun.DB) error { + return nil + } + + err := run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor) + require.NoError(t, err) + + // Must be idempotent + err = run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor) + require.NoError(t, err) +} diff --git a/components/operator/docs/crd.md b/components/operator/docs/crd.md index 76937d0b8d..8fe4e1ec1c 100644 --- a/components/operator/docs/crd.md +++ b/components/operator/docs/crd.md @@ -683,7 +683,6 @@ _Appears in:_ | `ready` _boolean_ | | | `info` _string_ | | | `conditions` _[Condition](#condition) array_ | | -| `version` _string_ | | diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_auths.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_auths.formance.com.yaml index 366eb0ef88..a605f59ba3 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_auths.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_auths.formance.com.yaml @@ -167,8 +167,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml index d7bdb2d24c..dc045a8a77 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_gateways.formance.com.yaml @@ -165,8 +165,6 @@ spec: items: type: string type: array - version: - type: string required: - authEnabled type: object diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_ledgers.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_ledgers.formance.com.yaml index d274e47d92..cafc5b3fde 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_ledgers.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_ledgers.formance.com.yaml @@ -165,12 +165,8 @@ spec: type: array info: type: string - isMigratedOnV2: - type: boolean ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_orchestrations.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_orchestrations.formance.com.yaml index 1a0ef94288..0de6942809 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_orchestrations.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_orchestrations.formance.com.yaml @@ -137,8 +137,6 @@ spec: type: boolean temporalURI: type: string - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_payments.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_payments.formance.com.yaml index c39e00bed1..c0289e4d3a 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_payments.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_payments.formance.com.yaml @@ -137,8 +137,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_reconciliations.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_reconciliations.formance.com.yaml index d2941d0e1a..c2c461cec5 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_reconciliations.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_reconciliations.formance.com.yaml @@ -135,8 +135,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_searches.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_searches.formance.com.yaml index 94e36e9ad2..405520d012 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_searches.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_searches.formance.com.yaml @@ -147,8 +147,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stargates.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stargates.formance.com.yaml index 061ff87d46..24f8b64aeb 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stargates.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_stargates.formance.com.yaml @@ -152,8 +152,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_wallets.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_wallets.formance.com.yaml index 0e0732f41d..fe06709fbd 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_wallets.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_wallets.formance.com.yaml @@ -135,8 +135,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_webhooks.formance.com.yaml b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_webhooks.formance.com.yaml index 3c79536c2e..0f5e704adb 100644 --- a/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_webhooks.formance.com.yaml +++ b/components/operator/helm/templates/gen/apiextensions.k8s.io_v1_customresourcedefinition_webhooks.formance.com.yaml @@ -135,8 +135,6 @@ spec: type: string ready: type: boolean - version: - type: string type: object type: object served: true diff --git a/libs/go-libs/bun/bunconnect/flags.go b/libs/go-libs/bun/bunconnect/flags.go index 58edfdbaee..a0716e23fc 100644 --- a/libs/go-libs/bun/bunconnect/flags.go +++ b/libs/go-libs/bun/bunconnect/flags.go @@ -30,6 +30,7 @@ func InitFlags(flags *pflag.FlagSet) { func ConnectionOptionsFromFlags(v *viper.Viper, output io.Writer, debug bool) (*ConnectionOptions, error) { var connector func(string) (driver.Connector, error) + if v.GetBool(PostgresAWSEnableIAMFlag) { cfg, err := config.LoadDefaultConfig(context.Background(), iam.LoadOptionFromViper(v)) if err != nil { diff --git a/libs/go-libs/bun/bunmigrate/command.go b/libs/go-libs/bun/bunmigrate/command.go index 6549e988e5..82e6ccd5b3 100644 --- a/libs/go-libs/bun/bunmigrate/command.go +++ b/libs/go-libs/bun/bunmigrate/command.go @@ -2,15 +2,8 @@ package bunmigrate import ( "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" - sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" - "github.com/pkg/errors" - - "github.com/formancehq/stack/libs/go-libs/service" "github.com/spf13/cobra" - "github.com/spf13/viper" "github.com/uptrace/bun" - "github.com/uptrace/bun/extra/bundebug" - // Import the postgres driver. _ "github.com/lib/pq" ) @@ -31,26 +24,3 @@ func NewDefaultCommand(executor Executor, options ...func(command *cobra.Command bunconnect.InitFlags(ret.Flags()) return ret } - -func Run(cmd *cobra.Command, args []string, executor Executor) error { - connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag)) - if err != nil { - return errors.Wrap(err, "evaluating connection options") - } - - db, err := bunconnect.OpenSQLDB(*connectionOptions) - if err != nil { - return errors.Wrap(err, "opening database") - } - defer func() { - err := db.Close() - if err != nil { - sharedlogging.FromContext(cmd.Context()).Errorf("Closing database: %s", err) - } - }() - if viper.GetBool(service.DebugFlag) { - db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(cmd.OutOrStdout()))) - } - - return errors.Wrap(executor(cmd, args, db), "executing migration") -} diff --git a/libs/go-libs/bun/bunmigrate/run.go b/libs/go-libs/bun/bunmigrate/run.go new file mode 100644 index 0000000000..93ab374aa8 --- /dev/null +++ b/libs/go-libs/bun/bunmigrate/run.go @@ -0,0 +1,88 @@ +package bunmigrate + +import ( + "context" + "database/sql" + "fmt" + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + sharedlogging "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/pointer" + "github.com/formancehq/stack/libs/go-libs/service" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/uptrace/bun" + "github.com/uptrace/bun/extra/bundebug" + "github.com/xo/dburl" + "io" +) + +func ensureDatabaseExists(ctx context.Context, connectionOptions bunconnect.ConnectionOptions) error { + url, err := dburl.Parse(connectionOptions.DatabaseSourceName) + if err != nil { + return err + } + originalPath := url.Path + url.Path = "postgres" // notes(gfyrag): default "postgres" database (most of the time?) + connectionOptions.DatabaseSourceName = url.String() + + db, err := bunconnect.OpenSQLDB(connectionOptions) + if err != nil { + return errors.Wrap(err, "opening database") + } + defer func() { + err := db.Close() + if err != nil { + sharedlogging.FromContext(ctx).Errorf("Closing database: %s", err) + } + }() + + row := db.QueryRowContext(ctx, `SELECT datname FROM pg_database WHERE datname = ?`, originalPath[1:]) + if row.Err() != nil { + return row.Err() + } + + if err := row.Scan(pointer.For("")); err != nil { + if !errors.Is(err, sql.ErrNoRows) { + return err + } + + _, err = db.ExecContext(ctx, fmt.Sprintf(`CREATE DATABASE "%s"`, originalPath[1:])) + if err != nil { + return err + } + } + + return nil +} + +func run(ctx context.Context, output io.Writer, args []string, connectionOptions *bunconnect.ConnectionOptions, + executor func(args []string, db *bun.DB) error) error { + + if err := ensureDatabaseExists(ctx, *connectionOptions); err != nil { + return err + } + + db, err := bunconnect.OpenSQLDB(*connectionOptions) + if err != nil { + return errors.Wrap(err, "opening database") + } + defer func() { + _ = db.Close() + }() + if viper.GetBool(service.DebugFlag) { + db.AddQueryHook(bundebug.NewQueryHook(bundebug.WithWriter(output))) + } + + return errors.Wrap(executor(args, db), "executing migration") +} + +func Run(cmd *cobra.Command, args []string, executor Executor) error { + connectionOptions, err := bunconnect.ConnectionOptionsFromFlags(viper.GetViper(), cmd.OutOrStdout(), viper.GetBool(service.DebugFlag)) + if err != nil { + return errors.Wrap(err, "evaluating connection options") + } + return run(cmd.Context(), cmd.OutOrStdout(), args, connectionOptions, func(args []string, db *bun.DB) error { + return executor(cmd, args, db) + }) +} diff --git a/libs/go-libs/bun/bunmigrate/run_test.go b/libs/go-libs/bun/bunmigrate/run_test.go new file mode 100644 index 0000000000..afc794ddde --- /dev/null +++ b/libs/go-libs/bun/bunmigrate/run_test.go @@ -0,0 +1,34 @@ +package bunmigrate + +import ( + "github.com/formancehq/stack/libs/go-libs/bun/bunconnect" + "github.com/formancehq/stack/libs/go-libs/logging" + "github.com/formancehq/stack/libs/go-libs/pgtesting" + "github.com/stretchr/testify/require" + "github.com/uptrace/bun" + "os" + "testing" +) + +func TestRunMigrate(t *testing.T) { + require.NoError(t, pgtesting.CreatePostgresServer()) + t.Cleanup(func() { + require.NoError(t, pgtesting.DestroyPostgresServer()) + }) + + connectionOptions := &bunconnect.ConnectionOptions{ + DatabaseSourceName: pgtesting.Server().GetDatabaseDSN("testing"), + Debug: testing.Verbose(), + Writer: os.Stdout, + } + executor := func(args []string, db *bun.DB) error { + return nil + } + + err := run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor) + require.NoError(t, err) + + // Must be idempotent + err = run(logging.TestingContext(), os.Stdout, []string{}, connectionOptions, executor) + require.NoError(t, err) +} diff --git a/releases/sdks/go/.speakeasy/gen.lock b/releases/sdks/go/.speakeasy/gen.lock index 438db82258..a172c5459b 100755 --- a/releases/sdks/go/.speakeasy/gen.lock +++ b/releases/sdks/go/.speakeasy/gen.lock @@ -1,5 +1,5 @@ lockVersion: 2.0.0 -id: 7eac0a45-60a2-40bb-9e85-26bd77ec2a6d +id: 82f13198-0dba-47b8-bce6-ea6e95df109d management: docChecksum: e64f058f8ed1617c198d1fe84ea95670 docVersion: INTERNAL