Skip to content

Commit

Permalink
Database schema source (#107)
Browse files Browse the repository at this point in the history
  • Loading branch information
bplunkett-stripe authored Feb 7, 2024
1 parent 43a6d6f commit 97fffc8
Show file tree
Hide file tree
Showing 8 changed files with 261 additions and 50 deletions.
14 changes: 9 additions & 5 deletions internal/migration_acceptance_tests/acceptance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ type (
empty bool
}

planFactory func(ctx context.Context, connPool sqldb.Queryable, tempDbFactory tempdb.Factory, newSchemaDDL []string, opts ...diff.PlanOpt) (diff.Plan, error)

acceptanceTestCase struct {
name string
oldSchemaDDL []string
Expand All @@ -54,11 +56,13 @@ type (

// vanillaExpectations refers to the expectations of the migration if no additional opts are used
vanillaExpectations expectations
// dataPackingExpectations refers to the expectations of the migration if table packing is used
// dataPackingExpectations refers to the expectations of the migration if table packing is used. We should
// aim to deprecate this and just split out a separate set of individual tests for data packing.
dataPackingExpectations expectations

// use old generate plan func
useOldGeneratePlan bool
// planFactory is used to generate the actual plan. This is useful for testing different plan generation paths
// outside of the normal path. If not specified, a plan will be generated using a default.
planFactory planFactory
}

acceptanceTestSuite struct {
Expand Down Expand Up @@ -124,8 +128,8 @@ func (suite *acceptanceTestSuite) runSubtest(tc acceptanceTestCase, expects expe
suite.Require().NoError(tempDbFactory.Close())
}(tempDbFactory)

generatePlanFn := diff.GeneratePlan
if !tc.useOldGeneratePlan {
generatePlanFn := tc.planFactory
if generatePlanFn == nil {
generatePlanFn = func(ctx context.Context, connPool sqldb.Queryable, tempDbFactory tempdb.Factory, newSchemaDDL []string, opts ...diff.PlanOpt) (diff.Plan, error) {
return diff.Generate(ctx, connPool, diff.DDLSchemaSource(newSchemaDDL),
append(planOpts,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ var backCompatAcceptanceTestCases = []acceptanceTestCase{
},

// Ensure that we're maintaining backwards compatibility with the old generate plan func
useOldGeneratePlan: true,
planFactory: diff.GeneratePlan,
},
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,130 @@
package migration_acceptance_tests

import (
"context"
"fmt"

"github.com/stripe/pg-schema-diff/pkg/diff"
"github.com/stripe/pg-schema-diff/pkg/sqldb"
"github.com/stripe/pg-schema-diff/pkg/tempdb"
)

func databaseSchemaSourcePlanFactory(ctx context.Context, connPool sqldb.Queryable, tempDbFactory tempdb.Factory, newSchemaDDL []string, opts ...diff.PlanOpt) (_ diff.Plan, retErr error) {
newSchemaDb, err := tempDbFactory.Create(ctx)
if err != nil {
return diff.Plan{}, fmt.Errorf("creating temp database: %w", err)
}

defer func() {
tempDbErr := newSchemaDb.Close(ctx)
if retErr == nil {
retErr = tempDbErr
}
}()

for _, stmt := range newSchemaDDL {
if _, err := newSchemaDb.ConnPool.ExecContext(ctx, stmt); err != nil {
return diff.Plan{}, fmt.Errorf("running DDL: %w", err)
}
}

// Clone the opts so we don't modify the original.
opts = append([]diff.PlanOpt(nil), opts...)
opts = append(opts, diff.WithTempDbFactory(tempDbFactory))
for _, o := range newSchemaDb.ExcludeMetadatOptions {
opts = append(opts, diff.WithGetSchemaOpts(o))
}

return diff.Generate(ctx, connPool, diff.DBSchemaSource(newSchemaDb.ConnPool), opts...)
}

var databaseSchemaSourceTestCases = []acceptanceTestCase{
{
name: "Drop partitioned table, Add partitioned table with local keys",
oldSchemaDDL: []string{
`
CREATE TABLE fizz();
CREATE TABLE foobar(
id INT,
bar SERIAL NOT NULL,
foo VARCHAR(255) DEFAULT 'some default' NOT NULL CHECK (LENGTH(foo) > 0),
fizz TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (foo, id),
UNIQUE (foo, bar)
) PARTITION BY LIST(foo);
CREATE TABLE foobar_1 PARTITION of foobar(
fizz NOT NULL
) FOR VALUES IN ('foobar_1_val_1', 'foobar_1_val_2');
-- partitioned indexes
CREATE INDEX foobar_normal_idx ON foobar(foo, bar);
CREATE UNIQUE INDEX foobar_unique_idx ON foobar(foo, fizz);
-- local indexes
CREATE INDEX foobar_1_local_idx ON foobar_1(foo, bar);
CREATE table bar(
id VARCHAR(255) PRIMARY KEY,
foo VARCHAR(255),
bar DOUBLE PRECISION NOT NULL DEFAULT 8.8,
fizz TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
buzz REAL NOT NULL CHECK (buzz IS NOT NULL),
FOREIGN KEY (foo, fizz) REFERENCES foobar (foo, fizz)
);
CREATE INDEX bar_normal_idx ON bar(bar);
CREATE INDEX bar_another_normal_id ON bar(bar, fizz);
CREATE UNIQUE INDEX bar_unique_idx on bar(fizz, buzz);
`,
},
newSchemaDDL: []string{
`
CREATE TABLE fizz();
CREATE SCHEMA schema_1;
CREATE TABLE schema_1.foobar(
bar TIMESTAMPTZ NOT NULL,
fizz TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
id INT,
foo VARCHAR(255) DEFAULT 'some default' NOT NULL CHECK (LENGTH(foo) > 0),
UNIQUE (foo, bar)
) PARTITION BY LIST(foo);
CREATE TABLE schema_1.foobar_1 PARTITION of schema_1.foobar(
fizz NOT NULL,
PRIMARY KEY (foo, bar)
) FOR VALUES IN ('foobar_1_val_1', 'foobar_1_val_2');
-- local indexes
CREATE INDEX foobar_1_local_idx ON schema_1.foobar_1(foo, bar);
-- partitioned indexes
CREATE INDEX foobar_normal_idx ON schema_1.foobar(foo, bar);
CREATE UNIQUE INDEX foobar_unique_idx ON schema_1.foobar(foo, fizz);
CREATE table bar(
id VARCHAR(255) PRIMARY KEY,
foo VARCHAR(255),
bar DOUBLE PRECISION NOT NULL DEFAULT 8.8,
fizz TIMESTAMPTZ DEFAULT CURRENT_TIMESTAMP,
buzz REAL NOT NULL CHECK (buzz IS NOT NULL),
FOREIGN KEY (foo, fizz) REFERENCES schema_1.foobar (foo, fizz)
);
CREATE INDEX bar_normal_idx ON bar(bar);
CREATE INDEX bar_another_normal_id ON bar(bar, fizz);
CREATE UNIQUE INDEX bar_unique_idx on bar(fizz, buzz);
`,
},
expectedHazardTypes: []diff.MigrationHazardType{
diff.MigrationHazardTypeAcquiresShareRowExclusiveLock,
diff.MigrationHazardTypeDeletesData,
},

planFactory: databaseSchemaSourcePlanFactory,
},
}

func (suite *acceptanceTestSuite) TestDatabaseSchemaSourceTestCases() {
suite.runTestCases(databaseSchemaSourceTestCases)
}
13 changes: 13 additions & 0 deletions pkg/diff/plan_generator.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
_ "github.com/jackc/pgx/v4/stdlib"
"github.com/kr/pretty"
"github.com/stripe/pg-schema-diff/internal/schema"
externalschema "github.com/stripe/pg-schema-diff/pkg/schema"

"github.com/stripe/pg-schema-diff/pkg/log"
"github.com/stripe/pg-schema-diff/pkg/sqldb"
Expand Down Expand Up @@ -80,6 +81,18 @@ func WithSchemas(schemas ...string) PlanOpt {
}
}

func WithExcludeSchemas(schemas ...string) PlanOpt {
return func(opts *planOptions) {
opts.getSchemaOpts = append(opts.getSchemaOpts, schema.WithExcludeSchemas(schemas...))
}
}

func WithGetSchemaOpts(getSchemaOpts ...externalschema.GetSchemaOpt) PlanOpt {
return func(opts *planOptions) {
opts.getSchemaOpts = append(opts.getSchemaOpts, getSchemaOpts...)
}
}

// deprecated: GeneratePlan generates a migration plan to migrate the database to the target schema. This function only
// diffs the public schemas.
//
Expand Down
Loading

0 comments on commit 97fffc8

Please sign in to comment.