Skip to content

Commit

Permalink
Migration/schema handling improvements around migration 21 (#6288)
Browse files Browse the repository at this point in the history
## Motivation

Migration 21 has performance issues even with VACUUM INTO being used for migration.
Schema drift detection code prints bad diffs
  • Loading branch information
ivan4th committed Aug 22, 2024
1 parent 82b206e commit 4ab0d09
Show file tree
Hide file tree
Showing 5 changed files with 52 additions and 37 deletions.
7 changes: 3 additions & 4 deletions sql/database_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -345,7 +345,7 @@ var _ Migration = &faultyMigration{}

func (m *faultyMigration) Apply(db Executor, logger *zap.Logger) error {
if m.interceptVacuumInto {
db.(Database).Intercept("crashOnVacuum", func(query string) error {
db.(*sqliteTx).db.Intercept("crashOnVacuum", func(query string) error {
if strings.Contains(strings.ToLower(query), "vacuum into") {
panic("simulated crash")
}
Expand Down Expand Up @@ -563,7 +563,7 @@ func TestSchemaDrift(t *testing.T) {
WithLogger(logger),
)
require.Error(t, err)
require.Regexp(t, `.*\n.*\+.*CREATE TABLE newtbl \(id int\);`, err.Error())
require.Contains(t, err.Error(), "newtbl")
require.Equal(t, 0, observedLogs.Len(), "expected 0 log messages")

db, err = Open("file:"+dbFile,
Expand All @@ -575,8 +575,7 @@ func TestSchemaDrift(t *testing.T) {
require.NoError(t, err)
require.Equal(t, 1, observedLogs.Len(), "expected 1 log messages")
require.Equal(t, "database schema drift detected", observedLogs.All()[0].Message)
require.Regexp(t, `.*\n.*\+.*CREATE TABLE newtbl \(id int\);`,
observedLogs.All()[0].ContextMap()["diff"])
require.Contains(t, observedLogs.All()[0].ContextMap()["diff"], "newtbl")
}

func TestExclusive(t *testing.T) {
Expand Down
11 changes: 6 additions & 5 deletions sql/schema.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,9 @@ type Schema struct {
// Diff diffs the database schema against the actual schema.
// If there's no differences, it returns an empty string.
func (s *Schema) Diff(actualScript string) string {
opt := cmp.Comparer(func(x, y string) bool {
return strings.Join(strings.Fields(x), "") == strings.Join(strings.Fields(y), "")
})
return cmp.Diff(s.Script, actualScript, opt)
// If the difference is only in whitespaces, consider the schemas equal.
return cmp.Diff(strings.Join(strings.Fields(s.Script), " "),
strings.Join(strings.Fields(actualScript), " "))
}

// WriteToFile writes the schema to the corresponding updated schema file.
Expand Down Expand Up @@ -189,7 +188,9 @@ func (s *Schema) MigrateTempDB(logger *zap.Logger, db Database, before int) erro
}

if _, ok := s.skipMigration[m.Order()]; !ok {
if err := m.Apply(db, logger); err != nil {
if err := db.WithTx(context.Background(), func(tx Transaction) error {
return m.Apply(tx, logger)
}); err != nil {
return fmt.Errorf("apply %s: %w", m.Name(), err)
}
}
Expand Down
38 changes: 27 additions & 11 deletions sql/statesql/migrations/state_0021_migration.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,26 +61,42 @@ func (m *migration0021) Apply(db sql.Executor, logger *zap.Logger) error {
progress := float64(processed) * 100.0 / float64(total)
logger.Info("processed ATXs", zap.Float64("progress [%]", progress))
if processed >= total {
return nil
break
}
}

if err := m.applyIndices(db); err != nil {
return err
}

return nil
}

const tableSQL = `CREATE TABLE posts (
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL
);`

func (m *migration0021) applySql(db sql.Executor) error {
query := `CREATE TABLE posts (
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL,
UNIQUE (atxid, pubkey)
);`
_, err := db.Exec(query, nil, nil)
_, err := db.Exec(tableSQL, nil, nil)
if err != nil {
return fmt.Errorf("creating posts table: %w", err)
}

query = "CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);"
return nil
}

func (m *migration0021) applyIndices(db sql.Executor) error {
query := "CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);"
_, err := db.Exec(query, nil, nil)
if err != nil {
return fmt.Errorf("creating index `posts_by_atxid_by_pubkey`: %w", err)
}

query = "CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);"
_, err = db.Exec(query, nil, nil)
if err != nil {
return fmt.Errorf("creating index `posts_by_atxid_by_pubkey`: %w", err)
Expand Down
17 changes: 8 additions & 9 deletions sql/statesql/schema/migrations/0021_atx_posts.sql
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
-- Table showing the PoST commitment by a smesher in given ATX.
-- It shows the exact number of space units committed and the previous ATX id.
CREATE TABLE posts (
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL,
UNIQUE (atxid, pubkey)
);

CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL
);
CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);
CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);

ALTER TABLE atxs DROP COLUMN prev_id;
16 changes: 8 additions & 8 deletions sql/statesql/schema/schema.sql
Original file line number Diff line number Diff line change
Expand Up @@ -105,14 +105,14 @@ CREATE TABLE poets
);
CREATE INDEX poets_by_service_id_by_round_id ON poets (service_id, round_id);
CREATE TABLE posts (
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL,
UNIQUE (atxid, pubkey)
);
CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);
atxid CHAR(32) NOT NULL,
pubkey CHAR(32) NOT NULL,
prev_atxid CHAR(32),
prev_atx_index INT,
units INT NOT NULL
);
CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);
CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);
CREATE TABLE proposal_transactions
(
tid CHAR(32),
Expand Down

0 comments on commit 4ab0d09

Please sign in to comment.