Migration/schema handling improvements around migration 21 (#6288)

## Motivation

Migration 21 has performance issues even with VACUUM INTO being used for migration.
Schema drift detection code prints bad diffs

sql/database_test.go

@@ -345,7 +345,7 @@ var _ Migration = &faultyMigration{}
 
 func (m *faultyMigration) Apply(db Executor, logger *zap.Logger) error {
 	if m.interceptVacuumInto {
-		db.(Database).Intercept("crashOnVacuum", func(query string) error {
+		db.(*sqliteTx).db.Intercept("crashOnVacuum", func(query string) error {
 			if strings.Contains(strings.ToLower(query), "vacuum into") {
 				panic("simulated crash")
 			}
@@ -563,7 +563,7 @@ func TestSchemaDrift(t *testing.T) {
 		WithLogger(logger),
 	)
 	require.Error(t, err)
-	require.Regexp(t, `.*\n.*\+.*CREATE TABLE newtbl \(id int\);`, err.Error())
+	require.Contains(t, err.Error(), "newtbl")
 	require.Equal(t, 0, observedLogs.Len(), "expected 0 log messages")
 
 	db, err = Open("file:"+dbFile,
@@ -575,8 +575,7 @@ func TestSchemaDrift(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 1, observedLogs.Len(), "expected 1 log messages")
 	require.Equal(t, "database schema drift detected", observedLogs.All()[0].Message)
-	require.Regexp(t, `.*\n.*\+.*CREATE TABLE newtbl \(id int\);`,
-		observedLogs.All()[0].ContextMap()["diff"])
+	require.Contains(t, observedLogs.All()[0].ContextMap()["diff"], "newtbl")
 }
 
 func TestExclusive(t *testing.T) {

sql/schema.go

@@ -56,10 +56,9 @@ type Schema struct {
 // Diff diffs the database schema against the actual schema.
 // If there's no differences, it returns an empty string.
 func (s *Schema) Diff(actualScript string) string {
-	opt := cmp.Comparer(func(x, y string) bool {
-		return strings.Join(strings.Fields(x), "") == strings.Join(strings.Fields(y), "")
-	})
-	return cmp.Diff(s.Script, actualScript, opt)
+	// If the difference is only in whitespaces, consider the schemas equal.
+	return cmp.Diff(strings.Join(strings.Fields(s.Script), " "),
+		strings.Join(strings.Fields(actualScript), " "))
 }
 
 // WriteToFile writes the schema to the corresponding updated schema file.
@@ -189,7 +188,9 @@ func (s *Schema) MigrateTempDB(logger *zap.Logger, db Database, before int) erro
 		}
 
 		if _, ok := s.skipMigration[m.Order()]; !ok {
-			if err := m.Apply(db, logger); err != nil {
+			if err := db.WithTx(context.Background(), func(tx Transaction) error {
+				return m.Apply(tx, logger)
+			}); err != nil {
 				return fmt.Errorf("apply %s: %w", m.Name(), err)
 			}
 		}

sql/statesql/migrations/state_0021_migration.go

@@ -61,26 +61,42 @@ func (m *migration0021) Apply(db sql.Executor, logger *zap.Logger) error {
 		progress := float64(processed) * 100.0 / float64(total)
 		logger.Info("processed ATXs", zap.Float64("progress [%]", progress))
 		if processed >= total {
-			return nil
+			break
 		}
 	}
+
+	if err := m.applyIndices(db); err != nil {
+		return err
+	}
+
+	return nil
 }
 
+const tableSQL = `CREATE TABLE posts (
+    atxid CHAR(32) NOT NULL,
+    pubkey CHAR(32) NOT NULL,
+    prev_atxid CHAR(32),
+    prev_atx_index INT,
+    units INT NOT NULL
+);`
+
 func (m *migration0021) applySql(db sql.Executor) error {
-	query := `CREATE TABLE posts (
-		atxid  CHAR(32) NOT NULL,
-		pubkey CHAR(32) NOT NULL,
-		prev_atxid  CHAR(32),
-		prev_atx_index INT,
-		units  INT NOT NULL,
-		UNIQUE (atxid, pubkey)
-	);`
-	_, err := db.Exec(query, nil, nil)
+	_, err := db.Exec(tableSQL, nil, nil)
 	if err != nil {
 		return fmt.Errorf("creating posts table: %w", err)
 	}
 
-	query = "CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);"
+	return nil
+}
+
+func (m *migration0021) applyIndices(db sql.Executor) error {
+	query := "CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);"
+	_, err := db.Exec(query, nil, nil)
+	if err != nil {
+		return fmt.Errorf("creating index `posts_by_atxid_by_pubkey`: %w", err)
+	}
+
+	query = "CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);"
 	_, err = db.Exec(query, nil, nil)
 	if err != nil {
 		return fmt.Errorf("creating index `posts_by_atxid_by_pubkey`: %w", err)

sql/statesql/schema/migrations/0021_atx_posts.sql

@@ -1,14 +1,13 @@
 -- Table showing the PoST commitment by a smesher in given ATX.
 -- It shows the exact number of space units committed and the previous ATX id.
 CREATE TABLE posts (
-		atxid  CHAR(32) NOT NULL,
-		pubkey CHAR(32) NOT NULL,
-		prev_atxid  CHAR(32),
-		prev_atx_index INT,
-		units  INT NOT NULL,
-		UNIQUE (atxid, pubkey)
-	);
-
-CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);
+    atxid CHAR(32) NOT NULL,
+    pubkey CHAR(32) NOT NULL,
+    prev_atxid CHAR(32),
+    prev_atx_index INT,
+    units INT NOT NULL
+);
+CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);
+CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);
 
 ALTER TABLE atxs DROP COLUMN prev_id;

sql/statesql/schema/schema.sql

@@ -105,14 +105,14 @@ CREATE TABLE poets
 );
 CREATE INDEX poets_by_service_id_by_round_id ON poets (service_id, round_id);
 CREATE TABLE posts (
-		atxid  CHAR(32) NOT NULL,
-		pubkey CHAR(32) NOT NULL,
-		prev_atxid  CHAR(32),
-		prev_atx_index INT,
-		units  INT NOT NULL,
-		UNIQUE (atxid, pubkey)
-	);
-CREATE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey, prev_atxid);
+    atxid CHAR(32) NOT NULL,
+    pubkey CHAR(32) NOT NULL,
+    prev_atxid CHAR(32),
+    prev_atx_index INT,
+    units INT NOT NULL
+);
+CREATE UNIQUE INDEX posts_by_atxid_by_pubkey ON posts (atxid, pubkey);
+CREATE INDEX posts_by_atxid_by_pubkey_prev_atxid ON posts (atxid, pubkey, prev_atxid);
 CREATE TABLE proposal_transactions
 (
     tid     CHAR(32),