From 78ae83bea98bee36973eb43dd2e3116eed73a964 Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Thu, 23 Oct 2025 13:59:43 +0300 Subject: [PATCH 1/2] MDEV-34124: Improve sequences replication with Galera - use shared key for sequence update certification - employ native replication's code to apply changes for sequences which handles all corner cases properly - fix the tests to allow more transactions using sequences to be accepted That way the sequence is always updated to the maximum value independent of the order of updates, and shared certification keys allow to improve acceptance ratio of concurrent transactions that use sequences. It's reflected in the test changes. --- .../suite/galera/r/galera_sequences.result | 2 +- .../r/galera_sequences_transaction.result | 28 +++++++++++++++---- mysql-test/suite/galera/r/mdev-22063.result | 2 +- .../suite/galera/t/galera_sequences.test | 11 ++------ .../t/galera_sequences_transaction.test | 2 -- sql/log_event_server.cc | 20 ++++++++----- storage/innobase/handler/ha_innodb.cc | 19 ++++++++----- storage/innobase/include/ha_prototypes.h | 6 ++-- storage/innobase/row/row0ins.cc | 2 +- 9 files changed, 57 insertions(+), 35 deletions(-) diff --git a/mysql-test/suite/galera/r/galera_sequences.result b/mysql-test/suite/galera/r/galera_sequences.result index 7cdeffff1e30d..bacd0f07c9709 100644 --- a/mysql-test/suite/galera/r/galera_sequences.result +++ b/mysql-test/suite/galera/r/galera_sequences.result @@ -206,7 +206,7 @@ t CREATE TABLE `t` ( ) ENGINE=InnoDB SEQUENCE=1 connection node_1; DROP SEQUENCE t; -CREATE SEQUENCE t INCREMENT BY 1 NOCACHE ENGINE=INNODB; +CREATE SEQUENCE t INCREMENT BY 0 CACHE 1 ENGINE=INNODB; CREATE TABLE t1(a int not null primary key default nextval(t), b int) engine=innodb; connection node_2; # Wait DDL to replicate diff --git a/mysql-test/suite/galera/r/galera_sequences_transaction.result b/mysql-test/suite/galera/r/galera_sequences_transaction.result index c1cfdc4aa2092..80bfe44093e47 100644 --- a/mysql-test/suite/galera/r/galera_sequences_transaction.result +++ b/mysql-test/suite/galera/r/galera_sequences_transaction.result @@ -72,7 +72,7 @@ LASTVAL(s) connection node_1a; SELECT LASTVAL(s); LASTVAL(s) -79 +81 connection node_1; SELECT * FROM t1; f1 f2 @@ -106,7 +106,6 @@ f1 f2 56 1 58 1 60 1 -61 1 63 1 65 1 67 1 @@ -116,6 +115,7 @@ f1 f2 75 1 77 1 79 1 +81 1 connection node_2; SELECT * FROM t1; f1 f2 @@ -149,7 +149,6 @@ f1 f2 56 1 58 1 60 1 -61 1 63 1 65 1 67 1 @@ -159,6 +158,7 @@ f1 f2 75 1 77 1 79 1 +81 1 connection node_1; DROP TABLE t1; DROP SEQUENCE s; @@ -299,10 +299,8 @@ connection node_1a; ROLLBACK; connection node_2; COMMIT; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction connection node_2a; ROLLBACK; -ERROR 40001: Deadlock found when trying to get lock; try restarting transaction connection node_2; SELECT LASTVAL(s); LASTVAL(s) @@ -332,6 +330,16 @@ f1 f2 15 1 17 1 19 1 +22 1 +24 1 +26 1 +28 1 +30 1 +32 1 +34 1 +36 1 +38 1 +40 1 connection node_2; SELECT * FROM t1; f1 f2 @@ -345,6 +353,16 @@ f1 f2 15 1 17 1 19 1 +22 1 +24 1 +26 1 +28 1 +30 1 +32 1 +34 1 +36 1 +38 1 +40 1 connection node_1; DROP TABLE t1; DROP SEQUENCE s; diff --git a/mysql-test/suite/galera/r/mdev-22063.result b/mysql-test/suite/galera/r/mdev-22063.result index 228f63d6688bc..886a84b05e305 100644 --- a/mysql-test/suite/galera/r/mdev-22063.result +++ b/mysql-test/suite/galera/r/mdev-22063.result @@ -22,7 +22,7 @@ SELECT * FROM t1; a SELECT * FROM s; next_not_cached_value minimum_value maximum_value start_value increment cache_size cycle_option cycle_count -1 1 9223372036854775806 1 1 1000 0 0 +1 1 9223372036854775806 1 1 0 0 0 connection node_1; SET GLOBAL WSREP_MODE='REPLICATE_ARIA,REPLICATE_MYISAM'; DROP TABLE t1; diff --git a/mysql-test/suite/galera/t/galera_sequences.test b/mysql-test/suite/galera/t/galera_sequences.test index 9e18353893b2d..c9ab8db237aab 100644 --- a/mysql-test/suite/galera/t/galera_sequences.test +++ b/mysql-test/suite/galera/t/galera_sequences.test @@ -163,7 +163,8 @@ SHOW CREATE TABLE t; --connection node_1 DROP SEQUENCE t; -CREATE SEQUENCE t INCREMENT BY 1 NOCACHE ENGINE=INNODB; +# Don't use NOCACHE as it may produce duplicate entries from multiple nodes +CREATE SEQUENCE t INCREMENT BY 0 CACHE 1 ENGINE=INNODB; CREATE TABLE t1(a int not null primary key default nextval(t), b int) engine=innodb; --connection node_2 @@ -193,15 +194,11 @@ SET SESSION wsrep_sync_wait=0; while ($count) { --connection node_1 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (1); --connection node_2 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (2); ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (2); --connection node_1 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (1); --dec $count } @@ -251,15 +248,11 @@ SET SESSION wsrep_sync_wait=0; while ($count) { --connection node_1 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (1),(2),(3),(4),(5),(6),(7),(8),(9); --connection node_2 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (21),(22),(23),(24),(25),(26),(27),(28),(29); ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (21),(22),(23),(24),(25),(26),(27),(28),(29); --connection node_1 ---error 0,ER_LOCK_WAIT_TIMEOUT,ER_LOCK_DEADLOCK INSERT INTO t1(b) values (1),(2),(3),(4),(5),(6),(7),(8),(9); --dec $count } diff --git a/mysql-test/suite/galera/t/galera_sequences_transaction.test b/mysql-test/suite/galera/t/galera_sequences_transaction.test index f3dc7d51285a8..facf65a02fef5 100644 --- a/mysql-test/suite/galera/t/galera_sequences_transaction.test +++ b/mysql-test/suite/galera/t/galera_sequences_transaction.test @@ -230,10 +230,8 @@ COMMIT; --connection node_1a ROLLBACK; --connection node_2 ---error ER_LOCK_DEADLOCK COMMIT; --connection node_2a ---error ER_LOCK_DEADLOCK ROLLBACK; --connection node_2 diff --git a/sql/log_event_server.cc b/sql/log_event_server.cc index 3700cb55ac0a4..eb55fde1e8060 100644 --- a/sql/log_event_server.cc +++ b/sql/log_event_server.cc @@ -7666,15 +7666,21 @@ int Rows_log_event::update_sequence() bool old_master= false; int err= 0; - if (!bitmap_is_set(table->rpl_write_set, MIN_VALUE_FIELD_NO) || - ( -#if defined(WITH_WSREP) - ! WSREP(thd) && + rpl_group_info *table_rgi= +#ifdef WITH_WSREP + WSREP(thd) ? thd->wsrep_rgi : #endif - table->in_use->rgi_slave && - !(table->in_use->rgi_slave->gtid_ev_flags2 & Gtid_log_event::FL_DDL) && + table->in_use->rgi_slave; + rpl_group_info *thd_rgi= +#ifdef WITH_WSREP + WSREP(thd) ? thd->wsrep_rgi : +#endif + thd->rgi_slave; + if (!bitmap_is_set(table->rpl_write_set, MIN_VALUE_FIELD_NO) || + (table_rgi && + !(table_rgi->gtid_ev_flags2 & Gtid_log_event::FL_DDL) && !(old_master= - rpl_master_has_bug(thd->rgi_slave->rli, + rpl_master_has_bug(thd_rgi->rli, 29621, FALSE, FALSE, FALSE, TRUE)))) { /* This event come from a setval function executed on the master. diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 19582b79b2eeb..e5d51be516951 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8527,12 +8527,15 @@ wsrep_calc_row_hash( return(0); } -/** Append table-level exclusive key. +/** Append table-level exclusive/shared key. @param thd MySQL thread handle @param table table +@param exclusive Exclusive not shared certification key. @retval false on success @retval true on failure */ -ATTRIBUTE_COLD bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &table) +ATTRIBUTE_COLD bool wsrep_append_table_key(MYSQL_THD thd, + const dict_table_t &table, + bool exclusive) { char db_buf[NAME_LEN + 1]; char tbl_buf[NAME_LEN + 1]; @@ -8545,9 +8548,11 @@ ATTRIBUTE_COLD bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &ta return true; } - /* Append table-level exclusive key */ - const int rcode = wsrep_thd_append_table_key(thd, db_buf, - tbl_buf, WSREP_SERVICE_KEY_EXCLUSIVE); + /* Append table-level key */ + const enum Wsrep_service_key_type key_type = exclusive + ? WSREP_SERVICE_KEY_EXCLUSIVE + : WSREP_SERVICE_KEY_SHARED; + const int rcode = wsrep_thd_append_table_key(thd, db_buf, tbl_buf, key_type); if (rcode) { WSREP_ERROR("Appending table key failed: %s, %d", @@ -8710,10 +8715,10 @@ ha_innobase::update_row( && (thd_sql_command(m_user_thd) != SQLCOM_LOAD || thd_binlog_format(m_user_thd) == BINLOG_FORMAT_ROW)) { - /* We use table-level exclusive key for SEQUENCES + /* We use table-level shared key for SEQUENCES and normal key append for others. */ if (table->s->table_type == TABLE_TYPE_SEQUENCE) { - if (wsrep_append_table_key(m_user_thd, *m_prebuilt->table)) + if (wsrep_append_table_key(m_user_thd, *m_prebuilt->table, false)) DBUG_RETURN(HA_ERR_INTERNAL_ERROR); } else if (wsrep_append_keys(m_user_thd, wsrep_protocol_version >= 4 diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 098986febdf71..548a8fd38a05b 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -436,13 +436,15 @@ char *dict_table_lookup(LEX_CSTRING db, LEX_CSTRING name, dict_table_t **table, mem_heap_t *heap) noexcept; #ifdef WITH_WSREP -/** Append table-level exclusive key. +/** Append table-level exclusive/shared key. @param thd MySQL thread handle @param table table +@param exclusive Exclusive not shared certification key. @retval false on success @retval true on failure */ struct dict_table_t; -bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &table); +bool wsrep_append_table_key(MYSQL_THD thd, const dict_table_t &table, + bool exclusive); #endif /* WITH_WSREP */ #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index 31cefab99cd2e..687d572f79425 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -2770,7 +2770,7 @@ row_ins_clust_index_entry_low( #ifdef WITH_WSREP if (trx->is_wsrep() && - wsrep_append_table_key(trx->mysql_thd, *index->table)) + wsrep_append_table_key(trx->mysql_thd, *index->table, true)) { trx->error_state = DB_ROLLBACK; goto err_exit; From 4dcbf8b049b96f5e7b034d4bc65ca083976c3a16 Mon Sep 17 00:00:00 2001 From: Denis Protivensky Date: Fri, 24 Oct 2025 16:23:28 +0300 Subject: [PATCH 2/2] MDEV-34124: Test sequences recovery after crash in Galera cluster --- .../galera/r/galera_sequences_recovery.result | 132 ++++++++++++++++++ .../galera/t/galera_sequences_recovery.cnf | 9 ++ .../t/galera_sequences_recovery.combinations | 5 + .../galera/t/galera_sequences_recovery.test | 123 ++++++++++++++++ 4 files changed, 269 insertions(+) create mode 100644 mysql-test/suite/galera/r/galera_sequences_recovery.result create mode 100644 mysql-test/suite/galera/t/galera_sequences_recovery.cnf create mode 100644 mysql-test/suite/galera/t/galera_sequences_recovery.combinations create mode 100644 mysql-test/suite/galera/t/galera_sequences_recovery.test diff --git a/mysql-test/suite/galera/r/galera_sequences_recovery.result b/mysql-test/suite/galera/r/galera_sequences_recovery.result new file mode 100644 index 0000000000000..1125f3faf8117 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_sequences_recovery.result @@ -0,0 +1,132 @@ +connection node_2; +connection node_1; +connection node_1; +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY DEFAULT NEXTVAL(s), f2 INT) ENGINE=InnoDB; +connection node_1; +BEGIN; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +connection node_2; +BEGIN; +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +Killing server ... +SELECT NEXTVAL(s); +NEXTVAL(s) +12 +connection node_1; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +COMMIT; +SELECT LASTVAL(s); +LASTVAL(s) +29 +SELECT * FROM t1; +f1 f2 +1 1 +3 1 +5 1 +7 1 +9 1 +21 1 +23 1 +25 1 +27 1 +29 1 +connection node_2; +SELECT LASTVAL(s); +LASTVAL(s) +12 +SELECT NEXTVAL(s); +NEXTVAL(s) +32 +SELECT * FROM t1; +f1 f2 +1 1 +3 1 +5 1 +7 1 +9 1 +21 1 +23 1 +25 1 +27 1 +29 1 +connection node_1; +DROP TABLE t1; +DROP SEQUENCE s; +connection node_1; +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY DEFAULT NEXTVAL(s), f2 INT) ENGINE=InnoDB; +connection node_1; +BEGIN; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +connection node_2; +BEGIN; +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +Killing server ... +connection node_1; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +COMMIT; +SELECT LASTVAL(s); +LASTVAL(s) +19 +SELECT * FROM t1; +f1 f2 +1 1 +3 1 +5 1 +7 1 +9 1 +11 1 +13 1 +15 1 +17 1 +19 1 +connection node_2; +SELECT NEXTVAL(s); +NEXTVAL(s) +22 +SELECT * FROM t1; +f1 f2 +1 1 +3 1 +5 1 +7 1 +9 1 +11 1 +13 1 +15 1 +17 1 +19 1 +connection node_1; +SELECT LASTVAL(s); +LASTVAL(s) +19 +SELECT NEXTVAL(s); +NEXTVAL(s) +33 +connection node_1; +DROP TABLE t1; +DROP SEQUENCE s; diff --git a/mysql-test/suite/galera/t/galera_sequences_recovery.cnf b/mysql-test/suite/galera/t/galera_sequences_recovery.cnf new file mode 100644 index 0000000000000..8701e86db5f0e --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_recovery.cnf @@ -0,0 +1,9 @@ +!include ../galera_2nodes.cnf + +[mysqld.1] +auto-increment-increment=2 +auto-increment-offset=1 + +[mysqld.2] +auto-increment-increment=2 +auto-increment-offset=2 diff --git a/mysql-test/suite/galera/t/galera_sequences_recovery.combinations b/mysql-test/suite/galera/t/galera_sequences_recovery.combinations new file mode 100644 index 0000000000000..cef98e75213f7 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_recovery.combinations @@ -0,0 +1,5 @@ +[binlogon] +log-bin +log-slave-updates + +[binlogoff] diff --git a/mysql-test/suite/galera/t/galera_sequences_recovery.test b/mysql-test/suite/galera/t/galera_sequences_recovery.test new file mode 100644 index 0000000000000..4b59d0ad1afed --- /dev/null +++ b/mysql-test/suite/galera/t/galera_sequences_recovery.test @@ -0,0 +1,123 @@ +# +# Test sequences crash recovery. +# + +--source include/galera_cluster.inc +--source include/have_sequence.inc +--source include/big_test.inc + +--disable_ps2_protocol + +# +# Case 1: Crash a node during a transaction, bring the +# sequence in sync by receiving higher current value +# +--connection node_1 +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY DEFAULT NEXTVAL(s), f2 INT) ENGINE=InnoDB; + +--connection node_1 +BEGIN; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); + +--connection node_2 +BEGIN; +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); + +# Crash and restart the node, losing the transaciton +--source include/kill_galera.inc +--let $start_mysqld_params = +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--source include/start_mysqld.inc + +# Check and update the last written sequence value +SELECT NEXTVAL(s); + +--connection node_1 +# Update the sequence value further +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); + +COMMIT; +SELECT LASTVAL(s); +SELECT * FROM t1; + +--connection node_2 +# The next value should be in sync now, but the last +# value is still the same +SELECT LASTVAL(s); +SELECT NEXTVAL(s); +SELECT * FROM t1; + +--connection node_1 +DROP TABLE t1; +DROP SEQUENCE s; + +# +# Case 2: Crash a node during a transaction, bring the +# sequence in sync by sending higher current value +# +--connection node_1 +CREATE SEQUENCE s INCREMENT=0 CACHE=5 ENGINE=InnoDB; +CREATE TABLE t1 (f1 INT PRIMARY KEY DEFAULT NEXTVAL(s), f2 INT) ENGINE=InnoDB; + +--connection node_1 +BEGIN; +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); + +--connection node_2 +# Exhaust the cached values once, but no replication happens even +# though the sequnce table is updated +BEGIN; +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); +INSERT INTO t1(f2) values (2); + +# Crash and restart the node, losing the transaciton +--source include/kill_galera.inc +--let $start_mysqld_params = +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.2.expect +--source include/start_mysqld.inc + +--connection node_1 +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); +INSERT INTO t1(f2) values (1); + +COMMIT; +SELECT LASTVAL(s); +SELECT * FROM t1; + +--connection node_2 +# Check and update the last written sequence value, it's still +# higher than the one from the other node +SELECT NEXTVAL(s); +SELECT * FROM t1; + +--connection node_1 +# The next value should be in sync now, but the last +# value is still the same +SELECT LASTVAL(s); +SELECT NEXTVAL(s); + +--connection node_1 +DROP TABLE t1; +DROP SEQUENCE s;