From 0ad0a466ad08c20f8a71b1fdd501c339eda751d7 Mon Sep 17 00:00:00 2001 From: Arthur Schreiber Date: Sat, 8 Jun 2024 11:03:07 +0000 Subject: [PATCH] Do not load table stats when booting `vttablet`. Signed-off-by: Arthur Schreiber --- go/mysql/flavor_mysql.go | 6 +- go/vt/vtexplain/vtexplain_vttablet.go | 5 + .../tabletserver/health_streamer_test.go | 28 ++++ .../tabletserver/query_engine_test.go | 11 ++ .../tabletserver/query_executor_test.go | 9 ++ go/vt/vttablet/tabletserver/schema/engine.go | 2 +- .../tabletserver/schema/engine_test.go | 135 ++++++++++++------ .../vttablet/tabletserver/schema/main_test.go | 1 + .../tabletserver/tabletserver_test.go | 8 ++ 9 files changed, 155 insertions(+), 50 deletions(-) diff --git a/go/mysql/flavor_mysql.go b/go/mysql/flavor_mysql.go index 612bb04fac7..5c07031bdff 100644 --- a/go/mysql/flavor_mysql.go +++ b/go/mysql/flavor_mysql.go @@ -299,9 +299,11 @@ func (mysqlFlavor) readBinlogEvent(c *Conn) (BinlogEvent, error) { // baseShowTables is part of the Flavor interface. func (mysqlFlavor) baseShowTables() string { - return "SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()" + return BaseShowTables } +const BaseShowTables = `SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()` + // TablesWithSize56 is a query to select table along with size for mysql 5.6 const TablesWithSize56 = `SELECT table_name, table_type, @@ -340,8 +342,6 @@ LEFT OUTER JOIN ( WHERE t.table_schema = database() GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment` -// TablesWithSize80 is a query to select table along with size for mysql 8.0 -// // Note the following: // - We use a single query to fetch both partitioned and non-partitioned tables. This is because // accessing `information_schema.innodb_tablespaces` is expensive on servers with many tablespaces, diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 24c17b8f969..aa33875657b 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -438,6 +438,11 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) (*tablet Rows: showTableRows, }) + tEnv.addResult(mysql.BaseShowTables, &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields[0:4], + Rows: showTableRows, + }) + indexRows := make([][]sqltypes.Value, 0, 4) for _, ddl := range ddls { table := sqlparser.String(ddl.GetTable().Name) diff --git a/go/vt/vttablet/tabletserver/health_streamer_test.go b/go/vt/vttablet/tabletserver/health_streamer_test.go index 7f19d640d7f..3efd0001a4f 100644 --- a/go/vt/vttablet/tabletserver/health_streamer_test.go +++ b/go/vt/vttablet/tabletserver/health_streamer_test.go @@ -249,6 +249,17 @@ func TestReloadSchema(t *testing.T) { "product|BASE TABLE|1684735966||114688|114688", "users|BASE TABLE|1684735966||114688|114688", )) + + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT", + "varchar|varchar|int64|varchar", + ), + "product|BASE TABLE|1684735966|", + "users|BASE TABLE|1684735966|", + )) + db.AddQueryPattern("SELECT COLUMN_NAME as column_name.*", sqltypes.MakeTestResult( sqltypes.MakeTestFields( "column_name", @@ -293,6 +304,16 @@ func TestReloadSchema(t *testing.T) { "users|BASE TABLE|1684735967||114688|114688", )) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT", + "varchar|varchar|int64|varchar", + ), + "product|BASE TABLE|1684735967|", + "users|BASE TABLE|1684735967|", + )) + var wg sync.WaitGroup wg.Add(1) go func() { @@ -359,6 +380,13 @@ func TestReloadView(t *testing.T) { "varchar|varchar|int64|varchar|int64|int64", ), )) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + sqltypes.MakeTestResult( + sqltypes.MakeTestFields( + "TABLE_NAME | TABLE_TYPE | UNIX_TIMESTAMP(t.create_time) | TABLE_COMMENT", + "varchar|varchar|int64|varchar", + ), + )) db.AddQueryPattern("SELECT COLUMN_NAME as column_name.*", sqltypes.MakeTestResult( sqltypes.MakeTestFields( "column_name", diff --git a/go/vt/vttablet/tabletserver/query_engine_test.go b/go/vt/vttablet/tabletserver/query_engine_test.go index 33849c3f5e6..53236db3230 100644 --- a/go/vt/vttablet/tabletserver/query_engine_test.go +++ b/go/vt/vttablet/tabletserver/query_engine_test.go @@ -121,6 +121,17 @@ func addSchemaEngineQueries(db *fakesqldb.DB) { mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }}) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) db.AddQuery("show status like 'Innodb_rows_read'", sqltypes.MakeTestResult(sqltypes.MakeTestFields( "Variable_name|Value", "varchar|int64"), diff --git a/go/vt/vttablet/tabletserver/query_executor_test.go b/go/vt/vttablet/tabletserver/query_executor_test.go index 3ab653bf50c..8f4e75bb126 100644 --- a/go/vt/vttablet/tabletserver/query_executor_test.go +++ b/go/vt/vttablet/tabletserver/query_executor_test.go @@ -1557,6 +1557,15 @@ func initQueryExecutorTestDB(db *fakesqldb.DB) { mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }, }) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) db.AddQuery("show status like 'Innodb_rows_read'", sqltypes.MakeTestResult(sqltypes.MakeTestFields( "Variable_name|Value", "varchar|int64"), diff --git a/go/vt/vttablet/tabletserver/schema/engine.go b/go/vt/vttablet/tabletserver/schema/engine.go index 0ca6701d38d..a64748f8b6f 100644 --- a/go/vt/vttablet/tabletserver/schema/engine.go +++ b/go/vt/vttablet/tabletserver/schema/engine.go @@ -254,7 +254,7 @@ func (se *Engine) Open() error { } se.notifiers = make(map[string]notifier) - if err := se.reload(ctx, true); err != nil { + if err := se.reload(ctx, false); err != nil { return err } if !se.SkipMetaCheck { diff --git a/go/vt/vttablet/tabletserver/schema/engine_test.go b/go/vt/vttablet/tabletserver/schema/engine_test.go index 950619f95de..6e33af32c1b 100644 --- a/go/vt/vttablet/tabletserver/schema/engine_test.go +++ b/go/vt/vttablet/tabletserver/schema/engine_test.go @@ -54,21 +54,23 @@ func TestOpenAndReload(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - db.AddQueryPattern(baseShowTablesPattern, - &sqltypes.Result{ - Fields: mysql.BaseShowTablesFields, - RowsAffected: 0, - InsertID: 0, - Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesRow("test_table_01", false, ""), - mysql.BaseShowTablesRow("test_table_02", false, ""), - mysql.BaseShowTablesRow("test_table_03", false, ""), - mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), - }, - SessionStateChanges: "", - StatusFlags: 0, - }) + + db.RejectQueryPattern(baseShowTablesPattern, "Opening schema engine should query tables without size information") + + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + SessionStateChanges: "", + StatusFlags: 0, + }) // advance to one second after the default 1427325875. db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( @@ -84,8 +86,8 @@ func TestOpenAndReload(t *testing.T) { want := initialSchema() mustMatch(t, want, se.GetSchema()) - assert.Equal(t, int64(100), se.tableFileSizeGauge.Counts()["msg"]) - assert.Equal(t, int64(150), se.tableAllocatedSizeGauge.Counts()["msg"]) + assert.Equal(t, int64(0), se.tableFileSizeGauge.Counts()["msg"]) + assert.Equal(t, int64(0), se.tableAllocatedSizeGauge.Counts()["msg"]) // Advance time some more. db.AddQuery("select unix_timestamp()", sqltypes.MakeTestResult(sqltypes.MakeTestFields( @@ -116,6 +118,9 @@ func TestOpenAndReload(t *testing.T) { mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), }, }) + + db.AddRejectedQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", fmt.Errorf("Reloading schema engine should query tables with size information")) + db.MockQueriesForTable("test_table_03", &sqltypes.Result{ Fields: []*querypb.Field{{ Name: "pk1", @@ -171,6 +176,15 @@ func TestOpenAndReload(t *testing.T) { assert.EqualValues(t, secondReadRowsValue, se.innoDbReadRowsCounter.Get()) + want["seq"].FileSize = 100 + want["seq"].AllocatedSize = 150 + + want["test_table_01"].FileSize = 100 + want["test_table_01"].AllocatedSize = 150 + + want["test_table_02"].FileSize = 100 + want["test_table_02"].AllocatedSize = 150 + want["test_table_03"] = &Table{ Name: sqlparser.NewIdentifierCS("test_table_03"), Fields: []*querypb.Field{{ @@ -228,6 +242,17 @@ func TestOpenAndReload(t *testing.T) { mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), }, }) + + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_04", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + }, + }) + db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{ Fields: mysql.ShowPrimaryFields, Rows: [][]sqltypes.Value{ @@ -251,21 +276,23 @@ func TestReloadWithSwappedTables(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - db.AddQueryPattern(baseShowTablesPattern, - &sqltypes.Result{ - Fields: mysql.BaseShowTablesFields, - RowsAffected: 0, - InsertID: 0, - Rows: [][]sqltypes.Value{ - mysql.BaseShowTablesRow("test_table_01", false, ""), - mysql.BaseShowTablesRow("test_table_02", false, ""), - mysql.BaseShowTablesRow("test_table_03", false, ""), - mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), - mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), - }, - SessionStateChanges: "", - StatusFlags: 0, - }) + + db.RejectQueryPattern(baseShowTablesPattern, "Opening schema engine should query tables without size information") + + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + RowsAffected: 0, + InsertID: 0, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table_01", false, ""), + mysql.BaseShowTablesRow("test_table_02", false, ""), + mysql.BaseShowTablesRow("test_table_03", false, ""), + mysql.BaseShowTablesRow("seq", false, "vitess_sequence"), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + SessionStateChanges: "", + StatusFlags: 0, + }) firstReadRowsValue := 12 AddFakeInnoDBReadRowsResult(db, firstReadRowsValue) @@ -319,6 +346,22 @@ func TestReloadWithSwappedTables(t *testing.T) { }) err := se.Reload(context.Background()) require.NoError(t, err) + + want["msg"].FileSize = 100 + want["msg"].AllocatedSize = 150 + + want["seq"].FileSize = 100 + want["seq"].AllocatedSize = 150 + + want["test_table_01"].FileSize = 100 + want["test_table_01"].AllocatedSize = 150 + + want["test_table_02"].FileSize = 100 + want["test_table_02"].AllocatedSize = 150 + + want["test_table_03"].FileSize = 100 + want["test_table_03"].AllocatedSize = 150 + want["test_table_04"] = &Table{ Name: sqlparser.NewIdentifierCS("test_table_04"), Fields: []*querypb.Field{{ @@ -418,7 +461,7 @@ func TestOpenFailedDueToExecErr(t *testing.T) { defer db.Close() schematest.AddDefaultQueries(db) want := "injected error" - db.RejectQueryPattern(baseShowTablesPattern, want) + db.AddRejectedQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", fmt.Errorf(want)) se := newEngine(10, 1*time.Second, 1*time.Second, 0, db) err := se.Open() if err == nil || !strings.Contains(err.Error(), want) { @@ -433,7 +476,7 @@ func TestOpenFailedDueToLoadTableErr(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesRow("test_table", false, ""), @@ -468,7 +511,7 @@ func TestOpenNoErrorDueToInvalidViews(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{ + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, Rows: [][]sqltypes.Value{ mysql.BaseShowTablesRow("foo_view", true, "VIEW"), @@ -526,7 +569,7 @@ func TestSchemaEngineCloseTickRace(t *testing.T) { db := fakesqldb.New(t) defer db.Close() schematest.AddDefaultQueries(db) - db.AddQueryPattern(baseShowTablesPattern, + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{ Fields: mysql.BaseShowTablesFields, RowsAffected: 0, @@ -602,8 +645,8 @@ func initialSchema() map[string]*Table { }}, PKColumns: []int{0}, CreateTime: 1427325875, - FileSize: 0x64, - AllocatedSize: 0x96, + FileSize: 0, + AllocatedSize: 0, }, "test_table_02": { Name: sqlparser.NewIdentifierCS("test_table_02"), @@ -613,8 +656,8 @@ func initialSchema() map[string]*Table { }}, PKColumns: []int{0}, CreateTime: 1427325875, - FileSize: 0x64, - AllocatedSize: 0x96, + FileSize: 0, + AllocatedSize: 0, }, "test_table_03": { Name: sqlparser.NewIdentifierCS("test_table_03"), @@ -624,8 +667,8 @@ func initialSchema() map[string]*Table { }}, PKColumns: []int{0}, CreateTime: 1427325875, - FileSize: 0x64, - AllocatedSize: 0x96, + FileSize: 0, + AllocatedSize: 0, }, "seq": { Name: sqlparser.NewIdentifierCS("seq"), @@ -645,8 +688,8 @@ func initialSchema() map[string]*Table { }}, PKColumns: []int{0}, CreateTime: 1427325875, - FileSize: 0x64, - AllocatedSize: 0x96, + FileSize: 0, + AllocatedSize: 0, SequenceInfo: &SequenceInfo{}, }, "msg": { @@ -673,8 +716,8 @@ func initialSchema() map[string]*Table { }}, PKColumns: []int{0}, CreateTime: 1427325875, - FileSize: 0x64, - AllocatedSize: 0x96, + FileSize: 0, + AllocatedSize: 0, MessageInfo: &MessageInfo{ Fields: []*querypb.Field{{ Name: "id", diff --git a/go/vt/vttablet/tabletserver/schema/main_test.go b/go/vt/vttablet/tabletserver/schema/main_test.go index 19fc66c36d1..a2469b947df 100644 --- a/go/vt/vttablet/tabletserver/schema/main_test.go +++ b/go/vt/vttablet/tabletserver/schema/main_test.go @@ -35,6 +35,7 @@ func getTestSchemaEngine(t *testing.T, schemaMaxAgeSeconds int64) (*Engine, *fak "1427325876", )) db.AddQueryPattern(baseShowTablesPattern, &sqltypes.Result{}) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", &sqltypes.Result{}) db.AddQuery(mysql.BaseShowPrimary, &sqltypes.Result{}) AddFakeInnoDBReadRowsResult(db, 1) se := newEngine(10, 10*time.Second, 10*time.Second, schemaMaxAgeSeconds, db) diff --git a/go/vt/vttablet/tabletserver/tabletserver_test.go b/go/vt/vttablet/tabletserver/tabletserver_test.go index 51504ffb8ed..d20d6a63da9 100644 --- a/go/vt/vttablet/tabletserver/tabletserver_test.go +++ b/go/vt/vttablet/tabletserver/tabletserver_test.go @@ -2353,6 +2353,14 @@ func setupFakeDB(t *testing.T) *fakesqldb.DB { mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), }, }) + db.AddQuery("SELECT table_name, table_type, unix_timestamp(create_time), table_comment FROM information_schema.tables WHERE table_schema = database()", + &sqltypes.Result{ + Fields: mysql.BaseShowTablesFields, + Rows: [][]sqltypes.Value{ + mysql.BaseShowTablesRow("test_table", false, ""), + mysql.BaseShowTablesRow("msg", false, "vitess_message,vt_ack_wait=30,vt_purge_after=120,vt_batch_size=1,vt_cache_size=10,vt_poller_interval=30"), + }, + }) db.AddQuery("show status like 'Innodb_rows_read'", sqltypes.MakeTestResult(sqltypes.MakeTestFields( "Variable_name|Value", "varchar|int64"),