Merge #131124

131124: tablemetadatacache: retrieve live data bytes from total stats r=xinhaoz a=xinhaoz Previously we were incorrectly reading live data bytes from ApproximateTotalStats from the span stats response. That field is the approximation of the post-replicated data. For this field we want the unreplicated number. Epic: CRDB-37558 Release note: None Co-authored-by: Xin Hao Zhang <[email protected]>
cockroachdb · Sep 22, 2024 · d8f9982 · d8f9982
2 parents 663af90 + c45574b
commit d8f9982
Show file tree

Hide file tree

Showing 4 changed files with 51 additions and 15 deletions.
diff --git a/pkg/sql/tablemetadatacache/BUILD.bazel b/pkg/sql/tablemetadatacache/BUILD.bazel
@@ -45,6 +45,7 @@ go_test(
         "//pkg/base",
         "//pkg/jobs",
         "//pkg/jobs/jobspb",
+        "//pkg/kv/kvserver",
         "//pkg/security/securityassets",
         "//pkg/security/securitytest",
         "//pkg/server",

diff --git a/pkg/sql/tablemetadatacache/table_metadata_updater.go b/pkg/sql/tablemetadatacache/table_metadata_updater.go
@@ -185,8 +185,9 @@ func (q *tableMetadataBatchUpsertQuery) addRow(row *tableMetadataIterRow) error
 
 	livePercentage := float64(0)
 	total := stats.TotalStats.Total()
+	liveBytes := stats.TotalStats.LiveBytes
 	if total > 0 {
-		livePercentage = float64(stats.ApproximateTotalStats.LiveBytes) / float64(total)
+		livePercentage = float64(liveBytes) / float64(total)
 	}
 
 	// TODO (xinhaoz): Get store ids from span stats after
@@ -197,20 +198,20 @@ func (q *tableMetadataBatchUpsertQuery) addRow(row *tableMetadataIterRow) error
 	}
 
 	args := []interface{}{
-		row.dbID,                              // db_id
-		row.tableID,                           // table_id
-		row.dbName,                            // db_name
-		row.schemaName,                        // schema_name,
-		row.tableName,                         // table_name
-		row.columnCount,                       // total_columns
-		row.indexCount,                        // total_indexes
-		row.tableType,                         // table_type
-		storeIds,                              // storeIds
-		stats.ApproximateDiskBytes,            // replication_size_bytes
-		stats.RangeCount,                      // total_ranges
-		stats.ApproximateTotalStats.LiveBytes, // total_live_data_bytes
-		total,                                 // total_data_bytes
-		livePercentage,                        // perc_live_data
+		row.dbID,                   // db_id
+		row.tableID,                // table_id
+		row.dbName,                 // db_name
+		row.schemaName,             // schema_name,
+		row.tableName,              // table_name
+		row.columnCount,            // total_columns
+		row.indexCount,             // total_indexes
+		row.tableType,              // table_type
+		storeIds,                   // storeIds
+		stats.ApproximateDiskBytes, // replication_size_bytes
+		stats.RangeCount,           // total_ranges
+		liveBytes,                  // total_live_data_bytes
+		total,                      // total_data_bytes
+		livePercentage,             // perc_live_data
 	}
 
 	if len(q.args) > 0 {

diff --git a/pkg/sql/tablemetadatacache/table_metadata_updater_test.go b/pkg/sql/tablemetadatacache/table_metadata_updater_test.go
@@ -16,6 +16,7 @@ import (
 	"testing"
 
 	"github.com/cockroachdb/cockroach/pkg/base"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/sql/isql"
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
@@ -66,6 +67,16 @@ func TestDataDrivenTableMetadataCacheUpdater(t *testing.T) {
 					return err.Error()
 				}
 				return fmt.Sprintf("pruned %d table(s)", pruned)
+			case "flush-stores":
+				// Flush store so that bytes return non-zero from span stats.
+				s := s.StorageLayer()
+				err := s.GetStores().(*kvserver.Stores).VisitStores(func(store *kvserver.Store) error {
+					return store.TODOEngine().Flush()
+				})
+				if err != nil {
+					return err.Error()
+				}
+				return "success"
 			default:
 				return "unknown command"
 			}

diff --git a/pkg/sql/tablemetadatacache/testdata/update_table_metadata b/pkg/sql/tablemetadatacache/testdata/update_table_metadata
@@ -24,6 +24,14 @@ query
 CREATE SEQUENCE myseq
 ----
 
+query
+INSERT INTO mytable VALUES (1, 2), (3, 4), (5, 6)
+----
+
+flush-stores
+----
+success
+
 update-cache
 ----
 updated 62 table(s)
@@ -113,6 +121,21 @@ users system public 1 4 4 1 TABLE {1} 1 <nil>
 web_sessions system public 1 19 9 4 TABLE {1} 1 <nil>
 zones system public 1 5 2 0 TABLE {1} 1 <nil>
 
+query
+SELECT count(*) FROM system.table_metadata WHERE perc_live_data > 1
+----
+0
+
+query
+SELECT count(*) FROM system.table_metadata WHERE replication_size_bytes > 0
+----
+62
+
+query
+SELECT count(*) FROM system.table_metadata WHERE total_live_data_bytes > total_data_bytes
+----
+0
+
 query
 DROP TABLE mytable
 ----