More advanced refactoring of the Run method

Signed-off-by: Florent Poinsard <[email protected]>

go/summarize/markdown.go

@@ -0,0 +1,216 @@
+/*
+Copyright 2024 The Vitess Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package summarize
+
+import (
+	"fmt"
+	"maps"
+	"slices"
+	"sort"
+	"strconv"
+
+	"vitess.io/vitess/go/vt/vtgate/planbuilder/operators"
+
+	"github.com/vitessio/vt/go/keys"
+	"github.com/vitessio/vt/go/markdown"
+)
+
+func renderHotQueries(md *markdown.MarkDown, queries []keys.QueryAnalysisResult, metricReader getMetric) {
+	if len(queries) == 0 {
+		return
+	}
+
+	hasTime := false
+	// Sort the queries in descending order of hotness
+	sort.Slice(queries, func(i, j int) bool {
+		if queries[i].QueryTime != 0 {
+			hasTime = true
+		}
+		return metricReader(queries[i]) > metricReader(queries[j])
+	})
+
+	if !hasTime {
+		return
+	}
+
+	md.PrintHeader("Top Queries", 2)
+
+	// Prepare table headers and rows
+	headers := []string{"Query ID", "Usage Count", "Total Query Time (ms)", "Avg Query Time (ms)", "Total Rows Examined"}
+	var rows [][]string
+
+	for i, query := range queries {
+		queryID := fmt.Sprintf("Q%d", i+1)
+		avgQueryTime := query.QueryTime / float64(query.UsageCount)
+		rows = append(rows, []string{
+			queryID,
+			strconv.Itoa(query.UsageCount),
+			fmt.Sprintf("%.2f", query.QueryTime),
+			fmt.Sprintf("%.2f", avgQueryTime),
+			strconv.Itoa(query.RowsExamined),
+		})
+	}
+
+	// Print the table
+	md.PrintTable(headers, rows)
+
+	// After the table, list the full queries with their IDs
+	md.PrintHeader("Query Details", 3)
+	for i, query := range queries {
+		queryID := fmt.Sprintf("Q%d", i+1)
+		md.PrintHeader(queryID, 4)
+		md.Println("```sql")
+		md.Println(query.QueryStructure)
+		md.Println("```")
+		md.NewLine()
+	}
+}
+
+func renderTableUsage(md *markdown.MarkDown, tableSummaries []*TableSummary, includeRowCount bool) {
+	if len(tableSummaries) == 0 {
+		return
+	}
+
+	sort.Slice(tableSummaries, func(i, j int) bool {
+		if tableSummaries[i].UseCount() == tableSummaries[j].UseCount() {
+			return tableSummaries[i].Table < tableSummaries[j].Table
+		}
+		return tableSummaries[i].UseCount() > tableSummaries[j].UseCount()
+	})
+
+	md.PrintHeader("Tables", 2)
+	renderTableOverview(md, tableSummaries, includeRowCount)
+
+	md.PrintHeader("Column Usage", 3)
+	for _, summary := range tableSummaries {
+		renderColumnUsageTable(md, summary)
+	}
+}
+
+func renderTableOverview(md *markdown.MarkDown, tableSummaries []*TableSummary, includeRowCount bool) {
+	headers := []string{"Table Name", "Reads", "Writes"}
+	if includeRowCount {
+		headers = append(headers, "Number of Rows")
+	}
+	var rows [][]string
+	for _, summary := range tableSummaries {
+		thisRow := []string{
+			summary.Table,
+			strconv.Itoa(summary.ReadQueryCount),
+			strconv.Itoa(summary.WriteQueryCount),
+		}
+		if includeRowCount {
+			thisRow = append(thisRow, strconv.Itoa(summary.RowCount))
+		}
+
+		rows = append(rows, thisRow)
+	}
+	md.PrintTable(headers, rows)
+}
+
+func renderColumnUsageTable(md *markdown.MarkDown, summary *TableSummary) {
+	md.PrintHeader(fmt.Sprintf("Table: `%s` (%d reads and %d writes)", summary.Table, summary.ReadQueryCount, summary.WriteQueryCount), 4)
+
+	headers := []string{"Column", "Position", "Used %"}
+	var rows [][]string
+	var lastName string
+	for colInfo, usage := range summary.GetColumns() {
+		name := colInfo.Name
+		if lastName == name {
+			name = ""
+		} else {
+			lastName = name
+		}
+		rows = append(rows, []string{
+			name,
+			colInfo.Pos.String(),
+			fmt.Sprintf("%.0f%%", usage.Percentage),
+		})
+	}
+
+	md.PrintTable(headers, rows)
+}
+
+func renderTablesJoined(md *markdown.MarkDown, summary *Summary) {
+	if len(summary.queryGraph) > 0 {
+		md.PrintHeader("Tables Joined", 2)
+	}
+
+	type joinDetails struct {
+		Tbl1, Tbl2  string
+		Occurrences int
+		predicates  []operators.JoinPredicate
+	}
+
+	var joins []joinDetails
+	for tables, predicates := range summary.queryGraph {
+		occurrences := 0
+		for _, count := range predicates {
+			occurrences += count
+		}
+		joinPredicates := slices.Collect(maps.Keys(predicates))
+		sort.Slice(joinPredicates, func(i, j int) bool {
+			return joinPredicates[i].String() < joinPredicates[j].String()
+		})
+		joins = append(joins, joinDetails{
+			Tbl1:        tables.Tbl1,
+			Tbl2:        tables.Tbl2,
+			Occurrences: occurrences,
+			predicates:  joinPredicates,
+		})
+	}
+
+	sort.Slice(joins, func(i, j int) bool {
+		if joins[i].Occurrences != joins[j].Occurrences {
+			return joins[i].Occurrences > joins[j].Occurrences
+		}
+		if joins[i].Tbl1 != joins[j].Tbl1 {
+			return joins[i].Tbl1 < joins[j].Tbl1
+		}
+		return joins[i].Tbl2 < joins[j].Tbl2
+	})
+
+	md.Println("```")
+	for _, join := range joins {
+		md.Printf("%s ↔ %s (Occurrences: %d)\n", join.Tbl1, join.Tbl2, join.Occurrences)
+		for i, pred := range join.predicates {
+			var s string
+			if i == len(join.predicates)-1 {
+				s = "└─"
+			} else {
+				s = "├─"
+			}
+			md.Printf("%s %s\n", s, pred.String())
+		}
+		md.NewLine()
+	}
+	md.Println("```")
+}
+
+func renderFailures(md *markdown.MarkDown, failures []FailuresSummary) {
+	if len(failures) == 0 {
+		return
+	}
+	md.PrintHeader("Failures", 2)
+
+	headers := []string{"Error", "Count"}
+	var rows [][]string
+	for _, failure := range failures {
+		rows = append(rows, []string{failure.Error, strconv.Itoa(failure.Count)})
+	}
+	md.PrintTable(headers, rows)
+}

go/summarize/reading.go

@@ -19,49 +19,35 @@ package summarize
 import (
 	"encoding/json"
 	"errors"
-	"io"
 	"os"
 	"sort"
 	"strconv"
 
 	"github.com/vitessio/vt/go/keys"
+	"github.com/vitessio/vt/go/schema"
 )
 
-func readTraceFile(fi fileInfo) (readingSummary, error) {
+func readTraceFile(fi fileInfo) traceSummary {
 	switch fi.fileType {
 	case traceFile:
-		return readTracedQueryFile(fi.filename), nil
-	case keysFile:
-		return readAnalysedQueryFile(fi.filename), nil
+		return readTracedQueryFile(fi.filename)
 	default:
-		return readingSummary{}, errors.New("unknown file format")
+		panic("Unsupported file type")
 	}
 }
 
-func getDecoderAndDelim(file *os.File) (*json.Decoder, json.Delim) {
-	// Create a decoder
-	decoder := json.NewDecoder(file)
-
-	// Read the opening bracket
-	val, err := decoder.Token()
-	if err != nil {
-		exit("Error reading json: " + err.Error())
-	}
-	delim, ok := val.(json.Delim)
-	if !ok {
-		exit("Error reading json: expected delimiter")
-	}
-
-	// Reset the file pointer to the beginning
-	_, err = file.Seek(0, io.SeekStart)
-	if err != nil {
-		exit("Error rewinding file: " + err.Error())
+func readFile(fi fileInfo) (func(s *Summary) error, error) {
+	switch fi.fileType {
+	case keysFile:
+		return readAnalysedQueryFile(fi.filename), nil
+	case dbInfoFile:
+		return readDBInfoFile(fi.filename), nil
+	default:
+		return nil, errors.New("unknown file format")
 	}
-	decoder = json.NewDecoder(file)
-	return decoder, delim
 }
 
-func readTracedQueryFile(fileName string) readingSummary {
+func readTracedQueryFile(fileName string) traceSummary {
 	c, err := os.ReadFile(fileName)
 	if err != nil {
 		exit("Error opening file: " + err.Error())
@@ -89,13 +75,13 @@ func readTracedQueryFile(fileName string) readingSummary {
 		return a < b
 	})
 
-	return readingSummary{
+	return traceSummary{
 		Name:          fileName,
 		TracedQueries: to.Queries,
 	}
 }
 
-func readAnalysedQueryFile(fileName string) readingSummary {
+func readAnalysedQueryFile(fileName string) func(s *Summary) error {
 	c, err := os.ReadFile(fileName)
 	if err != nil {
 		exit("Error opening file: " + err.Error())
@@ -107,8 +93,30 @@ func readAnalysedQueryFile(fileName string) readingSummary {
 		exit("Error parsing json: " + err.Error())
 	}
 
-	return readingSummary{
-		Name:            fileName,
-		AnalysedQueries: &ko,
+	return func(s *Summary) error {
+		s.analyzedFiles = append(s.analyzedFiles, fileName)
+		summarizeKeysQueries(s, &ko)
+		return nil
+	}
+}
+
+func readDBInfoFile(fileName string) func(s *Summary) error {
+	schemaInfo, err := schema.Load(fileName)
+	if err != nil {
+		panic(err)
+	}
+
+	return func(s *Summary) error {
+		s.analyzedFiles = append(s.analyzedFiles, fileName)
+		s.hasRowCount = true
+		for _, ti := range schemaInfo.Tables {
+			table := s.GetTable(ti.Name)
+			if table == nil {
+				table = &TableSummary{Table: ti.Name}
+				s.AddTable(table)
+			}
+			table.RowCount = ti.Rows
+		}
+		return nil
 	}
 }