From 0202c24e3aa66f7e8dd2f3735702960dc5992323 Mon Sep 17 00:00:00 2001 From: Thearas Date: Fri, 15 Nov 2024 12:27:24 +0800 Subject: [PATCH] chore: remove unique dump because no one use Change-Id: Ic7e10c8401a75dddbd3fce8c2018ecc86e998824 --- cmd/dump.go | 27 +++++++++------------------ src/auditlog.go | 34 ++-------------------------------- src/auditlog_test.go | 17 ----------------- src/db.go | 3 +-- 4 files changed, 12 insertions(+), 69 deletions(-) diff --git a/cmd/dump.go b/cmd/dump.go index 4de27b1..97fb540 100644 --- a/cmd/dump.go +++ b/cmd/dump.go @@ -52,17 +52,15 @@ type Dump struct { SSHPassword string SSHPrivateKey string - DumpSchema bool - DumpStats bool - DumpQuery bool - QueryOutputMode string - QueryUniqueNormalize bool - QueryMinDuration_ time.Duration - QueryMinDurationMs int - QueryStates []string - OnlySelect bool - Strict bool - From, To string + DumpSchema bool + DumpStats bool + DumpQuery bool + QueryMinDuration_ time.Duration + QueryMinDurationMs int + QueryStates []string + OnlySelect bool + Strict bool + From, To string Clean bool } @@ -144,8 +142,6 @@ func init() { pFlags.BoolVar(&DumpConfig.DumpSchema, "dump-schema", false, "Dump schema") pFlags.BoolVar(&DumpConfig.DumpStats, "dump-stats", true, "Dump schema stats, only take effect when '--dump-schema=true'") pFlags.BoolVar(&DumpConfig.DumpQuery, "dump-query", false, "Dump query from audit log") - pFlags.StringVar(&DumpConfig.QueryOutputMode, "query-output-mode", "default", "Dump query output mode, one of [default, unique]") - pFlags.BoolVar(&DumpConfig.QueryUniqueNormalize, "query-unique-normalize", false, "Regard 'select 1 from b where a = 1' as 'select ? from b where a = ?' for unique, only take effect when '--query-output-mode=unique'") pFlags.DurationVar(&DumpConfig.QueryMinDuration_, "query-min-duration", 0, "Dump queries which execution duration is greater than or equal to") pFlags.StringSliceVar(&DumpConfig.QueryStates, "query-states", []string{}, "Dump queries with states, like 'ok', 'eof' and 'err'") pFlags.BoolVar(&DumpConfig.OnlySelect, "only-select", true, "Only dump SELECT queries") @@ -361,8 +357,6 @@ func dumpQueries(ctx context.Context) ([][]string, error) { DBs: GlobalConfig.DBs, QueryMinDurationMs: DumpConfig.QueryMinDurationMs, QueryStates: DumpConfig.QueryStates, - Unique: DumpConfig.QueryOutputMode == "unique", - UniqueNormalize: DumpConfig.QueryUniqueNormalize, Unescape: DumpConfig.AuditLogUnescape, OnlySelect: DumpConfig.OnlySelect, Strict: DumpConfig.Strict, @@ -380,9 +374,6 @@ func dumpQueriesFromTable(ctx context.Context, opts src.AuditLogScanOpts) ([][]s if opts.From == "" || opts.To == "" { return nil, errors.New("Must specific both '--from' and '--to' when dumping from audit log table") } - if opts.Unique { - return nil, errors.New("Not yet support '--query-output-mode=unique' with '--audit-log-table'") - } dbTable := strings.SplitN(DumpConfig.AuditLogTable, ".", 2) dbname, table := dbTable[0], dbTable[1] diff --git a/src/auditlog.go b/src/auditlog.go index 9dc040a..515b741 100644 --- a/src/auditlog.go +++ b/src/auditlog.go @@ -12,7 +12,6 @@ import ( "github.com/dlclark/regexp2" "github.com/edsrzf/mmap-go" - tidbparser "github.com/pingcap/tidb/pkg/parser" "github.com/samber/lo" "github.com/sirupsen/logrus" "github.com/zeebo/blake3" @@ -30,9 +29,6 @@ var ( // Tested on v2.0.x and v2.1.x. Not sure if it also works on others Doris version. stmtMatchFmt = `^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d*) \[[^\]]+?\] \|Client=([^|]+?)\|User=([^|]+?)\|(?:.+?)\|Db=(%s?)\|State=%s\|(?:.+?)\|Time(?:\(ms\))?=(\d*)\|(?:.+?)\|QueryId=([a-z0-9-]+)\|IsQuery=%s\|(?:.+?)\|Stmt=(.+?)\|CpuTimeMS=` - // The cols read from audit log table - captureFieldCols = []string{"time", "client_ip", "user", "db", "query_time", "query_id", "stmt"} - unescapeReplacer = strings.NewReplacer( "\\n", "\n", "\\t", "\t", @@ -60,9 +56,8 @@ type AuditLogScanOpts struct { OnlySelect bool From, To string - Unique, UniqueNormalize bool - Unescape bool - Strict bool + Unescape bool + Strict bool } func (opts *AuditLogScanOpts) sqlConditions() string { @@ -250,13 +245,6 @@ func (s *SimpleAuditLogScanner) ScanOne(oneLog []byte) error { } func (s *SimpleAuditLogScanner) Result() []string { - if s.Unique { - // append number of occurrences of each sql - for _, v := range s.uniqsqls { - sqlCount := fmt.Sprintf(" -- count: %d", v.count) - s.sqls[v.sqlIdx] += sqlCount - } - } return s.sqls } @@ -281,30 +269,12 @@ func (s *SimpleAuditLogScanner) onMatch(caps []string, skipOptsFilter bool) { stmt = unescapeReplacer.Replace(stmt) } if s.Strict && s.validateSQL(queryId, stmt) != nil { - print("asdsadad\n") return } - // unique sqls - if s.Unique { - var h [32]byte - if s.UniqueNormalize { - h = hashstr(s.hash, tidbparser.NormalizeKeepHint(stmt)) - } else { - h = hashstr(s.hash, stmt) - } - if _, ok := s.uniqsqls[h]; ok { - s.uniqsqls[h].count++ - return - } else { - s.uniqsqls[h] = &uniqSql{count: 1, sqlIdx: len(s.sqls)} - } - } - // add leading meta comment outputStmt := EncodeReplaySql(time, client, user, db, queryId, stmt) - // not unique sqls s.sqls = append(s.sqls, outputStmt) } diff --git a/src/auditlog_test.go b/src/auditlog_test.go index c4ffe8d..e66f833 100644 --- a/src/auditlog_test.go +++ b/src/auditlog_test.go @@ -23,8 +23,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) { queryMinCpuTimeMs int queryStates []string parallel int - unique bool - uniqueNormalize bool unescape bool onlySelect bool strict bool @@ -48,19 +46,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) { }, want: []int{8}, }, - { - name: "unique", - args: args{ - auditlogPaths: []string{"fixture/fe.audit.log"}, - encoding: "auto", - unique: true, - uniqueNormalize: true, - unescape: true, - onlySelect: true, - strict: true, - }, - want: []int{7}, - }, { name: "not_only_select", args: args{ @@ -95,8 +80,6 @@ func TestExtractQueriesFromAuditLogs(t *testing.T) { OnlySelect: tt.args.onlySelect, From: tt.args.from, To: tt.args.to, - Unique: tt.args.unique, - UniqueNormalize: tt.args.uniqueNormalize, Unescape: tt.args.unescape, Strict: tt.args.strict, } diff --git a/src/db.go b/src/db.go index 3c3ef74..259961b 100644 --- a/src/db.go +++ b/src/db.go @@ -445,8 +445,7 @@ func getDBAuditLogs( conditions string, limit, offset int, ) (string, string, error) { - stmt := fmt.Sprintf("SELECT %s FROM `%s`.`%s` WHERE %s LIMIT %d OFFSET %d ORDER BY time asc, query_id asc", - strings.Join(captureFieldCols, ", "), + stmt := fmt.Sprintf("SELECT time, client_ip, user, db, query_time, query_id, stmt FROM `%s`.`%s` WHERE %s LIMIT %d OFFSET %d ORDER BY time asc, query_id asc", dbname, table, conditions,