Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[WIP] Parse query comments and pass metadata for max time query #1

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 20 additions & 9 deletions class.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,11 @@ type Class struct {
UniqueQueries uint // unique number of queries in class
Example *Example `json:",omitempty"` // sample query with max Query_time
// --
outliers uint64
lastDb string
sample bool
outliers uint64
lastDb string
sample bool
maxQueryTime float64
MaxQueryCommentMetadata map[string]string
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Let's call this just Metadata. The max query time stuff is something we do internally.

}

// A Example is a real query and its database, timestamp, and Query_time.
Expand All @@ -40,12 +42,14 @@ type Example struct {
// If sample is true, the query with the greatest Query_time is saved.
func NewClass(id, fingerprint string, sample bool) *Class {
return &Class{
Id: id,
Fingerprint: fingerprint,
Metrics: NewMetrics(),
TotalQueries: 0,
Example: &Example{},
sample: sample,
Id: id,
Fingerprint: fingerprint,
Metrics: NewMetrics(),
TotalQueries: 0,
Example: &Example{},
sample: sample,
maxQueryTime: 0,
MaxQueryCommentMetadata: map[string]string{},
}
}

Expand Down Expand Up @@ -82,6 +86,13 @@ func (c *Class) AddEvent(e Event, outlier bool) {
}
}
}

if queryTime, ok := e.TimeMetrics["Query_time"]; ok {
if queryTime > c.maxQueryTime {
c.MaxQueryCommentMetadata = e.CommentMetadata
}
}

}

// Finalize calculates all metric statistics. Call this function when done
Expand Down
25 changes: 13 additions & 12 deletions event.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,19 @@ package slowlog
// event is expected to define the query and Query_time metric. Other metrics
// and metadata vary according to MySQL version, distro, and configuration.
type Event struct {
Offset uint64 // byte offset in file at which event starts
Ts string // raw timestamp of event
Admin bool // true if Query is admin command
Query string // SQL query or admin command
User string
Host string
Db string
TimeMetrics map[string]float64 // *_time and *_wait metrics
NumberMetrics map[string]uint64 // most metrics
BoolMetrics map[string]bool // yes/no metrics
RateType string // Percona Server rate limit type
RateLimit uint // Percona Server rate limit value
Offset uint64 // byte offset in file at which event starts
Ts string // raw timestamp of event
Admin bool // true if Query is admin command
Query string // SQL query or admin command
User string
Host string
Db string
TimeMetrics map[string]float64 // *_time and *_wait metrics
NumberMetrics map[string]uint64 // most metrics
BoolMetrics map[string]bool // yes/no metrics
RateType string // Percona Server rate limit type
RateLimit uint // Percona Server rate limit value
CommentMetadata map[string]string
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same as above: just Metadata.

}

// NewEvent returns a new Event with initialized metric maps.
Expand Down
88 changes: 88 additions & 0 deletions parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ package slowlog

import (
"bufio"
"bytes"
"errors"
"fmt"
"io"
Expand Down Expand Up @@ -396,6 +397,92 @@ func (p *FileParser) parseAdmin(line string) {
}
}

// Tested here: https://play.golang.org/p/u7XZnxu2LLL
func parseComments(queryWithComments string) map[string]string {
startIndex := strings.Index(queryWithComments, "/*")
if startIndex != -1 {
endIndex := strings.LastIndex(queryWithComments, "*/")
sqlComments := queryWithComments[startIndex:endIndex]
return parseKeyValuePairToMap(sqlComments)
}
return map[string]string{}
}

// Copied verbatim from https://gist.github.com/alexisvisco/4b846978c9346e4eaf618bb632c0693a
func parseKeyValuePairToMap(msg string) map[string]string {

type kv struct {
key, val string
}

var pair *kv = nil
pairs := make(map[string]string)
buf := bytes.NewBuffer([]byte{})

var (
escape = false
garbage = false
quoted = false
)

completePair := func(buffer *bytes.Buffer, pair *kv) kv {
if pair != nil {
return kv{pair.key, buffer.String()}
} else {
return kv{buffer.String(), ""}
}
}

for _, c := range msg {
if !quoted && c == ' ' {
if buf.Len() != 0 {
if !garbage {
p := completePair(buf, pair)
pairs[p.key] = p.val
pair = nil
}
buf.Reset()
}
garbage = false
} else if !quoted && c == '=' {
if buf.Len() != 0 {
pair = &kv{key: buf.String(), val: ""}
buf.Reset()
} else {
garbage = true
}
} else if quoted && c == '\\' {
escape = true
} else if c == '"' {
if escape {
buf.WriteRune(c)
escape = false
} else {
quoted = !quoted
}
} else {
if escape {
buf.WriteRune('\\')
escape = false
}
buf.WriteRune(c)
}
}

if !garbage {
p := completePair(buf, pair)
pairs[p.key] = p.val
}

// Remove any key value pair where either of them is a blank string
for key, value := range pairs {
if key == "" || value == "" {
delete(pairs, key)
}
}
return pairs
}

func (p *FileParser) sendEvent(inHeader bool, inQuery bool) {
if Debug {
log.Println("send event")
Expand All @@ -421,6 +508,7 @@ func (p *FileParser) sendEvent(inHeader bool, inQuery bool) {
// Clean up the event.
p.event.Db = strings.TrimSuffix(p.event.Db, ";\n")
p.event.Query = strings.TrimSuffix(p.event.Query, ";")
p.event.CommentMetadata = parseComments(p.event.Query)

// Send the event. This will block.
select {
Expand Down