Skip to content

Commit

Permalink
Add LogQL support for Loki v3.1.0 (observatorium#723)
Browse files Browse the repository at this point in the history
  • Loading branch information
@periklis
periklis committed Jul 11, 2024
1 parent 7cc1cf6 commit 02d067e
Show file tree
Hide file tree
Showing 6 changed files with 781 additions and 472 deletions.
96 changes: 84 additions & 12 deletions logql/v2/ast.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,9 @@ type LogFilterExpr struct {
filter string
filterOp string
value string
isNested bool
chainOp string
right []*LogFilterExpr
}

func (LogFilterExpr) logQLExpr() {}
Expand All @@ -124,11 +127,22 @@ func newLogFilterExpr(filter, filterOp, value string) *LogFilterExpr {
return &LogFilterExpr{filter: filter, filterOp: filterOp, value: value}
}

func (l *LogFilterExpr) chain(op string, expr *LogFilterExpr) *LogFilterExpr {
expr.isNested = true
expr.chainOp = op
l.right = append(l.right, expr)

return l
}

func (l *LogFilterExpr) String() string {
var sb strings.Builder

sb.WriteString(l.filter)
sb.WriteString(" ")
// Render filter only on first filter and not nested or filters
if !l.isNested {
sb.WriteString(l.filter)
sb.WriteString(" ")
}

if l.filterOp != "" {
sb.WriteString(l.filterOp)
Expand All @@ -139,25 +153,85 @@ func (l *LogFilterExpr) String() string {
sb.WriteString(strconv.Quote(l.value))
}

for i, r := range l.right {
switch r.chainOp {
case "or":
sb.WriteString(" ")
sb.WriteString(r.chainOp)
sb.WriteString(" ")
sb.WriteString(r.String())
}

if i == len(l.right) {
sb.WriteString(" ")
}
}

return sb.String()
}

func (l *LogFilterExpr) Walk(fn WalkFn) {
fn(l)
}

type FilterValueType string

const (
TypeNumber FilterValueType = "number"
TypeDuration FilterValueType = "duration"
TypeText FilterValueType = "text"
)

type LogLabelFilterValue struct {
filterType FilterValueType
numberVal *LogNumberExpr
strVal string
durVal time.Duration
}

func newLogLabelFilterValue(t FilterValueType, numberVal *LogNumberExpr, strVal string, durVal time.Duration) *LogLabelFilterValue {
switch t {
case TypeNumber:
return &LogLabelFilterValue{filterType: t, numberVal: numberVal}
case TypeDuration:
return &LogLabelFilterValue{filterType: t, durVal: durVal}
case TypeText:
return &LogLabelFilterValue{filterType: t, strVal: strVal}
default:
return &LogLabelFilterValue{}
}
}

func (l *LogLabelFilterValue) String() string {
var sb strings.Builder
switch l.filterType {
case TypeNumber:
sb.WriteString(l.numberVal.String())
case TypeDuration:
sb.WriteString(l.durVal.String())
case TypeText:
sb.WriteString(`"`)
sb.WriteString(l.strVal)
sb.WriteString(`"`)
default:
return ""
}

return sb.String()
}

type LogLabelFilterExpr struct {
defaultLogQLExpr // nolint:unused
labelName string
comparisonOp string
filterOp string
labelValue string
labelValue *LogLabelFilterValue
isNested bool
chainOp string
right []*LogLabelFilterExpr
}

func newLogLabelFilter(identifier, comparisonOp, filterOp, value string) *LogLabelFilterExpr {
func newLogLabelFilter(identifier, comparisonOp, filterOp string, value *LogLabelFilterValue) *LogLabelFilterExpr {
return &LogLabelFilterExpr{labelName: identifier, comparisonOp: comparisonOp, filterOp: filterOp, labelValue: value}
}

Expand All @@ -181,19 +255,17 @@ func (l *LogLabelFilterExpr) String() string {
}

sb.WriteString(l.labelName)
sb.WriteString(" ")
sb.WriteString(l.comparisonOp)
sb.WriteString(" ")

if l.filterOp != "" {
sb.WriteString(l.filterOp)
sb.WriteString("(")
sb.WriteString(`"`)
sb.WriteString(l.labelValue)
sb.WriteString(`"`)
sb.WriteString(l.labelValue.String())
sb.WriteString(")")
} else {
sb.WriteString(`"`)
sb.WriteString(l.labelValue)
sb.WriteString(`"`)
sb.WriteString(l.labelValue.String())
}

for i, r := range l.right {
Expand Down Expand Up @@ -535,13 +607,13 @@ func (l *LogQueryExpr) AppendPipelineMatchers(matchers []*labels.Matcher, chainO
matchersFilter := LogLabelFilterExpr{
labelName: matchers[0].Name,
comparisonOp: matchers[0].Type.String(),
labelValue: matchers[0].Value,
labelValue: newLogLabelFilterValue(TypeText, nil, matchers[0].Value, 0),
}
for _, m := range matchers[1:] {
matchersFilter.right = append(matchersFilter.right, &LogLabelFilterExpr{
labelName: m.Name,
comparisonOp: m.Type.String(),
labelValue: m.Value,
labelValue: newLogLabelFilterValue(TypeText, nil, m.Value, 0),
isNested: true,
chainOp: chainOp,
})
Expand Down
16 changes: 8 additions & 8 deletions logql/v2/ast_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,8 +76,8 @@ func Test_AstWalker_AppendMatcher(t *testing.T) {
output: `{first="value", second="next"} |= "other" |= ip("8.8.8.8")`,
},
{
input: `{ first = "value" }|logfmt|addr>=ip("1.1.1.1")`,
output: `{first="value", second="next"} | logfmt | addr>=ip("1.1.1.1")`,
input: `{ first = "value" }|logfmt|addr >= ip("1.1.1.1")`,
output: `{first="value", second="next"} | logfmt | addr >= ip("1.1.1.1")`,
},
// log metric expressions
{
Expand Down Expand Up @@ -169,24 +169,24 @@ func Test_AstWalker_AppendORMatcher(t *testing.T) {
// log selector expressions
{
input: `{first="value"}`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar"`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar"`,
},
{
input: `{first="value"} |= "other" |= ip("8.8.8.8")`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar" |= "other" |= ip("8.8.8.8")`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar" |= "other" |= ip("8.8.8.8")`,
},
{
input: `{ first = "value" }|logfmt|addr>=ip("1.1.1.1")`,
output: `{first="value"} | second=~"foo|bar" or third=~"foo|bar" | logfmt | addr>=ip("1.1.1.1")`,
input: `{ first = "value" }|logfmt|addr >= ip("1.1.1.1")`,
output: `{first="value"} | second =~ "foo|bar" or third =~ "foo|bar" | logfmt | addr >= ip("1.1.1.1")`,
},
// log metric expressions
{
input: `sum(rate({first="value"}[5m]))`,
output: `sum(rate({first="value"}[5m] | second=~"foo|bar" or third=~"foo|bar"))`,
output: `sum(rate({first="value"}[5m] | second =~ "foo|bar" or third =~ "foo|bar"))`,
},
{
input: `max without (second) (count_over_time({first="value"}[5h]))`,
output: `max without(second) (count_over_time({first="value"}[5h] | second=~"foo|bar" or third=~"foo|bar"))`,
output: `max without(second) (count_over_time({first="value"}[5h] | second =~ "foo|bar" or third =~ "foo|bar"))`,
},
}
for _, tc := range tc {
Expand Down
30 changes: 18 additions & 12 deletions logql/v2/expr.y
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ import (
%token <duration> DURATION
%token <val> MATCHERS LABELS EQ RE NRE OPEN_BRACE CLOSE_BRACE OPEN_BRACKET CLOSE_BRACKET COMMA DOT
OPEN_PARENTHESIS CLOSE_PARENTHESIS COUNT_OVER_TIME RATE RATE_COUNTER SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK SORT SORT_DESC
BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE_MATCH PIPE_EXACT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE_MATCH PIPE_EXACT PIPE_MATCH_PATTERN PIPE_NOT_MATCH_PATTERN PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME
MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME
BY WITHOUT VECTOR LABEL_REPLACE IP UNPACK PATTERN OFFSET BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV ON IGNORING GROUP_LEFT GROUP_RIGHT
DECOLORIZE DROP KEEP
Expand Down Expand Up @@ -132,16 +132,20 @@ logStageExpr:
;

logFilterExpr:
filter STRING { $$ = newLogFilterExpr($1, "", $2) }
| filter IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, OpIP, $4) }
filter STRING { $$ = newLogFilterExpr($1, "", $2) }
| filter IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, OpIP, $4) }
| filter STRING OR STRING { $$ = newLogFilterExpr($1, "", $2).chain("or", newLogFilterExpr($1, "", $4)) }
| filter STRING OR IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogFilterExpr($1, "", $2).chain("or", newLogFilterExpr($1, OpIP, $6)) }
;

logLabelFilterExpr:
IDENTIFIER comparisonOp STRING { $$ = newLogLabelFilter($1, $2, "", $3) }
| IDENTIFIER comparisonOp IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogLabelFilter($1, $2, OpIP, $5) }
| logLabelFilterExpr AND logLabelFilterExpr { $$ = $1.chain("and", $3) }
| logLabelFilterExpr OR logLabelFilterExpr { $$ = $1.chain("or", $3) }
| logLabelFilterExpr COMMA logLabelFilterExpr { $$ = $1.chain(",", $3) }
IDENTIFIER comparisonOp STRING { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeText, nil, $3, 0)) }
| IDENTIFIER comparisonOp DURATION { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeDuration, nil, "", $3)) }
| IDENTIFIER comparisonOp logNumberExpr { $$ = newLogLabelFilter($1, $2, "", newLogLabelFilterValue(TypeNumber, &$3, "", 0)) }
| IDENTIFIER comparisonOp IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLogLabelFilter($1, $2, OpIP, newLogLabelFilterValue(TypeText, nil, $5, 0)) }
| logLabelFilterExpr AND logLabelFilterExpr { $$ = $1.chain("and", $3) }
| logLabelFilterExpr OR logLabelFilterExpr { $$ = $1.chain("or", $3) }
| logLabelFilterExpr COMMA logLabelFilterExpr { $$ = $1.chain(",", $3) }
;

logFormatExpr:
Expand Down Expand Up @@ -396,10 +400,12 @@ metricOp:
;

filter:
PIPE_MATCH { $$ = "|~" }
| PIPE_EXACT { $$ = "|=" }
| NRE { $$ = "!~" }
| NEQ { $$ = "!=" }
PIPE_MATCH { $$ = "|~" }
| PIPE_EXACT { $$ = "|=" }
| PIPE_MATCH_PATTERN { $$ = "|>" }
| PIPE_NOT_MATCH_PATTERN { $$ = "!>" }
| NRE { $$ = "!~" }
| NEQ { $$ = "!=" }
;

comparisonOp:
Expand Down
Loading

0 comments on commit 02d067e

Please sign in to comment.