diff --git a/Insanedocfile b/Insanedocfile index 8c4d85fa1..ea312f2d0 100644 --- a/Insanedocfile +++ b/Insanedocfile @@ -2,10 +2,16 @@ extractors: config-params: '"config-params" /json:\"([a-z_]+)\"/ #2 /default:\"([^"]+)\"/ /(required):\"true\"/ /options:\"([^"]+)\"/' fn-list: '"fn-list" #4 /Plugin\)\s(.+)\s{/' match-modes: '"match-modes" /MatchMode(.*),/ /\"(.*)\"/' + do-if-node: '"do-if-node" /DoIfNode(\w+)\s/' + do-if-field-op: '"do-if-field-op" /doIfField(\w+)OpBytes\s/' + do-if-logical-op: '"do-if-logical-op" /doIfLogical(\w+)Bytes\s/' decorators: config-params: '_ _ /*`%s`* / /*`default=%s`* / /*`%s`* / /*`options=%s`* /' fn-list: '_ _ /`%s`/' match-modes: '_ /%s/ /`match_mode: %s`/' + do-if-node: '_ /%s/' + do-if-field-op: '_ /%s/' + do-if-logical-op: '_ /%s/' templates: - template: docs/*.idoc.md files: ["../pipeline/*.go"] diff --git a/_sidebar.idoc.md b/_sidebar.idoc.md index 42abcb341..bb2e9630e 100644 --- a/_sidebar.idoc.md +++ b/_sidebar.idoc.md @@ -19,6 +19,10 @@ - Output @global-contents-table-plugin-output|links-list +- **Pipeline** + - [Match modes](pipeline/README.md#match-modes) + - [Experimental: Do If rules](pipeline/README.md#experimental-do-if-rules) + - **Other** - [Contributing](/docs/contributing.md) - [License](/docs/license.md) \ No newline at end of file diff --git a/_sidebar.md b/_sidebar.md index d73659f88..3b3716536 100644 --- a/_sidebar.md +++ b/_sidebar.md @@ -58,6 +58,10 @@ - [stdout](plugin/output/stdout/README.md) +- **Pipeline** + - [Match modes](pipeline/README.md#match-modes) + - [Experimental: Do If rules](pipeline/README.md#experimental-do-if-rules) + - **Other** - [Contributing](/docs/contributing.md) - [License](/docs/license.md) \ No newline at end of file diff --git a/fd/file.d.go b/fd/file.d.go index f0cf9bafb..64b5024e2 100644 --- a/fd/file.d.go +++ b/fd/file.d.go @@ -157,6 +157,11 @@ func (f *FileD) setupAction(p *pipeline.Pipeline, index int, t string, actionJSO logger.Infof("creating action with type %q for pipeline %q", t, p.Name) info := f.plugins.GetActionByType(t) + doIfChecker, err := extractDoIfChecker(actionJSON.Get("do_if")) + if err != nil { + logger.Fatalf(`failed to extract "do_if" conditions for action %d/%s in pipeline %q: %s`, index, t, p.Name, err.Error()) + } + matchMode := extractMatchMode(actionJSON) if matchMode == pipeline.MatchModeUnknown { logger.Fatalf("unknown match_mode value for action %d/%s in pipeline %q", index, t, p.Name) @@ -191,6 +196,7 @@ func (f *FileD) setupAction(p *pipeline.Pipeline, index int, t string, actionJSO MetricLabels: metricLabels, MetricSkipStatus: skipStatus, MatchInvert: matchInvert, + DoIfChecker: doIfChecker, }) } diff --git a/fd/util.go b/fd/util.go index af0b7754d..57e93d431 100644 --- a/fd/util.go +++ b/fd/util.go @@ -3,6 +3,7 @@ package fd import ( "bytes" "encoding/json" + "errors" "fmt" "time" @@ -180,6 +181,95 @@ func extractMetrics(actionJSON *simplejson.Json) (string, []string, bool) { return metricName, metricLabels, skipStatus } +var ( + doIfLogicalOpNodes = map[string]struct{}{ + "and": struct{}{}, + "not": struct{}{}, + "or": struct{}{}, + } + doIfFieldOpNodes = map[string]struct{}{ + "equal": struct{}{}, + "contains": struct{}{}, + "prefix": struct{}{}, + "suffix": struct{}{}, + "regex": struct{}{}, + } +) + +func extractFieldOpNode(opName string, jsonNode *simplejson.Json) (pipeline.DoIfNode, error) { + var result pipeline.DoIfNode + var err error + fieldPath := jsonNode.Get("field").MustString() + caseSensitiveNode, has := jsonNode.CheckGet("case_sensitive") + caseSensitive := true + if has { + caseSensitive = caseSensitiveNode.MustBool() + } + values := jsonNode.Get("values") + vals := make([][]byte, 0) + for i := range values.MustArray() { + curValue := values.GetIndex(i).Interface() + if curValue == nil { + vals = append(vals, nil) + } else { + vals = append(vals, []byte(curValue.(string))) + } + } + result, err = pipeline.NewFieldOpNode(opName, fieldPath, caseSensitive, vals) + if err != nil { + return nil, fmt.Errorf("failed to init field op: %w", err) + } + + return result, nil +} + +func extractLogicalOpNode(opName string, jsonNode *simplejson.Json) (pipeline.DoIfNode, error) { + var result, operand pipeline.DoIfNode + var err error + operands := jsonNode.Get("operands") + operandsList := make([]pipeline.DoIfNode, 0) + for i := range operands.MustArray() { + opNode := operands.GetIndex(i) + operand, err = extractDoIfNode(opNode) + if err != nil { + return nil, fmt.Errorf("failed to extract operand node for logical op %q", opName) + } + operandsList = append(operandsList, operand) + } + result, err = pipeline.NewLogicalNode(opName, operandsList) + if err != nil { + return nil, fmt.Errorf("failed to init logical node: %w", err) + } + return result, nil +} + +func extractDoIfNode(jsonNode *simplejson.Json) (pipeline.DoIfNode, error) { + opNameNode, has := jsonNode.CheckGet("op") + if !has { + return nil, errors.New(`"op" field not found`) + } + opName := opNameNode.MustString() + if _, has := doIfLogicalOpNodes[opName]; has { + return extractLogicalOpNode(opName, jsonNode) + } else if _, has := doIfFieldOpNodes[opName]; has { + return extractFieldOpNode(opName, jsonNode) + } + return nil, fmt.Errorf("unknown op %q", opName) +} + +func extractDoIfChecker(actionJSON *simplejson.Json) (*pipeline.DoIfChecker, error) { + if actionJSON.MustMap() == nil { + return nil, nil + } + + root, err := extractDoIfNode(actionJSON) + if err != nil { + return nil, fmt.Errorf("failed to extract nodes: %w", err) + } + result := pipeline.NewDoIfChecker(root) + return result, nil +} + func makeActionJSON(actionJSON *simplejson.Json) []byte { actionJSON.Del("type") actionJSON.Del("match_fields") @@ -188,6 +278,7 @@ func makeActionJSON(actionJSON *simplejson.Json) []byte { actionJSON.Del("metric_labels") actionJSON.Del("metric_skip_status") actionJSON.Del("match_invert") + actionJSON.Del("do_if") configJson, err := actionJSON.Encode() if err != nil { logger.Panicf("can't create action json") diff --git a/fd/util_test.go b/fd/util_test.go index 596ce6704..332b8e0ac 100644 --- a/fd/util_test.go +++ b/fd/util_test.go @@ -1,10 +1,14 @@ package fd import ( + "bytes" + "errors" + "fmt" "testing" "github.com/bitly/go-simplejson" "github.com/ozontech/file.d/pipeline" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -33,3 +37,215 @@ func Test_extractConditions(t *testing.T) { } require.Equal(t, expected, got) } + +type doIfTreeNode struct { + fieldOp string + fieldName string + caseSensitive bool + values [][]byte + + logicalOp string + operands []*doIfTreeNode +} + +// nolint:gocritic +func buildDoIfTree(node *doIfTreeNode) (pipeline.DoIfNode, error) { + if node.fieldOp != "" { + return pipeline.NewFieldOpNode( + node.fieldOp, + node.fieldName, + node.caseSensitive, + node.values, + ) + } else if node.logicalOp != "" { + operands := make([]pipeline.DoIfNode, 0) + for _, operandNode := range node.operands { + operand, err := buildDoIfTree(operandNode) + if err != nil { + return nil, fmt.Errorf("failed to build tree: %w", err) + } + operands = append(operands, operand) + } + return pipeline.NewLogicalNode( + node.logicalOp, + operands, + ) + } + return nil, errors.New("unknown type of node") +} + +func Test_extractDoIfChecker(t *testing.T) { + type args struct { + cfgStr string + } + + tests := []struct { + name string + args args + want *doIfTreeNode + wantErr bool + }{ + { + name: "ok", + args: args{ + cfgStr: ` + { + "op": "not", + "operands": [ + { + "op": "and", + "operands": [ + { + "op": "equal", + "field": "service", + "values": [null, ""], + "case_sensitive": false + }, + { + "op": "prefix", + "field": "log.msg", + "values": ["test-1", "test-2"], + "case_sensitive": false + }, + { + "op": "or", + "operands": [ + { + "op": "suffix", + "field": "service", + "values": ["test-svc-1", "test-svc-2"], + "case_sensitive": true + }, + { + "op": "contains", + "field": "pod", + "values": ["test"] + }, + { + "op": "regex", + "field": "message", + "values": ["test-\\d+", "test-msg-\\d+"] + } + ] + } + ] + } + ] + } + `, + }, + want: &doIfTreeNode{ + logicalOp: "not", + operands: []*doIfTreeNode{ + { + logicalOp: "and", + operands: []*doIfTreeNode{ + { + fieldOp: "equal", + fieldName: "service", + values: [][]byte{nil, []byte("")}, + caseSensitive: false, + }, + { + fieldOp: "prefix", + fieldName: "log.msg", + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + caseSensitive: false, + }, + { + logicalOp: "or", + operands: []*doIfTreeNode{ + { + fieldOp: "suffix", + fieldName: "service", + values: [][]byte{[]byte("test-svc-1"), []byte("test-svc-2")}, + caseSensitive: true, + }, + { + fieldOp: "contains", + fieldName: "pod", + values: [][]byte{[]byte("test")}, + caseSensitive: true, + }, + { + fieldOp: "regex", + fieldName: "message", + values: [][]byte{[]byte(`test-\d+`), []byte(`test-msg-\d+`)}, + caseSensitive: true, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "ok_not_map", + args: args{ + cfgStr: `[{"field":"val"}]`, + }, + wantErr: false, + }, + { + name: "error_no_op_field", + args: args{ + cfgStr: `{"field": "val"}`, + }, + wantErr: true, + }, + { + name: "error_invalid_op_name", + args: args{ + cfgStr: `{"op": "invalid"}`, + }, + wantErr: true, + }, + { + name: "error_invalid_field_op", + args: args{ + cfgStr: `{"op": "equal"}`, + }, + wantErr: true, + }, + { + name: "error_invalid_logical_op", + args: args{ + cfgStr: `{"op": "or"}`, + }, + wantErr: true, + }, + { + name: "error_invalid_logical_op_operand", + args: args{ + cfgStr: `{"op": "or", "operands": [{"op": "equal"}]}`, + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + reader := bytes.NewBufferString(tt.args.cfgStr) + actionJSON, err := simplejson.NewFromReader(reader) + require.NoError(t, err) + got, err := extractDoIfChecker(actionJSON) + if (err != nil) != tt.wantErr { + t.Errorf("extractDoIfChecker() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + if tt.want == nil { + assert.Nil(t, got) + return + } + wantTree, err := buildDoIfTree(tt.want) + require.NoError(t, err) + wantDoIfChecker := pipeline.NewDoIfChecker(wantTree) + assert.NoError(t, wantDoIfChecker.IsEqualTo(got)) + }) + } +} diff --git a/pipeline/README.idoc.md b/pipeline/README.idoc.md index e35179ddb..abb775a21 100644 --- a/pipeline/README.idoc.md +++ b/pipeline/README.idoc.md @@ -1,2 +1,24 @@ -### Match modes +## Match modes @match-modes|header-description + +## Experimental: Do If rules + +This is experimental feature and represents an advanced version of `match_fields`. +The Do If rules are a tree of nodes. The tree is stored in the Do If Checker instance. +When Do If Checker's Match func is called it calls to the root Match func and then +the chain of Match func calls are performed across the whole tree. + +### Node types +@do-if-node|description + +### Field op node +@do-if-field-op-node + +### Field operations +@do-if-field-op|description + +### Logical op node +@do-if-logical-op-node + +### Logical operations +@do-if-logical-op|description diff --git a/pipeline/README.md b/pipeline/README.md index 7fd40a443..cc2858192 100755 --- a/pipeline/README.md +++ b/pipeline/README.md @@ -1,4 +1,4 @@ -### Match modes +## Match modes #### And `match_mode: and` — matches fields with AND operator @@ -105,4 +105,287 @@ result:
+## Experimental: Do If rules + +This is experimental feature and represents an advanced version of `match_fields`. +The Do If rules are a tree of nodes. The tree is stored in the Do If Checker instance. +When Do If Checker's Match func is called it calls to the root Match func and then +the chain of Match func calls are performed across the whole tree. + +### Node types +**`FieldOp`** Type of node where matching rules for fields are stored. + +
+ +**`LogicalOp`** Type of node where logical rules for applying other rules are stored. + +
+ + +### Field op node +DoIf field op node is considered to always be a leaf in the DoIf tree. +It contains operation to be checked on the field value, the field name to extract data and +the values to check against. + +Params: + - `field_op` - value from field operations list. Required. + - `field` - name of the field to apply operation. Required. + - `values` - list of values to check field. Required non-empty. + - `case_sensitive` - flag indicating whether checks are performed in case sensitive way. Default `true`. + Note: case insensitive checks can cause CPU and memory overhead since every field value will be converted to lower letters. + +Example: +```yaml +pipelines: + tests: + actions: + - type: discard + do_if: + - field_op: suffix + field: pod + values: [pod-1, pod-2] + case_sensitive: true +``` + + +### Field operations +**`Equal`** checks whether the field value is equal to one of the elements in the values list. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - field_op: equal + field: pod + values: [test-pod-1, test-pod-2] +``` + +result: +``` +{"pod":"test-pod-1","service":"test-service"} # discarded +{"pod":"test-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`Contains`** checks whether the field value contains one of the elements the in values list. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - field_op: contains + field: pod + values: [my-pod, my-test] +``` + +result: +``` +{"pod":"test-my-pod-1","service":"test-service"} # discarded +{"pod":"test-not-my-pod","service":"test-service-2"} # discarded +{"pod":"my-test-pod","service":"test-service"} # discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`Prefix`** checks whether the field value has prefix equal to one of the elements in the values list. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - field_op: prefix + field: pod + values: [test-1, test-2] +``` + +result: +``` +{"pod":"test-1-pod-1","service":"test-service"} # discarded +{"pod":"test-2-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`Suffix`** checks whether the field value has suffix equal to one of the elements in the values list. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - field_op: suffix + field: pod + values: [pod-1, pod-2] +``` + +result: +``` +{"pod":"test-1-pod-1","service":"test-service"} # discarded +{"pod":"test-2-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`Regex`** checks whether the field matches any regex from the values list. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - field_op: regex + field: pod + values: [pod-\d, my-test.*] +``` + +result: +``` +{"pod":"test-1-pod-1","service":"test-service"} # discarded +{"pod":"test-2-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"my-test-pod","service":"test-service-1"} # discarded +{"pod":"my-test-instance","service":"test-service-1"} # discarded +{"pod":"service123","service":"test-service-1"} # not discarded +``` + +
+ + +### Logical op node +DoIf logical op node is a node considered to be the root or an edge between nodes. +It always has at least one operand which are other nodes and calls their checks +to apply logical operation on their results. + +Params: + - `logical_op` - value from logical operations list. Required. + - `operands` - list of another do-if nodes. Required non-empty. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - logical_op: and + operands: + - field_op: equal + field: pod + values: [test-pod-1, test-pod-2] + case_sensitive: true + - field_op: equal + field: service + values: [test-service] + case_sensitive: true +``` + + +### Logical operations +**`Or`** accepts at least one operand and returns true on the first returned true from its operands. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - logical_op: or + operands: + - field_op: equal + field: pod + values: [test-pod-1, test-pod-2] + - field_op: equal + field: service + values: [test-service] +``` + +result: +``` +{"pod":"test-pod-1","service":"test-service"} # discarded +{"pod":"test-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`And`** accepts at least one operand and returns true if all operands return true +(in other words returns false on the first returned false from its operands). + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - logical_op: and + operands: + - field_op: equal + field: pod + values: [test-pod-1, test-pod-2] + - field_op: equal + field: service + values: [test-service] +``` + +result: +``` +{"pod":"test-pod-1","service":"test-service"} # discarded +{"pod":"test-pod-2","service":"test-service-2"} # not discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"test-pod","service":"test-service-1"} # not discarded +``` + +
+ +**`Not`** accepts exactly one operand and returns inverted result of its operand. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - logical_op: not + operands: + - field_op: equal + field: service + values: [test-service] +``` + +result: +``` +{"pod":"test-pod-1","service":"test-service"} # not discarded +{"pod":"test-pod-2","service":"test-service-2"} # discarded +{"pod":"test-pod","service":"test-service"} # not discarded +{"pod":"test-pod","service":"test-service-1"} # discarded +``` + +
+ +
*Generated using [__insane-doc__](https://github.com/vitkovskii/insane-doc)* \ No newline at end of file diff --git a/pipeline/do_if.go b/pipeline/do_if.go new file mode 100644 index 000000000..be72bba10 --- /dev/null +++ b/pipeline/do_if.go @@ -0,0 +1,678 @@ +package pipeline + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "slices" + + "github.com/ozontech/file.d/cfg" + insaneJSON "github.com/vitkovskii/insane-json" +) + +// ! do-if-node +// ^ do-if-node + +type DoIfNodeType int + +const ( + DoIfNodeEmpty DoIfNodeType = iota + + // > Type of node where matching rules for fields are stored. + DoIfNodeFieldOp // * + + // > Type of node where logical rules for applying other rules are stored. + DoIfNodeLogicalOp // * +) + +type DoIfNode interface { + Type() DoIfNodeType + Check(*insaneJSON.Root) bool + isEqualTo(DoIfNode, int) error +} + +// ! do-if-field-op +// ^ do-if-field-op + +type doIfFieldOpType int + +const ( + doIfFieldUnknownOp doIfFieldOpType = iota + doIfFieldEqualOp + doIfFieldContainsOp + doIfFieldPrefixOp + doIfFieldSuffixOp + doIfFieldRegexOp +) + +func (t doIfFieldOpType) String() string { + switch t { + case doIfFieldEqualOp: + return "equal" + case doIfFieldContainsOp: + return "contains" + case doIfFieldPrefixOp: + return "prefix" + case doIfFieldSuffixOp: + return "suffix" + case doIfFieldRegexOp: + return "regex" + } + return "unknown" +} + +var ( + // > checks whether the field value is equal to one of the elements in the values list. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - field_op: equal + // > field: pod + // > values: [test-pod-1, test-pod-2] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-pod-1","service":"test-service"} # discarded + // > {"pod":"test-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfFieldEqualOpBytes = []byte(`equal`) // * + + // > checks whether the field value contains one of the elements the in values list. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - field_op: contains + // > field: pod + // > values: [my-pod, my-test] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-my-pod-1","service":"test-service"} # discarded + // > {"pod":"test-not-my-pod","service":"test-service-2"} # discarded + // > {"pod":"my-test-pod","service":"test-service"} # discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfFieldContainsOpBytes = []byte(`contains`) // * + + // > checks whether the field value has prefix equal to one of the elements in the values list. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - field_op: prefix + // > field: pod + // > values: [test-1, test-2] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-1-pod-1","service":"test-service"} # discarded + // > {"pod":"test-2-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfFieldPrefixOpBytes = []byte(`prefix`) // * + + // > checks whether the field value has suffix equal to one of the elements in the values list. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - field_op: suffix + // > field: pod + // > values: [pod-1, pod-2] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-1-pod-1","service":"test-service"} # discarded + // > {"pod":"test-2-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfFieldSuffixOpBytes = []byte(`suffix`) // * + + // > checks whether the field matches any regex from the values list. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - field_op: regex + // > field: pod + // > values: [pod-\d, my-test.*] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-1-pod-1","service":"test-service"} # discarded + // > {"pod":"test-2-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"my-test-pod","service":"test-service-1"} # discarded + // > {"pod":"my-test-instance","service":"test-service-1"} # discarded + // > {"pod":"service123","service":"test-service-1"} # not discarded + // > ``` + doIfFieldRegexOpBytes = []byte(`regex`) // * +) + +/*{ do-if-field-op-node +DoIf field op node is considered to always be a leaf in the DoIf tree. +It contains operation to be checked on the field value, the field name to extract data and +the values to check against. + +Params: + - `field_op` - value from field operations list. Required. + - `field` - name of the field to apply operation. Required. + - `values` - list of values to check field. Required non-empty. + - `case_sensitive` - flag indicating whether checks are performed in case sensitive way. Default `true`. + Note: case insensitive checks can cause CPU and memory overhead since every field value will be converted to lower letters. + +Example: +```yaml +pipelines: + tests: + actions: + - type: discard + do_if: + - field_op: suffix + field: pod + values: [pod-1, pod-2] + case_sensitive: true +``` + +}*/ + +type doIfFieldOpNode struct { + op doIfFieldOpType + fieldPath []string + fieldPathStr string + caseSensitive bool + values [][]byte + valuesBySize map[int][][]byte + reValues []*regexp.Regexp + + minValLen int + maxValLen int +} + +func NewFieldOpNode(op string, field string, caseSensitive bool, values [][]byte) (DoIfNode, error) { + if field == "" { + return nil, errors.New("field is not specified") + } + if len(values) == 0 { + return nil, errors.New("values are not provided") + } + var vals [][]byte + var valsBySize map[int][][]byte + var reValues []*regexp.Regexp + var minValLen, maxValLen int + var fop doIfFieldOpType + + fieldPath := cfg.ParseFieldSelector(field) + + opBytes := []byte(op) + switch { + case bytes.Equal(opBytes, doIfFieldEqualOpBytes): + fop = doIfFieldEqualOp + case bytes.Equal(opBytes, doIfFieldContainsOpBytes): + fop = doIfFieldContainsOp + case bytes.Equal(opBytes, doIfFieldPrefixOpBytes): + fop = doIfFieldPrefixOp + case bytes.Equal(opBytes, doIfFieldSuffixOpBytes): + fop = doIfFieldSuffixOp + case bytes.Equal(opBytes, doIfFieldRegexOpBytes): + fop = doIfFieldRegexOp + reValues = make([]*regexp.Regexp, 0, len(values)) + for _, v := range values { + re, err := regexp.Compile(string(v)) + if err != nil { + return nil, fmt.Errorf("failed to compile regex %q: %w", v, err) + } + reValues = append(reValues, re) + } + default: + return nil, fmt.Errorf("unknown field op %q", op) + } + + if fop != doIfFieldRegexOp { + minValLen = len(values[0]) + maxValLen = len(values[0]) + if fop == doIfFieldEqualOp { + valsBySize = make(map[int][][]byte) + } else { + vals = make([][]byte, len(values)) + } + for i := range values { + var curVal []byte + if values[i] != nil { + curVal = make([]byte, len(values[i])) + copy(curVal, values[i]) + } + if !caseSensitive && curVal != nil { + curVal = bytes.ToLower(curVal) + } + if len(values[i]) < minValLen { + minValLen = len(values[i]) + } + if len(values[i]) > maxValLen { + maxValLen = len(values[i]) + } + if fop == doIfFieldEqualOp { + valsBySize[len(curVal)] = append(valsBySize[len(curVal)], curVal) + } else { + vals[i] = curVal + } + } + } + + return &doIfFieldOpNode{ + op: fop, + fieldPath: fieldPath, + fieldPathStr: field, + caseSensitive: caseSensitive, + values: vals, + valuesBySize: valsBySize, + reValues: reValues, + minValLen: minValLen, + maxValLen: maxValLen, + }, nil +} + +func (n *doIfFieldOpNode) Type() DoIfNodeType { + return DoIfNodeFieldOp +} + +func (n *doIfFieldOpNode) Check(eventRoot *insaneJSON.Root) bool { + var data []byte + node := eventRoot.Dig(n.fieldPath...) + if !node.IsNull() { + data = node.AsBytes() + } + // fast check for data + if n.op != doIfFieldRegexOp && len(data) < n.minValLen { + return false + } + switch n.op { + case doIfFieldEqualOp: + vals, ok := n.valuesBySize[len(data)] + if !ok { + return false + } + if !n.caseSensitive && data != nil { + data = bytes.ToLower(data) + } + for _, val := range vals { + // null and empty strings are considered as different values + // null can also come if field value is absent + if (data == nil && val != nil) || (data != nil && val == nil) { + continue + } + if bytes.Equal(data, val) { + return true + } + } + case doIfFieldContainsOp: + if !n.caseSensitive { + data = bytes.ToLower(data) + } + for _, val := range n.values { + if bytes.Contains(data, val) { + return true + } + } + case doIfFieldPrefixOp: + // check only necessary amount of bytes + if len(data) > n.maxValLen { + data = data[:n.maxValLen] + } + if !n.caseSensitive { + data = bytes.ToLower(data) + } + for _, val := range n.values { + if bytes.HasPrefix(data, val) { + return true + } + } + case doIfFieldSuffixOp: + // check only necessary amount of bytes + if len(data) > n.maxValLen { + data = data[len(data)-n.maxValLen:] + } + if !n.caseSensitive { + data = bytes.ToLower(data) + } + for _, val := range n.values { + if bytes.HasSuffix(data, val) { + return true + } + } + case doIfFieldRegexOp: + for _, re := range n.reValues { + if re.Match(data) { + return true + } + } + } + return false +} + +func (n *doIfFieldOpNode) isEqualTo(n2 DoIfNode, _ int) error { + n2f, ok := n2.(*doIfFieldOpNode) + if !ok { + return errors.New("nodes have different types expected: fieldOpNode") + } + if n.op != n2f.op { + return fmt.Errorf("nodes have different op expected: %q", n.op) + } + if n.caseSensitive != n2f.caseSensitive { + return fmt.Errorf("nodes have different caseSensitive expected: %v", n.caseSensitive) + } + if n.fieldPathStr != n2f.fieldPathStr || slices.Compare[[]string](n.fieldPath, n2f.fieldPath) != 0 { + return fmt.Errorf("nodes have different fieldPathStr expected: fieldPathStr=%q fieldPath=%v", + n.fieldPathStr, n.fieldPath, + ) + } + if len(n.values) != len(n2f.values) { + return fmt.Errorf("nodes have different values slices len expected: %d", len(n.values)) + } + for i := 0; i < len(n.values); i++ { + if !bytes.Equal(n.values[i], n2f.values[i]) { + return fmt.Errorf("nodes have different data in values expected: %v on position", n.values) + } + } + if len(n.valuesBySize) != len(n2f.valuesBySize) { + return fmt.Errorf("nodes have different valuesBySize len expected: %d", len(n.valuesBySize)) + } + for k, v := range n.valuesBySize { + if v2, has := n2f.valuesBySize[k]; !has { + return fmt.Errorf("nodes have different valuesBySize keys expected key: %d", k) + } else if len(v) != len(v2) { + return fmt.Errorf("nodes have different valuesBySize values len under key %d expected: %d", k, len(v)) + } else { + for i := 0; i < len(v); i++ { + if !bytes.Equal(v[i], v2[i]) { + return fmt.Errorf("nodes have different valuesBySize data under key %d: %v", k, v) + } + } + } + } + if len(n.reValues) != len(n2f.reValues) { + return fmt.Errorf("nodes have different reValues len expected: %d", len(n.reValues)) + } + for i := 0; i < len(n.reValues); i++ { + if n.reValues[i].String() != n2f.reValues[i].String() { + return fmt.Errorf("nodes have different reValues data expected: %v", n.reValues) + } + } + if n.minValLen != n2f.minValLen { + return fmt.Errorf("nodes have different minValLem expected: %d", n.minValLen) + } + if n.maxValLen != n2f.maxValLen { + return fmt.Errorf("nodes have different maxValLem expected: %d", n.maxValLen) + } + return nil +} + +// ! do-if-logical-op +// ^ do-if-logical-op + +type doIfLogicalOpType int + +const ( + doIfLogicalOpUnknown doIfLogicalOpType = iota + doIfLogicalOr + doIfLogicalAnd + doIfLogicalNot +) + +func (t doIfLogicalOpType) String() string { + switch t { + case doIfLogicalOr: + return "or" + case doIfLogicalAnd: + return "and" + case doIfLogicalNot: + return "not" + } + return "unknown" +} + +var ( + // > accepts at least one operand and returns true on the first returned true from its operands. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - logical_op: or + // > operands: + // > - field_op: equal + // > field: pod + // > values: [test-pod-1, test-pod-2] + // > - field_op: equal + // > field: service + // > values: [test-service] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-pod-1","service":"test-service"} # discarded + // > {"pod":"test-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfLogicalOrBytes = []byte(`or`) // * + + // > accepts at least one operand and returns true if all operands return true + // > (in other words returns false on the first returned false from its operands). + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - logical_op: and + // > operands: + // > - field_op: equal + // > field: pod + // > values: [test-pod-1, test-pod-2] + // > - field_op: equal + // > field: service + // > values: [test-service] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-pod-1","service":"test-service"} # discarded + // > {"pod":"test-pod-2","service":"test-service-2"} # not discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"test-pod","service":"test-service-1"} # not discarded + // > ``` + doIfLogicalAndBytes = []byte(`and`) // * + + // > accepts exactly one operand and returns inverted result of its operand. + // > + // > Example: + // > ```yaml + // > pipelines: + // > test: + // > actions: + // > - type: discard + // > do_if: + // > - logical_op: not + // > operands: + // > - field_op: equal + // > field: service + // > values: [test-service] + // > ``` + // > + // > result: + // > ``` + // > {"pod":"test-pod-1","service":"test-service"} # not discarded + // > {"pod":"test-pod-2","service":"test-service-2"} # discarded + // > {"pod":"test-pod","service":"test-service"} # not discarded + // > {"pod":"test-pod","service":"test-service-1"} # discarded + // > ``` + doIfLogicalNotBytes = []byte(`not`) // * +) + +/*{ do-if-logical-op-node +DoIf logical op node is a node considered to be the root or an edge between nodes. +It always has at least one operand which are other nodes and calls their checks +to apply logical operation on their results. + +Params: + - `logical_op` - value from logical operations list. Required. + - `operands` - list of another do-if nodes. Required non-empty. + +Example: +```yaml +pipelines: + test: + actions: + - type: discard + do_if: + - logical_op: and + operands: + - field_op: equal + field: pod + values: [test-pod-1, test-pod-2] + case_sensitive: true + - field_op: equal + field: service + values: [test-service] + case_sensitive: true +``` + +}*/ + +type doIfLogicalNode struct { + op doIfLogicalOpType + operands []DoIfNode +} + +func NewLogicalNode(op string, operands []DoIfNode) (DoIfNode, error) { + if len(operands) == 0 { + return nil, errors.New("logical op must have at least one operand") + } + var lop doIfLogicalOpType + opBytes := []byte(op) + switch { + case bytes.Equal(opBytes, doIfLogicalOrBytes): + lop = doIfLogicalOr + case bytes.Equal(opBytes, doIfLogicalAndBytes): + lop = doIfLogicalAnd + case bytes.Equal(opBytes, doIfLogicalNotBytes): + lop = doIfLogicalNot + if len(operands) > 1 { + return nil, fmt.Errorf("logical not must have exactly one operand, got %d", len(operands)) + } + default: + return nil, fmt.Errorf("unknown logical op %q", op) + } + return &doIfLogicalNode{ + op: lop, + operands: operands, + }, nil +} + +func (n *doIfLogicalNode) Type() DoIfNodeType { + return DoIfNodeLogicalOp +} + +func (n *doIfLogicalNode) Check(eventRoot *insaneJSON.Root) bool { + switch n.op { + case doIfLogicalOr: + for _, op := range n.operands { + if op.Check(eventRoot) { + return true + } + } + return false + case doIfLogicalAnd: + for _, op := range n.operands { + if !op.Check(eventRoot) { + return false + } + } + return true + case doIfLogicalNot: + return !n.operands[0].Check(eventRoot) + } + return false +} + +func (n *doIfLogicalNode) isEqualTo(n2 DoIfNode, level int) error { + n2l, ok := n2.(*doIfLogicalNode) + if !ok { + return errors.New("nodes have different types expected: logicalNode") + } + if n.op != n2l.op { + return fmt.Errorf("nodes have different op expected: %q", n.op) + } + if len(n.operands) != len(n2l.operands) { + return fmt.Errorf("nodes have different operands len expected: %d", len(n.operands)) + } + for i := 0; i < len(n.operands); i++ { + if err := n.operands[i].isEqualTo(n2l.operands[i], level+1); err != nil { + tabs := make([]byte, 0, level) + for j := 0; j < level; j++ { + tabs = append(tabs, '\t') + } + return fmt.Errorf("nodes with op %q have different operand nodes on position %d:\n%s%w", n.op, i, tabs, err) + } + } + return nil +} + +type DoIfChecker struct { + root DoIfNode +} + +func NewDoIfChecker(root DoIfNode) *DoIfChecker { + return &DoIfChecker{ + root: root, + } +} + +func (c *DoIfChecker) IsEqualTo(c2 *DoIfChecker) error { + return c.root.isEqualTo(c2.root, 1) +} + +func (c *DoIfChecker) Check(eventRoot *insaneJSON.Root) bool { + if eventRoot == nil { + return false + } + return c.root.Check(eventRoot) +} diff --git a/pipeline/do_if_test.go b/pipeline/do_if_test.go new file mode 100644 index 000000000..0d79dcbdf --- /dev/null +++ b/pipeline/do_if_test.go @@ -0,0 +1,1015 @@ +package pipeline + +import ( + "errors" + "fmt" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + insaneJSON "github.com/vitkovskii/insane-json" +) + +type treeNode struct { + fieldOp string + fieldName string + caseSensitive bool + values [][]byte + + logicalOp string + operands []treeNode +} + +// nolint:gocritic +func buildTree(node treeNode) (DoIfNode, error) { + if node.fieldOp != "" { + return NewFieldOpNode( + node.fieldOp, + node.fieldName, + node.caseSensitive, + node.values, + ) + } else if node.logicalOp != "" { + operands := make([]DoIfNode, 0) + for _, operandNode := range node.operands { + operand, err := buildTree(operandNode) + if err != nil { + return nil, fmt.Errorf("failed to build tree: %w", err) + } + operands = append(operands, operand) + } + return NewLogicalNode( + node.logicalOp, + operands, + ) + } + return nil, errors.New("unknown type of node") +} + +func checkDoIfNode(t *testing.T, want, got DoIfNode) { + require.Equal(t, want.Type(), got.Type()) + switch want.Type() { + case DoIfNodeFieldOp: + wantNode := want.(*doIfFieldOpNode) + gotNode := got.(*doIfFieldOpNode) + assert.Equal(t, wantNode.op, gotNode.op) + assert.Equal(t, 0, slices.Compare[[]string](wantNode.fieldPath, gotNode.fieldPath)) + assert.Equal(t, wantNode.fieldPathStr, gotNode.fieldPathStr) + assert.Equal(t, wantNode.caseSensitive, gotNode.caseSensitive) + if wantNode.values == nil { + assert.Equal(t, wantNode.values, gotNode.values) + } else { + require.Equal(t, len(wantNode.values), len(gotNode.values)) + for i := 0; i < len(wantNode.values); i++ { + wantValues := wantNode.values[i] + gotValues := gotNode.values[i] + assert.Equal(t, 0, slices.Compare[[]byte](wantValues, gotValues)) + } + } + if wantNode.valuesBySize == nil { + assert.Equal(t, wantNode.valuesBySize, gotNode.valuesBySize) + } else { + require.Equal(t, len(wantNode.valuesBySize), len(gotNode.valuesBySize)) + for k, wantVals := range wantNode.valuesBySize { + gotVals, ok := gotNode.valuesBySize[k] + assert.True(t, ok, "values by key %d not present in got node", k) + if ok { + require.Equal(t, len(wantVals), len(gotVals)) + for i := 0; i < len(wantVals); i++ { + assert.Equal(t, 0, slices.Compare[[]byte](wantVals[i], gotVals[i])) + } + } + } + } + assert.Equal(t, wantNode.minValLen, gotNode.minValLen) + assert.Equal(t, wantNode.maxValLen, gotNode.maxValLen) + case DoIfNodeLogicalOp: + wantNode := want.(*doIfLogicalNode) + gotNode := got.(*doIfLogicalNode) + assert.Equal(t, wantNode.op, gotNode.op) + require.Equal(t, len(wantNode.operands), len(gotNode.operands)) + for i := 0; i < len(wantNode.operands); i++ { + checkDoIfNode(t, wantNode.operands[i], gotNode.operands[i]) + } + } +} + +func TestBuildDoIfNodes(t *testing.T) { + tests := []struct { + name string + tree treeNode + want DoIfNode + wantErr bool + }{ + { + name: "ok_field_op_node", + tree: treeNode{ + fieldOp: "equal", + fieldName: "log.pod", + caseSensitive: true, + values: [][]byte{[]byte(`test-111`), []byte(`test-2`), []byte(`test-3`), []byte(`test-12345`)}, + }, + want: &doIfFieldOpNode{ + op: doIfFieldEqualOp, + fieldPath: []string{"log", "pod"}, + fieldPathStr: "log.pod", + caseSensitive: true, + values: nil, + valuesBySize: map[int][][]byte{ + 6: [][]byte{ + []byte(`test-2`), + []byte(`test-3`), + }, + 8: [][]byte{ + []byte(`test-111`), + }, + 10: [][]byte{ + []byte(`test-12345`), + }, + }, + minValLen: 6, + maxValLen: 10, + }, + }, + { + name: "ok_field_op_node_case_insensitive", + tree: treeNode{ + fieldOp: "equal", + fieldName: "log.pod", + caseSensitive: false, + values: [][]byte{[]byte(`TEST-111`), []byte(`Test-2`), []byte(`tesT-3`), []byte(`TeSt-12345`)}, + }, + want: &doIfFieldOpNode{ + op: doIfFieldEqualOp, + fieldPath: []string{"log", "pod"}, + fieldPathStr: "log.pod", + caseSensitive: false, + values: nil, + valuesBySize: map[int][][]byte{ + 6: [][]byte{ + []byte(`test-2`), + []byte(`test-3`), + }, + 8: [][]byte{ + []byte(`test-111`), + }, + 10: [][]byte{ + []byte(`test-12345`), + }, + }, + minValLen: 6, + maxValLen: 10, + }, + }, + { + name: "ok_logical_op_node_or", + tree: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "log.pod", + caseSensitive: true, + values: [][]byte{[]byte(`test-111`), []byte(`test-2`), []byte(`test-3`), []byte(`test-12345`)}, + }, + { + fieldOp: "contains", + fieldName: "service.msg", + caseSensitive: true, + values: [][]byte{[]byte(`test-0987`), []byte(`test-11`)}, + }, + }, + }, + want: &doIfLogicalNode{ + op: doIfLogicalOr, + operands: []DoIfNode{ + &doIfFieldOpNode{ + op: doIfFieldEqualOp, + fieldPath: []string{"log", "pod"}, + fieldPathStr: "log.pod", + caseSensitive: true, + values: nil, + valuesBySize: map[int][][]byte{ + 6: [][]byte{ + []byte(`test-2`), + []byte(`test-3`), + }, + 8: [][]byte{ + []byte(`test-111`), + }, + 10: [][]byte{ + []byte(`test-12345`), + }, + }, + minValLen: 6, + maxValLen: 10, + }, + &doIfFieldOpNode{ + op: doIfFieldContainsOp, + fieldPath: []string{"service", "msg"}, + fieldPathStr: "service.msg", + caseSensitive: true, + values: [][]byte{ + []byte(`test-0987`), + []byte(`test-11`), + }, + minValLen: 7, + maxValLen: 9, + }, + }, + }, + }, + { + name: "err_field_op_node_empty_field", + tree: treeNode{ + fieldOp: "equal", + }, + wantErr: true, + }, + { + name: "err_field_op_node_empty_values", + tree: treeNode{ + fieldOp: "equal", + fieldName: "pod", + }, + wantErr: true, + }, + { + name: "err_field_op_node_invalid_regex", + tree: treeNode{ + fieldOp: "regex", + fieldName: "pod", + values: [][]byte{[]byte(`\`)}, + }, + wantErr: true, + }, + { + name: "err_field_op_node_invalid_op_type", + tree: treeNode{ + fieldOp: "noop", + fieldName: "pod", + values: [][]byte{[]byte(`test`)}, + }, + wantErr: true, + }, + { + name: "err_logical_op_node_empty_operands", + tree: treeNode{ + logicalOp: "or", + }, + wantErr: true, + }, + { + name: "err_logical_op_node_invalid_op_type", + tree: treeNode{ + logicalOp: "noop", + operands: []treeNode{ + { + fieldOp: "contains", + fieldName: "service.msg", + caseSensitive: true, + values: [][]byte{[]byte(`test-0987`), []byte(`test-11`)}, + }, + }, + }, + wantErr: true, + }, + { + name: "err_logical_op_node_too_much_operands", + tree: treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + fieldOp: "contains", + fieldName: "service.msg", + caseSensitive: true, + values: [][]byte{[]byte(`test-0987`), []byte(`test-11`)}, + }, + { + fieldOp: "contains", + fieldName: "service.msg", + caseSensitive: true, + values: [][]byte{[]byte(`test-0987`), []byte(`test-11`)}, + }, + }, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got, err := buildTree(tt.tree) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + checkDoIfNode(t, tt.want, got) + }) + } +} + +func TestCheck(t *testing.T) { + type argsResp struct { + eventStr string + want bool + } + + tests := []struct { + name string + tree treeNode + data []argsResp + wantNewNodeErr bool + }{ + { + name: "ok_equal", + tree: treeNode{ + fieldOp: "equal", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2"), []byte("test-pod-123"), []byte("po-32")}, + }, + data: []argsResp{ + {eventStr: `{"pod":"test-1"}`, want: true}, + {eventStr: `{"pod":"test-2"}`, want: true}, + {eventStr: `{"pod":"test-3"}`, want: false}, + {eventStr: `{"pod":"TEST-2"}`, want: false}, + {eventStr: `{"pod":"test-pod-123"}`, want: true}, + {eventStr: `{"pod":"po-32"}`, want: true}, + {eventStr: `{"pod":"p-32"}`, want: false}, + {eventStr: `{"service":"test-1"}`, want: false}, + {eventStr: `{"pod":"test-123456789"}`, want: false}, + {eventStr: ``, want: false}, + }, + }, + { + name: "ok_contains", + tree: treeNode{ + fieldOp: "contains", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + data: []argsResp{ + {`{"pod":"my-test-1-pod"}`, true}, + {`{"pod":"my-test-2-pod"}`, true}, + {`{"pod":"my-test-3-pod"}`, false}, + {`{"pod":"my-TEST-2-pod"}`, false}, + }, + }, + { + name: "ok_prefix", + tree: treeNode{ + fieldOp: "prefix", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + data: []argsResp{ + {`{"pod":"test-1-pod"}`, true}, + {`{"pod":"test-2-pod"}`, true}, + {`{"pod":"test-3-pod"}`, false}, + {`{"pod":"TEST-2-pod"}`, false}, + }, + }, + { + name: "ok_suffix", + tree: treeNode{ + fieldOp: "suffix", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + data: []argsResp{ + {`{"pod":"my-test-1"}`, true}, + {`{"pod":"my-test-2"}`, true}, + {`{"pod":"my-test-3"}`, false}, + {`{"pod":"my-TEST-2"}`, false}, + }, + }, + { + name: "ok_regex", + tree: treeNode{ + fieldOp: "regex", + fieldName: "pod", + values: [][]byte{[]byte(`test-\d`)}, + }, + data: []argsResp{ + {`{"pod":"my-test-1-pod"}`, true}, + {`{"pod":"my-test-2-pod"}`, true}, + {`{"pod":"my-test-3-pod"}`, true}, + {`{"pod":"my-test-pod"}`, false}, + {`{"pod":"my-pod-3-pod"}`, false}, + {`{"pod":"my-TEST-4-pod"}`, false}, + }, + }, + { + name: "ok_or", + tree: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + { + fieldOp: "equal", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-3"), []byte("test-4")}, + }, + }, + }, + data: []argsResp{ + {`{"pod":"test-1"}`, true}, + {`{"pod":"test-2"}`, true}, + {`{"pod":"test-3"}`, true}, + {`{"pod":"test-4"}`, true}, + {`{"pod":"test-5"}`, false}, + {`{"pod":"TEST-1"}`, false}, + {`{"pod":"TEST-3"}`, false}, + }, + }, + { + name: "ok_and", + tree: treeNode{ + logicalOp: "and", + operands: []treeNode{ + { + fieldOp: "prefix", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test")}, + }, + { + fieldOp: "suffix", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("pod")}, + }, + }, + }, + data: []argsResp{ + {`{"pod":"test-1-pod"}`, true}, + {`{"pod":"test-2-pod"}`, true}, + {`{"pod":"test-3"}`, false}, + {`{"pod":"service-test-4-pod"}`, false}, + {`{"pod":"service-test-5"}`, false}, + {`{"pod":"TEST-6-pod"}`, false}, + {`{"pod":"test-7-POD"}`, false}, + }, + }, + { + name: "ok_not", + tree: treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "pod", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + }, + }, + data: []argsResp{ + {`{"pod":"test-1"}`, false}, + {`{"pod":"test-2"}`, false}, + {`{"pod":"TEST-2"}`, true}, + {`{"pod":"test-3"}`, true}, + {`{"pod":"test-4"}`, true}, + }, + }, + { + name: "ok_equal_case_insensitive", + tree: treeNode{ + fieldOp: "equal", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{[]byte("Test-1"), []byte("tesT-2")}, + }, + data: []argsResp{ + {eventStr: `{"pod":"tEST-1"}`, want: true}, + {eventStr: `{"pod":"test-2"}`, want: true}, + {eventStr: `{"pod":"test-3"}`, want: false}, + {eventStr: `{"pod":"TEST-2"}`, want: true}, + }, + }, + { + name: "ok_contains_case_insensitive", + tree: treeNode{ + fieldOp: "contains", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{[]byte("Test-1"), []byte("tesT-2")}, + }, + data: []argsResp{ + {`{"pod":"my-tEST-1-pod"}`, true}, + {`{"pod":"my-test-2-pod"}`, true}, + {`{"pod":"my-test-3-pod"}`, false}, + {`{"pod":"my-TEST-2-pod"}`, true}, + }, + }, + { + name: "ok_prefix_case_insensitive", + tree: treeNode{ + fieldOp: "prefix", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{[]byte("Test-1"), []byte("tesT-2")}, + }, + data: []argsResp{ + {`{"pod":"tEST-1-pod"}`, true}, + {`{"pod":"test-2-pod"}`, true}, + {`{"pod":"test-3-pod"}`, false}, + {`{"pod":"TEST-2-pod"}`, true}, + }, + }, + { + name: "ok_suffix_case_insensitive", + tree: treeNode{ + fieldOp: "suffix", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{[]byte("Test-1"), []byte("tesT-2")}, + }, + data: []argsResp{ + {`{"pod":"my-teST-1"}`, true}, + {`{"pod":"my-test-2"}`, true}, + {`{"pod":"my-test-3"}`, false}, + {`{"pod":"my-TEST-2"}`, true}, + }, + }, + { + name: "ok_equal_nil_or_empty_string", + tree: treeNode{ + fieldOp: "equal", + fieldName: "test-field", + caseSensitive: false, + values: [][]byte{nil, []byte("")}, + }, + data: []argsResp{ + {`{"pod":"my-teST-1"}`, true}, + {`{"pod":"my-test-2","test-field":null}`, true}, + {`{"pod":"my-test-3","test-field":""}`, true}, + {`{"pod":"my-TEST-2","test-field":"non-empty"}`, false}, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + var root DoIfNode + var eventRoot *insaneJSON.Root + var err error + t.Parallel() + root, err = buildTree(tt.tree) + if tt.wantNewNodeErr { + require.Error(t, err) + return + } + require.NoError(t, err) + checker := NewDoIfChecker(root) + for _, d := range tt.data { + if d.eventStr == "" { + eventRoot = nil + } else { + eventRoot, err = insaneJSON.DecodeString(d.eventStr) + require.NoError(t, err) + } + got := checker.Check(eventRoot) + assert.Equal(t, d.want, got, "invalid result for event %q", d.eventStr) + } + }) + } +} + +func TestDoIfNodeIsEqual(t *testing.T) { + singleNode := treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + } + twoNodes := treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + }, + } + multiNodes := treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{nil, []byte(""), []byte("null")}, + }, + { + fieldOp: "contains", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{[]byte("pod-1"), []byte("pod-2")}, + }, + { + logicalOp: "and", + operands: []treeNode{ + { + fieldOp: "prefix", + fieldName: "message", + caseSensitive: true, + values: [][]byte{[]byte("test-msg-1"), []byte("test-msg-2")}, + }, + { + fieldOp: "suffix", + fieldName: "message", + caseSensitive: true, + values: [][]byte{[]byte("test-msg-3"), []byte("test-msg-4")}, + }, + { + fieldOp: "regex", + fieldName: "msg", + caseSensitive: true, + values: [][]byte{[]byte("test-\\d+"), []byte("test-000-\\d+")}, + }, + }, + }, + }, + }, + }, + } + tests := []struct { + name string + t1 treeNode + t2 treeNode + wantErr bool + }{ + { + name: "equal_single_node", + t1: singleNode, + t2: singleNode, + wantErr: false, + }, + { + name: "equal_two_nodes", + t1: twoNodes, + t2: twoNodes, + wantErr: false, + }, + { + name: "equal_multiple_nodes", + t1: multiNodes, + t2: multiNodes, + wantErr: false, + }, + { + name: "not_equal_type_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + t2: treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + wantErr: true, + }, + { + name: "not_equal_field_op_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + t2: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_op_mismatch_2", + t1: treeNode{ + fieldOp: "prefix", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + t2: treeNode{ + fieldOp: "suffix", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_op_mismatch_3", + t1: treeNode{ + fieldOp: "regex", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + t2: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_case_sensitive_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{[]byte("test-1")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_field_path_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "log.msg", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "log.svc", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_slice_len_mismatch", + t1: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_slice_vals_mismatch", + t1: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "contains", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_by_size_len_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-22")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_by_size_vals_key_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-11")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_by_size_vals_len_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_values_by_size_vals_mismatch", + t1: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "equal", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_reValues_len_mismatch", + t1: treeNode{ + fieldOp: "regex", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1"), []byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "regex", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_field_reValues_vals_mismatch", + t1: treeNode{ + fieldOp: "regex", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-2")}, + }, + t2: treeNode{ + fieldOp: "regex", + fieldName: "service", + caseSensitive: true, + values: [][]byte{[]byte("test-1")}, + }, + wantErr: true, + }, + { + name: "not_equal_logical_op_mismatch", + t1: treeNode{ + logicalOp: "not", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + t2: treeNode{ + logicalOp: "and", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + wantErr: true, + }, + { + name: "not_equal_logical_operands_len_mismatch", + t1: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + t2: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + wantErr: true, + }, + { + name: "not_equal_logical_operands_mismatch_field_name", + t1: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "service", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + t2: treeNode{ + logicalOp: "or", + operands: []treeNode{ + { + fieldOp: "equal", + fieldName: "pod", + caseSensitive: false, + values: [][]byte{nil}, + }, + }, + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + root1, err := buildTree(tt.t1) + require.NoError(t, err) + root2, err := buildTree(tt.t2) + require.NoError(t, err) + c1 := NewDoIfChecker(root1) + c2 := NewDoIfChecker(root2) + err1 := c1.IsEqualTo(c2) + err2 := c2.IsEqualTo(c1) + if tt.wantErr { + assert.Error(t, err1, "tree1 expected to be not equal to tree2") + assert.Error(t, err2, "tree2 expected to be not equal to tree1") + } else { + assert.NoError(t, err1, "tree1 expected to be equal to tree2") + assert.NoError(t, err2, "tree2 expected to be equal to tree1") + } + }) + } +} diff --git a/pipeline/pipeline.go b/pipeline/pipeline.go index 4f2f51f11..239b3fa41 100644 --- a/pipeline/pipeline.go +++ b/pipeline/pipeline.go @@ -610,7 +610,9 @@ func (p *Pipeline) expandProcs() { p.logger.Warn("too many processors", zap.Int32("new", to)) } - for x := 0; x < int(to-from); x++ { + // proc IDs are added starting from the next after the last one + // so all procs have unique IDs + for x := 1; x <= int(to-from); x++ { proc := p.newProc(p.Procs[from-1].id + x) p.Procs = append(p.Procs, proc) proc.start(p.actionParams, p.logger.Sugar()) diff --git a/pipeline/plugin.go b/pipeline/plugin.go index 90ac0f180..40d84b1cb 100644 --- a/pipeline/plugin.go +++ b/pipeline/plugin.go @@ -96,6 +96,8 @@ type ActionPluginStaticInfo struct { MatchConditions MatchConditions MatchMode MatchMode MatchInvert bool + + DoIfChecker *DoIfChecker } type ActionPluginInfo struct { diff --git a/pipeline/processor.go b/pipeline/processor.go index df8711385..492e3f6f8 100644 --- a/pipeline/processor.go +++ b/pipeline/processor.go @@ -263,6 +263,11 @@ func (p *processor) countEvent(event *Event, actionIndex int, status eventStatus func (p *processor) isMatch(index int, event *Event) bool { info := p.actionInfos[index] + + if info.DoIfChecker != nil { + return info.DoIfChecker.Check(event.Root) + } + conds := info.MatchConditions mode := info.MatchMode match := false