diff --git a/.chloggen/add-log-body-cc-filtering.yaml b/.chloggen/add-log-body-cc-filtering.yaml new file mode 100644 index 00000000..0180455a --- /dev/null +++ b/.chloggen/add-log-body-cc-filtering.yaml @@ -0,0 +1,18 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: transformprocessor + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Extend the credit card filtering example with log body filtering + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [357] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: diff --git a/config_examples/masking_logbody.yaml b/config_examples/masking_logbody.yaml new file mode 100644 index 00000000..457145e9 --- /dev/null +++ b/config_examples/masking_logbody.yaml @@ -0,0 +1,36 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 +exporters: + otlphttp: + endpoint: ${env:DT_ENDPOINT} + headers: + Authorization: "Api-Token ${env:API_TOKEN}" + +processors: + transform/cc: + error_mode: ignore + trace_statements: + log_statements: + - context: log + statements: + - replace_pattern(body, "(^|\\s)3\\s*[47](\\s*[0-9]){9}((\\s*[0-9]){4})($|\\s)", "$$1**** $$3$$5") + - replace_pattern(body, "(^|\\s)(5[1-5]([0-9]){2}|222[1-9]|22[3-9]\\d|2[3-6]\\d{2}|27[0-1]\\d|2720)(\\s*[0-9]){8}\\s*([0-9]{4})($|\\s)", "$$1**** $$5$$6") + - replace_pattern(body, "(^|\\s)4(\\s*[0-9]){8,14}\\s*(([0-9]\\s*?){4})($|\\s)", "$$1**** $$3$$5") + transform/iban: + error_mode: ignore + trace_statements: + log_statements: + - context: log + statements: + - replace_pattern(body, "(^|\\s)([A-Z]{2})[0-9]{2}((\\s*[A-Z0-9]){4,26})\\s*((\\s*[A-Z0-9]){4})($|\\s)", "$$1$$2 **** $$5$$7") + + +service: + pipelines: + logs: + receivers: [otlp] + processors: [transform/iban, transform/cc] + exporters: [otlphttp] diff --git a/internal/testbed/integration/filtering_test.go b/internal/testbed/integration/filtering_test.go index 4d830e59..4f72de9f 100644 --- a/internal/testbed/integration/filtering_test.go +++ b/internal/testbed/integration/filtering_test.go @@ -155,6 +155,177 @@ func TestFilteringCreditCard(t *testing.T) { } } +func TestFilteringLogBody(t *testing.T) { + logsCCPlain := []string{ + "card_master_spaces1 2367 8901 2345 6789", + "card_master_spaces2 5105 1051 0510 5100", + "card_master_spaces3 2720 1051 0510 5100", + "card_master_spaces4 2720 1051 0520 5100 some text after", + "card_master_no_spaces1 2367890123456789", + "card_master_no_spaces2 5105105105105100", + "card_master_no_spaces3 2720105105105100", + "card_master_no_spaces4 2720145107105100 testy test", + "card_visa_spaces1 4539 1488 0343 6467", + "card_visa_spaces2 4539 1488 0343 6", + "card_visa_spaces3 4539 1488 0343 6467 234", + "card_visa_spaces4 4539 1498 0343 6457 234 testy test", + "card_visa_no_spaces1 4539148803436467", + "card_visa_no_spaces2 4539148803436", + "card_visa_no_spaces3 4539148803436467234", + "card_visa_no_spaces4 4539148812336467234 asdf asdf", + "card_amex_spaces1 3714 496353 98431", + "card_amex_spaces2 3487 344936 71000", + "card_amex_spaces3 3782 822463 10005", + "card_amex_spaces4 3782 822463 12305 testy test.", + "card_amex_no_spaces1 371449635398431", + "card_amex_no_spaces2 348734493671000", + "card_amex_no_spaces3 378282246310005", + "card_amex_no_spaces4 378282243210005 some postfix", + "safe_attribute1 371", + "safe_attribute2 37810005", + "safe_attribute3 987346 some postfix test", + } + + logsCCFiltered := []string{ + "card_master_spaces1 **** 6789", + "card_master_spaces2 **** 5100", + "card_master_spaces3 **** 5100", + "card_master_spaces4 **** 5100 some text after", + "card_master_no_spaces1 **** 6789", + "card_master_no_spaces2 **** 5100", + "card_master_no_spaces3 **** 5100", + "card_master_no_spaces4 **** 5100 testy test", + "card_visa_spaces1 **** 6467", + "card_visa_spaces2 **** 343 6", + "card_visa_spaces3 **** 7 234", + "card_visa_spaces4 **** 7 234 testy test", + "card_visa_no_spaces1 **** 6467", + "card_visa_no_spaces2 **** 3436", + "card_visa_no_spaces3 **** 7234", + "card_visa_no_spaces4 **** 7234 asdf asdf", + "card_amex_spaces1 **** 8431", + "card_amex_spaces2 **** 1000", + "card_amex_spaces3 **** 0005", + "card_amex_spaces4 **** 2305 testy test.", + "card_amex_no_spaces1 **** 8431", + "card_amex_no_spaces2 **** 1000", + "card_amex_no_spaces3 **** 0005", + "card_amex_no_spaces4 **** 0005 some postfix", + "safe_attribute1 371", + "safe_attribute2 37810005", + "safe_attribute3 987346 some postfix test", + } + + logsIbanPlain := []string{ + "iban1 DE89 3704 0044 0532 0130 00", + "iban2 FR14 2004 1010 0505 0001 3M02 606", + "iban3 ES91 2100 0418 4502 0005 1332", + "iban4 IT60 X054 2811 1010 0000 0123 456", + "iban5 NL91 ABNA 0417 1643 00", + "iban6 BE71 0961 2345 6769", + "iban7 AT48 3200 0000 0123 4568 testy test", + "iban8 SE72 8000 0810 0340 0978 3242", + "iban9 PL61 1090 1014 0000 0712 1981 2874", + "iban10 GB29 NWBK 6016 1331 9268 19", + "iban11 AL47 2121 1009 0000 0002 3569 8741 asdf asdf", + "iban12 CY17 0020 0128 0000 0012 0052 7600", + "iban13 KW81 CBKU 0000 0000 0000 1234 5601 01", + "iban14 LU28 0019 4006 4475 0000", + "iban15 NO93 8601 1117 947", + "iban16 DE89370400440532013000", + "iban17 FR1420041010050500013M02606 test222", + "iban18 ES9121000418450200051332", + "iban19 IT60X0542811101000000123456", + "iban20 NL91ABNA0417164300 text text", + "iban21 BE71096123456769", + "iban22 AT483200000001234568", + "iban23 SE7280000810034009783242", + "iban24 PL61109010140000071219812874", + "iban25 GB29NWBK60161331926819 postfix text", + "iban26 AL47212110090000000235698741", + "iban27 CY17002001280000001200527600", + "iban28 KW81CBKU0000000000001234560101", + "iban29 LU280019400644750000", + "iban30 NO9386011117947", + "non-iban no4444 ds", + } + + logsIbanFiltered := []string{ + "iban1 DE **** 30 00", + "iban2 FR **** 2 606", + "iban3 ES **** 1332", + "iban4 IT **** 3 456", + "iban5 NL **** 43 00", + "iban6 BE **** 6769", + "iban7 AT **** 4568 testy test", + "iban8 SE **** 3242", + "iban9 PL **** 2874", + "iban10 GB **** 68 19", + "iban11 AL **** 8741 asdf asdf", + "iban12 CY **** 7600", + "iban13 KW **** 01 01", + "iban14 LU **** 0000", + "iban15 NO **** 7 947", + "iban16 DE **** 3000", + "iban17 FR **** 2606 test222", + "iban18 ES **** 1332", + "iban19 IT **** 3456", + "iban20 NL **** 4300 text text", + "iban21 BE **** 6769", + "iban22 AT **** 4568", + "iban23 SE **** 3242", + "iban24 PL **** 2874", + "iban25 GB **** 6819 postfix text", + "iban26 AL **** 8741", + "iban27 CY **** 7600", + "iban28 KW **** 0101", + "iban29 LU **** 0000", + "iban30 NO **** 7947", + "non-iban no4444 ds", + } + + creditCardTransformConfig := "masking_logbody.yaml" + + tests := []struct { + name string + dataProvider testbed.DataProvider + sender SenderFunc + receiver ReceiverFunc + validator testbed.TestCaseValidator + configName string + }{ + { + name: "CC log bodies transform", + dataProvider: NewSampleConfigsLogsDataProvider(generateLogsWithBodies(logsCCPlain)), + sender: NewOTLPLogsDataSenderWrapper, + receiver: testbed.NewOTLPHTTPDataReceiver, + validator: NewLogsValidator(t, []plog.Logs{generateLogsWithBodies(logsCCFiltered)}), + configName: creditCardTransformConfig, + }, + { + name: "IBAN log bodies transform", + dataProvider: NewSampleConfigsLogsDataProvider(generateLogsWithBodies(logsIbanPlain)), + sender: NewOTLPLogsDataSenderWrapper, + receiver: testbed.NewOTLPHTTPDataReceiver, + validator: NewLogsValidator(t, []plog.Logs{generateLogsWithBodies(logsIbanFiltered)}), + configName: creditCardTransformConfig, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + FilteringScenario( + t, + test.dataProvider, + test.sender, + test.receiver, + test.validator, + test.configName, + ) + }) + } +} + func TestFilteringDTAPITokenRedactionProcessor(t *testing.T) { configName := "redaction_api_token.yaml" @@ -661,3 +832,20 @@ func generateBasicLogsWithAttributes(attributes pcommon.Map) plog.Logs { return logs } + +func generateLogsWithBodies(bodies []string) plog.Logs { + logs := plog.NewLogs() + rl := logs.ResourceLogs().AppendEmpty() + + for _, body := range bodies { + logRecords := rl.ScopeLogs().AppendEmpty().LogRecords() + logRecords.EnsureCapacity(1) + + record := logRecords.AppendEmpty() + record.SetSeverityNumber(plog.SeverityNumberInfo3) + record.SetSeverityText("INFO") + record.Body().SetStr(body) + } + + return logs +}