Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

update add_field_to function for improved error handling #696

Merged
merged 38 commits into from
Nov 14, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
012dd6e
update add_field_to function for improved error handling
dtrai2 Nov 6, 2024
0adad9c
let add_field_to always raise FieldExistsWarning on failure
dtrai2 Nov 7, 2024
33f402d
fix field_manager tests
dtrai2 Nov 7, 2024
64cf75a
fix domain_label_extractor tests
dtrai2 Nov 7, 2024
f3da215
fix generic_adder tests
dtrai2 Nov 7, 2024
28fd58c
fix geoip_enricher tests
dtrai2 Nov 7, 2024
ad52e08
fig grokker processor
dtrai2 Nov 8, 2024
817d980
fig ip_informer processor
dtrai2 Nov 8, 2024
ad8d0d4
fig labeler tests processor
dtrai2 Nov 8, 2024
9dfd7ab
fig list_comparison processor tests
dtrai2 Nov 8, 2024
d7582f9
fig pre_detector processor tests
dtrai2 Nov 8, 2024
b124195
fix requester processor tests
dtrai2 Nov 8, 2024
1ea2526
fix ProcessingWarning init
dtrai2 Nov 8, 2024
adfd509
fix timestamper processor
dtrai2 Nov 8, 2024
f68d7c3
fix template_replacer processor
dtrai2 Nov 8, 2024
66f6696
fix input connector tests
dtrai2 Nov 8, 2024
113bb25
fix FieldExistsWarning init tests
dtrai2 Nov 8, 2024
1db38f3
fix auto_rule_tester
dtrai2 Nov 8, 2024
770a80a
fix and refactor generic_resolver
dtrai2 Nov 11, 2024
52d1823
fix and refactor hyperscan_resolver
dtrai2 Nov 11, 2024
28d12b0
fix auto_rule_tester
dtrai2 Nov 11, 2024
04e115d
fix labeler and add new test
dtrai2 Nov 11, 2024
bb7cbd0
clean up
dtrai2 Nov 11, 2024
bcae7ba
replace else statement with early return
dtrai2 Nov 12, 2024
297aa97
remove silent fail functions for field addition
dtrai2 Nov 12, 2024
4cd6db7
Refactor overwrite argument for field addition functions
dtrai2 Nov 12, 2024
0507313
add assertions to verify document state after exceptions
dtrai2 Nov 12, 2024
f56d45b
optimize imports
dtrai2 Nov 12, 2024
ea781f5
update add_field_to signature
dtrai2 Nov 12, 2024
bdb445f
enable `add_field_to` to always take a batch of fields
dtrai2 Nov 13, 2024
90038c0
revert exception signature and add rule to add_field_to method as arg…
dtrai2 Nov 13, 2024
6439b8c
revert key indexing in dissector back to original
dtrai2 Nov 13, 2024
ad2bc78
add tests for multiple field additions in add_field_to
dtrai2 Nov 13, 2024
a22bf0a
renamed 'add_field_to' to 'add_fields_to'
dtrai2 Nov 13, 2024
0facf61
fix typo in StringSplitterRule
dtrai2 Nov 13, 2024
827b7f4
remove unused conflicting_fields list
dtrai2 Nov 13, 2024
8405edd
rename _add_one_field_to to _add_field_to for clarity
dtrai2 Nov 13, 2024
2506df6
fix CHANGELOG.md
dtrai2 Nov 13, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix auto_rule_tester
- one test is still broken, needs further investigation why
  • Loading branch information
dtrai2 committed Nov 13, 2024
commit 1db38f3fc7cb3842a21b1d04b5a70d87a80e9c24
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
filter: message
clusterer:
id: clusterer-rule-2
source_fields: [message]
pattern: '(bytes|Bytes|Byte)'
repl: 'byte'
description: '...'
tests:
raw: 'Byte is a Bytes is a bytes is a byte'
result: 'byte is a byte is a byte is a byte'
result: 'byte is a byte is a byte is a byte'
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
filter: message
clusterer:
id: clusterer-rule-1
source_fields: [message]
pattern: '(bytes|Bytes|Byte)'
repl: 'byte'
description: '...'
tests:
raw: 'Byte is a Bytes is a bytes is a byte'
result: 'byte is a byte is a byte is a byte'
result: 'byte is a byte is a byte is a byte'
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
[{
"filter": "message",
"dissector": {
"id": "dissector-1",
"mapping": {
"message": "%{source}-%{target}"
}
},
"description": "Test-rule with matching auto-test"
}]
}]
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
[{
"filter": "message",
"dissector": {
"id": "dissector-2",
"mapping": {
"message": "%{source} %{target}"
}
},
"description": "Test-rule with matching auto-test"
}]
}]
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
{
"filter": "drop_me",
"dropper": {
"id": "dropper-1",
"drop": [
"drop_me"
]
}
}
]
]
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,10 @@
{
"filter": "drop_me",
"dropper": {
"id": "dropper-2",
"drop": [
"drop_me"
]
}
}
]
]
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
{
"filter": "some_field: (stop OR end)",
"labeler": {
"id": "labeler-1",
"label": {
"action": [
"terminate"
Expand All @@ -10,4 +11,4 @@
},
"description": "Test-rule with matching auto-test"
}
]
]
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
{
"filter": "some_field: (stop OR end)",
"labeler": {
"id": "labeler-2",
"label": {
"action": [
"terminate"
Expand All @@ -10,4 +11,4 @@
},
"description": "Test-rule with matching auto-test"
}
]
]
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
[{
"filter": "some_field",
"pre_detector": {
"id": "SOME_TEST_RULE_ID",
"id": "SOME_TEST_RULE_ID_1",
"title": "SOME_TEST_RULE",
"severity": "critical",
"mitre": [],
"case_condition": "directly"
},
"sigma_fields": true,
"description": "Test-rule with mismatching auto-test"
}]
}]
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
[{
"filter": "some_field",
"pre_detector": {
"id": "SOME_TEST_RULE_ID",
"id": "SOME_TEST_RULE_ID_2",
"title": "SOME_TEST_RULE",
"severity": "critical",
"mitre": [],
"case_condition": "directly"
},
"sigma_fields": true,
"description": "Test-rule without auto-test"
}]
}]
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
[{
"filter": "winlog.provider_name: \"the provider\" AND winlog.event_id: 123",
"template_replacer": {},
"template_replacer": {
"id": "template-replacer-1"
},
"description": ""
}]
}]
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
[
{
"filter": "winlog.provider_name: \"the provider\" AND winlog.event_id: 123",
"template_replacer": {},
"template_replacer": {
"id": "template-replacer-2"
},
"description": ""
}
]
]
11 changes: 11 additions & 0 deletions tests/unit/processor/template_replacer/test_template_replacer.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,3 +163,14 @@ def _create_template_replacer(self, config):
template_replacer = Factory.create({"test instance": config})
template_replacer.setup()
return template_replacer

def test_replace_message_via_template(self):
document = {
"winlog": {"channel": "System", "provider_name": "Test", "event_id": 123},
"message": "foo",
}

self.object.process(document)

assert document.get("message")
assert document["message"] == "Test %1 Test %2"
2 changes: 2 additions & 0 deletions tests/unit/util/test_auto_rule_tester.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
import pytest

from logprep.util.auto_rule_tester.auto_rule_tester import AutoRuleTester
from logprep.util.configuration import Configuration

LOGGER = logging.getLogger()


@pytest.fixture(name="auto_rule_tester")
def fixture_auto_rule_tester():
config_path = "tests/testdata/config/config-auto-tests.yml"
Configuration.from_source(config_path)._verify()
return AutoRuleTester(config_path)


Expand Down