Skip to content

Commit

Permalink
fix inits of ProecssingWarnings and FieldExistsWarnings
Browse files Browse the repository at this point in the history
  • Loading branch information
dtrai2 committed Oct 19, 2023
1 parent 2a2f826 commit cece1cf
Show file tree
Hide file tree
Showing 17 changed files with 24 additions and 27 deletions.
6 changes: 3 additions & 3 deletions logprep/abc/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

from logprep.abc.component import Component
from logprep.framework.rule_tree.rule_tree import RuleTree, RuleTreeType
from logprep.metrics.metrics import CounterMetric, HistogramMetric
from logprep.metrics.metrics import HistogramMetric
from logprep.processor.base.exceptions import (
FieldExistsWarning,
ProcessingCriticalError,
Expand Down Expand Up @@ -293,7 +293,7 @@ def _handle_warning_error(self, event, rule, error, failure_tags=None):
if isinstance(error, ProcessingWarning):
self._logger.warning(str(error))
else:
self._logger.warning(str(ProcessingWarning(self, str(error), rule, event)))
self._logger.warning(str(ProcessingWarning(str(error), rule, event)))

def _has_missing_values(self, event, rule, source_field_dict):
missing_fields = list(
Expand Down Expand Up @@ -324,4 +324,4 @@ def _write_target_field(self, event: dict, rule: "Rule", result: any) -> None:
overwrite_output_field=rule.overwrite_target,
)
if not add_successful:
raise FieldExistsWarning(self, rule, event, [rule.target_field])
raise FieldExistsWarning(rule, event, [rule.target_field])
2 changes: 1 addition & 1 deletion logprep/processor/domain_label_extractor/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def _apply_rules(self, event, rule: DomainLabelExtractorRule):
)

if not add_successful:
raise FieldExistsWarning(self, rule, event, [output_field])
raise FieldExistsWarning(rule, event, [output_field])
else:
tagging_field.append(f"invalid_domain_in_{rule.source_fields[0].replace('.', '_')}")
event[self._config.tagging_field_name] = tagging_field
Expand Down
4 changes: 2 additions & 2 deletions logprep/processor/field_manager/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def _write_to_multiple_targets(self, event, target_fields, field_values, rule, r
unsuccessful_targets = [
x for i, x in enumerate(target_fields) if i in unsuccessful_indices
]
raise FieldExistsWarning(self, rule, event, unsuccessful_targets)
raise FieldExistsWarning(rule, event, unsuccessful_targets)

def _write_to_single_target(self, args, extend_target_list, overwrite_target, rule):
if extend_target_list and overwrite_target:
Expand All @@ -116,7 +116,7 @@ def _add_field_to(self, *args, rule):
field_values = field_values.pop()
successful = add_field_to(event, target_field, field_values, False, False)
if not successful:
raise FieldExistsWarning(self, rule, event, [target_field])
raise FieldExistsWarning(rule, event, [target_field])

def _overwrite_target_with_source_field_values(self, event, target_field, field_values):
if len(field_values) == 1:
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/generic_adder/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ def _apply_rules(self, event: dict, rule: GenericAdderRule):
conflicting_fields.append(dotted_field)

if conflicting_fields:
raise FieldExistsWarning(self, rule, event, conflicting_fields)
raise FieldExistsWarning(rule, event, conflicting_fields)

def _try_adding_from_db(self, event: dict, items_to_add: list, rule: GenericAdderRule):
"""Get the sub part of the value from the event using a regex pattern"""
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/generic_resolver/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def _apply_rules(self, event, rule):
break

if conflicting_fields:
raise FieldExistsWarning(self, rule, event, conflicting_fields)
raise FieldExistsWarning(rule, event, conflicting_fields)

def ensure_rules_from_file(self, rule):
"""loads rules from file"""
Expand Down
4 changes: 2 additions & 2 deletions logprep/processor/geoip_enricher/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ def _apply_rules(self, event, rule):
ip_string = get_dotted_field_value(event, rule.source_fields[0])
if ip_string is None:
raise ProcessingWarning(
self, f"Value of IP field '{rule.source_fields[0]}' is 'None'", rule, event
f"Value of IP field '{rule.source_fields[0]}' is 'None'", rule, event
)
geoip_data = self._try_getting_geoip_data(ip_string)
if not geoip_data:
Expand All @@ -140,4 +140,4 @@ def _apply_rules(self, event, rule):
overwrite_output_field=rule.overwrite_target,
)
if not adding_was_successful:
raise FieldExistsWarning(self, rule, event, [full_output_field])
raise FieldExistsWarning(rule, event, [full_output_field])
4 changes: 2 additions & 2 deletions logprep/processor/grokker/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,9 +95,9 @@ def _apply_rules(self, event: dict, rule: GrokkerRule):
if not success:
conflicting_fields.append(dotted_field)
if conflicting_fields:
raise FieldExistsWarning(self, rule, event, conflicting_fields)
raise FieldExistsWarning(rule, event, conflicting_fields)
if not matches:
raise ProcessingWarning(self, "no grok pattern matched", rule, event)
raise ProcessingWarning("no grok pattern matched", rule, event)

def setup(self):
"""Loads the action mapping. Has to be called before processing"""
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/hyperscan_resolver/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ def _apply_rules(self, event: dict, rule: HyperscanResolverRule):
if has_conflict:
conflicting_fields.append(split_dotted_keys[idx])
if conflicting_fields:
raise FieldExistsWarning(self, rule, event, conflicting_fields)
raise FieldExistsWarning(rule, event, conflicting_fields)

@staticmethod
def _try_adding_value_to_existing_field(
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/ip_informer/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ def _apply_rules(self, event: dict, rule: IpInformerRule) -> None:
if results:
self._write_target_field(event, rule, results)
for msg, error in self._processing_warnings:
raise ProcessingWarning(self, msg, rule, event) from error
raise ProcessingWarning(msg, rule, event) from error

def _get_results(self, ip_address_list: Iterable, rule: IpInformerRule) -> dict:
results = [(ip, self._ip_properties(ip, rule)) for ip in ip_address_list]
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/list_comparison/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ def _apply_rules(self, event, rule):
output_field = f"{ rule.target_field }.{ comparison_key }"
field_possible = add_field_to(event, output_field, comparison_result, True)
if not field_possible:
raise FieldExistsWarning(self, rule, event, [output_field])
raise FieldExistsWarning(rule, event, [output_field])

def _list_comparison(self, rule: ListComparisonRule, event: dict):
"""
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/normalizer/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -377,7 +377,7 @@ def _apply_field_copy(self, event: dict, source_field: str, target_field: str):

def _raise_warning_if_fields_already_existed(self, rule, event):
if self._conflicting_fields:
raise FieldExistsWarning(self, rule, event, self._conflicting_fields)
raise FieldExistsWarning(rule, event, self._conflicting_fields)

def shut_down(self):
"""
Expand Down
2 changes: 1 addition & 1 deletion logprep/processor/requester/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def _handle_response(self, event, rule, response):
if not successful:
conflicting_fields.append(rule.target_field)
if conflicting_fields:
raise FieldExistsWarning(self, rule, event, [rule.target_field])
raise FieldExistsWarning(rule, event, [rule.target_field])

def _request(self, event, rule, kwargs):
try:
Expand Down
4 changes: 2 additions & 2 deletions logprep/processor/string_splitter/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ def _apply_rules(self, event: dict, rule: StringSplitterRule):
source_field_content = get_dotted_field_value(event, source_field)
if not isinstance(source_field_content, str):
raise ProcessingWarning(
self, f"source_field '{source_field}' is not a string", rule, event
f"source_field '{source_field}' is not a string", rule, event
)
result = source_field_content.split(rule.delimeter)
successful = add_field_to(
Expand All @@ -54,4 +54,4 @@ def _apply_rules(self, event: dict, rule: StringSplitterRule):
overwrite_output_field=rule.overwrite_target,
)
if not successful:
raise FieldExistsWarning(self, rule, event, [target_field])
raise FieldExistsWarning(rule, event, [target_field])
2 changes: 1 addition & 1 deletion logprep/processor/template_replacer/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -160,5 +160,5 @@ def _perform_replacement(self, event: dict, replacement: str, rule: TemplateRepl
event[subfield] = {}
event = event[subfield]
else:
raise FieldExistsWarning(self, event, rule, [subfield])
raise FieldExistsWarning(rule, event, [subfield])
event[self._target_field_split[-1]] = replacement
2 changes: 1 addition & 1 deletion logprep/processor/timestamp_differ/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ def _apply_rules(self, event, rule):
overwrite_output_field=rule.overwrite_target,
)
if not add_successful:
raise FieldExistsWarning(self, rule, event, [rule.target_field])
raise FieldExistsWarning(rule, event, [rule.target_field])

@staticmethod
def _create_timestamp_object(source: Union[str, int], format_str: str) -> datetime:
Expand Down
4 changes: 2 additions & 2 deletions logprep/processor/timestamper/processor.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,14 +58,14 @@ def _apply_rules(self, event, rule):
parsed_successfully = True
break
if not parsed_successfully:
raise ProcessingWarning(self, str("Could not parse timestamp"), rule, event)
raise ProcessingWarning(str("Could not parse timestamp"), rule, event)

def _get_source_field(self, event, rule):
source_field = rule.source_fields[0]
source_field_value = get_dotted_field_value(event, source_field)
if not source_field_value:
raise ProcessingWarning(
self, f"'{source_field}' does not exist or is falsy value", rule, event
f"'{source_field}' does not exist or is falsy value", rule, event
)

return source_field_value
5 changes: 1 addition & 4 deletions tests/unit/framework/test_pipeline.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
Output,
WarningOutputError,
)
from logprep.abc.processor import Processor
from logprep.factory import Factory
from logprep.framework.pipeline import MultiprocessingPipeline, Pipeline, SharedCounter
from logprep.processor.base.exceptions import ProcessingCriticalError, ProcessingWarning
Expand Down Expand Up @@ -229,9 +228,7 @@ def test_output_warning_error_is_logged_but_processing_continues(self, mock_warn
def test_processor_warning_error_is_logged_but_processing_continues(self, mock_warning, _):
self.pipeline._setup()
self.pipeline._input.get_next.return_value = ({"message": "test"}, None)
processing_warning = ProcessingWarning(
self.pipeline._pipeline[1], "not so bad", None, {"message": "test"}
)
processing_warning = ProcessingWarning("not so bad", None, {"message": "test"})

def raise_processing_warning(_):
raise processing_warning
Expand Down

0 comments on commit cece1cf

Please sign in to comment.