WIP

logprep/abc/component.py

@@ -8,7 +8,7 @@
 from attrs import asdict
 from schedule import Scheduler
 
-from logprep.metrics.metrics import Metric, MetricType
+from logprep.metrics.metrics import CounterMetric, HistogramMetric
 from logprep.util.helper import camel_to_snake
 
 
@@ -29,34 +29,39 @@ class Metrics:
         _labels: dict
         _prefix: str = "logprep_"
 
-        number_of_processed_events: Metric = field(
-            factory=lambda: Metric(
-                type=MetricType.COUNTER,
+        number_of_processed_events: CounterMetric = field(
+            factory=lambda: CounterMetric(
                 description="Number of events that were processed",
                 name="number_of_processed_events",
             )
         )
         """Number of events that were processed"""
 
-        number_of_failed_events: Metric = field(
-            factory=lambda: Metric(
-                type=MetricType.COUNTER,
+        number_of_failed_events: CounterMetric = field(
+            factory=lambda: CounterMetric(
                 description="Number of events that were send to error output",
                 name="number_of_failed_events",
             )
         )
         """Number of events that were send to error output"""
 
+        processing_time_per_event: HistogramMetric = field(
+            factory=lambda: HistogramMetric(
+                description="Time in seconds that it took to process an event",
+                name="processing_time_per_event",
+            )
+        )
+        """Time in seconds that it took to process an event"""
+
         def __attrs_post_init__(self):
             for attribute in asdict(self):
                 attribute = getattr(self, attribute)
-                if isinstance(attribute, Metric):
+                if isinstance(attribute, CounterMetric):
                     attribute.labels = self._labels
-                    attribute.tracker = attribute.type(
+                    attribute.tracker = attribute(
                         name=f"{self._prefix}{attribute.name}",
                         documentation=attribute.description,
                         labelnames=attribute.labels.keys(),
-                        registry=None,
                     )
                     attribute.tracker.labels(**attribute.labels)
 
@@ -144,3 +149,6 @@ def _schedule_task(
     def run_pending_tasks(cls) -> None:
         """Starts all pending tasks. This is called in :code:`pipeline.py`"""
         cls._scheduler.run_pending()
+
+
+tri

logprep/abc/processor.py

@@ -1,9 +1,8 @@
 """Abstract module for processors"""
-import copy
 import time
 from abc import abstractmethod
 from functools import reduce
-from logging import DEBUG, Logger
+from logging import Logger
 from pathlib import Path
 from typing import TYPE_CHECKING, List, Optional
 

logprep/framework/rule_tree/rule_tree.py

@@ -10,6 +10,7 @@
 from logprep.abc.component import Component
 from logprep.framework.rule_tree.node import Node
 from logprep.framework.rule_tree.rule_parser import RuleParser
+from logprep.metrics.metrics import CounterMetric, MetricType
 from logprep.util import getter
 
 if TYPE_CHECKING:
@@ -30,7 +31,7 @@ class RuleTree:
     """Represent a set of rules using a rule tree model."""
 
     @define(kw_only=True)
-    class RuleTreeMetrics(Component.Metrics):
+    class Metrics(Component.Metrics):
         """Tracks statistics about the current rule tree"""
 
         number_of_processed_events = field(default=None)
@@ -48,7 +49,7 @@ class RuleTreeMetrics(Component.Metrics):
     )
 
     rule_parser: Optional[RuleParser]
-    metrics: RuleTreeMetrics
+    metrics: Metrics
     priority_dict: dict
     rule_tree_type: RuleTreeType
     _rule_mapping: dict
@@ -83,7 +84,7 @@ def __init__(
         self._processor_name = processor_name
         self.rule_tree_type = rule_tree_type
         self._setup()
-        self.metrics = self.RuleTreeMetrics(labels=self.metric_labels)
+        self.metrics = self.Metrics(labels=self.metric_labels)
 
         if root:
             self._root = root

logprep/metrics/metrics.py

@@ -1,5 +1,5 @@
 """This module tracks, calculates, exposes and resets logprep metrics"""
-import logging
+from abc import ABC, abstractmethod
 from enum import Enum
 
 from attr import asdict, define, field, validators
@@ -35,24 +35,8 @@ def get_settable_metrics(metric_object):
     return metric_dict
 
 
-class MetricType(Enum):
-    COUNTER = 1
-    HISTOGRAM = 2
-    GAUGE = 3
-    INFO = 4
-
-
-PROMETHEUS_METRIC_TYPES = {
-    MetricType.COUNTER.value: Counter,
-    MetricType.GAUGE.value: Gauge,
-    MetricType.INFO.value: Info,
-    MetricType.HISTOGRAM.value: Histogram,
-}
-
-
 @define(kw_only=True)
-class Metric:
-    type: MetricType = field(converter=lambda x: PROMETHEUS_METRIC_TYPES.get(x.value))
+class Metric(ABC):
     name: str = field(validator=validators.instance_of(str))
     description: str = field(validator=validators.instance_of(str))
     labels: dict = field(
@@ -67,6 +51,36 @@ class Metric:
     )
     tracker: object = field(default=None)
 
+    @abstractmethod
+    def __add__(self, other):
+        """Add"""
+
+
+@define(kw_only=True)
+class CounterMetric(Metric):
+    def __attrs_post_init__(self):
+        self.tracker = Counter(
+            name=self.name,
+            documentation=self.description,
+            labelnames=self.labels.keys(),
+            registry=None,
+        )
+
+    def __add__(self, other):
+        self.tracker.labels(**self.labels).inc(other)
+        return self
+
+
+@define(kw_only=True)
+class HistogramMetric(Metric):
+    def __attrs_post_init__(self):
+        self.tracker = Histogram(
+            name=self.name,
+            documentation=self.description,
+            labelnames=self.labels.keys(),
+            registry=None,
+        )
+
     def __add__(self, other):
         self.tracker.labels(**self.labels).inc(other)
         return self

logprep/util/configuration.py

@@ -7,9 +7,11 @@
 from pathlib import Path
 from typing import List
 
+from attrs import define, field
 from colorama import Fore
 from ruamel.yaml.scanner import ScannerError
 
+from logprep.abc.component import Component
 from logprep.abc.getter import Getter
 from logprep.abc.processor import Processor
 from logprep.factory import Factory
@@ -51,34 +53,13 @@ def __init__(self, key: str):
         super().__init__(f"Required option is missing: {key}")
 
 
-class InvalidLabelingSchemaError(InvalidConfigurationError):
-    """Raise if labeling schema is invalid."""
-
-    def __init__(self, message: str):
-        super().__init__(f"Invalid labeling schema: {message}")
-
-
-class InvalidRulesError(InvalidConfigurationError):
-    """Raise if set of rules is invalid."""
-
-    def __init__(self, message: str):
-        super().__init__(f"Invalid rule set: {message}")
-
-
 class InvalidProcessorConfigurationError(InvalidConfigurationError):
     """Raise if processor configuration is invalid."""
 
     def __init__(self, message: str):
         super().__init__(f"Invalid processor configuration: {message}")
 
 
-class InvalidConnectorConfigurationError(InvalidConfigurationError):
-    """Raise if connector configuration is invalid."""
-
-    def __init__(self, message: str):
-        super().__init__(f"Invalid connector configuration: {message}")
-
-
 class InvalidInputConnectorConfigurationError(InvalidConfigurationError):
     """Raise if input connector configuration is invalid."""
 
@@ -93,13 +74,6 @@ def __init__(self, message: str):
         super().__init__(f"Invalid output connector configuration: {message}")
 
 
-class IncalidMetricsConfigurationError(InvalidConfigurationError):
-    """Raise if status_logger configuration is invalid."""
-
-    def __init__(self, message: str):
-        super().__init__(f"Invalid metrics configuration: {message}")
-
-
 class MissingEnvironmentError(InvalidConfigurationError):
     """Raise if environment variables are missing"""
 

tests/unit/metrics/test_metric_exposer.py

@@ -80,11 +80,11 @@ def test_aggregate_metrics_combines_list_of_metrics_to_one(self):
         rule_metrics_one = Rule.RuleMetrics(labels={"type": "generic"})
         rule_metrics_one._number_of_matches = 1
         rule_metrics_one.update_mean_processing_time(1)
-        rule_tree_one = RuleTree.RuleTreeMetrics(labels={"type": "tree"}, rules=[rule_metrics_one])
+        rule_tree_one = RuleTree.Metrics(labels={"type": "tree"}, rules=[rule_metrics_one])
         rule_metrics_two = Rule.RuleMetrics(labels={"type": "generic"})
         rule_metrics_two._number_of_matches = 2
         rule_metrics_two.update_mean_processing_time(2)
-        rule_tree_two = RuleTree.RuleTreeMetrics(labels={"type": "tree"}, rules=[rule_metrics_two])
+        rule_tree_two = RuleTree.Metrics(labels={"type": "tree"}, rules=[rule_metrics_two])
         self.exposer._store_metrics(rule_tree_one)
         self.exposer._store_metrics(rule_tree_two)
         metrics = self.exposer._aggregate_metrics()

tests/unit/metrics/test_metrics.py

@@ -7,12 +7,12 @@
 from prometheus_client import REGISTRY
 from prometheus_client.registry import Collector
 
-from logprep.metrics.metrics import Metric, MetricType
+from logprep.metrics.metrics import CounterMetric, MetricType
 
 
 class TestsMetrics:
     def test_converts_enum_to_prometheus_metric(self):
-        metric = Metric(
+        metric = CounterMetric(
             name="testmetric",
             type=MetricType.COUNTER,
             description="empty description",
@@ -21,7 +21,7 @@ def test_converts_enum_to_prometheus_metric(self):
         assert issubclass(metric.type, Collector)
 
     def test_counter_metric_sets_labels(self):
-        metric = Metric(
+        metric = CounterMetric(
             type=MetricType.COUNTER,
             name="bla",
             description="empty description",
@@ -31,7 +31,7 @@ def test_counter_metric_sets_labels(self):
         assert ("pipeline-1",) in metric.tracker._metrics
 
     def test_counter_metric_increments_correctly(self):
-        metric = Metric(
+        metric = CounterMetric(
             type=MetricType.COUNTER,
             name="bla",
             description="empty description",

tests/unit/processor/dissector/test_dissector.py

@@ -5,7 +5,7 @@
 import pytest
 from prometheus_client import Counter
 
-from logprep.metrics.metrics import Metric
+from logprep.metrics.metrics import CounterMetric
 from tests.unit.processor.base import BaseProcessorTestCase
 
 test_cases = [  # testcase, rule, event, expected
@@ -721,6 +721,6 @@ def test_testcases_failure_handling(self, caplog, testcase, rule, event, expecte
         assert event == expected, testcase
 
     def test_has_number_of_processed_events_metric(self):
-        assert isinstance(self.object.metrics.number_of_processed_events, Metric)
+        assert isinstance(self.object.metrics.number_of_processed_events, CounterMetric)
         self.object.process({"test": "event"})
         assert True