fix pseudonymizer and domain_resolver tests

logprep/metrics/metrics.py

@@ -1,41 +1,14 @@
 """This module tracks, calculates, exposes and resets logprep metrics"""
 from abc import ABC, abstractmethod
 
-from attr import asdict, define, field, validators
+from attr import define, field, validators
 from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram
 
 
-def is_public(attribute, _):
-    """If an attribute name starts with an underscore it is considered private"""
-    return not attribute.name.startswith("_")
-
-
-def is_writable(attribute):
-    """Checks if an attribute is of type property and has a setter method"""
-    return isinstance(attribute, property) and attribute.fset is not None
-
-
-def get_exposable_metrics(metric_object):
-    """Retrieves exposable attributes by checking if they are public"""
-    metric_dict = asdict(metric_object, filter=is_public)
-    all_attributes = vars(type(metric_object)).items()
-    # include properties as they are not part of asdict
-    properties = {n: p.__get__(metric_object) for n, p in all_attributes if isinstance(p, property)}
-    metric_dict.update(properties)
-    return metric_dict
-
-
-def get_settable_metrics(metric_object):
-    """Retrieves writable attributes by checking have a setter method"""
-    metric_dict = asdict(metric_object)
-    all_attributes = vars(type(metric_object)).items()
-    # include properties as they are not part of asdict
-    metric_dict.update({n: p.__get__(metric_object) for n, p in all_attributes if is_writable(p)})
-    return metric_dict
-
-
 @define(kw_only=True)
 class Metric(ABC):
+    """Metric base class"""
+
     name: str = field(validator=validators.instance_of(str))
     description: str = field(validator=validators.instance_of(str))
     labels: dict = field(
@@ -54,9 +27,11 @@ class Metric(ABC):
 
     @property
     def fullname(self):
+        """returns the fullname"""
         return f"{self._prefix}{self.name}"
 
     def init_tracker(self):
+        """initializes the tracker and adds it to the trackers dict"""
         tracker = None
         try:
             if isinstance(self, CounterMetric):
@@ -94,6 +69,8 @@ def __add__(self, other):
 
 @define(kw_only=True)
 class CounterMetric(Metric):
+    """Wrapper for prometheus Counter metric"""
+
     trackers: dict = {}
 
     def __add__(self, other):
@@ -103,6 +80,8 @@ def __add__(self, other):
 
 @define(kw_only=True)
 class HistogramMetric(Metric):
+    """Wrapper for prometheus Histogram metric"""
+
     trackers: dict = {}
 
     def __add__(self, other):
@@ -112,17 +91,10 @@ def __add__(self, other):
 
 @define(kw_only=True)
 class GaugeMetric(Metric):
+    """Wrapper for prometheus Gauge metric""" ""
+
     trackers: dict = {}
 
     def __add__(self, other):
         self.trackers.get(self.fullname).labels(**self.labels).set(other)
         return self
-
-
-def calculate_new_average(current_average, next_sample, sample_counter):
-    """Calculate a new average by combining a new sample with a sample counter"""
-    average_multiple = current_average * sample_counter
-    extended_average_multiple = average_multiple + next_sample
-    sample_counter += 1
-    new_average = extended_average_multiple / sample_counter
-    return new_average, sample_counter

logprep/processor/amides/processor.py

@@ -85,20 +85,14 @@
 from functools import cached_property, lru_cache
 from multiprocessing import current_process
 from pathlib import Path
-from time import time
 from typing import List, Tuple
 from zipfile import ZipFile
 
 import joblib
 from attr import define, field, validators
 
 from logprep.abc.processor import Processor
-from logprep.metrics.metrics import (
-    CounterMetric,
-    GaugeMetric,
-    HistogramMetric,
-    calculate_new_average,
-)
+from logprep.metrics.metrics import CounterMetric, GaugeMetric, HistogramMetric
 from logprep.processor.amides.detection import MisuseDetector, RuleAttributor
 from logprep.processor.amides.normalize import CommandLineNormalizer
 from logprep.processor.amides.rule import AmidesRule

logprep/processor/domain_resolver/processor.py

@@ -48,6 +48,7 @@
 from tldextract import TLDExtract
 
 from logprep.abc.processor import Processor
+from logprep.metrics.metrics import CounterMetric
 from logprep.processor.domain_resolver.rule import DomainResolverRule
 from logprep.util.cache import Cache
 from logprep.util.getter import GetterFactory
@@ -101,13 +102,33 @@ class Config(Processor.Config):
     class Metrics(Processor.Metrics):
         """Tracks statistics about the DomainResolver"""
 
-        total_urls: int = 0
+        total_urls: CounterMetric = field(
+            factory=lambda: CounterMetric(
+                description="Number of all resolved urls",
+                name="domain_resolver_total_urls",
+            )
+        )
         """Number of all resolved urls"""
-        resolved_new: int = 0
+        resolved_new: CounterMetric = field(
+            factory=lambda: CounterMetric(
+                description="Number of urls that had to be resolved newly",
+                name="domain_resolver_resolved_new",
+            )
+        )
         """Number of urls that had to be resolved newly"""
-        resolved_cached: int = 0
+        resolved_cached: CounterMetric = field(
+            factory=lambda: CounterMetric(
+                description="Number of urls that were resolved from cache",
+                name="domain_resolver_resolved_cached",
+            )
+        )
         """Number of urls that were resolved from cache"""
-        timeouts: int = 0
+        timeouts: CounterMetric = field(
+            factory=lambda: CounterMetric(
+                description="Number of timeouts that occurred while resolving a url",
+                name="domain_resolver_timeouts",
+            )
+        )
         """Number of timeouts that occurred while resolving a url"""
 
     __slots__ = ["_domain_ip_map"]

logprep/processor/pseudonymizer/processor.py

@@ -47,6 +47,7 @@
 from urlextract import URLExtract
 
 from logprep.abc.processor import Processor
+from logprep.metrics.metrics import CounterMetric
 from logprep.processor.pseudonymizer.encrypter import DualPKCS1HybridEncrypter
 from logprep.processor.pseudonymizer.rule import PseudonymizerRule
 from logprep.util.cache import Cache
@@ -129,7 +130,12 @@ class Config(Processor.Config):
     class Metrics(Processor.Metrics):
         """Tracks statistics about the Pseudonymizer"""
 
-        pseudonymized_urls: int = 0
+        pseudonymized_urls: CounterMetric = field(
+            factory=lambda: CounterMetric(
+                description="Number of urls that were pseudonymized",
+                name="pseudonymizer_pseudonymized_urls",
+            )
+        )
         """Number urls that were pseudonymized"""
 
     __slots__ = [