diff --git a/logprep/metrics/metrics.py b/logprep/metrics/metrics.py index c1ef442aa..84723c190 100644 --- a/logprep/metrics/metrics.py +++ b/logprep/metrics/metrics.py @@ -1,41 +1,14 @@ """This module tracks, calculates, exposes and resets logprep metrics""" from abc import ABC, abstractmethod -from attr import asdict, define, field, validators +from attr import define, field, validators from prometheus_client import CollectorRegistry, Counter, Gauge, Histogram -def is_public(attribute, _): - """If an attribute name starts with an underscore it is considered private""" - return not attribute.name.startswith("_") - - -def is_writable(attribute): - """Checks if an attribute is of type property and has a setter method""" - return isinstance(attribute, property) and attribute.fset is not None - - -def get_exposable_metrics(metric_object): - """Retrieves exposable attributes by checking if they are public""" - metric_dict = asdict(metric_object, filter=is_public) - all_attributes = vars(type(metric_object)).items() - # include properties as they are not part of asdict - properties = {n: p.__get__(metric_object) for n, p in all_attributes if isinstance(p, property)} - metric_dict.update(properties) - return metric_dict - - -def get_settable_metrics(metric_object): - """Retrieves writable attributes by checking have a setter method""" - metric_dict = asdict(metric_object) - all_attributes = vars(type(metric_object)).items() - # include properties as they are not part of asdict - metric_dict.update({n: p.__get__(metric_object) for n, p in all_attributes if is_writable(p)}) - return metric_dict - - @define(kw_only=True) class Metric(ABC): + """Metric base class""" + name: str = field(validator=validators.instance_of(str)) description: str = field(validator=validators.instance_of(str)) labels: dict = field( @@ -54,9 +27,11 @@ class Metric(ABC): @property def fullname(self): + """returns the fullname""" return f"{self._prefix}{self.name}" def init_tracker(self): + """initializes the tracker and adds it to the trackers dict""" tracker = None try: if isinstance(self, CounterMetric): @@ -94,6 +69,8 @@ def __add__(self, other): @define(kw_only=True) class CounterMetric(Metric): + """Wrapper for prometheus Counter metric""" + trackers: dict = {} def __add__(self, other): @@ -103,6 +80,8 @@ def __add__(self, other): @define(kw_only=True) class HistogramMetric(Metric): + """Wrapper for prometheus Histogram metric""" + trackers: dict = {} def __add__(self, other): @@ -112,17 +91,10 @@ def __add__(self, other): @define(kw_only=True) class GaugeMetric(Metric): + """Wrapper for prometheus Gauge metric""" "" + trackers: dict = {} def __add__(self, other): self.trackers.get(self.fullname).labels(**self.labels).set(other) return self - - -def calculate_new_average(current_average, next_sample, sample_counter): - """Calculate a new average by combining a new sample with a sample counter""" - average_multiple = current_average * sample_counter - extended_average_multiple = average_multiple + next_sample - sample_counter += 1 - new_average = extended_average_multiple / sample_counter - return new_average, sample_counter diff --git a/logprep/processor/amides/processor.py b/logprep/processor/amides/processor.py index 8f7c1f65a..057f8a566 100644 --- a/logprep/processor/amides/processor.py +++ b/logprep/processor/amides/processor.py @@ -85,7 +85,6 @@ from functools import cached_property, lru_cache from multiprocessing import current_process from pathlib import Path -from time import time from typing import List, Tuple from zipfile import ZipFile @@ -93,12 +92,7 @@ from attr import define, field, validators from logprep.abc.processor import Processor -from logprep.metrics.metrics import ( - CounterMetric, - GaugeMetric, - HistogramMetric, - calculate_new_average, -) +from logprep.metrics.metrics import CounterMetric, GaugeMetric, HistogramMetric from logprep.processor.amides.detection import MisuseDetector, RuleAttributor from logprep.processor.amides.normalize import CommandLineNormalizer from logprep.processor.amides.rule import AmidesRule diff --git a/logprep/processor/domain_resolver/processor.py b/logprep/processor/domain_resolver/processor.py index a10a95047..7151f9484 100644 --- a/logprep/processor/domain_resolver/processor.py +++ b/logprep/processor/domain_resolver/processor.py @@ -48,6 +48,7 @@ from tldextract import TLDExtract from logprep.abc.processor import Processor +from logprep.metrics.metrics import CounterMetric from logprep.processor.domain_resolver.rule import DomainResolverRule from logprep.util.cache import Cache from logprep.util.getter import GetterFactory @@ -101,13 +102,33 @@ class Config(Processor.Config): class Metrics(Processor.Metrics): """Tracks statistics about the DomainResolver""" - total_urls: int = 0 + total_urls: CounterMetric = field( + factory=lambda: CounterMetric( + description="Number of all resolved urls", + name="domain_resolver_total_urls", + ) + ) """Number of all resolved urls""" - resolved_new: int = 0 + resolved_new: CounterMetric = field( + factory=lambda: CounterMetric( + description="Number of urls that had to be resolved newly", + name="domain_resolver_resolved_new", + ) + ) """Number of urls that had to be resolved newly""" - resolved_cached: int = 0 + resolved_cached: CounterMetric = field( + factory=lambda: CounterMetric( + description="Number of urls that were resolved from cache", + name="domain_resolver_resolved_cached", + ) + ) """Number of urls that were resolved from cache""" - timeouts: int = 0 + timeouts: CounterMetric = field( + factory=lambda: CounterMetric( + description="Number of timeouts that occurred while resolving a url", + name="domain_resolver_timeouts", + ) + ) """Number of timeouts that occurred while resolving a url""" __slots__ = ["_domain_ip_map"] diff --git a/logprep/processor/pseudonymizer/processor.py b/logprep/processor/pseudonymizer/processor.py index 06e95cd88..54cdc3049 100644 --- a/logprep/processor/pseudonymizer/processor.py +++ b/logprep/processor/pseudonymizer/processor.py @@ -47,6 +47,7 @@ from urlextract import URLExtract from logprep.abc.processor import Processor +from logprep.metrics.metrics import CounterMetric from logprep.processor.pseudonymizer.encrypter import DualPKCS1HybridEncrypter from logprep.processor.pseudonymizer.rule import PseudonymizerRule from logprep.util.cache import Cache @@ -129,7 +130,12 @@ class Config(Processor.Config): class Metrics(Processor.Metrics): """Tracks statistics about the Pseudonymizer""" - pseudonymized_urls: int = 0 + pseudonymized_urls: CounterMetric = field( + factory=lambda: CounterMetric( + description="Number of urls that were pseudonymized", + name="pseudonymizer_pseudonymized_urls", + ) + ) """Number urls that were pseudonymized""" __slots__ = [