diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0267270..f628bd4 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -14,7 +14,7 @@ "forwardPorts": [ 3000 ], - "postCreateCommand": "pip3 install --user -r /workspaces/loki-logger-handler/requirements.txt", + "postCreateCommand": "pip3 install --user -r /workspaces/loki-logger-handler/requirements.txt && pip3 install pytest", "customizations": { "vscode": { "extensions": [ diff --git a/README.md b/README.md index d9451c7..248176e 100644 --- a/README.md +++ b/README.md @@ -27,8 +27,9 @@ A logging handler that sends log messages to **(Grafana) Loki** in JSON format. * default_formatter (logging.Formatter, optional): Formatter for the log records. If not provided,`LoggerFormatter` or `LoguruFormatter` will be used. * enable_self_errors (bool, optional): Set to True to show Hanlder errors on console. Defaults to False ### Loki 3.0 -* enable_structured_metadata (bool, optional): Whether to include structured metadata in the logs. Defaults to False. Only supported for Loki 3.0 and above -* global_metadata (type, optional): Description of global_metadata. Defaults to None. Only supported for Loki 3.0 and above +* enable_structured_loki_metadata (bool, optional): Whether to include structured loki_metadata in the logs. Defaults to False. Only supported for Loki 3.0 and above +* loki_metadata (dict, optional): Default loki_metadata values. Defaults to None. Only supported for Loki 3.0 and above +* loki_metadata_keys (arrray, optional): Specific log record keys to extract as loki_metadata. Only supported for Loki 3.0 and above ## Formatters * **LoggerFormatter**: Formatter for default python logging implementation @@ -162,7 +163,7 @@ We can add metadata in 3 ways: 1. Defile static loki_metadata that will be injected into all logs lines 2. Use logger extra options adding metadata inside `loki_metadata` key -3. +3. Use logger `loki_metadata_keys` to move logs keys to loki metadata. ### Example global metadata @@ -181,7 +182,7 @@ custom_handler = LokiLoggerHandler( labels={"application": "Test", "environment": "Develop"}, label_keys={}, timeout=10, - enable_structured_metadata=True, + enable_structured_loki_metadata=True, loki_metadata={"service": "user-service", "version": "1.0.0"} ) @@ -189,7 +190,7 @@ logger.addHandler(custom_handler) ``` -In this example, the `loki_metadata` dictionary includes metadata that will be attached to every log message. The `enable_structured_metadata` flag ensures that this metadata is included in the logs. +In this example, the `loki_metadata` dictionary includes metadata that will be attached to every log message. The `enable_structured_loki_metadata` flag ensures that this metadata is included in the logs. ### Example log metadata @@ -209,7 +210,7 @@ custom_handler = LokiLoggerHandler( labels={"application": "Test", "environment": "Develop"}, label_keys={}, timeout=10, - enable_structured_metadata=True, + enable_structured_loki_metadata=True, loki_metadata={"service": "user-service", "version": "1.0.0"} ) @@ -234,7 +235,7 @@ custom_handler = LokiLoggerHandler( labels={"application": "Test", "environment": "Develop"}, label_keys={}, timeout=10, - enable_structured_metadata=True, + enable_structured_loki_metadata=True, loki_metadata={"service": "user-service", "version": "1.0.0"}, loki_metadata_keys=["trace_id"] ) diff --git a/loki_logger_handler/formatters/logger_formatter.py b/loki_logger_handler/formatters/logger_formatter.py index 640fce4..5f668f7 100644 --- a/loki_logger_handler/formatters/logger_formatter.py +++ b/loki_logger_handler/formatters/logger_formatter.py @@ -1,5 +1,5 @@ import traceback - +import time class LoggerFormatter: """ @@ -41,6 +41,7 @@ def format(self, record): Returns: dict: A dictionary representation of the log record. """ + formatted = { "message": record.getMessage(), "timestamp": record.created, @@ -53,8 +54,11 @@ def format(self, record): } # Capture any custom fields added to the log record - record_keys = set(record.__dict__.keys()) - custom_fields = record_keys - self.LOG_RECORD_FIELDS + custom_fields = { + key: value for key, value in record.__dict__.items() + if key not in self.LOG_RECORD_FIELDS + } + loki_metadata = {} for key in custom_fields: diff --git a/loki_logger_handler/loki_request.py b/loki_logger_handler/loki_request.py index 3732715..e386677 100644 --- a/loki_logger_handler/loki_request.py +++ b/loki_logger_handler/loki_request.py @@ -1,11 +1,6 @@ -import requests import gzip -import sys +import requests -try: - from io import BytesIO as IO # For Python 3 -except ImportError: - from StringIO import StringIO as IO # For Python 2 class LokiRequest: @@ -49,14 +44,11 @@ def send(self, data): try: if self.compressed: self.headers["Content-Encoding"] = "gzip" - buf = IO() - with gzip.GzipFile(fileobj=buf, mode="wb") as f: - f.write(data.encode("utf-8")) - data = buf.getvalue() - + data = gzip.compress(data.encode("utf-8")) + response = self.session.post(self.url, data=data, headers=self.headers) response.raise_for_status() - + except requests.RequestException as e: if response is not None: diff --git a/loki_logger_handler/stream.py b/loki_logger_handler/stream.py index b950bba..c939908 100644 --- a/loki_logger_handler/stream.py +++ b/loki_logger_handler/stream.py @@ -1,5 +1,5 @@ -import json import time +import json # Compatibility for Python 2 and 3 try: @@ -69,7 +69,7 @@ def append_value(self, value, metadata=None): except (TypeError, ValueError): # Fallback to the current time in nanoseconds if the timestamp is missing or invalid timestamp = str(time_ns()) - + formatted_value = json.dumps(value, ensure_ascii=False) if self.message_in_json_format else value if metadata or self.loki_metadata: # Ensure both metadata and self.loki_metadata are dictionaries (default to empty dict if None) diff --git a/loki_logger_handler/streams.py b/loki_logger_handler/streams.py index 7c28ebc..5c93a8d 100644 --- a/loki_logger_handler/streams.py +++ b/loki_logger_handler/streams.py @@ -7,12 +7,12 @@ class _LokiRequestEncoder(json.JSONEncoder): This internal class is used to handle the serialization of Streams objects into the JSON format expected by Loki. """ - def default(self, obj): - if isinstance(obj, Streams): + def default(self, o): + if isinstance(o, Streams): # Convert the Streams object to a dictionary format suitable for Loki - return {"streams": [stream.__dict__ for stream in obj.streams]} + return {"streams": [stream.__dict__ for stream in o.streams]} # Use the default serialization method for other objects - return super(_LokiRequestEncoder, self).default(obj) + return super(_LokiRequestEncoder, self).default(o) class Streams(object): # Explicitly inherit from object for Python 2 compatibility diff --git a/setup.py b/setup.py index c7968dd..181edd1 100644 --- a/setup.py +++ b/setup.py @@ -6,7 +6,7 @@ setup( name="loki-logger-handler", - version="1.0.2", + version="1.1.0", author="Xente", description="Handler designed for transmitting logs to Grafana Loki in JSON format.", long_description=long_description,