diff --git a/pyproject.toml b/pyproject.toml index 54a791e..21f2c46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,10 @@ dependencies = [ "pydantic~=2.1", "pyyaml~=6.0", "opentelemetry-api~=1.21", + "opentelemetry-sdk~=1.21", + "opentelemetry-exporter-otlp~=1.21", "opentelemetry-semantic-conventions~=0.42b0", + "opentelemetry-instrumentation-logging~=0.42b0", ] [project.urls] diff --git a/src/etos_lib/logging/log_processors.py b/src/etos_lib/logging/log_processors.py new file mode 100644 index 0000000..6ec607e --- /dev/null +++ b/src/etos_lib/logging/log_processors.py @@ -0,0 +1,35 @@ +# Copyright Axis Communications AB. +# +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Custom log processors for use with Open Telemetry logging signals.""" + +from opentelemetry.sdk._logs import LogData, LogRecordProcessor + + +class ToStringProcessor(LogRecordProcessor): + """Simple log record processor to convert all log records to type string.""" + + def emit(self, log_data: LogData) -> None: + """Change record body to string and emit the `LogData`.""" + record = log_data.log_record + if not isinstance(record.body, (str, bool, int, float)): + record.body = str(record.body) + + def force_flush(self, _timeout_millis: int = 30000) -> bool: + """Export all the received, but not yet exported, logs to the configured Exporter.""" + return True + + def shutdown(self) -> None: + """Logger shutdown procedures.""" diff --git a/src/etos_lib/logging/logger.py b/src/etos_lib/logging/logger.py index fe50637..9d37b46 100644 --- a/src/etos_lib/logging/logger.py +++ b/src/etos_lib/logging/logger.py @@ -28,19 +28,27 @@ >>> [2020-12-16 10:35:00][cb7c8cd9-40a6-4ecc-8321-a1eae6beae35] INFO: Hello! """ -import sys import atexit -from pathlib import Path -import threading import logging import logging.config -from yaml import load, SafeLoader +import sys +import threading +from pathlib import Path + +from opentelemetry.exporter.otlp.proto.grpc._log_exporter import OTLPLogExporter +from opentelemetry.instrumentation.logging import LoggingInstrumentor +from opentelemetry.sdk._logs import LoggerProvider, LoggingHandler +from opentelemetry.sdk._logs.export import BatchLogRecordProcessor +from opentelemetry.sdk.resources import Resource +from yaml import SafeLoader, load + +from etos_lib.lib.config import Config +from etos_lib.lib.debug import Debug from etos_lib.logging.filter import EtosFilter from etos_lib.logging.formatter import EtosLogFormatter -from etos_lib.logging.rabbitmq_handler import RabbitMQHandler +from etos_lib.logging.log_processors import ToStringProcessor from etos_lib.logging.log_publisher import RabbitMQLogPublisher -from etos_lib.lib.config import Config -from etos_lib.lib.debug import Debug +from etos_lib.logging.rabbitmq_handler import RabbitMQHandler DEFAULT_CONFIG = Path(__file__).parent.joinpath("default_config.yaml") DEFAULT_LOG_PATH = Debug().default_log_path @@ -48,7 +56,7 @@ FORMAT_CONFIG = threading.local() -def setup_file_logging(config, log_filter): +def setup_file_logging(config: dict, log_filter: EtosFilter) -> None: """Set up logging to file using the ETOS log formatter. Cofiguration file parameters ('file' must exist or no file handler is set up): @@ -78,7 +86,9 @@ def setup_file_logging(config, log_filter): root_logger = logging.getLogger() file_handler = logging.handlers.RotatingFileHandler( - logfile, maxBytes=max_bytes, backupCount=max_files + logfile, + maxBytes=max_bytes, + backupCount=max_files, ) file_handler.setFormatter(EtosLogFormatter()) file_handler.setLevel(loglevel) @@ -86,7 +96,7 @@ def setup_file_logging(config, log_filter): root_logger.addHandler(file_handler) -def setup_stream_logging(config, log_filter): +def setup_stream_logging(config: dict, log_filter: EtosFilter) -> None: """Set up logging to stdout stream. Cofiguration file parameters ('stream' must exist or no stream handler is set up): @@ -109,7 +119,8 @@ def setup_stream_logging(config, log_filter): loglevel = getattr(logging, config.get("loglevel", "INFO")) logformat = config.get( - "logformat", "[%(asctime)s][%(identifier)s] %(levelname)s:%(name)s: %(message)s" + "logformat", + "[%(asctime)s][%(identifier)s] %(levelname)s:%(name)s: %(message)s", ) dateformat = config.get("dateformat", "%Y-%m-%d %H:%M:%S") root_logger = logging.getLogger() @@ -120,7 +131,7 @@ def setup_stream_logging(config, log_filter): root_logger.addHandler(stream_handler) -def setup_rabbitmq_logging(log_filter): +def setup_rabbitmq_logging(log_filter: EtosFilter) -> None: """Set up rabbitmq logging. :param log_filter: Logfilter to add to stream handler. @@ -150,7 +161,38 @@ def setup_rabbitmq_logging(log_filter): root_logger.addHandler(rabbit_handler) -def setup_logging(application, version, environment, config_file=DEFAULT_CONFIG): +def setup_otel_logging( + log_filter: EtosFilter, + resource: Resource, + log_level: int = logging.INFO, +) -> None: + """Set up OpenTelemetry logging signals. + + :param log_filter: Logfilter to add to OpenTelemetry handler. + :param resource: OpenTelemetry Resource to use when instrumenting logs + :param log_level: Log level to set in the OpenTelemetry log handler + """ + logger_provider = LoggerProvider(resource) + logger_provider.add_log_record_processor(ToStringProcessor()) + logger_provider.add_log_record_processor(BatchLogRecordProcessor(OTLPLogExporter())) + otel_log_handler = LoggingHandler(logger_provider=logger_provider) + + otel_log_handler.setFormatter(EtosLogFormatter()) + otel_log_handler.addFilter(log_filter) + otel_log_handler.setLevel(log_level) + + logging.getLogger().addHandler(otel_log_handler) + + LoggingInstrumentor().instrument(set_logging_format=False) + + +def setup_logging( + application: str, + version: str, + environment: str, + otel_resource: Resource = None, + config_file: Path = DEFAULT_CONFIG, +) -> None: """Set up basic logging. :param application: Name of application to setup logging for. @@ -182,9 +224,11 @@ def setup_logging(application, version, environment, config_file=DEFAULT_CONFIG) if logging_config.get("file"): setup_file_logging(logging_config.get("file"), log_filter) setup_rabbitmq_logging(log_filter) + if otel_resource: + setup_otel_logging(log_filter, otel_resource) -def close_rabbit(rabbit): +def close_rabbit(rabbit: RabbitMQLogPublisher) -> None: """Close down a rabbitmq connection.""" rabbit.wait_for_unpublished_events() rabbit.close()