diff --git a/docs/BasisModule/Trace/Debug.md b/docs/BasisModule/Trace/Debug.md index 11d4239e3..9006c07fc 100644 --- a/docs/BasisModule/Trace/Debug.md +++ b/docs/BasisModule/Trace/Debug.md @@ -51,4 +51,40 @@ System.setProperty("APPBUILDER_LOGLFILE", "/tmp/appbuilder.log"); ```golang // golang os.Setenv("APPBUILDER_LOGLEVEL", "/tmp/appbuilder.log") -``` \ No newline at end of file +``` + + +## log高阶配置 + +`setLogConfig`功能可以设置日志的输出方式、是否滚动日志、滚动日志参数等。 + +参数说明: +- rolling (bool, optional): 是否启用滚动日志. 默认为True. +- update_interval (int, optional): 更新日志文件的间隔时间. 默认为1. +- update_time (str, optional): 更新日志文件的时间间隔单位. 默认为'midnight'，每日凌晨更新. + - 可选值: + - 'S' - Seconds + - 'M' - Minutes + - 'H' - Hours + - 'D' - Days + - 'W0'-'W6' - Weekday (0=Monday, 6=Sunday) + - 'midnight' - Roll over at midnight +- backup_count (int, optional): 备份日志文件的数量. 默认为无穷大. +- filename (str, optional): 日志文件的名称. 默认为空字符串. + +需在代码中设置 + +```python +appbuilder.logger.setLoglevel("DEBUG") +""" +这里设置: + rolling=Ture - 启用滚动日志(运行此段代码，默认使用滚动日志) + filename="appbuilder.log" - 此优先级最高会覆盖之前的设置，若未传参则使用之前已经设置的日志文件，若之前未设置则使用默认的"tmp.log"日志文件 + update_interval = 1 - 更新日志文件的间隔数量级 + update_time = 'midnight' - 更新日志文件的间隔单位为秒级，每日凌晨滚动日志 +""" +appbuilder.logger.setLogConfig(filename="appbuilder.log",update_interval=1, update_time='midnight') +``` + +## 新增功能 +日志功能会自动开启日志的分离功能，独立创建一个`error.(filename).log'文件，用于存储`WARNING`、`ERROR`级别日志，同时会兼容日志的滚动功能。 \ No newline at end of file diff --git a/python/tests/test_utils_logging_util.py b/python/tests/test_utils_logging_util.py index 0022e25dd..649c73cc5 100644 --- a/python/tests/test_utils_logging_util.py +++ b/python/tests/test_utils_logging_util.py @@ -12,15 +12,24 @@ # See the License for the specific language governing permissions and # limitations under the License. import unittest +import json import os +import copy -from appbuilder.utils.logger_util import LoggerWithLoggerId,LOGGING_CONFIG + +from appbuilder.utils.logger_util import LoggerWithLoggerId, LOGGING_CONFIG @unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") class TestTestUtilsLoggingUtil(unittest.TestCase): def setUp(self): - self.logger = LoggerWithLoggerId(LOGGING_CONFIG["loggers"]["appbuilder"], {}, 'DEBUG') + self.original_logging_config = copy.deepcopy(LOGGING_CONFIG) + print(json.dumps(LOGGING_CONFIG, indent=4, ensure_ascii=False)) + self.logger = LoggerWithLoggerId(self.original_logging_config["loggers"]["appbuilder"], {}, 'DEBUG') + + def tearDown(self): + global LOGGING_CONFIG + LOGGING_CONFIG = copy.deepcopy(self.original_logging_config) def test_set_auto_logid(self): self.logger.set_auto_logid() @@ -31,13 +40,18 @@ def test_set_logid(self): def test_get_logid(self): self.logger.set_auto_logid() - # def test_level(self): - # level=self.logger.level() - def test_process(self): msg,kwargs=self.logger.process(msg='test',kwargs={}) msg,kwargs=self.logger.process(msg='test',kwargs={'extra':{'logid':'test'}}) - + + def test_set_log_config_rolling_false(self): + self.logger.setLogConfig( + rolling=False, + filename='test.log', + update_interval = -1, + update_time='M', + backup_count=-1 + ) if __name__ == '__main__': diff --git a/python/tests/test_utils_new_logging_util.py b/python/tests/test_utils_new_logging_util.py new file mode 100644 index 000000000..4efc08635 --- /dev/null +++ b/python/tests/test_utils_new_logging_util.py @@ -0,0 +1,60 @@ +# Copyright (c) 2023 Baidu, Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +import json +import os +import copy + +from appbuilder.utils.logger_util import LoggerWithLoggerId, LOGGING_CONFIG + + +@unittest.skipUnless(os.getenv("TEST_CASE", "UNKNOWN") == "CPU_PARALLEL", "") +class TestTestUtilsLoggingUtil(unittest.TestCase): + def setUp(self): + self.original_logging_config = copy.deepcopy(LOGGING_CONFIG) + print(json.dumps(LOGGING_CONFIG, indent=4, ensure_ascii=False)) + self.logger = LoggerWithLoggerId(self.original_logging_config["loggers"]["appbuilder"], {}, 'DEBUG') + + def tearDown(self): + global LOGGING_CONFIG + LOGGING_CONFIG = copy.deepcopy(self.original_logging_config) + + + def test_set_log_config_01(self): + self.logger.setLogConfig( + update_interval = -1, + update_time='M', + backup_count=-1 + ) + + def test_set_log_config_02(self): + self.logger.setLogConfig( + filename='test.log', + update_interval = -1, + update_time='M', + backup_count=-1 + ) + + def test_set_log_config_03(self): + with self.assertRaises(ValueError): + self.logger.setLogConfig( + update_interval = -1, + update_time='Test', + backup_count=-1 + ) + + +if __name__ == '__main__': + unittest.main() + \ No newline at end of file diff --git a/python/utils/logger_util.py b/python/utils/logger_util.py index 3396567d4..c0919d528 100644 --- a/python/utils/logger_util.py +++ b/python/utils/logger_util.py @@ -17,9 +17,11 @@ 日志 """ import uuid +import json import os import sys import logging.config +from logging.handlers import RotatingFileHandler, TimedRotatingFileHandler from threading import current_thread @@ -37,13 +39,7 @@ "class": "logging.StreamHandler", "formatter": "standard", "stream": "ext://sys.stdout", # Use standard output - }, - "file": { - "level": "INFO", - "class": "logging.FileHandler", - "filename": "tmp.log", - "formatter": "standard", - }, + } }, "loggers": { "appbuilder": { @@ -55,19 +51,91 @@ } +TIME_HANDLERS_FILE = { + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'formatter': 'standard', + 'level': 'INFO', + 'filename': "", + 'when': 'midnight', # 可选项: 'S', 'M', 'H', 'D', 'W0'-'W6', 'midnight' + 'interval': 1, # 每1天滚动一次 + 'backupCount': 5, # 保留5个备份 + 'encoding': 'utf-8', +} + +TIME_HANDLERS_FILE_ERROR = { + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'formatter': 'standard', + 'level': 'ERROR', + 'filename': "", + 'when': 'midnight', # 可选项: 'S', 'M', 'H', 'D', 'W0'-'W6', 'midnight' + 'interval': 1, # 每1天滚动一次 + 'backupCount': 5, # 保留5个备份 + 'encoding': 'utf-8', +} + +SIMPLE_HANDLERS_FILE = { + "level": "INFO", + "class": "logging.FileHandler", + "filename": "", + "formatter": "standard", +} + +SIMPLE_HANDLERS_FILE_ERROR = { + "level": "ERROR", + "class": "logging.FileHandler", + "filename": "", + "formatter": "standard", +} + +def _update_error_file_name(filename:str): + """ + 更新文件名，使其以 "error." 开头。 + + Args: + filename (str): 原始文件名。 + + Returns: + str: 更新后的文件名。 + + Raises: + 无 + + """ + filenames = filename.split('/') + filenames[-1] = f"error.{filenames[-1]}" + return '/'.join(filenames) + class LoggerWithLoggerId(logging.LoggerAdapter): """ logger with logid """ def __init__(self, logger, extra, loglevel): """ - init + 初始化LoggerAdapter实例。 + + Args: + logger (logging.Logger): 日志记录器实例。 + extra (dict): 用于日志记录的额外上下文信息。 + loglevel (str): 日志级别，如'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'。 + + Returns: + None + """ log_file = os.environ.get("APPBUILDER_LOGFILE", "") if log_file: - LOGGING_CONFIG["handlers"]["file"]["filename"] = log_file - LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") - LOGGING_CONFIG["handlers"]["file"]["level"] = loglevel + LOGGING_CONFIG["loggers"]["appbuilder"]["handler"] = ["console"] # 默认使用console + + SIMPLE_HANDLERS_FILE["level"] = loglevel + TIME_HANDLERS_FILE['level'] = loglevel + SIMPLE_HANDLERS_FILE["filename"] = log_file + SIMPLE_HANDLERS_FILE_ERROR["filename"] = _update_error_file_name(log_file) + LOGGING_CONFIG["handlers"]["file"] = SIMPLE_HANDLERS_FILE + LOGGING_CONFIG["handlers"]["error_file"] = SIMPLE_HANDLERS_FILE_ERROR + if "file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") + if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") LOGGING_CONFIG['handlers']['console']['level'] = loglevel LOGGING_CONFIG['loggers']['appbuilder']['level'] = loglevel logging.config.dictConfig(LOGGING_CONFIG) @@ -102,13 +170,128 @@ def level(self): """ return self.logger.level + def setLogConfig( + self, + rolling:bool=True, + update_interval:int=1, + update_time:str='midnight', + backup_count:int=0, + filename:str='' + ): + """ + 设置日志配置。 + + Args: + rolling (bool): 是否开启日志滚动。默认为True。 + update_interval (int): 日志滚动更新的时间间隔。默认为1。 + update_time (str): 日志滚动更新的时间单位。默认为'midnight'。 + backup_count (int): 日志备份数量。默认为0。 + filename (str): 日志文件名。默认为空字符串。 + + Returns: + None + + Raises: + ValueError: 如果update_time参数的值不在预期的范围内。 + + """ + # 配置控制台输出 + if "file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") + if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + + # 确定备份数量 + if backup_count <= 0 or not isinstance(backup_count, int): + backup_count = sys.maxsize # 默认为无穷大 + + # 确定滚动时间 + if update_interval < 1: + update_interval = 1 + if update_time: + update_time = update_time.lower() + if update_time not in ['s', 'm', 'h', 'd', 'midnight'] and not (update_time.startswith('w') and update_time[1:].isdigit() and 0 <= int(update_time[1:]) <= 6): + raise ValueError("expected update_time in [S, M, H, D, midnight, WX], where X is between 0~6, but got %s") + else: + update_time = update_time.upper() + + # 设置filename + if not filename: + + filename = (SIMPLE_HANDLERS_FILE.get("filename") or + TIME_HANDLERS_FILE.get("filename") or + "tmp.log") + + + # 创建处理器 + if rolling: + if update_time and update_interval: + TIME_HANDLERS_FILE['when'] = update_time + TIME_HANDLERS_FILE_ERROR['when'] = update_time + TIME_HANDLERS_FILE['interval'] = update_interval + TIME_HANDLERS_FILE_ERROR['interval'] = update_interval + TIME_HANDLERS_FILE['backupCount'] = backup_count + TIME_HANDLERS_FILE_ERROR['backupCount'] = backup_count + TIME_HANDLERS_FILE['filename'] = filename + TIME_HANDLERS_FILE_ERROR['filename'] = _update_error_file_name(filename) + if 'file' in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("file") + if 'error_file' in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("error_file") + if not "timed_file" in LOGGING_CONFIG["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append('timed_file') + if not "error_timed_file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append('error_timed_file') + LOGGING_CONFIG["handlers"]["timed_file"] = TIME_HANDLERS_FILE + LOGGING_CONFIG["handlers"]["error_timed_file"] = TIME_HANDLERS_FILE_ERROR + LOGGING_CONFIG["handlers"]["timed_file"]["level"] = LOGGING_CONFIG['loggers']['appbuilder']['level'] + else: + SIMPLE_HANDLERS_FILE["filename"] = filename + SIMPLE_HANDLERS_FILE_ERROR["filename"] = _update_error_file_name(filename) + LOGGING_CONFIG["handlers"]["file"] = SIMPLE_HANDLERS_FILE + LOGGING_CONFIG["handlers"]["error_file"] = SIMPLE_HANDLERS_FILE_ERROR + if "timed_file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("timed_file") + if "error_timed_file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("error_timed_file") + if "file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") + if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + LOGGING_CONFIG["handlers"]["file"]["level"] = LOGGING_CONFIG['loggers']['appbuilder']['level'] + + logging.config.dictConfig(LOGGING_CONFIG) + def setFilename(self, filename): """ - set filename + 设置日志文件和错误日志文件的文件名，并取消日志文件的滚动功能。 + + Args: + filename (str): 要设置的日志文件和错误日志文件的文件名。 + + Returns: + None + + 说明: + 1. 如果LOGGING_CONFIG配置中存在"timed_file"和"error_timed_file"处理器，则从"appbuilder"日志记录器的处理器列表中移除它们。 + 2. 如果"file"和"error_file"处理器不在"appbuilder"日志记录器的处理器列表中，则将它们添加到列表中。 + 3. 更新SIMPLE_HANDLERS_FILE和SIMPLE_HANDLERS_FILE_ERROR配置中的"filename"为传入的文件名。 + 4. 使用_update_error_file_name函数更新错误日志文件的名称。 + 5. 更新LOGGING_CONFIG配置中的"handlers"下的"file"和"error_file"处理器配置。 + 6. 调用logging.config.dictConfig函数重新配置日志系统。 """ + if "timed_file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("timed_file") + if "error_timed_file" in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].remove("error_timed_file") if "file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("file") - LOGGING_CONFIG["handlers"]["file"]["filename"] = filename + if "error_file" not in LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"]: + LOGGING_CONFIG["loggers"]["appbuilder"]["handlers"].append("error_file") + SIMPLE_HANDLERS_FILE["filename"] = filename + SIMPLE_HANDLERS_FILE_ERROR["filename"] = _update_error_file_name(filename) + LOGGING_CONFIG["handlers"]["file"] = SIMPLE_HANDLERS_FILE + LOGGING_CONFIG["handlers"]["error_file"] = SIMPLE_HANDLERS_FILE_ERROR logging.config.dictConfig(LOGGING_CONFIG) def setLoglevel(self, level): @@ -122,7 +305,8 @@ def setLoglevel(self, level): log_level = log_level.upper() LOGGING_CONFIG['handlers']['console']['level'] = log_level LOGGING_CONFIG['loggers']['appbuilder']['level'] = log_level - LOGGING_CONFIG["handlers"]["file"]["level"] = log_level + SIMPLE_HANDLERS_FILE["level"] = log_level + TIME_HANDLERS_FILE['level'] = log_level logging.config.dictConfig(LOGGING_CONFIG) def process(self, msg, kwargs):