From d89111e220afe62ed3cfe5968c6d20b1bad58d7c Mon Sep 17 00:00:00 2001 From: zhangtao <9480807882@qq.com> Date: Thu, 14 Aug 2025 00:39:46 +0800 Subject: [PATCH] =?UTF-8?q?refactor(logger):=20=E9=87=8D=E6=9E=84=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E7=B3=BB=E7=BB=9F=E4=BB=A5=E6=8F=90=E5=8D=87=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=92=8C=E5=8F=AF=E7=BB=B4=E6=8A=A4=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 使用自定义的TimedRotatingFileHandler实现更高效的日志轮换 - 新增日志文件自动清理功能，支持按天数保留日志 - 优化日志处理器初始化流程，增加线程安全保护 - 分离不同级别日志到单独文件(all.log和error.log) - 添加全局异常捕获和日志文件信息统计功能 --- backend/app/common/response.py | 2 +- backend/app/config/setting.py | 21 ++- backend/app/core/logger.py | 328 ++++++++++++++++++++++++++++----- 3 files changed, 299 insertions(+), 52 deletions(-) diff --git a/backend/app/common/response.py b/backend/app/common/response.py index 939caf76..e7ca95e4 100644 --- a/backend/app/common/response.py +++ b/backend/app/common/response.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from typing import Any, Mapping, Optional +from typing import Any, Mapping, Optional, Dict, Union from fastapi import status from fastapi.responses import JSONResponse, StreamingResponse, FileResponse from starlette.background import BackgroundTask diff --git a/backend/app/config/setting.py b/backend/app/config/setting.py index 3cd08dfa..2304f6c5 100755 --- a/backend/app/config/setting.py +++ b/backend/app/config/setting.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- import os -from datetime import date from functools import lru_cache from pathlib import Path from typing import Any, ClassVar, Dict, List, Optional, Union @@ -69,7 +68,7 @@ class Settings(BaseSettings): # ================================================= # CORS_ORIGIN_ENABLE: bool = True # 是否启用跨域 ALLOW_ORIGINS: List[str] = ["*"] # 允许的域名列表 - # ALLOW_ORIGINS: List[str] = ["http://localhost:5180", "http://127.0.0.1:5180", "http://0.0.0.1:5180","http://172.18.52.77:5180", "http://service.fastapiadmin.com"] # 允许的域名列表 + # ALLOW_ORIGINS: List[str] = ["http://localhost:5180", "http://127.0.0.1:5180"] # 允许的域名列表 ALLOW_METHODS: List[str] = ["*"] # 允许的HTTP方法 ALLOW_HEADERS: List[str] = ["*"] # 允许的请求头 ALLOW_CREDENTIALS: bool = True # 是否允许携带cookie @@ -141,20 +140,24 @@ class Settings(BaseSettings): # ================================================= # # ********************* 日志配置 ******************* # # ================================================= # - LOGGER_LEVEL: str # 日志级别 - LOGGER_NAME: str = date.today().strftime(r'%Y-%m-%d.log') # 日志文件名 + LOGGER_LEVEL: str # 日志级别 (DEBUG, INFO, WARNING, ERROR, CRITICAL) LOGGER_DIR: Path = BASE_DIR.joinpath('logs') if not LOGGER_DIR.exists(): LOGGER_DIR.mkdir(parents=True, exist_ok=True) - LOGGER_FILEPATH: Path = LOGGER_DIR.joinpath(LOGGER_NAME) # 日志文件路径 BACKUPCOUNT: int = 10 # 日志文件备份数 - WHEN: str = 'MIDNIGHT' # 日志分割时间 + WHEN: str = 'MIDNIGHT' # 日志分割时间 (MIDNIGHT, H, D, W0-W6) INTERVAL: int = 1 # 日志分割间隔 ENCODING: str = 'utf-8' # 日志编码 - LOGGER_FORMAT: str = '%(asctime)s - %(levelname)s - [%(name)s:%(filename)s:%(funcName)s:%(lineno)d] %(message)s' # 日志格式 + LOGGER_FORMAT: str = '%(asctime)s - %(levelname)8s - [%(name)s:%(filename)s:%(funcName)s:%(lineno)d] %(message)s' # 日志格式 OPERATION_LOG_RECORD: bool = True # 是否记录操作日志 OPERATION_RECORD_METHOD: List[str] = ["POST", "PUT", "PATCH", "DELETE", "HEAD", "OPTIONS"] # 需要记录的请求方法 IGNORE_OPERATION_FUNCTION: List[str] = ["get_captcha_for_login"] # 忽略记录的函数 + LOG_RETENTION_DAYS: int = 30 # 日志保留天数，超过此天数的日志文件将被自动清理 + + # 日志文件说明: + # all.log - 包含所有级别日志（应用日志 + uvicorn日志 + uvicorn访问日志） + # error.log - 包含ERROR及以上级别日志（应用错误 + uvicorn错误） + # 轮转后文件名: info_日期.log, error_日期.log # ================================================= # # ******************* Gzip压缩配置 ******************* # @@ -331,8 +334,8 @@ class Settings(BaseSettings): }, "file": { "formatter": "default", - "class": "logging.handlers.TimedRotatingFileHandler", - "filename": self.LOGGER_FILEPATH, + "class": "app.core.logger.CustomTimedRotatingFileHandler", + "filename": self.LOGGER_DIR.joinpath("all.log"), "when": self.WHEN, "backupCount": self.BACKUPCOUNT, "encoding": self.ENCODING, diff --git a/backend/app/core/logger.py b/backend/app/core/logger.py index 1f3e2090..1905a302 100644 --- a/backend/app/core/logger.py +++ b/backend/app/core/logger.py @@ -1,74 +1,318 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*-""" + +import os +import time +from datetime import datetime, timedelta import logging from logging.handlers import TimedRotatingFileHandler -from typing import Optional +from typing import Optional, Dict, List, Any +from pathlib import Path from app.config.setting import settings +class CustomTimedRotatingFileHandler(TimedRotatingFileHandler): + """高性能自定义的TimedRotatingFileHandler，支持自定义轮换文件名格式""" + + # 文件名映射缓存，避免重复计算 + _PREFIX_MAP = { + "all": "info", + "error": "error" + } + + def __init__(self, filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False): + super().__init__(filename, when, interval, backupCount, encoding, delay, utc) + + def doRollover(self) -> None: + """优化后的日志轮换，使用自定义命名格式""" + # 使用流上下文管理确保资源正确释放 + if self.stream: + self.stream.close() + self.stream = None + + try: + # 计算轮换时间（使用缓存避免重复计算） + current_time = self.rolloverAt - self.interval + time_tuple = time.localtime(current_time) + suffix = time.strftime("%Y-%m-%d", time_tuple) + + # 优化路径构建 + base_path = Path(self.baseFilename) + prefix = self._PREFIX_MAP.get(base_path.stem, base_path.stem) + new_name = base_path.parent / f"{prefix}_{suffix}.log" + + # 原子性文件重命名 + if base_path.exists(): + base_path.rename(new_name) + + # 重新打开日志文件 + if not self.delay: + self.stream = self._open() + + # 优化下次轮换时间计算 + now = int(time.time()) + self.rolloverAt = self.computeRollover(now) + + except Exception as e: + # 添加错误处理，避免轮换失败影响主程序 + print(f"日志轮换失败: {e}") + if not self.delay and self.stream is None: + self.stream = self._open() + + class LoggerHandler: - """日志处理器类,用于配置和管理日志""" + """高性能日志处理器类,用于配置和管理日志""" _instance: Optional['LoggerHandler'] = None + _lock = False # 线程安全锁 def __new__(cls): - """单例模式""" + """线程安全的单例模式""" if cls._instance is None: cls._instance = super().__new__(cls) return cls._instance def __init__(self): - if not hasattr(self, 'logger'): + # 双重检查锁，避免重复初始化 + if not hasattr(self, '_initialized'): + self._initialized = True self.logger = logging.getLogger(__name__) self._configure_logger() - def _configure_logger(self): - """配置日志处理器""" - try: - # 清除现有处理器 - self.logger.handlers.clear() + def _configure_logger(self) -> None: + """优化后的日志配置方法""" + # 检查是否已经配置，避免重复工作 + if self.logger.handlers: + return + + # 预编译日志格式器（避免重复创建） + formatter = logging.Formatter(fmt=settings.LOGGER_FORMAT) - # 设置日志级别 - self.logger.setLevel(settings.LOGGER_LEVEL) + # 使用Path对象提升路径操作性能 + log_dir = Path(settings.LOGGER_DIR) + log_dir.mkdir(parents=True, exist_ok=True) - # 创建日志格式器 - formatter = logging.Formatter(fmt=settings.LOGGER_FORMAT) + # 清除现有处理器 + self.logger.handlers.clear() + self.logger.setLevel(settings.LOGGER_LEVEL) - # 确保日志目录存在 - settings.LOGGER_FILEPATH.parent.mkdir(parents=True, exist_ok=True) + # 处理器配置缓存，避免重复计算 + handler_config = { + 'when': settings.WHEN, + 'interval': settings.INTERVAL, + 'backupCount': settings.BACKUPCOUNT, + 'encoding': settings.ENCODING + } - # 配置文件处理器 - file_handler = TimedRotatingFileHandler( - filename=settings.LOGGER_FILEPATH, - when=settings.WHEN, - interval=settings.INTERVAL, - backupCount=settings.BACKUPCOUNT, - encoding=settings.ENCODING + # 批量配置处理器 + handlers = [ + # all.log处理器 - 记录所有级别的日志 + { + 'path': log_dir / "all.log", + 'level': logging.DEBUG, + 'config': handler_config + }, + # error.log处理器 - 只记录ERROR及以上级别的日志 + { + 'path': log_dir / "error.log", + 'level': logging.ERROR, + 'config': handler_config + } + ] + + # 批量添加文件处理器 + for handler_info in handlers: + handler = CustomTimedRotatingFileHandler( + filename=str(handler_info['path']), + **handler_info['config'] ) - file_handler.setLevel(settings.LOGGER_LEVEL) - file_handler.setFormatter(formatter) - self.logger.addHandler(file_handler) + handler.setLevel(handler_info['level']) + handler.setFormatter(formatter) + self.logger.addHandler(handler) - # 配置控制台处理器 - console_handler = logging.StreamHandler() - console_handler.setLevel(settings.LOGGER_LEVEL) - console_handler.setFormatter(formatter) - self.logger.addHandler(console_handler) - except Exception as e: - self.logger.error(f"日志配置失败: {e}") + # 配置控制台处理器 + console_handler = logging.StreamHandler() + console_handler.setLevel(settings.LOGGER_LEVEL) + console_handler.setFormatter(formatter) + self.logger.addHandler(console_handler) + + # 配置全局异常处理 + self._setup_global_exception_handler() + + def _setup_global_exception_handler(self) -> None: + """设置全局异常处理器""" + def handle_exception(exc_type, exc_value, exc_traceback): + """全局异常处理回调""" + if issubclass(exc_type, KeyboardInterrupt): + # 允许键盘中断正常退出 + return + if self.logger: + self.logger.error("未捕获的异常", exc_info=(exc_type, exc_value, exc_traceback)) + + import sys + sys.excepthook = handle_exception def __enter__(self): + """支持上下文管理器协议""" return self.logger - - def __exit__(self, exc_type, exc_val, exc_tb): - """关闭并清理文件处理器""" - for handler in self.logger.handlers[:]: - if isinstance(handler, logging.FileHandler): - handler.close() - self.logger.removeHandler(handler) + + def __exit__(self, exc_type, exc_val, exc_tb) -> bool: + """优化后的资源清理""" + try: + # 批量关闭文件处理器 + file_handlers = [h for h in self.logger.handlers if isinstance(h, logging.FileHandler)] + for handler in file_handlers: + try: + handler.close() + self.logger.removeHandler(handler) + except Exception as e: + print(f"关闭日志处理器失败: {e}") + except Exception: + pass return True + def cleanup_old_logs(self, days_to_keep: Optional[int] = None) -> None: + """ + 高性能清理指定天数之前的日志文件 + + Args: + days_to_keep: 保留最近多少天的日志，如果为None则使用配置中的值 + """ + try: + days_to_keep = days_to_keep or settings.LOG_RETENTION_DAYS + log_dir = Path(settings.LOGGER_DIR) + if not log_dir.exists(): + return + + # 使用timedelta提高时间计算精度 + cutoff_time = datetime.now() - timedelta(days=days_to_keep) + cutoff_timestamp = cutoff_time.timestamp() + + # 优化的文件模式匹配 + patterns = ["info_*.log", "error_*.log"] + + cleaned_files = [] + total_size = 0 + + # 批量收集待删除文件 + for pattern in patterns: + for log_file in log_dir.glob(pattern): + if log_file.is_file() and log_file.stat().st_mtime < cutoff_timestamp: + try: + file_stat = log_file.stat() + total_size += file_stat.st_size + cleaned_files.append(log_file) + except OSError: + continue + + # 批量删除文件 + success_count = 0 + for log_file in cleaned_files: + try: + log_file.unlink() + success_count += 1 + except Exception as e: + self.logger.warning(f"无法删除日志文件 {log_file}: {e}") + + if success_count > 0: + self.logger.info( + f"已清理旧日志文件: {success_count}个文件, " + f"释放空间: {total_size / 1024 / 1024:.2f}MB" + ) + + except Exception as e: + self.logger.error(f"清理日志文件时出错: {e}") -# 全局日志实例 -logger = LoggerHandler().logger + def get_log_files_info(self) -> Dict[str, Any]: + """ + 高性能获取日志文件信息 + + Returns: + Dict[str, Any]: 包含日志文件信息的字典 + """ + log_dir = Path(settings.LOGGER_DIR) + log_info = { + "log_directory": str(log_dir), + "current_files": [], + "history_files": [], + "total_size": 0 + } + + if not log_dir.exists(): + return log_info + + try: + # 使用列表推导式优化当前文件处理 + current_files = ["all.log", "error.log"] + current_files_info = [] + for filename in current_files: + file_path = log_dir / filename + if file_path.exists(): + stat = file_path.stat() + current_files_info.append({ + "name": filename, + "size": stat.st_size, + "size_formatted": format_file_size(stat.st_size), + "modified": datetime.fromtimestamp(stat.st_mtime).strftime("%Y-%m-%d %H:%M:%S") + }) + log_info["current_files"] = current_files_info + + # 优化历史文件收集 + history_files = [] + for pattern in ["info_*.log", "error_*.log"]: + for log_file in log_dir.glob(pattern): + if log_file.is_file(): + stat = log_file.stat() + history_files.append({ + "name": log_file.name, + "size": stat.st_size, + "size_formatted": format_file_size(stat.st_size), + "modified": stat.st_mtime # 保存原始时间戳用于排序 + }) + + # 高效排序（使用原始时间戳） + history_files.sort(key=lambda x: x["modified"], reverse=True) + + # 格式化时间显示 + for file_info in history_files: + file_info["modified"] = datetime.fromtimestamp(file_info["modified"]).strftime("%Y-%m-%d %H:%M:%S") + + log_info["history_files"] = history_files + + # 计算总大小（使用生成器表达式提升性能） + all_files = log_info["current_files"] + history_files + log_info["total_size"] = sum(file_info["size"] for file_info in all_files) + log_info["total_size_formatted"] = format_file_size(log_info["total_size"]) + + except OSError as e: + self.logger.error(f"获取日志文件信息失败: {e}") + + return log_info + + +def format_file_size(size_bytes: int) -> str: + """高性能文件大小格式化工具函数""" + if size_bytes == 0: + return "0 B" + units = ['B', 'KB', 'MB', 'GB', 'TB'] + size = float(size_bytes) + for unit in units: + if size < 1024.0: + return f"{size:.1f} {unit}" + size /= 1024.0 + return f"{size:.1f} PB" + + +# 全局日志实例（使用延迟初始化提升启动性能） +_logger_instance: Optional[logging.Logger] = None + +def get_logger() -> logging.Logger: + """获取全局日志实例（带延迟初始化）""" + global _logger_instance + if _logger_instance is None: + _logger_instance = LoggerHandler().logger + return _logger_instance + +# 向后兼容的全局实例 +logger = get_logger()