This commit is contained in:
jxxghp
2025-08-06 16:24:47 +08:00
parent c20bd84edd
commit 37f8ba7d72
+58 -16
View File
@@ -6,6 +6,7 @@ import threading
import time import time
from concurrent.futures import ThreadPoolExecutor from concurrent.futures import ThreadPoolExecutor
from datetime import datetime from datetime import datetime
from logging.handlers import RotatingFileHandler
from pathlib import Path from pathlib import Path
from typing import Dict, Any, Optional from typing import Dict, Any, Optional
@@ -117,10 +118,11 @@ class LogEntry:
class NonBlockingFileHandler: class NonBlockingFileHandler:
""" """
非阻塞文件处理器 - 透明地处理协程环境中的文件写入 非阻塞文件处理器 - 使用RotatingFileHandler实现日志滚动
""" """
_instance = None _instance = None
_lock = threading.Lock() _lock = threading.Lock()
_rotating_handlers = {} # 缓存RotatingFileHandler实例
def __new__(cls): def __new__(cls):
if cls._instance is None: if cls._instance is None:
@@ -137,14 +139,36 @@ class NonBlockingFileHandler:
self._write_queue = queue.Queue(maxsize=log_settings.ASYNC_FILE_QUEUE_SIZE) self._write_queue = queue.Queue(maxsize=log_settings.ASYNC_FILE_QUEUE_SIZE)
self._executor = ThreadPoolExecutor(max_workers=log_settings.ASYNC_FILE_WORKERS, self._executor = ThreadPoolExecutor(max_workers=log_settings.ASYNC_FILE_WORKERS,
thread_name_prefix="LogWriter") thread_name_prefix="LogWriter")
self._batch_buffer = {}
self._last_flush = {}
self._running = True self._running = True
# 启动后台写入线程 # 启动后台写入线程
self._write_thread = threading.Thread(target=self._batch_writer, daemon=True) self._write_thread = threading.Thread(target=self._batch_writer, daemon=True)
self._write_thread.start() self._write_thread.start()
def _get_rotating_handler(self, file_path: Path) -> RotatingFileHandler:
"""
获取或创建RotatingFileHandler实例
"""
if file_path not in self._rotating_handlers:
# 确保目录存在
file_path.parent.mkdir(parents=True, exist_ok=True)
# 创建RotatingFileHandler
handler = RotatingFileHandler(
filename=str(file_path),
maxBytes=log_settings.LOG_MAX_FILE_SIZE_BYTES,
backupCount=log_settings.LOG_BACKUP_COUNT,
encoding='utf-8'
)
# 设置格式化器
formatter = logging.Formatter(log_settings.LOG_FILE_FORMAT)
handler.setFormatter(formatter)
self._rotating_handlers[file_path] = handler
return self._rotating_handlers[file_path]
def write_log(self, level: str, message: str, file_path: Path): def write_log(self, level: str, message: str, file_path: Path):
""" """
写入日志 - 自动检测协程环境并使用合适的方式 写入日志 - 自动检测协程环境并使用合适的方式
@@ -186,16 +210,23 @@ class NonBlockingFileHandler:
同步写入日志 同步写入日志
""" """
try: try:
# 确保目录存在 # 获取RotatingFileHandler实例
entry.file_path.parent.mkdir(parents=True, exist_ok=True) handler = NonBlockingFileHandler()._get_rotating_handler(entry.file_path)
# 格式化时间戳 # 格式化时间戳
timestamp = entry.timestamp.strftime('%Y-%m-%d %H:%M:%S,') + f"{entry.timestamp.microsecond // 1000:03d}" timestamp = entry.timestamp.strftime('%Y-%m-%d %H:%M:%S,') + f"{entry.timestamp.microsecond // 1000:03d}"
line = f"{entry.level.upper()}{timestamp} - {entry.message}\n" line = f"{entry.level.upper()}{timestamp} - {entry.message}\n"
# 写入文件 # 使用RotatingFileHandler的emit方法
with open(entry.file_path, 'a', encoding='utf-8') as f: handler.emit(logging.LogRecord(
f.write(line) name='',
level=getattr(logging, entry.level.upper(), logging.INFO),
pathname='',
lineno=0,
msg=line,
args=(),
exc_info=None
))
except Exception as e: except Exception as e:
# 如果文件写入失败,至少输出到控制台 # 如果文件写入失败,至少输出到控制台
print(f"日志写入失败 {entry.file_path}: {e}") print(f"日志写入失败 {entry.file_path}: {e}")
@@ -240,16 +271,24 @@ class NonBlockingFileHandler:
# 批量写入每个文件 # 批量写入每个文件
for file_path, entries in file_groups.items(): for file_path, entries in file_groups.items():
try: try:
# 确保目录存在 # 获取RotatingFileHandler
file_path.parent.mkdir(parents=True, exist_ok=True) handler = self._get_rotating_handler(file_path)
# 批量写入 # 批量写入
with open(file_path, 'a', encoding='utf-8') as f: for entry in entries:
for entry in entries: timestamp = entry.timestamp.strftime(
timestamp = entry.timestamp.strftime( '%Y-%m-%d %H:%M:%S,') + f"{entry.timestamp.microsecond // 1000:03d}"
'%Y-%m-%d %H:%M:%S,') + f"{entry.timestamp.microsecond // 1000:03d}" line = f"{entry.level.upper()}{timestamp} - {entry.message}\n"
line = f"{entry.level.upper()}{timestamp} - {entry.message}\n" # 使用RotatingFileHandler的emit方法
f.write(line) handler.emit(logging.LogRecord(
name='',
level=getattr(logging, entry.level.upper(), logging.INFO),
pathname='',
lineno=0,
msg=line,
args=(),
exc_info=None
))
except Exception as e: except Exception as e:
print(f"批量写入失败 {file_path}: {e}") print(f"批量写入失败 {file_path}: {e}")
# 回退到逐个写入 # 回退到逐个写入
@@ -266,6 +305,9 @@ class NonBlockingFileHandler:
if self._executor: if self._executor:
self._executor.shutdown(wait=True) self._executor.shutdown(wait=True)
# 清理缓存
self._rotating_handlers.clear()
class LoggerManager: class LoggerManager:
""" """