feat: 实现日志自动删除功能并更新配置管理

本次提交主要包含以下内容： 1. **日志自动删除功能**: * 新增环境变量 (`AUTO_DELETE_ERROR_LOGS_ENABLED`, `AUTO_DELETE_ERROR_LOGS_DAYS`, `AUTO_DELETE_REQUEST_LOGS_ENABLED`, `AUTO_DELETE_REQUEST_LOGS_DAYS`) 用于控制错误日志和请求日志的自动删除策略。 * 在 `app/config/config.py` 中添加了对这些新配置项的支持和验证逻辑 (Pydantic `validator` 更新为 `field_validator`)。 * 修改了 `app/log/logger.py` 以适应新的日志配置。 * 新增 `app/scheduler/scheduled_tasks.py` 用于执行定期的日志清理任务。 * 新增 `app/service/error_log/error_log_service.py` 和 `app/service/request_log/request_log_service.py` 来处理具体的日志删除逻辑。 * 更新了 `app/router/error_log_routes.py` 和 `app/router/scheduler_routes.py` 以集成新功能。 2. **前端配置页面更新**: * 在 `app/templates/config_editor.html` 和 `app/static/js/config_editor.js` 中添加了用于配置日志自动删除选项的用户界面元素。 3. **代码和文件结构调整**: * 删除了不再使用的 `app/scheduler/key_checker.py` 文件。 * 在 `.gitignore` 文件中添加了 `default_db` 以忽略该目录。 4. **其他**: * 对 `app/core/application.py` 进行了相应调整。该更新旨在增强应用的日志管理能力，提供更灵活的日志保留策略，并优化了配置界面的用户体验。
2026-09-05 07:26:39 +08:00 · 2025-05-08 00:31:17 +08:00
parent b86eac839d
commit e1c068ed9e
13 changed files with 3315 additions and 1835 deletions
@@ -1,15 +1,29 @@
 """
 应用程序配置模块
 """
+
 import datetime
 import json
-from typing import List, Any, Dict, Type
+from typing import Any, Dict, List, Type

-from pydantic import ValidationError, validator
+from pydantic import ValidationError, ValidationInfo, field_validator
 from pydantic_settings import BaseSettings
-from sqlalchemy import insert, update, select
+from sqlalchemy import insert, select, update

-from app.core.constants import API_VERSION, DEFAULT_CREATE_IMAGE_MODEL, DEFAULT_FILTER_MODELS, DEFAULT_MODEL, DEFAULT_SAFETY_SETTINGS, DEFAULT_STREAM_CHUNK_SIZE, DEFAULT_STREAM_LONG_TEXT_THRESHOLD, DEFAULT_STREAM_MAX_DELAY, DEFAULT_STREAM_MIN_DELAY, DEFAULT_STREAM_SHORT_TEXT_THRESHOLD, DEFAULT_TIMEOUT, MAX_RETRIES
+from app.core.constants import (
+    API_VERSION,
+    DEFAULT_CREATE_IMAGE_MODEL,
+    DEFAULT_FILTER_MODELS,
+    DEFAULT_MODEL,
+    DEFAULT_SAFETY_SETTINGS,
+    DEFAULT_STREAM_CHUNK_SIZE,
+    DEFAULT_STREAM_LONG_TEXT_THRESHOLD,
+    DEFAULT_STREAM_MAX_DELAY,
+    DEFAULT_STREAM_MIN_DELAY,
+    DEFAULT_STREAM_SHORT_TEXT_THRESHOLD,
+    DEFAULT_TIMEOUT,
+    MAX_RETRIES,
+)
 from app.log.logger import Logger


@@ -23,13 +37,17 @@ class Settings(BaseSettings):
    MYSQL_PASSWORD: str = ""
    MYSQL_DATABASE: str = ""
    MYSQL_SOCKET: str = ""
-    
+
    # 验证 MySQL 配置
-    @validator('MYSQL_HOST', 'MYSQL_PORT', 'MYSQL_USER', 'MYSQL_PASSWORD', 'MYSQL_DATABASE')
-    def validate_mysql_config(cls, v, values):
-        if values.get('DATABASE_TYPE') == 'mysql':
+    @field_validator(
+        "MYSQL_HOST", "MYSQL_PORT", "MYSQL_USER", "MYSQL_PASSWORD", "MYSQL_DATABASE"
+    )
+    def validate_mysql_config(cls, v: Any, info: ValidationInfo) -> Any:
+        if info.data.get("DATABASE_TYPE") == "mysql":
            if v is None or v == "":
-                raise ValueError(f"MySQL configuration is required when DATABASE_TYPE is 'mysql'")
+                raise ValueError(
+                    "MySQL configuration is required when DATABASE_TYPE is 'mysql'"
+                )
        return v

    # API相关配置
@@ -41,7 +59,7 @@ class Settings(BaseSettings):
    TEST_MODEL: str = DEFAULT_MODEL
    TIME_OUT: int = DEFAULT_TIMEOUT
    MAX_RETRIES: int = MAX_RETRIES
-    PROXIES: List[str] = [] # 新增：代理服务器列表
+    PROXIES: List[str] = []  # 新增：代理服务器列表

    # 模型相关配置
    SEARCH_MODELS: List[str] = ["gemini-2.0-flash-exp"]
@@ -50,9 +68,9 @@ class Settings(BaseSettings):
    TOOLS_CODE_EXECUTION_ENABLED: bool = False
    SHOW_SEARCH_LINK: bool = True
    SHOW_THINKING_PROCESS: bool = True
-    THINKING_MODELS: List[str] = [] # 新增：用于思考过程的模型列表
-    THINKING_BUDGET_MAP: Dict[str, float] = {} # 新增：模型对应的预算映射
-    
+    THINKING_MODELS: List[str] = []  # 新增：用于思考过程的模型列表
+    THINKING_BUDGET_MAP: Dict[str, float] = {}  # 新增：模型对应的预算映射
+
    # 图像生成相关配置
    PAID_KEY: str = ""
    CREATE_IMAGE_MODEL: str = DEFAULT_CREATE_IMAGE_MODEL
@@ -61,7 +79,7 @@ class Settings(BaseSettings):
    PICGO_API_KEY: str = ""
    CLOUDFLARE_IMGBED_URL: str = ""
    CLOUDFLARE_IMGBED_AUTH_CODE: str = ""
-    
+
    # 流式输出优化器配置
    STREAM_OPTIMIZER_ENABLED: bool = False
    STREAM_MIN_DELAY: float = DEFAULT_STREAM_MIN_DELAY
@@ -71,16 +89,20 @@ class Settings(BaseSettings):
    STREAM_CHUNK_SIZE: int = DEFAULT_STREAM_CHUNK_SIZE

    # 调度器配置
-    CHECK_INTERVAL_HOURS: int = 1 # 默认检查间隔为1小时
-    TIMEZONE: str = "Asia/Shanghai" # 默认时区
-    
-    # github 
+    CHECK_INTERVAL_HOURS: int = 1  # 默认检查间隔为1小时
+    TIMEZONE: str = "Asia/Shanghai"  # 默认时区
+
+    # github
    GITHUB_REPO_OWNER: str = "snailyp"
    GITHUB_REPO_NAME: str = "gemini-balance"

    # 日志配置
-    LOG_LEVEL: str = "INFO" # 默认日志级别
-    SAFETY_SETTINGS: List[Dict[str, str]] = DEFAULT_SAFETY_SETTINGS # 新增：安全设置
+    LOG_LEVEL: str = "INFO"  # 默认日志级别
+    AUTO_DELETE_ERROR_LOGS_ENABLED: bool = True  # 是否开启自动删除错误日志
+    AUTO_DELETE_ERROR_LOGS_DAYS: int = 7  # 自动删除多少天前的错误日志 (1, 7, 30)
+    AUTO_DELETE_REQUEST_LOGS_ENABLED: bool = False  # 是否开启自动删除请求日志
+    AUTO_DELETE_REQUEST_LOGS_DAYS: int = 30  # 自动删除多少天前的请求日志 (1, 7, 30)
+    SAFETY_SETTINGS: List[Dict[str, str]] = DEFAULT_SAFETY_SETTINGS  # 新增：安全设置

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
@@ -88,13 +110,16 @@ class Settings(BaseSettings):
        if not self.AUTH_TOKEN and self.ALLOWED_TOKENS:
            self.AUTH_TOKEN = self.ALLOWED_TOKENS[0]

+
 # 创建全局配置实例
 settings = Settings()

+
 def _parse_db_value(key: str, db_value: str, target_type: Type) -> Any:
    """尝试将数据库字符串值解析为目标 Python 类型"""
-    from app.log.logger import get_config_logger # 函数内导入
-    logger = get_config_logger() # 函数内初始化
+    from app.log.logger import get_config_logger  # 函数内导入
+
+    logger = get_config_logger()  # 函数内初始化
    try:
        # 处理 List[str]
        if target_type == List[str]:
@@ -103,9 +128,11 @@ def _parse_db_value(key: str, db_value: str, target_type: Type) -> Any:
                if isinstance(parsed, list):
                    return [str(item) for item in parsed]
            except json.JSONDecodeError:
-                return [item.strip() for item in db_value.split(',') if item.strip()]
-            logger.warning(f"Could not parse '{db_value}' as List[str] for key '{key}', falling back to comma split or empty list.")
-            return [item.strip() for item in db_value.split(',') if item.strip()]
+                return [item.strip() for item in db_value.split(",") if item.strip()]
+            logger.warning(
+                f"Could not parse '{db_value}' as List[str] for key '{key}', falling back to comma split or empty list."
+            )
+            return [item.strip() for item in db_value.split(",") if item.strip()]
        # 处理 Dict[str, float]
        elif target_type == Dict[str, float]:
            parsed_dict = {}
@@ -115,24 +142,34 @@ def _parse_db_value(key: str, db_value: str, target_type: Type) -> Any:
                if isinstance(parsed, dict):
                    parsed_dict = {str(k): float(v) for k, v in parsed.items()}
                else:
-                     logger.warning(f"Parsed DB value for key '{key}' is not a dictionary type. Value: {db_value}")
+                    logger.warning(
+                        f"Parsed DB value for key '{key}' is not a dictionary type. Value: {db_value}"
+                    )
            except (json.JSONDecodeError, ValueError, TypeError) as e1:
                # Second attempt: try replacing single quotes if JSONDecodeError occurred
                if isinstance(e1, json.JSONDecodeError) and "'" in db_value:
-                    logger.warning(f"Failed initial JSON parse for key '{key}'. Attempting to replace single quotes. Error: {e1}")
+                    logger.warning(
+                        f"Failed initial JSON parse for key '{key}'. Attempting to replace single quotes. Error: {e1}"
+                    )
                    try:
                        corrected_db_value = db_value.replace("'", '"')
                        parsed = json.loads(corrected_db_value)
                        if isinstance(parsed, dict):
-                             parsed_dict = {str(k): float(v) for k, v in parsed.items()}
+                            parsed_dict = {str(k): float(v) for k, v in parsed.items()}
                        else:
-                            logger.warning(f"Parsed DB value (after quote replacement) for key '{key}' is not a dictionary type. Value: {corrected_db_value}")
+                            logger.warning(
+                                f"Parsed DB value (after quote replacement) for key '{key}' is not a dictionary type. Value: {corrected_db_value}"
+                            )
                    except (json.JSONDecodeError, ValueError, TypeError) as e2:
-                         logger.error(f"Could not parse '{db_value}' as Dict[str, float] for key '{key}' even after replacing quotes: {e2}. Returning empty dict.")
+                        logger.error(
+                            f"Could not parse '{db_value}' as Dict[str, float] for key '{key}' even after replacing quotes: {e2}. Returning empty dict."
+                        )
                else:
                    # Log other errors (ValueError, TypeError) or JSON errors without single quotes
-                    logger.error(f"Could not parse '{db_value}' as Dict[str, float] for key '{key}': {e1}. Returning empty dict.")
-            return parsed_dict # Return the parsed dict or an empty one if all attempts fail
+                    logger.error(
+                        f"Could not parse '{db_value}' as Dict[str, float] for key '{key}': {e1}. Returning empty dict."
+                    )
+            return parsed_dict  # Return the parsed dict or an empty one if all attempts fail
        # 处理 List[Dict[str, str]]
        elif target_type == List[Dict[str, str]]:
            try:
@@ -140,28 +177,36 @@ def _parse_db_value(key: str, db_value: str, target_type: Type) -> Any:
                if isinstance(parsed, list):
                    # 验证列表中的每个元素是否为字典，并且键和值都是字符串
                    valid = all(
-                        isinstance(item, dict) and
-                        all(isinstance(k, str) for k in item.keys()) and
-                        all(isinstance(v, str) for v in item.values())
+                        isinstance(item, dict)
+                        and all(isinstance(k, str) for k in item.keys())
+                        and all(isinstance(v, str) for v in item.values())
                        for item in parsed
                    )
                    if valid:
                        return parsed
                    else:
-                        logger.warning(f"Invalid structure in List[Dict[str, str]] for key '{key}'. Value: {db_value}")
-                        return [] # 或者返回默认值？这里返回空列表
+                        logger.warning(
+                            f"Invalid structure in List[Dict[str, str]] for key '{key}'. Value: {db_value}"
+                        )
+                        return []  # 或者返回默认值？这里返回空列表
                else:
-                     logger.warning(f"Parsed DB value for key '{key}' is not a list type. Value: {db_value}")
-                     return []
+                    logger.warning(
+                        f"Parsed DB value for key '{key}' is not a list type. Value: {db_value}"
+                    )
+                    return []
            except json.JSONDecodeError:
-                logger.error(f"Could not parse '{db_value}' as JSON for List[Dict[str, str]] for key '{key}'. Returning empty list.")
+                logger.error(
+                    f"Could not parse '{db_value}' as JSON for List[Dict[str, str]] for key '{key}'. Returning empty list."
+                )
                return []
            except Exception as e:
-                logger.error(f"Error parsing List[Dict[str, str]] for key '{key}': {e}. Value: {db_value}. Returning empty list.")
+                logger.error(
+                    f"Error parsing List[Dict[str, str]] for key '{key}': {e}. Value: {db_value}. Returning empty list."
+                )
                return []
        # 处理 bool
        elif target_type == bool:
-            return db_value.lower() in ('true', '1', 'yes', 'on')
+            return db_value.lower() in ("true", "1", "yes", "on")
        # 处理 int
        elif target_type == int:
            return int(db_value)
@@ -172,8 +217,11 @@ def _parse_db_value(key: str, db_value: str, target_type: Type) -> Any:
        else:
            return db_value
    except (ValueError, TypeError, json.JSONDecodeError) as e:
-        logger.warning(f"Failed to parse db_value '{db_value}' for key '{key}' as type {target_type}: {e}. Using original string value.")
-        return db_value # 解析失败则返回原始字符串
+        logger.warning(
+            f"Failed to parse db_value '{db_value}' for key '{key}' as type {target_type}: {e}. Using original string value."
+        )
+        return db_value  # 解析失败则返回原始字符串
+

 async def sync_initial_settings():
    """
@@ -182,8 +230,9 @@ async def sync_initial_settings():
    2. 将数据库设置合并到内存 settings (数据库优先)。
    3. 将最终的内存 settings 同步回数据库。
    """
-    from app.log.logger import get_config_logger # 函数内导入
-    logger = get_config_logger() # 函数内初始化
+    from app.log.logger import get_config_logger  # 函数内导入
+
+    logger = get_config_logger()  # 函数内初始化
    # 延迟导入以避免循环依赖和确保数据库连接已初始化
    from app.database.connection import database
    from app.database.models import Settings as SettingsModel
@@ -196,7 +245,9 @@ async def sync_initial_settings():
            await database.connect()
            logger.info("Database connection established for initial sync.")
        except Exception as e:
-            logger.error(f"Failed to connect to database for initial settings sync: {e}. Skipping sync.")
+            logger.error(
+                f"Failed to connect to database for initial settings sync: {e}. Skipping sync."
+            )
            return

    try:
@@ -205,13 +256,19 @@ async def sync_initial_settings():
        try:
            query = select(SettingsModel.key, SettingsModel.value)
            results = await database.fetch_all(query)
-            db_settings_raw = [{"key": row["key"], "value": row["value"]} for row in results]
+            db_settings_raw = [
+                {"key": row["key"], "value": row["value"]} for row in results
+            ]
            logger.info(f"Fetched {len(db_settings_raw)} settings from database.")
        except Exception as e:
-            logger.error(f"Failed to fetch settings from database: {e}. Proceeding with environment/dotenv settings.")
+            logger.error(
+                f"Failed to fetch settings from database: {e}. Proceeding with environment/dotenv settings."
+            )
            # 即使数据库读取失败，也要继续执行，确保基于 env/dotenv 的配置能同步到数据库

-        db_settings_map: Dict[str, str] = {s['key']: s['value'] for s in db_settings_raw}
+        db_settings_map: Dict[str, str] = {
+            s["key"]: s["value"] for s in db_settings_raw
+        }

        # 2. 将数据库设置合并到内存 settings (数据库优先)
        updated_in_memory = False
@@ -229,35 +286,52 @@ async def sync_initial_settings():
                        if parsed_db_value != memory_value:
                            # 检查类型是否匹配，以防解析函数返回了不兼容的类型
                            type_match = False
-                            if target_type == List[str] and isinstance(parsed_db_value, list):
+                            if target_type == List[str] and isinstance(
+                                parsed_db_value, list
+                            ):
                                type_match = True
-                            elif target_type == Dict[str, float] and isinstance(parsed_db_value, dict):
+                            elif target_type == Dict[str, float] and isinstance(
+                                parsed_db_value, dict
+                            ):
                                type_match = True
-                            elif target_type not in (List[str], Dict[str, float]) and isinstance(parsed_db_value, target_type):
+                            elif target_type not in (
+                                List[str],
+                                Dict[str, float],
+                            ) and isinstance(parsed_db_value, target_type):
                                type_match = True

                            if type_match:
                                setattr(settings, key, parsed_db_value)
-                                logger.debug(f"Updated setting '{key}' in memory from database value ({target_type}).")
+                                logger.debug(
+                                    f"Updated setting '{key}' in memory from database value ({target_type})."
+                                )
                                updated_in_memory = True
                            else:
-                                logger.warning(f"Parsed DB value type mismatch for key '{key}'. Expected {target_type}, got {type(parsed_db_value)}. Skipping update.")
+                                logger.warning(
+                                    f"Parsed DB value type mismatch for key '{key}'. Expected {target_type}, got {type(parsed_db_value)}. Skipping update."
+                                )

                    except Exception as e:
-                        logger.error(f"Error processing database setting for key '{key}': {e}")
+                        logger.error(
+                            f"Error processing database setting for key '{key}': {e}"
+                        )
            else:
-                 logger.warning(f"Database setting '{key}' not found in Settings model definition. Ignoring.")
-
+                logger.warning(
+                    f"Database setting '{key}' not found in Settings model definition. Ignoring."
+                )

        # 如果内存中有更新，重新验证 Pydantic 模型（可选但推荐）
        if updated_in_memory:
            try:
                # 重新加载以确保类型转换和验证
                settings = Settings(**settings.model_dump())
-                logger.info("Settings object re-validated after merging database values.")
+                logger.info(
+                    "Settings object re-validated after merging database values."
+                )
            except ValidationError as e:
-                 logger.error(f"Validation error after merging database settings: {e}. Settings might be inconsistent.")
-
+                logger.error(
+                    f"Validation error after merging database settings: {e}. Settings might be inconsistent."
+                )

        # 3. 将最终的内存 settings 同步回数据库
        final_memory_settings = settings.model_dump()
@@ -269,20 +343,22 @@ async def sync_initial_settings():

        for key, value in final_memory_settings.items():
            # 序列化值为字符串或 JSON 字符串
-            if isinstance(value, (list, dict)): # 处理列表和字典
-                db_value = json.dumps(value, ensure_ascii=False) # 使用 ensure_ascii=False 以支持非 ASCII 字符
+            if isinstance(value, (list, dict)):  # 处理列表和字典
+                db_value = json.dumps(
+                    value, ensure_ascii=False
+                )  # 使用 ensure_ascii=False 以支持非 ASCII 字符
            elif isinstance(value, bool):
                db_value = str(value).lower()
-            elif value is None: # 处理 None 值
-                db_value = "" # 或者根据需要设为 NULL 或其他标记
+            elif value is None:  # 处理 None 值
+                db_value = ""  # 或者根据需要设为 NULL 或其他标记
            else:
                db_value = str(value)

            data = {
-                'key': key,
-                'value': db_value,
-                'description': f"{key} configuration setting", # 默认描述
-                'updated_at': now
+                "key": key,
+                "value": db_value,
+                "description": f"{key} configuration setting",  # 默认描述
+                "updated_at": now,
            }

            if key in existing_db_keys:
@@ -291,7 +367,7 @@ async def sync_initial_settings():
                    settings_to_update.append(data)
            else:
                # 如果键不在数据库中，则插入
-                data['created_at'] = now
+                data["created_at"] = now
                settings_to_insert.append(data)

        # 在事务中执行批量插入和更新
@@ -300,48 +376,78 @@ async def sync_initial_settings():
                async with database.transaction():
                    if settings_to_insert:
                        # 获取现有描述以避免覆盖
-                        query_existing = select(SettingsModel.key, SettingsModel.description).where(SettingsModel.key.in_([s['key'] for s in settings_to_insert]))
-                        existing_desc = {row['key']: row['description'] for row in await database.fetch_all(query_existing)}
+                        query_existing = select(
+                            SettingsModel.key, SettingsModel.description
+                        ).where(
+                            SettingsModel.key.in_(
+                                [s["key"] for s in settings_to_insert]
+                            )
+                        )
+                        existing_desc = {
+                            row["key"]: row["description"]
+                            for row in await database.fetch_all(query_existing)
+                        }
                        for item in settings_to_insert:
-                            item['description'] = existing_desc.get(item['key'], item['description'])
+                            item["description"] = existing_desc.get(
+                                item["key"], item["description"]
+                            )

                        query_insert = insert(SettingsModel).values(settings_to_insert)
                        await database.execute(query=query_insert)
-                        logger.info(f"Synced (inserted) {len(settings_to_insert)} settings to database.")
+                        logger.info(
+                            f"Synced (inserted) {len(settings_to_insert)} settings to database."
+                        )

                    if settings_to_update:
                        # 获取现有描述以避免覆盖
-                        query_existing = select(SettingsModel.key, SettingsModel.description).where(SettingsModel.key.in_([s['key'] for s in settings_to_update]))
-                        existing_desc = {row['key']: row['description'] for row in await database.fetch_all(query_existing)}
+                        query_existing = select(
+                            SettingsModel.key, SettingsModel.description
+                        ).where(
+                            SettingsModel.key.in_(
+                                [s["key"] for s in settings_to_update]
+                            )
+                        )
+                        existing_desc = {
+                            row["key"]: row["description"]
+                            for row in await database.fetch_all(query_existing)
+                        }

                        for setting_data in settings_to_update:
-                            setting_data['description'] = existing_desc.get(setting_data['key'], setting_data['description'])
+                            setting_data["description"] = existing_desc.get(
+                                setting_data["key"], setting_data["description"]
+                            )
                            query_update = (
                                update(SettingsModel)
-                                .where(SettingsModel.key == setting_data['key'])
+                                .where(SettingsModel.key == setting_data["key"])
                                .values(
-                                    value=setting_data['value'],
-                                    description=setting_data['description'],
-                                    updated_at=setting_data['updated_at']
+                                    value=setting_data["value"],
+                                    description=setting_data["description"],
+                                    updated_at=setting_data["updated_at"],
                                )
                            )
                            await database.execute(query=query_update)
-                        logger.info(f"Synced (updated) {len(settings_to_update)} settings to database.")
+                        logger.info(
+                            f"Synced (updated) {len(settings_to_update)} settings to database."
+                        )
            except Exception as e:
-                logger.error(f"Failed to sync settings to database during startup: {str(e)}")
+                logger.error(
+                    f"Failed to sync settings to database during startup: {str(e)}"
+                )
        else:
-            logger.info("No setting changes detected between memory and database during initial sync.")
+            logger.info(
+                "No setting changes detected between memory and database during initial sync."
+            )

        # 刷新日志等级
        Logger.update_log_levels(final_memory_settings.get("LOG_LEVEL"))
-        
+
    except Exception as e:
        logger.error(f"An unexpected error occurred during initial settings sync: {e}")
    finally:
        if database.is_connected:
-             try:
-                 pass
-             except Exception as e:
-                 logger.error(f"Error disconnecting database after initial sync: {e}")
+            try:
+                pass
+            except Exception as e:
+                logger.error(f"Error disconnecting database after initial sync: {e}")

    logger.info("Initial settings synchronization finished.")