fix(monitor,transfer): 修复 FUSE 挂载无响应导致的监控冻死、整理链锁死与漏件 (#6276)

* wip(v3): 移植监控与整理韧性修复到 v3 基线

包含:监控看门狗隔离/挂载探测、整理队列持久化、文件系统子进程代理、
写入原子化。迁移重挂到 v3 链 8a4c7e1d2f90 -> 7f5c1d2e3a4b -> e3d9f4b7c806。
tmdb 相关测试尚未通过,待定位。

* fix(v3): 修正移植引入的 16 项测试失败

- poller.py:合并时我方保留的行仍用旧变量名 merged_snapshot,而 v3 已统一
  改名为 current_snapshot,导致 NameError 被外层 except 吞掉、快照从未保存
- smb.py:采纳 f-string 拆分写法,恢复 Python 3.11 可解析
- dispatcher 测试:历史查重由 _should_skip_by_history 统一承担,mock 点随之调整
- tmdb 缓存测试:补充 v3 新增的 media_source/media_id 字段
- tmdb 重试测试:为 fake 补充 match_multi/async_match_multi

尚余 3 项与 v3 识别流程的连接失败处理有关,待单独判断。

* fix(v3): 测试适配 v3 的 media_source/media_id 重构

v3 将媒体标识从 tmdbid 统一重构为 media_source + media_id,recognize_media
的 tmdbid 参数已被 **kwargs 静默吞掉——传了也不生效,流程会误降级到名称搜索。
tmdb 重试用例改用新参数后恢复正确路径。

同时修正 fake 的 match_multi 语义:真实实现(tmdbapi.match_multi)吞掉所有
异常并返回 None,连接失败与「未找到」在该路径上本就不可区分,fake 需保持一致。

至此移植引入的 19 项失败全部清零。

---------

Co-authored-by: Aqr-K <Aqr-K@users.noreply.github.com>
This commit is contained in:
Aqr-K
2026-08-13 08:19:54 +08:00
committed by GitHub
co-authored by Aqr-K
parent 4d11a38496
commit a2e70b443d
62 changed files with 7889 additions and 360 deletions
+3
View File
@@ -50,3 +50,6 @@ pylint-report.json
# Superpowers 设计/计划文档(本地协作产物,不纳入仓库) # Superpowers 设计/计划文档(本地协作产物,不纳入仓库)
docs/superpowers/ docs/superpowers/
# 本地前端构建产物(docker build 用,不入库)
frontend-dist/
+3
View File
@@ -28,6 +28,7 @@ from app.db.user_oper import (
get_current_active_superuser_async, get_current_active_superuser_async,
) )
from app.helper.progress import ProgressHelper from app.helper.progress import ProgressHelper
from app.helper.transferhistory import clear_transfer_failures
from app.schemas.types import EventType from app.schemas.types import EventType
from app.utils.jieba import cut as jieba_cut from app.utils.jieba import cut as jieba_cut
@@ -263,6 +264,8 @@ def delete_transfer_history(
) )
# 删除记录 # 删除记录
TransferHistory.delete(db, history_in.id) TransferHistory.delete(db, history_in.id)
# 删除记录是用户显式要求重来,失败重试计数一并清零,否则重整仍会受上一轮次数限制
clear_transfer_failures(history.src, history.src_storage)
return schemas.Response(success=True) return schemas.Response(success=True)
+326 -80
View File
@@ -25,11 +25,15 @@ from app.db.downloadhistory_oper import DownloadHistoryOper
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
from app.db.models.transferhistory import TransferHistory from app.db.models.transferhistory import TransferHistory
from app.db.systemconfig_oper import SystemConfigOper from app.db.systemconfig_oper import SystemConfigOper
from app.db.transferpending_oper import TransferPendingOper
from app.db.transferhistory_oper import TransferHistoryOper from app.db.transferhistory_oper import TransferHistoryOper
from app.helper.directory import DirectoryHelper from app.helper.directory import DirectoryHelper
from app.helper.audio import AudioMetadataHelper from app.helper.audio import AudioMetadataHelper
from app.helper.format import EpisodeFormatRuleHelper, FormatParser from app.helper.format import EpisodeFormatRuleHelper, FormatParser
from app.helper.progress import ProgressHelper from app.helper.progress import ProgressHelper
from app.helper.transferhistory import (clear_transfer_failures, describe_history_gate,
evaluate_history_gate, is_skip_action,
record_transfer_failure, resolve_history)
from app.log import logger from app.log import logger
from app.schemas import StorageOperSelectionEventData from app.schemas import StorageOperSelectionEventData
from app.schemas import ( from app.schemas import (
@@ -829,6 +833,22 @@ class JobManager:
with job_lock: with job_lock:
return sum([len(job.tasks) for job in self._job_view.values()]) return sum([len(job.tasks) for job in self._job_view.values()])
def pending_total(self) -> int:
"""
获取未到终态的任务总数。
作业要等关联任务全部终态才整体移除,追更/分批场景下已完成任务会
跨批次残留在视图中;批次统计若用全量 total() 会把历史任务计入
「当前共 N 个文件」并压低进度百分比,因此只数未终态任务。
"""
with job_lock:
return sum(
1
for job in self._job_view.values()
for task in job.tasks
if task.state not in ("completed", "failed")
)
def list_jobs(self) -> List[TransferJob]: def list_jobs(self) -> List[TransferJob]:
""" """
获取所有作业的任务列表 获取所有作业的任务列表
@@ -996,6 +1016,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self.jobview = JobManager() self.jobview = JobManager()
# Agent重试管理器 # Agent重试管理器
self.retry_scheduler = FailedRetryScheduler() self.retry_scheduler = FailedRetryScheduler()
# 待整理文件落盘登记,用于进程重启后回放内存队列里未完成的任务
self._pendingoper = TransferPendingOper()
# 转移成功的文件清单 # 转移成功的文件清单
self._success_target_files: Dict[Tuple, List[str]] = {} self._success_target_files: Dict[Tuple, List[str]] = {}
# 批次级刮削缓冲,避免同一批多文件入库重复触发目录刮削 # 批次级刮削缓冲,避免同一批多文件入库重复触发目录刮削
@@ -1339,6 +1361,34 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
return not mounted_filesystem_cache[source_directory] return not mounted_filesystem_cache[source_directory]
@staticmethod
def __is_overwrite_declined(task: TransferTask, transferinfo: TransferInfo,
transferhis: TransferHistoryOper) -> bool:
"""
判断本次未入库是否为「同路径已有成功记录 + 覆盖模式裁定不覆盖」。
只有同路径此前已成功整理过才需要保护:这类文件是查重闸放行的同路径新版本,
媒体库中的原有版本仍然在位,不应因一次不覆盖裁决把成功记录改写成失败记录。
没有成功记录时(如目标同名文件来自其他源路径)保持原有失败语义,
用户仍能在历史与通知中看到裁决结果。
:param task: 整理任务
:param transferinfo: 整理结果
:param transferhis: 历史操作对象
:return: True 表示应保留原成功记录
"""
if not transferinfo.overwrite_skipped or not task.fileitem:
return False
try:
history = resolve_history(
task.fileitem.path,
storage=task.fileitem.storage,
transfer_history_oper=transferhis,
)
except Exception as err:
logger.error(f"查询整理历史失败: {task.fileitem.path} - {err}")
return False
return bool(history and history.status)
def __default_callback( def __default_callback(
self, task: TransferTask, transferinfo: TransferInfo, / self, task: TransferTask, transferinfo: TransferInfo, /
) -> Tuple[bool, str]: ) -> Tuple[bool, str]:
@@ -1390,86 +1440,108 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
# 转移失败 # 转移失败
if not transferinfo.success: if not transferinfo.success:
logger.warn(f"{task.fileitem.name} 入库失败:{transferinfo.message}") # 查重闸放行同路径新版本后由 overwrite_mode 判定不覆盖,是一次正常裁决而非故障:
# 媒体库里原有版本仍然在位,写失败记录会用 add_force 顶掉原成功记录,此后该路径
# 新增转移失败历史记录 # 永远处于失败态,每个新事件都会重试并重推失败通知。此时保留原记录、不写历史、
history = transferhis.add_fail( # 不发事件与通知、不触发重试,仅把任务置为未入库
fileitem=task.fileitem, overwrite_declined = self.__is_overwrite_declined(
mode=transferinfo.transfer_type if transferinfo else "", task, transferinfo, transferhis
downloader=task.downloader,
download_hash=task.download_hash,
meta=task.meta,
mediainfo=task.mediainfo,
transferinfo=transferinfo,
) )
history = None
if overwrite_declined:
logger.info(
f"{task.fileitem.name} 未入库并保留原整理记录:{transferinfo.message}"
)
else:
logger.warn(f"{task.fileitem.name} 入库失败:{transferinfo.message}")
# 整理失败事件 # 累计失败次数,达到上限后查重闸不再自动放行重试
if self._is_primary_media_file(task.fileitem, task.mediainfo): record_transfer_failure(
# 主要媒体文件整理失败事件 task.fileitem.path if task.fileitem else None,
self.eventmanager.send_event( task.fileitem.storage if task.fileitem else None,
EventType.TransferFailed, file_size=task.fileitem.size if task.fileitem else None,
{ file_modify_time=task.fileitem.modify_time if task.fileitem else None,
"fileitem": task.fileitem, fileid=task.fileitem.fileid if task.fileitem else None,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
elif self.__is_subtitle_file(task.fileitem):
# 字幕整理失败事件
self.eventmanager.send_event(
EventType.SubtitleTransferFailed,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
elif self.__is_audio_file(task.fileitem):
# 音频文件整理失败事件
self.eventmanager.send_event(
EventType.AudioTransferFailed,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
) )
# 发送失败消息 # 新增转移失败历史记录
self.post_message( history = transferhis.add_fail(
Notification( fileitem=task.fileitem,
mtype=NotificationType.Manual, mode=transferinfo.transfer_type if transferinfo else "",
title=f"{task.mediainfo.title_year} {task.meta.season_episode} 入库失败!", downloader=task.downloader,
text="\n".join( download_hash=task.download_hash,
[ meta=task.meta,
f"原因:{transferinfo.message or '未知'}", mediainfo=task.mediainfo,
( transferinfo=transferinfo,
f"如果按钮不可用,可回复:\n```\n/redo {history.id}\n```" )
if history
else "" # 整理失败事件
), if self.__is_media_file(task.fileitem):
] # 主要媒体文件整理失败事件
).strip(), self.eventmanager.send_event(
image=task.mediainfo.get_message_image(), EventType.TransferFailed,
username=task.username, {
link=settings.MP_DOMAIN("#/history"), "fileitem": task.fileitem,
buttons=self.build_failed_transfer_buttons( "meta": task.meta,
history.id if history else None "mediainfo": task.mediainfo,
), "transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
elif self.__is_subtitle_file(task.fileitem):
# 字幕整理失败事件
self.eventmanager.send_event(
EventType.SubtitleTransferFailed,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
elif self.__is_audio_file(task.fileitem):
# 音频文件整理失败事件
self.eventmanager.send_event(
EventType.AudioTransferFailed,
{
"fileitem": task.fileitem,
"meta": task.meta,
"mediainfo": task.mediainfo,
"transferinfo": transferinfo,
"downloader": task.downloader,
"download_hash": task.download_hash,
"transfer_history_id": history.id if history else None,
},
)
# 发送失败消息
self.post_message(
Notification(
mtype=NotificationType.Manual,
title=f"{task.mediainfo.title_year} {task.meta.season_episode} 入库失败!",
text="\n".join(
[
f"原因:{transferinfo.message or '未知'}",
(
f"如果按钮不可用,可回复:\n```\n/redo {history.id}\n```"
if history
else ""
),
]
).strip(),
image=task.mediainfo.get_message_image(),
username=task.username,
link=settings.MP_DOMAIN("#/history"),
buttons=self.build_failed_transfer_buttons(
history.id if history else None
),
)
) )
)
# 设置任务失败 # 设置任务失败
self.jobview.fail_task(task) self.jobview.fail_task(task)
@@ -1506,6 +1578,12 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
# 转移成功 # 转移成功
logger.info(f"{task.fileitem.name} 入库成功:{target_dir_path or ''}") logger.info(f"{task.fileitem.name} 入库成功:{target_dir_path or ''}")
# 整理成功即认为此前的连续失败已恢复,重置计数让后续故障重新获得完整重试额度
clear_transfer_failures(
task.fileitem.path if task.fileitem else None,
task.fileitem.storage if task.fileitem else None,
)
# 新增task转移成功历史记录 # 新增task转移成功历史记录
history = transferhis.add_success( history = transferhis.add_success(
fileitem=task.fileitem, fileitem=task.fileitem,
@@ -1670,8 +1748,126 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
self.__register_scrape_batch_task(task) self.__register_scrape_batch_task(task)
# 添加到队列 # 添加到队列
self._queue.put(TransferQueue(task=task, callback=self.__default_callback)) self._queue.put(TransferQueue(task=task, callback=self.__default_callback))
# 落盘登记:队列是纯内存的,进程重启(挂载挂死后的人工重启、升级、OOM)
# 会让队列连同「这些文件还没整理」这个事实一起蒸发,而已稳定落地的文件
# 不会再产生任何监控事件,等于永久漏件。登记放在入队之后,宁可多留一条
# 由回放时的整理历史查重挡掉,也不制造「已入队但未登记」的窗口
self.__register_pending(task)
return True return True
def replay_pending(self):
"""
回放上次进程退出时仍未整理完的文件。
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
挂死状态,同步执行会把整个启动流程堵住。
"""
threading.Thread(
target=self.__replay_pending,
name="MoviePilot-TransferReplay",
daemon=True
).start()
def __replay_pending(self):
"""
把落盘登记的待整理文件重新送回整理入口。
只回放「存储 + 源路径」这一最小事实,重新走完整的识别与整理流程,
已经整理完成的由整理历史查重挡掉,因此不存在重复整理的问题。
"""
try:
pendings = self._pendingoper.list_all()
except Exception as err:
logger.error(f"读取待整理文件登记失败:{err}")
return
if not pendings:
return
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
replayed = 0
for storage, src_path in pendings:
try:
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
if not fileitem:
if should_discard:
# 源文件确认已消失,注销登记避免每次启动重复回放
self._pendingoper.discard(storage=storage, src_path=src_path)
continue
self.do_transfer(fileitem=fileitem)
replayed += 1
except Exception as err:
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
@staticmethod
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
"""
为回放构造文件项。
必须用 stat 的异常类型区分「文件真的没了」和「挂载暂时读不到」,不能用
Path.exists():它在任何 OSError 下都返回 False,会把挂载抖动
Transport endpoint is not connected)误判成文件消失,进而注销登记
——那等于在故障期间主动丢件,正是本表要防的事。
:param storage: 存储
:param src_path: 源文件路径,以 / 结尾表示蓝光原盘目录
:return: (文件项, 是否应注销登记)。文件项为 None 表示本次不回放;
只有确认源文件已经消失时才注销登记
"""
# 蓝光原盘目录在登记时保留了尾部斜杠,这里据此还原类型
is_dir = src_path.endswith("/")
path = Path(src_path)
size, modify_time = None, None
if storage == "local":
try:
file_stat = path.stat()
size, modify_time = file_stat.st_size, file_stat.st_mtime
except FileNotFoundError:
logger.info(f"待整理文件已不存在,注销登记:{src_path}")
return None, True
except OSError as err:
# 挂载未就绪或无响应属于暂时性故障,保留登记等下次启动再回放
logger.warn(f"读取待整理文件失败,保留登记等待下次回放:{src_path} - {err}")
return None, False
return FileItem(
storage=storage,
path=src_path if is_dir else path.as_posix(),
type="dir" if is_dir else "file",
name=path.name,
basename=path.stem,
extension=path.suffix[1:] if not is_dir else None,
size=size,
modify_time=modify_time,
), False
def __register_pending(self, task: TransferTask):
"""
落盘登记一个待整理文件,登记失败不影响正常入队。
:param task: 任务信息
"""
fileitem = task.fileitem if task else None
if not fileitem or not fileitem.path:
return
try:
self._pendingoper.register(storage=fileitem.storage, src_path=fileitem.path)
except Exception as err:
# 登记只是重启后的补救手段,失败不能阻断正常整理
logger.debug(f"登记待整理文件失败: {fileitem.path} - {err}")
def __discard_pending(self, task: TransferTask):
"""
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
失败的文件不靠本表回放:整理历史里已有失败记录,分发器的历史门控会按
重试预算重新送入整理链;留在本表反而会每次重启都重复回放。
:param task: 任务信息
"""
fileitem = task.fileitem if task else None
if not fileitem or not fileitem.path:
return
try:
self._pendingoper.discard(storage=fileitem.storage, src_path=fileitem.path)
except Exception as err:
logger.debug(f"注销待整理文件登记失败: {fileitem.path} - {err}")
def __put_to_jobview(self, task: TransferTask) -> bool: def __put_to_jobview(self, task: TransferTask) -> bool:
""" """
添加到作业视图 添加到作业视图
@@ -1942,6 +2138,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
marker = getattr(self.jobview, "finish_execution", None) marker = getattr(self.jobview, "finish_execution", None)
if marker: if marker:
marker(task) marker(task)
# 任务已到终态,落盘登记到此作废(未登记过的实时整理路径为无害空操作)
self.__discard_pending(task)
def __expire_stale_transfer_tasks(self): def __expire_stale_transfer_tasks(self):
"""清理外部接管后失去状态心跳的运行中整理任务。""" """清理外部接管后失去状态心跳的运行中整理任务。"""
@@ -1987,8 +2185,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
fileitem = task.fileitem fileitem = task.fileitem
with task_lock: with task_lock:
# 获取当前最新总数 # 批次总数 = 本批已处理数 + 未终态数。作业视图会残留上一批
current_total = self.jobview.total() # 已完成的任务(作业要等关联任务全部终态才移除),用全量
# total() 会把历史任务计入本批(如显示 8 个实际只处理 2 个),
# 且进度分母虚高导致百分比走不满
current_total = self._processed_num + self.jobview.pending_total()
# 更新总数,取当前总数和当前已处理+运行中+队列中的最大值 # 更新总数,取当前总数和当前已处理+运行中+队列中的最大值
self._total_num = max(self._total_num, current_total) self._total_num = max(self._total_num, current_total)
@@ -2155,6 +2356,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
if not mediainfo: if not mediainfo:
if task.preview: if task.preview:
return False, "未识别到媒体信息" return False, "未识别到媒体信息"
# 未识别同样是整理失败,计入重试次数:TMDB 瞬断属于可恢复故障,
# 但文件名永远识别不出时不能无限重试刷通知
record_transfer_failure(
task.fileitem.path if task.fileitem else None,
task.fileitem.storage if task.fileitem else None,
file_size=task.fileitem.size if task.fileitem else None,
file_modify_time=task.fileitem.modify_time if task.fileitem else None,
fileid=task.fileitem.fileid if task.fileitem else None,
)
# 新增整理失败历史记录 # 新增整理失败历史记录
his = transferhis.add_fail( his = transferhis.add_fail(
fileitem=task.fileitem, fileitem=task.fileitem,
@@ -3156,9 +3366,12 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
include_move_dest: bool = False, include_move_dest: bool = False,
) -> Optional[TransferHistory]: ) -> Optional[TransferHistory]:
"""查询文件源路径历史,并兼容从成功移动后的目标现址重新整理。""" """查询文件源路径历史,并兼容从成功移动后的目标现址重新整理。"""
history = transfer_history_oper.get_by_src( # resolve_history 在命中失败记录时会再确认一次有无成功记录,
# 避免 get_by_src 无排序导致同源多行时返回哪条不确定
history = resolve_history(
fileitem.path, fileitem.path,
storage=fileitem.storage, storage=fileitem.storage,
transfer_history_oper=transfer_history_oper,
) )
if history or not include_move_dest: if history or not include_move_dest:
return history return history
@@ -3228,6 +3441,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
): ):
return False, f"{dest_fileitem.path} 删除失败" return False, f"{dest_fileitem.path} 删除失败"
transfer_history_oper.delete(history.id) transfer_history_oper.delete(history.id)
# 删除记录是用户显式要求重来,失败计数一并清零,否则重整仍会受上一轮次数限制
clear_transfer_failures(history.src, history.src_storage)
return True, "" return True, ""
def do_transfer( def do_transfer(
@@ -3786,7 +4001,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
raise OperationInterrupted() raise OperationInterrupted()
file_path = Path(file_item.path) file_path = Path(file_item.path)
# 自动整理继续按全部历史去重;手动整理可清理失败记录,或按用户确认清理成功记录。 # 自动整理按 app/helper/transferhistory.py 的统一判定去重(失败记录放行重试、
# 成功但源文件已变化放行交 overwrite_mode 决断);手动整理可清理失败记录,
# 或按用户确认清理成功记录。
if (not force or reorganize) and not preview: if (not force or reorganize) and not preview:
transfer_history_oper = TransferHistoryOper() transfer_history_oper = TransferHistoryOper()
transferd = self._get_manual_transfer_history( transferd = self._get_manual_transfer_history(
@@ -3813,10 +4030,37 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
transferd = None transferd = None
if transferd:
history_description = describe_history_gate(
transferd,
file_size=file_item.size,
file_modify_time=file_item.modify_time,
fileid=file_item.fileid,
)
if transferd and not manual:
# 自动路径(目录监控、下载器轮询)与监控分发共用同一套判定,
# 否则监控层刚放行的失败重试与升级请求会在这里被全额收回
gate_action = evaluate_history_gate(
transferd,
file_size=file_item.size,
file_modify_time=file_item.modify_time,
fileid=file_item.fileid,
)
if not is_skip_action(gate_action):
logger.info(
f"{file_item.path} 命中"
f"{history_description}"
f",重新送入整理"
)
transferd = None
if transferd: if transferd:
skipped_history_count += 1 skipped_history_count += 1
if not transferd.status: if not transferd.status:
all_success = False all_success = False
# 失败记录能走到这里说明重试次数已用尽,此时同样要打已整理标签让种子
# 退出轮询,否则下载器每一轮都会重新扫描并在这里被拦一次,空转且刷屏
candidate_hash = download_hash or transferd.download_hash candidate_hash = download_hash or transferd.download_hash
candidate_downloader = downloader or transferd.downloader candidate_downloader = downloader or transferd.downloader
if candidate_hash and candidate_downloader: if candidate_hash and candidate_downloader:
@@ -3824,7 +4068,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
(candidate_hash, candidate_downloader) (candidate_hash, candidate_downloader)
) )
logger.info( logger.info(
f"{file_item.path} 已整理过,如需重新处理,请删除整理记录。" f"{file_item.path} 已整理过"
f"{history_description}"
f"),如需重新处理,请删除整理记录。"
) )
err_msgs.append(f"{file_item.name} 已整理过") err_msgs.append(f"{file_item.name} 已整理过")
continue continue
+8 -2
View File
@@ -7,7 +7,7 @@ from abc import ABC, abstractmethod
from contextlib import contextmanager, asynccontextmanager from contextlib import contextmanager, asynccontextmanager
from functools import wraps from functools import wraps
from pathlib import Path from pathlib import Path
from typing import Any, Dict, Optional, Generator, AsyncGenerator, Tuple, Literal, Union from typing import Any, Callable, Dict, Optional, Generator, AsyncGenerator, Tuple, Literal, Union
import aiofiles import aiofiles
import aioshutil import aioshutil
@@ -1123,7 +1123,8 @@ def AsyncCache(cache_type: Literal['ttl', 'lru'] = 'ttl',
def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Optional[int] = None, def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Optional[int] = None,
skip_none: Optional[bool] = True, skip_empty: Optional[bool] = False, shared_key: Optional[str] = None): skip_none: Optional[bool] = True, skip_empty: Optional[bool] = False, shared_key: Optional[str] = None,
skip_if: Optional[Callable[[Any], bool]] = None):
""" """
自定义缓存装饰器,支持配置缓存区域的 maxsize 和每个 key 的 ttl 自定义缓存装饰器,支持配置缓存区域的 maxsize 和每个 key 的 ttl
@@ -1133,6 +1134,9 @@ def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Opt
:param skip_none: 跳过 None 缓存,默认为 True :param skip_none: 跳过 None 缓存,默认为 True
:param skip_empty: 跳过空值缓存(如 None, [], {}, "", set()),默认为 False :param skip_empty: 跳过空值缓存(如 None, [], {}, "", set()),默认为 False
:param shared_key: 同步/异步函数共享缓存的键,默认使用函数名(异步函数名会标准化为同步格式,如移除 `async_` 前缀) :param shared_key: 同步/异步函数共享缓存的键,默认使用函数名(异步函数名会标准化为同步格式,如移除 `async_` 前缀)
:param skip_if: 按返回值判断是否跳过缓存的谓词,返回真值时不缓存;用于
「结构合法但业务失败」的返回值(如 TMDB 的 success=false 响应),
这类值无法用 skip_none/skip_empty 表达
:return: 装饰器函数 :return: 装饰器函数
""" """
@@ -1158,6 +1162,8 @@ def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Opt
# if skip_empty and value in [None, [], {}, "", set()]: # if skip_empty and value in [None, [], {}, "", set()]:
if skip_empty and not value: if skip_empty and not value:
return False return False
if skip_if and skip_if(value):
return False
return True return True
def is_valid_cache_value(_cache_key: str, _cached_value: Any, _cache_region: str) -> bool: def is_valid_cache_value(_cache_key: str, _cached_value: Any, _cache_region: str) -> bool:
+34
View File
@@ -386,6 +386,19 @@ class ConfigModel(BaseModel):
# 下载器临时文件后缀 # 下载器临时文件后缀
DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: [".!qb", ".part"]) DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: [".!qb", ".part"])
# ==================== 目录监控配置 ====================
# 允许网络文件系统使用快速模式(inotify)。部分 FUSE 实现(如 CloudDrive2
# 会正常下发内核通知,快速模式可用,且比每 N 秒 stat 全部目录的轮询对挂载后端
# 的压力小得多,由用户确认后开启
MONITOR_NETWORK_FAST_MODE: bool = False
# 网络文件系统的轮询扫描间隔(毫秒),0 表示使用内置默认值
MONITOR_POLL_DELAY_NETWORK: int = 0
# 新增目录延迟重扫的轮次延迟秒数,多个使用,分隔,如 "30,120,600,1800"。
# FUSE 挂载(如 CloudDrive2)上超大目录树呈现完整内容可能超过分钟级,
# 默认值在常见的 30/120 秒窗口后追加两轮成本极低的长延迟轮次兜底;
# 解析失败(如格式非法、包含非正整数)时回退默认值并记录 warn 日志
MONITOR_RESCAN_DELAYS: str = "30,120,600,1800"
# ==================== CookieCloud配置 ==================== # ==================== CookieCloud配置 ====================
# CookieCloud是否启动本地服务 # CookieCloud是否启动本地服务
COOKIECLOUD_ENABLE_LOCAL: Optional[bool] = False COOKIECLOUD_ENABLE_LOCAL: Optional[bool] = False
@@ -405,6 +418,27 @@ class ConfigModel(BaseModel):
# ==================== 整理配置 ==================== # ==================== 整理配置 ====================
# 文件整理线程数 # 文件整理线程数
TRANSFER_THREADS: int = 1 TRANSFER_THREADS: int = 1
# 本地文件操作是否走可强杀的子进程代理。
# FUSE/网络挂载进入「请求永不返回」状态时,stat/listdir 这类调用会永久悬挂,
# 而 Python 既不能中断已发出的系统调用、也不能强杀线程,阻塞其上的线程无法回收。
# 走子进程后超时可以 SIGKILL 真正回收,block 型故障被转换成各层已能处理的
# crash 型故障(OSError)。出现兼容问题时可关闭回到直接调用。
FS_PROXY_ENABLED: bool = True
# 单次本地快操作(stat/listdir/删除/重命名)的超时秒数,
# 超时即判定挂载无响应并回收代理进程
FS_PROXY_TIMEOUT: int = 30
# 复制大文件时两次进度上报之间的最长间隔秒数。代理每秒上报一次进度作为心跳,
# 因此这个阈值判定的是「传输完全没有推进」而不是「传输很慢」,
# 复制几小时的大文件也不会被误杀
FS_PROXY_STALL_TIMEOUT: int = 120
# 自动整理(目录监控、下载器轮询)遇到失败整理记录时,同一源路径允许自动重试的最大次数。
# 一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)不该让文件永久漏整理,因此必须重试;
# 但永远识别不出的文件重试再多也不会成功,只会重复推送失败通知,批量导入时更会刷屏,
# 因此必须有界。取值区间 1-10,越界或非法值会被钳制到边界并记录 warn:
# 既不支持关闭重试,也不支持无限重试。整理成功或删除整理记录时计数清零。
# 与本项无关的是同路径新版本:已成功整理的文件源大小变化时一律放行,
# 由整理链的 overwrite_mode 决断是否覆盖
TRANSFER_MAX_FAILED_RETRIES: int = 3
# 外部接管的运行中整理任务无状态心跳超时(分钟),0 表示禁用 # 外部接管的运行中整理任务无状态心跳超时(分钟),0 表示禁用
TRANSFER_TASK_TIMEOUT: int = 120 TRANSFER_TASK_TIMEOUT: int = 120
# 电影重命名格式 # 电影重命名格式
+1
View File
@@ -14,6 +14,7 @@ from .subscribe import Subscribe
from .subscribehistory import SubscribeHistory from .subscribehistory import SubscribeHistory
from .systemconfig import SystemConfig from .systemconfig import SystemConfig
from .transferhistory import TransferHistory from .transferhistory import TransferHistory
from .transferpending import TransferPending
from .user import User from .user import User
from .userconfig import UserConfig from .userconfig import UserConfig
from .workflow import Workflow from .workflow import Workflow
+51 -5
View File
@@ -27,7 +27,7 @@ class TransferHistory(Base):
# 源路径 # 源路径
src = Column(String, index=True) src = Column(String, index=True)
# 源存储 # 源存储
src_storage = Column(String) src_storage = Column(String, nullable=False, default="local")
# 源文件项 # 源文件项
src_fileitem = Column(JSON, default=dict) src_fileitem = Column(JSON, default=dict)
# 目标路径 # 目标路径
@@ -89,6 +89,7 @@ class TransferHistory(Base):
Index('ix_transferhistory_status_date', 'status', 'date'), Index('ix_transferhistory_status_date', 'status', 'date'),
Index('ix_transferhistory_date_id', 'date', 'id'), Index('ix_transferhistory_date_id', 'date', 'id'),
Index('ix_transferhistory_media_identity', 'media_source', 'media_id'), Index('ix_transferhistory_media_identity', 'media_source', 'media_id'),
Index('ux_transferhistory_src_storage', 'src', 'src_storage', unique=True),
) )
@classmethod @classmethod
@@ -207,10 +208,30 @@ class TransferHistory(Base):
:return: 命中的整理记录未命中时返回 None :return: 命中的整理记录未命中时返回 None
""" """
if storage: if storage:
return db.query(cls).filter(cls.src == src, query = db.query(cls).filter(cls.src == src, cls.src_storage == storage)
cls.src_storage == storage).first()
else: else:
return db.query(cls).filter(cls.src == src).first() query = db.query(cls).filter(cls.src == src)
return query.order_by(cls.id.desc()).first()
@classmethod
@db_query
def get_success_by_src(
cls, db: Session, src: str, storage: Optional[str] = None
) -> Optional["TransferHistory"]:
"""
按源路径和存储查询成功的整理记录源路径原样精确匹配
list_success_by_src 不同这里不对源路径做归一化蓝光原盘目录记录
带尾斜杠归一化后反而匹配不到
:param db: 数据库会话
:param src: 源路径
:param storage: 源存储类型
:return: 命中的成功整理记录未命中时返回 None
"""
query = db.query(cls).filter(cls.src == src, cls.status.is_(True))
if storage:
query = query.filter(cls.src_storage == storage)
return query.order_by(cls.id.desc()).first()
@classmethod @classmethod
@db_query @db_query
@@ -228,7 +249,7 @@ class TransferHistory(Base):
query = db.query(cls).filter(cls.dest == dest) query = db.query(cls).filter(cls.dest == dest)
if storage: if storage:
query = query.filter(cls.dest_storage == storage) query = query.filter(cls.dest_storage == storage)
return query.first() return query.order_by(cls.id.desc()).first()
@classmethod @classmethod
@db_query @db_query
@@ -565,6 +586,31 @@ class TransferHistory(Base):
} }
) )
@classmethod
@db_update
def replace_by_src(cls, db: Session, **kwargs) -> "TransferHistory":
"""
用同源存储的新记录原子替换旧整理历史
同一源路径在一个存储中只能对应一条最新整理记录先在同一事务内清理旧行再
插入避免旧的查询一条再删除一条在遗留重复数据下留下脏记录
:param db: 数据库会话
:param kwargs: 整理历史字段
:return: 新创建的整理历史
"""
src = kwargs.get("src")
src_storage = kwargs.get("src_storage") or "local"
kwargs["src_storage"] = src_storage
if src:
db.query(cls).filter(
cls.src == src,
cls.src_storage == src_storage,
).delete(synchronize_session=False)
history = cls(**kwargs)
db.add(history)
db.flush()
return history
@classmethod @classmethod
@db_query @db_query
def list_by_date(cls, db: Session, date: str): def list_by_date(cls, db: Session, date: str):
+102
View File
@@ -0,0 +1,102 @@
from typing import List, Optional
from sqlalchemy import Column, Index, String
from sqlalchemy.orm import Session
from app.db import Base, db_query, db_update, get_id_column
class TransferPending(Base):
"""
待整理文件登记
整理队列是纯内存的 queue.Queue进程一旦重启挂载挂死后的人工重启版本
升级OOM宿主重启队列里的任务会连同这些文件还没整理这个事实一起
蒸发而已经稳定落地的文件不会再产生任何监控事件也不会有新的补偿扫描起点
结果就是永久漏件只能靠人工比对补整理
这里只落盘最小事实存储与源文件路径重启后重新走一遍整理入口由整理历史
查重挡掉已经完成的因此不需要序列化 meta/mediainfo 这些重对象也不存在
识别结果陈旧的问题
"""
id = get_id_column()
# 存储
storage = Column(String, nullable=False)
# 源文件路径
src_path = Column(String, nullable=False)
# 登记时间
created_at = Column(String)
__table_args__ = (
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
Index("ux_transferpending_storage_path", "storage", "src_path", unique=True),
)
@classmethod
@db_update
def register(cls, db: Session, storage: str, src_path: str,
now_time: str) -> Optional["TransferPending"]:
"""
登记一个待整理文件已存在时保持原登记时间不变
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:param now_time: 当前时间
:return: 登记记录
"""
if not storage or not src_path:
return None
pending = db.query(cls).filter(
cls.storage == storage, cls.src_path == src_path
).first()
if pending:
return pending
pending = cls(storage=storage, src_path=src_path, created_at=now_time)
db.add(pending)
return pending
@classmethod
@db_update
def discard(cls, db: Session, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记整理到达终态成功或失败时调用
:param db: 数据库会话
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
if not storage or not src_path:
return 0
return db.query(cls).filter(
cls.storage == storage, cls.src_path == src_path
).delete(synchronize_session=False)
@classmethod
@db_query
def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]:
"""
列出全部待整理登记供启动回放使用
按登记时间升序回放保持与原入队顺序一致上限避免异常积压时
一次性把整理链压垮
:param db: 数据库会话
:param limit: 单次回放上限
:return: 待整理登记列表
"""
return (
db.query(cls)
.order_by(cls.created_at.asc(), cls.id.asc())
.limit(limit)
.all()
)
@classmethod
@db_update
def clear(cls, db: Session) -> int:
"""
清空全部待整理登记
:param db: 数据库会话
:return: 删除的记录数
"""
return db.query(cls).delete(synchronize_session=False)
+25 -7
View File
@@ -91,6 +91,17 @@ class TransferHistoryOper(DbOper):
""" """
return TransferHistory.get_by_src(self._db, src, storage) return TransferHistory.get_by_src(self._db, src, storage)
def get_success_by_src(
self, src: str, storage: Optional[str] = None
) -> Optional[TransferHistory]:
"""
按源查询成功的转移记录源路径原样精确匹配
:param src: 数据key
:param storage: 存储类型
:return: 命中的成功整理记录未命中时返回 None
"""
return TransferHistory.get_success_by_src(self._db, src, storage)
def get_by_dest( def get_by_dest(
self, dest: str, storage: Optional[str] = None self, dest: str, storage: Optional[str] = None
) -> Optional[TransferHistory]: ) -> Optional[TransferHistory]:
@@ -215,18 +226,25 @@ class TransferHistoryOper(DbOper):
def add_force(self, **kwargs) -> TransferHistory: def add_force(self, **kwargs) -> TransferHistory:
""" """
新增转移历史相同源目录的记录会被删除 新增转移历史并以同源存储的记录为准替换旧记录
""" """
kwargs = normalize_media_identity_payload(kwargs) kwargs = normalize_media_identity_payload(kwargs)
if kwargs.get("src"): # 文件项的默认存储是 local;归一化旧调用传入的 None,确保运行时语义与
transferhistory = TransferHistory.get_by_src(self._db, kwargs.get("src")) # (src, src_storage) 唯一索引一致。
if transferhistory: kwargs["src_storage"] = kwargs.get("src_storage") or "local"
transferhistory.delete(self._db, transferhistory.id) # 旧记录的清理交给 replace_by_src 按 (src, src_storage) 处理:
# 仅按 src 删除会连带删掉其他存储下同路径的记录。
kwargs.update({ kwargs.update({
"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) "date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
}) })
TransferHistory(**kwargs).create(self._db) TransferHistory.replace_by_src(self._db, **kwargs)
return TransferHistory.get_by_src(self._db, kwargs.get("src")) # 保持 add_force 的既有返回契约:返回可被调用方安全读取字段的查询结果,
# 而非事务提交后可能已脱离会话的新建实例。
return TransferHistory.get_by_src(
self._db,
kwargs.get("src"),
kwargs["src_storage"],
)
def update_download_hash(self, historyid, download_hash): def update_download_hash(self, historyid, download_hash):
""" """
+59
View File
@@ -0,0 +1,59 @@
from datetime import datetime
from typing import List, Optional, Tuple
from app.db import DbOper
from app.db.models.transferpending import TransferPending
class TransferPendingOper(DbOper):
"""
待整理文件登记管理
只保存存储 + 源文件路径这一最小事实用于在进程重启后把没走完整理链的
文件重新送回去避免挂载故障重启后永久漏件
"""
def register(self, storage: str, src_path: str) -> Optional[TransferPending]:
"""
登记一个待整理文件
:param storage: 存储
:param src_path: 源文件路径
:return: 登记记录
"""
return TransferPending.register(
self._db,
storage=storage,
src_path=src_path,
now_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
)
def discard(self, storage: str, src_path: str) -> int:
"""
注销一个待整理文件登记
:param storage: 存储
:param src_path: 源文件路径
:return: 删除的记录数
"""
return TransferPending.discard(self._db, storage=storage, src_path=src_path)
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
"""
列出全部待整理登记供启动回放使用
返回纯元组而不是 ORM 实例回放发生在会话之外ORM 实例脱离 session
后访问属性会触发 DetachedInstanceError
:param limit: 单次回放上限
:return: (存储, 源文件路径) 列表
"""
return [
(item.storage, item.src_path)
for item in TransferPending.list_all(self._db, limit=limit) or []
if item and item.storage and item.src_path
]
def clear(self) -> int:
"""
清空全部待整理登记
:return: 删除的记录数
"""
return TransferPending.clear(self._db)
+422
View File
@@ -0,0 +1,422 @@
from typing import Any, Dict, Optional
from app.core.cache import TTLCache
from app.core.config import settings
from app.db.models.transferhistory import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper
from app.log import logger
# 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多
# 也不会成功,只会重复推送失败通知,批量导入场景下会刷屏,所以不允许无限重试
MIN_FAILED_RETRIES = 1
MAX_FAILED_RETRIES = 10
# 同一源路径的连续整理失败状态。整理链在写失败历史时累计、整理成功或删除历史时清零,
# 查重闸只读不写,避免监控层与整理链对同一个事件重复计数。缓存值会同时保存文件指纹,
# 因此同一路径的新版本天然获得独立预算;内存缓存会随进程重启清空,Redis 后端则保留到 TTL 到期。
FAILED_RETRY_TTL = 24 * 3600
_failed_retry_counts = TTLCache(region="transfer_failed_retry", maxsize=5000, ttl=FAILED_RETRY_TTL)
class HistoryGateAction:
"""
整理历史查重闸的判定结果
监控分发app/monitor/dispatcher.py与整理链计划整理段app/chain/transfer.py
共用本模块避免两处各写一套去重策略后互相对冲上游放行的文件被下游按
存在记录即拦全额收回等于放行逻辑完全失效
"""
# 没有整理记录
PASS_NO_RECORD = "pass_no_record"
# 上次整理失败且重试次数未用尽,放行重试
PASS_FAILED = "pass_failed"
# 上次整理失败但源文件已变为新版本,放行并重置该版本的重试预算
PASS_FAILED_VERSION_CHANGED = "pass_failed_version_changed"
# 已整理成功但源文件已变化,放行交由 overwrite_mode 决断
PASS_SIZE_CHANGED = "pass_size_changed"
# 上次整理失败且重试次数已用尽,跳过
SKIP_RETRY_EXHAUSTED = "skip_retry_exhausted"
# 已整理成功且源文件未变化,跳过
SKIP = "skip"
def is_skip_action(action: str) -> bool:
"""
判断查重闸判定是否为跳过整理
:param action: HistoryGateAction 之一
:return: True 表示跳过
"""
return action in (HistoryGateAction.SKIP, HistoryGateAction.SKIP_RETRY_EXHAUSTED)
def max_failed_retries() -> int:
"""
读取失败重试上限并钳制到合法区间
配置为负数0 或超过上界时都会被钳制并记录 warn关闭重试会让瞬时故障造成
永久漏件无限重试会让永久失败的文件反复刷通知两端都不接受
:return: 合法的最大重试次数
"""
raw = settings.TRANSFER_MAX_FAILED_RETRIES
try:
value = int(raw)
except (TypeError, ValueError):
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 配置非法({raw!r}),"
f"已回退为 {MIN_FAILED_RETRIES}")
return MIN_FAILED_RETRIES
if value < MIN_FAILED_RETRIES:
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能小于 {MIN_FAILED_RETRIES}"
f"(当前 {value}),已按 {MIN_FAILED_RETRIES} 处理")
return MIN_FAILED_RETRIES
if value > MAX_FAILED_RETRIES:
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能大于 {MAX_FAILED_RETRIES}"
f"(当前 {value}),已按 {MAX_FAILED_RETRIES} 处理")
return MAX_FAILED_RETRIES
return value
def failed_retry_key(src_path: Optional[str], storage: Optional[str] = None) -> Optional[str]:
"""
生成失败重试计数的缓存键
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:return: 缓存键源路径为空时返回 None
"""
if not src_path:
return None
return f"{storage or 'local'}:{src_path}"
def coerce_modify_time(modify_time: Any) -> Optional[float]:
"""
统一转换文件修改时间无法转换时返回 None
:param modify_time: 原始修改时间值
:return: 文件修改时间
"""
if modify_time is None:
return None
try:
return float(modify_time)
except (TypeError, ValueError):
return None
def coerce_fileid(fileid: Any) -> Optional[str]:
"""
统一转换文件唯一标识空值视为不可比对
:param fileid: 原始文件唯一标识
:return: 非空文件唯一标识
"""
if fileid is None:
return None
value = str(fileid).strip()
return value or None
def file_fingerprint(
file_size: Any = None,
file_modify_time: Any = None,
fileid: Any = None,
) -> Dict[str, Any]:
"""
生成用于区分同一路径文件版本的稳定指纹
大小是所有存储器都尽量提供的最小指纹两端均有数据时修改时间和文件 ID 还可
识别同大小替换只保留可比较字段避免缺失元数据把同一文件误判成新版本
:param file_size: 文件大小
:param file_modify_time: 文件修改时间
:param fileid: 存储器文件唯一标识
:return: 非空且可比较的指纹字段
"""
fingerprint = {}
size = coerce_size(file_size)
if size is not None:
fingerprint["size"] = size
modify_time = coerce_modify_time(file_modify_time)
if modify_time is not None:
fingerprint["modify_time"] = modify_time
normalized_fileid = coerce_fileid(fileid)
if normalized_fileid is not None:
fingerprint["fileid"] = normalized_fileid
return fingerprint
def _retry_state(value: Any) -> tuple[int, Dict[str, Any]]:
"""将新旧缓存值统一转换为失败次数与文件指纹。"""
if isinstance(value, dict):
raw_count = value.get("count", 0)
raw_fingerprint = value.get("fingerprint")
else:
# 兼容已写入 Redis 或内存的旧整数计数;下次带指纹写入时会自动升级结构。
raw_count = value
raw_fingerprint = None
try:
count = max(int(raw_count or 0), 0)
except (TypeError, ValueError):
count = 0
fingerprint = (
file_fingerprint(
file_size=raw_fingerprint.get("size"),
file_modify_time=raw_fingerprint.get("modify_time"),
fileid=raw_fingerprint.get("fileid"),
)
if isinstance(raw_fingerprint, dict)
else {}
)
return count, fingerprint
def _is_file_version_changed(
recorded_fingerprint: Dict[str, Any],
current_fingerprint: Dict[str, Any],
) -> bool:
"""判断两个可比文件指纹是否指向不同版本。"""
for field in ("fileid", "modify_time", "size"):
recorded_value = recorded_fingerprint.get(field)
current_value = current_fingerprint.get(field)
if (
recorded_value is not None
and current_value is not None
and recorded_value != current_value
):
return True
return False
def failed_retry_count(src_path: Optional[str], storage: Optional[str] = None,
file_size: Any = None, file_modify_time: Any = None,
fileid: Any = None) -> int:
"""
读取同一源路径已累计的连续整理失败次数
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 当前文件版本已失败次数无记录时为 0
"""
key = failed_retry_key(src_path, storage)
if not key:
return 0
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if (
recorded_fingerprint
and current_fingerprint
and _is_file_version_changed(recorded_fingerprint, current_fingerprint)
):
return 0
return count
def record_transfer_failure(src_path: Optional[str], storage: Optional[str] = None,
file_size: Any = None, file_modify_time: Any = None,
fileid: Any = None) -> int:
"""
累计一次整理失败
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 当前文件版本累计后的失败次数
"""
key = failed_retry_key(src_path, storage)
if not key:
return 0
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if current_fingerprint and (
not recorded_fingerprint
or _is_file_version_changed(recorded_fingerprint, current_fingerprint)
):
count = 0
count += 1
if current_fingerprint:
_failed_retry_counts[key] = {
"count": count,
"fingerprint": current_fingerprint,
}
elif recorded_fingerprint:
_failed_retry_counts[key] = {
"count": count,
"fingerprint": recorded_fingerprint,
}
else:
_failed_retry_counts[key] = count
return count
def clear_transfer_failures(src_path: Optional[str], storage: Optional[str] = None) -> None:
"""
清空同一源路径的失败计数整理成功或用户删除整理记录显式要求重来时调用
:param src_path: 整理记录使用的源路径
:param storage: 源存储
"""
key = failed_retry_key(src_path, storage)
if key:
# 缺省值必须是 0 而不是 NoneCacheBackend.pop 把「default 为 None」当成「未提供
# default」,键不存在时会抛 KeyError。整理成功路径上绝大多数文件从未失败过,
# 传 None 会让每一次首次成功整理都炸掉成功回调
_failed_retry_counts.pop(key, 0)
def coerce_size(size: Any) -> Optional[int]:
"""
统一转换文件大小无法转换时返回 None视为不可比对
:param size: 原始大小值
:return: 文件大小
"""
if size is None:
return None
try:
return int(size)
except (TypeError, ValueError):
return None
def history_src_size(history: TransferHistory) -> Optional[int]:
"""
读取整理记录中的源文件大小
src_fileitem JSON 历史数据可能为空 size 键甚至不是字典
取不到时统一返回 None 交由调用方保守处理
:param history: 整理记录
:return: 源文件大小取不到时为 None
"""
return history_src_fingerprint(history).get("size")
def history_src_fingerprint(history: TransferHistory) -> Dict[str, Any]:
"""
读取整理记录中的源文件版本指纹
:param history: 整理记录
:return: 源文件的可比较指纹字段
"""
src_fileitem = getattr(history, "src_fileitem", None)
if not isinstance(src_fileitem, dict):
return {}
return file_fingerprint(
file_size=src_fileitem.get("size"),
file_modify_time=src_fileitem.get("modify_time"),
fileid=src_fileitem.get("fileid"),
)
def resolve_history(src_path: str, storage: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
查询源路径对应的整理记录
新表通过 (src, src_storage) 唯一索引保证单条记录仍保留对成功记录的二次确认
兼容升级前可能残留的重复数据避免把已整理成功的文件重复整理查询异常不在
此处吞掉由调用方按各自的重试策略处理
:param src_path: 整理记录使用的源路径
:param storage: 存储
:param transfer_history_oper: 复用的历史操作对象未传时新建
:return: 命中的整理记录未命中时为 None
"""
oper = transfer_history_oper or TransferHistoryOper()
history = oper.get_by_src(src_path, storage=storage)
if history is not None and not history.status:
history = oper.get_success_by_src(src_path, storage=storage) or history
return history
def evaluate_history_gate(history: Optional[TransferHistory],
file_size: Optional[float] = None,
file_modify_time: Optional[float] = None,
fileid: Optional[str] = None,
retry_count: Optional[int] = None) -> str:
"""
依据整理历史判断本次是否跳过整理
成功记录不能简单地存在即跳过同路径重新上传的新版本会因此没有机会走到
整理链的 overwrite_mode 判定升级永远无法入库故任一可比文件指纹变化时一律放行
失败记录按文件版本使用有界重试新版本先放行并在下一次失败时从 1 重新计数
同一版本未达上限时继续重试让瞬时故障网络/识别/移动自愈达到上限后跳过
避免永久失败的文件反复刷失败通知
:param history: 整理记录未命中时为 None
:param file_size: 当前文件大小蓝光目录等场景可能为 None
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:param retry_count: 已累计的失败次数None 表示按记录源路径实时查询
:return: HistoryGateAction 之一
"""
if history is None:
return HistoryGateAction.PASS_NO_RECORD
recorded_fingerprint = history_src_fingerprint(history)
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if not history.status:
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
return HistoryGateAction.PASS_FAILED_VERSION_CHANGED
if retry_count is None:
retry_count = failed_retry_count(
getattr(history, "src", None),
getattr(history, "src_storage", None),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if retry_count >= max_failed_retries():
return HistoryGateAction.SKIP_RETRY_EXHAUSTED
# 监控事件是稀疏驱动的(落地事件/延迟重扫/补偿扫描),入口还有 TTL 去重兜底,
# 配合失败次数上限,重试频率与总量都可控
return HistoryGateAction.PASS_FAILED
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
# 同路径换成了另一个版本(如升级为更高码率),是否覆盖交给整理链的
# overwrite_mode 决断,查重闸不做替代判断
return HistoryGateAction.PASS_SIZE_CHANGED
# 无法比对大小(蓝光目录、历史记录缺 size)时保守跳过,避免重复整理
return HistoryGateAction.SKIP
def describe_history_gate(history: Optional[TransferHistory],
file_size: Optional[float] = None,
file_modify_time: Optional[float] = None,
fileid: Optional[str] = None) -> str:
"""
生成查重闸判定的可读说明供日志定位到底是哪条记录在拦
:param history: 整理记录
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 说明文本
"""
if history is None:
return "无整理记录"
recorded_fingerprint = history_src_fingerprint(history)
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if not history.status:
count = failed_retry_count(
getattr(history, "src", None),
getattr(history, "src_storage", None),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
return f"失败记录 #{history.id},文件版本已变化,重试预算将重置"
return f"失败记录 #{history.id},已重试 {count}/{max_failed_retries()}"
recorded_size = recorded_fingerprint.get("size")
current_size = current_fingerprint.get("size")
if recorded_size is None and current_size is None:
return f"成功记录 #{history.id},大小不可比对"
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"
+411
View File
@@ -0,0 +1,411 @@
"""
本地文件系统操作代理
FUSE/网络挂载有两种故障形态crash 调用抛错可捕获可重试 block
调用既不返回错误也不返回结果永久悬挂**Python 无法中断一个已经发出的
系统调用也无法强杀线程**所以 block 型故障下阻塞的线程永远无法回收这正是
整理消费线程停摆监控自愈路径自冻的根因
本模块把这些调用放进一个常驻子进程执行子进程可以被 SIGKILL因此超时后能真正
回收对调用方而言超时表现为一个普通的 OSError 子类FileSystemTimeout
换句话说**把不可处理的 block 型故障转换成系统各层已经能正确处理的 crash
故障**退避重启登记待重试这些既有机制立刻就能接管
第一版只放行安全的操作
- 只读stat/exists/listdir强杀不产生任何副作用
- 同存储 rename内核保证原子性强杀后要么完全成功要么完全没发生
跨存储的复制+删除不在此列它需要单独的可恢复语义临时名 + 完成后 rename
"""
import errno as errno_module
import json
import os
import selectors
import shutil
import subprocess
import sys
import threading
from pathlib import Path
from typing import Any, Callable, Dict, List, Optional
from app.core.config import settings
from app.log import logger
# worker 脚本路径。用绝对路径直接执行,而不是 -m 或 import:
# 直接执行文件不会触发 app/__init__.py 的导入链,代理启动才是毫秒级的
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
# 单次快操作(stat/listdir/rename/unlink 等)的默认超时秒数
DEFAULT_TIMEOUT = 30
# 长耗时操作(复制)两次进度上报之间的最长间隔秒数。
# worker 每秒上报一次心跳,因此这个阈值判定的是「传输完全没有推进」,
# 而不是「传输很慢」——大文件复制几小时也不会误杀
DEFAULT_STALL_TIMEOUT = 120
# 强杀代理后等待它消失的宽限秒数,不能无限等待
_KILL_GRACE = 5
class FileSystemTimeout(OSError):
"""
文件系统操作在代理中超时未返回判定挂载无响应
继承 OSError 是刻意的整理链监控 watcher 等各层对 OSError 已有完整的
退避重试与登记逻辑block 型故障经此转换后可以直接复用它们
"""
class FileSystemProxy:
"""
常驻子进程文件系统代理
请求-响应严格串行一个代理同时只处理一个请求由锁保证超时即强杀代理
下一次请求自动重启一个新的启动成本是毫秒级因为 worker 只依赖标准库
"""
def __init__(self, timeout: Optional[float] = None,
stall_timeout: Optional[float] = None):
"""
:param timeout: 单次快操作的超时秒数None 表示实时跟随系统设置
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数
None 表示实时跟随系统设置
"""
self._timeout_override = timeout
self._stall_timeout_override = stall_timeout
self._process: Optional[subprocess.Popen] = None
self._selector: Optional[selectors.BaseSelector] = None
self._lock = threading.Lock()
# ------------------------------------------------------------------ #
# 对外操作
# ------------------------------------------------------------------ #
def stat(self, path: Path) -> Dict[str, Any]:
"""
读取路径属性
:param path: 目标路径
:return: {"size", "mtime", "is_dir", "is_file"}
"""
return self._call("stat", path=str(path))
def exists(self, path: Path) -> bool:
"""
判断路径是否存在
只有 FileNotFoundError 才算不存在其余 OSError含超时原样抛出
避免像 Path.exists() 那样把挂载抖动误判成文件消失
:param path: 目标路径
:return: 是否存在
"""
try:
self._call("exists", path=str(path))
return True
except FileNotFoundError:
return False
def listdir(self, path: Path) -> List[str]:
"""
列出目录条目名
:param path: 目标目录
:return: 条目名列表
"""
return self._call("listdir", path=str(path))
def rename(self, src: Path, dst: Path) -> bool:
"""
同一存储内重命名/移动跨存储会抛 OSError(EXDEV)由调用方走原有路径
:param src: 源路径
:param dst: 目标路径
:return: 是否成功
"""
return self._call("rename", src=str(src), dst=str(dst))
def copy(self, src: Path, dst: Path,
progress_cb: Optional[Callable[[float], None]] = None,
cancel_cb: Optional[Callable[[], bool]] = None,
chunk_size: Optional[int] = None) -> Any:
"""
复制文件内容并保留时间戳进度无推进判定挂死
复制大文件可能持续几小时固定超时无法区分正常但慢已经挂死
worker 每秒上报一次进度作为心跳这里判定的是**两次上报之间的间隔**
超过 stall 阈值收不到任何一行才认定挂载无响应并强杀 worker
取消检查放在父进程worker 里读不到 global_vars 的传输取消标记而父进程
每收到一次进度就能检查一次要取消直接杀掉 worker 即可比在子进程里
轮询标记更干净
:param src: 源文件
:param dst: 目标文件调用方应传临时名完成后自行原子替换
:param progress_cb: 进度回调入参为百分比
:param cancel_cb: 取消检查回调返回 True 表示应中止
:param chunk_size: 分块大小
:return: 成功时为 {"copied", "total"}被取消或通信失败时为 False
"""
payload = {"src": str(src), "dst": str(dst)}
if chunk_size:
payload["chunk_size"] = chunk_size
if not self._enabled():
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
with self._lock:
try:
return self._request_stream(payload, progress_cb, cancel_cb)
except FileSystemTimeout:
raise
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
self._shutdown()
return False
def _request_stream(self, payload: Dict[str, Any],
progress_cb: Optional[Callable[[float], None]],
cancel_cb: Optional[Callable[[], bool]]) -> Any:
"""
发起一次流式请求逐行消费进度直到终态
:param payload: 请求参数
:param progress_cb: 进度回调
:param cancel_cb: 取消检查回调
:return: 操作结果
"""
self._ensure_worker()
message = json.dumps({"op": "copy", **payload}) + "\n"
self._process.stdin.write(message.encode("utf-8"))
self._process.stdin.flush()
while True:
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
progress = response.get("progress")
if progress is not None:
if cancel_cb is not None and cancel_cb():
logger.info(f"复制已取消: {payload.get('src')}")
# 取消就地生效:杀掉 worker 立刻中断传输,不必等它读完整个文件
self._shutdown()
return False
if progress_cb is not None:
total = progress.get("total") or 0
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
continue
if response.get("ok"):
return response.get("result")
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
@staticmethod
def _direct_copy(src: Path, dst: Path,
progress_cb: Optional[Callable[[float], None]],
cancel_cb: Optional[Callable[[], bool]],
chunk_size: Optional[int]) -> bool:
"""
不经代理直接复制供代理关闭时使用
"""
info = os.stat(src)
total = info.st_size
copied = 0
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
while True:
if cancel_cb is not None and cancel_cb():
return False
buf = fsrc.read(chunk_size or 1024 * 1024)
if not buf:
break
fdst.write(buf)
copied += len(buf)
if progress_cb is not None and total:
progress_cb(copied / total * 100)
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
return True
def unlink(self, path: Path) -> bool:
"""
删除单个文件unlink 是原子操作强杀后没有中间状态
:param path: 目标文件
:return: 是否成功
"""
return self._call("unlink", path=str(path))
def rmtree(self, path: Path) -> bool:
"""
递归删除目录容忍部分失败可重复执行直到成功
:param path: 目标目录
:return: 是否成功
"""
return self._call("rmtree", path=str(path))
def close(self):
"""
关闭代理进程
"""
with self._lock:
self._shutdown()
# ------------------------------------------------------------------ #
# 内部实现
# ------------------------------------------------------------------ #
@property
def _timeout(self) -> float:
"""
单次快操作的超时秒数
实时读取而不是构造时固定这三项都暴露在前端设置里用户改完保存后
必须立刻生效否则会出现改了没反应的困惑
"""
if self._timeout_override is not None:
return self._timeout_override
return float(getattr(settings, "FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
@property
def _stall_timeout(self) -> float:
"""
长耗时操作两次进度上报之间的最长间隔秒数同样实时跟随系统设置
"""
if self._stall_timeout_override is not None:
return self._stall_timeout_override
return float(getattr(settings, "FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT))
@staticmethod
def _enabled() -> bool:
"""
代理是否启用关闭时退回直接调用行为与引入代理之前完全一致
"""
return bool(getattr(settings, "FS_PROXY_ENABLED", True))
@staticmethod
def _direct(op: str, payload: Dict[str, Any]) -> Any:
"""
不经代理直接执行操作供代理关闭时使用
:param op: 操作名
:param payload: 操作参数
:return: 操作结果
"""
if op == "stat":
path = payload["path"]
info = os.stat(path)
return {
"size": info.st_size,
"mtime": info.st_mtime,
"is_dir": os.path.isdir(path),
"is_file": os.path.isfile(path),
}
if op == "exists":
os.stat(payload["path"])
return True
if op == "listdir":
return sorted(os.listdir(payload["path"]))
if op == "rename":
os.rename(payload["src"], payload["dst"])
return True
if op == "unlink":
os.unlink(payload["path"])
return True
if op == "rmtree":
shutil.rmtree(payload["path"], ignore_errors=True)
return True
raise ValueError(f"unknown op: {op}")
def _call(self, op: str, **payload) -> Any:
"""
执行一次代理调用
:param op: 操作名
:param payload: 操作参数
:return: 操作结果
"""
if not self._enabled():
return self._direct(op, payload)
with self._lock:
try:
return self._request(op, payload)
except FileSystemTimeout:
# 超时说明挂载正在挂死,重试只会再冻一次,直接上报给调用方
raise
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
# 代理进程意外退出或响应损坏,重启后重试一次
logger.debug(f"文件系统代理通信异常,重启后重试: {op} - {err}")
self._shutdown()
return self._request(op, payload)
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
"""
发送请求并等待响应
:param op: 操作名
:param payload: 操作参数
:return: 操作结果
"""
self._ensure_worker()
message = json.dumps({"op": op, **payload}) + "\n"
self._process.stdin.write(message.encode("utf-8"))
self._process.stdin.flush()
response = json.loads(self._read_line().decode("utf-8"))
if response.get("ok"):
return response.get("result")
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类,
# 调用方沿用原有的异常分支即可,无需感知代理的存在
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
def _read_line(self, timeout: Optional[float] = None) -> bytes:
"""
读取一行响应超时即强杀代理
:param timeout: 本次读取的超时秒数默认用单次操作超时
:return: 响应行
"""
timeout = self._timeout if timeout is None else timeout
if not self._selector.select(timeout=timeout):
logger.error(f"文件系统操作 {timeout} 秒无响应,判定挂载挂死,正在回收代理进程")
self._shutdown()
raise FileSystemTimeout(
errno_module.ETIMEDOUT,
f"文件系统操作超过 {timeout} 秒无响应,挂载可能已无响应"
)
line = self._process.stdout.readline()
if not line:
raise BrokenPipeError("文件系统代理进程已退出")
return line
def _ensure_worker(self):
"""
确保代理进程可用不可用时重新启动
"""
if self._process is not None and self._process.poll() is None:
return
self._shutdown()
self._process = subprocess.Popen(
[sys.executable, str(_WORKER_PATH)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.DEVNULL,
bufsize=0,
)
self._selector = selectors.DefaultSelector()
self._selector.register(self._process.stdout, selectors.EVENT_READ)
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
def _shutdown(self):
"""
回收代理进程冻在挂载上的进程用 SIGKILL且不无限等待它消失
否则可放弃的代理又变回一次不可放弃的阻塞
"""
if self._selector is not None:
try:
self._selector.close()
except Exception: # noqa: BLE001
pass
self._selector = None
process, self._process = self._process, None
if process is None:
return
for stream in (process.stdin, process.stdout):
try:
if stream:
stream.close()
except Exception: # noqa: BLE001
pass
if process.poll() is not None:
return
try:
process.kill()
process.wait(timeout=_KILL_GRACE)
except subprocess.TimeoutExpired:
logger.warn(f"文件系统代理进程未能及时退出,交由系统回收: pid={process.pid}")
except Exception as err: # noqa: BLE001
logger.debug(f"回收文件系统代理进程失败: {err}")
# 全局单例:local 存储本身是单例,代理也只需要一个
# 不传超时参数:让它实时跟随系统设置,前端改完保存即刻生效
fsproxy = FileSystemProxy()
+180
View File
@@ -0,0 +1,180 @@
"""
文件系统操作代理 worker
**本文件不能被 import只能作为独立脚本执行**fsproxy
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动直接执行文件
路径不会触发 `app/__init__.py` 的导入链因此这个进程只依赖标准库启动是
毫秒级的一旦走 import 就会把整个应用的依赖拉进来代理被强杀后的重启成本
会高到无法接受
存在的理由FUSE/网络挂载进入 block 型故障时`stat`/`listdir`/`rename` 这类
系统调用既不返回错误也不返回结果 Python 没有中断线程的手段阻塞其上的
线程永远无法回收放进独立进程后父进程可以在超时后 SIGKILL 掉它
不可处理的 block转换成可处理的 crash
协议stdin/stdout 逐行 JSON
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
成功 {"ok": true, "result": {...}}
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
"""
import json
import os
import shutil
import sys
import time
def _stat(payload, _emit):
"""
读取路径的基本属性
"""
path = payload["path"]
info = os.stat(path)
return {
"size": info.st_size,
"mtime": info.st_mtime,
"is_dir": os.path.isdir(path),
"is_file": os.path.isfile(path),
}
def _exists(payload, _emit):
"""
判断路径是否存在
os.stat 而不是 os.path.exists后者会把任意 OSError 都归为不存在
挂载抖动会被误判成文件消失这里让异常原样抛出由父进程按 errno 区分
"""
os.stat(payload["path"])
return True
def _listdir(payload, _emit):
"""
列出目录下的条目名
"""
return sorted(os.listdir(payload["path"]))
def _copy(payload, emit):
"""
分块复制文件内容并周期上报进度
进度上报同时充当心跳复制大文件可能持续几小时父进程无法用固定超时判断
挂死只能看两次上报之间隔了多久因此这里按固定时间间隔上报即使
某一秒没读到数据也照常发一旦挂载卡住read/write 不返回上报自然断流
父进程据此判定并强杀本进程
只复制内容和时间戳不复制权限目标目录的默认权限与继承 ACL 是媒体库的
访问策略用源文件权限覆盖会清除已继承的 ACL
"""
src, dst = payload["src"], payload["dst"]
chunk_size = payload.get("chunk_size") or 1024 * 1024
interval = payload.get("progress_interval") or 1.0
info = os.stat(src)
total = info.st_size
copied = 0
# 先发一次 0%:既让心跳立刻开始,也保证父进程在传输开始前就有一次检查
# 取消的机会——否则小文件会在首次定时上报之前就复制完,取消形同虚设
emit({"ok": True, "progress": {"copied": 0, "total": total}})
last_emit = time.monotonic()
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
while True:
buf = fsrc.read(chunk_size)
if not buf:
break
fdst.write(buf)
copied += len(buf)
now = time.monotonic()
if now - last_emit >= interval:
last_emit = now
emit({"ok": True, "progress": {"copied": copied, "total": total}})
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
return {"copied": copied, "total": total}
def _rename(payload, _emit):
"""
同一存储内重命名/移动
这是第一版唯一放行的写操作同文件系统内的 rename 由内核保证原子性
进程被强杀后要么完全成功要么完全没发生不存在需要清理的中间状态
跨存储的复制+删除不走这里它需要单独的可恢复语义
"""
src, dst = payload["src"], payload["dst"]
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
raise OSError(18, "Cross-device rename is not handled by the proxy")
os.rename(src, dst)
return True
def _unlink(payload, _emit):
"""
删除单个文件unlink 是原子操作强杀后要么删掉了要么没删没有中间状态
"""
os.unlink(payload["path"])
return True
def _rmtree(payload, _emit):
"""
递归删除目录
这一项不是原子的强杀可能只删掉一部分放行的理由是删除被中断的后果
残留若干文件远轻于写入被中断留下叫最终文件名的半成品而且调用方
本来就以 ignore_errors 容忍部分失败可以重复执行直到成功
"""
shutil.rmtree(payload["path"], ignore_errors=True)
return True
_HANDLERS = {
"stat": _stat,
"exists": _exists,
"listdir": _listdir,
"copy": _copy,
"rename": _rename,
"unlink": _unlink,
"rmtree": _rmtree,
"ping": lambda _payload, _emit: True,
}
def _write(message):
"""
输出一行响应
"""
sys.stdout.write(json.dumps(message) + "\n")
sys.stdout.flush()
def main():
"""
请求循环每读一行处理一个请求直到 stdin 关闭
一个请求可能对应多行响应长耗时操作先流式发若干 progress 最后发一行
终态result error父进程据此区分还在推进已经挂死
"""
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
payload = json.loads(line)
handler = _HANDLERS.get(payload.get("op"))
if handler is None:
response = {"ok": False, "errno": 0,
"error": f"unknown op: {payload.get('op')}"}
else:
response = {"ok": True, "result": handler(payload, _write)}
except OSError as err:
response = {"ok": False, "errno": err.errno or 0,
"error": err.strerror or str(err)}
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
response = {"ok": False, "errno": 0, "error": str(err)}
_write(response)
if __name__ == "__main__":
main()
+8 -2
View File
@@ -8,6 +8,7 @@ from app import schemas
from app.helper.progress import ProgressHelper from app.helper.progress import ProgressHelper
from app.helper.storage import StorageHelper from app.helper.storage import StorageHelper
from app.log import logger from app.log import logger
from app.schemas.exception import StorageQueryError
from app.utils.crypto import HashUtils from app.utils.crypto import HashUtils
@@ -179,9 +180,13 @@ class StorageBase(metaclass=ABCMeta):
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]: def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
""" """
获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError 获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError
默认实现不区分不存在查询失败由具体存储按需覆写
默认保守失败未覆写的存储无法区分不存在查询失败沿用
get_item() 会让 overwrite_mode=size 的覆盖保护在查询失败时被绕过
无法确认当成目标不存在而放行覆盖具体存储必须先实现
确认不存在的判定再覆写本方法
""" """
return self.get_item(path) raise StorageQueryError(f"存储 {self.schema} 未实现严格查询,无法确认目标状态: {path}")
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]: def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
""" """
@@ -337,6 +342,7 @@ class StorageBase(metaclass=ABCMeta):
files_info[_fileitm.path] = { files_info[_fileitm.path] = {
'size': _fileitm.size or 0, 'size': _fileitm.size or 0,
'modify_time': getattr(_fileitm, 'modify_time', 0), 'modify_time': getattr(_fileitm, 'modify_time', 0),
'fileid': getattr(_fileitm, 'fileid', None),
'type': _fileitm.type 'type': _fileitm.type
} }
+49 -9
View File
@@ -10,7 +10,7 @@ from app.core.cache import cached
from app.core.config import settings, global_vars from app.core.config import settings, global_vars
from app.log import logger from app.log import logger
from app.modules.filemanager.storages import StorageBase, transfer_process from app.modules.filemanager.storages import StorageBase, transfer_process
from app.schemas.exception import OperationInterrupted from app.schemas.exception import OperationInterrupted, StorageQueryError
from app.schemas.types import StorageSchema from app.schemas.types import StorageSchema
from app.utils.http import RequestUtils from app.utils.http import RequestUtils
from app.utils.singleton import WeakSingleton from app.utils.singleton import WeakSingleton
@@ -471,18 +471,58 @@ class Alist(StorageBase, metaclass=WeakSingleton):
) )
return None return None
return self.__build_fileitem(path, result["data"])
def __build_fileitem(self, path: Path, data: dict) -> schemas.FileItem:
"""
根据接口返回数据构建文件项
:param path: 文件路径
:param data: 接口返回的 data 字段
:return: 文件项
"""
return schemas.FileItem( return schemas.FileItem(
storage=self.schema.value, storage=self.schema.value,
type="dir" if result["data"]["is_dir"] else "file", type="dir" if data["is_dir"] else "file",
path=path.as_posix() + ("/" if result["data"]["is_dir"] else ""), path=path.as_posix() + ("/" if data["is_dir"] else ""),
name=result["data"]["name"], name=data["name"],
basename=Path(result["data"]["name"]).stem, basename=Path(data["name"]).stem,
extension=Path(result["data"]["name"]).suffix[1:], extension=Path(data["name"]).suffix[1:],
size=result["data"]["size"], size=data["size"],
modify_time=self.__parse_timestamp(result["data"]["modified"]), modify_time=self.__parse_timestamp(data["modified"]),
thumbnail=result["data"]["thumb"], thumbnail=data["thumb"],
) )
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError
只有接口明确回报对象不存在才是确定结果连接失败HTTP 异常与其他
业务错误都无法确认目标状态必须保守失败以免覆盖保护被绕过
"""
resp = RequestUtils(headers=self.__get_header_with_token()).post_res(
self.__get_api_url("/api/fs/get"),
json={
"path": path.as_posix(),
"password": "",
"page": 1,
"per_page": 0,
"refresh": False,
},
)
if resp is None:
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,无法连接服务")
if resp.status_code != 200:
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,状态码:{resp.status_code}")
try:
result = resp.json()
except Exception as err:
raise StorageQueryError(f"【OpenList】解析查询结果失败: {path} - {err}") from err
if result.get("code") != 200:
message = str(result.get("message") or "")
if "not found" in message.lower() or "not exist" in message.lower():
return None
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败:{message}")
return self.__build_fileitem(path, result["data"])
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]: def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
""" """
获取父目录 获取父目录
+145 -46
View File
@@ -1,5 +1,6 @@
import os import os
import shutil import shutil
import time
from pathlib import Path from pathlib import Path
from typing import Optional, List from typing import Optional, List
@@ -7,6 +8,7 @@ from app import schemas
from app.core.config import global_vars, settings from app.core.config import global_vars, settings
from app.helper.directory import DirectoryHelper from app.helper.directory import DirectoryHelper
from app.log import logger from app.log import logger
from app.modules.filemanager.fsproxy import fsproxy
from app.modules.filemanager.storages import StorageBase, transfer_process from app.modules.filemanager.storages import StorageBase, transfer_process
from app.schemas.exception import StorageQueryError from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema from app.schemas.types import StorageSchema
@@ -47,6 +49,10 @@ class LocalStorage(StorageBase):
""" """
获取文件项 获取文件项
""" """
# 走代理读取:挂载挂死时这一步会在超时后抛 OSError,而不是永久悬挂线程。
# 顺带只 stat 一次——原先 size 与 modify_time 各 stat 一次,在网络挂载上
# 等于把这个热点路径的开销翻倍
info = fsproxy.stat(path)
return schemas.FileItem( return schemas.FileItem(
storage=self.schema.value, storage=self.schema.value,
type="file", type="file",
@@ -54,8 +60,8 @@ class LocalStorage(StorageBase):
name=path.name, name=path.name,
basename=path.stem, basename=path.stem,
extension=path.suffix[1:], extension=path.suffix[1:],
size=path.stat().st_size, size=info["size"],
modify_time=path.stat().st_mtime, modify_time=info["mtime"],
) )
def __get_diritem(self, path: Path) -> schemas.FileItem: def __get_diritem(self, path: Path) -> schemas.FileItem:
@@ -68,7 +74,7 @@ class LocalStorage(StorageBase):
path=path.as_posix() + "/", path=path.as_posix() + "/",
name=path.name, name=path.name,
basename=path.stem, basename=path.stem,
modify_time=path.stat().st_mtime, modify_time=fsproxy.stat(path)["mtime"],
) )
def list(self, fileitem: schemas.FileItem) -> List[schemas.FileItem]: def list(self, fileitem: schemas.FileItem) -> List[schemas.FileItem]:
@@ -100,12 +106,14 @@ class LocalStorage(StorageBase):
# 遍历目录 # 遍历目录
path_obj = Path(path) path_obj = Path(path)
if not path_obj.exists(): try:
info = fsproxy.stat(path_obj)
except (FileNotFoundError, NotADirectoryError):
logger.warn(f"【本地】目录不存在:{path}") logger.warn(f"【本地】目录不存在:{path}")
return [] return []
# 如果是文件 # 如果是文件
if path_obj.is_file(): if info["is_file"]:
ret_items.append(self.__get_fileitem(path_obj)) ret_items.append(self.__get_fileitem(path_obj))
return ret_items return ret_items
@@ -143,9 +151,11 @@ class LocalStorage(StorageBase):
""" """
获取文件或目录不存在返回None 获取文件或目录不存在返回None
""" """
if not path.exists(): try:
info = fsproxy.stat(path)
except (FileNotFoundError, NotADirectoryError):
return None return None
if path.is_file(): if info["is_file"]:
return self.__get_fileitem(path) return self.__get_fileitem(path)
return self.__get_diritem(path) return self.__get_diritem(path)
@@ -154,9 +164,11 @@ class LocalStorage(StorageBase):
获取文件或目录无法确认状态时抛出 StorageQueryError 获取文件或目录无法确认状态时抛出 StorageQueryError
Path.exists() 会把部分 errno EBADF/ELOOP归入不存在 Path.exists() 会把部分 errno EBADF/ELOOP归入不存在
网络/FUSE 挂载抖动时会误判这里用 stat 显式区分 网络/FUSE 挂载抖动时会误判这里用 stat 显式区分
挂载完全无响应时代理会超时并抛 FileSystemTimeoutOSError 子类
同样落入下面的分支转化成调用方能处理的查询失败
""" """
try: try:
path.stat() fsproxy.stat(path)
except (FileNotFoundError, NotADirectoryError): except (FileNotFoundError, NotADirectoryError):
return None return None
except OSError as e: except OSError as e:
@@ -182,13 +194,18 @@ class LocalStorage(StorageBase):
if not fileitem.path: if not fileitem.path:
return False return False
path_obj = Path(fileitem.path) path_obj = Path(fileitem.path)
if not path_obj.exists():
return True
try: try:
if path_obj.is_file(): info = fsproxy.stat(path_obj)
path_obj.unlink() except (FileNotFoundError, NotADirectoryError):
return True
except OSError as e:
logger.error(f"【本地】读取待删除文件状态失败:{e}")
return False
try:
if info["is_file"]:
fsproxy.unlink(path_obj)
else: else:
shutil.rmtree(path_obj, ignore_errors=True) fsproxy.rmtree(path_obj)
except Exception as e: except Exception as e:
logger.error(f"【本地】删除文件失败:{e}") logger.error(f"【本地】删除文件失败:{e}")
return False return False
@@ -199,10 +216,10 @@ class LocalStorage(StorageBase):
重命名文件 重命名文件
""" """
path_obj = Path(fileitem.path) path_obj = Path(fileitem.path)
if not path_obj.exists():
return False
try: try:
path_obj.rename(path_obj.parent / name) fsproxy.rename(path_obj, path_obj.parent / name)
except (FileNotFoundError, NotADirectoryError):
return False
except Exception as e: except Exception as e:
logger.error(f"【本地】重命名文件失败:{e}") logger.error(f"【本地】重命名文件失败:{e}")
return False return False
@@ -214,6 +231,93 @@ class LocalStorage(StorageBase):
""" """
return Path(fileitem.path) return Path(fileitem.path)
# 写入中的临时文件后缀。点开头(隐藏)+ 专用后缀双重保证:即使进程被
# SIGKILL、临时文件残留,媒体库也不会把半成品当成媒体收录
PARTIAL_SUFFIX = ".mp-partial"
# 临时文件被认定为中断残留的时长(秒)。正常失败路径会自行清理,只有被
# 强杀才会残留;阈值取得宽松,避免误删仍在写入的大文件
PARTIAL_STALE_SECONDS = 24 * 3600
@classmethod
def _partial_path(cls, dest: Path) -> Path:
"""
生成写入中的临时文件路径
必须与目标同目录os.replace 只有在同一文件系统内才是原子的放到
/tmp 之类的地方会退化成一次完整拷贝原子性荡然无存 PID 是为了
避免多进程同时写同一目标时互相踩踏
:param dest: 目标文件路径
:return: 临时文件路径
"""
return dest.parent / f".{dest.name}.{os.getpid()}{cls.PARTIAL_SUFFIX}"
@classmethod
def _cleanup_stale_partials(cls, directory: Path):
"""
清理目录下中断残留的临时文件
只做局部清理而不是全库扫描在网络挂载上遍历整个媒体库代价不可接受
而残留只可能出现在曾经写入过的目录里因此每次写入时顺带清理即可
本方法是尽力而为的旁路操作任何失败都不影响主流程
:param directory: 目标目录
"""
try:
threshold = time.time() - cls.PARTIAL_STALE_SECONDS
for item in directory.glob(f"*{cls.PARTIAL_SUFFIX}"):
try:
if item.stat().st_mtime < threshold:
item.unlink()
logger.info(f"【本地】已清理中断残留的临时文件:{item}")
except OSError:
continue
except Exception as err:
logger.debug(f"【本地】清理临时文件失败:{directory} - {err}")
def _write_atomically(self, src: Path, dest: Path) -> bool:
"""
写临时名 os.replace的方式把源文件内容落到目标
直接写目标路径的话进程被杀OOM重启宿主断电SIGKILL会在媒体库
里留下一个**叫最终文件名的半截文件**媒体库会把它扫进去后续的
目标已存在判断也会把它当成完成品os.replace 在同目录内由内核保证
原子性因此目标要么完整存在要么根本不存在
:param src: 源文件路径
:param dest: 目标文件路径
:return: 是否成功
"""
self._cleanup_stale_partials(dest.parent)
partial = self._partial_path(dest)
# 进度只在需要展示时才回调 UI,但代理内部始终按固定间隔上报——那是判定
# 「传输是否还在推进」的心跳,不能因为不展示进度就关掉
progress_callback = (
transfer_process(src.as_posix())
if self.__should_show_progress(src, dest) else None
)
try:
copied = fsproxy.copy(
src, partial,
progress_cb=progress_callback,
cancel_cb=lambda: global_vars.is_transfer_stopped(src.as_posix()),
chunk_size=self.chunk_size,
)
if not copied:
logger.info(f"【本地】{src} 复制未完成")
return False
os.replace(partial, dest)
return True
except Exception as err:
logger.error(f"【本地】复制文件失败:{err}")
return False
finally:
if progress_callback:
progress_callback(100)
# 失败路径留下的临时文件就地清掉;成功时 replace 已经把它移走
try:
if partial.exists():
partial.unlink()
except OSError:
pass
@staticmethod @staticmethod
def _copy_with_target_permissions(src: Path, dest: Path) -> Path: def _copy_with_target_permissions(src: Path, dest: Path) -> Path:
""" """
@@ -276,12 +380,13 @@ class LocalStorage(StorageBase):
try: try:
dir_path = Path(fileitem.path) dir_path = Path(fileitem.path)
target_path = dir_path / (new_name or path.name) target_path = dir_path / (new_name or path.name)
if self._copy_with_progress(path, target_path): # 先原子地把内容落到目标,确认完整之后才删源
if self._write_atomically(path, target_path):
# 上传删除源文件 # 上传删除源文件
path.unlink() path.unlink()
return self.get_item(target_path) return self.get_item(target_path)
except Exception as err: except Exception as err:
logger.error(f"【本地】移动文件失败:{err}") logger.error(f"【本地】上传文件失败:{err}")
return None return None
@staticmethod @staticmethod
@@ -304,18 +409,7 @@ class LocalStorage(StorageBase):
""" """
复制文件带进度 复制文件带进度
""" """
try: return self._write_atomically(Path(fileitem.path), path / new_name)
src = Path(fileitem.path)
dest = path / new_name
if self.__should_show_progress(src, dest):
if self._copy_with_progress(src, dest):
return True
else:
self._copy_with_target_permissions(src, dest)
return True
except Exception as err:
logger.error(f"【本地】复制文件失败:{err}")
return False
def move( def move(
self, self,
@@ -326,23 +420,28 @@ class LocalStorage(StorageBase):
""" """
移动文件带进度 移动文件带进度
""" """
src = Path(fileitem.path)
dest = path / new_name
if src == dest:
# 目标和源文件相同,直接返回成功,不做任何操作
return True
try: try:
src = Path(fileitem.path) # 同一文件系统内 rename 是原子操作:中断后要么完全成功、要么完全
dest = path / new_name # 没发生,既不需要临时文件也不会留下半成品。直接尝试而不预先比较
if src == dest: # st_dev,省掉挂载上的两次 stat——跨设备会以 EXDEV 失败并落到下面
# 目标和源文件相同,直接返回成功,不做任何操作 os.replace(src, dest)
return True return True
if self.__should_show_progress(src, dest): except OSError as err:
if self._copy_with_progress(src, dest): logger.debug(f"【本地】直接移动未成功,降级为复制:{src} -> {dest} - {err}")
# 复制成功删除源文件 # 跨文件系统:先原子地把内容落到目标,确认完整之后才删源。
src.unlink() # 顺序不能反——先删源再失败就是永久丢件
return True if not self._write_atomically(src, dest):
else: return False
shutil.move(src, dest, copy_function=self._copy_with_target_permissions) try:
return True src.unlink()
except Exception as err: except OSError as err:
logger.error(f"【本地】移动文件失败:{err}") logger.warn(f"【本地】移动已完成但删除源文件失败:{src} - {err}")
return False return True
def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool: def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool:
""" """
@@ -10,6 +10,7 @@ from app import schemas
from app.core.config import settings from app.core.config import settings
from app.log import logger from app.log import logger
from app.modules.filemanager.storages import StorageBase, transfer_process from app.modules.filemanager.storages import StorageBase, transfer_process
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema from app.schemas.types import StorageSchema
from app.utils.string import StringUtils from app.utils.string import StringUtils
from app.utils.system import SystemUtils from app.utils.system import SystemUtils
@@ -299,6 +300,38 @@ class Rclone(StorageBase):
logger.debug(f"【rclone】获取文件项失败:{err}") logger.debug(f"【rclone】获取文件项失败:{err}")
return None return None
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError
rclone 用退出码 3/4 表示目录/文件不存在其余非零退出无法区分
不存在查询失败必须保守失败以免覆盖保护被绕过
"""
try:
ret = subprocess.run(
[
'rclone', 'lsjson',
f'MP:{path.parent}'
],
capture_output=True,
startupinfo=self.__get_hidden_shell()
)
except Exception as err:
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {err}") from err
if ret.returncode in (3, 4):
# 目录或文件不存在,是确定结果
return None
if ret.returncode != 0:
errmsg = (ret.stderr or b"").decode(errors="ignore").strip()
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {errmsg}")
try:
items = json.loads(ret.stdout)
except Exception as err:
raise StorageQueryError(f"【rclone】解析查询结果失败: {path} - {err}") from err
for item in items:
if item.get("Name") == path.name:
return self.__get_rcloneitem(item, parent=str(path.parent) + "/")
return None
def delete(self, fileitem: schemas.FileItem) -> bool: def delete(self, fileitem: schemas.FileItem) -> bool:
""" """
删除文件 删除文件
+37 -1
View File
@@ -1,3 +1,4 @@
import errno
import threading import threading
import time import time
from pathlib import Path from pathlib import Path
@@ -16,6 +17,7 @@ from app.core.config import settings, global_vars
from app.log import logger from app.log import logger
from app.modules.filemanager import StorageBase from app.modules.filemanager import StorageBase
from app.modules.filemanager.storages import transfer_process from app.modules.filemanager.storages import transfer_process
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema from app.schemas.types import StorageSchema
from app.utils.singleton import WeakSingleton from app.utils.singleton import WeakSingleton
@@ -163,7 +165,8 @@ class SMB(StorageBase, metaclass=WeakSingleton):
# 构建完整的SMB路径 # 构建完整的SMB路径
if path_str: if path_str:
return f"{self._server_path}\\{path_str.replace('/', '\\')}" normalized_path = path_str.replace("/", "\\")
return f"{self._server_path}\\{normalized_path}"
else: else:
return self._server_path return self._server_path
@@ -379,6 +382,39 @@ class SMB(StorageBase, metaclass=WeakSingleton):
logger.debug(f"【SMB】获取文件项失败: {e}") logger.debug(f"【SMB】获取文件项失败: {e}")
return None return None
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
"""
获取文件或目录确认不存在返回None无法确认状态时抛出 StorageQueryError
只有 ENOENT/ENOTDIR 才是确认不存在连接中断认证失败等都无法确认
目标状态必须保守失败以免覆盖保护被绕过
"""
try:
self._check_connection()
# 处理根目录
if str(path) == "/":
return schemas.FileItem(
storage=self.schema.value,
type="dir",
path="/",
name="",
basename="",
modify_time=int(time.time()),
)
smb_path = self._normalize_path(str(path).rstrip("/"))
try:
stat_result = smbclient.stat(smb_path)
except OSError as err:
if err.errno in (errno.ENOENT, errno.ENOTDIR):
return None
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {err}") from err
return self._create_fileitem(stat_result, smb_path, Path(path).name)
except StorageQueryError:
raise
except Exception as e:
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {e}") from e
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]: def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
""" """
获取文件详情 获取文件详情
+3
View File
@@ -545,6 +545,7 @@ class TransHandler:
fail_list=[fileitem.path], fail_list=[fileitem.path],
transfer_type=transfer_type, transfer_type=transfer_type,
need_notify=need_notify, need_notify=need_notify,
overwrite_skipped=True,
) )
return result return result
elif overwrite_mode == "always": elif overwrite_mode == "always":
@@ -571,6 +572,7 @@ class TransHandler:
fail_list=[fileitem.path], fail_list=[fileitem.path],
transfer_type=transfer_type, transfer_type=transfer_type,
need_notify=need_notify, need_notify=need_notify,
overwrite_skipped=True,
) )
return result return result
else: else:
@@ -614,6 +616,7 @@ class TransHandler:
fail_list=[fileitem.path], fail_list=[fileitem.path],
transfer_type=transfer_type, transfer_type=transfer_type,
need_notify=need_notify, need_notify=need_notify,
overwrite_skipped=True,
) )
return result return result
elif overwrite_mode == "latest": elif overwrite_mode == "latest":
+71 -14
View File
@@ -13,6 +13,7 @@ from app.modules.themoviedb.category import CategoryHelper
from app.modules.themoviedb.scraper import TmdbScraper from app.modules.themoviedb.scraper import TmdbScraper
from app.modules.themoviedb.tmdb_cache import TmdbCache from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.modules.themoviedb.tmdbapi import TmdbApi from app.modules.themoviedb.tmdbapi import TmdbApi
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbConnectionError
from app.schemas.category import CategoryConfig from app.schemas.category import CategoryConfig
from app.schemas.types import ( from app.schemas.types import (
MediaImageType, MediaImageType,
@@ -193,16 +194,42 @@ class TheMovieDbModule(_ModuleBase):
media.season = meta.begin_season media.season = meta.begin_season
return medias return medias
def _safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
"""
查询指定类型的媒体详情"确认TMDB连接失败""确认查无此项"区分开
:param mtype: 媒体类型电影或电视剧
:param tmdbid: TMDB的ID
:return: (媒体信息或None, 本次查询是否因TMDB连接失败而没有得到确定结果)
"""
try:
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
except TMDbConnectionError:
return None, True
async def _async_safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
"""
查询指定类型的媒体详情"确认TMDB连接失败""确认查无此项"区分开异步版本
"""
try:
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
except TMDbConnectionError:
return None, True
def _get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType], def _get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
meta: Optional[MetaBase]) -> Optional[dict]: meta: Optional[MetaBase]) -> Optional[dict]:
""" """
根据tmdbid查询媒体信息当类型未知且同时存在电影和电视剧时通过元数据消歧 根据tmdbid查询媒体信息当类型未知且同时存在电影和电视剧时通过元数据消歧
:raises TMDbConnectionError: 电影电视剧两路查询都没有得到确定结果且至少一路
是因TMDB连接失败导致的此时不能断言"条目不存在"交由上层报网络故障
""" """
if mtype: if mtype:
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid) return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
# 类型未知,分别查询电影和电视剧 # 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
info_tv = self.tmdb.get_info(mtype=MediaType.TV, tmdbid=tmdbid) # 避免一路瞬时抖动掩盖另一路已经得到的确定结果
info_movie = self.tmdb.get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid) info_tv, tv_conn_error = self._safe_get_info_by_type(MediaType.TV, tmdbid)
info_movie, movie_conn_error = self._safe_get_info_by_type(MediaType.MOVIE, tmdbid)
if info_tv and info_movie: if info_tv and info_movie:
# 同时存在,尝试通过元数据消歧 # 同时存在,尝试通过元数据消歧
result = self._disambiguate_by_meta(info_tv, info_movie, meta) result = self._disambiguate_by_meta(info_tv, info_movie, meta)
@@ -210,18 +237,26 @@ class TheMovieDbModule(_ModuleBase):
return result return result
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧") logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
return None return None
return info_tv or info_movie or None if info_tv or info_movie:
return info_tv or info_movie
if tv_conn_error or movie_conn_error:
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
return None
async def _async_get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType], async def _async_get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
meta: Optional[MetaBase]) -> Optional[dict]: meta: Optional[MetaBase]) -> Optional[dict]:
""" """
根据tmdbid查询媒体信息当类型未知且同时存在电影和电视剧时通过元数据消歧异步版本 根据tmdbid查询媒体信息当类型未知且同时存在电影和电视剧时通过元数据消歧异步版本
:raises TMDbConnectionError: 电影电视剧两路查询都没有得到确定结果且至少一路
是因TMDB连接失败导致的此时不能断言"条目不存在"交由上层报网络故障
""" """
if mtype: if mtype:
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid) return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
# 类型未知,分别查询电影和电视剧 # 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
info_tv = await self.tmdb.async_get_info(mtype=MediaType.TV, tmdbid=tmdbid) # 避免一路瞬时抖动掩盖另一路已经得到的确定结果
info_movie = await self.tmdb.async_get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid) info_tv, tv_conn_error = await self._async_safe_get_info_by_type(MediaType.TV, tmdbid)
info_movie, movie_conn_error = await self._async_safe_get_info_by_type(MediaType.MOVIE, tmdbid)
if info_tv and info_movie: if info_tv and info_movie:
# 同时存在,尝试通过元数据消歧 # 同时存在,尝试通过元数据消歧
result = self._disambiguate_by_meta(info_tv, info_movie, meta) result = self._disambiguate_by_meta(info_tv, info_movie, meta)
@@ -229,7 +264,11 @@ class TheMovieDbModule(_ModuleBase):
return result return result
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧") logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
return None return None
return info_tv or info_movie or None if info_tv or info_movie:
return info_tv or info_movie
if tv_conn_error or movie_conn_error:
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
return None
@staticmethod @staticmethod
def _disambiguate_by_meta(info_tv: dict, info_movie: dict, def _disambiguate_by_meta(info_tv: dict, info_movie: dict,
@@ -525,10 +564,15 @@ class TheMovieDbModule(_ModuleBase):
# 识别匹配 # 识别匹配
if not cache_info or not cache: if not cache_info or not cache:
info = None info = None
connection_error = False
# 缓存没有或者强制不使用缓存 # 缓存没有或者强制不使用缓存
if tmdbid: if tmdbid:
# 直接查询详情,支持同ID电影/电视剧消歧 # 直接查询详情,支持同ID电影/电视剧消歧
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta) try:
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
except TMDbConnectionError as err:
logger.error(f"tmdb_id:{tmdbid} {err}")
connection_error = True
if not info and meta and not tmdbid: if not info and meta and not tmdbid:
# 准备搜索名称 # 准备搜索名称
names = self._prepare_search_names(meta) names = self._prepare_search_names(meta)
@@ -542,7 +586,11 @@ class TheMovieDbModule(_ModuleBase):
info = self.tmdb.get_info(mtype=info.get("media_type"), info = self.tmdb.get_info(mtype=info.get("media_type"),
tmdbid=info.get("id")) tmdbid=info.get("id"))
elif not info: elif not info:
if tmdbid: if connection_error:
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
elif tmdbid:
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败") logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
else: else:
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid") logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
@@ -624,10 +672,15 @@ class TheMovieDbModule(_ModuleBase):
# 识别匹配 # 识别匹配
if not cache_info or not cache: if not cache_info or not cache:
info = None info = None
connection_error = False
# 缓存没有或者强制不使用缓存 # 缓存没有或者强制不使用缓存
if tmdbid: if tmdbid:
# 直接查询详情,支持同ID电影/电视剧消歧 # 直接查询详情,支持同ID电影/电视剧消歧
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta) try:
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
except TMDbConnectionError as err:
logger.error(f"tmdb_id:{tmdbid} {err}")
connection_error = True
if not info and meta and not tmdbid: if not info and meta and not tmdbid:
# 准备搜索名称 # 准备搜索名称
names = self._prepare_search_names(meta) names = self._prepare_search_names(meta)
@@ -641,7 +694,11 @@ class TheMovieDbModule(_ModuleBase):
info = await self.tmdb.async_get_info(mtype=info.get("media_type"), info = await self.tmdb.async_get_info(mtype=info.get("media_type"),
tmdbid=info.get("id")) tmdbid=info.get("id"))
elif not info: elif not info:
if tmdbid: if connection_error:
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
elif tmdbid:
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败") logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
else: else:
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid") logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
+41 -1
View File
@@ -3,6 +3,7 @@ import traceback
from math import ceil from math import ceil
from threading import RLock from threading import RLock
from time import time from time import time
from typing import Any
from app.core.cache import FileCache, TTLCache from app.core.cache import FileCache, TTLCache
from app.core.config import settings from app.core.config import settings
@@ -144,6 +145,30 @@ class TmdbCache(metaclass=WeakSingleton):
media_id = meta.media_id if meta.media_source == MediaSource.TMDB else None media_id = meta.media_id if meta.media_source == MediaSource.TMDB else None
return f"[{meta.type.value if meta.type else '未知'}][{settings.TMDB_LOCALE}]{media_id or meta.name}-{meta.year}-{meta.begin_season}" return f"[{meta.type.value if meta.type else '未知'}][{settings.TMDB_LOCALE}]{media_id or meta.name}-{meta.year}-{meta.begin_season}"
@staticmethod
def __is_type_conflicted(meta: MetaBase, media_type: Any, tmdb_id: Any) -> bool:
"""
判断媒体类型是否与元数据声明的类型冲突
只有元数据判定为电视剧结果却是电影才算冲突反向不算名称识别在
电影分支查不到时会回退到电视剧查询识别缓存正是用来记住这个纠正结果
一律要求 key value 类型一致会让这类条目每次都被丢弃反复回源而电视
剧分支恒定写入电视剧类型`[电视剧]` 键下出现电影只可能来自 tmdbid 消歧
或共享识别回填的脏写会让整季剧集被当成电影反复整理失败
:param meta: 元数据
:param media_type: 待校验的媒体类型
:param tmdb_id: 对应的 TMDB ID为空表示负缓存不带类型信息
:return: 是否冲突
"""
if meta.type != MediaType.TV or not tmdb_id:
return False
if not isinstance(media_type, MediaType):
try:
media_type = MediaType(media_type)
except (TypeError, ValueError):
return False
return media_type == MediaType.MOVIE
def get(self, meta: MetaBase): def get(self, meta: MetaBase):
""" """
根据KEY值获取缓存值 根据KEY值获取缓存值
@@ -154,7 +179,17 @@ class TmdbCache(metaclass=WeakSingleton):
cache_data = self._cache.get(key) cache_data = self._cache.get(key)
if not cache_data and self._expires_at.pop(key, None) is not None: if not cache_data and self._expires_at.pop(key, None) is not None:
self._dirty = True self._dirty = True
return cache_data or {} if not cache_data or not isinstance(cache_data, dict):
return {}
if self.__is_type_conflicted(meta, cache_data.get("type"), cache_data.get("id")):
# 脏条目不丢弃就会被无限期沿用,正确的识别逻辑永远没有执行机会
logger.warn(f"识别缓存类型与元数据冲突,已丢弃并重新识别:{key} -> "
f"{cache_data.get('title')}({cache_data.get('type')})")
self._cache.delete(key)
self._expires_at.pop(key, None)
self._dirty = True
return {}
return cache_data
def delete(self, key: str) -> dict: def delete(self, key: str) -> dict:
""" """
@@ -193,6 +228,11 @@ class TmdbCache(metaclass=WeakSingleton):
""" """
key = self.__get_key(meta) key = self.__get_key(meta)
if info: if info:
if self.__is_type_conflicted(meta, info.get("media_type"), info.get("id")):
# 拒绝写入而不是改写键:识别结果照常返回,只是不把矛盾条目留给下一次
logger.warn(f"识别结果类型与元数据冲突,不写入识别缓存:{key} -> "
f"{info.get('title')}({info.get('media_type')})")
return
# 缓存标题 # 缓存标题
cache_title = info.get("title") \ cache_title = info.get("title") \
if info.get("media_type") == MediaType.MOVIE else info.get("name") if info.get("media_type") == MediaType.MOVIE else info.get("name")
+53 -15
View File
@@ -8,7 +8,7 @@ from app.schemas.types import MediaType
from app.utils.string import StringUtils from app.utils.string import StringUtils
from app.utils.zhconv import convert as zhconv_convert from app.utils.zhconv import convert as zhconv_convert
from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
from .tmdbv3api.exceptions import TMDbException from .tmdbv3api.exceptions import TMDbException, TMDbConnectionError
class TmdbApi: class TmdbApi:
@@ -584,11 +584,14 @@ class TmdbApi:
def get_info(self, def get_info(self,
mtype: MediaType, mtype: MediaType,
tmdbid: int) -> dict: tmdbid: int,
raise_on_connection_error: bool = False) -> dict:
""" """
给定TMDB号查询一条媒体信息 给定TMDB号查询一条媒体信息
:param mtype: 类型电影电视剧为空时都查此时用不上年份 :param mtype: 类型电影电视剧为空时都查此时用不上年份
:param tmdbid: TMDB的ID有tmdbid时优先使用tmdbid否则使用年份和标题 :param tmdbid: TMDB的ID有tmdbid时优先使用tmdbid否则使用年份和标题
:param raise_on_connection_error: 为True时遇到TMDB连接失败区别于404等业务错误
将抛出TMDbConnectionError而不是吞掉返回None默认False与既有调用方行为完全一致
""" """
def __get_genre_ids(genres: list) -> list: def __get_genre_ids(genres: list) -> list:
@@ -604,16 +607,16 @@ class TmdbApi:
# 查询TMDB详情 # 查询TMDB详情
if mtype == MediaType.MOVIE: if mtype == MediaType.MOVIE:
tmdb_info = self.__get_movie_detail(tmdbid) tmdb_info = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info: if tmdb_info:
tmdb_info['media_type'] = MediaType.MOVIE tmdb_info['media_type'] = MediaType.MOVIE
elif mtype == MediaType.TV: elif mtype == MediaType.TV:
tmdb_info = self.__get_tv_detail(tmdbid) tmdb_info = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info: if tmdb_info:
tmdb_info['media_type'] = MediaType.TV tmdb_info['media_type'] = MediaType.TV
else: else:
tmdb_info_tv = self.__get_tv_detail(tmdbid) tmdb_info_tv = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
tmdb_info_movie = self.__get_movie_detail(tmdbid) tmdb_info_movie = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info_tv and tmdb_info_movie: if tmdb_info_tv and tmdb_info_movie:
tmdb_info = None tmdb_info = None
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧") logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
@@ -797,10 +800,13 @@ class TmdbApi:
"alternative_titles," "alternative_titles,"
"translations," "translations,"
"release_dates," "release_dates,"
"external_ids") -> Optional[dict]: "external_ids",
raise_on_connection_error: bool = False) -> Optional[dict]:
""" """
获取电影的详情 获取电影的详情
:param tmdbid: TMDB ID :param tmdbid: TMDB ID
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError
而不是像默认那样吞掉返回None404等TMDB业务错误不受影响始终返回None
:return: TMDB信息 :return: TMDB信息
""" """
""" """
@@ -899,6 +905,11 @@ class TmdbApi:
if tmdbinfo: if tmdbinfo:
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}") logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
return tmdbinfo or {} return tmdbinfo or {}
except TMDbConnectionError as err:
logger.error(str(err))
if raise_on_connection_error:
raise
return None
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
return None return None
@@ -911,10 +922,13 @@ class TmdbApi:
"translations," "translations,"
"content_ratings," "content_ratings,"
"external_ids," "external_ids,"
"episode_groups") -> Optional[dict]: "episode_groups",
raise_on_connection_error: bool = False) -> Optional[dict]:
""" """
获取电视剧的详情 获取电视剧的详情
:param tmdbid: TMDB ID :param tmdbid: TMDB ID
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError
而不是像默认那样吞掉返回None404等TMDB业务错误不受影响始终返回None
:return: TMDB信息 :return: TMDB信息
""" """
""" """
@@ -1084,6 +1098,11 @@ class TmdbApi:
if tmdbinfo: if tmdbinfo:
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}") logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
return tmdbinfo or {} return tmdbinfo or {}
except TMDbConnectionError as err:
logger.error(str(err))
if raise_on_connection_error:
raise
return None
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
return None return None
@@ -1666,10 +1685,13 @@ class TmdbApi:
"alternative_titles," "alternative_titles,"
"translations," "translations,"
"release_dates," "release_dates,"
"external_ids") -> Optional[dict]: "external_ids",
raise_on_connection_error: bool = False) -> Optional[dict]:
""" """
获取电影的详情异步版本 获取电影的详情异步版本
:param tmdbid: TMDB ID :param tmdbid: TMDB ID
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError
而不是像默认那样吞掉返回None404等TMDB业务错误不受影响始终返回None
:return: TMDB信息 :return: TMDB信息
""" """
if not self.movie: if not self.movie:
@@ -1680,6 +1702,11 @@ class TmdbApi:
if tmdbinfo: if tmdbinfo:
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}") logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
return tmdbinfo or {} return tmdbinfo or {}
except TMDbConnectionError as err:
logger.error(str(err))
if raise_on_connection_error:
raise
return None
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
return None return None
@@ -1692,10 +1719,13 @@ class TmdbApi:
"translations," "translations,"
"content_ratings," "content_ratings,"
"external_ids," "external_ids,"
"episode_groups") -> Optional[dict]: "episode_groups",
raise_on_connection_error: bool = False) -> Optional[dict]:
""" """
获取电视剧的详情异步版本 获取电视剧的详情异步版本
:param tmdbid: TMDB ID :param tmdbid: TMDB ID
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError
而不是像默认那样吞掉返回None404等TMDB业务错误不受影响始终返回None
:return: TMDB信息 :return: TMDB信息
""" """
if not self.tv: if not self.tv:
@@ -1706,6 +1736,11 @@ class TmdbApi:
if tmdbinfo: if tmdbinfo:
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}") logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
return tmdbinfo or {} return tmdbinfo or {}
except TMDbConnectionError as err:
logger.error(str(err))
if raise_on_connection_error:
raise
return None
except Exception as e: except Exception as e:
logger.error(str(e)) logger.error(str(e))
return None return None
@@ -1922,11 +1957,14 @@ class TmdbApi:
async def async_get_info(self, async def async_get_info(self,
mtype: MediaType, mtype: MediaType,
tmdbid: int) -> dict: tmdbid: int,
raise_on_connection_error: bool = False) -> dict:
""" """
给定TMDB号查询一条媒体信息异步版本 给定TMDB号查询一条媒体信息异步版本
:param mtype: 类型电影电视剧为空时都查此时用不上年份 :param mtype: 类型电影电视剧为空时都查此时用不上年份
:param tmdbid: TMDB的ID有tmdbid时优先使用tmdbid否则使用年份和标题 :param tmdbid: TMDB的ID有tmdbid时优先使用tmdbid否则使用年份和标题
:param raise_on_connection_error: 为True时遇到TMDB连接失败区别于404等业务错误
将抛出TMDbConnectionError而不是吞掉返回None默认False与既有调用方行为完全一致
""" """
def __get_genre_ids(genres: list) -> list: def __get_genre_ids(genres: list) -> list:
@@ -1942,16 +1980,16 @@ class TmdbApi:
# 查询TMDB详情 # 查询TMDB详情
if mtype == MediaType.MOVIE: if mtype == MediaType.MOVIE:
tmdb_info = await self.__async_get_movie_detail(tmdbid) tmdb_info = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info: if tmdb_info:
tmdb_info['media_type'] = MediaType.MOVIE tmdb_info['media_type'] = MediaType.MOVIE
elif mtype == MediaType.TV: elif mtype == MediaType.TV:
tmdb_info = await self.__async_get_tv_detail(tmdbid) tmdb_info = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info: if tmdb_info:
tmdb_info['media_type'] = MediaType.TV tmdb_info['media_type'] = MediaType.TV
else: else:
tmdb_info_tv = await self.__async_get_tv_detail(tmdbid) tmdb_info_tv = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
tmdb_info_movie = await self.__async_get_movie_detail(tmdbid) tmdb_info_movie = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
if tmdb_info_tv and tmdb_info_movie: if tmdb_info_tv and tmdb_info_movie:
tmdb_info = None tmdb_info = None
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧") logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
@@ -1,2 +1,16 @@
class TMDbException(Exception): class TMDbException(Exception):
pass pass
class TMDbConnectionError(TMDbException):
"""
TMDB连接失败异常
仅在确认为传输层/响应格式问题如底层HTTP请求失败响应无法解析为JSON时抛出
与TMDB业务层明确返回的错误如404条目不存在参数错误等仍抛出普通TMDbException
区分开便于上层区分"网络故障,请重试""条目确实不存在"两类完全不同的处理与文案
继承自TMDbException因此现有 `except TMDbException` 代码路径无需修改即可
继续捕获本异常保持向后兼容
"""
pass
+48 -10
View File
@@ -12,10 +12,30 @@ import requests.exceptions
from app.core.cache import cached, fresh, async_fresh from app.core.cache import cached, fresh, async_fresh
from app.core.config import settings from app.core.config import settings
from app.utils.http import RequestUtils, AsyncRequestUtils from app.utils.http import RequestUtils, AsyncRequestUtils
from .exceptions import TMDbException from .exceptions import TMDbException, TMDbConnectionError
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
# 单次重试前的退避等待时间(秒)。NAS+FUSE网盘等环境下TMDB连接的失败大多是数秒内
# 可自愈的瞬时抖动,零间隔重试(或异步完全不重试)基本无法穿越这类抖动窗口;
# 识别链路是同步阻塞调用,1-3秒的等待可接受,超出3秒则会让识别耗时明显变长,
# 故取区间内的经验值。
RETRY_BACKOFF_SECONDS = 2
def _is_business_failure_snapshot(snapshot) -> bool:
"""
判断响应快照是否为TMDB业务失败success=false如404/限流/服务端错误的合法JSON
这类响应若入缓存会把瞬时失败固化整个TTL周期如12小时期间同key请求
直接命中失败快照跳过缓存让下次请求重新确认真正的负缓存条目确认
不存在由上层 TmdbCache 负责request 层不做失败记忆
"""
if not isinstance(snapshot, dict):
return False
json_data = snapshot.get("json")
return isinstance(json_data, dict) and json_data.get("success") is False
class TMDb(object): class TMDb(object):
_RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__" _RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__"
@@ -136,15 +156,21 @@ class TMDb(object):
def wait_on_rate_limit(self, wait_on_rate_limit): def wait_on_rate_limit(self, wait_on_rate_limit):
self._wait_on_rate_limit = bool(wait_on_rate_limit) self._wait_on_rate_limit = bool(wait_on_rate_limit)
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True) @cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
skip_if=_is_business_failure_snapshot)
def request(self, method, url, data, json, **kwargs): def request(self, method, url, data, json, **kwargs):
req = self._request_once(method, url, data, json) req = self._request_once(method, url, data, json)
if req is None and method == "GET" and self._owns_session: if req is None and method == "GET" and self._owns_session:
logger.debug("TMDB同步请求失败,重建会话重试一次") logger.debug(f"TMDB同步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重建会话重试一次")
# 同步阻塞识别链线程;1-3秒的退避等待可接受,能显著提升对瞬时抖动的容错,
# 详见模块级常量 RETRY_BACKOFF_SECONDS 的说明。
time.sleep(RETRY_BACKOFF_SECONDS)
self._reset_owned_session() self._reset_owned_session()
req = self._request_once(method, url, data, json) req = self._request_once(method, url, data, json)
if req is None: if req is None:
raise TMDbException("无法连接TheMovieDb,请检查网络连接!") # 抛出更具体的连接异常子类,供上层(如TMDB详情查询)区分"网络故障"
# 与"TMDB业务层明确返回的错误"(如404条目不存在),两者不能混为一谈。
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
return self._snapshot_response(req) return self._snapshot_response(req)
def _request_once(self, method, url, data, json): def _request_once(self, method, url, data, json):
@@ -155,16 +181,28 @@ class TMDb(object):
return self._req.get_res(url, params=data, json=json) return self._req.get_res(url, params=data, json=json)
return self._req.post_res(url, data=data, json=json) return self._req.post_res(url, data=data, json=json)
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True) @cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
skip_if=_is_business_failure_snapshot)
async def async_request(self, method, url, data, json, **kwargs): async def async_request(self, method, url, data, json, **kwargs):
if method == "GET": req = await self._async_request_once(method, url, data, json)
req = await self._async_req.get_res(url, params=data, json=json)
else:
req = await self._async_req.post_res(url, data=data, json=json)
if req is None: if req is None:
raise TMDbException("无法连接TheMovieDb,请检查网络连接!") logger.debug(f"TMDB异步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重试一次")
# 异步会话(AsyncRequestUtils)不像同步会话那样支持按需重建,
# 这里退化为原会话上的纯重试,同样以退避等待应对瞬时抖动。
await asyncio.sleep(RETRY_BACKOFF_SECONDS)
req = await self._async_request_once(method, url, data, json)
if req is None:
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
return self._snapshot_response(req) return self._snapshot_response(req)
async def _async_request_once(self, method, url, data, json):
"""
执行一次TMDB异步请求调用方负责决定是否重试
"""
if method == "GET":
return await self._async_req.get_res(url, params=data, json=json)
return await self._async_req.post_res(url, data=data, json=json)
@classmethod @classmethod
def _snapshot_response(cls, response): def _snapshot_response(cls, response):
""" """
+1
View File
@@ -6,6 +6,7 @@
- snapshot.py 远程快照存取与比对 - snapshot.py 远程快照存取与比对
- dispatcher.py 监控事件到整理链的分发 - dispatcher.py 监控事件到整理链的分发
- poller.py 远程目录轮询监控 - poller.py 远程目录轮询监控
- recovery.py 触碰挂载的恢复动作的可放弃执行单元block 型故障隔离
- monitor.py Monitor 门面装配生命周期与健康检查 - monitor.py Monitor 门面装配生命周期与健康检查
""" """
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
+229 -64
View File
@@ -2,13 +2,16 @@ import re
import traceback import traceback
from pathlib import Path from pathlib import Path
from threading import Lock from threading import Lock
from typing import Any, Dict, List, Optional from typing import Any, Dict, List, Optional, Tuple
from app.chain.transfer import TransferChain from app.chain.transfer import TransferChain
from app.core.cache import TTLCache from app.core.cache import TTLCache
from app.core.config import settings from app.core.config import settings
from app.db.transferhistory_oper import TransferHistoryOper from app.db.transferhistory_oper import TransferHistoryOper
from app.helper.directory import DirectoryHelper from app.helper.directory import DirectoryHelper
from app.helper.transferhistory import (HistoryGateAction, describe_history_gate,
evaluate_history_gate, is_skip_action,
max_failed_retries, resolve_history)
from app.log import logger from app.log import logger
from app.schemas import FileItem from app.schemas import FileItem
from app.schemas.types import MediaType from app.schemas.types import MediaType
@@ -80,17 +83,57 @@ class TransferDispatcher:
return f"{event_path.as_posix()}/" return f"{event_path.as_posix()}/"
return event_path.as_posix() return event_path.as_posix()
@staticmethod @classmethod
def _has_transfer_history(storage: str, src_path: str) -> Optional[bool]: def _should_skip_by_history(cls, storage: str, src_path: str,
file_size: Optional[float] = None,
file_modify_time: Optional[float] = None,
fileid: Optional[str] = None) -> Optional[bool]:
""" """
判断源文件是否已经存在整理记录 依据整理历史判断本次是否跳过整理
:return: True/False 查询成功None 查询失败
判定策略由 app/helper/transferhistory.py 统一提供整理链的计划整理段使用
同一套判定避免此处放行的文件在下游被另一套存在记录即拦的策略收回
:param storage: 存储
:param src_path: 整理记录使用的源路径
:param file_size: 当前文件大小蓝光目录等场景可能为 None
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: True 跳过整理False 放行整理None 查询失败
""" """
try: try:
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage)) history = resolve_history(src_path, storage=storage,
transfer_history_oper=TransferHistoryOper())
except Exception as err: except Exception as err:
logger.error(f"查询整理历史失败: {src_path} - {err}") logger.error(f"查询整理历史失败: {src_path} - {err}")
return None return None
action = evaluate_history_gate(
history,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
history_description = describe_history_gate(
history,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if action == HistoryGateAction.PASS_FAILED:
logger.debug(f"上次整理失败({history_description}),"
f"本次重新送入整理链: {src_path}")
elif action == HistoryGateAction.PASS_FAILED_VERSION_CHANGED:
logger.info(f"上次整理失败但文件版本已变化({history_description}),"
f"本次重新送入整理链: {src_path}")
elif action == HistoryGateAction.PASS_SIZE_CHANGED:
logger.info(f"已整理过但文件版本已变化({history_description}),"
f"重新送入整理链: {src_path}")
elif action == HistoryGateAction.SKIP_RETRY_EXHAUSTED:
# 放弃自动重试意味着该文件需要人介入,不能只留 debug 日志重蹈静默漏件的覆辙
logger.warn(f"整理连续失败 {max_failed_retries()} 次已达上限,不再自动重试,"
f"请手动整理或删除整理记录: {src_path}")
elif action == HistoryGateAction.SKIP:
logger.debug(f"已整理过且文件未变化,跳过: {src_path}")
return is_skip_action(action)
@staticmethod @staticmethod
def _pending_key(storage: str, event_path: Path) -> str: def _pending_key(storage: str, event_path: Path) -> str:
@@ -132,12 +175,18 @@ class TransferDispatcher:
) )
return None return None
def _register_pending(self, storage: str, event_path: Path, file_size: float = None):
def _register_pending(self, storage: str, event_path: Path, file_size: float = None,
file_modify_time: float = None, fileid: Optional[str] = None,
reason: str = "整理历史查询失败"):
""" """
登记历史查询失败的文件待重试重复失败累计次数超限后放弃 登记暂时性故障的文件待重试重复失败累计次数超限后放弃
:param storage: 存储 :param storage: 存储
:param event_path: 原始事件路径 :param event_path: 原始事件路径
:param file_size: 文件大小 :param file_size: 文件大小None 表示重试时需要重新读取
:param file_modify_time: 文件修改时间
:param fileid: 文件唯一标识
:param reason: 登记原因用于日志
""" """
key = self._pending_key(storage, event_path) key = self._pending_key(storage, event_path)
with self._pending_guard: with self._pending_guard:
@@ -146,7 +195,7 @@ class TransferDispatcher:
entry["attempts"] += 1 entry["attempts"] += 1
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS: if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
self._pending_retries.pop(key, None) self._pending_retries.pop(key, None)
logger.error(f"整理历史查询持续失败,已放弃重试: {key}") logger.error(f"{reason}持续失败,已放弃重试: {key}")
return return
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES: if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
logger.error(f"整理重试队列已满,丢弃: {key}") logger.error(f"整理重试队列已满,丢弃: {key}")
@@ -155,9 +204,39 @@ class TransferDispatcher:
"storage": storage, "storage": storage,
"event_path": event_path, "event_path": event_path,
"file_size": file_size, "file_size": file_size,
"file_modify_time": file_modify_time,
"fileid": fileid,
"attempts": 1 "attempts": 1
} }
logger.warn(f"整理历史查询失败,已登记待重试: {key}") logger.warn(f"{reason},已登记待重试: {key}")
def register_unreadable(self, storage: str, event_path: Path):
"""
登记读取失败的监控事件待重试
FUSE/网络挂载抖动时 stat 会瞬时失败直接丢弃事件就是永久漏件
因此复用待重试队列由健康检查周期重新读取
:param storage: 存储
:param event_path: 事件文件路径
"""
self._register_pending(storage=storage, event_path=event_path,
file_size=None, reason="读取监控事件文件失败")
@staticmethod
def _resolve_file_state(event_path: Path) -> Tuple[Optional[int], Optional[float], bool]:
"""
重新读取本地文件指纹
:param event_path: 文件路径
:return: (文件大小, 修改时间, 文件是否仍然存在)大小为 None 表示本次读取仍然失败
"""
try:
file_stat = Path(event_path).stat()
return file_stat.st_size, file_stat.st_mtime, True
except FileNotFoundError:
return None, None, False
except OSError as err:
logger.debug(f"重试读取文件大小失败: {event_path} - {err}")
return None, None, True
def _discard_pending(self, storage: str, event_path: Path): def _discard_pending(self, storage: str, event_path: Path):
""" """
@@ -168,6 +247,17 @@ class TransferDispatcher:
with self._pending_guard: with self._pending_guard:
self._pending_retries.pop(self._pending_key(storage, event_path), None) self._pending_retries.pop(self._pending_key(storage, event_path), None)
def clear_pending(self):
"""
清空待重试队列监控停止或配置重载时调用避免已移除的监控目录
在数据库恢复后仍被看门狗送入整理链
"""
with self._pending_guard:
if not self._pending_retries:
return
logger.debug(f"清理整理重试队列,丢弃 {len(self._pending_retries)} 个待重试条目")
self._pending_retries.clear()
def retry_pending(self): def retry_pending(self):
""" """
重试历史查询失败的文件由健康检查周期驱动 重试历史查询失败的文件由健康检查周期驱动
@@ -176,74 +266,149 @@ class TransferDispatcher:
with self._pending_guard: with self._pending_guard:
items = list(self._pending_retries.values()) items = list(self._pending_retries.values())
for item in items: for item in items:
logger.info(f"重试整理: {item['storage']}:{item['event_path']}") storage = item["storage"]
self.handle_file(storage=item["storage"], event_path=item["event_path"], event_path = item["event_path"]
file_size=item["file_size"]) file_size = item["file_size"]
file_modify_time = item.get("file_modify_time")
fileid = item.get("fileid")
if file_size is None and storage == "local":
# 因读取失败入队的事件没有大小,重试时必须重新读取
file_size, file_modify_time, exists = self._resolve_file_state(event_path)
if not exists:
logger.debug(f"待重试文件已不存在,放弃: {storage}:{event_path}")
self._discard_pending(storage=storage, event_path=event_path)
continue
if file_size is None:
# 仍然读不到,累计失败次数后等下个周期,超限由登记逻辑放弃
self._register_pending(storage=storage, event_path=event_path,
reason="读取监控事件文件失败")
continue
logger.info(f"重试整理: {storage}:{event_path}")
self.handle_file(
storage=storage,
event_path=event_path,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
def handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool: def handle_file(self, storage: str, event_path: Path, file_size: float = None,
file_modify_time: float = None, fileid: Optional[str] = None) -> bool:
""" """
整理一个文件 整理一个文件
:param storage: 存储 :param storage: 存储
:param event_path: 事件文件路径 :param event_path: 事件文件路径
:param file_size: 文件大小 :param file_size: 文件大小
:param file_modify_time: 文件修改时间
:param fileid: 文件唯一标识
:return: 是否进入整理链 :return: 是否进入整理链
""" """
with self._lock: # 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行 origin_path = event_path
origin_path = event_path is_bluray_folder = False
is_bluray_folder = False # 蓝光原盘文件处理
# 蓝光原盘文件处理 if self._is_bluray_sub(event_path):
if self._is_bluray_sub(event_path): event_path = self._get_bluray_dir(event_path)
event_path = self._get_bluray_dir(event_path) if not event_path:
if not event_path:
return False
is_bluray_folder = True
elif not self.is_transfer_candidate_path(event_path):
return False return False
is_bluray_folder = True
elif not self.is_transfer_candidate_path(event_path):
return False
# TTL缓存控重 # TTL 缓存控重。这是本方法唯一需要互斥的临界区,锁只保护「查缓存 + 写缓存」
# 这一步的原子性。
#
# 锁的范围绝不能扩大到下面的历史查询与整理调用:整理的规划阶段会访问挂载
# do_transfer 内的 get_parent_item / list_files),FUSE 进入「请求永不
# 返回」状态时这些调用永远不返回,持锁线程就把这把锁永久攥在手里,连带
# 锁死所有 watcher 线程的事件派发、监控恢复后的补偿扫描和重试队列——监控层
# 即使完成自愈也送不进任何文件,漏件永远补不回来。
#
# 并发是安全的:TTL 去重保证同一路径不会并发进入;TransferChain 是单例,
# 内部用 job_lock/task_lock 保护共享状态、入队走线程安全的 queue.Queue
# 本来就被下载完成事件、定时任务与工作流并发调用。
with self._lock:
if self._cache.get(str(event_path)): if self._cache.get(str(event_path)):
return False return False
self._cache[str(event_path)] = True self._cache[str(event_path)] = True
src_path = self._build_transfer_src_path( src_path = self._build_transfer_src_path(
event_path=event_path, event_path=event_path,
is_bluray_folder=is_bluray_folder, is_bluray_folder=is_bluray_folder,
) )
has_transfer_history = self._has_transfer_history( skip_by_history = self._should_skip_by_history(
storage=storage,
src_path=src_path,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if skip_by_history is None:
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
self._register_pending(
storage=storage, storage=storage,
src_path=src_path, event_path=origin_path,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
) )
if has_transfer_history is None: return False
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过 if skip_by_history:
self._register_pending(storage=storage, event_path=origin_path, file_size=file_size)
return False
self._discard_pending(storage=storage, event_path=origin_path) self._discard_pending(storage=storage, event_path=origin_path)
if has_transfer_history: return False
return False
try: try:
if is_bluray_folder: if is_bluray_folder:
logger.info(f"开始整理蓝光原盘: {event_path}") logger.info(f"开始整理蓝光原盘: {event_path}")
else: else:
logger.info(f"开始整理文件: {event_path}") logger.info(f"开始整理文件: {event_path}")
# 开始整理 # 开始整理
TransferChain().do_transfer( TransferChain().do_transfer(
fileitem=FileItem( fileitem=FileItem(
storage=storage, storage=storage,
path=src_path, path=src_path,
type="file" if not is_bluray_folder else "dir", type="file" if not is_bluray_folder else "dir",
name=event_path.name, name=event_path.name,
basename=event_path.stem, basename=event_path.stem,
extension=event_path.suffix[1:], extension=event_path.suffix[1:],
size=file_size size=file_size,
), modify_time=file_modify_time,
mtype=self._get_monitor_media_type( fileid=fileid,
storage=storage, ),
event_path=event_path, mtype=self._get_monitor_media_type(
), storage=storage,
) event_path=event_path,
return True ),
except Exception as e: )
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc())) # 整理已执行完毕,此前因暂时性故障登记的重试条目到此作废
return False self._discard_pending(storage=storage, event_path=origin_path)
return True
except Exception as e:
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
# 去重缓存在入口已写入,整理抛异常时必须失效,否则 TTL 窗口内该文件的
# 后续事件会被静默吞掉,等于一次异常就丢一个文件
self._invalidate_cache(str(event_path))
# 已稳定落地的文件不会再产生任何事件,批量整理期间撞上一次 DB/网络瞬断
# 就是永久丢件,因此与历史查询失败同样登记待重试;登记用原始事件路径,
# 重试时重新解析蓝光目录并重走完整流程。异常未清空登记,重试次数会持续
# 累计,达到上限后由 _register_pending 放弃,不会无限重试
self._register_pending(storage=storage, event_path=origin_path,
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
reason="整理执行异常")
return False
def _invalidate_cache(self, key: str):
"""
使去重缓存条目失效兼容缓存后端与测试注入的字典
:param key: 缓存键
"""
try:
delete = getattr(self._cache, "delete", None)
if callable(delete):
delete(key)
return
self._cache.pop(key, None)
except Exception as err:
logger.debug(f"清理监控去重缓存失败: {key} - {err}")
+292 -21
View File
@@ -1,7 +1,9 @@
import time
import traceback import traceback
from functools import partial
from pathlib import Path from pathlib import Path
from threading import Lock from threading import Lock, Thread
from typing import Any, Dict, List, Optional from typing import Any, Callable, Dict, List, Optional, Tuple
from apscheduler.schedulers.background import BackgroundScheduler from apscheduler.schedulers.background import BackgroundScheduler
@@ -11,6 +13,7 @@ from app.helper.message import MessageHelper
from app.log import logger from app.log import logger
from app.monitor.dispatcher import TransferDispatcher from app.monitor.dispatcher import TransferDispatcher
from app.monitor.poller import RemotePoller from app.monitor.poller import RemotePoller
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
from app.monitor.snapshot import SnapshotStore from app.monitor.snapshot import SnapshotStore
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
from app.monitor.watcher import LocalDirectoryWatcher from app.monitor.watcher import LocalDirectoryWatcher
@@ -24,11 +27,33 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
""" """
目录监控门面单例模式装配本地/远程监控维护生命周期与健康检查 目录监控门面单例模式装配本地/远程监控维护生命周期与健康检查
""" """
CONFIG_WATCH = {SystemConfigKey.Directories.value} # 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
CONFIG_WATCH = {SystemConfigKey.Directories.value,
"MONITOR_NETWORK_FAST_MODE",
"MONITOR_POLL_DELAY_NETWORK"}
# 目录监控健康检查间隔(秒) # 目录监控健康检查间隔(秒)
WATCHDOG_INTERVAL = 60 WATCHDOG_INTERVAL = 60
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏 # 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
RECOVERY_STABLE_CYCLES = 5 RECOVERY_STABLE_CYCLES = 5
# 补偿扫描的时间回溯余量(秒),覆盖心跳与文件落地之间的时间差
COMPENSATION_MARGIN = 60
# 监控内部退避重启的停摆窗口无法从外部精确观测(重启在 watcher 线程内部完成,
# 健康检查只能看到累计重启次数),保守取「健康检查周期 + 最长退避 + 余量」
RESTART_STALL_LOOKBACK = WATCHDOG_INTERVAL + max(LocalDirectoryWatcher.RESTART_BACKOFF) + COMPENSATION_MARGIN
# 单次补偿扫描最多送入整理链的文件数,避免超大目录把整理链和数据库压垮
MAX_COMPENSATION_FILES = 2000
# 恢复动作(重建监控、重试队列、挂载探测)单个健康检查周期的最长等待秒数。
# 这些动作全部会触碰挂载,一律在一次性工作线程里执行,看门狗只等待有限时间;
# 取健康检查周期的一半,保证看门狗自身永远不会被拖过下一个周期
RECOVERY_TIMEOUT = WATCHDOG_INTERVAL // 2
# 隔离目录的挂载探测超时秒数(子进程,超时可被 kill)
MOUNT_PROBE_TIMEOUT = 10
# 恢复动作的 key 前缀/常量,用于按目录隔离在途动作
REBUILD_KEY_PREFIX = "rebuild:"
PROBE_KEY = "probe"
PENDING_KEY = "pending"
def __init__(self): def __init__(self):
super().__init__() super().__init__()
@@ -38,12 +63,16 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self._watcher_lock = Lock() self._watcher_lock = Lock()
# 启动失败待重试的本地监控配置 # 启动失败待重试的本地监控配置
self._pending_locals: List[Dict[str, Any]] = [] self._pending_locals: List[Dict[str, Any]] = []
# 已告警的监控目录,避免重复推送 # 已告警的监控目录及其告警阶段,避免重复推送,同时保证故障升级能再推一次
self._alerted_paths: set = set() self._alerted_paths: Dict[str, str] = {}
# 各监控目录已告警过的自动重启次数 # 各监控目录已告警过的自动重启次数
self._restart_marks: Dict[str, int] = {} self._restart_marks: Dict[str, int] = {}
# 各监控目录连续稳定的健康检查周期数 # 各监控目录连续稳定的健康检查周期数
self._stable_cycles: Dict[str, int] = {} self._stable_cycles: Dict[str, int] = {}
# 判定为挂载级故障、已暂停一切访问的监控目录(path -> 隔离状态)
self._isolated: Dict[str, Dict[str, Any]] = {}
# 触碰挂载的恢复动作执行器,把 block 型故障挡在看门狗线程之外
self._recovery = RecoveryExecutor()
# 定时服务 # 定时服务
self._scheduler = None self._scheduler = None
# 整理分发器 # 整理分发器
@@ -252,7 +281,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力 # 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
poll_delay_ms = None poll_delay_ms = None
if use_polling and SystemUtils.is_network_filesystem(mon_path): if use_polling and SystemUtils.is_network_filesystem(mon_path):
poll_delay_ms = LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS poll_delay_ms = (settings.MONITOR_POLL_DELAY_NETWORK
or LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS)
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}") logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
watcher = LocalDirectoryWatcher( watcher = LocalDirectoryWatcher(
@@ -314,23 +344,38 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
def watchdog(self): def watchdog(self):
""" """
目录监控健康检查重建崩溃或静默失效的监控线程并重试启动失败的监控目录 目录监控健康检查检测监控线程状态并驱动恢复
本方法是全局自愈的单点因此**只做纯内存的检测与判定**任何会触碰挂载
的动作重建监控重试启动重试整理挂载探测都交给一次性工作线程
看门狗只等待有限时间FUSE 挂载进入请求永不返回状态时内联执行这些
动作会把看门狗冻死在它自己要修复的挂载上随后停滞检测告警重试驱动
全部静默失效这正是全进程雪崩的起点
""" """
try: try:
self.__check_watchers() broken = self.__check_watchers()
self.__retry_pending_locals() self.__drive_recovery(broken)
self._dispatcher.retry_pending()
except Exception as e: except Exception as e:
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}") logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
def __check_watchers(self): def __check_watchers(self) -> List[LocalDirectoryWatcher]:
""" """
检查本地目录监控线程状态异常时重建 检查本地目录监控线程状态返回需要重建的监控
全程只读内存状态线程存活标志心跳时间戳重启计数不做任何文件
系统访问确保挂载无响应时检测环节本身永远不会被阻塞
:return: 需要重建的监控列表
""" """
with self._watcher_lock: with self._watcher_lock:
watchers = list(self._watchers) watchers = list(self._watchers)
isolated = set(self._isolated)
broken: List[LocalDirectoryWatcher] = []
for watcher in watchers: for watcher in watchers:
key = str(watcher.watch_path) key = str(watcher.watch_path)
if key in isolated:
# 已判定挂载级故障:重建只会再冻死一个线程,等探测确认挂载
# 恢复应答后由 __probe_isolated 统一重建
continue
if watcher.is_stalled(): if watcher.is_stalled():
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效" reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
elif not watcher.is_alive(): elif not watcher.is_alive():
@@ -343,6 +388,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
self.__send_alert(watcher.watch_path, self.__send_alert(watcher.watch_path,
f"目录监控发生错误并已自动重启" f"目录监控发生错误并已自动重启"
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}") f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
# 内部退避重启同样是停摆:轮询模式下重启会重建基线快照,停摆窗口内
# 落地的文件会被新基线静默吸收,永远不再产生事件,必须补扫。
# 重启计数只在增长时进入本分支,同一次重启不会被反复补扫
self.__start_compensation(mon_path=watcher.watch_path,
since=time.time() - self.RESTART_STALL_LOOKBACK)
else: else:
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏 # 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1 self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
@@ -352,7 +402,102 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...") logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
self.__send_alert(watcher.watch_path, self.__send_alert(watcher.watch_path,
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控") f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
self.__rebuild_watcher(watcher) broken.append(watcher)
return broken
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
"""
把所有会触碰挂载的恢复动作派发到一次性工作线程并等待有限时间
重建动作超时或上一轮的重建仍冻着即判定为挂载级故障这已经不是单个
目录的问题继续每 60 秒重试只会不断泄漏冻死的线程必须转入隔离改由
可放弃的子进程探测来确认挂载何时恢复
:param broken: 需要重建的监控列表
"""
actions: Dict[str, Callable[[], None]] = {}
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
for watcher in broken:
key = f"{self.REBUILD_KEY_PREFIX}{watcher.watch_path}"
rebuilds[key] = watcher
actions[key] = partial(self.__rebuild_watcher, watcher)
if self._isolated:
actions[self.PROBE_KEY] = self.__probe_isolated
actions[self.PENDING_KEY] = self.__drive_pending
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
for key, state in results.items():
if state is RecoveryState.COMPLETED:
continue
watcher = rebuilds.get(key)
if watcher is not None:
self.__enter_isolation(watcher)
else:
# 探测与重试驱动超时不升级为隔离:它们跨多个目录,无法归因到
# 具体挂载,下个周期由执行器的 BUSY 判定自动跳过,不会泄漏线程
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
f"{state.value}),将在后续健康检查周期重试: {key}")
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
"""
将一个监控目录转入挂载级故障隔离停止对它的一切新访问等待探测恢复
:param watcher: 重建未能返回的监控
"""
key = str(watcher.watch_path)
with self._watcher_lock:
if key in self._isolated:
return
self._isolated[key] = {
"watcher": watcher,
"since": time.time(),
"failures": 0,
}
logger.error(f"目录监控重建在挂载上无响应,判定为挂载级故障,"
f"已暂停对该目录的所有访问并转入周期探测: {watcher.watch_path}")
self.__send_alert(watcher.watch_path,
f"目录监控挂载无响应: {watcher.watch_path}\n"
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
stage="isolated")
def __probe_isolated(self):
"""
对隔离中的监控目录做可放弃探测挂载恢复应答后解除隔离并重建监控
运行在恢复工作线程里探测本身由子进程执行 recovery.probe_path
超时可被 kill因此本线程不会像内联 stat 那样永久冻死探测通过后的重建
仍有极小概率再次卡住届时本线程会被下一轮的 BUSY 判定跳过不再泄漏
"""
with self._watcher_lock:
keys = list(self._isolated)
for key in keys:
mon_path = Path(key)
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
with self._watcher_lock:
entry = self._isolated.get(key)
if not entry:
continue
entry["failures"] += 1
failures = entry["failures"]
since = entry["since"]
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
continue
with self._watcher_lock:
entry = self._isolated.pop(key, None)
if not entry:
continue
logger.info(f"✓ 挂载探测通过,解除隔离并重建目录监控: {mon_path}")
self.__clear_alert(mon_path, f"目录监控挂载已恢复响应,正在重建监控: {mon_path}")
# 重建内部会按 watcher 的最后心跳时间发起补偿扫描,补回隔离期间落地的文件
self.__rebuild_watcher(entry["watcher"])
def __drive_pending(self):
"""
驱动两条待重试队列两者都会访问挂载启动重试走目录遍历与 exists
整理重试走 stat必须在恢复工作线程里执行而不是看门狗线程里
"""
self.__retry_pending_locals()
self._dispatcher.retry_pending()
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher): def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
""" """
@@ -381,12 +526,102 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
}) })
return return
with self._watcher_lock: with self._watcher_lock:
self._watchers = [new_watcher if item is watcher else item for item in self._watchers] registered = any(item is watcher for item in self._watchers)
if registered:
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
if not registered:
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
# (健康检查永远看不到它)、也没人调用 stop(),会变成与旧监控重复
# 派发事件的孤儿线程,必须就地停掉
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
new_watcher.stop()
return
# 新监控的重启计数从零开始,同步重置告警基准 # 新监控的重启计数从零开始,同步重置告警基准
self._restart_marks.pop(str(watcher.watch_path), None) self._restart_marks.pop(str(watcher.watch_path), None)
self._stable_cycles.pop(str(watcher.watch_path), None) self._stable_cycles.pop(str(watcher.watch_path), None)
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}") logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}") self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
# 重建只恢复未来的事件,停摆期间落地的文件不会再产生任何事件,必须补扫
self.__start_compensation(mon_path=watcher.watch_path,
since=watcher.last_activity_time)
def __start_compensation(self, mon_path: Path, since: float):
"""
在后台线程发起补偿扫描避免遍历目录阻塞健康检查周期
:param mon_path: 监控目录
:param since: 停摆起点墙钟时间戳
"""
if not since:
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
return
Thread(
target=self.__compensate_scan,
kwargs={"mon_path": mon_path, "since": since},
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
daemon=True
).start()
def __compensate_scan(self, mon_path: Path, since: float):
"""
补扫监控停摆期间落地的文件
不能用 mtime 判定停摆期间落地CloudDrive2/115 等网盘挂载在转存移动
文件时保留原始 mtime可能是几年前 mtime 过滤会让补偿扫描完全空转
因此把目录内所有候选文件都送入整理链由分发器的 TTL 去重与整理历史查重挡掉
已处理过的文件代价是每个候选文件一次历史查询故按 mtime 从新到旧排序并用
MAX_COMPENSATION_FILES 限制单次规模让名额优先给最可能是新落地的文件
:param mon_path: 监控目录
:param since: 停摆起点墙钟时间戳仅用于统计与日志
"""
candidates = self.__collect_compensation_files(mon_path)
if candidates is None:
return
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
candidates.sort(key=lambda item: item[1], reverse=True)
if len(candidates) > self.MAX_COMPENSATION_FILES:
logger.warn(f"补偿扫描候选文件 {len(candidates)} 个,超过单次上限 "
f"{self.MAX_COMPENSATION_FILES},本次只处理最新的一批: {mon_path}")
candidates = candidates[:self.MAX_COMPENSATION_FILES]
threshold = since - self.COMPENSATION_MARGIN
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
handled = 0
for file_path, file_modify_time, file_size in candidates:
if self._dispatcher.handle_file(
storage="local",
event_path=file_path,
file_size=file_size,
file_modify_time=file_modify_time,
):
handled += 1
logger.info(f"✓ 目录监控补偿扫描完成,{len(candidates)} 个候选文件"
f"(其中 {changed_count} 个修改时间落在停摆期间)中有 {handled} 个进入整理链: {mon_path}")
def __collect_compensation_files(self, mon_path: Path) -> Optional[List[Tuple[Path, float, int]]]:
"""
收集补偿扫描的候选文件
:param mon_path: 监控目录
:return: (文件路径, 修改时间, 文件大小) 列表目录遍历失败时返回 None
"""
candidates: List[Tuple[Path, float, int]] = []
try:
for file_path in mon_path.rglob("*"):
# 扩展名判断是纯字符串运算,先过滤能省掉大量 FUSE 挂载上昂贵的 stat
if not self._dispatcher.is_transfer_candidate_path(file_path):
continue
try:
if not file_path.is_file():
continue
file_stat = file_path.stat()
except OSError as err:
logger.debug(f"补偿扫描读取文件失败: {file_path} - {err}")
continue
candidates.append((file_path, file_stat.st_mtime, file_stat.st_size))
except OSError as err:
logger.error(f"补偿扫描失败: {mon_path} - {err}")
return None
return candidates
def __retry_pending_locals(self): def __retry_pending_locals(self):
""" """
@@ -394,7 +629,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
""" """
with self._watcher_lock: with self._watcher_lock:
pending = list(self._pending_locals) pending = list(self._pending_locals)
isolated = set(self._isolated)
for item in pending: for item in pending:
if str(item["mon_path"]) in isolated:
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
# 在「请求永不返回」的挂载上会再冻死一个线程
continue
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏 # 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
if item.get("skip_cycles", 0) > 0: if item.get("skip_cycles", 0) > 0:
item["skip_cycles"] -= 1 item["skip_cycles"] -= 1
@@ -404,17 +644,20 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
item["attempts"] = item.get("attempts", 0) + 1 item["attempts"] = item.get("attempts", 0) + 1
item["skip_cycles"] = min(item["attempts"], 10) item["skip_cycles"] = min(item["attempts"], 10)
def __send_alert(self, mon_path: Path, message: str): def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
""" """
推送目录监控异常告警同一目录仅在状态变化时推送一次 推送目录监控异常告警同一目录在同一阶段仅推送一次
:param mon_path: 监控目录 :param mon_path: 监控目录
:param message: 告警内容 :param message: 告警内容
:param stage: 告警阶段故障升级为挂载级隔离是用户必须知道的状态变化
监控已暂停访问等待挂载恢复只按目录去重会把这条
关键消息吞掉因此阶段变化时重新推送
""" """
key = str(mon_path) key = str(mon_path)
with self._watcher_lock: with self._watcher_lock:
if key in self._alerted_paths: if self._alerted_paths.get(key) == stage:
return return
self._alerted_paths.add(key) self._alerted_paths[key] = stage
MessageHelper().put(message, title="目录监控") MessageHelper().put(message, title="目录监控")
@staticmethod @staticmethod
@@ -437,7 +680,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
with self._watcher_lock: with self._watcher_lock:
if key not in self._alerted_paths: if key not in self._alerted_paths:
return return
self._alerted_paths.discard(key) self._alerted_paths.pop(key, None)
logger.info(message) logger.info(message)
MessageHelper().put(message, title="目录监控") MessageHelper().put(message, title="目录监控")
@@ -474,8 +717,30 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
return return
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)): if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
return return
file_modify_time = None
try:
file_modify_time = Path(event_path).stat().st_mtime
except OSError as err:
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
# 整理文件 # 整理文件
self._dispatcher.handle_file(storage="local", event_path=Path(event_path), file_size=file_size) handle_kwargs = {
"storage": "local",
"event_path": Path(event_path),
"file_size": file_size,
}
if file_modify_time is not None:
handle_kwargs["file_modify_time"] = file_modify_time
self._dispatcher.handle_file(**handle_kwargs)
def event_unreadable(self, event_path: Path):
"""
处理读取失败的监控事件登记待重试
:param event_path: 事件文件路径
"""
event_path = Path(event_path)
if not self._dispatcher.is_transfer_candidate_path(event_path):
return
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
def stop(self): def stop(self):
""" """
@@ -491,13 +756,19 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
except Exception as e: except Exception as e:
logger.error(f"停止定时服务出现了错误:{e}") logger.error(f"停止定时服务出现了错误:{e}")
self._scheduler = None self._scheduler = None
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
self._dispatcher.clear_pending()
with self._watcher_lock: with self._watcher_lock:
watchers = self._watchers watchers = self._watchers
self._watchers = [] self._watchers = []
self._pending_locals = [] self._pending_locals = []
self._alerted_paths = set() self._alerted_paths = {}
self._restart_marks = {} self._restart_marks = {}
self._stable_cycles = {} self._stable_cycles = {}
self._isolated = {}
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
# 被残留记录误判为 BUSY 而永远拿不到重建机会
self._recovery.clear()
if watchers: if watchers:
logger.info("正在停止本地目录监控服务...") logger.info("正在停止本地目录监控服务...")
for watcher in watchers: for watcher in watchers:
+45 -11
View File
@@ -137,13 +137,17 @@ class RemotePoller:
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件") logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***") logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
# 保存当前完整基线 # 保存合并后的基线。增量游标是整个存储共用的,若本轮有路径失败仍让
if not self._store.save(storage, current_snapshot, file_count, last_snapshot_time): # 游标跟随成功路径前进,失败路径中时间落在新旧游标之间的变更会被
# 后续增量查询永久跳过,因此部分失败时把游标固定在旧值
pinned_time = last_snapshot_time if failed_paths else None
if not self._store.save(storage, current_snapshot, file_count, last_snapshot_time,
snapshot_time=pinned_time):
self._note_failure(storage, "保存快照基线失败") self._note_failure(storage, "保存快照基线失败")
return None return None
if failed_paths: if failed_paths:
# 部分路径失败:成功路径已合并,失败路径保留旧基线,下轮重试 # 部分路径失败:成功路径已合并,失败路径保留旧基线与旧游标,下轮重试
self._note_failure( self._note_failure(
storage, storage,
f"部分路径快照失败: {','.join(str(path) for path in failed_paths)}" f"部分路径快照失败: {','.join(str(path) for path in failed_paths)}"
@@ -181,7 +185,15 @@ class RemotePoller:
for new_file in added_files: for new_file in added_files:
file_info = new_snapshot.get(new_file, {}) file_info = new_snapshot.get(new_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(new_file), file_size=file_size): file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
if self._dispatcher.handle_file(
storage=storage,
event_path=Path(new_file),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
):
handled_added_count += 1 handled_added_count += 1
# 处理修改文件 # 处理修改文件
@@ -189,7 +201,15 @@ class RemotePoller:
for modified_file in modified_files: for modified_file in modified_files:
file_info = new_snapshot.get(modified_file, {}) file_info = new_snapshot.get(modified_file, {})
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size): file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
if self._dispatcher.handle_file(
storage=storage,
event_path=Path(modified_file),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
):
handled_modified_count += 1 handled_modified_count += 1
if handled_added_count or handled_modified_count: if handled_added_count or handled_modified_count:
@@ -228,8 +248,15 @@ class RemotePoller:
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)): if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
continue continue
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
if self._dispatcher.handle_file(storage=storage, event_path=Path(file_path), file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
file_size=file_size): fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
if self._dispatcher.handle_file(
storage=storage,
event_path=Path(file_path),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
):
processed_count += 1 processed_count += 1
except Exception as e: except Exception as e:
logger.error(f"处理文件 {file_path} 失败: {e}") logger.error(f"处理文件 {file_path} 失败: {e}")
@@ -237,11 +264,18 @@ class RemotePoller:
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件") logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
# 全量扫描覆盖单个路径,与已有基线合并后落盘,避免覆盖其他监控路径的基线 # 全量扫描覆盖单个路径,必须与已有基线合并后落盘。读取失败时无法
# 区分「基线不存在」与「读取异常」,此时落盘会抹掉同存储下其他监控
# 路径的基线,因此直接判定失败
old_snapshot_data, load_ok = self._store.load_checked(storage) old_snapshot_data, load_ok = self._store.load_checked(storage)
old_snapshot = old_snapshot_data.get('snapshot', {}) if (load_ok and old_snapshot_data) else {} if not load_ok:
merged_snapshot = {**old_snapshot, **new_snapshot} logger.error(f"读取快照基线失败,已跳过落盘以避免覆盖其他监控路径: {storage}:{mon_path}")
self._store.save(storage, merged_snapshot, len(merged_snapshot)) return False
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
current_snapshot = {**old_snapshot, **new_snapshot}
if not self._store.save(storage, current_snapshot, len(current_snapshot)):
logger.error(f"保存快照基线失败,全量扫描未完成: {storage}:{mon_path}")
return False
return True return True
+178
View File
@@ -0,0 +1,178 @@
"""
监控恢复动作的可放弃执行单元
FUSE/网络挂载有两种故障形态下游程序的免疫力完全不同
- crash 调用抛错 Transport endpoint is not connected异常能被捕获
退避重启即可自愈 watcher 自身的重启循环覆盖
- block 调用既不返回错误也不返回结果永久悬挂**没有任何超时参数能救
一个已经发出的 stat**阻塞其上的线程无法被 Python 回收线程没有强杀接口
本模块提供 block 型故障下唯一可行的两种自保手段
1. RecoveryExecutor 把会触碰挂载的动作放进一次性守护线程执行调用方只
等待有限时间超时即放弃该线程它会作为守护线程悬挂到进程退出换取
调用方健康检查这个全局自愈单点永远活着
2. probe_path 用子进程而非线程做挂载探测子进程可以被 kill因此探测
本身是可放弃的隔离期间可以无限次周期重试而不累积不可回收的资源
"""
import subprocess
import sys
import threading
import time
from enum import Enum
from pathlib import Path
from typing import Callable, Dict, Optional
from app.log import logger
# 探测子进程执行的脚本:只对目标路径做一次 stat,成功退出 0,失败退出非 0。
# 用 sys.executable 而不是 test/stat 等外部命令,避免依赖发行版的 coreutils 布局。
_PROBE_SCRIPT = "import os, sys; os.stat(sys.argv[1])"
# 探测子进程超时后,等待它响应 SIGKILL 的宽限秒数。挂在 FUSE 上的进程可能一时
# 收不掉,宽限期满就不再等待,残留进程由后续 Popen 自动回收,绝不能无限等待
# ——否则「可放弃的探测」又变回一次不可放弃的阻塞。
_PROBE_KILL_GRACE = 5
class RecoveryState(str, Enum):
"""
一次恢复动作的执行结论
"""
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
COMPLETED = "completed"
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
TIMEOUT = "timeout"
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
BUSY = "busy"
class RecoveryExecutor:
"""
key 隔离的一次性恢复线程执行器
每个 key 同一时刻最多有一个在途动作上一个还冻着就不再提交新的否则每个
健康检查周期都会在同一个死挂载上多泄漏一个线程
"""
def __init__(self):
# key -> 该 key 最近一次提交的执行线程
self._running: Dict[str, threading.Thread] = {}
self._lock = threading.Lock()
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
"""
并发执行一批恢复动作整批最多等待 timeout
并发而非串行是必需的串行等待会让总耗时随监控目录数线性增长
13 个目录都挂死时健康检查会被拖过下一个周期等于又一次自我冻结
:param actions: key -> 无参恢复动作
:param timeout: 整批动作的最长等待秒数
:return: key -> 执行结论
"""
results: Dict[str, RecoveryState] = {}
started = []
for key, action in actions.items():
thread = self._start(key, action)
if thread is None:
results[key] = RecoveryState.BUSY
logger.warn(f"上一次恢复动作仍未返回,本轮跳过以避免线程泄漏: {key}")
continue
started.append((key, thread))
deadline = time.monotonic() + timeout
for key, thread in started:
thread.join(timeout=max(0.0, deadline - time.monotonic()))
if thread.is_alive():
results[key] = RecoveryState.TIMEOUT
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
else:
results[key] = RecoveryState.COMPLETED
return results
def discard(self, key: str):
"""
丢弃一个 key 的在途记录监控停止或配置重载时调用避免残留条目
让重建后的同名目录被误判为 BUSY
:param key: 动作标识
"""
with self._lock:
self._running.pop(key, None)
def clear(self):
"""
清空全部在途记录已经冻死的线程无法回收这里只是不再跟踪它们
"""
with self._lock:
self._running.clear()
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
"""
为一个 key 启动执行线程 key 仍有在途动作时不启动
:param key: 动作标识
:param action: 无参恢复动作
:return: 执行线程未启动时为 None
"""
with self._lock:
running = self._running.get(key)
if running is not None and running.is_alive():
return None
thread = threading.Thread(
target=self._execute,
args=(key, action),
name=f"MoviePilot-MonitorRecovery-{key}"[:120],
daemon=True
)
self._running[key] = thread
thread.start()
return thread
@staticmethod
def _execute(key: str, action: Callable[[], None]):
"""
执行一个恢复动作异常只记录不外抛避免一个目录的失败连累整批恢复
:param key: 动作标识
:param action: 无参恢复动作
"""
try:
action()
except Exception as err:
logger.error(f"执行目录监控恢复动作失败: {key} - {err}")
def probe_path(path: Path, timeout: float) -> bool:
"""
用可放弃的子进程探测一个路径是否仍能被访问
必须是子进程在本线程里直接 statblock 型故障下这个调用永不返回探测
线程就成了又一个不可回收的悬挂线程子进程可以在超时后被 kill因此隔离
期间可以无限次周期探测
:param path: 待探测路径
:param timeout: 探测超时秒数
:return: 路径是否可访问
"""
try:
process = subprocess.Popen(
[sys.executable, "-c", _PROBE_SCRIPT, str(path)],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL
)
except Exception as err:
logger.error(f"启动挂载探测子进程失败: {path} - {err}")
return False
try:
return process.wait(timeout=timeout) == 0
except subprocess.TimeoutExpired:
logger.warn(f"挂载探测 {timeout} 秒无响应,判定挂载仍未恢复: {path}")
process.kill()
try:
# 不能用无超时的 wait():进程若卡在 FUSE 上收不掉 SIGKILL
# 这里就会替它把调用线程也一起挂住
process.wait(timeout=_PROBE_KILL_GRACE)
except subprocess.TimeoutExpired:
logger.warn(f"挂载探测子进程未能及时退出,交由系统回收: {path}")
return False
except Exception as err:
logger.error(f"挂载探测执行失败: {path} - {err}")
return False
+20 -6
View File
@@ -21,20 +21,25 @@ class SnapshotStore:
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots") self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
def save(self, storage: str, snapshot: Dict, file_count: int = 0, def save(self, storage: str, snapshot: Dict, file_count: int = 0,
last_snapshot_time: Optional[float] = None) -> bool: last_snapshot_time: Optional[float] = None,
snapshot_time: Optional[float] = None) -> bool:
""" """
保存快照到文件缓存 保存快照到文件缓存
:param storage: 存储名称 :param storage: 存储名称
:param snapshot: 快照数据 :param snapshot: 快照数据
:param file_count: 文件数量用于调整监控间隔 :param file_count: 文件数量用于调整监控间隔
:param last_snapshot_time: 上次快照时间戳 :param last_snapshot_time: 上次快照时间戳
:param snapshot_time: 强制指定的增量游标用于部分路径失败时固定游标不前进
:return: 是否保存成功 :return: 是否保存成功
""" """
try: try:
snapshot_time = max( if snapshot_time is None:
last_snapshot_time or 0, # 取「上次游标」与「本轮最大 mtime」的较大者:本轮全是旧文件时
max((item.get('modify_time', 0) for item in snapshot.values()), default=0) # 游标不能回退,否则已处理过的变更会被重新判定为新增
) snapshot_time = max(
last_snapshot_time or 0,
max((item.get('modify_time', 0) for item in snapshot.values()), default=0)
)
if not snapshot_time: if not snapshot_time:
snapshot_time = time.time() snapshot_time = time.time()
snapshot_data = { snapshot_data = {
@@ -131,7 +136,16 @@ class SnapshotStore:
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0 old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0 new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
if old_size != new_size or (old_time and new_time and old_time != new_time): # 支持文件唯一标识的存储器可用它识别同大小且修改时间未变化的替换文件。
# 旧快照缺少 fileid 时保持保守,避免升级后首次补齐元数据触发全量重整。
old_fileid = old_info.get('fileid') if isinstance(old_info, dict) else None
new_fileid = new_info.get('fileid') if isinstance(new_info, dict) else None
if (
old_size != new_size
or (old_time and new_time and old_time != new_time)
or (old_fileid and new_fileid and old_fileid != new_fileid)
):
changes['modified'].append(file_path) changes['modified'].append(file_path)
return changes return changes
+5 -1
View File
@@ -3,6 +3,7 @@ import platform
from pathlib import Path from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple from typing import Any, Dict, List, Optional, Tuple
from app.core.config import settings
from app.log import logger from app.log import logger
from app.utils.system import SystemUtils from app.utils.system import SystemUtils
@@ -123,7 +124,10 @@ def decide_monitor_mode(directory: Path,
# 检查网络文件系统 # 检查网络文件系统
if SystemUtils.is_network_filesystem(directory): if SystemUtils.is_network_filesystem(directory):
return True, "检测到网络文件系统,建议使用兼容模式", None, None if not settings.MONITOR_NETWORK_FAST_MODE:
return True, "检测到网络文件系统,建议使用兼容模式", None, None
# 用户已确认该挂载支持 inotify,继续走快速模式的系统限制检查
logger.info(f"检测到网络文件系统,但已配置允许快速模式: {directory}")
limits = check_system_limits() limits = check_system_limits()
file_count, dir_count = count_directory_entries(directory) file_count, dir_count = count_directory_entries(directory)
+223 -9
View File
@@ -7,6 +7,7 @@ from typing import Any, Optional
from watchfiles import Change, DefaultFilter, watch from watchfiles import Change, DefaultFilter, watch
from app.core.config import settings
from app.log import logger from app.log import logger
@@ -35,6 +36,17 @@ class LocalDirectoryWatcher:
POLL_DELAY_LOCAL_MS = 300 POLL_DELAY_LOCAL_MS = 300
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力 # 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
POLL_DELAY_NETWORK_MS = 5000 POLL_DELAY_NETWORK_MS = 5000
# 新增目录延迟重扫的间隔秒数默认值:FUSE 上目录内容的可见性有延迟,首次展开时
# 看不到的文件不会再产生任何事件,只能靠延迟重扫补回。默认在常见的 30/120 秒
# 窗口后追加两轮成本极低的长延迟轮次(600s/1800s),应对超大目录树的极端延迟;
# 实际生效值可通过 MONITOR_RESCAN_DELAYS 配置覆盖,见 DIRECTORY_RESCAN_DELAYS
DEFAULT_RESCAN_DELAYS = (30, 120, 600, 1800)
# 待重扫目录队列上限,避免大批量移入时无限增长
MAX_PENDING_RESCANS = 100
# 单个重扫条目允许的连续「整体扫描失败」次数上限:扫描失败(如 FUSE 瞬时抖动)
# 时条目会原地重试而不消耗重扫轮次,必须设置上限,避免目录被删除或长期不可
# 访问时无限重试、占满队列
MAX_RESCAN_FAILURES = 5
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None, def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
poll_delay_ms: Optional[int] = None): poll_delay_ms: Optional[int] = None):
@@ -54,8 +66,12 @@ class LocalDirectoryWatcher:
self._watch_filter = DefaultFilter() self._watch_filter = DefaultFilter()
# 最近一次监控循环活动时间(monotonic),用于检测静默失效 # 最近一次监控循环活动时间(monotonic),用于检测静默失效
self._last_activity: float = 0.0 self._last_activity: float = 0.0
# 最近一次活动的墙钟时间,供监控重建后的补偿扫描定位停摆起点
self._last_activity_wall: float = 0.0
# 累计自动重启次数 # 累计自动重启次数
self._restart_count: int = 0 self._restart_count: int = 0
# 待延迟重扫的新增目录
self._pending_rescans: list[dict] = []
@property @property
def watch_path(self) -> Path: def watch_path(self) -> Path:
@@ -141,11 +157,20 @@ class LocalDirectoryWatcher:
return False return False
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
@property
def last_activity_time(self) -> float:
"""
获取最近一次监控循环活动的墙钟时间
:return: Unix 时间戳从未活动过时为 0
"""
return self._last_activity_wall
def _mark_activity(self): def _mark_activity(self):
""" """
记录一次监控循环活动时间作为静默失效检测的心跳 记录一次监控循环活动时间作为静默失效检测的心跳
""" """
self._last_activity = time.monotonic() self._last_activity = time.monotonic()
self._last_activity_wall = time.time()
def _run(self): def _run(self):
""" """
@@ -199,6 +224,8 @@ class LocalDirectoryWatcher:
self._mark_activity() self._mark_activity()
if self._stop_event.is_set(): if self._stop_event.is_set():
break break
# 空转周期也要推进延迟重扫,否则移入目录后没有新事件就永远不会补扫
self._process_pending_rescans()
if not changes: if not changes:
continue continue
self._handle_changes(changes) self._handle_changes(changes)
@@ -209,7 +236,13 @@ class LocalDirectoryWatcher:
watchfiles 原始变更转换为目录监控事件 watchfiles 原始变更转换为目录监控事件
:param changes: watchfiles 返回的变更集合 :param changes: watchfiles 返回的变更集合
""" """
changes = self._expand_added_directories(changes) self._dispatch_changes(self._expand_added_directories(changes))
def _dispatch_changes(self, changes: set[tuple[Change, str]]):
"""
将变更集合逐个派发给回调
:param changes: 已展开的变更集合
"""
for change_type, path_str in sorted(changes, key=lambda item: item[1]): for change_type, path_str in sorted(changes, key=lambda item: item[1]):
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效 # 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
self._mark_activity() self._mark_activity()
@@ -217,10 +250,17 @@ class LocalDirectoryWatcher:
continue continue
event_path = Path(path_str) event_path = Path(path_str)
event = self._build_event(change_type=change_type, event_path=event_path) event = self._build_event(change_type=change_type, event_path=event_path)
if not event or event.is_directory: if not event:
# 「文件已消失」与「挂载抖动瞬时不可见」在此刻无法区分,静默丢弃就是
# 永久漏件,一律登记重试:真删除的文件由重试队列在下个周期确认后自动放弃
self._notify_unreadable(event_path)
continue
if event.is_directory:
continue continue
file_size = self._get_file_size(event_path) file_size = self._get_file_size(event_path)
if file_size is None: if file_size is None:
# 读取失败通常是挂载抖动,直接丢弃就是永久漏件,交给回调登记重试
self._notify_unreadable(event_path)
continue continue
text = self._change_text(change_type) text = self._change_text(change_type)
try: try:
@@ -233,6 +273,44 @@ class LocalDirectoryWatcher:
except Exception as err: except Exception as err:
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}") logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
@property
def DIRECTORY_RESCAN_DELAYS(self) -> tuple[int, ...]: # noqa: N802 保持与原类常量同名,兼容既有引用/测试
"""
获取当前生效的重扫轮次延迟秒数每次访问都会重新解析 MONITOR_RESCAN_DELAYS
配置配置热更新后无需重建监控线程即可生效解析失败或未配置时回退默认值
:return: 重扫轮次延迟秒数元组
"""
return self._parse_rescan_delays(getattr(settings, "MONITOR_RESCAN_DELAYS", None))
@classmethod
def _parse_rescan_delays(cls, raw: Optional[str]) -> tuple[int, ...]:
"""
解析 MONITOR_RESCAN_DELAYS 配置为重扫轮次延迟秒数元组
:param raw: 配置原始字符串形如 "30,120,600,1800"
:return: 重扫轮次延迟秒数元组解析失败或为空时回退 DEFAULT_RESCAN_DELAYS
"""
if not raw or not raw.strip():
return cls.DEFAULT_RESCAN_DELAYS
try:
delays = tuple(int(part.strip()) for part in raw.split(",") if part.strip())
if not delays or any(delay <= 0 for delay in delays):
raise ValueError(f"重扫延迟必须是正整数: {raw}")
return delays
except (TypeError, ValueError) as err:
logger.warn(f"MONITOR_RESCAN_DELAYS 配置无效({raw!r}),"
f"回退默认值 {cls.DEFAULT_RESCAN_DELAYS}: {err}")
return cls.DEFAULT_RESCAN_DELAYS
@staticmethod
def _is_descendant_of_any(path: Path, candidates: set[Path]) -> bool:
"""
判断 path 是否是 candidates 中某个目录的子孙路径不含自身
:param path: 待判断路径
:param candidates: 候选祖先目录集合
:return: 是否存在祖先命中
"""
return any(candidate in path.parents for candidate in candidates)
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]: def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
""" """
将整体移入监控范围的新增目录展开为内部文件事件 将整体移入监控范围的新增目录展开为内部文件事件
@@ -240,6 +318,12 @@ class LocalDirectoryWatcher:
:return: 包含目录内新增文件的变更集合 :return: 包含目录内新增文件的变更集合
""" """
expanded_changes = set(changes) expanded_changes = set(changes)
# 大目录树整体移入监控范围时,changes 里每一层子目录都会各自产生一次
# added 事件;先收集本批全部新增目录路径,用于判断某个新增目录是否还有
# 祖先目录也在本批新增中——只有「顶层」新增目录需要登记重扫,顶层目录的
# rglob 已经递归覆盖了全部子孙目录的内容,子目录重扫条目纯属冗余,还会在
# 大目录树场景下迅速打满重扫队列
added_dirs = {Path(path_str) for change_type, path_str in changes if change_type == Change.added}
for change_type, path_str in changes: for change_type, path_str in changes:
if change_type != Change.added: if change_type != Change.added:
continue continue
@@ -247,15 +331,145 @@ class LocalDirectoryWatcher:
try: try:
if not event_path.is_dir(): if not event_path.is_dir():
continue continue
for nested_path in event_path.rglob("*"): except OSError as err:
logger.debug(f"读取新增路径类型失败: {event_path} - {err}")
continue
nested_paths, _, _ = self._collect_directory_files(event_path, exclude=set())
for nested_path_str in nested_paths:
expanded_changes.add((Change.added, nested_path_str))
if self._is_descendant_of_any(event_path, added_dirs):
continue
# 目录内容在 FUSE 上可能延迟可见,安排延迟重扫补齐本次看不到的文件
self._schedule_rescan(event_path, seen=nested_paths)
return expanded_changes
def _collect_directory_files(self, directory: Path, exclude: set[str]) -> tuple[set[str], bool, bool]:
"""
收集目录内需要处理的文件路径
:param directory: 目录
:param exclude: 需要排除的路径已处理过的
:return: 三元组 (文件路径集合, 目录是否已不存在终态不算失败,
目录整体扫描是否失败如顶层 rglob 抛出 OSError通常是 FUSE
瞬时抖动属于可重试的暂时性失败与单个条目读取失败区分开
后者不影响整体遍历不计入失败
"""
collected: set[str] = set()
try:
if not directory.is_dir():
# 目录已被删除或从未存在,是终态而非抖动,调用方应据此让条目
# 直接出队,不再计入失败重试
return collected, True, False
except OSError as err:
logger.debug(f"读取目录状态失败,暂视为可重试的扫描失败: {directory} - {err}")
return collected, False, True
try:
for nested_path in directory.rglob("*"):
try:
if not nested_path.is_file(): if not nested_path.is_file():
continue continue
nested_path_str = nested_path.as_posix() except OSError as err:
if self._watch_filter(Change.added, nested_path_str): # 单个条目读取失败不应中断整个目录的遍历,不算整体扫描失败
expanded_changes.add((Change.added, nested_path_str)) logger.debug(f"读取目录内条目失败: {nested_path} - {err}")
except OSError as err: continue
logger.debug(f"扫描新增目录失败: {event_path} - {err}") nested_path_str = nested_path.as_posix()
return expanded_changes if nested_path_str in exclude:
continue
if self._watch_filter(Change.added, nested_path_str):
collected.add(nested_path_str)
except OSError as err:
# 顶层 rglob 中断代表整个目录遍历失败(如 FUSE 瞬时抖动),与「目录
# 已删除」区分开:这里应视为可重试的暂时性失败,而不是静默返回空结果
logger.debug(f"扫描新增目录失败: {directory} - {err}")
return collected, False, True
return collected, False, False
def _schedule_rescan(self, directory: Path, seen: set[str]):
"""
安排一个新增目录的延迟重扫
:param directory: 新增目录
:param seen: 首次展开时已处理的文件路径
"""
if not self.DIRECTORY_RESCAN_DELAYS:
return
if any(item["path"] == directory for item in self._pending_rescans):
# readdir 闪断等原因可能让同一目录产生两次 added 事件,从而被重复
# 展开、重复调用到这里;重复登记只会造成队列膨胀和重复扫描——新事件
# 覆盖到的文件已经在本次展开时派发过,保留已有条目、由它继续推进
# 重扫轮次即可
logger.debug(f"目录已在待重扫队列中,跳过重复登记: {directory}")
return
if self._is_descendant_of_any(directory, {item["path"] for item in self._pending_rescans}):
# 祖先目录的重扫会用 rglob 递归覆盖到这里,无需为子孙目录单独登记
logger.debug(f"目录的祖先已在待重扫队列中,跳过重复登记: {directory}")
return
if len(self._pending_rescans) >= self.MAX_PENDING_RESCANS:
# 队列打满意味着可能有目录的重扫机会被挤掉,之后可能永久漏文件,
# 需要 warn 级别可见,而不是默默丢弃
logger.warn(f"新增目录重扫队列已满(上限 {self.MAX_PENDING_RESCANS}),跳过登记: {directory}")
return
self._pending_rescans.append({
"path": directory,
"seen": set(seen),
"round": 0,
"due": time.monotonic() + self.DIRECTORY_RESCAN_DELAYS[0],
"failures": 0,
})
def _process_pending_rescans(self):
"""
对到期的新增目录做延迟重扫补回首次展开时尚不可见的文件
"""
if not self._pending_rescans:
return
now = time.monotonic()
due_items = [item for item in self._pending_rescans if item["due"] <= now]
if not due_items:
return
self._pending_rescans = [item for item in self._pending_rescans if item["due"] > now]
for item in due_items:
directory = item["path"]
new_paths, is_missing, scan_failed = self._collect_directory_files(directory, exclude=item["seen"])
if is_missing:
# 目录已不存在,是终态:直接出队,不再重新入队,也不计入失败重试
logger.debug(f"待重扫目录已不存在,结束重扫: {directory}")
continue
if scan_failed:
# 整体扫描失败(通常是 FUSE 瞬时抖动):本轮不消耗重扫轮次,
# 让条目原样重新入队,下一个监控周期立即再试;但要有失败上限,
# 避免目录长期不可访问时无限重试、占满队列
item["failures"] = item.get("failures", 0) + 1
if item["failures"] >= self.MAX_RESCAN_FAILURES:
logger.warn(
f"目录延迟重扫连续失败 {item['failures']} 次,放弃重扫: {directory}")
continue
logger.debug(
f"目录延迟重扫本轮扫描失败(第 {item['failures']} 次),不消耗轮次,稍后重试: {directory}")
self._pending_rescans.append(item)
continue
# 本轮扫描成功,重置失败计数
item["failures"] = 0
if new_paths:
logger.info(f"新增目录延迟重扫发现 {len(new_paths)} 个此前不可见的文件: {directory}")
self._dispatch_changes({(Change.added, path_str) for path_str in new_paths})
item["seen"].update(new_paths)
next_round = item["round"] + 1
if next_round < len(self.DIRECTORY_RESCAN_DELAYS):
item["round"] = next_round
item["due"] = now + self.DIRECTORY_RESCAN_DELAYS[next_round]
self._pending_rescans.append(item)
def _notify_unreadable(self, event_path: Path):
"""
通知回调登记读取失败的事件等待重试
:param event_path: 事件文件路径
"""
handler = getattr(self._callback, "event_unreadable", None)
if not callable(handler):
return
try:
handler(event_path=event_path)
except Exception as err:
logger.error(f"登记待重试监控事件失败: {event_path} - {err}")
@staticmethod @staticmethod
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]: def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
+3
View File
@@ -168,6 +168,9 @@ class TransferInfo(BaseModel):
need_scrape: Optional[bool] = False need_scrape: Optional[bool] = False
# 是否需要通知 # 是否需要通知
need_notify: Optional[bool] = False need_notify: Optional[bool] = False
# 是否因覆盖模式判定「不覆盖」而放弃整理。
# 这是一次正常的策略裁决而非整理故障,调用方据此决定是否写失败历史与推送失败通知
overwrite_skipped: Optional[bool] = False
def to_dict(self): def to_dict(self):
""" """
+3
View File
@@ -33,6 +33,7 @@ from app.startup.scheduler_initializer import (
init_scheduler, init_scheduler,
init_plugin_scheduler, init_plugin_scheduler,
) )
from app.startup.transfer_initializer import replay_pending_transfers
from app.startup.workflow_initializer import init_workflow, stop_workflow from app.startup.workflow_initializer import init_workflow, stop_workflow
from app.utils.http import aclose_shared_async_transports from app.utils.http import aclose_shared_async_transports
@@ -91,6 +92,8 @@ async def lifespan(app: FastAPI):
init_scheduler() init_scheduler()
# 初始化监控器 # 初始化监控器
init_monitor() init_monitor()
# 回放上次未整理完的文件(后台线程,不阻塞启动)
replay_pending_transfers()
# 初始化命令 # 初始化命令
init_command() init_command()
# 初始化工作流 # 初始化工作流
+13
View File
@@ -0,0 +1,13 @@
from app.chain.transfer import TransferChain
def replay_pending_transfers():
"""
回放上次进程退出时仍未整理完的文件
整理队列是纯内存的挂载挂死后的人工重启版本升级OOM宿主重启都会让
队列连同这些文件还没整理这个事实一起蒸发而已稳定落地的文件不会再产生
任何监控事件也不会有新的补偿扫描起点结果就是永久漏件
回放本身在后台线程执行不阻塞启动流程
"""
TransferChain().replay_pending()
+141
View File
@@ -0,0 +1,141 @@
"""3.0.2
整理历史按源存储与源路径唯一
Revision ID: 7f5c1d2e3a4b
Revises: 8a4c7e1d2f90
Create Date: 2026-08-04
"""
from collections import defaultdict
from alembic import op
import sqlalchemy as sa
revision = "7f5c1d2e3a4b"
down_revision = "8a4c7e1d2f90"
branch_labels = None
depends_on = None
TABLE_NAME = "transferhistory"
INDEX_NAME = "ux_transferhistory_src_storage"
INDEX_COLUMNS = ["src", "src_storage"]
transferhistory = sa.table(
TABLE_NAME,
sa.column("id", sa.Integer()),
sa.column("src", sa.String()),
sa.column("src_storage", sa.String()),
sa.column("status", sa.Boolean()),
)
def _table_exists(inspector: sa.Inspector) -> bool:
"""检查整理历史表是否存在。"""
return TABLE_NAME in inspector.get_table_names()
def _has_unique_index(inspector: sa.Inspector) -> bool:
"""检查源路径与源存储的唯一索引是否已存在。"""
return any(
tuple(index.get("column_names") or []) == tuple(INDEX_COLUMNS)
and bool(index.get("unique"))
for index in inspector.get_indexes(TABLE_NAME)
)
def _deduplicate_rows() -> None:
"""归一化旧存储值并按现有查重语义清理重复历史。"""
bind = op.get_bind()
bind.execute(
transferhistory.update()
.where(
sa.or_(
transferhistory.c.src_storage.is_(None),
transferhistory.c.src_storage == "",
)
)
.values(src_storage="local")
)
rows = bind.execute(
sa.select(
transferhistory.c.id,
transferhistory.c.src,
transferhistory.c.src_storage,
transferhistory.c.status,
).where(transferhistory.c.src.is_not(None))
).mappings()
grouped_rows = defaultdict(list)
for row in rows:
grouped_rows[(row["src"], row["src_storage"])].append(row)
duplicate_ids = []
for group in grouped_rows.values():
# 旧运行时在同源混有成功和失败记录时优先返回成功记录;保留其中 ID 最新的一条,
# 既延续这一保护语义,也让唯一索引能安全建立。
retained = max(
group,
key=lambda row: (bool(row["status"]), row["id"]),
)
duplicate_ids.extend(
row["id"]
for row in group
if row["id"] != retained["id"]
)
if duplicate_ids:
bind.execute(
transferhistory.delete().where(transferhistory.c.id.in_(duplicate_ids))
)
def _make_src_storage_required(inspector: sa.Inspector) -> None:
"""将源存储设为非空,使唯一索引同样约束本地存储记录。"""
column = next(
(
current
for current in inspector.get_columns(TABLE_NAME)
if current["name"] == "src_storage"
),
None,
)
if not column or not column.get("nullable"):
return
with op.batch_alter_table(TABLE_NAME) as batch_op:
batch_op.alter_column(
"src_storage",
existing_type=sa.String(),
nullable=False,
server_default="local",
)
def upgrade() -> None:
"""归一化并唯一化整理历史的源路径记录。"""
inspector = sa.inspect(op.get_bind())
if not _table_exists(inspector):
return
_deduplicate_rows()
_make_src_storage_required(sa.inspect(op.get_bind()))
inspector = sa.inspect(op.get_bind())
if not _has_unique_index(inspector):
op.create_index(INDEX_NAME, TABLE_NAME, INDEX_COLUMNS, unique=True)
def downgrade() -> None:
"""移除整理历史源路径唯一约束。"""
inspector = sa.inspect(op.get_bind())
if not _table_exists(inspector):
return
if _has_unique_index(inspector):
op.drop_index(INDEX_NAME, table_name=TABLE_NAME)
with op.batch_alter_table(TABLE_NAME) as batch_op:
batch_op.alter_column(
"src_storage",
existing_type=sa.String(),
nullable=True,
server_default=None,
)
+57
View File
@@ -0,0 +1,57 @@
"""3.0.3
新增待整理文件登记表
整理队列是纯内存的 queue.Queue进程重启会让队列里的任务连同这些文件还没
整理这个事实一起蒸发而已稳定落地的文件不会再产生任何监控事件等于永久
漏件该表只登记存储 + 源文件路径这一最小事实供启动时回放
Revision ID: e3d9f4b7c806
Revises: 7f5c1d2e3a4b
Create Date: 2026-08-10
"""
from alembic import op
import sqlalchemy as sa
revision = "e3d9f4b7c806"
down_revision = "7f5c1d2e3a4b"
branch_labels = None
depends_on = None
def _has_table(table_name: str) -> bool:
"""检查数据表是否已存在。"""
inspector = sa.inspect(op.get_bind())
return table_name in inspector.get_table_names()
def upgrade() -> None:
"""
创建待整理文件登记表
"""
if _has_table("transferpending"):
return
op.create_table(
"transferpending",
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
sa.Column("storage", sa.String, nullable=False),
sa.Column("src_path", sa.String, nullable=False),
sa.Column("created_at", sa.String),
)
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
op.create_index(
"ux_transferpending_storage_path",
"transferpending",
["storage", "src_path"],
unique=True,
)
def downgrade() -> None:
"""
删除待整理文件登记表
"""
if not _has_table("transferpending"):
return
op.drop_index("ux_transferpending_storage_path", table_name="transferpending")
op.drop_table("transferpending")
+8 -2
View File
@@ -98,9 +98,15 @@ FROM prepare_package AS prepare_code
WORKDIR /app WORKDIR /app
COPY . . COPY . .
RUN FRONTEND_VERSION=$(sed -n "s/^FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/p" /app/version.py) \ # 构建上下文中存在 frontend-dist/ 时优先使用本地前端产物(fork 自有前端),
# 否则回退下载官方 release 的 dist.zip
RUN if [ -d /app/frontend-dist ]; then \
mv /app/frontend-dist /public; \
else \
FRONTEND_VERSION=$(sed -n "s/^FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/p" /app/version.py) \
&& curl -sL "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${FRONTEND_VERSION}/dist.zip" | busybox unzip -d / - \ && curl -sL "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${FRONTEND_VERSION}/dist.zip" | busybox unzip -d / - \
&& mv /dist /public \ && mv /dist /public; \
fi \
&& curl -sL "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \ && curl -sL "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \
&& mv -f /tmp/MoviePilot-Plugins-main/plugins.v2/* /app/app/plugins/ \ && mv -f /tmp/MoviePilot-Plugins-main/plugins.v2/* /app/app/plugins/ \
&& cat /tmp/MoviePilot-Plugins-main/package.json | jq -r 'to_entries[] | select(.value.v2 == true) | .key' | awk '{print tolower($0)}' | \ && cat /tmp/MoviePilot-Plugins-main/package.json | jq -r 'to_entries[] | select(.value.v2 == true) | .key' | awk '{print tolower($0)}' | \
+224
View File
@@ -0,0 +1,224 @@
"""
本地文件系统操作代理测试
代理存在的唯一理由FUSE 挂载 block 型故障下stat/listdir 这类调用永不返回
Python 既不能中断已发出的系统调用也不能强杀线程放进子进程后超时可以
真正 SIGKILL 回收block 型故障被转换成各层已能处理的 crash 型故障
这些测试固定三项不变量语义与直接调用一致超时可放弃代理死亡后能自愈
"""
import errno
import json
import os
import subprocess
import sys
import time
from pathlib import Path
import pytest
from app.modules.filemanager.fsproxy import FileSystemProxy, FileSystemTimeout
@pytest.fixture
def proxy():
"""
每个用例一个独立代理用完回收进程
"""
instance = FileSystemProxy(timeout=30)
yield instance
instance.close()
# --------------------------------------------------------------------------- #
# 语义等价:代理调用的结果必须与直接调用一致
# --------------------------------------------------------------------------- #
def test_stat_matches_direct_call(tmp_path, proxy):
"""
stat 的结果要与直接调用一致调用方才能无感替换
"""
media = tmp_path / "Movie.2024.mkv"
media.write_bytes(b"x" * 42)
result = proxy.stat(media)
assert result["size"] == 42
assert result["is_file"] is True
assert result["is_dir"] is False
assert result["mtime"] == pytest.approx(media.stat().st_mtime, abs=1)
def test_stat_on_directory(tmp_path, proxy):
"""
目录的 stat 要正确标记类型
"""
result = proxy.stat(tmp_path)
assert result["is_dir"] is True
assert result["is_file"] is False
def test_stat_missing_raises_file_not_found(tmp_path, proxy):
"""
errno 必须还原成具体的 OSError 子类调用方的既有异常分支才能继续生效
"""
with pytest.raises(FileNotFoundError):
proxy.stat(tmp_path / "nope.mkv")
def test_exists_true_and_false(tmp_path, proxy):
"""
exists 的基本语义
"""
media = tmp_path / "a.mkv"
media.write_bytes(b"x")
assert proxy.exists(media) is True
assert proxy.exists(tmp_path / "missing.mkv") is False
def test_listdir_matches_direct_call(tmp_path, proxy):
"""
listdir 的结果要与直接调用一致
"""
for name in ("b.mkv", "a.mkv", "c.srt"):
(tmp_path / name).write_bytes(b"x")
assert proxy.listdir(tmp_path) == sorted(os.listdir(tmp_path))
def test_listdir_missing_raises_file_not_found(tmp_path, proxy):
"""
目录不存在要抛 FileNotFoundError
"""
with pytest.raises(FileNotFoundError):
proxy.listdir(tmp_path / "nodir")
def test_rename_within_same_storage(tmp_path, proxy):
"""
同存储 rename 是第一版唯一放行的写操作内核保证原子性
强杀后要么完全成功要么完全没发生不需要恢复语义
"""
src = tmp_path / "old.mkv"
dst = tmp_path / "new.mkv"
src.write_bytes(b"data")
assert proxy.rename(src, dst) is True
assert dst.read_bytes() == b"data"
assert not src.exists()
# --------------------------------------------------------------------------- #
# 核心价值:超时可放弃
# --------------------------------------------------------------------------- #
def test_timeout_raises_and_kills_worker(tmp_path, monkeypatch):
"""
这是整个代理存在的意义挂载不返回时调用方必须在有限时间内拿到异常
而且冻住的进程要被真正杀掉不能像线程那样永久悬挂
"""
import app.modules.filemanager.fsproxy as fsproxy_module
# 用一个必定挂死的 worker 替身,模拟 stat 永不返回的挂载
stuck_worker = tmp_path / "stuck_worker.py"
stuck_worker.write_text("import time\nwhile True:\n time.sleep(60)\n", encoding="utf-8")
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck_worker)
proxy = FileSystemProxy(timeout=0.5)
try:
started = time.monotonic()
with pytest.raises(FileSystemTimeout) as excinfo:
proxy.stat(tmp_path / "whatever.mkv")
elapsed = time.monotonic() - started
assert elapsed < 10, "超时后没有及时放弃"
assert excinfo.value.errno == errno.ETIMEDOUT
# 冻住的代理必须已被回收,否则每次故障都会泄漏一个进程
assert proxy._process is None
finally:
proxy.close()
def test_timeout_is_an_oserror():
"""
超时必须是 OSError 子类整理链与监控 watcher OSError 已有完整的退避
重试与登记逻辑block 型故障经此转换后可直接复用无需改动各层调用点
"""
assert issubclass(FileSystemTimeout, OSError)
def test_proxy_recovers_after_timeout(tmp_path, monkeypatch):
"""
超时杀掉代理后下一次调用要能用新代理正常工作否则一次挂载抖动
就会让文件操作永久不可用
"""
import app.modules.filemanager.fsproxy as fsproxy_module
stuck_worker = tmp_path / "stuck_worker.py"
stuck_worker.write_text("import time\nwhile True:\n time.sleep(60)\n", encoding="utf-8")
real_worker = fsproxy_module._WORKER_PATH
proxy = FileSystemProxy(timeout=0.5)
try:
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck_worker)
with pytest.raises(FileSystemTimeout):
proxy.stat(tmp_path)
# 挂载恢复:换回真正的 worker,代理应自动重启并正常服务
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", real_worker)
media = tmp_path / "recovered.mkv"
media.write_bytes(b"ok")
assert proxy.stat(media)["size"] == 2
finally:
proxy.close()
def test_proxy_restarts_after_worker_dies(tmp_path, proxy):
"""
代理进程被外部杀掉OOM误杀下一次调用要能自动重启
"""
media = tmp_path / "a.mkv"
media.write_bytes(b"x")
assert proxy.stat(media)["size"] == 1
# 模拟代理进程被外部杀死
proxy._process.kill()
proxy._process.wait(timeout=5)
assert proxy.stat(media)["size"] == 1
def test_worker_does_not_import_app_package():
"""
worker 必须只依赖标准库一旦触发 app/__init__.py 的导入链启动成本会从
毫秒级涨到秒级代理被强杀后的重启就不再可行
"""
worker = Path("app/modules/filemanager/fsworker.py").read_text(encoding="utf-8")
assert "from app." not in worker
assert "import app" not in worker
def test_worker_runs_standalone_without_app_on_path(tmp_path):
"""
直接执行 worker 脚本必须成功且不依赖项目根在 sys.path
这是绕开 app 导入链这一设计前提的实证
"""
worker_path = Path("app/modules/filemanager/fsworker.py").resolve()
media = tmp_path / "x.mkv"
media.write_bytes(b"xyz")
result = subprocess.run(
[sys.executable, str(worker_path)],
input=json.dumps({"op": "stat", "path": str(media)}) + "\n",
capture_output=True,
text=True,
timeout=30,
cwd=str(tmp_path),
)
response = json.loads(result.stdout.strip())
assert response["ok"] is True
assert response["result"]["size"] == 3
+194
View File
@@ -0,0 +1,194 @@
"""
文件系统代理的流式复制测试
复制大文件可能持续几小时固定超时无法区分正常但慢已经挂死
worker 每秒上报一次进度作为心跳父进程判定的是**两次上报之间的间隔**
超过 stall 阈值收不到任何一行才认定挂载无响应并强杀 worker
这些测试固定的不变量内容与时间戳正确进度可回调慢传输不被误杀
挂死能被判定并回收取消可即时生效
"""
import os
import time
from pathlib import Path
import pytest
from app.modules.filemanager.fsproxy import FileSystemProxy, FileSystemTimeout
@pytest.fixture
def proxy():
"""
每个用例一个独立代理用完回收进程
"""
instance = FileSystemProxy(timeout=30, stall_timeout=30)
yield instance
instance.close()
def test_copy_preserves_content_and_mtime(tmp_path, proxy):
"""
复制结果的内容与修改时间必须与源一致
时间戳要保留但权限不能覆盖目标目录的默认权限与继承 ACL 是媒体库的访问
策略用源文件权限盖掉会清除已继承的 ACL
"""
src = tmp_path / "Movie.2024.mkv"
src.write_bytes(b"payload" * 5000)
old = time.time() - 86400
os.utime(src, (old, old))
dst = tmp_path / "out.mkv"
size = src.stat().st_size
assert proxy.copy(src, dst) == {"copied": size, "total": size}
assert dst.read_bytes() == src.read_bytes()
assert dst.stat().st_mtime == pytest.approx(src.stat().st_mtime, abs=1)
def test_copy_empty_file(tmp_path, proxy):
"""
空文件不能因为没有任何数据块就走进异常分支
"""
src = tmp_path / "empty.mkv"
src.write_bytes(b"")
dst = tmp_path / "out.mkv"
assert proxy.copy(src, dst)["total"] == 0
assert dst.exists()
def test_copy_reports_progress(tmp_path, proxy):
"""
进度必须被回调出来且单调递增不超过 100
"""
src = tmp_path / "big.mkv"
src.write_bytes(b"x" * (3 * 1024 * 1024))
dst = tmp_path / "out.mkv"
seen = []
# 用小 chunk 让传输持续足够久,确保能观察到进度上报
assert proxy.copy(src, dst, progress_cb=seen.append, chunk_size=4096)
assert seen == sorted(seen), "进度不是单调递增的"
assert all(0 <= value <= 100 for value in seen)
def test_copy_missing_source_raises_file_not_found(tmp_path, proxy):
"""
源文件不存在要还原成 FileNotFoundError调用方的既有分支才能生效
"""
with pytest.raises(FileNotFoundError):
proxy.copy(tmp_path / "nope.mkv", tmp_path / "out.mkv")
def test_slow_transfer_is_not_killed(tmp_path):
"""
关键区分之一传输很慢但仍在推进时绝不能被判定为挂死
stall 阈值取得远小于可能的总耗时只要心跳不断就必须一路跑完
"""
src = tmp_path / "slow.mkv"
src.write_bytes(b"x" * (2 * 1024 * 1024))
dst = tmp_path / "out.mkv"
proxy = FileSystemProxy(timeout=30, stall_timeout=3)
try:
assert proxy.copy(src, dst, chunk_size=8192)
assert dst.read_bytes() == src.read_bytes()
finally:
proxy.close()
def test_stalled_transfer_is_detected_and_killed(tmp_path, monkeypatch):
"""
关键区分之二传输彻底不推进时必须被判定并强杀而不是永久等待
"""
import app.modules.filemanager.fsproxy as fsproxy_module
# worker 替身:读到请求后完全不响应,模拟卡死在挂载上的传输
stuck = tmp_path / "stuck_worker.py"
stuck.write_text(
"import sys, time\nsys.stdin.readline()\nwhile True:\n time.sleep(60)\n",
encoding="utf-8"
)
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck)
proxy = FileSystemProxy(timeout=30, stall_timeout=0.5)
try:
started = time.monotonic()
with pytest.raises(FileSystemTimeout):
proxy.copy(tmp_path / "a.mkv", tmp_path / "b.mkv")
assert time.monotonic() - started < 15, "挂死的传输没有被及时放弃"
assert proxy._process is None, "冻住的代理进程没有被回收"
finally:
proxy.close()
def test_copy_can_be_cancelled_midway(tmp_path):
"""
取消要即时生效父进程收到进度就检查取消标记命中即杀掉 worker 中断传输
不必等它把整个文件读完
"""
src = tmp_path / "big.mkv"
src.write_bytes(b"x" * (4 * 1024 * 1024))
dst = tmp_path / "out.mkv"
proxy = FileSystemProxy(timeout=30, stall_timeout=30)
try:
assert proxy.copy(src, dst, cancel_cb=lambda: True, chunk_size=4096) is False
# 取消后代理已被回收,下一次调用会重启一个干净的 worker
assert proxy._process is None
finally:
proxy.close()
def test_copy_falls_back_to_direct_when_disabled(tmp_path, monkeypatch):
"""
代理关闭时复制退回进程内直接执行行为与引入代理之前一致
"""
import app.modules.filemanager.fsproxy as fsproxy_module
monkeypatch.setattr(fsproxy_module.settings, "FS_PROXY_ENABLED", False, raising=False)
src = tmp_path / "a.mkv"
src.write_bytes(b"direct" * 100)
dst = tmp_path / "b.mkv"
proxy = FileSystemProxy()
try:
assert proxy.copy(src, dst) is True
assert dst.read_bytes() == src.read_bytes()
# 关闭时不应启动任何子进程
assert proxy._process is None
finally:
proxy.close()
def test_direct_copy_honours_cancel(tmp_path, monkeypatch):
"""
关闭代理时取消同样要生效否则关掉开关就丢了取消能力
"""
import app.modules.filemanager.fsproxy as fsproxy_module
monkeypatch.setattr(fsproxy_module.settings, "FS_PROXY_ENABLED", False, raising=False)
src = tmp_path / "a.mkv"
src.write_bytes(b"x" * 8192)
dst = tmp_path / "b.mkv"
proxy = FileSystemProxy()
try:
assert proxy.copy(src, dst, cancel_cb=lambda: True, chunk_size=1024) is False
finally:
proxy.close()
def test_worker_still_standalone_after_streaming_support():
"""
加了流式协议之后 worker 仍须只依赖标准库一旦引入 app 导入链
强杀后的重启成本会从毫秒级涨到秒级整个代理方案就不成立了
"""
worker = Path("app/modules/filemanager/fsworker.py").read_text(encoding="utf-8")
assert "from app." not in worker
assert "import app" not in worker
+207
View File
@@ -0,0 +1,207 @@
"""
本地存储写入原子性测试
copy / 跨盘 move / upload 原先直接写目标路径进程被杀OOM重启宿主断电
SIGKILL时目标目录会留下一个**半截的媒体文件而且叫最终文件名**媒体库会
把它扫进去后续的目标已存在检查也会把它当成完成品
改成写临时名 os.replaceos.replace 在同目录内由内核保证原子性
中断只可能留下一个带专用后缀的隐藏文件最终文件要么完整存在要么根本不存在
"""
import os
from pathlib import Path
import pytest
from app.modules.filemanager.storages.local import LocalStorage
from app.schemas import FileItem
def _fileitem(path: Path) -> FileItem:
"""
构造本地文件项
:param path: 文件路径
:return: 文件项
"""
return FileItem(
storage="local",
type="file",
path=path.as_posix(),
name=path.name,
basename=path.stem,
extension=path.suffix[1:],
)
@pytest.fixture
def storage():
"""
本地存储实例
"""
return LocalStorage()
def test_copy_produces_complete_file(tmp_path, storage):
"""
正常复制的结果必须与直接复制一致
"""
src = tmp_path / "src" / "Movie.2024.mkv"
src.parent.mkdir()
src.write_bytes(b"payload" * 100)
dest_dir = tmp_path / "library"
dest_dir.mkdir()
assert storage.copy(_fileitem(src), dest_dir, "Movie.2024.mkv") is True
assert (dest_dir / "Movie.2024.mkv").read_bytes() == b"payload" * 100
# 复制完成后不能残留任何临时文件
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
def test_copy_leaves_no_final_name_on_failure(tmp_path, storage, monkeypatch):
"""
复制中途失败时目标文件名绝不能出现这是本次修复的核心
失败点选在内容写完替换之前正是原实现留下完整文件名的半成品的位置
"""
src = tmp_path / "Movie.2024.mkv"
src.write_bytes(b"x" * 1000)
dest_dir = tmp_path / "library"
dest_dir.mkdir()
def boom(*_args, **_kwargs):
"""
模拟替换阶段被打断
"""
raise OSError(28, "No space left on device")
monkeypatch.setattr(os, "replace", boom)
assert storage.copy(_fileitem(src), dest_dir, "Movie.2024.mkv") is False
# 最终文件名不存在,媒体库不会扫到半成品
assert not (dest_dir / "Movie.2024.mkv").exists()
# 临时文件也要清掉,不留垃圾
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
def test_partial_file_is_hidden_and_suffixed(tmp_path, storage):
"""
临时文件必须同时满足点开头隐藏+ 专用后缀
媒体库按扩展名识别媒体文件点开头又能让多数扫描器跳过两者叠加保证
即使进程被 SIGKILL临时文件残留也不会被当成媒体收录
"""
partial = storage._partial_path(tmp_path / "Movie.2024.mkv")
assert partial.name.startswith(".")
assert partial.name.endswith(LocalStorage.PARTIAL_SUFFIX)
# 必须与目标同目录,os.replace 才是原子的(跨文件系统会退化成拷贝)
assert partial.parent == tmp_path
assert partial.suffix != ".mkv"
def test_move_produces_complete_file_and_removes_source(tmp_path, storage, monkeypatch):
"""
跨盘移动完成后目标完整源被删除
"""
src = tmp_path / "src" / "Movie.2024.mkv"
src.parent.mkdir()
src.write_bytes(b"data" * 50)
dest_dir = tmp_path / "library"
dest_dir.mkdir()
# 强制走跨盘路径(copy + unlink):只让「源 → 最终目标」的直接 rename 以
# EXDEV 失败,临时文件到目标的替换仍需正常工作
real_replace = os.replace
def exdev_for_source(source, target, *args, **kwargs):
"""
模拟源与目标不在同一文件系统
"""
if Path(source) == src:
raise OSError(18, "Invalid cross-device link")
return real_replace(source, target, *args, **kwargs)
monkeypatch.setattr(os, "replace", exdev_for_source)
assert storage.move(_fileitem(src), dest_dir, "Movie.2024.mkv") is True
assert (dest_dir / "Movie.2024.mkv").read_bytes() == b"data" * 50
assert not src.exists()
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
def test_move_keeps_source_when_copy_fails(tmp_path, storage, monkeypatch):
"""
跨盘移动失败时源文件必须保留先删源再失败就是永久丢件
"""
src = tmp_path / "src" / "Movie.2024.mkv"
src.parent.mkdir()
src.write_bytes(b"data")
dest_dir = tmp_path / "library"
dest_dir.mkdir()
def boom(*_args, **_kwargs):
"""
模拟直接移动与临时文件替换都失败磁盘写满
"""
raise OSError(28, "No space left on device")
monkeypatch.setattr(os, "replace", boom)
assert storage.move(_fileitem(src), dest_dir, "Movie.2024.mkv") is False
assert src.read_bytes() == b"data", "移动失败却删掉了源文件,等于永久丢件"
assert not (dest_dir / "Movie.2024.mkv").exists()
def test_same_path_move_is_noop(tmp_path, storage):
"""
源与目标相同时应直接成功不能把文件写没了
"""
media = tmp_path / "Movie.2024.mkv"
media.write_bytes(b"keep")
assert storage.move(_fileitem(media), tmp_path, "Movie.2024.mkv") is True
assert media.read_bytes() == b"keep"
def test_cleanup_removes_stale_partials_only(tmp_path, storage):
"""
清理只针对陈旧的临时文件正在写入的临时文件与正常媒体文件都不能被误删
不做全库扫描只在实际写入的目录做局部清理全库遍历在网络挂载上代价
不可接受而残留只会出现在曾经写入过的目录里
"""
stale = tmp_path / f".Old.mkv.1234{LocalStorage.PARTIAL_SUFFIX}"
fresh = tmp_path / f".New.mkv.5678{LocalStorage.PARTIAL_SUFFIX}"
media = tmp_path / "Keep.mkv"
for item in (stale, fresh, media):
item.write_bytes(b"x")
# 把陈旧临时文件的 mtime 推到阈值之前
old_time = os.stat(stale).st_mtime - LocalStorage.PARTIAL_STALE_SECONDS - 60
os.utime(stale, (old_time, old_time))
storage._cleanup_stale_partials(tmp_path)
assert not stale.exists(), "陈旧临时文件没有被清理"
assert fresh.exists(), "正在写入的临时文件被误删了"
assert media.exists(), "正常媒体文件被误删了"
def test_cleanup_never_raises(tmp_path, storage, monkeypatch):
"""
清理是尽力而为的旁路操作任何失败都不能影响主流程
"""
def boom(*_args, **_kwargs):
"""
模拟目录不可读
"""
raise OSError(5, "Input/output error")
monkeypatch.setattr(Path, "glob", boom)
# 不抛异常即为通过
storage._cleanup_stale_partials(tmp_path)
+16 -3
View File
@@ -95,16 +95,29 @@ def test_cross_device_move_keeps_target_permissions(tmp_path: Path) -> None:
source_content = source.read_bytes() source_content = source.read_bytes()
storage = _make_storage() storage = _make_storage()
real_replace = local_storage_module.os.replace
def _exdev_for_source(src_path, dst_path, *args, **kwargs):
"""
只让 最终目标的直接移动以 EXDEV 失败降级路径里
临时文件 目标的替换仍需正常工作否则整个移动都会失败
"""
if Path(src_path) == source:
raise OSError(errno.EXDEV, "跨设备移动")
return real_replace(src_path, dst_path, *args, **kwargs)
with ( with (
patch.object( patch.object(
local_storage_module.LocalStorage, local_storage_module.LocalStorage,
"_LocalStorage__should_show_progress", "_LocalStorage__should_show_progress",
return_value=False, return_value=False,
), ),
# move 现在显式用 os.replace 做同盘原子移动,EXDEV 失败才降级为
# 「写临时名 → 替换」的复制路径,因此在这里注入跨设备错误
patch.object( patch.object(
shutil.os, local_storage_module.os,
"rename", "replace",
side_effect=OSError(errno.EXDEV, "跨设备移动"), side_effect=_exdev_for_source,
), ),
): ):
result = storage.move( result = storage.move(
+193 -10
View File
@@ -5,7 +5,14 @@ from app.api.endpoints.transfer import (
query_manual_transfer_history, query_manual_transfer_history,
) )
from app.chain.transfer import TransferChain from app.chain.transfer import TransferChain
from app.core.config import settings
from app.db.transferhistory_oper import TransferHistoryOper from app.db.transferhistory_oper import TransferHistoryOper
from app.helper.transferhistory import (
clear_transfer_failures,
failed_retry_count,
max_failed_retries,
record_transfer_failure,
)
from app.schemas import FileItem, ManualTransferItem from app.schemas import FileItem, ManualTransferItem
from tests.test_transfer_sync_extra_files import ( from tests.test_transfer_sync_extra_files import (
FakeMeta, FakeMeta,
@@ -14,6 +21,11 @@ from tests.test_transfer_sync_extra_files import (
) )
def _reset_failed_retries(src_path, storage=None):
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
clear_transfer_failures(src_path, storage)
def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, deleted): def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, deleted):
"""隔离整理规划依赖,并记录历史及旧目标清理动作。""" """隔离整理规划依赖,并记录历史及旧目标清理动作。"""
monkeypatch.setattr( monkeypatch.setattr(
@@ -42,6 +54,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del
history_oper = SimpleNamespace( history_oper = SimpleNamespace(
get_by_src=lambda src, storage=None: history, get_by_src=lambda src, storage=None: history,
get_success_by_src=lambda src, storage=None: history if history and history.status else None,
get_by_dest=lambda dest, storage=None: None, get_by_dest=lambda dest, storage=None: None,
delete=lambda history_id: deleted.append(("history", history_id)), delete=lambda history_id: deleted.append(("history", history_id)),
) )
@@ -286,6 +299,8 @@ def test_manual_transfer_removes_failed_history_before_retry(monkeypatch):
dest_fileitem=old_dest.model_dump(), dest_fileitem=old_dest.model_dump(),
download_hash=None, download_hash=None,
downloader=None, downloader=None,
src=None,
src_storage=None,
) )
planned = [] planned = []
deleted = [] deleted = []
@@ -314,8 +329,8 @@ def test_manual_transfer_removes_failed_history_before_retry(monkeypatch):
assert planned == [fileitem.path] assert planned == [fileitem.path]
def test_automatic_transfer_keeps_failed_history(monkeypatch): def test_automatic_transfer_retries_failed_history_within_retry_budget(monkeypatch):
"""自动整理仍应保留失败历史并跳过,避免失败任务自动循环""" """失败重试次数未达上限(默认计数为 0)时,自动整理遇到失败记录应放行重试,避免一次瞬时故障永久锁死文件"""
chain = make_transfer_chain() chain = make_transfer_chain()
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv") fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
history = SimpleNamespace( history = SimpleNamespace(
@@ -327,6 +342,8 @@ def test_automatic_transfer_keeps_failed_history(monkeypatch):
).model_dump(), ).model_dump(),
download_hash=None, download_hash=None,
downloader=None, downloader=None,
src=fileitem.path,
src_storage=fileitem.storage,
) )
planned = [] planned = []
deleted = [] deleted = []
@@ -339,17 +356,177 @@ def test_automatic_transfer_keeps_failed_history(monkeypatch):
deleted, deleted,
) )
state, message = TransferChain.do_transfer( _reset_failed_retries(fileitem.path, fileitem.storage)
try:
state, message = TransferChain.do_transfer(
chain,
fileitem=fileitem,
background=False,
manual=False,
)
assert state is True
assert message == ""
assert deleted == []
assert planned == [fileitem.path]
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_automatic_transfer_skips_failed_history_when_retry_budget_exhausted(monkeypatch):
"""失败重试次数已达上限时,自动整理遇到失败记录仍应被拦截、不进入整理链,避免失败任务自动循环。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
chain = make_transfer_chain()
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
history = SimpleNamespace(
id=13,
status=False,
mode="copy",
dest_fileitem=make_fileitem(
"/library/Test Show/Test.Show.S01E01.mkv"
).model_dump(),
download_hash=None,
downloader=None,
src=fileitem.path,
src_storage=fileitem.storage,
)
planned = []
deleted = []
_patch_transfer_planning(
monkeypatch,
chain, chain,
fileitem=fileitem, fileitem,
background=False, history,
manual=False, planned,
deleted,
) )
assert state is False _reset_failed_retries(fileitem.path, fileitem.storage)
assert message == f"{fileitem.name} 已整理过" try:
assert deleted == [] record_transfer_failure(fileitem.path, fileitem.storage)
assert planned == []
state, message = TransferChain.do_transfer(
chain,
fileitem=fileitem,
background=False,
manual=False,
)
assert state is False
assert message == f"{fileitem.name} 已整理过"
assert deleted == []
assert planned == []
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_automatic_transfer_new_version_bypasses_exhausted_retry_budget(monkeypatch):
"""失败预算耗尽后同路径新版本必须进入整理,且不能给下载任务补打已整理标签。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
chain = make_transfer_chain()
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
fileitem.size = 200
history = SimpleNamespace(
id=15,
status=False,
mode="copy",
dest_fileitem=None,
download_hash="abc123",
downloader="qbittorrent",
src=fileitem.path,
src_storage=fileitem.storage,
src_fileitem={"size": 100},
)
planned = []
deleted = []
_patch_transfer_planning(
monkeypatch,
chain,
fileitem,
history,
planned,
deleted,
)
completed = []
chain.transfer_completed = lambda download_hash, downloader: completed.append(
(download_hash, downloader)
)
_reset_failed_retries(fileitem.path, fileitem.storage)
try:
record_transfer_failure(fileitem.path, fileitem.storage, file_size=100)
state, message = TransferChain.do_transfer(
chain,
fileitem=fileitem,
downloader="qbittorrent",
download_hash="abc123",
background=False,
manual=False,
)
assert state is True
assert message == ""
assert deleted == []
assert planned == [fileitem.path]
assert completed == []
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_manual_transfer_bypasses_retry_budget_when_exhausted(monkeypatch):
"""
手动整理不受重试次数上限限制manual=True 时查重闸根本不参与判定
do_transfer `if transferd and not manual` 已把 manual 路径排除在外
即使失败计数已达上限自动路径会因此拦截仍应清理失败记录重置计数并放行重试
"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
chain = make_transfer_chain()
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
old_dest = make_fileitem("/library/Test Show/Test.Show.S01E01.mkv")
history = SimpleNamespace(
id=14,
status=False,
mode="copy",
dest_fileitem=old_dest.model_dump(),
download_hash=None,
downloader=None,
src=fileitem.path,
src_storage=fileitem.storage,
)
planned = []
deleted = []
_patch_transfer_planning(
monkeypatch,
chain,
fileitem,
history,
planned,
deleted,
)
_reset_failed_retries(fileitem.path, fileitem.storage)
try:
record_transfer_failure(fileitem.path, fileitem.storage)
assert failed_retry_count(fileitem.path, fileitem.storage) >= max_failed_retries()
state, message = TransferChain.do_transfer(
chain,
fileitem=fileitem,
background=False,
manual=True,
)
assert state is True
assert message == ""
assert deleted == [
("target", old_dest.path),
("history", history.id),
]
assert planned == [fileitem.path]
assert failed_retry_count(fileitem.path, fileitem.storage) == 0
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_manual_transfer_keeps_success_history_without_confirmation(monkeypatch): def test_manual_transfer_keeps_success_history_without_confirmation(monkeypatch):
@@ -402,6 +579,8 @@ def test_manual_reorganize_removes_success_history_and_old_target(monkeypatch):
dest_fileitem=old_dest.model_dump(), dest_fileitem=old_dest.model_dump(),
download_hash=None, download_hash=None,
downloader=None, downloader=None,
src=None,
src_storage=None,
) )
planned = [] planned = []
deleted = [] deleted = []
@@ -442,6 +621,8 @@ def test_manual_reorganize_keeps_successful_move_target_as_source(monkeypatch):
dest_fileitem=fileitem.model_dump(), dest_fileitem=fileitem.model_dump(),
download_hash=None, download_hash=None,
downloader=None, downloader=None,
src=None,
src_storage=None,
) )
planned = [] planned = []
deleted = [] deleted = []
@@ -480,6 +661,8 @@ def test_forced_manual_reorganize_still_removes_history(monkeypatch):
dest_fileitem=old_dest.model_dump(), dest_fileitem=old_dest.model_dump(),
download_hash=None, download_hash=None,
downloader=None, downloader=None,
src=None,
src_storage=None,
) )
planned = [] planned = []
deleted = [] deleted = []
+278
View File
@@ -0,0 +1,278 @@
"""目录监控分发器的整理历史查重与整理异常重试测试。"""
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import MagicMock
from app.core.config import settings
from app.helper.transferhistory import (
clear_transfer_failures,
failed_retry_count,
record_transfer_failure,
)
from app.monitor.dispatcher import TransferDispatcher
def _build_dispatcher() -> TransferDispatcher:
"""
构造测试用整理分发器使用普通字典充当去重缓存
:return: 整理分发器
"""
return TransferDispatcher(all_exts=[".mkv"], cache={})
def _history(status: bool = True, size=None, src_fileitem=..., src=None, src_storage=None,
history_id: int = 1):
"""
构造整理历史记录替身
:param status: 整理是否成功
:param size: 记录中的源文件大小
:param src_fileitem: 直接指定源文件项默认按 size 生成
:param src: 记录源路径未指定时查重闸的失败重试计数按 None 处理恒为 0
:param src_storage: 记录源存储
:param history_id: 记录 IDdescribe_history_gate 的日志文案需要
:return: 整理历史记录替身
"""
if src_fileitem is ...:
src_fileitem = {"size": size}
return SimpleNamespace(id=history_id, status=status, src_fileitem=src_fileitem,
src=src, src_storage=src_storage)
def _reset_failed_retries(src_path, storage=None):
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
clear_transfer_failures(src_path, storage)
def _patch_history(monkeypatch, record=None, success_record=None) -> MagicMock:
"""
替换整理历史查询返回指定记录
:param monkeypatch: pytest monkeypatch
:param record: get_by_src 返回的记录
:param success_record: 成功记录二次确认的返回值
:return: 整理历史操作替身
"""
oper = MagicMock()
oper.get_by_src.return_value = record
oper.get_success_by_src.return_value = success_record
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", MagicMock(return_value=oper))
return oper
def _patch_chain(monkeypatch, side_effect=None) -> MagicMock:
"""
替换整理链记录整理调用
:param monkeypatch: pytest monkeypatch
:param side_effect: do_transfer 的副作用
:return: 整理链替身
"""
chain = MagicMock()
if side_effect is not None:
chain.do_transfer.side_effect = side_effect
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", MagicMock(return_value=chain))
return chain
def test_no_history_goes_to_transfer(monkeypatch):
"""没有任何整理记录的文件应直接进入整理链。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=None)
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is True
chain.do_transfer.assert_called_once()
def test_failed_history_is_retried_within_retry_budget(monkeypatch):
"""失败重试次数未达上限(默认计数为 0)时,失败的整理记录不得永久锁死文件,应放行重试。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=_history(status=False, size=100))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is True
chain.do_transfer.assert_called_once()
def test_failed_history_is_skipped_when_retry_budget_exhausted(monkeypatch):
"""失败重试次数已达上限时,失败的整理记录仍应被拦截、跳过整理。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
src_path = "/downloads/a.mkv"
_reset_failed_retries(src_path, "local")
try:
record_transfer_failure(src_path, "local")
dispatcher = _build_dispatcher()
_patch_history(monkeypatch,
record=_history(status=False, size=100, src=src_path, src_storage="local"))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path(src_path), file_size=100) is False
chain.do_transfer.assert_not_called()
finally:
_reset_failed_retries(src_path, "local")
def test_failed_history_new_version_bypasses_exhausted_retry_budget(monkeypatch):
"""失败预算耗尽后新版本仍应进入整理链,而不是被错误标记为已处理。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
src_path = "/downloads/failed-new-version.mkv"
_reset_failed_retries(src_path, "local")
try:
record_transfer_failure(src_path, "local", file_size=100)
dispatcher = _build_dispatcher()
_patch_history(
monkeypatch,
record=_history(
status=False,
size=100,
src=src_path,
src_storage="local",
),
)
chain = _patch_chain(monkeypatch)
assert (
dispatcher.handle_file(
storage="local",
event_path=Path(src_path),
file_size=200,
)
is True
)
chain.do_transfer.assert_called_once()
finally:
_reset_failed_retries(src_path, "local")
def test_should_skip_by_history_returns_true_when_retry_budget_exhausted(monkeypatch):
"""直接调用 _should_skip_by_history:失败计数达到上限时应判定为跳过。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
src_path = "/downloads/b.mkv"
_reset_failed_retries(src_path, "local")
try:
record_transfer_failure(src_path, "local")
_patch_history(monkeypatch,
record=_history(status=False, size=100, src=src_path, src_storage="local"))
result = TransferDispatcher._should_skip_by_history(
storage="local", src_path=src_path, file_size=100
)
assert result is True
finally:
_reset_failed_retries(src_path, "local")
def test_success_history_with_changed_size_is_retried(monkeypatch):
"""同路径重新上传的不同版本应放行,由整理链的覆盖模式决断。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=_history(status=True, size=100))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=200) is True
chain.do_transfer.assert_called_once()
def test_success_history_without_change_is_skipped(monkeypatch):
"""已成功整理且文件未变化时跳过,并留下 debug 痕迹便于排查。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=_history(status=True, size=100))
chain = _patch_chain(monkeypatch)
recorder = MagicMock()
monkeypatch.setattr("app.monitor.dispatcher.logger", recorder)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
chain.do_transfer.assert_not_called()
assert any("跳过" in str(call.args[0]) for call in recorder.debug.call_args_list)
def test_success_history_without_size_info_is_skipped(monkeypatch):
"""记录中缺少源文件大小时无法比对,保守跳过而不是重复整理。"""
chain_calls = []
for src_fileitem in (None, {}, {"size": None}, {"size": "未知"}, "bad-json"):
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=_history(status=True, src_fileitem=src_fileitem))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"),
file_size=100) is False
chain_calls.append(chain.do_transfer.call_count)
assert chain_calls == [0, 0, 0, 0, 0]
def test_bluray_folder_without_file_size_is_skipped(monkeypatch):
"""蓝光原盘目录没有文件大小可比对,已成功整理过时应跳过。"""
dispatcher = _build_dispatcher()
oper = _patch_history(monkeypatch, record=_history(status=True, size=100))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local",
event_path=Path("/downloads/Movie/BDMV/STREAM/00000.m2ts"),
file_size=None) is False
chain.do_transfer.assert_not_called()
# 蓝光目录的整理记录源路径带尾斜杠,查询必须原样传入
assert oper.get_by_src.call_args.args[0] == "/downloads/Movie/"
def test_success_history_wins_over_failed_history(monkeypatch):
"""同一源路径同时存在成功与失败记录时,以成功记录为准。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch,
record=_history(status=False, size=100),
success_record=_history(status=True, size=100))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
chain.do_transfer.assert_not_called()
def test_history_query_error_registers_pending(monkeypatch):
"""整理历史查询异常仍应登记待重试,而不是被当作已整理跳过。"""
dispatcher = _build_dispatcher()
oper = MagicMock()
oper.get_by_src.side_effect = RuntimeError("数据库不可用")
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", MagicMock(return_value=oper))
chain = _patch_chain(monkeypatch)
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
chain.do_transfer.assert_not_called()
assert list(dispatcher._pending_retries) == ["local:/downloads/a.mkv"]
def test_transfer_exception_registers_pending_and_retries(monkeypatch):
"""整理执行抛异常的文件必须登记待重试,否则已落地的文件永久丢失。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=None)
chain = _patch_chain(monkeypatch, side_effect=RuntimeError("数据库瞬断"))
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
assert list(dispatcher._pending_retries) == ["local:/downloads/a.mkv"]
# 故障恢复后由健康检查周期驱动重试
chain.do_transfer.side_effect = None
dispatcher.retry_pending()
assert chain.do_transfer.call_count == 2
assert dispatcher._pending_retries == {}
def test_transfer_exception_retry_keeps_attempt_count(monkeypatch):
"""整理持续异常时重试次数要累计,避免无限重试。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=None)
_patch_chain(monkeypatch, side_effect=RuntimeError("持续失败"))
dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100)
dispatcher.retry_pending()
assert dispatcher._pending_retries["local:/downloads/a.mkv"]["attempts"] == 2
def test_bluray_retry_uses_origin_event_path(monkeypatch):
"""蓝光原盘整理异常后应按原始事件路径重试,重试时重新解析目录。"""
dispatcher = _build_dispatcher()
_patch_history(monkeypatch, record=None)
_patch_chain(monkeypatch, side_effect=RuntimeError("整理失败"))
event_path = Path("/downloads/Movie/BDMV/STREAM/00000.m2ts")
dispatcher.handle_file(storage="local", event_path=event_path, file_size=None)
assert list(dispatcher._pending_retries) == [f"local:{event_path.as_posix()}"]
+49
View File
@@ -0,0 +1,49 @@
"""
分发瞬间 flap-out 事件不丢弃测试(§5.2)
FUSE 挂载抖动时,事件到达派发环节的瞬间文件可能恰好从视图中消失
(exists() False)原实现在路径"确认不存在"时静默丢弃事件,
flap-out 与真删除在这一瞬间无法区分,丢弃就是永久漏件
正确行为:一律登记待重试,由重试队列在 60s 周期里区分
"恢复可见(继续整理)""确实已删除(自动放弃)"
"""
import unittest
from pathlib import Path
from tempfile import TemporaryDirectory
from unittest.mock import Mock
from watchfiles import Change
from app.monitor.watcher import LocalDirectoryWatcher
class FlapOutDispatchTest(unittest.TestCase):
def setUp(self):
self._tmpdir = TemporaryDirectory()
self.mon_path = Path(self._tmpdir.name)
self.callback = Mock()
self.watcher = LocalDirectoryWatcher(
mon_path=self.mon_path, callback=self.callback)
def tearDown(self):
self._tmpdir.cleanup()
def test_vanished_file_event_registers_retry(self):
"""事件路径已不可见(flap-out 或删除)时必须登记重试,而非静默丢弃。"""
vanished = self.mon_path / "vanished.mkv"
self.watcher._dispatch_changes({(Change.added, vanished.as_posix())})
self.callback.event_unreadable.assert_called_once_with(event_path=vanished)
self.callback.event_handler.assert_not_called()
def test_existing_file_event_still_dispatched(self):
"""正常存在的文件事件照常派发,不受重试登记逻辑影响。"""
existing = self.mon_path / "existing.mkv"
existing.write_bytes(b"data")
self.watcher._dispatch_changes({(Change.added, existing.as_posix())})
self.callback.event_handler.assert_called_once()
self.callback.event_unreadable.assert_not_called()
if __name__ == "__main__":
unittest.main()
+259
View File
@@ -0,0 +1,259 @@
import os
import time
from unittest.mock import MagicMock
from watchfiles import Change
from app.core.config import settings
from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.dispatcher import TransferDispatcher
from app.monitor.recovery import RecoveryExecutor
from app.monitor.syslimits import decide_monitor_mode
from app.utils.system import SystemUtils
def _build_monitor(handle_file: MagicMock = None):
"""
构造带分发器的测试用 Monitor 骨架
:param handle_file: 替换分发器 handle_file 的替身
:return: (Monitor 骨架, 分发器)
"""
from threading import Lock
monitor = object.__new__(Monitor)
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
if handle_file is not None:
dispatcher.handle_file = handle_file
monitor._dispatcher = dispatcher
monitor._watchers = []
monitor._watcher_lock = Lock()
monitor._alerted_paths = {}
monitor._restart_marks = {}
monitor._stable_cycles = {}
monitor._isolated = {}
monitor._recovery = RecoveryExecutor()
return monitor, dispatcher
def _fake_watcher(mon_path, restart_count=0):
"""
构造存活且未静默失效的监控线程替身
:param mon_path: 监控目录
:param restart_count: 累计自动重启次数
:return: 监控线程替身
"""
watcher = MagicMock()
watcher.watch_path = mon_path
watcher.is_alive.return_value = True
watcher.is_stalled.return_value = False
watcher.restart_count = restart_count
return watcher
def test_compensation_scan_covers_files_with_old_mtime(tmp_path):
"""
网盘挂载转存/移动文件会保留原始 mtime补偿扫描不能因 mtime 旧就漏掉文件
但仍应按 mtime 从新到旧优先处理
"""
stale = tmp_path / "old.mkv"
fresh = tmp_path / "new.mkv"
stale.write_bytes(b"x")
fresh.write_bytes(b"y")
now = time.time()
os.utime(stale, (now - 86400 * 365, now - 86400 * 365))
os.utime(fresh, (now, now))
handled = MagicMock(return_value=True)
monitor, _ = _build_monitor(handle_file=handled)
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=now - 600)
handled_paths = [call.kwargs["event_path"] for call in handled.call_args_list]
assert handled_paths == [fresh, stale]
def test_compensation_scan_limits_single_batch(tmp_path, monkeypatch):
"""候选文件超过上限时只处理最新的一批,避免大目录把整理链与数据库压垮。"""
monkeypatch.setattr(Monitor, "MAX_COMPENSATION_FILES", 2)
now = time.time()
files = []
for index in range(3):
target = tmp_path / f"{index}.mkv"
target.write_bytes(b"x")
os.utime(target, (now - index * 60, now - index * 60))
files.append(target)
handled = MagicMock(return_value=True)
monitor, _ = _build_monitor(handle_file=handled)
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=now - 600)
handled_paths = [call.kwargs["event_path"] for call in handled.call_args_list]
assert handled_paths == [files[0], files[1]]
def test_watchdog_compensates_after_internal_restart(tmp_path, monkeypatch):
"""
watcher 内部退避重启期间落地的文件会被新基线快照静默吸收健康检查发现
重启计数增长时应补扫一次且同一次重启不得反复补扫
"""
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock())
monitor, _ = _build_monitor(handle_file=MagicMock(return_value=True))
watcher = _fake_watcher(tmp_path, restart_count=1)
monitor._watchers = [watcher]
started = []
setattr(monitor, "_Monitor__start_compensation", lambda **kwargs: started.append(kwargs))
monitor._Monitor__check_watchers()
assert len(started) == 1
assert started[0]["mon_path"] == tmp_path
# 停摆起点无法精确观测,应保守回溯到最坏情况
assert 0 < time.time() - started[0]["since"] <= Monitor.RESTART_STALL_LOOKBACK + 5
monitor._Monitor__check_watchers()
monitor._Monitor__check_watchers()
assert len(started) == 1
watcher.restart_count = 2
monitor._Monitor__check_watchers()
assert len(started) == 2
def test_compensation_scan_skips_non_candidate_files(tmp_path):
"""补偿扫描应跳过不属于监控扩展名的文件。"""
other = tmp_path / "note.txt"
other.write_bytes(b"x")
handled = MagicMock(return_value=True)
monitor, _ = _build_monitor(handle_file=handled)
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=time.time() - 60)
handled.assert_not_called()
def test_compensation_skipped_without_activity_record(tmp_path, monkeypatch):
"""没有活动记录就没有可靠的停摆起点,应跳过补偿扫描。"""
monitor, _ = _build_monitor(handle_file=MagicMock())
started = []
monkeypatch.setattr("app.monitor.monitor.Thread",
lambda **kwargs: started.append(kwargs) or MagicMock())
monitor._Monitor__start_compensation(mon_path=tmp_path, since=0)
assert started == []
monitor._Monitor__start_compensation(mon_path=tmp_path, since=time.time())
assert len(started) == 1
def test_unreadable_event_is_queued_instead_of_dropped(tmp_path, monkeypatch):
"""读取文件大小失败的事件应登记待重试,而不是被静默丢弃。"""
target = tmp_path / "a.mkv"
target.write_bytes(b"x")
monitor, dispatcher = _build_monitor(handle_file=MagicMock(return_value=True))
watcher = LocalDirectoryWatcher(tmp_path, callback=monitor, force_polling=True)
monkeypatch.setattr(LocalDirectoryWatcher, "_get_file_size", staticmethod(lambda _p: None))
watcher._dispatch_changes({(Change.added, target.as_posix())})
assert len(dispatcher._pending_retries) == 1
entry = next(iter(dispatcher._pending_retries.values()))
assert entry["file_size"] is None
def test_retry_pending_reresolves_missing_file_size(tmp_path):
"""重试时要重新读取文件大小,再把文件送入整理链。"""
target = tmp_path / "a.mkv"
target.write_bytes(b"12345")
handled = MagicMock(return_value=True)
_, dispatcher = _build_monitor(handle_file=handled)
dispatcher.register_unreadable(storage="local", event_path=target)
dispatcher.retry_pending()
assert handled.call_args.kwargs["file_size"] == 5
def test_retry_pending_drops_vanished_file(tmp_path):
"""待重试文件已经消失时应放弃登记,不再无谓重试。"""
target = tmp_path / "gone.mkv"
handled = MagicMock(return_value=True)
_, dispatcher = _build_monitor(handle_file=handled)
dispatcher.register_unreadable(storage="local", event_path=target)
dispatcher.retry_pending()
assert dispatcher._pending_retries == {}
handled.assert_not_called()
def test_transfer_failure_invalidates_dedup_cache(tmp_path, monkeypatch):
"""整理抛异常时去重缓存必须失效,否则 TTL 窗口内的后续事件会被吞掉。"""
target = tmp_path / "a.mkv"
target.write_bytes(b"x")
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
monkeypatch.setattr(dispatcher, "_should_skip_by_history", lambda **kwargs: False)
class _FailingChain:
"""整理时固定抛异常的整理链替身。"""
@staticmethod
def do_transfer(**kwargs):
"""模拟整理过程抛出异常。"""
raise RuntimeError("整理失败")
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", _FailingChain)
assert dispatcher.handle_file(storage="local", event_path=target, file_size=1) is False
assert dispatcher._cache == {}
def test_delayed_rescan_picks_up_late_visible_files(tmp_path, monkeypatch):
"""新增目录首次展开时不可见的文件,应由延迟重扫补回。"""
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
new_dir = tmp_path / "season"
new_dir.mkdir()
first = new_dir / "E03.mkv"
first.write_bytes(b"x")
recorder = MagicMock()
watcher = LocalDirectoryWatcher(tmp_path, callback=recorder, force_polling=True)
watcher._handle_changes({(Change.added, new_dir.as_posix())})
first_paths = {call.kwargs["event_path"] for call in recorder.event_handler.call_args_list}
assert first.as_posix() in first_paths
# 目录内容延迟可见:第二个文件此时才出现,不会再产生任何 watchfiles 事件
late = new_dir / "E04.mkv"
late.write_bytes(b"y")
recorder.event_handler.reset_mock()
watcher._process_pending_rescans()
rescan_paths = {call.kwargs["event_path"] for call in recorder.event_handler.call_args_list}
assert rescan_paths == {late.as_posix()}
def test_network_filesystem_forces_polling_by_default(tmp_path, monkeypatch):
"""默认情况下网络文件系统仍应强制兼容模式。"""
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", False)
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "fast")
assert use_polling is True
def test_network_filesystem_can_opt_into_fast_mode(tmp_path, monkeypatch):
"""用户确认挂载支持 inotify 后应允许快速模式。"""
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", True)
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "fast")
assert use_polling is False
def test_compatibility_mode_still_wins_over_fast_mode_override(tmp_path, monkeypatch):
"""用户显式配置兼容模式时,快速模式开关不得反向覆盖。"""
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", True)
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "compatibility")
assert use_polling is True
+509
View File
@@ -0,0 +1,509 @@
"""
挂载级(block )故障下的监控自愈测试
背景CloudDrive2/115 FUSE 挂载会进入请求不返回错误永不返回的挂死
状态此时监控自愈体系的检测环节仍然有效但恢复环节重建监控线程重试
整理队列本身要访问挂载一旦内联执行就会把全局自愈的单点健康检查
冻死在它自己要修复的挂载上随后停滞检测告警重试驱动全部静默失效
crash 挂载抛 Transport endpoint is not connected的自愈由
test_monitor_resilience.py 覆盖这些测试固定 block 型的两项不变量
看门狗零挂载访问挂载级故障隔离/探测/恢复
"""
import threading
import time
from pathlib import Path
from threading import Lock, Thread
from unittest.mock import MagicMock
import pytest
from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
def _build_monitor(monkeypatch, put_recorder=None):
"""
构造测试用 Monitor 骨架绕过单例初始化
:param monkeypatch: pytest monkeypatch
:param put_recorder: 消息推送记录器
:return: Monitor 骨架
"""
put_recorder = put_recorder or MagicMock()
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
monitor = object.__new__(Monitor)
monitor._dispatcher = MagicMock()
monitor._watchers = []
monitor._watcher_lock = Lock()
monitor._pending_locals = []
monitor._alerted_paths = {}
monitor._restart_marks = {}
monitor._stable_cycles = {}
monitor._isolated = {}
monitor._recovery = RecoveryExecutor()
return monitor
def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
"""
构造测试用监控线程替身
:param mon_path: 监控目录
:param alive: 线程是否存活
:param stalled: 是否静默失效
:param restart_count: 自动重启次数
:return: 监控线程替身
"""
watcher = MagicMock()
watcher.watch_path = mon_path
watcher.is_alive.return_value = alive
watcher.is_stalled.return_value = stalled
watcher.restart_count = restart_count
watcher.last_activity_time = time.time()
return watcher
def _run_watchdog(monitor, timeout=10.0):
"""
在独立线程里跑一次健康检查返回它是否在限定时间内结束
:param monitor: Monitor 骨架
:param timeout: 最长等待秒数
:return: 健康检查是否已返回
"""
done = threading.Event()
def runner():
"""
执行一次健康检查并标记结束
"""
try:
monitor.watchdog()
finally:
done.set()
Thread(target=runner, daemon=True, name="test-watchdog").start()
return done.wait(timeout=timeout)
# --------------------------------------------------------------------------- #
# P0:看门狗线程零挂载访问
# --------------------------------------------------------------------------- #
def test_watchdog_returns_while_rebuild_blocks_forever(tmp_path, monkeypatch):
"""
根因回归重建动作卡在死挂载上永不返回时健康检查本身必须在有限时间内
返回事故中看门狗内联执行重建冻死后 13 个目录的停滞检测与告警全部失效
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
monitor = _build_monitor(monkeypatch)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
entered = threading.Event()
release = threading.Event()
def blocking_rebuild(_target):
"""
模拟 block 型挂载进入后永不返回
"""
entered.set()
release.wait()
setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)
try:
assert _run_watchdog(monitor), "看门狗被重建动作冻死,未能在限定时间内返回"
assert entered.is_set(), "重建动作没有被真正发起"
finally:
release.set()
def test_watchdog_returns_while_pending_retry_blocks_forever(tmp_path, monkeypatch):
"""
重试驱动整理重试队列的 stat同样会卡在死挂载上也必须移出看门狗线程
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
monitor = _build_monitor(monkeypatch)
release = threading.Event()
monitor._dispatcher.retry_pending.side_effect = lambda: release.wait()
try:
assert _run_watchdog(monitor), "看门狗被整理重试驱动冻死,未能在限定时间内返回"
finally:
release.set()
def test_watchdog_returns_while_local_retry_blocks_forever(tmp_path, monkeypatch):
"""
启动失败目录的重试会走 decide_monitor_mode os.walk watcher.start()
exists()同样触碰挂载必须移出看门狗线程
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
monitor = _build_monitor(monkeypatch)
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
release = threading.Event()
def blocking_start(**_kwargs):
"""
模拟启动重试卡在死挂载的目录遍历上
"""
release.wait()
return False
setattr(monitor, "_Monitor__start_local_monitor", blocking_start)
try:
assert _run_watchdog(monitor), "看门狗被本地监控重试冻死,未能在限定时间内返回"
finally:
release.set()
def test_check_watchers_performs_no_filesystem_access(tmp_path, monkeypatch):
"""
检测与判定必须是纯内存运算只读线程存活标志心跳与重启计数
不做任何文件系统访问否则检测环节本身也会被 block 型故障拖死
"""
monitor = _build_monitor(monkeypatch)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
def forbidden(*_args, **_kwargs):
"""
任何真实的文件系统调用都应视为检测环节的缺陷
"""
raise AssertionError("检测环节不允许访问文件系统")
for name in ("stat", "exists", "is_dir", "iterdir", "rglob"):
monkeypatch.setattr(Path, name, forbidden, raising=False)
broken = monitor._Monitor__check_watchers()
assert broken == [watcher]
def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch):
"""
端到端回归不替换任何恢复逻辑只让 Path.exists() 永不返回等价于对 FUSE
守护进程 kill -STOP 后挂载的表现走真实的
__rebuild_watcher -> LocalDirectoryWatcher.start() -> exists() 调用链
事故当天冻结的就是这一行watcher.py start() 入口校验修复后看门狗
必须照常返回并把该目录转入隔离
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
monitor = _build_monitor(monkeypatch)
# 用真实 watcher:只把底层线程换成替身,让它被判定为「存活但静默失效」
watcher = LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=True)
watcher._thread = MagicMock()
watcher._thread.is_alive.return_value = True
watcher._mark_activity()
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
monitor._watchers = [watcher]
release = threading.Event()
real_exists = Path.exists
def blocking_exists(self, *args, **kwargs):
"""
模拟 block 型挂载对监控目录的 exists() 永不返回
"""
if self == tmp_path:
release.wait()
return real_exists(self, *args, **kwargs)
monkeypatch.setattr(Path, "exists", blocking_exists)
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
try:
assert _run_watchdog(monitor), "看门狗冻死在真实的重建调用链上"
assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离"
finally:
release.set()
# --------------------------------------------------------------------------- #
# P0.5:挂载级故障隔离
# --------------------------------------------------------------------------- #
def test_rebuild_timeout_isolates_directory(tmp_path, monkeypatch):
"""
重建在挂载上超时未返回 = 挂载级故障该目录转入隔离后续周期不再对它
发起任何新的挂载访问避免每 60 秒泄漏一个冻死的线程
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.3)
monitor = _build_monitor(monkeypatch)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
calls = []
release = threading.Event()
def blocking_rebuild(target):
"""
模拟 block 型挂载上的重建进入后永不返回
"""
calls.append(target)
release.wait()
setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
try:
assert _run_watchdog(monitor)
assert str(tmp_path) in monitor._isolated, "重建超时后目录没有转入隔离"
# 第二个周期:隔离中的目录不得再被提交重建
assert _run_watchdog(monitor)
assert len(calls) == 1, "隔离中的目录仍在被反复重建,会持续泄漏冻死的线程"
finally:
release.set()
def test_isolated_directory_recovers_after_probe_succeeds(tmp_path, monkeypatch):
"""
隔离期间用可放弃的子进程探测挂载探测通过即解除隔离重建监控
并复用既有的补偿扫描补回停摆期间落地的文件
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
monitor = _build_monitor(monkeypatch)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 3}}
rebuilt = []
setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: True)
assert _run_watchdog(monitor)
assert str(tmp_path) not in monitor._isolated, "探测通过后没有解除隔离"
assert rebuilt == [watcher], "解除隔离后没有重建监控"
def test_isolated_directory_stays_isolated_while_probe_fails(tmp_path, monkeypatch):
"""
探测未通过说明挂载仍未恢复应答必须保持隔离并累计失败次数
"""
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
monitor = _build_monitor(monkeypatch)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher]
monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 0}}
rebuilt = []
setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
assert _run_watchdog(monitor)
assert monitor._isolated[str(tmp_path)]["failures"] == 1
assert rebuilt == [], "挂载尚未恢复就重建监控,只会再冻死一个线程"
def test_pending_locals_skip_isolated_directories(tmp_path, monkeypatch):
"""
隔离中的目录不能再走启动重试路径否则又会在同一个挂载上冻死
"""
monitor = _build_monitor(monkeypatch)
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
monitor._isolated = {str(tmp_path): {"watcher": _fake_watcher(tmp_path), "since": time.time(),
"failures": 0}}
start = MagicMock(return_value=False)
setattr(monitor, "_Monitor__start_local_monitor", start)
monitor._Monitor__retry_pending_locals()
start.assert_not_called()
def test_isolation_alert_is_pushed_after_fault_alert(tmp_path, monkeypatch):
"""
故障隔离是状态升级必须再推一条告警沿用同目录只告警一次会让
用户完全看不到监控已暂停访问正在等待挂载恢复这个关键状态变化
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
monitor._Monitor__send_alert(tmp_path, "目录监控异常")
monitor._Monitor__send_alert(tmp_path, "目录监控异常")
assert put_recorder.put.call_count == 1
monitor._Monitor__send_alert(tmp_path, "挂载无响应,已隔离", stage="isolated")
assert put_recorder.put.call_count == 2
monitor._Monitor__clear_alert(tmp_path, "已恢复")
assert put_recorder.put.call_count == 3
assert str(tmp_path) not in monitor._alerted_paths
# --------------------------------------------------------------------------- #
# 整理分发器:一个文件卡死不得锁死整条整理链
# --------------------------------------------------------------------------- #
def test_stuck_transfer_does_not_block_other_files(monkeypatch):
"""
根因回归整理的规划阶段do_transfer 里的 get_parent_item / list_files
访问挂载 block 型故障下永不返回若分发器的互斥锁包住这段调用这把锁
就会被永久持有 13 watcher 的事件派发补偿扫描和重试队列一起锁死
监控层即使自愈成功也送不进任何文件漏件永远补不回来
锁只应保护 TTL 去重的 check-and-set
"""
from app.monitor.dispatcher import TransferDispatcher
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=False))
release = threading.Event()
stuck_entered = threading.Event()
transferred = []
class FakeChain:
"""
整理链替身指定文件的整理进入后永不返回
"""
@staticmethod
def do_transfer(fileitem, **_kwargs):
"""
模拟规划阶段卡在死挂载的 list_files
"""
transferred.append(fileitem.path)
if fileitem.path.endswith("stuck.mkv"):
stuck_entered.set()
release.wait()
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", FakeChain)
def feed(name):
"""
向分发器送入一个文件
"""
dispatcher.handle_file(storage="local", event_path=Path(f"/mnt/cd2/{name}"), file_size=1)
Thread(target=feed, args=("stuck.mkv",), daemon=True).start()
assert stuck_entered.wait(timeout=5), "卡死的整理没有真正进入"
done = threading.Event()
Thread(target=lambda: (feed("other.mkv"), done.set()), daemon=True).start()
try:
assert done.wait(timeout=5), "一个文件卡在死挂载上就锁死了整条整理链"
assert "/mnt/cd2/other.mkv" in transferred
finally:
release.set()
# --------------------------------------------------------------------------- #
# 恢复执行器与挂载探测
# --------------------------------------------------------------------------- #
def test_recovery_executor_reports_timeout_without_blocking():
"""
永不返回的动作只应消耗一次 timeout执行器必须放弃它并如实报告
"""
executor = RecoveryExecutor()
release = threading.Event()
started = time.monotonic()
results = executor.run({"stuck": release.wait}, timeout=0.3)
elapsed = time.monotonic() - started
try:
assert results == {"stuck": RecoveryState.TIMEOUT}
assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作"
finally:
release.set()
def test_recovery_executor_skips_key_with_running_task():
"""
同一个 key 的上一个动作还冻着时不能再提交新线程否则每个健康检查
周期都会在同一个死挂载上泄漏一个线程
"""
executor = RecoveryExecutor()
release = threading.Event()
calls = []
def stuck():
"""
模拟永不返回的恢复动作
"""
calls.append(1)
release.wait()
try:
assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.TIMEOUT}
assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.BUSY}
assert len(calls) == 1
finally:
release.set()
def test_recovery_executor_runs_actions_concurrently():
"""
一个周期内多个目录的恢复动作必须并发执行否则 13 个目录串行等待会把
健康检查拖过下一个周期
"""
executor = RecoveryExecutor()
release = threading.Event()
started = time.monotonic()
results = executor.run({f"k{i}": release.wait for i in range(5)}, timeout=0.4)
elapsed = time.monotonic() - started
try:
assert set(results.values()) == {RecoveryState.TIMEOUT}
assert elapsed < 1.5, "恢复动作是串行等待的,总耗时随目录数增长"
finally:
release.set()
def test_recovery_executor_reports_completion_and_swallows_errors():
"""
正常完成的动作报告 COMPLETED动作内部抛异常不能让执行器崩溃
否则一个目录的失败会连累整批恢复
"""
executor = RecoveryExecutor()
done = []
def boom():
"""
模拟恢复动作内部异常
"""
raise RuntimeError("rebuild failed")
results = executor.run({"ok": lambda: done.append(1), "bad": boom}, timeout=5)
assert results == {"ok": RecoveryState.COMPLETED, "bad": RecoveryState.COMPLETED}
assert done == [1]
def test_probe_path_succeeds_on_existing_directory(tmp_path):
"""
挂载可用时探测应通过
"""
assert probe_path(tmp_path, timeout=30) is True
def test_probe_path_fails_on_missing_path(tmp_path):
"""
路径不存在时探测应失败而不是抛异常
"""
assert probe_path(tmp_path / "does-not-exist", timeout=30) is False
@pytest.mark.skipif(not Path("/bin/sh").exists(), reason="需要 POSIX 环境")
def test_probe_path_is_abandonable_on_timeout(tmp_path, monkeypatch):
"""
探测的关键性质卡住时可以被放弃线程做的 stat 无法回收只有子进程能在
超时后被 kill这正是隔离期间能持续探测而不泄漏资源的前提
"""
import app.monitor.recovery as recovery
# 用一个必定超时的探测脚本替换真实探测逻辑,验证超时路径的可放弃性
monkeypatch.setattr(recovery, "_PROBE_SCRIPT", "import time; time.sleep(60)")
started = time.monotonic()
assert probe_path(tmp_path, timeout=0.5) is False
assert time.monotonic() - started < 10, "探测超时后没有及时放弃子进程"
+321
View File
@@ -0,0 +1,321 @@
"""
目录监控延迟重扫机制的边界缺陷回归测试
覆盖点
- 同目录重复登记去重4.1
- 整体扫描失败不燃烧轮次 + 失败次数上限4.2
- 目录删除后条目出队不计入失败重试4.2
- MONITOR_RESCAN_DELAYS 配置解析合法/非法回退4.3
- 目录树整体移入时只登记顶层目录祖先已登记时跳过子孙目录4.4
- 待重扫队列溢出时日志升级为 warn4.4
"""
import shutil
from pathlib import Path
from unittest.mock import MagicMock
from watchfiles import Change
from app.core.config import settings
from app.monitor.watcher import LocalDirectoryWatcher
def _build_watcher(tmp_path, force_polling=True):
"""
构造测试用目录监控
:param tmp_path: 监控目录
:param force_polling: 是否强制轮询
:return: 目录监控
"""
return LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=force_polling)
# ==================== 4.1 同目录重复登记去重 ====================
def test_schedule_rescan_dedups_same_directory(tmp_path):
"""
readdir 闪断可能让同一目录产生两次 added 事件重复调用 _schedule_rescan
只应保留一条待重扫记录不应重复登记
"""
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(directory, seen=set())
watcher._schedule_rescan(directory, seen={"already-seen"})
assert len(watcher._pending_rescans) == 1
# 保留原条目即可,不需要用新事件的 seen 覆盖
assert watcher._pending_rescans[0]["seen"] == set()
def test_expand_added_directories_dedups_repeated_added_event_across_batches(tmp_path):
"""
同一目录在不同批次的 changes 中各出现一次 added 事件时模拟 readdir 闪断
整体展开流程也不应重复登记重扫
"""
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._expand_added_directories({(Change.added, directory.as_posix())})
watcher._expand_added_directories({(Change.added, directory.as_posix())})
assert len(watcher._pending_rescans) == 1
# ==================== 4.2 扫描失败不燃烧轮次 + 失败上限 + 目录删除终态 ====================
def test_collect_directory_files_reports_missing_directory(tmp_path):
"""
目录已不存在时应识别为终态missing=True而不是当作扫描失败
"""
missing = tmp_path / "gone"
watcher = _build_watcher(tmp_path)
collected, is_missing, scan_failed = watcher._collect_directory_files(missing, exclude=set())
assert collected == set()
assert is_missing is True
assert scan_failed is False
def test_collect_directory_files_reports_scan_failure(tmp_path, monkeypatch):
"""
顶层 rglob OSError 应识别为整体扫描失败scan_failed=True
与目录已删除的终态区分开
"""
directory = tmp_path / "season"
directory.mkdir()
def failing_rglob(self, pattern):
raise OSError("FUSE 抖动")
monkeypatch.setattr(Path, "rglob", failing_rglob)
watcher = _build_watcher(tmp_path)
collected, is_missing, scan_failed = watcher._collect_directory_files(directory, exclude=set())
assert collected == set()
assert is_missing is False
assert scan_failed is True
def test_process_pending_rescans_does_not_burn_round_on_scan_failure(tmp_path, monkeypatch):
"""
整体扫描失败时不应消耗重扫轮次条目应保持在原轮次继续重试
并累计一次失败计数
"""
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0, 100))
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(directory, seen=set())
def failing_rglob(self, pattern):
raise OSError("FUSE 抖动")
monkeypatch.setattr(Path, "rglob", failing_rglob)
watcher._process_pending_rescans()
assert len(watcher._pending_rescans) == 1
item = watcher._pending_rescans[0]
assert item["round"] == 0
assert item["failures"] == 1
def test_process_pending_rescans_drops_item_after_max_failures(tmp_path, monkeypatch):
"""
连续扫描失败达到 MAX_RESCAN_FAILURES 上限后应放弃重扫并 warn
避免目录长期不可访问时无限重试
"""
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
monkeypatch.setattr(LocalDirectoryWatcher, "MAX_RESCAN_FAILURES", 2)
logger_warn = MagicMock()
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(directory, seen=set())
def failing_rglob(self, pattern):
raise OSError("FUSE 抖动")
monkeypatch.setattr(Path, "rglob", failing_rglob)
watcher._process_pending_rescans()
assert len(watcher._pending_rescans) == 1
watcher._process_pending_rescans()
assert watcher._pending_rescans == []
logger_warn.assert_called_once()
def test_process_pending_rescans_resets_failure_count_after_success(tmp_path, monkeypatch):
"""
扫描恢复成功后应重置失败计数不应带着历史失败次数继续累积
"""
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0, 100))
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(directory, seen=set())
watcher._pending_rescans[0]["failures"] = 3
watcher._process_pending_rescans()
assert len(watcher._pending_rescans) == 1
assert watcher._pending_rescans[0]["failures"] == 0
def test_process_pending_rescans_drops_deleted_directory_without_failure(tmp_path, monkeypatch):
"""
目录已被删除是终态应直接出队不计入失败重试次数
"""
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
directory = tmp_path / "season"
directory.mkdir()
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(directory, seen=set())
shutil.rmtree(directory)
watcher._process_pending_rescans()
assert watcher._pending_rescans == []
# ==================== 4.3 重扫窗口可配置 ====================
def test_parse_rescan_delays_accepts_valid_string():
"""
合法的逗号分隔正整数字符串应正确解析为元组
"""
assert LocalDirectoryWatcher._parse_rescan_delays("30,120,600,1800") == (30, 120, 600, 1800)
assert LocalDirectoryWatcher._parse_rescan_delays(" 5 , 10 ") == (5, 10)
def test_parse_rescan_delays_falls_back_on_empty():
"""
空字符串/None 应回退到默认值
"""
assert LocalDirectoryWatcher._parse_rescan_delays("") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
assert LocalDirectoryWatcher._parse_rescan_delays(None) == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
def test_parse_rescan_delays_falls_back_on_invalid_format(monkeypatch):
"""
非法格式无法转换为整数应回退默认值并记录 warn 日志
"""
logger_warn = MagicMock()
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
assert LocalDirectoryWatcher._parse_rescan_delays("abc,def") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
logger_warn.assert_called_once()
def test_parse_rescan_delays_rejects_non_positive_values(monkeypatch):
"""
包含非正整数0 或负数视为非法配置应回退默认值并记录 warn 日志
"""
logger_warn = MagicMock()
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
assert LocalDirectoryWatcher._parse_rescan_delays("30,-5") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
logger_warn.assert_called_once()
def test_directory_rescan_delays_property_reads_settings(tmp_path, monkeypatch):
"""
DIRECTORY_RESCAN_DELAYS 应实时反映 MONITOR_RESCAN_DELAYS 配置
"""
monkeypatch.setattr(settings, "MONITOR_RESCAN_DELAYS", "5,10")
watcher = _build_watcher(tmp_path)
assert watcher.DIRECTORY_RESCAN_DELAYS == (5, 10)
def test_directory_rescan_delays_property_falls_back_on_invalid_settings(tmp_path, monkeypatch):
"""
配置非法时属性访问应回退默认值而不是抛异常影响监控主流程
"""
logger_warn = MagicMock()
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
monkeypatch.setattr(settings, "MONITOR_RESCAN_DELAYS", "not-a-number")
watcher = _build_watcher(tmp_path)
assert watcher.DIRECTORY_RESCAN_DELAYS == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
# ==================== 4.4 目录树移入只登记顶层 + 溢出 warn ====================
def test_expand_added_directories_only_schedules_top_level(tmp_path):
"""
大目录树整体移入时changes 里每一层子目录都会各自产生一次 added 事件
但只有顶层目录需要登记重扫子目录内容已被顶层目录的 rglob 覆盖
"""
top = tmp_path / "task"
nested = top / "season1"
nested2 = nested / "sub"
nested2.mkdir(parents=True)
(top / "movie.mkv").write_bytes(b"x")
watcher = _build_watcher(tmp_path)
watcher._expand_added_directories({
(Change.added, top.as_posix()),
(Change.added, nested.as_posix()),
(Change.added, nested2.as_posix()),
})
assert len(watcher._pending_rescans) == 1
assert watcher._pending_rescans[0]["path"] == top
def test_schedule_rescan_skips_descendant_of_pending_ancestor(tmp_path):
"""
某祖先目录已经在待重扫队列中时其子孙目录不应再单独登记
祖先条目的 rglob 会递归覆盖到子孙目录的新文件
"""
parent = tmp_path / "task"
child = parent / "season"
child.mkdir(parents=True)
watcher = _build_watcher(tmp_path)
watcher._schedule_rescan(parent, seen=set())
watcher._schedule_rescan(child, seen=set())
assert len(watcher._pending_rescans) == 1
assert watcher._pending_rescans[0]["path"] == parent
def test_is_descendant_of_any_does_not_match_sibling_with_prefix_name(tmp_path):
"""
路径前缀相同但并非真实父子关系的兄弟目录 task / task2不应被
误判为祖先命中避免字符串前缀匹配导致的误跳过
"""
parent = tmp_path / "task"
sibling = tmp_path / "task2"
assert LocalDirectoryWatcher._is_descendant_of_any(sibling, {parent}) is False
def test_schedule_rescan_overflow_logs_warning(tmp_path, monkeypatch):
"""
待重扫队列已满时应放弃登记且日志级别应为 warn原实现仅 debug
可能导致文件永久漏扫却难以被发现
"""
monkeypatch.setattr(LocalDirectoryWatcher, "MAX_PENDING_RESCANS", 1)
logger_warn = MagicMock()
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
watcher = _build_watcher(tmp_path)
first_dir = tmp_path / "a"
second_dir = tmp_path / "b"
first_dir.mkdir()
second_dir.mkdir()
watcher._schedule_rescan(first_dir, seen=set())
watcher._schedule_rescan(second_dir, seen=set())
assert len(watcher._pending_rescans) == 1
assert watcher._pending_rescans[0]["path"] == first_dir
logger_warn.assert_called_once()
+58 -14
View File
@@ -2,6 +2,7 @@ from pathlib import Path
from unittest.mock import MagicMock from unittest.mock import MagicMock
from app.monitor import LocalDirectoryWatcher, Monitor from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.recovery import RecoveryExecutor
def _build_watcher(tmp_path, force_polling): def _build_watcher(tmp_path, force_polling):
@@ -130,12 +131,16 @@ def _build_monitor(monkeypatch, put_recorder):
from threading import Lock from threading import Lock
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder)) monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
monitor = object.__new__(Monitor) monitor = object.__new__(Monitor)
# 自动重启后健康检查会发起补偿扫描,骨架需要一个分发器替身
monitor._dispatcher = MagicMock()
monitor._watchers = [] monitor._watchers = []
monitor._watcher_lock = Lock() monitor._watcher_lock = Lock()
monitor._pending_locals = [] monitor._pending_locals = []
monitor._alerted_paths = set() monitor._alerted_paths = {}
monitor._restart_marks = {} monitor._restart_marks = {}
monitor._stable_cycles = {} monitor._stable_cycles = {}
monitor._isolated = {}
monitor._recovery = RecoveryExecutor()
return monitor return monitor
@@ -158,35 +163,31 @@ def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch): def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch):
""" """
监控线程退出后健康检查应重建线程并告警 监控线程退出后健康检查应判定为待重建并告警
重建本身会触碰挂载已移出看门狗线程因此检测环节只负责判定与告警
真正的重建由 __drive_recovery 派发到一次性工作线程执行
""" """
put_recorder = MagicMock() put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder) monitor = _build_monitor(monkeypatch, put_recorder)
watcher = _fake_watcher(tmp_path, alive=False) watcher = _fake_watcher(tmp_path, alive=False)
monitor._watchers = [watcher] monitor._watchers = [watcher]
rebuild = MagicMock()
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
monitor._Monitor__check_watchers() assert monitor._Monitor__check_watchers() == [watcher]
rebuild.assert_called_once_with(watcher)
put_recorder.put.assert_called_once() put_recorder.put.assert_called_once()
def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch): def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch):
""" """
静默失效的监控线程也应被健康检查重建 静默失效的监控线程也应被健康检查判定为待重建
""" """
put_recorder = MagicMock() put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder) monitor = _build_monitor(monkeypatch, put_recorder)
watcher = _fake_watcher(tmp_path, alive=True, stalled=True) watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
monitor._watchers = [watcher] monitor._watchers = [watcher]
rebuild = MagicMock()
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
monitor._Monitor__check_watchers() assert monitor._Monitor__check_watchers() == [watcher]
rebuild.assert_called_once_with(watcher)
def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch): def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch):
@@ -235,7 +236,7 @@ def test_dispatcher_retries_after_history_query_failure(monkeypatch):
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={}) dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv") event_path = Path("/downloads/movie.mkv")
history = MagicMock(side_effect=[None, False]) history = MagicMock(side_effect=[None, False])
monkeypatch.setattr(dispatcher, "_has_transfer_history", history) monkeypatch.setattr(dispatcher, "_should_skip_by_history", history)
transfer_chain_instance = MagicMock() transfer_chain_instance = MagicMock()
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", monkeypatch.setattr("app.monitor.dispatcher.TransferChain",
MagicMock(return_value=transfer_chain_instance)) MagicMock(return_value=transfer_chain_instance))
@@ -253,6 +254,38 @@ def test_dispatcher_retries_after_history_query_failure(monkeypatch):
assert dispatcher._pending_retries == {} assert dispatcher._pending_retries == {}
def test_dispatcher_clear_pending_drops_stale_entries(monkeypatch):
"""
停止/配置重载时应清空待重试队列避免已移除的监控目录在数据库恢复后
仍被送入整理链
"""
from app.monitor.dispatcher import TransferDispatcher
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=None))
dispatcher.handle_file(storage="local", event_path=Path("/removed/movie.mkv"), file_size=1)
assert dispatcher._pending_retries
dispatcher.clear_pending()
assert dispatcher._pending_retries == {}
dispatcher.retry_pending()
assert dispatcher._pending_retries == {}
def test_monitor_stop_clears_dispatcher_pending(monkeypatch):
"""
Monitor.stop 应连带清理分发器的待重试队列
"""
put_recorder = MagicMock()
monitor = _build_monitor(monkeypatch, put_recorder)
monitor._scheduler = None
monitor._dispatcher = MagicMock()
monitor.stop()
monitor._dispatcher.clear_pending.assert_called_once()
def test_dispatcher_drops_pending_after_max_attempts(monkeypatch): def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
""" """
历史查询持续失败达到上限后应放弃重试避免队列无限累积 历史查询持续失败达到上限后应放弃重试避免队列无限累积
@@ -260,7 +293,7 @@ def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
from app.monitor.dispatcher import TransferDispatcher from app.monitor.dispatcher import TransferDispatcher
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={}) dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
event_path = Path("/downloads/movie.mkv") event_path = Path("/downloads/movie.mkv")
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=None)) monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=None))
dispatcher.handle_file(storage="local", event_path=event_path, file_size=1) dispatcher.handle_file(storage="local", event_path=event_path, file_size=1)
key = f"local:{event_path.as_posix()}" key = f"local:{event_path.as_posix()}"
@@ -271,3 +304,14 @@ def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
dispatcher.retry_pending() dispatcher.retry_pending()
assert dispatcher._pending_retries == {} assert dispatcher._pending_retries == {}
def test_monitor_watches_mode_env_keys_for_hot_reload():
"""
快速模式/轮询间隔只在监控线程创建时读取,必须监听对应 env 变更触发
init() 重建才能热生效;防止上游合并时丢失监听键
"""
from app.schemas.types import SystemConfigKey
assert SystemConfigKey.Directories.value in Monitor.CONFIG_WATCH
assert "MONITOR_NETWORK_FAST_MODE" in Monitor.CONFIG_WATCH
assert "MONITOR_POLL_DELAY_NETWORK" in Monitor.CONFIG_WATCH
+65
View File
@@ -254,6 +254,71 @@ def test_poll_failure_alert_threshold_and_recovery(monkeypatch):
assert "已恢复" in alert_cb.call_args.args[1] assert "已恢复" in alert_cb.call_args.args[1]
def test_poll_partial_failure_pins_incremental_cursor(monkeypatch):
"""
部分路径失败时增量游标必须固定在旧值否则失败路径中时间落在新旧游标
之间的变更会被后续增量查询永久跳过
"""
poller, store, _ = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [None, {'/mon2/b.mkv': {'size': 2, 'modify_time': 999}}])
poller.poll("u115", [Path("/mon"), Path("/mon2")])
# 游标固定为旧基线时间 100,而不是成功路径产生的 999
assert store.save.call_args.kwargs["snapshot_time"] == 100
def test_poll_full_success_advances_cursor(monkeypatch):
"""
全部路径成功时不固定游标由快照内容推进增量游标
"""
poller, store, _ = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 999}}])
poller.poll("u115", [Path("/mon")])
assert store.save.call_args.kwargs["snapshot_time"] is None
def test_force_full_scan_aborts_on_baseline_load_error(monkeypatch):
"""
全量扫描读取基线失败时必须放弃落盘否则会抹掉同存储其他监控路径的基线
"""
poller, store, _ = _build_poller()
store.load_checked.return_value = (None, False)
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
assert poller.force_full_scan("u115", Path("/mon")) is False
store.save.assert_not_called()
def test_force_full_scan_reports_save_failure(monkeypatch):
"""
全量扫描持久化失败必须传播为失败不能返回成功掩盖基线未更新
"""
poller, store, _ = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
store.save.return_value = False
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}])
assert poller.force_full_scan("u115", Path("/mon")) is False
def test_force_full_scan_merges_with_existing_baseline(monkeypatch):
"""
全量扫描只覆盖单个路径应与已有基线合并后落盘
"""
poller, store, _ = _build_poller()
store.load_checked.return_value = (dict(BASELINE), True)
_mock_storage_chain(monkeypatch, [{'/mon2/b.mkv': {'size': 2, 'modify_time': 200}}])
assert poller.force_full_scan("u115", Path("/mon2")) is True
saved_snapshot = store.save.call_args.args[1]
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon2/b.mkv'}
def test_watcher_poll_delay_defaults_and_override(tmp_path): def test_watcher_poll_delay_defaults_and_override(tmp_path):
""" """
轮询扫描间隔默认取本地值显式传入网络值时生效 轮询扫描间隔默认取本地值显式传入网络值时生效
+3 -1
View File
@@ -326,7 +326,9 @@ def test_handle_file_prefers_music_type_from_monitor_directory(monkeypatch):
] ]
transfer_chain_instance = MagicMock() transfer_chain_instance = MagicMock()
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=False)) # 历史查重已由 _should_skip_by_history 统一承担(含失败重试预算与版本变化判定),
# 这里放行以便验证 mtype 的传递
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=False))
monkeypatch.setattr( monkeypatch.setattr(
"app.monitor.dispatcher.DirectoryHelper", "app.monitor.dispatcher.DirectoryHelper",
MagicMock(return_value=MagicMock(get_download_dirs=MagicMock(return_value=directories))), MagicMock(return_value=MagicMock(get_download_dirs=MagicMock(return_value=directories))),
@@ -0,0 +1,37 @@
import importlib
from pathlib import Path
from types import SimpleNamespace
import pytest
from app.modules.filemanager.storages import StorageBase
from app.schemas.exception import StorageQueryError
# 在树存储模块,全部必须实现严格查询
BUILTIN_STORAGE_MODULES = ["alipan", "alist", "local", "rclone", "smb", "u115"]
def test_base_get_item_strict_fails_conservatively():
"""未覆写严格查询的存储必须保守失败,而不是回退到宽松查询。"""
stub = SimpleNamespace(schema="dummy")
with pytest.raises(StorageQueryError):
StorageBase.get_item_strict(stub, Path("/media/示例.mkv"))
@pytest.mark.parametrize("module_name", BUILTIN_STORAGE_MODULES)
def test_builtin_storage_overrides_strict_query(module_name):
"""每个在树存储都必须覆写严格查询,否则整理会被基类保守拒绝。"""
module = importlib.import_module(f"app.modules.filemanager.storages.{module_name}")
storage_classes = [
obj for obj in vars(module).values()
if isinstance(obj, type)
and issubclass(obj, StorageBase)
and obj is not StorageBase
and obj.__module__ == module.__name__
]
assert storage_classes, f"{module_name} 未定义存储类"
for storage_class in storage_classes:
assert "get_item_strict" in vars(storage_class), \
f"{storage_class.__name__} 未实现 get_item_strict"
+122
View File
@@ -0,0 +1,122 @@
from types import SimpleNamespace
from app.core.config import settings
from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.schemas.types import MediaSource, MediaType
class _MemoryCacheStub:
"""TMDB 识别缓存测试用的最小内存后端。"""
def __init__(self, data: dict = None):
"""使用给定字典初始化测试缓存。"""
self.data = data if data is not None else {}
self.ttls = {}
def get(self, key: str):
"""读取指定缓存条目。"""
return self.data.get(key)
def set(self, key: str, value, ttl=None):
"""写入缓存条目并记录使用的 TTL。"""
self.data[key] = value
self.ttls[key] = ttl
def delete(self, key: str):
"""删除指定缓存条目。"""
self.data.pop(key, None)
def items(self):
"""返回全部缓存条目。"""
return self.data.items()
@staticmethod
def is_redis() -> bool:
"""测试后端不是 Redis。"""
return False
def _build_cache(data: dict = None) -> TmdbCache:
"""构造绕过单例初始化的 TMDB 缓存实例。"""
cache = object.__new__(TmdbCache)
cache._cache = _MemoryCacheStub(data)
cache.ttl = 43200
cache._expires_at = {}
cache._dirty = False
cache.save = lambda force=False: None
return cache
def _build_meta(mtype: MediaType, begin_season=1) -> SimpleNamespace:
"""构造识别缓存所需的最小元数据。"""
# v3 起缓存键改用 media_source/media_id 统一标识媒体来源,
# 非 TMDB 来源不参与 TMDB 缓存键构造
return SimpleNamespace(type=mtype, tmdbid=329809, year="2022",
begin_season=begin_season, name="死神",
media_source=MediaSource.TMDB, media_id="329809")
def _key(type_name: str, begin_season=1) -> str:
"""按缓存键格式构造测试用键。"""
return f"[{type_name}][{settings.TMDB_LOCALE}]329809-2022-{begin_season}"
def test_get_discards_movie_value_cached_under_tv_key():
"""电视剧元数据命中电影缓存值属于脏条目,应丢弃并回源。"""
meta = _build_meta(MediaType.TV)
key = _key("电视剧")
cache = _build_cache({key: {"id": 329809, "type": MediaType.MOVIE,
"title": "白鼬", "year": "2015"}})
assert cache.get(meta) == {}
assert cache._cache.get(key) is None
def test_get_discards_movie_value_stored_as_plain_string():
"""缓存值经序列化后类型退化为字符串时,同样要识别出冲突。"""
meta = _build_meta(MediaType.TV)
key = _key("电视剧")
cache = _build_cache({key: {"id": 329809, "type": "电影", "title": "白鼬"}})
assert cache.get(meta) == {}
def test_get_keeps_tv_value_cached_under_movie_key():
"""电影元数据命中电视剧缓存值是名称识别的正常纠正结果,必须保留。"""
meta = _build_meta(MediaType.MOVIE, begin_season=None)
key = _key("电影", begin_season=None)
cache = _build_cache({key: {"id": 1, "type": MediaType.TV, "title": "某剧"}})
assert cache.get(meta)["type"] == MediaType.TV
def test_get_keeps_negative_cache_entry():
"""负缓存不带类型信息,不应被类型校验误删。"""
meta = _build_meta(MediaType.TV)
cache = _build_cache({_key("电视剧"): {"id": 0}})
assert cache.get(meta) == {"id": 0}
def test_update_refuses_to_cache_movie_result_for_tv_meta():
"""电视剧元数据得到电影识别结果时不得写入缓存,避免固化脏条目。"""
meta = _build_meta(MediaType.TV)
cache = _build_cache()
cache.update(meta, {"id": 329809, "media_type": MediaType.MOVIE,
"title": "白鼬", "release_date": "2015-01-01"})
assert cache._cache.data == {}
def test_update_caches_tv_result_for_movie_meta():
"""电影元数据得到电视剧结果是正常纠正,应照常写入缓存。"""
meta = _build_meta(MediaType.MOVIE, begin_season=None)
cache = _build_cache()
cache.update(meta, {"id": 1, "media_type": MediaType.TV,
"name": "某剧", "first_air_date": "2022-01-01"})
stored = cache._cache.get(_key("电影", begin_season=None))
assert stored["type"] == MediaType.TV
assert stored["title"] == "某剧"
+72
View File
@@ -0,0 +1,72 @@
"""
TMDB request 层业务失败响应不缓存测试(§6.4)
TMDB 404 等业务失败返回合法 JSON(success=false),原实现会随快照缓存
12 小时;瞬时的服务端错误也会被同样固化,期间同 key 请求直接命中失败快照
业务失败响应必须跳过缓存,允许下次请求重新确认
"""
import unittest
from unittest.mock import patch
from app.core.cache import cached
from app.modules.themoviedb.tmdbv3api.tmdb import TMDb
from tests.test_tmdb_response_cache import _FakeResponse
_NOT_FOUND_PAYLOAD = {
"success": False,
"status_code": 34,
"status_message": "The resource you requested could not be found.",
}
_HEADERS = {"Content-Type": "application/json"}
class CachedSkipIfTest(unittest.TestCase):
def test_skip_if_prevents_caching_matching_values(self):
"""skip_if 命中的返回值不入缓存,后续调用重新执行函数。"""
calls = {"bad": 0, "good": 0}
@cached(region="test_skip_if", ttl=60,
skip_if=lambda value: value.get("bad"))
def fetch(kind: str) -> dict:
calls[kind] += 1
return {"bad": kind == "bad"}
fetch("bad")
fetch("bad")
self.assertEqual(calls["bad"], 2)
fetch("good")
fetch("good")
self.assertEqual(calls["good"], 1)
class TmdbFailureSnapshotCacheTest(unittest.TestCase):
@staticmethod
def _make_tmdb() -> TMDb:
tmdb = TMDb()
tmdb.api_key = "test-key"
return tmdb
def test_business_failure_response_is_not_cached(self):
"""404 业务失败 JSON 不入缓存,同参数再次请求会重新访问 TMDB。"""
tmdb = self._make_tmdb()
fake = _FakeResponse(_NOT_FOUND_PAYLOAD, _HEADERS, status_code=404)
with patch.object(TMDb, "_request_once", return_value=fake) as req:
tmdb.request("GET", "https://api.tmdb.test/failure-not-cached", None, None)
tmdb.request("GET", "https://api.tmdb.test/failure-not-cached", None, None)
self.assertEqual(req.call_count, 2)
def test_success_response_is_still_cached(self):
"""成功响应保持缓存,同参数第二次请求命中快照。"""
tmdb = self._make_tmdb()
fake = _FakeResponse({"id": 98865, "title": "Test"}, _HEADERS)
with patch.object(TMDb, "_request_once", return_value=fake) as req:
tmdb.request("GET", "https://api.tmdb.test/success-cached", None, None)
tmdb.request("GET", "https://api.tmdb.test/success-cached", None, None)
self.assertEqual(req.call_count, 1)
if __name__ == "__main__":
unittest.main()
+503
View File
@@ -0,0 +1,503 @@
# -*- coding: utf-8 -*-
"""
覆盖TMDB识别链路两个已核实缺陷的回归测试
§6.1 零退避重试
同步 `TMDb.request` 在连接失败重建会话后立即重试两次尝试间隔为0
异步 `TMDb.async_request` 遇到连接失败时完全没有重试
NAS+FUSE网盘环境下TMDB连接偶发的都是数秒内可自愈的瞬时抖动零退避/不重试
基本无法覆盖这类抖动窗口
§6.2 网络故障与"条目不存在"文案混淆
`TmdbApi.__get_movie_detail`/`__get_tv_detail` 对所有异常含真实网络故障与
TMDB返回的404"资源不存在"一律 `except Exception: return None`两类完全不同
性质的失败被折叠成同一个 None`TheMovieDbModule._get_info_by_tmdbid` 又用
`info_tv or info_movie or None` 把结果进一步折叠最终识别失败统一报
"无法确定媒体类型,识别失败"用户无法判断该等网络恢复还是该确认条目不存在
"""
import asyncio
from unittest.mock import AsyncMock, Mock
import pytest
import app.modules.themoviedb as themoviedb_module
from app.core.metainfo import MetaInfo
from app.modules.themoviedb import TheMovieDbModule
from app.modules.themoviedb.tmdb_cache import TmdbCache
from app.modules.themoviedb.tmdbapi import TmdbApi
from app.modules.themoviedb.tmdbv3api import tmdb as tmdb_module
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbConnectionError, TMDbException
from app.modules.themoviedb.tmdbv3api.tmdb import TMDb
from app.schemas.types import MediaSource, MediaType
class _FakeResponse:
"""测试用响应对象,模拟 requests/httpx 响应的最小接口。"""
def __init__(self, payload, headers: dict = None):
"""初始化响应内容。"""
self._payload = payload
self.headers = headers or {}
self.status_code = 200
self.text = ""
def json(self):
"""返回预置JSON内容。"""
return self._payload
# ---------------------------------------------------------------------------
# §6.1 零退避重试
# ---------------------------------------------------------------------------
def test_tmdb_connection_error_is_tmdb_exception_subclass():
"""
TMDbConnectionError必须是TMDbException的子类
否则现有 `except TMDbException` 代码路径会因新增异常类型而失效
"""
assert issubclass(TMDbConnectionError, TMDbException)
def test_request_sleeps_before_retry_after_connection_failure(monkeypatch):
"""
同步请求失败后重建会话重试前应等待 RETRY_BACKOFF_SECONDS
而不是零间隔立即重试
"""
tmdb = TMDb()
response = _FakeResponse(payload={"id": 1})
request_results = [None, response]
tmdb._request_once = lambda method, url, data, json: request_results.pop(0)
sleep_calls = []
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: sleep_calls.append(seconds))
result = TMDb.request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
assert result["json"] == {"id": 1}
assert sleep_calls == [tmdb_module.RETRY_BACKOFF_SECONDS]
assert 1 <= tmdb_module.RETRY_BACKOFF_SECONDS <= 3
def test_request_raises_connection_error_after_retry_exhausted(monkeypatch):
"""
重试后依旧失败时应抛出更具体的 TMDbConnectionError而不仅是笼统的 TMDbException
供上层区分"网络故障"与TMDB业务层错误对外仍保持"最终失败抛异常"的语义不变
"""
tmdb = TMDb()
tmdb._request_once = lambda method, url, data, json: None
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: None)
with pytest.raises(TMDbConnectionError, match="无法连接TheMovieDb"):
TMDb.request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
def test_async_request_retries_once_after_connection_failure(monkeypatch):
"""
异步请求当前完全没有重试修复后首次失败应等待 RETRY_BACKOFF_SECONDS
后重试一次重试成功则返回正常结果
"""
tmdb = TMDb()
response = _FakeResponse(payload={"id": 2})
outcomes = [None, response]
async def _fake_once(method, url, data, json):
return outcomes.pop(0)
tmdb._async_request_once = _fake_once
sleep_calls = []
async def _fake_sleep(seconds):
sleep_calls.append(seconds)
monkeypatch.setattr(tmdb_module.asyncio, "sleep", _fake_sleep)
result = asyncio.run(
TMDb.async_request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
)
assert result["json"] == {"id": 2}
assert sleep_calls == [tmdb_module.RETRY_BACKOFF_SECONDS]
def test_async_request_raises_connection_error_after_retry_exhausted(monkeypatch):
"""异步请求重试一次后仍失败,应抛出 TMDbConnectionError,保持最终失败语义。"""
tmdb = TMDb()
async def _fake_once(method, url, data, json):
return None
tmdb._async_request_once = _fake_once
async def _fake_sleep(seconds):
return None
monkeypatch.setattr(tmdb_module.asyncio, "sleep", _fake_sleep)
with pytest.raises(TMDbConnectionError, match="无法连接TheMovieDb"):
asyncio.run(TMDb.async_request.__wrapped__(tmdb, "GET", "https://example.com", None, None))
def test_request_exception_path_is_not_cached(monkeypatch):
"""
`request`/`async_request` `@cached(skip_none=True)`抛异常时函数没有
正常返回值缓存装饰器的 `cache_backend.set` 分支不会被执行到天然不会缓存
这里通过真实缓存路径不使用 __wrapped__验证连续两次同参数请求都会真实
触发底层请求而不是第二次直接命中一个"异常快照"
"""
tmdb = TMDb()
call_count = {"n": 0}
def _always_fail(method, url, data, json):
call_count["n"] += 1
return None
tmdb._request_once = _always_fail
tmdb._reset_owned_session = lambda: None
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: None)
for _ in range(2):
with pytest.raises(TMDbConnectionError):
tmdb.request("GET", "https://example.com/exc-not-cached", None, None)
# 两次调用都真实触发了底层请求(各自都经历了一次重试),说明异常没有被当作
# 缓存命中而跳过;即 2 次调用 × 2 次尝试 = 4 次底层请求。
assert call_count["n"] == 4
# ---------------------------------------------------------------------------
# §6.2 网络故障与"条目不存在"文案混淆
# ---------------------------------------------------------------------------
def test_request_obj_raises_plain_tmdb_exception_for_not_found_status():
"""
调研结论TMDB对404"资源不存在"的响应体形如
`{"success": false, "status_code": 34, "status_message": "..."}`
`TMDb._handle_errors` 据此抛出的是普通 TMDbException而不是表示传输层失败的
TMDbConnectionError这与`request`/`async_request`在请求彻底失败返回None
时抛出的 TMDbConnectionError是两类不同的异常不应混淆处理
"""
tmdb = TMDb()
snapshot = {
TMDb._RESPONSE_SNAPSHOT_MARKER: True,
"headers": {},
"json": {
"success": False,
"status_code": 34,
"status_message": "The resource you requested could not be found.",
},
}
tmdb.request = lambda *args, **kwargs: snapshot
with pytest.raises(TMDbException) as exc_info:
tmdb._request_obj("/movie/999999999")
assert not isinstance(exc_info.value, TMDbConnectionError)
def test_get_movie_detail_propagates_connection_error_when_requested():
"""
`TmdbApi.get_info(..., raise_on_connection_error=True)` 遇到TMDB连接失败时
应该把 TMDbConnectionError 传播出去而不是像默认那样吞掉返回 None
"""
tmdb_api = TmdbApi()
tmdb_api.movie.details = Mock(
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
)
with pytest.raises(TMDbConnectionError):
tmdb_api.get_info(mtype=MediaType.MOVIE, tmdbid=1, raise_on_connection_error=True)
def test_get_movie_detail_swallows_connection_error_by_default():
"""
不传 `raise_on_connection_error` 默认False行为必须与修复前完全一致
任何异常都吞掉返回 None不破坏现有调用方
"""
tmdb_api = TmdbApi()
tmdb_api.movie.details = Mock(
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
)
assert tmdb_api.get_info(mtype=MediaType.MOVIE, tmdbid=1) is None
def test_get_tv_detail_not_found_error_still_returns_none_even_with_flag():
"""
即使显式要求 `raise_on_connection_error=True`TMDB业务层返回的"资源不存在"
普通TMDbException非TMDbConnectionError也不应被当作连接失败传播出去
应继续按现有语义返回 None
"""
tmdb_api = TmdbApi()
tmdb_api.tv.details = Mock(
side_effect=TMDbException("The resource you requested could not be found.")
)
result = tmdb_api.get_info(mtype=MediaType.TV, tmdbid=1, raise_on_connection_error=True)
assert result is None
def test_async_get_info_propagates_connection_error_when_requested():
"""异步版本的详情查询同样要支持连接失败的显式传播。"""
tmdb_api = TmdbApi()
tmdb_api.movie.async_details = AsyncMock(
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
)
with pytest.raises(TMDbConnectionError):
asyncio.run(
tmdb_api.async_get_info(mtype=MediaType.MOVIE, tmdbid=1, raise_on_connection_error=True)
)
def test_async_get_info_swallows_connection_error_by_default():
"""异步默认行为同样必须保持向后兼容。"""
tmdb_api = TmdbApi()
tmdb_api.movie.async_details = AsyncMock(
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
)
result = asyncio.run(tmdb_api.async_get_info(mtype=MediaType.MOVIE, tmdbid=1))
assert result is None
class _FakeTmdbApi:
"""
模拟 TmdbApi.get_info/async_get_info用给定结果驱动
`TheMovieDbModule._get_info_by_tmdbid` 的三种分支确定命中确认不存在连接失败
"""
def __init__(self, tv_outcome, movie_outcome):
"""使用电视剧、电影两路各自的结果初始化。"""
self.tv_outcome = tv_outcome
self.movie_outcome = movie_outcome
self.calls = []
def _resolve(self, outcome, raise_on_connection_error):
if isinstance(outcome, Exception):
if raise_on_connection_error and isinstance(outcome, TMDbConnectionError):
raise outcome
return None
return outcome
def match_multi(self, name: str):
"""
多类型名称匹配真实实现tmdbapi.match_multi会吞掉所有异常并返回 None
连接失败与未找到在这条路径上不可区分fake 必须保持同样语义
"""
return None
async def async_match_multi(self, name: str):
"""异步版多类型匹配,语义同上。"""
return None
async def async_match_multi(self, name: str):
"""异步版多类型匹配。"""
return self._multi_outcome()
def get_info(self, mtype, tmdbid, raise_on_connection_error=False):
"""同步查询:按mtype返回预置结果,忠实模拟 raise_on_connection_error 语义。"""
self.calls.append((mtype, raise_on_connection_error))
outcome = self.tv_outcome if mtype == MediaType.TV else self.movie_outcome
return self._resolve(outcome, raise_on_connection_error)
async def async_get_info(self, mtype, tmdbid, raise_on_connection_error=False):
"""异步查询:委托同步实现。"""
return self.get_info(mtype, tmdbid, raise_on_connection_error=raise_on_connection_error)
class _CacheProbe:
"""记录 update 调用参数,get 恒返回未命中,用于验证网络故障路径不会触碰缓存。"""
def __init__(self):
"""初始化调用记录列表。"""
self.update_calls = []
def get(self, meta):
"""始终返回缓存未命中。"""
return {}
def update(self, meta, info):
"""记录写入尝试,供测试断言从未被调用或调用了哪些内容。"""
self.update_calls.append(info)
def _build_module(tv_outcome, movie_outcome) -> TheMovieDbModule:
"""构造绕开真实HTTP/缓存的 TheMovieDbModule 测试实例。"""
module = TheMovieDbModule()
module.tmdb = _FakeTmdbApi(tv_outcome, movie_outcome)
module.cache = _CacheProbe()
return module
def _build_meta(tmdbid: int = 98865) -> MetaInfo:
"""构造识别所需的最小元数据。"""
meta = MetaInfo(title="测试标题")
meta.tmdbid = tmdbid
return meta
def test_get_info_by_tmdbid_raises_connection_error_when_both_types_fail_to_connect():
"""
电影电视剧两路查询都因TMDB连接失败而没有得到确定结果时
不能断言"条目不存在"应向上抛出 TMDbConnectionError
"""
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
with pytest.raises(TMDbConnectionError):
module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
def test_get_info_by_tmdbid_returns_none_when_both_confirmed_not_found():
"""
电影电视剧两路查询都明确返回"未查询到"空dict非连接异常
应保持原有"无法确定媒体类型"语义返回 None 且不抛异常
"""
module = _build_module(tv_outcome={}, movie_outcome={})
result = module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
assert result is None
def test_get_info_by_tmdbid_prefers_positive_result_over_partial_connection_error():
"""
电视剧一路连接失败但电影一路查到了确定结果时应直接返回电影结果
不能因为另一路的瞬时抖动就误判为整体失败
"""
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
movie_info = {
"id": 98865,
"media_type": MediaType.MOVIE,
"title": "测试电影",
"release_date": "2020-01-01",
"genres": [{"id": 28, "name": "动作"}],
}
module = _build_module(tv_outcome=conn_err, movie_outcome=movie_info)
result = module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
assert result is movie_info
def test_async_get_info_by_tmdbid_raises_connection_error_when_both_types_fail_to_connect():
"""异步版本同样要在两路都判定为连接失败时抛出 TMDbConnectionError。"""
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
with pytest.raises(TMDbConnectionError):
asyncio.run(
module._async_get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
)
def test_recognize_media_reports_network_error_message_and_skips_cache_on_connection_failure(
monkeypatch,
):
"""
端到端回归复现因果链中的场景tmdb_id:98865 双路查询均连接失败
识别应返回 None日志应给出明确的网络故障文案而不是"无法确定媒体类型"
且绝不能写入任何缓存正缓存负缓存都不能写
"""
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
mock_logger = Mock()
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
meta = MetaInfo(title="测试标题")
result = module.recognize_media(meta=meta, media_source=MediaSource.TMDB,
media_id="98865", cache=True)
assert result is None
# 网络故障场景不写入任何缓存条目(正缓存/负缓存皆不写)
assert module.cache.update_calls == []
logged_messages = " ".join(
str(call.args[0]) if call.args else "" for call in mock_logger.error.call_args_list
)
assert "连接TheMovieDb失败" in logged_messages or "连接 TheMovieDb 失败" in logged_messages
assert "无法确定媒体类型" not in logged_messages
def test_recognize_media_keeps_not_found_message_when_both_types_confirmed_absent(monkeypatch):
"""
电影电视剧都确认查无此项非连接失败应维持原有的
"无法确定媒体类型,识别失败"文案不能被网络错误文案顶替
"""
module = _build_module(tv_outcome={}, movie_outcome={})
mock_logger = Mock()
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
meta = MetaInfo(title="测试标题")
result = module.recognize_media(meta=meta, media_source=MediaSource.TMDB,
media_id="98865", cache=True)
assert result is None
assert module.cache.update_calls == []
logged_messages = " ".join(
str(call.args[0]) if call.args else "" for call in mock_logger.warn.call_args_list
)
assert "无法确定媒体类型" in logged_messages
def test_async_recognize_media_reports_network_error_message_and_skips_cache_on_connection_failure(
monkeypatch,
):
"""异步识别路径同样要区分网络故障文案,且不得写入缓存。"""
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
mock_logger = Mock()
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
meta = MetaInfo(title="测试标题")
result = asyncio.run(
module.async_recognize_media(meta=meta, media_source=MediaSource.TMDB,
media_id="98865", cache=True)
)
assert result is None
assert module.cache.update_calls == []
logged_messages = " ".join(
str(call.args[0]) if call.args else "" for call in mock_logger.error.call_args_list
)
assert "连接TheMovieDb失败" in logged_messages or "连接 TheMovieDb 失败" in logged_messages
assert "无法确定媒体类型" not in logged_messages
def test_cache_update_writes_nothing_for_connection_error_sentinel():
"""
回归锁定 `TmdbCache.update` 的既有安全语义info=None网络错误场景对应的
取值不应写入任何缓存条目负缓存都不写
"""
cache = object.__new__(TmdbCache)
cache._cache = Mock()
meta = MetaInfo(title="测试标题")
meta.tmdbid = 98865
cache.update(meta, None)
cache._cache.set.assert_not_called()
def test_cache_update_still_writes_negative_cache_for_confirmed_empty_result():
"""
对照组info={}确认查无此项时仍应写入负缓存 `{"id": 0}`
info=None网络错误的行为形成对照证明二者语义未被本次修复混淆
"""
cache = object.__new__(TmdbCache)
cache._cache = Mock()
meta = MetaInfo(title="测试标题")
meta.tmdbid = 98865
cache.update(meta, {})
cache._cache.set.assert_called_once()
args, _kwargs = cache._cache.set.call_args
assert args[1] == {"id": 0}
+134
View File
@@ -0,0 +1,134 @@
"""
有界重试预算的端到端行为验证
app/helper/transferhistory.py 的查重闸真值表与计数器 API 已在
tests/test_transfer_history_gate.py 逐项覆盖本文件换一个角度同一源路径
连续多个监控事件串成一条时间线验证瞬时故障能在预算内自愈耗尽预算后被拦
以及删除整理记录会让预算重新满额贴近真实使用场景
"""
from types import SimpleNamespace
from app import schemas
from app.core.config import settings
from app.helper.transferhistory import (
HistoryGateAction,
clear_transfer_failures,
evaluate_history_gate,
failed_retry_count,
record_transfer_failure,
)
def _reset_failed_retries(src_path, storage=None):
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
clear_transfer_failures(src_path, storage)
def _failed_history(history_id: int, src_path: str, storage: str):
"""构造一条持续失败的整理记录替身,模拟同一源路径屡次整理失败后落库的状态。"""
return SimpleNamespace(id=history_id, status=False, src_fileitem=None,
src=src_path, src_storage=storage)
def test_transient_failures_self_heal_within_retry_budget(monkeypatch):
"""
瞬时故障自愈上限为 3 连续 2 次失败后第 3 个事件仍应放行重试
3 次也失败后计数达到上限 4 个事件应被拦截
"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
src_path = "/downloads/retry-budget-self-heal.mkv"
storage = "local"
_reset_failed_retries(src_path, storage)
try:
history = _failed_history(1, src_path, storage)
# 事件 1:整理失败,登记第 1 次失败
record_transfer_failure(src_path, storage)
assert failed_retry_count(src_path, storage) == 1
# 事件 2:计数为 1,未达上限,放行重试;本次也失败,登记第 2 次失败
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
record_transfer_failure(src_path, storage)
assert failed_retry_count(src_path, storage) == 2
# 事件 3:连续失败 2 次后,计数为 2 仍未达上限(3),第 3 个事件仍应放行重试
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
# 第 3 次尝试也失败,登记第 3 次失败,计数达到上限
record_transfer_failure(src_path, storage)
assert failed_retry_count(src_path, storage) == 3
# 事件 4:计数达到上限(3),应被拦截,不再自动重试
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
finally:
_reset_failed_retries(src_path, storage)
def test_clearing_transfer_history_restores_full_retry_budget(monkeypatch):
"""删除整理记录清零:耗尽重试预算后调用 clear_transfer_failures,应重新获得放行资格。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
src_path = "/downloads/retry-budget-cleared-by-delete.mkv"
storage = "local"
_reset_failed_retries(src_path, storage)
try:
history = _failed_history(2, src_path, storage)
record_transfer_failure(src_path, storage)
record_transfer_failure(src_path, storage)
assert failed_retry_count(src_path, storage) == 2
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
# 用户删除整理记录,显式要求重来
clear_transfer_failures(src_path, storage)
assert failed_retry_count(src_path, storage) == 0
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
finally:
_reset_failed_retries(src_path, storage)
def test_delete_transfer_history_endpoint_clears_retry_count(monkeypatch):
"""
app/api/endpoints/history.py::delete_transfer_history 是用户删除整理记录的入口
删除时应连带清空失败重试计数否则重整仍会受上一轮次数限制
该端点依赖 SQLAlchemy Session 与鉴权依赖这里按仓库内既有做法参见
tests/test_manual_transfer_history.py app.api.endpoints.transfer 端点的用法
直接以关键字参数调用端点函数本身绕开 FastAPI 的依赖注入只替换端点内部
实际用到的 TransferHistory.get / TransferHistory.delete 两个类方法
"""
from app.api.endpoints.history import delete_transfer_history
src_path = "/downloads/retry-budget-delete-endpoint.mkv"
storage = "local"
history = SimpleNamespace(
id=101,
src=src_path,
src_storage=storage,
dest_fileitem=None,
src_fileitem=None,
download_hash=None,
)
monkeypatch.setattr("app.api.endpoints.history.TransferHistory.get",
lambda db, history_id: history)
monkeypatch.setattr("app.api.endpoints.history.TransferHistory.delete",
lambda db, history_id: None)
_reset_failed_retries(src_path, storage)
try:
record_transfer_failure(src_path, storage)
record_transfer_failure(src_path, storage)
assert failed_retry_count(src_path, storage) == 2
response = delete_transfer_history(
history_in=schemas.TransferHistory(id=101),
deletesrc=False,
deletedest=False,
db=object(),
_="token",
)
assert response.success is True
assert failed_retry_count(src_path, storage) == 0
finally:
_reset_failed_retries(src_path, storage)
+549
View File
@@ -0,0 +1,549 @@
"""
覆盖 app/helper/transferhistory.py 的整理历史查重闸
监控分发app/monitor/dispatcher.py与整理链计划整理段app/chain/transfer.py
共用这套判定本文件只测判定本身的真值表与查询辅助函数不涉及调用方
"""
from types import SimpleNamespace
import pytest
from app.core.config import settings
from app.helper import transferhistory as transferhistory_helper
from app.helper.transferhistory import (
HistoryGateAction,
clear_transfer_failures,
coerce_size,
describe_history_gate,
evaluate_history_gate,
failed_retry_count,
history_src_size,
is_skip_action,
max_failed_retries,
record_transfer_failure,
resolve_history,
)
def make_history(status: bool, size=1024, has_src_fileitem: bool = True,
history_id: int = 1, src_fileitem_override=None,
src=None, src_storage=None, modify_time=None, fileid=None):
"""构造用于查重闸判定的整理记录替身。"""
if src_fileitem_override is not None:
src_fileitem = src_fileitem_override
elif has_src_fileitem:
src_fileitem = {
"size": size,
"modify_time": modify_time,
"fileid": fileid,
}
else:
src_fileitem = None
return SimpleNamespace(id=history_id, status=status, src_fileitem=src_fileitem,
src=src, src_storage=src_storage)
def _reset_failed_retries(src_path, storage=None):
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
clear_transfer_failures(src_path, storage)
# ---------------------------------------------------------------------------
# evaluate_history_gate 真值表:无记录 / 成功记录
# ---------------------------------------------------------------------------
def test_evaluate_history_gate_passes_when_no_record():
"""没有整理记录时应放行整理。"""
action = evaluate_history_gate(None, file_size=1024)
assert action == HistoryGateAction.PASS_NO_RECORD
def test_evaluate_history_gate_passes_when_success_size_changed():
"""成功记录但源文件大小已变化时应放行,交由 overwrite_mode 决断。"""
history = make_history(status=True, size=1024)
action = evaluate_history_gate(history, file_size=2048)
assert action == HistoryGateAction.PASS_SIZE_CHANGED
def test_evaluate_history_gate_skips_when_success_size_unchanged():
"""成功记录且源文件大小未变化时应跳过。"""
history = make_history(status=True, size=1024)
action = evaluate_history_gate(history, file_size=1024)
assert action == HistoryGateAction.SKIP
def test_evaluate_history_gate_skips_when_recorded_size_missing():
"""成功记录缺少大小信息(如蓝光目录)时无法比对,保守跳过。"""
history = make_history(status=True, has_src_fileitem=False)
action = evaluate_history_gate(history, file_size=1024)
assert action == HistoryGateAction.SKIP
def test_evaluate_history_gate_skips_when_current_size_is_none():
"""当前文件大小取不到时无法比对,保守跳过。"""
history = make_history(status=True, size=1024)
action = evaluate_history_gate(history, file_size=None)
assert action == HistoryGateAction.SKIP
def test_evaluate_history_gate_skips_when_src_fileitem_is_not_dict():
"""src_fileitem 历史数据异常(不是字典)时无法比对,保守跳过。"""
history = make_history(status=True, src_fileitem_override="not-a-dict")
action = evaluate_history_gate(history, file_size=1024)
assert action == HistoryGateAction.SKIP
def test_evaluate_history_gate_size_changed_ignores_failed_retry_budget(monkeypatch):
"""
重试次数上限只影响失败记录的判定即便同路径的失败计数早已超过上限
成功记录 + 源文件大小变化时仍应放行交由 overwrite_mode 决断
"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
src_path = "/downloads/gate-test-size-changed-ignores-budget.mkv"
_reset_failed_retries(src_path, "local")
try:
for _ in range(5):
record_transfer_failure(src_path, "local")
assert failed_retry_count(src_path, "local") > max_failed_retries()
history = make_history(status=True, size=1024, src=src_path, src_storage="local")
action = evaluate_history_gate(history, file_size=2048)
assert action == HistoryGateAction.PASS_SIZE_CHANGED
finally:
_reset_failed_retries(src_path, "local")
# ---------------------------------------------------------------------------
# evaluate_history_gate 真值表:失败记录 —— 有界重试
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("retry_count, expected", [
(0, HistoryGateAction.PASS_FAILED),
(1, HistoryGateAction.PASS_FAILED),
(2, HistoryGateAction.PASS_FAILED),
(3, HistoryGateAction.SKIP_RETRY_EXHAUSTED),
(4, HistoryGateAction.SKIP_RETRY_EXHAUSTED),
])
def test_evaluate_history_gate_failed_record_retry_budget_truth_table(monkeypatch, retry_count, expected):
"""失败记录按有界重试判定:未达上限(含 0)放行重试,达到或超过上限后跳过。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
history = make_history(status=False, size=1024)
action = evaluate_history_gate(history, file_size=1024, retry_count=retry_count)
assert action == expected
def test_evaluate_history_gate_failed_record_queries_realtime_count_when_omitted(monkeypatch):
"""retry_count 省略(为 None)时应按 history.src / history.src_storage 实时查询失败计数。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
src_path = "/downloads/gate-test-realtime-count.mkv"
_reset_failed_retries(src_path, "local")
try:
history = make_history(status=False, size=1024, src=src_path, src_storage="local")
# 计数为 0:未达上限(2),放行重试
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.PASS_FAILED
# 累计一次失败,计数为 1:仍未达上限
record_transfer_failure(src_path, "local")
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.PASS_FAILED
# 累计第二次失败,计数为 2:达到上限,跳过
record_transfer_failure(src_path, "local")
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
finally:
_reset_failed_retries(src_path, "local")
def test_evaluate_history_gate_explicit_retry_count_overrides_realtime_lookup(monkeypatch):
"""显式传入 retry_count 时不应再触发实时查询,不受计数器实际状态影响。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
src_path = "/downloads/gate-test-explicit-overrides-realtime.mkv"
_reset_failed_retries(src_path, "local")
try:
for _ in range(10):
record_transfer_failure(src_path, "local")
history = make_history(status=False, size=1024, src=src_path, src_storage="local")
# 即使实时计数早已超限,显式传入的低 retry_count 仍应放行
action = evaluate_history_gate(history, file_size=1024, retry_count=0)
assert action == HistoryGateAction.PASS_FAILED
finally:
_reset_failed_retries(src_path, "local")
def test_failed_history_new_size_passes_and_resets_retry_budget(monkeypatch):
"""失败预算耗尽后同路径文件大小变化时,应放行新版本并从第 1 次失败重新计数。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
src_path = "/downloads/gate-test-failed-new-size.mkv"
_reset_failed_retries(src_path, "local")
try:
history = make_history(
status=False,
size=1024,
src=src_path,
src_storage="local",
)
record_transfer_failure(src_path, "local", file_size=1024)
record_transfer_failure(src_path, "local", file_size=1024)
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
assert (
evaluate_history_gate(history, file_size=2048)
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
)
assert record_transfer_failure(src_path, "local", file_size=2048) == 1
assert failed_retry_count(src_path, "local", file_size=2048) == 1
finally:
_reset_failed_retries(src_path, "local")
@pytest.mark.parametrize(
"current_fields",
[
{"file_modify_time": 200.0, "fileid": "same-id"},
{"file_modify_time": 100.0, "fileid": "new-id"},
],
)
def test_failed_history_same_size_new_fingerprint_passes(current_fields, monkeypatch):
"""大小相同但修改时间或文件 ID 改变时,也应视为失败文件的新版本。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
src_path = "/downloads/gate-test-failed-new-fingerprint.mkv"
_reset_failed_retries(src_path, "local")
try:
history = make_history(
status=False,
size=1024,
modify_time=100.0,
fileid="same-id",
src=src_path,
src_storage="local",
)
record_transfer_failure(
src_path,
"local",
file_size=1024,
file_modify_time=100.0,
fileid="same-id",
)
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
assert (
evaluate_history_gate(history, file_size=1024, **current_fields)
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
)
finally:
_reset_failed_retries(src_path, "local")
def test_legacy_integer_retry_count_is_upgraded_after_new_version_failure(monkeypatch):
"""Redis 中遗留的整数计数不得阻断新版本,并应在下次失败时升级为指纹状态。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
src_path = "/downloads/gate-test-legacy-retry-state.mkv"
_reset_failed_retries(src_path, "local")
try:
key = transferhistory_helper.failed_retry_key(src_path, "local")
transferhistory_helper._failed_retry_counts[key] = 2
history = make_history(
status=False,
size=1024,
src=src_path,
src_storage="local",
)
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
assert (
evaluate_history_gate(history, file_size=2048)
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
)
assert record_transfer_failure(src_path, "local", file_size=2048) == 1
assert failed_retry_count(src_path, "local", file_size=2048) == 1
finally:
_reset_failed_retries(src_path, "local")
# ---------------------------------------------------------------------------
# max_failed_retries 钳制
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("raw, expected", [
(-1, 1),
(0, 1),
(1, 1),
(3, 3),
(10, 10),
(11, 10),
])
def test_max_failed_retries_clamps_numeric_values(monkeypatch, raw, expected):
"""合法区间外的配置值应被钳制到 [1, 10],区间内的值原样返回。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", raw)
assert max_failed_retries() == expected
def test_max_failed_retries_falls_back_when_non_integer(monkeypatch):
"""非整数配置(如解析失败的字符串)应回退为下界 1。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", "abc")
assert max_failed_retries() == 1
# ---------------------------------------------------------------------------
# 失败计数器:record_transfer_failure / failed_retry_count / clear_transfer_failures
# ---------------------------------------------------------------------------
def test_record_transfer_failure_returns_incrementing_count():
"""连续记录失败应返回递增的累计次数。"""
src_path = "/downloads/gate-test-incrementing-count.mkv"
_reset_failed_retries(src_path, "local")
try:
assert record_transfer_failure(src_path, "local") == 1
assert record_transfer_failure(src_path, "local") == 2
assert record_transfer_failure(src_path, "local") == 3
assert failed_retry_count(src_path, "local") == 3
finally:
_reset_failed_retries(src_path, "local")
def test_clear_transfer_failures_resets_count_to_zero():
"""清空后应归零,且不再影响后续查询。"""
src_path = "/downloads/gate-test-clear-resets.mkv"
_reset_failed_retries(src_path, "local")
try:
record_transfer_failure(src_path, "local")
record_transfer_failure(src_path, "local")
assert failed_retry_count(src_path, "local") == 2
clear_transfer_failures(src_path, "local")
assert failed_retry_count(src_path, "local") == 0
finally:
_reset_failed_retries(src_path, "local")
def test_failed_retry_count_isolates_different_storages_for_same_path():
"""相同源路径但不同 storage 的失败计数应互不影响。"""
src_path = "/downloads/gate-test-storage-isolation.mkv"
_reset_failed_retries(src_path, "local")
_reset_failed_retries(src_path, "alist")
try:
record_transfer_failure(src_path, "local")
record_transfer_failure(src_path, "local")
record_transfer_failure(src_path, "alist")
assert failed_retry_count(src_path, "local") == 2
assert failed_retry_count(src_path, "alist") == 1
finally:
_reset_failed_retries(src_path, "local")
_reset_failed_retries(src_path, "alist")
def test_failed_retry_count_defaults_to_zero_without_record():
"""未记录过失败的路径应返回 0。"""
src_path = "/downloads/gate-test-no-record-yet.mkv"
_reset_failed_retries(src_path, "local")
assert failed_retry_count(src_path, "local") == 0
# ---------------------------------------------------------------------------
# is_skip_action
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("action, expected", [
(HistoryGateAction.PASS_NO_RECORD, False),
(HistoryGateAction.PASS_FAILED, False),
(HistoryGateAction.PASS_FAILED_VERSION_CHANGED, False),
(HistoryGateAction.PASS_SIZE_CHANGED, False),
(HistoryGateAction.SKIP_RETRY_EXHAUSTED, True),
(HistoryGateAction.SKIP, True),
])
def test_is_skip_action(action, expected):
"""跳过整理的判定应只对 SKIP 与 SKIP_RETRY_EXHAUSTED 返回 True。"""
assert is_skip_action(action) is expected
# ---------------------------------------------------------------------------
# coerce_size
# ---------------------------------------------------------------------------
def test_coerce_size_returns_none_for_none():
"""None 应原样返回 None,表示不可比对。"""
assert coerce_size(None) is None
def test_coerce_size_returns_none_for_non_numeric_string():
"""非数字字符串无法转换,应返回 None。"""
assert coerce_size("not-a-number") is None
def test_coerce_size_truncates_float():
"""浮点数应按 int() 截断转换。"""
assert coerce_size(1024.9) == 1024
def test_coerce_size_parses_numeric_string():
"""数字字符串应正确转换为整数。"""
assert coerce_size("2048") == 2048
# ---------------------------------------------------------------------------
# resolve_history
# ---------------------------------------------------------------------------
def test_resolve_history_upgrades_failed_hit_to_success_record():
"""get_by_src 命中失败记录且存在成功记录时,应返回成功记录。"""
failed_history = make_history(status=False, history_id=1)
success_history = make_history(status=True, history_id=2)
oper = SimpleNamespace(
get_by_src=lambda src, storage=None: failed_history,
get_success_by_src=lambda src, storage=None: success_history,
)
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
assert history is success_history
def test_resolve_history_keeps_failed_hit_when_no_success_record():
"""get_by_src 命中失败记录但不存在成功记录时,应返回原失败记录。"""
failed_history = make_history(status=False, history_id=1)
oper = SimpleNamespace(
get_by_src=lambda src, storage=None: failed_history,
get_success_by_src=lambda src, storage=None: None,
)
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
assert history is failed_history
def test_resolve_history_does_not_query_success_when_already_successful():
"""get_by_src 直接命中成功记录时,不应再查询 get_success_by_src。"""
success_history = make_history(status=True, history_id=3)
success_query_calls = []
def get_success_by_src(src, storage=None):
success_query_calls.append(src)
return success_history
oper = SimpleNamespace(
get_by_src=lambda src, storage=None: success_history,
get_success_by_src=get_success_by_src,
)
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
assert history is success_history
assert success_query_calls == []
def test_resolve_history_returns_none_when_no_record():
"""没有命中任何记录时应返回 None,不触发额外查询。"""
success_query_calls = []
def get_success_by_src(src, storage=None):
success_query_calls.append(src)
return None
oper = SimpleNamespace(
get_by_src=lambda src, storage=None: None,
get_success_by_src=get_success_by_src,
)
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
assert history is None
assert success_query_calls == []
# ---------------------------------------------------------------------------
# describe_history_gate
# ---------------------------------------------------------------------------
def test_describe_history_gate_reports_no_record():
"""没有记录时应给出明确的无记录说明。"""
assert describe_history_gate(None, file_size=1024) == "无整理记录"
def test_describe_history_gate_includes_status_and_sizes():
"""说明文案应包含记录状态、记录中的大小与当前大小两个数值。"""
history = make_history(status=True, size=1024, history_id=9)
description = describe_history_gate(history, file_size=2048)
assert "成功记录 #9" in description
assert str(history_src_size(history)) in description
assert str(coerce_size(2048)) in description
assert "1024" in description
assert "2048" in description
def test_describe_history_gate_reports_failed_status_with_retry_progress(monkeypatch):
"""失败记录的说明文案应标注记录号,并包含「已重试 n/max 次」的实时计数。"""
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
src_path = "/downloads/gate-test-describe-failed.mkv"
_reset_failed_retries(src_path, "local")
try:
record_transfer_failure(src_path, "local")
record_transfer_failure(src_path, "local")
history = make_history(status=False, size=1024, history_id=5,
src=src_path, src_storage="local")
description = describe_history_gate(history, file_size=1024)
assert "失败记录 #5" in description
assert "已重试" in description
assert "2/3" in description
finally:
_reset_failed_retries(src_path, "local")
def test_describe_history_gate_reports_incomparable_sizes():
"""两侧大小都取不到时应说明大小不可比对。"""
history = make_history(status=True, has_src_fileitem=False, history_id=7)
description = describe_history_gate(history, file_size=None)
assert description == "成功记录 #7,大小不可比对"
def test_clear_transfer_failures_is_safe_when_no_count_recorded():
"""
清空从未失败过的源路径不得抛异常
整理成功回调会对每个文件无条件清零而绝大多数文件从未失败过
底层 CacheBackend.pop default None当成未提供 default
键不存在时会抛 KeyError一旦回归就会让每一次首次成功整理都失败
"""
clear_transfer_failures("/downloads/gate-test-never-failed.mkv", "local")
assert failed_retry_count("/downloads/gate-test-never-failed.mkv", "local") == 0
def test_clear_transfer_failures_is_safe_for_empty_path():
"""源路径为空时清零应静默返回,不得抛异常。"""
clear_transfer_failures(None, None)
clear_transfer_failures("", "local")
+223 -20
View File
@@ -7,11 +7,21 @@ from app.core.config import settings
from app.core.context import MediaInfo from app.core.context import MediaInfo
from app.core.meta import MetaVideo from app.core.meta import MetaVideo
from app.chain.transfer import JobManager, TransferChain from app.chain.transfer import JobManager, TransferChain
from app.helper.transferhistory import (
clear_transfer_failures,
failed_retry_count,
record_transfer_failure,
)
from app.modules.filemanager.transhandler import TransHandler from app.modules.filemanager.transhandler import TransHandler
from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask
from app.schemas.types import EventType, MediaSource, MediaType from app.schemas.types import EventType, MediaSource, MediaType
def _reset_failed_retries(src_path, storage=None):
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
clear_transfer_failures(src_path, storage)
class FakeMeta: class FakeMeta:
def __init__(self, episode: int, season: int = 1): def __init__(self, episode: int, season: int = 1):
self.name = "Test Show" self.name = "Test Show"
@@ -600,12 +610,14 @@ class TransferJobManagerTest(unittest.TestCase):
fileitem = make_task(1).fileitem fileitem = make_task(1).fileitem
history = SimpleNamespace( history = SimpleNamespace(
id=1,
status=True, status=True,
download_hash="abc123", download_hash="abc123",
downloader="qbittorrent", downloader="qbittorrent",
) )
transfer_history_oper = SimpleNamespace( transfer_history_oper = SimpleNamespace(
get_by_src=lambda src, storage=None: history get_by_src=lambda src, storage=None: history,
get_success_by_src=lambda src, storage=None: history,
) )
system_config_oper = SimpleNamespace(get=lambda key: None) system_config_oper = SimpleNamespace(get=lambda key: None)
@@ -628,7 +640,11 @@ class TransferJobManagerTest(unittest.TestCase):
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg) self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
self.assertEqual([("abc123", "qbittorrent")], completed) self.assertEqual([("abc123", "qbittorrent")], completed)
def test_failed_history_skip_still_marks_downloader_hash_completed(self): def test_failed_history_is_retried_within_retry_budget(self):
"""
失败重试次数未达上限默认计数为 0失败记录不再跳过整理
会放行重新送入整理链由于种子还没有真正整理完成也不能连带把种子标记为已整理
"""
chain = make_transfer_chain() chain = make_transfer_chain()
completed = [] completed = []
@@ -641,35 +657,222 @@ class TransferJobManagerTest(unittest.TestCase):
(fileitem, False) (fileitem, False)
] ]
planned = []
def fake_handle_transfer(task, callback=None):
"""记录放行重试后实际进入整理执行阶段的文件。"""
planned.append(task.fileitem.path)
return True, ""
chain._TransferChain__handle_transfer = fake_handle_transfer
fileitem = make_task(1).fileitem fileitem = make_task(1).fileitem
history = SimpleNamespace( history = SimpleNamespace(
id=2,
status=False, status=False,
download_hash="abc123", download_hash="abc123",
downloader="qbittorrent", downloader="qbittorrent",
src=fileitem.path,
src_storage=fileitem.storage,
) )
transfer_history_oper = SimpleNamespace( transfer_history_oper = SimpleNamespace(
get_by_src=lambda src, storage=None: history get_by_src=lambda src, storage=None: history,
get_success_by_src=lambda src, storage=None: None,
)
download_history_oper = SimpleNamespace(
get_by_hash=lambda download_hash: None,
get_file_by_fullpath=lambda fullpath: None,
get_files_by_savepath=lambda savepath: [],
get_by_path=lambda path: None,
) )
system_config_oper = SimpleNamespace(get=lambda key: None) system_config_oper = SimpleNamespace(get=lambda key: None)
with patch( _reset_failed_retries(fileitem.path, fileitem.storage)
"app.chain.transfer.TransferHistoryOper", try:
return_value=transfer_history_oper, with patch(
), patch( "app.chain.transfer.TransferHistoryOper",
"app.chain.transfer.SystemConfigOper", return_value=transfer_history_oper,
return_value=system_config_oper, ), patch(
): "app.chain.transfer.DownloadHistoryOper",
state, errmsg = TransferChain.do_transfer( return_value=download_history_oper,
chain, ), patch(
fileitem=fileitem, "app.chain.transfer.SystemConfigOper",
downloader="qbittorrent", return_value=system_config_oper,
download_hash="abc123", ):
background=False, state, errmsg = TransferChain.do_transfer(
) chain,
fileitem=fileitem,
downloader="qbittorrent",
download_hash="abc123",
background=False,
)
self.assertFalse(state) self.assertTrue(state)
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg) self.assertEqual("", errmsg)
self.assertEqual([("abc123", "qbittorrent")], completed) self.assertEqual([fileitem.path], planned)
self.assertEqual([], completed)
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_failed_history_skip_marks_downloader_hash_when_retry_budget_exhausted(self):
"""
失败重试次数已达上限时失败记录仍会拦截整理但拦截意味着不再重试
此时仍要给种子打已整理标签让种子退出下载器轮询否则下载器每一轮都会
重新扫描到同一个失败记录空转且刷屏
"""
chain = make_transfer_chain()
completed = []
def fake_transfer_completed(hashs, downloader):
completed.append((hashs, downloader))
chain.transfer_completed = fake_transfer_completed
chain.list_torrents = lambda **kwargs: [SimpleNamespace(progress=100)]
chain._TransferChain__get_trans_fileitems = lambda fileitem, predicate: [
(fileitem, False)
]
planned = []
def fake_handle_transfer(task, callback=None):
"""达到重试上限时不应有文件进入实际整理执行阶段。"""
planned.append(task.fileitem.path)
return True, ""
chain._TransferChain__handle_transfer = fake_handle_transfer
fileitem = make_task(1).fileitem
history = SimpleNamespace(
id=3,
status=False,
download_hash="abc123",
downloader="qbittorrent",
src=fileitem.path,
src_storage=fileitem.storage,
)
transfer_history_oper = SimpleNamespace(
get_by_src=lambda src, storage=None: history,
get_success_by_src=lambda src, storage=None: None,
)
download_history_oper = SimpleNamespace(
get_by_hash=lambda download_hash: None,
get_file_by_fullpath=lambda fullpath: None,
get_files_by_savepath=lambda savepath: [],
get_by_path=lambda path: None,
)
system_config_oper = SimpleNamespace(get=lambda key: None)
_reset_failed_retries(fileitem.path, fileitem.storage)
try:
with patch.object(
settings, "TRANSFER_MAX_FAILED_RETRIES", 1,
):
record_transfer_failure(fileitem.path, fileitem.storage)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=transfer_history_oper,
), patch(
"app.chain.transfer.DownloadHistoryOper",
return_value=download_history_oper,
), patch(
"app.chain.transfer.SystemConfigOper",
return_value=system_config_oper,
):
state, errmsg = TransferChain.do_transfer(
chain,
fileitem=fileitem,
downloader="qbittorrent",
download_hash="abc123",
background=False,
)
self.assertFalse(state)
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
self.assertEqual([], planned)
self.assertEqual([("abc123", "qbittorrent")], completed)
finally:
_reset_failed_retries(fileitem.path, fileitem.storage)
def test_default_callback_failure_and_success_track_retry_counter(self):
"""
__default_callback 应在整理失败时累计连续失败次数整理成功时清零
避免瞬时故障与长期失败混用同一份计数导致误判上限
"""
chain = make_transfer_chain()
chain.eventmanager = MagicMock()
chain.post_message = MagicMock()
chain.transfer_completed = lambda *args, **kwargs: None
task = make_task(1)
task.mediainfo = FakeMedia()
# __default_callback 失败通知路径需要读取海报图,FakeMedia 本身不提供该接口
task.mediainfo.get_message_image = lambda: "poster.jpg"
task.background = False
task.manual = True
src_path = task.fileitem.path
storage = task.fileitem.storage
_reset_failed_retries(src_path, storage)
try:
self.assertEqual(0, failed_retry_count(src_path, storage))
failed_transferinfo = TransferInfo(
success=False,
fileitem=task.fileitem,
message="整理失败测试",
transfer_type="copy",
need_notify=False,
)
failed_history_oper = SimpleNamespace(
add_fail=lambda **kwargs: SimpleNamespace(id=1),
)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=failed_history_oper,
), patch(
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
), patch(
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
):
state, _ = chain._TransferChain__default_callback(task, failed_transferinfo)
self.assertFalse(state)
self.assertEqual(1, failed_retry_count(src_path, storage))
self.assertTrue(chain._TransferChain__put_to_jobview(task))
success_transferinfo = TransferInfo(
success=True,
fileitem=task.fileitem,
target_item=FileItem(
storage=storage,
path="/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv",
type="file",
name="Test.Show.S01E01.mkv",
extension="mkv",
),
target_diritem=FileItem(
storage=storage,
path="/library/Test Show (2026)/Season 1/",
type="dir",
name="Season 1",
),
file_list_new=[
"/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv"
],
transfer_type="copy",
need_scrape=False,
need_notify=False,
)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=SimpleNamespace(add_success=lambda **kwargs: SimpleNamespace(id=2)),
):
state, _ = chain._TransferChain__default_callback(task, success_transferinfo)
self.assertTrue(state)
self.assertEqual(0, failed_retry_count(src_path, storage))
finally:
_reset_failed_retries(src_path, storage)
def test_unrecognized_task_marks_downloader_hash_completed(self): def test_unrecognized_task_marks_downloader_hash_completed(self):
chain = make_transfer_chain() chain = make_transfer_chain()
+218
View File
@@ -0,0 +1,218 @@
"""
覆盖不覆盖裁决不应降级已有成功记录的行为
查重闸放行同路径新版本后 overwrite_mode 最终裁定不覆盖媒体库中原有的
成功版本仍然在位这是一次正常策略裁决而非整理故障TransferChain 内部的
__is_overwrite_declined 用于识别这一场景__default_callback 失败分支据此
决定是否写失败历史发送失败事件与失败通知本文件覆盖两者
"""
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from app.chain.transfer import TransferChain
from app.schemas import TransferInfo
from app.schemas.types import EventType
from tests.test_transfer_job_manager import FakeMedia, make_task, make_transfer_chain
def make_history_oper(history=None, success_history=None, raise_on_query: bool = False,
add_fail_calls=None):
"""构造 __is_overwrite_declined / __default_callback 查询与写入整理历史使用的替身。"""
def get_by_src(src, storage=None):
if raise_on_query:
raise RuntimeError("boom")
return history
def get_success_by_src(src, storage=None):
return success_history
def add_fail(**kwargs):
if add_fail_calls is not None:
add_fail_calls.append(kwargs)
return SimpleNamespace(id=1)
return SimpleNamespace(
get_by_src=get_by_src,
get_success_by_src=get_success_by_src,
add_fail=add_fail,
)
# ---------------------------------------------------------------------------
# TransferChain.__is_overwrite_declined
# ---------------------------------------------------------------------------
def test_overwrite_declined_false_when_flag_not_set():
"""overwrite_skipped 为假时直接判定为 False,且不应触发历史查询。"""
task = make_task(1)
transferinfo = TransferInfo(success=False, overwrite_skipped=False)
transferhis = make_history_oper(raise_on_query=True)
result = TransferChain._TransferChain__is_overwrite_declined(
task, transferinfo, transferhis
)
assert result is False
def test_overwrite_declined_true_when_success_history_exists():
"""overwrite_skipped 为真且同源已有成功记录时,应判定为保护场景。"""
task = make_task(1)
success_history = SimpleNamespace(id=1, status=True)
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
transferhis = make_history_oper(history=success_history)
result = TransferChain._TransferChain__is_overwrite_declined(
task, transferinfo, transferhis
)
assert result is True
def test_overwrite_declined_false_when_no_history():
"""overwrite_skipped 为真但没有任何整理记录时,不应判定为保护场景。"""
task = make_task(1)
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
transferhis = make_history_oper(history=None)
result = TransferChain._TransferChain__is_overwrite_declined(
task, transferinfo, transferhis
)
assert result is False
def test_overwrite_declined_false_when_only_failed_history():
"""overwrite_skipped 为真但只有失败记录时,不应判定为保护场景。"""
task = make_task(1)
failed_history = SimpleNamespace(id=2, status=False)
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
transferhis = make_history_oper(history=failed_history, success_history=None)
result = TransferChain._TransferChain__is_overwrite_declined(
task, transferinfo, transferhis
)
assert result is False
def test_overwrite_declined_false_when_query_raises():
"""查询整理历史异常时应保守返回 False,不阻断原有失败语义。"""
task = make_task(1)
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
transferhis = make_history_oper(raise_on_query=True)
result = TransferChain._TransferChain__is_overwrite_declined(
task, transferinfo, transferhis
)
assert result is False
# ---------------------------------------------------------------------------
# __default_callback 失败分支
# ---------------------------------------------------------------------------
def _make_failed_task():
"""构造一个失败回调测试所需的最小整理任务。"""
task = make_task(1)
task.mediainfo = FakeMedia()
# __default_callback 失败通知路径需要读取海报图,FakeMedia 本身不提供该接口
task.mediainfo.get_message_image = lambda: "poster.jpg"
task.background = False
task.manual = True
return task
def test_default_callback_skips_history_and_notification_when_overwrite_declined():
"""
同源已有成功记录时覆盖裁决不覆盖不应写失败历史不应发送失败事件与通知
"""
chain = make_transfer_chain()
chain.eventmanager = MagicMock()
chain.post_message = MagicMock()
task = _make_failed_task()
success_history = SimpleNamespace(id=99, status=True)
add_fail_calls = []
transfer_history_oper = make_history_oper(
history=success_history, add_fail_calls=add_fail_calls
)
transferinfo = TransferInfo(
success=False,
fileitem=task.fileitem,
message="目标已存在,按覆盖策略跳过覆盖",
transfer_type="copy",
overwrite_skipped=True,
need_notify=False,
)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=transfer_history_oper,
), patch(
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
), patch(
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
):
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
assert state is False
assert errmsg == transferinfo.message
assert add_fail_calls == []
assert chain.post_message.call_count == 0
transfer_failed_events = [
call
for call in chain.eventmanager.send_event.call_args_list
if call.args[0] == EventType.TransferFailed
]
assert transfer_failed_events == []
def test_default_callback_keeps_original_failure_semantics_without_success_history():
"""
没有已有成功记录时即使 overwrite_skipped 为真仍应按原有语义写失败历史并通知
"""
chain = make_transfer_chain()
chain.eventmanager = MagicMock()
chain.post_message = MagicMock()
task = _make_failed_task()
add_fail_calls = []
transfer_history_oper = make_history_oper(
history=None, add_fail_calls=add_fail_calls
)
transferinfo = TransferInfo(
success=False,
fileitem=task.fileitem,
message="目标已存在,按覆盖策略跳过覆盖",
transfer_type="copy",
overwrite_skipped=True,
need_notify=False,
)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=transfer_history_oper,
), patch(
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
), patch(
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
):
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
assert state is False
assert errmsg == transferinfo.message
assert len(add_fail_calls) == 1
assert chain.post_message.call_count == 1
transfer_failed_events = [
call
for call in chain.eventmanager.send_event.call_args_list
if call.args[0] == EventType.TransferFailed
]
assert len(transfer_failed_events) == 1
+13 -6
View File
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
import pytest import pytest
from app.modules.filemanager.storages import StorageBase
from app.modules.filemanager.storages.alipan import AliPan from app.modules.filemanager.storages.alipan import AliPan
from app.modules.filemanager.storages.local import LocalStorage from app.modules.filemanager.storages.local import LocalStorage
from app.modules.filemanager.storages.rclone import Rclone from app.modules.filemanager.storages.rclone import Rclone
@@ -87,13 +88,17 @@ def test_local_strict_raises_on_stat_error(tmp_path, monkeypatch):
""" """
target = tmp_path / "movie.mkv" target = tmp_path / "movie.mkv"
def raise_stat_error(self, *args, **kwargs): def raise_stat_error(*_args, **_kwargs):
""" """
模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE 模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE
""" """
raise OSError(131, "State not recoverable") raise OSError(131, "State not recoverable")
monkeypatch.setattr(Path, "stat", raise_stat_error) # 文件系统边界已下移到代理子进程,patch Path.stat 影响不到那里,
# 必须在代理这一层注入故障
monkeypatch.setattr(
"app.modules.filemanager.storages.local.fsproxy.stat", raise_stat_error
)
with pytest.raises(StorageQueryError): with pytest.raises(StorageQueryError):
_local().get_item_strict(target) _local().get_item_strict(target)
@@ -229,12 +234,14 @@ def test_alipan_strict_returns_item(monkeypatch):
assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM" assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM"
def test_storage_base_strict_defaults_to_get_item(): def test_storage_base_strict_fails_conservatively_without_override():
""" """
未覆写的存储沿用 get_item 判定行为不变 未覆写严格查询的存储必须保守失败沿用 get_item 会把查询失败当成
目标不存在 overwrite_mode=size 的覆盖保护被绕过
""" """
storage = object.__new__(Rclone) storage = object.__new__(Rclone)
storage.get_item = MagicMock(return_value=None) storage.get_item = MagicMock(return_value=None)
assert storage.get_item_strict(Path("/movie.mkv")) is None with pytest.raises(StorageQueryError):
storage.get_item.assert_called_once() StorageBase.get_item_strict(storage, Path("/movie.mkv"))
storage.get_item.assert_not_called()
+232
View File
@@ -0,0 +1,232 @@
"""
整理队列持久化与重启回放测试
整理队列是纯内存的 queue.Queue挂载挂死后的人工重启版本升级OOM宿主
重启都会让队列连同这些文件还没整理这个事实一起蒸发而已稳定落地的文件
不会再产生任何监控事件也不会有新的补偿扫描起点结果就是永久漏件
这些测试固定三项不变量入队即落盘登记终态即注销重启能回放
"""
from pathlib import Path
from unittest.mock import MagicMock
from app.chain.transfer import TransferChain
from app.schemas import FileItem, TransferTask
def _build_chain(pendingoper) -> TransferChain:
"""
构造绕过单例初始化的 TransferChain 骨架
:param pendingoper: 待整理登记管理替身
:return: TransferChain 骨架
"""
chain = object.__new__(TransferChain)
chain._pendingoper = pendingoper
return chain
def _task(path: str, storage: str = "local") -> TransferTask:
"""
构造测试用整理任务
:param path: 源文件路径
:param storage: 存储
:return: 整理任务
"""
file_path = Path(path)
return TransferTask(fileitem=FileItem(
storage=storage,
path=path,
type="file",
name=file_path.name,
basename=file_path.stem,
extension=file_path.suffix[1:],
))
def test_register_pending_records_storage_and_path():
"""
入队时必须落盘登记存储 + 源路径这一最小事实
"""
pendingoper = MagicMock()
chain = _build_chain(pendingoper)
chain._TransferChain__register_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
pendingoper.register.assert_called_once_with(
storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv"
)
def test_register_pending_failure_does_not_break_enqueue():
"""
落盘登记只是重启后的补救手段登记失败绝不能阻断正常整理
"""
pendingoper = MagicMock()
pendingoper.register.side_effect = RuntimeError("db locked")
chain = _build_chain(pendingoper)
# 不抛异常即为通过
chain._TransferChain__register_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
def test_discard_pending_on_terminal_state():
"""
整理到达终态后必须注销登记否则每次重启都会重复回放
"""
pendingoper = MagicMock()
chain = _build_chain(pendingoper)
chain._TransferChain__discard_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
pendingoper.discard.assert_called_once_with(
storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv"
)
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
"""
重启回放登记过的文件要重新送入整理链恢复被内存队列蒸发的任务
"""
media = tmp_path / "Movie.2024.mkv"
media.write_bytes(b"x" * 10)
pendingoper = MagicMock()
pendingoper.list_all.return_value = [("local", str(media))]
chain = _build_chain(pendingoper)
transferred = []
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
chain._TransferChain__replay_pending()
assert len(transferred) == 1
item = transferred[0]
assert item.path == media.as_posix()
assert item.storage == "local"
assert item.type == "file"
# 回放时重新读取当前大小,不依赖登记时的陈旧信息
assert item.size == 10
def test_replay_discards_vanished_files(tmp_path):
"""
源文件已消失的登记要注销否则每次启动都会重复回放一个不存在的文件
"""
pendingoper = MagicMock()
missing = tmp_path / "gone.mkv"
pendingoper.list_all.return_value = [("local", str(missing))]
chain = _build_chain(pendingoper)
chain.do_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
pendingoper.discard.assert_called_once_with(storage="local", src_path=str(missing))
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
"""
挂载未就绪时读取失败属于暂时性故障登记必须保留等下次启动或人工整理
这与文件已消失必须区别对待把挂载抖动误判成文件消失就等于主动丢件
"""
media = tmp_path / "Movie.2024.mkv"
media.write_bytes(b"x")
pendingoper = MagicMock()
pendingoper.list_all.return_value = [("local", str(media))]
chain = _build_chain(pendingoper)
chain.do_transfer = MagicMock()
def unreadable(self, *_args, **_kwargs):
"""
模拟挂载未就绪时的 stat 失败
"""
raise OSError(107, "Transport endpoint is not connected")
monkeypatch.setattr(Path, "stat", unreadable)
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
pendingoper.discard.assert_not_called()
def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
"""
蓝光原盘登记时保留尾部斜杠回放必须还原成目录类型否则会被当成单文件整理
"""
bluray = tmp_path / "Movie.2024.BluRay"
bluray.mkdir()
src_path = f"{bluray.as_posix()}/"
pendingoper = MagicMock()
pendingoper.list_all.return_value = [("local", src_path)]
chain = _build_chain(pendingoper)
transferred = []
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
chain._TransferChain__replay_pending()
assert len(transferred) == 1
assert transferred[0].type == "dir"
assert transferred[0].path == src_path
def test_replay_is_noop_without_registrations():
"""
没有登记时回放不应触碰整理链
"""
pendingoper = MagicMock()
pendingoper.list_all.return_value = []
chain = _build_chain(pendingoper)
chain.do_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
def test_replay_survives_db_failure():
"""
读取登记失败不能让启动流程报错
"""
pendingoper = MagicMock()
pendingoper.list_all.side_effect = RuntimeError("db gone")
chain = _build_chain(pendingoper)
chain.do_transfer = MagicMock()
chain._TransferChain__replay_pending()
chain.do_transfer.assert_not_called()
def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
"""
单个文件回放失败不能中断整批回放否则一个坏文件会拖住所有漏件的恢复
"""
first = tmp_path / "A.mkv"
second = tmp_path / "B.mkv"
for item in (first, second):
item.write_bytes(b"x")
pendingoper = MagicMock()
pendingoper.list_all.return_value = [("local", str(first)), ("local", str(second))]
chain = _build_chain(pendingoper)
handled = []
def flaky(**kw):
"""
第一个文件整理抛异常第二个正常
"""
if kw["fileitem"].name == "A.mkv":
raise RuntimeError("boom")
handled.append(kw["fileitem"].name)
monkeypatch.setattr(chain, "do_transfer", flaky)
chain._TransferChain__replay_pending()
assert handled == ["B.mkv"]
+65
View File
@@ -0,0 +1,65 @@
"""
整理队列批次计数口径测试
背景:作业视图中的任务完成后仅标记终态,作业要等关联任务全部终态才整体移除,
因此追更/分批场景下已完成任务会跨批次残留批次开始日志与进度分母若用全量
total(),会把历史任务计入当前共 N 个文件(如实际只处理 2 个却显示 8 ),
且进度百分比永远走不满批次统计必须只数未终态任务
"""
import unittest
from app.chain.transfer import JobManager
from tests.test_transfer_job_manager import make_task
class TransferQueueCountTest(unittest.TestCase):
@staticmethod
def _build_jobview_with_stale_tasks() -> JobManager:
"""
构造用户实测的 82 场景:同一作业 8 个任务,6 个已完成(作业因
仍有未终态任务不会被移除),2 个等待处理
"""
jobview = JobManager()
completed = [make_task(episode) for episode in range(1, 7)]
waiting = [make_task(episode) for episode in range(7, 9)]
for task in completed + waiting:
assert jobview.add_task(task)
for task in completed:
jobview.finish_task(task)
# 作业尚有未终态任务,不会被移除,已完成任务随作业残留
jobview.try_remove_job(task)
return jobview
def test_total_still_counts_terminal_tasks(self):
"""total() 保持全量语义(供作业视图展示),包含已完成任务。"""
jobview = self._build_jobview_with_stale_tasks()
self.assertEqual(jobview.total(), 8)
def test_pending_total_excludes_terminal_tasks(self):
"""pending_total() 只数未终态任务,不受跨批次残留影响。"""
jobview = self._build_jobview_with_stale_tasks()
self.assertEqual(jobview.pending_total(), 2)
def test_pending_total_excludes_failed_tasks(self):
"""失败任务同为终态,不应计入待处理数。"""
jobview = JobManager()
failed_task = make_task(1)
waiting_task = make_task(2)
assert jobview.add_task(failed_task)
assert jobview.add_task(waiting_task)
jobview.fail_task(failed_task)
self.assertEqual(jobview.pending_total(), 1)
def test_pending_total_counts_running_tasks(self):
"""运行中的任务属于本批,必须计入。"""
jobview = JobManager()
running_task = make_task(1)
assert jobview.add_task(running_task)
jobview.running_task(running_task)
self.assertEqual(jobview.pending_total(), 1)
if __name__ == "__main__":
unittest.main()