mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-05 23:47:41 +08:00
fix(monitor,transfer): 修复 FUSE 挂载无响应导致的监控冻死、整理链锁死与漏件 (#6276)
* wip(v3): 移植监控与整理韧性修复到 v3 基线 包含:监控看门狗隔离/挂载探测、整理队列持久化、文件系统子进程代理、 写入原子化。迁移重挂到 v3 链 8a4c7e1d2f90 -> 7f5c1d2e3a4b -> e3d9f4b7c806。 tmdb 相关测试尚未通过,待定位。 * fix(v3): 修正移植引入的 16 项测试失败 - poller.py:合并时我方保留的行仍用旧变量名 merged_snapshot,而 v3 已统一 改名为 current_snapshot,导致 NameError 被外层 except 吞掉、快照从未保存 - smb.py:采纳 f-string 拆分写法,恢复 Python 3.11 可解析 - dispatcher 测试:历史查重由 _should_skip_by_history 统一承担,mock 点随之调整 - tmdb 缓存测试:补充 v3 新增的 media_source/media_id 字段 - tmdb 重试测试:为 fake 补充 match_multi/async_match_multi 尚余 3 项与 v3 识别流程的连接失败处理有关,待单独判断。 * fix(v3): 测试适配 v3 的 media_source/media_id 重构 v3 将媒体标识从 tmdbid 统一重构为 media_source + media_id,recognize_media 的 tmdbid 参数已被 **kwargs 静默吞掉——传了也不生效,流程会误降级到名称搜索。 tmdb 重试用例改用新参数后恢复正确路径。 同时修正 fake 的 match_multi 语义:真实实现(tmdbapi.match_multi)吞掉所有 异常并返回 None,连接失败与「未找到」在该路径上本就不可区分,fake 需保持一致。 至此移植引入的 19 项失败全部清零。 --------- Co-authored-by: Aqr-K <Aqr-K@users.noreply.github.com>
This commit is contained in:
@@ -50,3 +50,6 @@ pylint-report.json
|
|||||||
|
|
||||||
# Superpowers 设计/计划文档(本地协作产物,不纳入仓库)
|
# Superpowers 设计/计划文档(本地协作产物,不纳入仓库)
|
||||||
docs/superpowers/
|
docs/superpowers/
|
||||||
|
|
||||||
|
# 本地前端构建产物(docker build 用,不入库)
|
||||||
|
frontend-dist/
|
||||||
|
|||||||
@@ -28,6 +28,7 @@ from app.db.user_oper import (
|
|||||||
get_current_active_superuser_async,
|
get_current_active_superuser_async,
|
||||||
)
|
)
|
||||||
from app.helper.progress import ProgressHelper
|
from app.helper.progress import ProgressHelper
|
||||||
|
from app.helper.transferhistory import clear_transfer_failures
|
||||||
from app.schemas.types import EventType
|
from app.schemas.types import EventType
|
||||||
from app.utils.jieba import cut as jieba_cut
|
from app.utils.jieba import cut as jieba_cut
|
||||||
|
|
||||||
@@ -263,6 +264,8 @@ def delete_transfer_history(
|
|||||||
)
|
)
|
||||||
# 删除记录
|
# 删除记录
|
||||||
TransferHistory.delete(db, history_in.id)
|
TransferHistory.delete(db, history_in.id)
|
||||||
|
# 删除记录是用户显式要求重来,失败重试计数一并清零,否则重整仍会受上一轮次数限制
|
||||||
|
clear_transfer_failures(history.src, history.src_storage)
|
||||||
return schemas.Response(success=True)
|
return schemas.Response(success=True)
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
+326
-80
@@ -25,11 +25,15 @@ from app.db.downloadhistory_oper import DownloadHistoryOper
|
|||||||
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
|
from app.db.models.downloadhistory import DownloadHistory, DownloadFiles
|
||||||
from app.db.models.transferhistory import TransferHistory
|
from app.db.models.transferhistory import TransferHistory
|
||||||
from app.db.systemconfig_oper import SystemConfigOper
|
from app.db.systemconfig_oper import SystemConfigOper
|
||||||
|
from app.db.transferpending_oper import TransferPendingOper
|
||||||
from app.db.transferhistory_oper import TransferHistoryOper
|
from app.db.transferhistory_oper import TransferHistoryOper
|
||||||
from app.helper.directory import DirectoryHelper
|
from app.helper.directory import DirectoryHelper
|
||||||
from app.helper.audio import AudioMetadataHelper
|
from app.helper.audio import AudioMetadataHelper
|
||||||
from app.helper.format import EpisodeFormatRuleHelper, FormatParser
|
from app.helper.format import EpisodeFormatRuleHelper, FormatParser
|
||||||
from app.helper.progress import ProgressHelper
|
from app.helper.progress import ProgressHelper
|
||||||
|
from app.helper.transferhistory import (clear_transfer_failures, describe_history_gate,
|
||||||
|
evaluate_history_gate, is_skip_action,
|
||||||
|
record_transfer_failure, resolve_history)
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.schemas import StorageOperSelectionEventData
|
from app.schemas import StorageOperSelectionEventData
|
||||||
from app.schemas import (
|
from app.schemas import (
|
||||||
@@ -829,6 +833,22 @@ class JobManager:
|
|||||||
with job_lock:
|
with job_lock:
|
||||||
return sum([len(job.tasks) for job in self._job_view.values()])
|
return sum([len(job.tasks) for job in self._job_view.values()])
|
||||||
|
|
||||||
|
def pending_total(self) -> int:
|
||||||
|
"""
|
||||||
|
获取未到终态的任务总数。
|
||||||
|
|
||||||
|
作业要等关联任务全部终态才整体移除,追更/分批场景下已完成任务会
|
||||||
|
跨批次残留在视图中;批次统计若用全量 total() 会把历史任务计入
|
||||||
|
「当前共 N 个文件」并压低进度百分比,因此只数未终态任务。
|
||||||
|
"""
|
||||||
|
with job_lock:
|
||||||
|
return sum(
|
||||||
|
1
|
||||||
|
for job in self._job_view.values()
|
||||||
|
for task in job.tasks
|
||||||
|
if task.state not in ("completed", "failed")
|
||||||
|
)
|
||||||
|
|
||||||
def list_jobs(self) -> List[TransferJob]:
|
def list_jobs(self) -> List[TransferJob]:
|
||||||
"""
|
"""
|
||||||
获取所有作业的任务列表
|
获取所有作业的任务列表
|
||||||
@@ -996,6 +1016,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
self.jobview = JobManager()
|
self.jobview = JobManager()
|
||||||
# Agent重试管理器
|
# Agent重试管理器
|
||||||
self.retry_scheduler = FailedRetryScheduler()
|
self.retry_scheduler = FailedRetryScheduler()
|
||||||
|
# 待整理文件落盘登记,用于进程重启后回放内存队列里未完成的任务
|
||||||
|
self._pendingoper = TransferPendingOper()
|
||||||
# 转移成功的文件清单
|
# 转移成功的文件清单
|
||||||
self._success_target_files: Dict[Tuple, List[str]] = {}
|
self._success_target_files: Dict[Tuple, List[str]] = {}
|
||||||
# 批次级刮削缓冲,避免同一批多文件入库重复触发目录刮削
|
# 批次级刮削缓冲,避免同一批多文件入库重复触发目录刮削
|
||||||
@@ -1339,6 +1361,34 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
)
|
)
|
||||||
return not mounted_filesystem_cache[source_directory]
|
return not mounted_filesystem_cache[source_directory]
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def __is_overwrite_declined(task: TransferTask, transferinfo: TransferInfo,
|
||||||
|
transferhis: TransferHistoryOper) -> bool:
|
||||||
|
"""
|
||||||
|
判断本次未入库是否为「同路径已有成功记录 + 覆盖模式裁定不覆盖」。
|
||||||
|
|
||||||
|
只有同路径此前已成功整理过才需要保护:这类文件是查重闸放行的同路径新版本,
|
||||||
|
媒体库中的原有版本仍然在位,不应因一次不覆盖裁决把成功记录改写成失败记录。
|
||||||
|
没有成功记录时(如目标同名文件来自其他源路径)保持原有失败语义,
|
||||||
|
用户仍能在历史与通知中看到裁决结果。
|
||||||
|
:param task: 整理任务
|
||||||
|
:param transferinfo: 整理结果
|
||||||
|
:param transferhis: 历史操作对象
|
||||||
|
:return: True 表示应保留原成功记录
|
||||||
|
"""
|
||||||
|
if not transferinfo.overwrite_skipped or not task.fileitem:
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
history = resolve_history(
|
||||||
|
task.fileitem.path,
|
||||||
|
storage=task.fileitem.storage,
|
||||||
|
transfer_history_oper=transferhis,
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"查询整理历史失败: {task.fileitem.path} - {err}")
|
||||||
|
return False
|
||||||
|
return bool(history and history.status)
|
||||||
|
|
||||||
def __default_callback(
|
def __default_callback(
|
||||||
self, task: TransferTask, transferinfo: TransferInfo, /
|
self, task: TransferTask, transferinfo: TransferInfo, /
|
||||||
) -> Tuple[bool, str]:
|
) -> Tuple[bool, str]:
|
||||||
@@ -1390,86 +1440,108 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
|
|
||||||
# 转移失败
|
# 转移失败
|
||||||
if not transferinfo.success:
|
if not transferinfo.success:
|
||||||
logger.warn(f"{task.fileitem.name} 入库失败:{transferinfo.message}")
|
# 查重闸放行同路径新版本后由 overwrite_mode 判定不覆盖,是一次正常裁决而非故障:
|
||||||
|
# 媒体库里原有版本仍然在位,写失败记录会用 add_force 顶掉原成功记录,此后该路径
|
||||||
# 新增转移失败历史记录
|
# 永远处于失败态,每个新事件都会重试并重推失败通知。此时保留原记录、不写历史、
|
||||||
history = transferhis.add_fail(
|
# 不发事件与通知、不触发重试,仅把任务置为未入库
|
||||||
fileitem=task.fileitem,
|
overwrite_declined = self.__is_overwrite_declined(
|
||||||
mode=transferinfo.transfer_type if transferinfo else "",
|
task, transferinfo, transferhis
|
||||||
downloader=task.downloader,
|
|
||||||
download_hash=task.download_hash,
|
|
||||||
meta=task.meta,
|
|
||||||
mediainfo=task.mediainfo,
|
|
||||||
transferinfo=transferinfo,
|
|
||||||
)
|
)
|
||||||
|
history = None
|
||||||
|
if overwrite_declined:
|
||||||
|
logger.info(
|
||||||
|
f"{task.fileitem.name} 未入库并保留原整理记录:{transferinfo.message}"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warn(f"{task.fileitem.name} 入库失败:{transferinfo.message}")
|
||||||
|
|
||||||
# 整理失败事件
|
# 累计失败次数,达到上限后查重闸不再自动放行重试
|
||||||
if self._is_primary_media_file(task.fileitem, task.mediainfo):
|
record_transfer_failure(
|
||||||
# 主要媒体文件整理失败事件
|
task.fileitem.path if task.fileitem else None,
|
||||||
self.eventmanager.send_event(
|
task.fileitem.storage if task.fileitem else None,
|
||||||
EventType.TransferFailed,
|
file_size=task.fileitem.size if task.fileitem else None,
|
||||||
{
|
file_modify_time=task.fileitem.modify_time if task.fileitem else None,
|
||||||
"fileitem": task.fileitem,
|
fileid=task.fileitem.fileid if task.fileitem else None,
|
||||||
"meta": task.meta,
|
|
||||||
"mediainfo": task.mediainfo,
|
|
||||||
"transferinfo": transferinfo,
|
|
||||||
"downloader": task.downloader,
|
|
||||||
"download_hash": task.download_hash,
|
|
||||||
"transfer_history_id": history.id if history else None,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
elif self.__is_subtitle_file(task.fileitem):
|
|
||||||
# 字幕整理失败事件
|
|
||||||
self.eventmanager.send_event(
|
|
||||||
EventType.SubtitleTransferFailed,
|
|
||||||
{
|
|
||||||
"fileitem": task.fileitem,
|
|
||||||
"meta": task.meta,
|
|
||||||
"mediainfo": task.mediainfo,
|
|
||||||
"transferinfo": transferinfo,
|
|
||||||
"downloader": task.downloader,
|
|
||||||
"download_hash": task.download_hash,
|
|
||||||
"transfer_history_id": history.id if history else None,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
elif self.__is_audio_file(task.fileitem):
|
|
||||||
# 音频文件整理失败事件
|
|
||||||
self.eventmanager.send_event(
|
|
||||||
EventType.AudioTransferFailed,
|
|
||||||
{
|
|
||||||
"fileitem": task.fileitem,
|
|
||||||
"meta": task.meta,
|
|
||||||
"mediainfo": task.mediainfo,
|
|
||||||
"transferinfo": transferinfo,
|
|
||||||
"downloader": task.downloader,
|
|
||||||
"download_hash": task.download_hash,
|
|
||||||
"transfer_history_id": history.id if history else None,
|
|
||||||
},
|
|
||||||
)
|
)
|
||||||
|
|
||||||
# 发送失败消息
|
# 新增转移失败历史记录
|
||||||
self.post_message(
|
history = transferhis.add_fail(
|
||||||
Notification(
|
fileitem=task.fileitem,
|
||||||
mtype=NotificationType.Manual,
|
mode=transferinfo.transfer_type if transferinfo else "",
|
||||||
title=f"{task.mediainfo.title_year} {task.meta.season_episode} 入库失败!",
|
downloader=task.downloader,
|
||||||
text="\n".join(
|
download_hash=task.download_hash,
|
||||||
[
|
meta=task.meta,
|
||||||
f"原因:{transferinfo.message or '未知'}",
|
mediainfo=task.mediainfo,
|
||||||
(
|
transferinfo=transferinfo,
|
||||||
f"如果按钮不可用,可回复:\n```\n/redo {history.id}\n```"
|
)
|
||||||
if history
|
|
||||||
else ""
|
# 整理失败事件
|
||||||
),
|
if self.__is_media_file(task.fileitem):
|
||||||
]
|
# 主要媒体文件整理失败事件
|
||||||
).strip(),
|
self.eventmanager.send_event(
|
||||||
image=task.mediainfo.get_message_image(),
|
EventType.TransferFailed,
|
||||||
username=task.username,
|
{
|
||||||
link=settings.MP_DOMAIN("#/history"),
|
"fileitem": task.fileitem,
|
||||||
buttons=self.build_failed_transfer_buttons(
|
"meta": task.meta,
|
||||||
history.id if history else None
|
"mediainfo": task.mediainfo,
|
||||||
),
|
"transferinfo": transferinfo,
|
||||||
|
"downloader": task.downloader,
|
||||||
|
"download_hash": task.download_hash,
|
||||||
|
"transfer_history_id": history.id if history else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
elif self.__is_subtitle_file(task.fileitem):
|
||||||
|
# 字幕整理失败事件
|
||||||
|
self.eventmanager.send_event(
|
||||||
|
EventType.SubtitleTransferFailed,
|
||||||
|
{
|
||||||
|
"fileitem": task.fileitem,
|
||||||
|
"meta": task.meta,
|
||||||
|
"mediainfo": task.mediainfo,
|
||||||
|
"transferinfo": transferinfo,
|
||||||
|
"downloader": task.downloader,
|
||||||
|
"download_hash": task.download_hash,
|
||||||
|
"transfer_history_id": history.id if history else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
elif self.__is_audio_file(task.fileitem):
|
||||||
|
# 音频文件整理失败事件
|
||||||
|
self.eventmanager.send_event(
|
||||||
|
EventType.AudioTransferFailed,
|
||||||
|
{
|
||||||
|
"fileitem": task.fileitem,
|
||||||
|
"meta": task.meta,
|
||||||
|
"mediainfo": task.mediainfo,
|
||||||
|
"transferinfo": transferinfo,
|
||||||
|
"downloader": task.downloader,
|
||||||
|
"download_hash": task.download_hash,
|
||||||
|
"transfer_history_id": history.id if history else None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
# 发送失败消息
|
||||||
|
self.post_message(
|
||||||
|
Notification(
|
||||||
|
mtype=NotificationType.Manual,
|
||||||
|
title=f"{task.mediainfo.title_year} {task.meta.season_episode} 入库失败!",
|
||||||
|
text="\n".join(
|
||||||
|
[
|
||||||
|
f"原因:{transferinfo.message or '未知'}",
|
||||||
|
(
|
||||||
|
f"如果按钮不可用,可回复:\n```\n/redo {history.id}\n```"
|
||||||
|
if history
|
||||||
|
else ""
|
||||||
|
),
|
||||||
|
]
|
||||||
|
).strip(),
|
||||||
|
image=task.mediainfo.get_message_image(),
|
||||||
|
username=task.username,
|
||||||
|
link=settings.MP_DOMAIN("#/history"),
|
||||||
|
buttons=self.build_failed_transfer_buttons(
|
||||||
|
history.id if history else None
|
||||||
|
),
|
||||||
|
)
|
||||||
)
|
)
|
||||||
)
|
|
||||||
|
|
||||||
# 设置任务失败
|
# 设置任务失败
|
||||||
self.jobview.fail_task(task)
|
self.jobview.fail_task(task)
|
||||||
@@ -1506,6 +1578,12 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
# 转移成功
|
# 转移成功
|
||||||
logger.info(f"{task.fileitem.name} 入库成功:{target_dir_path or ''}")
|
logger.info(f"{task.fileitem.name} 入库成功:{target_dir_path or ''}")
|
||||||
|
|
||||||
|
# 整理成功即认为此前的连续失败已恢复,重置计数让后续故障重新获得完整重试额度
|
||||||
|
clear_transfer_failures(
|
||||||
|
task.fileitem.path if task.fileitem else None,
|
||||||
|
task.fileitem.storage if task.fileitem else None,
|
||||||
|
)
|
||||||
|
|
||||||
# 新增task转移成功历史记录
|
# 新增task转移成功历史记录
|
||||||
history = transferhis.add_success(
|
history = transferhis.add_success(
|
||||||
fileitem=task.fileitem,
|
fileitem=task.fileitem,
|
||||||
@@ -1670,8 +1748,126 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
self.__register_scrape_batch_task(task)
|
self.__register_scrape_batch_task(task)
|
||||||
# 添加到队列
|
# 添加到队列
|
||||||
self._queue.put(TransferQueue(task=task, callback=self.__default_callback))
|
self._queue.put(TransferQueue(task=task, callback=self.__default_callback))
|
||||||
|
# 落盘登记:队列是纯内存的,进程重启(挂载挂死后的人工重启、升级、OOM)
|
||||||
|
# 会让队列连同「这些文件还没整理」这个事实一起蒸发,而已稳定落地的文件
|
||||||
|
# 不会再产生任何监控事件,等于永久漏件。登记放在入队之后,宁可多留一条
|
||||||
|
# 由回放时的整理历史查重挡掉,也不制造「已入队但未登记」的窗口
|
||||||
|
self.__register_pending(task)
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
def replay_pending(self):
|
||||||
|
"""
|
||||||
|
回放上次进程退出时仍未整理完的文件。
|
||||||
|
|
||||||
|
在后台线程执行:回放要 stat 源文件,而启动期挂载可能尚未就绪甚至处于
|
||||||
|
挂死状态,同步执行会把整个启动流程堵住。
|
||||||
|
"""
|
||||||
|
threading.Thread(
|
||||||
|
target=self.__replay_pending,
|
||||||
|
name="MoviePilot-TransferReplay",
|
||||||
|
daemon=True
|
||||||
|
).start()
|
||||||
|
|
||||||
|
def __replay_pending(self):
|
||||||
|
"""
|
||||||
|
把落盘登记的待整理文件重新送回整理入口。
|
||||||
|
|
||||||
|
只回放「存储 + 源路径」这一最小事实,重新走完整的识别与整理流程,
|
||||||
|
已经整理完成的由整理历史查重挡掉,因此不存在重复整理的问题。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
pendings = self._pendingoper.list_all()
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"读取待整理文件登记失败:{err}")
|
||||||
|
return
|
||||||
|
if not pendings:
|
||||||
|
return
|
||||||
|
logger.info(f"发现 {len(pendings)} 个上次未整理完的文件,正在重新送入整理链 ...")
|
||||||
|
replayed = 0
|
||||||
|
for storage, src_path in pendings:
|
||||||
|
try:
|
||||||
|
fileitem, should_discard = self.__build_replay_fileitem(storage, src_path)
|
||||||
|
if not fileitem:
|
||||||
|
if should_discard:
|
||||||
|
# 源文件确认已消失,注销登记避免每次启动重复回放
|
||||||
|
self._pendingoper.discard(storage=storage, src_path=src_path)
|
||||||
|
continue
|
||||||
|
self.do_transfer(fileitem=fileitem)
|
||||||
|
replayed += 1
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"回放待整理文件失败:{storage}:{src_path} - {err}")
|
||||||
|
logger.info(f"✓ 待整理文件回放完成,{replayed} 个文件已重新送入整理链")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def __build_replay_fileitem(storage: str, src_path: str) -> Tuple[Optional[FileItem], bool]:
|
||||||
|
"""
|
||||||
|
为回放构造文件项。
|
||||||
|
|
||||||
|
必须用 stat 的异常类型区分「文件真的没了」和「挂载暂时读不到」,不能用
|
||||||
|
Path.exists():它在任何 OSError 下都返回 False,会把挂载抖动
|
||||||
|
(Transport endpoint is not connected)误判成文件消失,进而注销登记
|
||||||
|
——那等于在故障期间主动丢件,正是本表要防的事。
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 源文件路径,以 / 结尾表示蓝光原盘目录
|
||||||
|
:return: (文件项, 是否应注销登记)。文件项为 None 表示本次不回放;
|
||||||
|
只有确认源文件已经消失时才注销登记
|
||||||
|
"""
|
||||||
|
# 蓝光原盘目录在登记时保留了尾部斜杠,这里据此还原类型
|
||||||
|
is_dir = src_path.endswith("/")
|
||||||
|
path = Path(src_path)
|
||||||
|
size, modify_time = None, None
|
||||||
|
if storage == "local":
|
||||||
|
try:
|
||||||
|
file_stat = path.stat()
|
||||||
|
size, modify_time = file_stat.st_size, file_stat.st_mtime
|
||||||
|
except FileNotFoundError:
|
||||||
|
logger.info(f"待整理文件已不存在,注销登记:{src_path}")
|
||||||
|
return None, True
|
||||||
|
except OSError as err:
|
||||||
|
# 挂载未就绪或无响应属于暂时性故障,保留登记等下次启动再回放
|
||||||
|
logger.warn(f"读取待整理文件失败,保留登记等待下次回放:{src_path} - {err}")
|
||||||
|
return None, False
|
||||||
|
return FileItem(
|
||||||
|
storage=storage,
|
||||||
|
path=src_path if is_dir else path.as_posix(),
|
||||||
|
type="dir" if is_dir else "file",
|
||||||
|
name=path.name,
|
||||||
|
basename=path.stem,
|
||||||
|
extension=path.suffix[1:] if not is_dir else None,
|
||||||
|
size=size,
|
||||||
|
modify_time=modify_time,
|
||||||
|
), False
|
||||||
|
|
||||||
|
def __register_pending(self, task: TransferTask):
|
||||||
|
"""
|
||||||
|
落盘登记一个待整理文件,登记失败不影响正常入队。
|
||||||
|
:param task: 任务信息
|
||||||
|
"""
|
||||||
|
fileitem = task.fileitem if task else None
|
||||||
|
if not fileitem or not fileitem.path:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._pendingoper.register(storage=fileitem.storage, src_path=fileitem.path)
|
||||||
|
except Exception as err:
|
||||||
|
# 登记只是重启后的补救手段,失败不能阻断正常整理
|
||||||
|
logger.debug(f"登记待整理文件失败: {fileitem.path} - {err}")
|
||||||
|
|
||||||
|
def __discard_pending(self, task: TransferTask):
|
||||||
|
"""
|
||||||
|
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
|
||||||
|
|
||||||
|
失败的文件不靠本表回放:整理历史里已有失败记录,分发器的历史门控会按
|
||||||
|
重试预算重新送入整理链;留在本表反而会每次重启都重复回放。
|
||||||
|
:param task: 任务信息
|
||||||
|
"""
|
||||||
|
fileitem = task.fileitem if task else None
|
||||||
|
if not fileitem or not fileitem.path:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
self._pendingoper.discard(storage=fileitem.storage, src_path=fileitem.path)
|
||||||
|
except Exception as err:
|
||||||
|
logger.debug(f"注销待整理文件登记失败: {fileitem.path} - {err}")
|
||||||
|
|
||||||
def __put_to_jobview(self, task: TransferTask) -> bool:
|
def __put_to_jobview(self, task: TransferTask) -> bool:
|
||||||
"""
|
"""
|
||||||
添加到作业视图
|
添加到作业视图
|
||||||
@@ -1942,6 +2138,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
marker = getattr(self.jobview, "finish_execution", None)
|
marker = getattr(self.jobview, "finish_execution", None)
|
||||||
if marker:
|
if marker:
|
||||||
marker(task)
|
marker(task)
|
||||||
|
# 任务已到终态,落盘登记到此作废(未登记过的实时整理路径为无害空操作)
|
||||||
|
self.__discard_pending(task)
|
||||||
|
|
||||||
def __expire_stale_transfer_tasks(self):
|
def __expire_stale_transfer_tasks(self):
|
||||||
"""清理外部接管后失去状态心跳的运行中整理任务。"""
|
"""清理外部接管后失去状态心跳的运行中整理任务。"""
|
||||||
@@ -1987,8 +2185,11 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
fileitem = task.fileitem
|
fileitem = task.fileitem
|
||||||
|
|
||||||
with task_lock:
|
with task_lock:
|
||||||
# 获取当前最新总数
|
# 批次总数 = 本批已处理数 + 未终态数。作业视图会残留上一批
|
||||||
current_total = self.jobview.total()
|
# 已完成的任务(作业要等关联任务全部终态才移除),用全量
|
||||||
|
# total() 会把历史任务计入本批(如显示 8 个实际只处理 2 个),
|
||||||
|
# 且进度分母虚高导致百分比走不满
|
||||||
|
current_total = self._processed_num + self.jobview.pending_total()
|
||||||
# 更新总数,取当前总数和当前已处理+运行中+队列中的最大值
|
# 更新总数,取当前总数和当前已处理+运行中+队列中的最大值
|
||||||
self._total_num = max(self._total_num, current_total)
|
self._total_num = max(self._total_num, current_total)
|
||||||
|
|
||||||
@@ -2155,6 +2356,15 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
if not mediainfo:
|
if not mediainfo:
|
||||||
if task.preview:
|
if task.preview:
|
||||||
return False, "未识别到媒体信息"
|
return False, "未识别到媒体信息"
|
||||||
|
# 未识别同样是整理失败,计入重试次数:TMDB 瞬断属于可恢复故障,
|
||||||
|
# 但文件名永远识别不出时不能无限重试刷通知
|
||||||
|
record_transfer_failure(
|
||||||
|
task.fileitem.path if task.fileitem else None,
|
||||||
|
task.fileitem.storage if task.fileitem else None,
|
||||||
|
file_size=task.fileitem.size if task.fileitem else None,
|
||||||
|
file_modify_time=task.fileitem.modify_time if task.fileitem else None,
|
||||||
|
fileid=task.fileitem.fileid if task.fileitem else None,
|
||||||
|
)
|
||||||
# 新增整理失败历史记录
|
# 新增整理失败历史记录
|
||||||
his = transferhis.add_fail(
|
his = transferhis.add_fail(
|
||||||
fileitem=task.fileitem,
|
fileitem=task.fileitem,
|
||||||
@@ -3156,9 +3366,12 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
include_move_dest: bool = False,
|
include_move_dest: bool = False,
|
||||||
) -> Optional[TransferHistory]:
|
) -> Optional[TransferHistory]:
|
||||||
"""查询文件源路径历史,并兼容从成功移动后的目标现址重新整理。"""
|
"""查询文件源路径历史,并兼容从成功移动后的目标现址重新整理。"""
|
||||||
history = transfer_history_oper.get_by_src(
|
# resolve_history 在命中失败记录时会再确认一次有无成功记录,
|
||||||
|
# 避免 get_by_src 无排序导致同源多行时返回哪条不确定
|
||||||
|
history = resolve_history(
|
||||||
fileitem.path,
|
fileitem.path,
|
||||||
storage=fileitem.storage,
|
storage=fileitem.storage,
|
||||||
|
transfer_history_oper=transfer_history_oper,
|
||||||
)
|
)
|
||||||
if history or not include_move_dest:
|
if history or not include_move_dest:
|
||||||
return history
|
return history
|
||||||
@@ -3228,6 +3441,8 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
):
|
):
|
||||||
return False, f"{dest_fileitem.path} 删除失败"
|
return False, f"{dest_fileitem.path} 删除失败"
|
||||||
transfer_history_oper.delete(history.id)
|
transfer_history_oper.delete(history.id)
|
||||||
|
# 删除记录是用户显式要求重来,失败计数一并清零,否则重整仍会受上一轮次数限制
|
||||||
|
clear_transfer_failures(history.src, history.src_storage)
|
||||||
return True, ""
|
return True, ""
|
||||||
|
|
||||||
def do_transfer(
|
def do_transfer(
|
||||||
@@ -3786,7 +4001,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
raise OperationInterrupted()
|
raise OperationInterrupted()
|
||||||
file_path = Path(file_item.path)
|
file_path = Path(file_item.path)
|
||||||
|
|
||||||
# 自动整理继续按全部历史去重;手动整理可清理失败记录,或按用户确认清理成功记录。
|
# 自动整理按 app/helper/transferhistory.py 的统一判定去重(失败记录放行重试、
|
||||||
|
# 成功但源文件已变化放行交 overwrite_mode 决断);手动整理可清理失败记录,
|
||||||
|
# 或按用户确认清理成功记录。
|
||||||
if (not force or reorganize) and not preview:
|
if (not force or reorganize) and not preview:
|
||||||
transfer_history_oper = TransferHistoryOper()
|
transfer_history_oper = TransferHistoryOper()
|
||||||
transferd = self._get_manual_transfer_history(
|
transferd = self._get_manual_transfer_history(
|
||||||
@@ -3813,10 +4030,37 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
)
|
)
|
||||||
transferd = None
|
transferd = None
|
||||||
|
|
||||||
|
if transferd:
|
||||||
|
history_description = describe_history_gate(
|
||||||
|
transferd,
|
||||||
|
file_size=file_item.size,
|
||||||
|
file_modify_time=file_item.modify_time,
|
||||||
|
fileid=file_item.fileid,
|
||||||
|
)
|
||||||
|
|
||||||
|
if transferd and not manual:
|
||||||
|
# 自动路径(目录监控、下载器轮询)与监控分发共用同一套判定,
|
||||||
|
# 否则监控层刚放行的失败重试与升级请求会在这里被全额收回
|
||||||
|
gate_action = evaluate_history_gate(
|
||||||
|
transferd,
|
||||||
|
file_size=file_item.size,
|
||||||
|
file_modify_time=file_item.modify_time,
|
||||||
|
fileid=file_item.fileid,
|
||||||
|
)
|
||||||
|
if not is_skip_action(gate_action):
|
||||||
|
logger.info(
|
||||||
|
f"{file_item.path} 命中"
|
||||||
|
f"{history_description}"
|
||||||
|
f",重新送入整理"
|
||||||
|
)
|
||||||
|
transferd = None
|
||||||
|
|
||||||
if transferd:
|
if transferd:
|
||||||
skipped_history_count += 1
|
skipped_history_count += 1
|
||||||
if not transferd.status:
|
if not transferd.status:
|
||||||
all_success = False
|
all_success = False
|
||||||
|
# 失败记录能走到这里说明重试次数已用尽,此时同样要打已整理标签让种子
|
||||||
|
# 退出轮询,否则下载器每一轮都会重新扫描并在这里被拦一次,空转且刷屏
|
||||||
candidate_hash = download_hash or transferd.download_hash
|
candidate_hash = download_hash or transferd.download_hash
|
||||||
candidate_downloader = downloader or transferd.downloader
|
candidate_downloader = downloader or transferd.downloader
|
||||||
if candidate_hash and candidate_downloader:
|
if candidate_hash and candidate_downloader:
|
||||||
@@ -3824,7 +4068,9 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
|
|||||||
(candidate_hash, candidate_downloader)
|
(candidate_hash, candidate_downloader)
|
||||||
)
|
)
|
||||||
logger.info(
|
logger.info(
|
||||||
f"{file_item.path} 已整理过,如需重新处理,请删除整理记录。"
|
f"{file_item.path} 已整理过("
|
||||||
|
f"{history_description}"
|
||||||
|
f"),如需重新处理,请删除整理记录。"
|
||||||
)
|
)
|
||||||
err_msgs.append(f"{file_item.name} 已整理过")
|
err_msgs.append(f"{file_item.name} 已整理过")
|
||||||
continue
|
continue
|
||||||
|
|||||||
+8
-2
@@ -7,7 +7,7 @@ from abc import ABC, abstractmethod
|
|||||||
from contextlib import contextmanager, asynccontextmanager
|
from contextlib import contextmanager, asynccontextmanager
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, Optional, Generator, AsyncGenerator, Tuple, Literal, Union
|
from typing import Any, Callable, Dict, Optional, Generator, AsyncGenerator, Tuple, Literal, Union
|
||||||
|
|
||||||
import aiofiles
|
import aiofiles
|
||||||
import aioshutil
|
import aioshutil
|
||||||
@@ -1123,7 +1123,8 @@ def AsyncCache(cache_type: Literal['ttl', 'lru'] = 'ttl',
|
|||||||
|
|
||||||
|
|
||||||
def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Optional[int] = None,
|
def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Optional[int] = None,
|
||||||
skip_none: Optional[bool] = True, skip_empty: Optional[bool] = False, shared_key: Optional[str] = None):
|
skip_none: Optional[bool] = True, skip_empty: Optional[bool] = False, shared_key: Optional[str] = None,
|
||||||
|
skip_if: Optional[Callable[[Any], bool]] = None):
|
||||||
"""
|
"""
|
||||||
自定义缓存装饰器,支持配置缓存区域的 maxsize 和每个 key 的 ttl
|
自定义缓存装饰器,支持配置缓存区域的 maxsize 和每个 key 的 ttl
|
||||||
|
|
||||||
@@ -1133,6 +1134,9 @@ def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Opt
|
|||||||
:param skip_none: 跳过 None 缓存,默认为 True
|
:param skip_none: 跳过 None 缓存,默认为 True
|
||||||
:param skip_empty: 跳过空值缓存(如 None, [], {}, "", set()),默认为 False
|
:param skip_empty: 跳过空值缓存(如 None, [], {}, "", set()),默认为 False
|
||||||
:param shared_key: 同步/异步函数共享缓存的键,默认使用函数名(异步函数名会标准化为同步格式,如移除 `async_` 前缀)
|
:param shared_key: 同步/异步函数共享缓存的键,默认使用函数名(异步函数名会标准化为同步格式,如移除 `async_` 前缀)
|
||||||
|
:param skip_if: 按返回值判断是否跳过缓存的谓词,返回真值时不缓存;用于
|
||||||
|
「结构合法但业务失败」的返回值(如 TMDB 的 success=false 响应),
|
||||||
|
这类值无法用 skip_none/skip_empty 表达
|
||||||
:return: 装饰器函数
|
:return: 装饰器函数
|
||||||
"""
|
"""
|
||||||
|
|
||||||
@@ -1158,6 +1162,8 @@ def cached(region: Optional[str] = None, maxsize: Optional[int] = 1024, ttl: Opt
|
|||||||
# if skip_empty and value in [None, [], {}, "", set()]:
|
# if skip_empty and value in [None, [], {}, "", set()]:
|
||||||
if skip_empty and not value:
|
if skip_empty and not value:
|
||||||
return False
|
return False
|
||||||
|
if skip_if and skip_if(value):
|
||||||
|
return False
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def is_valid_cache_value(_cache_key: str, _cached_value: Any, _cache_region: str) -> bool:
|
def is_valid_cache_value(_cache_key: str, _cached_value: Any, _cache_region: str) -> bool:
|
||||||
|
|||||||
@@ -386,6 +386,19 @@ class ConfigModel(BaseModel):
|
|||||||
# 下载器临时文件后缀
|
# 下载器临时文件后缀
|
||||||
DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: [".!qb", ".part"])
|
DOWNLOAD_TMPEXT: list = Field(default_factory=lambda: [".!qb", ".part"])
|
||||||
|
|
||||||
|
# ==================== 目录监控配置 ====================
|
||||||
|
# 允许网络文件系统使用快速模式(inotify)。部分 FUSE 实现(如 CloudDrive2)
|
||||||
|
# 会正常下发内核通知,快速模式可用,且比每 N 秒 stat 全部目录的轮询对挂载后端
|
||||||
|
# 的压力小得多,由用户确认后开启
|
||||||
|
MONITOR_NETWORK_FAST_MODE: bool = False
|
||||||
|
# 网络文件系统的轮询扫描间隔(毫秒),0 表示使用内置默认值
|
||||||
|
MONITOR_POLL_DELAY_NETWORK: int = 0
|
||||||
|
# 新增目录延迟重扫的轮次延迟秒数,多个使用,分隔,如 "30,120,600,1800"。
|
||||||
|
# FUSE 挂载(如 CloudDrive2)上超大目录树呈现完整内容可能超过分钟级,
|
||||||
|
# 默认值在常见的 30/120 秒窗口后追加两轮成本极低的长延迟轮次兜底;
|
||||||
|
# 解析失败(如格式非法、包含非正整数)时回退默认值并记录 warn 日志
|
||||||
|
MONITOR_RESCAN_DELAYS: str = "30,120,600,1800"
|
||||||
|
|
||||||
# ==================== CookieCloud配置 ====================
|
# ==================== CookieCloud配置 ====================
|
||||||
# CookieCloud是否启动本地服务
|
# CookieCloud是否启动本地服务
|
||||||
COOKIECLOUD_ENABLE_LOCAL: Optional[bool] = False
|
COOKIECLOUD_ENABLE_LOCAL: Optional[bool] = False
|
||||||
@@ -405,6 +418,27 @@ class ConfigModel(BaseModel):
|
|||||||
# ==================== 整理配置 ====================
|
# ==================== 整理配置 ====================
|
||||||
# 文件整理线程数
|
# 文件整理线程数
|
||||||
TRANSFER_THREADS: int = 1
|
TRANSFER_THREADS: int = 1
|
||||||
|
# 本地文件操作是否走可强杀的子进程代理。
|
||||||
|
# FUSE/网络挂载进入「请求永不返回」状态时,stat/listdir 这类调用会永久悬挂,
|
||||||
|
# 而 Python 既不能中断已发出的系统调用、也不能强杀线程,阻塞其上的线程无法回收。
|
||||||
|
# 走子进程后超时可以 SIGKILL 真正回收,block 型故障被转换成各层已能处理的
|
||||||
|
# crash 型故障(OSError)。出现兼容问题时可关闭回到直接调用。
|
||||||
|
FS_PROXY_ENABLED: bool = True
|
||||||
|
# 单次本地快操作(stat/listdir/删除/重命名)的超时秒数,
|
||||||
|
# 超时即判定挂载无响应并回收代理进程
|
||||||
|
FS_PROXY_TIMEOUT: int = 30
|
||||||
|
# 复制大文件时两次进度上报之间的最长间隔秒数。代理每秒上报一次进度作为心跳,
|
||||||
|
# 因此这个阈值判定的是「传输完全没有推进」而不是「传输很慢」,
|
||||||
|
# 复制几小时的大文件也不会被误杀
|
||||||
|
FS_PROXY_STALL_TIMEOUT: int = 120
|
||||||
|
# 自动整理(目录监控、下载器轮询)遇到失败整理记录时,同一源路径允许自动重试的最大次数。
|
||||||
|
# 一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)不该让文件永久漏整理,因此必须重试;
|
||||||
|
# 但永远识别不出的文件重试再多也不会成功,只会重复推送失败通知,批量导入时更会刷屏,
|
||||||
|
# 因此必须有界。取值区间 1-10,越界或非法值会被钳制到边界并记录 warn:
|
||||||
|
# 既不支持关闭重试,也不支持无限重试。整理成功或删除整理记录时计数清零。
|
||||||
|
# 与本项无关的是同路径新版本:已成功整理的文件源大小变化时一律放行,
|
||||||
|
# 由整理链的 overwrite_mode 决断是否覆盖
|
||||||
|
TRANSFER_MAX_FAILED_RETRIES: int = 3
|
||||||
# 外部接管的运行中整理任务无状态心跳超时(分钟),0 表示禁用
|
# 外部接管的运行中整理任务无状态心跳超时(分钟),0 表示禁用
|
||||||
TRANSFER_TASK_TIMEOUT: int = 120
|
TRANSFER_TASK_TIMEOUT: int = 120
|
||||||
# 电影重命名格式
|
# 电影重命名格式
|
||||||
|
|||||||
@@ -14,6 +14,7 @@ from .subscribe import Subscribe
|
|||||||
from .subscribehistory import SubscribeHistory
|
from .subscribehistory import SubscribeHistory
|
||||||
from .systemconfig import SystemConfig
|
from .systemconfig import SystemConfig
|
||||||
from .transferhistory import TransferHistory
|
from .transferhistory import TransferHistory
|
||||||
|
from .transferpending import TransferPending
|
||||||
from .user import User
|
from .user import User
|
||||||
from .userconfig import UserConfig
|
from .userconfig import UserConfig
|
||||||
from .workflow import Workflow
|
from .workflow import Workflow
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ class TransferHistory(Base):
|
|||||||
# 源路径
|
# 源路径
|
||||||
src = Column(String, index=True)
|
src = Column(String, index=True)
|
||||||
# 源存储
|
# 源存储
|
||||||
src_storage = Column(String)
|
src_storage = Column(String, nullable=False, default="local")
|
||||||
# 源文件项
|
# 源文件项
|
||||||
src_fileitem = Column(JSON, default=dict)
|
src_fileitem = Column(JSON, default=dict)
|
||||||
# 目标路径
|
# 目标路径
|
||||||
@@ -89,6 +89,7 @@ class TransferHistory(Base):
|
|||||||
Index('ix_transferhistory_status_date', 'status', 'date'),
|
Index('ix_transferhistory_status_date', 'status', 'date'),
|
||||||
Index('ix_transferhistory_date_id', 'date', 'id'),
|
Index('ix_transferhistory_date_id', 'date', 'id'),
|
||||||
Index('ix_transferhistory_media_identity', 'media_source', 'media_id'),
|
Index('ix_transferhistory_media_identity', 'media_source', 'media_id'),
|
||||||
|
Index('ux_transferhistory_src_storage', 'src', 'src_storage', unique=True),
|
||||||
)
|
)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@@ -207,10 +208,30 @@ class TransferHistory(Base):
|
|||||||
:return: 命中的整理记录,未命中时返回 None
|
:return: 命中的整理记录,未命中时返回 None
|
||||||
"""
|
"""
|
||||||
if storage:
|
if storage:
|
||||||
return db.query(cls).filter(cls.src == src,
|
query = db.query(cls).filter(cls.src == src, cls.src_storage == storage)
|
||||||
cls.src_storage == storage).first()
|
|
||||||
else:
|
else:
|
||||||
return db.query(cls).filter(cls.src == src).first()
|
query = db.query(cls).filter(cls.src == src)
|
||||||
|
return query.order_by(cls.id.desc()).first()
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_query
|
||||||
|
def get_success_by_src(
|
||||||
|
cls, db: Session, src: str, storage: Optional[str] = None
|
||||||
|
) -> Optional["TransferHistory"]:
|
||||||
|
"""
|
||||||
|
按源路径和存储查询成功的整理记录,源路径原样精确匹配。
|
||||||
|
|
||||||
|
与 list_success_by_src 不同,这里不对源路径做归一化,蓝光原盘目录记录
|
||||||
|
带尾斜杠,归一化后反而匹配不到。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:param src: 源路径
|
||||||
|
:param storage: 源存储类型
|
||||||
|
:return: 命中的成功整理记录,未命中时返回 None
|
||||||
|
"""
|
||||||
|
query = db.query(cls).filter(cls.src == src, cls.status.is_(True))
|
||||||
|
if storage:
|
||||||
|
query = query.filter(cls.src_storage == storage)
|
||||||
|
return query.order_by(cls.id.desc()).first()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@db_query
|
@db_query
|
||||||
@@ -228,7 +249,7 @@ class TransferHistory(Base):
|
|||||||
query = db.query(cls).filter(cls.dest == dest)
|
query = db.query(cls).filter(cls.dest == dest)
|
||||||
if storage:
|
if storage:
|
||||||
query = query.filter(cls.dest_storage == storage)
|
query = query.filter(cls.dest_storage == storage)
|
||||||
return query.first()
|
return query.order_by(cls.id.desc()).first()
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@db_query
|
@db_query
|
||||||
@@ -565,6 +586,31 @@ class TransferHistory(Base):
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_update
|
||||||
|
def replace_by_src(cls, db: Session, **kwargs) -> "TransferHistory":
|
||||||
|
"""
|
||||||
|
用同源存储的新记录原子替换旧整理历史。
|
||||||
|
|
||||||
|
同一源路径在一个存储中只能对应一条最新整理记录。先在同一事务内清理旧行再
|
||||||
|
插入,避免旧的“查询一条再删除一条”在遗留重复数据下留下脏记录。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:param kwargs: 整理历史字段
|
||||||
|
:return: 新创建的整理历史
|
||||||
|
"""
|
||||||
|
src = kwargs.get("src")
|
||||||
|
src_storage = kwargs.get("src_storage") or "local"
|
||||||
|
kwargs["src_storage"] = src_storage
|
||||||
|
if src:
|
||||||
|
db.query(cls).filter(
|
||||||
|
cls.src == src,
|
||||||
|
cls.src_storage == src_storage,
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
history = cls(**kwargs)
|
||||||
|
db.add(history)
|
||||||
|
db.flush()
|
||||||
|
return history
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
@db_query
|
@db_query
|
||||||
def list_by_date(cls, db: Session, date: str):
|
def list_by_date(cls, db: Session, date: str):
|
||||||
|
|||||||
@@ -0,0 +1,102 @@
|
|||||||
|
from typing import List, Optional
|
||||||
|
|
||||||
|
from sqlalchemy import Column, Index, String
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.db import Base, db_query, db_update, get_id_column
|
||||||
|
|
||||||
|
|
||||||
|
class TransferPending(Base):
|
||||||
|
"""
|
||||||
|
待整理文件登记。
|
||||||
|
|
||||||
|
整理队列是纯内存的 queue.Queue:进程一旦重启(挂载挂死后的人工重启、版本
|
||||||
|
升级、OOM、宿主重启),队列里的任务会连同「这些文件还没整理」这个事实一起
|
||||||
|
蒸发。而已经稳定落地的文件不会再产生任何监控事件,也不会有新的补偿扫描起点
|
||||||
|
——结果就是永久漏件,只能靠人工比对补整理。
|
||||||
|
|
||||||
|
这里只落盘最小事实:存储与源文件路径。重启后重新走一遍整理入口,由整理历史
|
||||||
|
查重挡掉已经完成的,因此不需要序列化 meta/mediainfo 这些重对象,也不存在
|
||||||
|
识别结果陈旧的问题。
|
||||||
|
"""
|
||||||
|
|
||||||
|
id = get_id_column()
|
||||||
|
# 存储
|
||||||
|
storage = Column(String, nullable=False)
|
||||||
|
# 源文件路径
|
||||||
|
src_path = Column(String, nullable=False)
|
||||||
|
# 登记时间
|
||||||
|
created_at = Column(String)
|
||||||
|
|
||||||
|
__table_args__ = (
|
||||||
|
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
|
||||||
|
Index("ux_transferpending_storage_path", "storage", "src_path", unique=True),
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_update
|
||||||
|
def register(cls, db: Session, storage: str, src_path: str,
|
||||||
|
now_time: str) -> Optional["TransferPending"]:
|
||||||
|
"""
|
||||||
|
登记一个待整理文件,已存在时保持原登记时间不变。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 源文件路径
|
||||||
|
:param now_time: 当前时间
|
||||||
|
:return: 登记记录
|
||||||
|
"""
|
||||||
|
if not storage or not src_path:
|
||||||
|
return None
|
||||||
|
pending = db.query(cls).filter(
|
||||||
|
cls.storage == storage, cls.src_path == src_path
|
||||||
|
).first()
|
||||||
|
if pending:
|
||||||
|
return pending
|
||||||
|
pending = cls(storage=storage, src_path=src_path, created_at=now_time)
|
||||||
|
db.add(pending)
|
||||||
|
return pending
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_update
|
||||||
|
def discard(cls, db: Session, storage: str, src_path: str) -> int:
|
||||||
|
"""
|
||||||
|
注销一个待整理文件登记,整理到达终态(成功或失败)时调用。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 源文件路径
|
||||||
|
:return: 删除的记录数
|
||||||
|
"""
|
||||||
|
if not storage or not src_path:
|
||||||
|
return 0
|
||||||
|
return db.query(cls).filter(
|
||||||
|
cls.storage == storage, cls.src_path == src_path
|
||||||
|
).delete(synchronize_session=False)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_query
|
||||||
|
def list_all(cls, db: Session, limit: Optional[int] = 5000) -> List["TransferPending"]:
|
||||||
|
"""
|
||||||
|
列出全部待整理登记,供启动回放使用。
|
||||||
|
|
||||||
|
按登记时间升序回放,保持与原入队顺序一致;上限避免异常积压时
|
||||||
|
一次性把整理链压垮。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:param limit: 单次回放上限
|
||||||
|
:return: 待整理登记列表
|
||||||
|
"""
|
||||||
|
return (
|
||||||
|
db.query(cls)
|
||||||
|
.order_by(cls.created_at.asc(), cls.id.asc())
|
||||||
|
.limit(limit)
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
@db_update
|
||||||
|
def clear(cls, db: Session) -> int:
|
||||||
|
"""
|
||||||
|
清空全部待整理登记。
|
||||||
|
:param db: 数据库会话
|
||||||
|
:return: 删除的记录数
|
||||||
|
"""
|
||||||
|
return db.query(cls).delete(synchronize_session=False)
|
||||||
@@ -91,6 +91,17 @@ class TransferHistoryOper(DbOper):
|
|||||||
"""
|
"""
|
||||||
return TransferHistory.get_by_src(self._db, src, storage)
|
return TransferHistory.get_by_src(self._db, src, storage)
|
||||||
|
|
||||||
|
def get_success_by_src(
|
||||||
|
self, src: str, storage: Optional[str] = None
|
||||||
|
) -> Optional[TransferHistory]:
|
||||||
|
"""
|
||||||
|
按源查询成功的转移记录,源路径原样精确匹配
|
||||||
|
:param src: 数据key
|
||||||
|
:param storage: 存储类型
|
||||||
|
:return: 命中的成功整理记录,未命中时返回 None
|
||||||
|
"""
|
||||||
|
return TransferHistory.get_success_by_src(self._db, src, storage)
|
||||||
|
|
||||||
def get_by_dest(
|
def get_by_dest(
|
||||||
self, dest: str, storage: Optional[str] = None
|
self, dest: str, storage: Optional[str] = None
|
||||||
) -> Optional[TransferHistory]:
|
) -> Optional[TransferHistory]:
|
||||||
@@ -215,18 +226,25 @@ class TransferHistoryOper(DbOper):
|
|||||||
|
|
||||||
def add_force(self, **kwargs) -> TransferHistory:
|
def add_force(self, **kwargs) -> TransferHistory:
|
||||||
"""
|
"""
|
||||||
新增转移历史,相同源目录的记录会被删除
|
新增转移历史,并以同源存储的记录为准替换旧记录。
|
||||||
"""
|
"""
|
||||||
kwargs = normalize_media_identity_payload(kwargs)
|
kwargs = normalize_media_identity_payload(kwargs)
|
||||||
if kwargs.get("src"):
|
# 文件项的默认存储是 local;归一化旧调用传入的 None,确保运行时语义与
|
||||||
transferhistory = TransferHistory.get_by_src(self._db, kwargs.get("src"))
|
# (src, src_storage) 唯一索引一致。
|
||||||
if transferhistory:
|
kwargs["src_storage"] = kwargs.get("src_storage") or "local"
|
||||||
transferhistory.delete(self._db, transferhistory.id)
|
# 旧记录的清理交给 replace_by_src 按 (src, src_storage) 处理:
|
||||||
|
# 仅按 src 删除会连带删掉其他存储下同路径的记录。
|
||||||
kwargs.update({
|
kwargs.update({
|
||||||
"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
"date": time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
|
||||||
})
|
})
|
||||||
TransferHistory(**kwargs).create(self._db)
|
TransferHistory.replace_by_src(self._db, **kwargs)
|
||||||
return TransferHistory.get_by_src(self._db, kwargs.get("src"))
|
# 保持 add_force 的既有返回契约:返回可被调用方安全读取字段的查询结果,
|
||||||
|
# 而非事务提交后可能已脱离会话的新建实例。
|
||||||
|
return TransferHistory.get_by_src(
|
||||||
|
self._db,
|
||||||
|
kwargs.get("src"),
|
||||||
|
kwargs["src_storage"],
|
||||||
|
)
|
||||||
|
|
||||||
def update_download_hash(self, historyid, download_hash):
|
def update_download_hash(self, historyid, download_hash):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -0,0 +1,59 @@
|
|||||||
|
from datetime import datetime
|
||||||
|
from typing import List, Optional, Tuple
|
||||||
|
|
||||||
|
from app.db import DbOper
|
||||||
|
from app.db.models.transferpending import TransferPending
|
||||||
|
|
||||||
|
|
||||||
|
class TransferPendingOper(DbOper):
|
||||||
|
"""
|
||||||
|
待整理文件登记管理。
|
||||||
|
|
||||||
|
只保存「存储 + 源文件路径」这一最小事实,用于在进程重启后把没走完整理链的
|
||||||
|
文件重新送回去,避免挂载故障重启后永久漏件。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def register(self, storage: str, src_path: str) -> Optional[TransferPending]:
|
||||||
|
"""
|
||||||
|
登记一个待整理文件。
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 源文件路径
|
||||||
|
:return: 登记记录
|
||||||
|
"""
|
||||||
|
return TransferPending.register(
|
||||||
|
self._db,
|
||||||
|
storage=storage,
|
||||||
|
src_path=src_path,
|
||||||
|
now_time=datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||||||
|
)
|
||||||
|
|
||||||
|
def discard(self, storage: str, src_path: str) -> int:
|
||||||
|
"""
|
||||||
|
注销一个待整理文件登记。
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 源文件路径
|
||||||
|
:return: 删除的记录数
|
||||||
|
"""
|
||||||
|
return TransferPending.discard(self._db, storage=storage, src_path=src_path)
|
||||||
|
|
||||||
|
def list_all(self, limit: Optional[int] = 5000) -> List[Tuple[str, str]]:
|
||||||
|
"""
|
||||||
|
列出全部待整理登记,供启动回放使用。
|
||||||
|
|
||||||
|
返回纯元组而不是 ORM 实例:回放发生在会话之外,ORM 实例脱离 session
|
||||||
|
后访问属性会触发 DetachedInstanceError。
|
||||||
|
:param limit: 单次回放上限
|
||||||
|
:return: (存储, 源文件路径) 列表
|
||||||
|
"""
|
||||||
|
return [
|
||||||
|
(item.storage, item.src_path)
|
||||||
|
for item in TransferPending.list_all(self._db, limit=limit) or []
|
||||||
|
if item and item.storage and item.src_path
|
||||||
|
]
|
||||||
|
|
||||||
|
def clear(self) -> int:
|
||||||
|
"""
|
||||||
|
清空全部待整理登记。
|
||||||
|
:return: 删除的记录数
|
||||||
|
"""
|
||||||
|
return TransferPending.clear(self._db)
|
||||||
@@ -0,0 +1,422 @@
|
|||||||
|
from typing import Any, Dict, Optional
|
||||||
|
|
||||||
|
from app.core.cache import TTLCache
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.db.models.transferhistory import TransferHistory
|
||||||
|
from app.db.transferhistory_oper import TransferHistoryOper
|
||||||
|
from app.log import logger
|
||||||
|
|
||||||
|
# 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)
|
||||||
|
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多
|
||||||
|
# 也不会成功,只会重复推送失败通知,批量导入场景下会刷屏,所以不允许无限重试
|
||||||
|
MIN_FAILED_RETRIES = 1
|
||||||
|
MAX_FAILED_RETRIES = 10
|
||||||
|
|
||||||
|
# 同一源路径的连续整理失败状态。整理链在写失败历史时累计、整理成功或删除历史时清零,
|
||||||
|
# 查重闸只读不写,避免监控层与整理链对同一个事件重复计数。缓存值会同时保存文件指纹,
|
||||||
|
# 因此同一路径的新版本天然获得独立预算;内存缓存会随进程重启清空,Redis 后端则保留到 TTL 到期。
|
||||||
|
FAILED_RETRY_TTL = 24 * 3600
|
||||||
|
_failed_retry_counts = TTLCache(region="transfer_failed_retry", maxsize=5000, ttl=FAILED_RETRY_TTL)
|
||||||
|
|
||||||
|
|
||||||
|
class HistoryGateAction:
|
||||||
|
"""
|
||||||
|
整理历史查重闸的判定结果。
|
||||||
|
|
||||||
|
监控分发(app/monitor/dispatcher.py)与整理链计划整理段(app/chain/transfer.py)
|
||||||
|
共用本模块,避免两处各写一套去重策略后互相对冲:上游放行的文件被下游按
|
||||||
|
「存在记录即拦」全额收回,等于放行逻辑完全失效。
|
||||||
|
"""
|
||||||
|
# 没有整理记录
|
||||||
|
PASS_NO_RECORD = "pass_no_record"
|
||||||
|
# 上次整理失败且重试次数未用尽,放行重试
|
||||||
|
PASS_FAILED = "pass_failed"
|
||||||
|
# 上次整理失败但源文件已变为新版本,放行并重置该版本的重试预算
|
||||||
|
PASS_FAILED_VERSION_CHANGED = "pass_failed_version_changed"
|
||||||
|
# 已整理成功但源文件已变化,放行交由 overwrite_mode 决断
|
||||||
|
PASS_SIZE_CHANGED = "pass_size_changed"
|
||||||
|
# 上次整理失败且重试次数已用尽,跳过
|
||||||
|
SKIP_RETRY_EXHAUSTED = "skip_retry_exhausted"
|
||||||
|
# 已整理成功且源文件未变化,跳过
|
||||||
|
SKIP = "skip"
|
||||||
|
|
||||||
|
|
||||||
|
def is_skip_action(action: str) -> bool:
|
||||||
|
"""
|
||||||
|
判断查重闸判定是否为跳过整理。
|
||||||
|
:param action: HistoryGateAction 之一
|
||||||
|
:return: True 表示跳过
|
||||||
|
"""
|
||||||
|
return action in (HistoryGateAction.SKIP, HistoryGateAction.SKIP_RETRY_EXHAUSTED)
|
||||||
|
|
||||||
|
|
||||||
|
def max_failed_retries() -> int:
|
||||||
|
"""
|
||||||
|
读取失败重试上限并钳制到合法区间。
|
||||||
|
|
||||||
|
配置为负数、0 或超过上界时都会被钳制并记录 warn:关闭重试会让瞬时故障造成
|
||||||
|
永久漏件,无限重试会让永久失败的文件反复刷通知,两端都不接受。
|
||||||
|
:return: 合法的最大重试次数
|
||||||
|
"""
|
||||||
|
raw = settings.TRANSFER_MAX_FAILED_RETRIES
|
||||||
|
try:
|
||||||
|
value = int(raw)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 配置非法({raw!r}),"
|
||||||
|
f"已回退为 {MIN_FAILED_RETRIES}")
|
||||||
|
return MIN_FAILED_RETRIES
|
||||||
|
if value < MIN_FAILED_RETRIES:
|
||||||
|
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能小于 {MIN_FAILED_RETRIES}"
|
||||||
|
f"(当前 {value}),已按 {MIN_FAILED_RETRIES} 处理")
|
||||||
|
return MIN_FAILED_RETRIES
|
||||||
|
if value > MAX_FAILED_RETRIES:
|
||||||
|
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能大于 {MAX_FAILED_RETRIES}"
|
||||||
|
f"(当前 {value}),已按 {MAX_FAILED_RETRIES} 处理")
|
||||||
|
return MAX_FAILED_RETRIES
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def failed_retry_key(src_path: Optional[str], storage: Optional[str] = None) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
生成失败重试计数的缓存键。
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param storage: 源存储
|
||||||
|
:return: 缓存键,源路径为空时返回 None
|
||||||
|
"""
|
||||||
|
if not src_path:
|
||||||
|
return None
|
||||||
|
return f"{storage or 'local'}:{src_path}"
|
||||||
|
|
||||||
|
|
||||||
|
def coerce_modify_time(modify_time: Any) -> Optional[float]:
|
||||||
|
"""
|
||||||
|
统一转换文件修改时间,无法转换时返回 None。
|
||||||
|
:param modify_time: 原始修改时间值
|
||||||
|
:return: 文件修改时间
|
||||||
|
"""
|
||||||
|
if modify_time is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return float(modify_time)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def coerce_fileid(fileid: Any) -> Optional[str]:
|
||||||
|
"""
|
||||||
|
统一转换文件唯一标识,空值视为不可比对。
|
||||||
|
:param fileid: 原始文件唯一标识
|
||||||
|
:return: 非空文件唯一标识
|
||||||
|
"""
|
||||||
|
if fileid is None:
|
||||||
|
return None
|
||||||
|
value = str(fileid).strip()
|
||||||
|
return value or None
|
||||||
|
|
||||||
|
|
||||||
|
def file_fingerprint(
|
||||||
|
file_size: Any = None,
|
||||||
|
file_modify_time: Any = None,
|
||||||
|
fileid: Any = None,
|
||||||
|
) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
生成用于区分同一路径文件版本的稳定指纹。
|
||||||
|
|
||||||
|
大小是所有存储器都尽量提供的最小指纹;两端均有数据时,修改时间和文件 ID 还可
|
||||||
|
识别“同大小替换”。只保留可比较字段,避免缺失元数据把同一文件误判成新版本。
|
||||||
|
:param file_size: 文件大小
|
||||||
|
:param file_modify_time: 文件修改时间
|
||||||
|
:param fileid: 存储器文件唯一标识
|
||||||
|
:return: 非空且可比较的指纹字段
|
||||||
|
"""
|
||||||
|
fingerprint = {}
|
||||||
|
size = coerce_size(file_size)
|
||||||
|
if size is not None:
|
||||||
|
fingerprint["size"] = size
|
||||||
|
modify_time = coerce_modify_time(file_modify_time)
|
||||||
|
if modify_time is not None:
|
||||||
|
fingerprint["modify_time"] = modify_time
|
||||||
|
normalized_fileid = coerce_fileid(fileid)
|
||||||
|
if normalized_fileid is not None:
|
||||||
|
fingerprint["fileid"] = normalized_fileid
|
||||||
|
return fingerprint
|
||||||
|
|
||||||
|
|
||||||
|
def _retry_state(value: Any) -> tuple[int, Dict[str, Any]]:
|
||||||
|
"""将新旧缓存值统一转换为失败次数与文件指纹。"""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
raw_count = value.get("count", 0)
|
||||||
|
raw_fingerprint = value.get("fingerprint")
|
||||||
|
else:
|
||||||
|
# 兼容已写入 Redis 或内存的旧整数计数;下次带指纹写入时会自动升级结构。
|
||||||
|
raw_count = value
|
||||||
|
raw_fingerprint = None
|
||||||
|
try:
|
||||||
|
count = max(int(raw_count or 0), 0)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
count = 0
|
||||||
|
fingerprint = (
|
||||||
|
file_fingerprint(
|
||||||
|
file_size=raw_fingerprint.get("size"),
|
||||||
|
file_modify_time=raw_fingerprint.get("modify_time"),
|
||||||
|
fileid=raw_fingerprint.get("fileid"),
|
||||||
|
)
|
||||||
|
if isinstance(raw_fingerprint, dict)
|
||||||
|
else {}
|
||||||
|
)
|
||||||
|
return count, fingerprint
|
||||||
|
|
||||||
|
|
||||||
|
def _is_file_version_changed(
|
||||||
|
recorded_fingerprint: Dict[str, Any],
|
||||||
|
current_fingerprint: Dict[str, Any],
|
||||||
|
) -> bool:
|
||||||
|
"""判断两个可比文件指纹是否指向不同版本。"""
|
||||||
|
for field in ("fileid", "modify_time", "size"):
|
||||||
|
recorded_value = recorded_fingerprint.get(field)
|
||||||
|
current_value = current_fingerprint.get(field)
|
||||||
|
if (
|
||||||
|
recorded_value is not None
|
||||||
|
and current_value is not None
|
||||||
|
and recorded_value != current_value
|
||||||
|
):
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def failed_retry_count(src_path: Optional[str], storage: Optional[str] = None,
|
||||||
|
file_size: Any = None, file_modify_time: Any = None,
|
||||||
|
fileid: Any = None) -> int:
|
||||||
|
"""
|
||||||
|
读取同一源路径已累计的连续整理失败次数。
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param storage: 源存储
|
||||||
|
:param file_size: 当前文件大小
|
||||||
|
:param file_modify_time: 当前文件修改时间
|
||||||
|
:param fileid: 当前文件唯一标识
|
||||||
|
:return: 当前文件版本已失败次数,无记录时为 0
|
||||||
|
"""
|
||||||
|
key = failed_retry_key(src_path, storage)
|
||||||
|
if not key:
|
||||||
|
return 0
|
||||||
|
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
|
||||||
|
current_fingerprint = file_fingerprint(
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
recorded_fingerprint
|
||||||
|
and current_fingerprint
|
||||||
|
and _is_file_version_changed(recorded_fingerprint, current_fingerprint)
|
||||||
|
):
|
||||||
|
return 0
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def record_transfer_failure(src_path: Optional[str], storage: Optional[str] = None,
|
||||||
|
file_size: Any = None, file_modify_time: Any = None,
|
||||||
|
fileid: Any = None) -> int:
|
||||||
|
"""
|
||||||
|
累计一次整理失败。
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param storage: 源存储
|
||||||
|
:param file_size: 当前文件大小
|
||||||
|
:param file_modify_time: 当前文件修改时间
|
||||||
|
:param fileid: 当前文件唯一标识
|
||||||
|
:return: 当前文件版本累计后的失败次数
|
||||||
|
"""
|
||||||
|
key = failed_retry_key(src_path, storage)
|
||||||
|
if not key:
|
||||||
|
return 0
|
||||||
|
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
|
||||||
|
current_fingerprint = file_fingerprint(
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if current_fingerprint and (
|
||||||
|
not recorded_fingerprint
|
||||||
|
or _is_file_version_changed(recorded_fingerprint, current_fingerprint)
|
||||||
|
):
|
||||||
|
count = 0
|
||||||
|
count += 1
|
||||||
|
if current_fingerprint:
|
||||||
|
_failed_retry_counts[key] = {
|
||||||
|
"count": count,
|
||||||
|
"fingerprint": current_fingerprint,
|
||||||
|
}
|
||||||
|
elif recorded_fingerprint:
|
||||||
|
_failed_retry_counts[key] = {
|
||||||
|
"count": count,
|
||||||
|
"fingerprint": recorded_fingerprint,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
_failed_retry_counts[key] = count
|
||||||
|
return count
|
||||||
|
|
||||||
|
|
||||||
|
def clear_transfer_failures(src_path: Optional[str], storage: Optional[str] = None) -> None:
|
||||||
|
"""
|
||||||
|
清空同一源路径的失败计数。整理成功、或用户删除整理记录(显式要求重来)时调用。
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param storage: 源存储
|
||||||
|
"""
|
||||||
|
key = failed_retry_key(src_path, storage)
|
||||||
|
if key:
|
||||||
|
# 缺省值必须是 0 而不是 None:CacheBackend.pop 把「default 为 None」当成「未提供
|
||||||
|
# default」,键不存在时会抛 KeyError。整理成功路径上绝大多数文件从未失败过,
|
||||||
|
# 传 None 会让每一次首次成功整理都炸掉成功回调
|
||||||
|
_failed_retry_counts.pop(key, 0)
|
||||||
|
|
||||||
|
|
||||||
|
def coerce_size(size: Any) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
统一转换文件大小,无法转换时返回 None(视为不可比对)。
|
||||||
|
:param size: 原始大小值
|
||||||
|
:return: 文件大小
|
||||||
|
"""
|
||||||
|
if size is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return int(size)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def history_src_size(history: TransferHistory) -> Optional[int]:
|
||||||
|
"""
|
||||||
|
读取整理记录中的源文件大小。
|
||||||
|
src_fileitem 是 JSON 列,历史数据可能为空、缺 size 键甚至不是字典,
|
||||||
|
取不到时统一返回 None 交由调用方保守处理。
|
||||||
|
:param history: 整理记录
|
||||||
|
:return: 源文件大小,取不到时为 None
|
||||||
|
"""
|
||||||
|
return history_src_fingerprint(history).get("size")
|
||||||
|
|
||||||
|
|
||||||
|
def history_src_fingerprint(history: TransferHistory) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
读取整理记录中的源文件版本指纹。
|
||||||
|
:param history: 整理记录
|
||||||
|
:return: 源文件的可比较指纹字段
|
||||||
|
"""
|
||||||
|
src_fileitem = getattr(history, "src_fileitem", None)
|
||||||
|
if not isinstance(src_fileitem, dict):
|
||||||
|
return {}
|
||||||
|
return file_fingerprint(
|
||||||
|
file_size=src_fileitem.get("size"),
|
||||||
|
file_modify_time=src_fileitem.get("modify_time"),
|
||||||
|
fileid=src_fileitem.get("fileid"),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def resolve_history(src_path: str, storage: Optional[str] = None,
|
||||||
|
transfer_history_oper: Optional[TransferHistoryOper] = None
|
||||||
|
) -> Optional[TransferHistory]:
|
||||||
|
"""
|
||||||
|
查询源路径对应的整理记录。
|
||||||
|
|
||||||
|
新表通过 (src, src_storage) 唯一索引保证单条记录;仍保留对成功记录的二次确认,
|
||||||
|
兼容升级前可能残留的重复数据,避免把已整理成功的文件重复整理。查询异常不在
|
||||||
|
此处吞掉,由调用方按各自的重试策略处理。
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param storage: 存储
|
||||||
|
:param transfer_history_oper: 复用的历史操作对象,未传时新建
|
||||||
|
:return: 命中的整理记录,未命中时为 None
|
||||||
|
"""
|
||||||
|
oper = transfer_history_oper or TransferHistoryOper()
|
||||||
|
history = oper.get_by_src(src_path, storage=storage)
|
||||||
|
if history is not None and not history.status:
|
||||||
|
history = oper.get_success_by_src(src_path, storage=storage) or history
|
||||||
|
return history
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_history_gate(history: Optional[TransferHistory],
|
||||||
|
file_size: Optional[float] = None,
|
||||||
|
file_modify_time: Optional[float] = None,
|
||||||
|
fileid: Optional[str] = None,
|
||||||
|
retry_count: Optional[int] = None) -> str:
|
||||||
|
"""
|
||||||
|
依据整理历史判断本次是否跳过整理。
|
||||||
|
|
||||||
|
成功记录不能简单地「存在即跳过」:同路径重新上传的新版本会因此没有机会走到
|
||||||
|
整理链的 overwrite_mode 判定,升级永远无法入库,故任一可比文件指纹变化时一律放行。
|
||||||
|
失败记录按文件版本使用有界重试:新版本先放行并在下一次失败时从 1 重新计数,
|
||||||
|
同一版本未达上限时继续重试,让瞬时故障(网络/识别/移动)自愈;达到上限后跳过,
|
||||||
|
避免永久失败的文件反复刷失败通知。
|
||||||
|
:param history: 整理记录,未命中时为 None
|
||||||
|
:param file_size: 当前文件大小,蓝光目录等场景可能为 None
|
||||||
|
:param file_modify_time: 当前文件修改时间
|
||||||
|
:param fileid: 当前文件唯一标识
|
||||||
|
:param retry_count: 已累计的失败次数,None 表示按记录源路径实时查询
|
||||||
|
:return: HistoryGateAction 之一
|
||||||
|
"""
|
||||||
|
if history is None:
|
||||||
|
return HistoryGateAction.PASS_NO_RECORD
|
||||||
|
recorded_fingerprint = history_src_fingerprint(history)
|
||||||
|
current_fingerprint = file_fingerprint(
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if not history.status:
|
||||||
|
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||||
|
return HistoryGateAction.PASS_FAILED_VERSION_CHANGED
|
||||||
|
if retry_count is None:
|
||||||
|
retry_count = failed_retry_count(
|
||||||
|
getattr(history, "src", None),
|
||||||
|
getattr(history, "src_storage", None),
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if retry_count >= max_failed_retries():
|
||||||
|
return HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
# 监控事件是稀疏驱动的(落地事件/延迟重扫/补偿扫描),入口还有 TTL 去重兜底,
|
||||||
|
# 配合失败次数上限,重试频率与总量都可控
|
||||||
|
return HistoryGateAction.PASS_FAILED
|
||||||
|
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||||
|
# 同路径换成了另一个版本(如升级为更高码率),是否覆盖交给整理链的
|
||||||
|
# overwrite_mode 决断,查重闸不做替代判断
|
||||||
|
return HistoryGateAction.PASS_SIZE_CHANGED
|
||||||
|
# 无法比对大小(蓝光目录、历史记录缺 size)时保守跳过,避免重复整理
|
||||||
|
return HistoryGateAction.SKIP
|
||||||
|
|
||||||
|
|
||||||
|
def describe_history_gate(history: Optional[TransferHistory],
|
||||||
|
file_size: Optional[float] = None,
|
||||||
|
file_modify_time: Optional[float] = None,
|
||||||
|
fileid: Optional[str] = None) -> str:
|
||||||
|
"""
|
||||||
|
生成查重闸判定的可读说明,供日志定位「到底是哪条记录在拦」。
|
||||||
|
:param history: 整理记录
|
||||||
|
:param file_size: 当前文件大小
|
||||||
|
:param file_modify_time: 当前文件修改时间
|
||||||
|
:param fileid: 当前文件唯一标识
|
||||||
|
:return: 说明文本
|
||||||
|
"""
|
||||||
|
if history is None:
|
||||||
|
return "无整理记录"
|
||||||
|
recorded_fingerprint = history_src_fingerprint(history)
|
||||||
|
current_fingerprint = file_fingerprint(
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if not history.status:
|
||||||
|
count = failed_retry_count(
|
||||||
|
getattr(history, "src", None),
|
||||||
|
getattr(history, "src_storage", None),
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||||
|
return f"失败记录 #{history.id},文件版本已变化,重试预算将重置"
|
||||||
|
return f"失败记录 #{history.id},已重试 {count}/{max_failed_retries()} 次"
|
||||||
|
recorded_size = recorded_fingerprint.get("size")
|
||||||
|
current_size = current_fingerprint.get("size")
|
||||||
|
if recorded_size is None and current_size is None:
|
||||||
|
return f"成功记录 #{history.id},大小不可比对"
|
||||||
|
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"
|
||||||
@@ -0,0 +1,411 @@
|
|||||||
|
"""
|
||||||
|
本地文件系统操作代理。
|
||||||
|
|
||||||
|
FUSE/网络挂载有两种故障形态:crash 型(调用抛错,可捕获、可重试)和 block 型
|
||||||
|
(调用既不返回错误也不返回结果,永久悬挂)。**Python 无法中断一个已经发出的
|
||||||
|
系统调用,也无法强杀线程**,所以 block 型故障下阻塞的线程永远无法回收——这正是
|
||||||
|
整理消费线程停摆、监控自愈路径自冻的根因。
|
||||||
|
|
||||||
|
本模块把这些调用放进一个常驻子进程执行。子进程可以被 SIGKILL,因此超时后能真正
|
||||||
|
回收;对调用方而言,超时表现为一个普通的 OSError 子类(FileSystemTimeout)。
|
||||||
|
换句话说:**把不可处理的 block 型故障,转换成系统各层已经能正确处理的 crash 型
|
||||||
|
故障**——退避重启、登记待重试这些既有机制立刻就能接管。
|
||||||
|
|
||||||
|
第一版只放行安全的操作:
|
||||||
|
- 只读(stat/exists/listdir)——强杀不产生任何副作用
|
||||||
|
- 同存储 rename——内核保证原子性,强杀后要么完全成功要么完全没发生
|
||||||
|
跨存储的复制+删除不在此列,它需要单独的可恢复语义(临时名 + 完成后 rename)。
|
||||||
|
"""
|
||||||
|
import errno as errno_module
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import selectors
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any, Callable, Dict, List, Optional
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.log import logger
|
||||||
|
|
||||||
|
# worker 脚本路径。用绝对路径直接执行,而不是 -m 或 import:
|
||||||
|
# 直接执行文件不会触发 app/__init__.py 的导入链,代理启动才是毫秒级的
|
||||||
|
_WORKER_PATH = Path(__file__).parent / "fsworker.py"
|
||||||
|
# 单次快操作(stat/listdir/rename/unlink 等)的默认超时秒数
|
||||||
|
DEFAULT_TIMEOUT = 30
|
||||||
|
# 长耗时操作(复制)两次进度上报之间的最长间隔秒数。
|
||||||
|
# worker 每秒上报一次心跳,因此这个阈值判定的是「传输完全没有推进」,
|
||||||
|
# 而不是「传输很慢」——大文件复制几小时也不会误杀
|
||||||
|
DEFAULT_STALL_TIMEOUT = 120
|
||||||
|
# 强杀代理后等待它消失的宽限秒数,不能无限等待
|
||||||
|
_KILL_GRACE = 5
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemTimeout(OSError):
|
||||||
|
"""
|
||||||
|
文件系统操作在代理中超时未返回,判定挂载无响应。
|
||||||
|
|
||||||
|
继承 OSError 是刻意的:整理链、监控 watcher 等各层对 OSError 已有完整的
|
||||||
|
退避重试与登记逻辑,block 型故障经此转换后可以直接复用它们。
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
class FileSystemProxy:
|
||||||
|
"""
|
||||||
|
常驻子进程文件系统代理。
|
||||||
|
|
||||||
|
请求-响应严格串行(一个代理同时只处理一个请求),由锁保证。超时即强杀代理,
|
||||||
|
下一次请求自动重启一个新的——启动成本是毫秒级,因为 worker 只依赖标准库。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, timeout: Optional[float] = None,
|
||||||
|
stall_timeout: Optional[float] = None):
|
||||||
|
"""
|
||||||
|
:param timeout: 单次快操作的超时秒数,None 表示实时跟随系统设置
|
||||||
|
:param stall_timeout: 长耗时操作两次进度上报之间的最长间隔秒数,
|
||||||
|
None 表示实时跟随系统设置
|
||||||
|
"""
|
||||||
|
self._timeout_override = timeout
|
||||||
|
self._stall_timeout_override = stall_timeout
|
||||||
|
self._process: Optional[subprocess.Popen] = None
|
||||||
|
self._selector: Optional[selectors.BaseSelector] = None
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 对外操作
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
def stat(self, path: Path) -> Dict[str, Any]:
|
||||||
|
"""
|
||||||
|
读取路径属性。
|
||||||
|
:param path: 目标路径
|
||||||
|
:return: {"size", "mtime", "is_dir", "is_file"}
|
||||||
|
"""
|
||||||
|
return self._call("stat", path=str(path))
|
||||||
|
|
||||||
|
def exists(self, path: Path) -> bool:
|
||||||
|
"""
|
||||||
|
判断路径是否存在。
|
||||||
|
|
||||||
|
只有 FileNotFoundError 才算「不存在」;其余 OSError(含超时)原样抛出,
|
||||||
|
避免像 Path.exists() 那样把挂载抖动误判成文件消失。
|
||||||
|
:param path: 目标路径
|
||||||
|
:return: 是否存在
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
self._call("exists", path=str(path))
|
||||||
|
return True
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def listdir(self, path: Path) -> List[str]:
|
||||||
|
"""
|
||||||
|
列出目录条目名。
|
||||||
|
:param path: 目标目录
|
||||||
|
:return: 条目名列表
|
||||||
|
"""
|
||||||
|
return self._call("listdir", path=str(path))
|
||||||
|
|
||||||
|
def rename(self, src: Path, dst: Path) -> bool:
|
||||||
|
"""
|
||||||
|
同一存储内重命名/移动。跨存储会抛 OSError(EXDEV),由调用方走原有路径。
|
||||||
|
:param src: 源路径
|
||||||
|
:param dst: 目标路径
|
||||||
|
:return: 是否成功
|
||||||
|
"""
|
||||||
|
return self._call("rename", src=str(src), dst=str(dst))
|
||||||
|
|
||||||
|
def copy(self, src: Path, dst: Path,
|
||||||
|
progress_cb: Optional[Callable[[float], None]] = None,
|
||||||
|
cancel_cb: Optional[Callable[[], bool]] = None,
|
||||||
|
chunk_size: Optional[int] = None) -> Any:
|
||||||
|
"""
|
||||||
|
复制文件内容并保留时间戳,按「进度无推进」判定挂死。
|
||||||
|
|
||||||
|
复制大文件可能持续几小时,固定超时无法区分「正常但慢」和「已经挂死」。
|
||||||
|
worker 每秒上报一次进度作为心跳,这里判定的是**两次上报之间的间隔**:
|
||||||
|
超过 stall 阈值收不到任何一行,才认定挂载无响应并强杀 worker。
|
||||||
|
|
||||||
|
取消检查放在父进程:worker 里读不到 global_vars 的传输取消标记,而父进程
|
||||||
|
每收到一次进度就能检查一次,要取消直接杀掉 worker 即可,比在子进程里
|
||||||
|
轮询标记更干净。
|
||||||
|
:param src: 源文件
|
||||||
|
:param dst: 目标文件(调用方应传临时名,完成后自行原子替换)
|
||||||
|
:param progress_cb: 进度回调,入参为百分比
|
||||||
|
:param cancel_cb: 取消检查回调,返回 True 表示应中止
|
||||||
|
:param chunk_size: 分块大小
|
||||||
|
:return: 成功时为 {"copied", "total"},被取消或通信失败时为 False
|
||||||
|
"""
|
||||||
|
payload = {"src": str(src), "dst": str(dst)}
|
||||||
|
if chunk_size:
|
||||||
|
payload["chunk_size"] = chunk_size
|
||||||
|
if not self._enabled():
|
||||||
|
return self._direct_copy(src, dst, progress_cb, cancel_cb, chunk_size)
|
||||||
|
with self._lock:
|
||||||
|
try:
|
||||||
|
return self._request_stream(payload, progress_cb, cancel_cb)
|
||||||
|
except FileSystemTimeout:
|
||||||
|
raise
|
||||||
|
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||||
|
logger.error(f"文件系统代理复制通信异常: {src} -> {dst} - {err}")
|
||||||
|
self._shutdown()
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _request_stream(self, payload: Dict[str, Any],
|
||||||
|
progress_cb: Optional[Callable[[float], None]],
|
||||||
|
cancel_cb: Optional[Callable[[], bool]]) -> Any:
|
||||||
|
"""
|
||||||
|
发起一次流式请求,逐行消费进度直到终态。
|
||||||
|
:param payload: 请求参数
|
||||||
|
:param progress_cb: 进度回调
|
||||||
|
:param cancel_cb: 取消检查回调
|
||||||
|
:return: 操作结果
|
||||||
|
"""
|
||||||
|
self._ensure_worker()
|
||||||
|
message = json.dumps({"op": "copy", **payload}) + "\n"
|
||||||
|
self._process.stdin.write(message.encode("utf-8"))
|
||||||
|
self._process.stdin.flush()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
response = json.loads(self._read_line(timeout=self._stall_timeout).decode("utf-8"))
|
||||||
|
progress = response.get("progress")
|
||||||
|
if progress is not None:
|
||||||
|
if cancel_cb is not None and cancel_cb():
|
||||||
|
logger.info(f"复制已取消: {payload.get('src')}")
|
||||||
|
# 取消就地生效:杀掉 worker 立刻中断传输,不必等它读完整个文件
|
||||||
|
self._shutdown()
|
||||||
|
return False
|
||||||
|
if progress_cb is not None:
|
||||||
|
total = progress.get("total") or 0
|
||||||
|
progress_cb(progress.get("copied", 0) / total * 100 if total else 0)
|
||||||
|
continue
|
||||||
|
if response.get("ok"):
|
||||||
|
return response.get("result")
|
||||||
|
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _direct_copy(src: Path, dst: Path,
|
||||||
|
progress_cb: Optional[Callable[[float], None]],
|
||||||
|
cancel_cb: Optional[Callable[[], bool]],
|
||||||
|
chunk_size: Optional[int]) -> bool:
|
||||||
|
"""
|
||||||
|
不经代理直接复制,供代理关闭时使用。
|
||||||
|
"""
|
||||||
|
info = os.stat(src)
|
||||||
|
total = info.st_size
|
||||||
|
copied = 0
|
||||||
|
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||||
|
while True:
|
||||||
|
if cancel_cb is not None and cancel_cb():
|
||||||
|
return False
|
||||||
|
buf = fsrc.read(chunk_size or 1024 * 1024)
|
||||||
|
if not buf:
|
||||||
|
break
|
||||||
|
fdst.write(buf)
|
||||||
|
copied += len(buf)
|
||||||
|
if progress_cb is not None and total:
|
||||||
|
progress_cb(copied / total * 100)
|
||||||
|
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||||
|
return True
|
||||||
|
|
||||||
|
def unlink(self, path: Path) -> bool:
|
||||||
|
"""
|
||||||
|
删除单个文件。unlink 是原子操作,强杀后没有中间状态。
|
||||||
|
:param path: 目标文件
|
||||||
|
:return: 是否成功
|
||||||
|
"""
|
||||||
|
return self._call("unlink", path=str(path))
|
||||||
|
|
||||||
|
def rmtree(self, path: Path) -> bool:
|
||||||
|
"""
|
||||||
|
递归删除目录,容忍部分失败(可重复执行直到成功)。
|
||||||
|
:param path: 目标目录
|
||||||
|
:return: 是否成功
|
||||||
|
"""
|
||||||
|
return self._call("rmtree", path=str(path))
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
"""
|
||||||
|
关闭代理进程。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._shutdown()
|
||||||
|
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
# 内部实现
|
||||||
|
# ------------------------------------------------------------------ #
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _timeout(self) -> float:
|
||||||
|
"""
|
||||||
|
单次快操作的超时秒数。
|
||||||
|
|
||||||
|
实时读取而不是构造时固定:这三项都暴露在前端设置里,用户改完保存后
|
||||||
|
必须立刻生效,否则会出现「改了没反应」的困惑。
|
||||||
|
"""
|
||||||
|
if self._timeout_override is not None:
|
||||||
|
return self._timeout_override
|
||||||
|
return float(getattr(settings, "FS_PROXY_TIMEOUT", DEFAULT_TIMEOUT))
|
||||||
|
|
||||||
|
@property
|
||||||
|
def _stall_timeout(self) -> float:
|
||||||
|
"""
|
||||||
|
长耗时操作两次进度上报之间的最长间隔秒数,同样实时跟随系统设置。
|
||||||
|
"""
|
||||||
|
if self._stall_timeout_override is not None:
|
||||||
|
return self._stall_timeout_override
|
||||||
|
return float(getattr(settings, "FS_PROXY_STALL_TIMEOUT", DEFAULT_STALL_TIMEOUT))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _enabled() -> bool:
|
||||||
|
"""
|
||||||
|
代理是否启用。关闭时退回直接调用,行为与引入代理之前完全一致。
|
||||||
|
"""
|
||||||
|
return bool(getattr(settings, "FS_PROXY_ENABLED", True))
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _direct(op: str, payload: Dict[str, Any]) -> Any:
|
||||||
|
"""
|
||||||
|
不经代理直接执行操作,供代理关闭时使用。
|
||||||
|
:param op: 操作名
|
||||||
|
:param payload: 操作参数
|
||||||
|
:return: 操作结果
|
||||||
|
"""
|
||||||
|
if op == "stat":
|
||||||
|
path = payload["path"]
|
||||||
|
info = os.stat(path)
|
||||||
|
return {
|
||||||
|
"size": info.st_size,
|
||||||
|
"mtime": info.st_mtime,
|
||||||
|
"is_dir": os.path.isdir(path),
|
||||||
|
"is_file": os.path.isfile(path),
|
||||||
|
}
|
||||||
|
if op == "exists":
|
||||||
|
os.stat(payload["path"])
|
||||||
|
return True
|
||||||
|
if op == "listdir":
|
||||||
|
return sorted(os.listdir(payload["path"]))
|
||||||
|
if op == "rename":
|
||||||
|
os.rename(payload["src"], payload["dst"])
|
||||||
|
return True
|
||||||
|
if op == "unlink":
|
||||||
|
os.unlink(payload["path"])
|
||||||
|
return True
|
||||||
|
if op == "rmtree":
|
||||||
|
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||||
|
return True
|
||||||
|
raise ValueError(f"unknown op: {op}")
|
||||||
|
|
||||||
|
def _call(self, op: str, **payload) -> Any:
|
||||||
|
"""
|
||||||
|
执行一次代理调用。
|
||||||
|
:param op: 操作名
|
||||||
|
:param payload: 操作参数
|
||||||
|
:return: 操作结果
|
||||||
|
"""
|
||||||
|
if not self._enabled():
|
||||||
|
return self._direct(op, payload)
|
||||||
|
with self._lock:
|
||||||
|
try:
|
||||||
|
return self._request(op, payload)
|
||||||
|
except FileSystemTimeout:
|
||||||
|
# 超时说明挂载正在挂死,重试只会再冻一次,直接上报给调用方
|
||||||
|
raise
|
||||||
|
except (BrokenPipeError, ConnectionError, json.JSONDecodeError, ValueError) as err:
|
||||||
|
# 代理进程意外退出或响应损坏,重启后重试一次
|
||||||
|
logger.debug(f"文件系统代理通信异常,重启后重试: {op} - {err}")
|
||||||
|
self._shutdown()
|
||||||
|
return self._request(op, payload)
|
||||||
|
|
||||||
|
def _request(self, op: str, payload: Dict[str, Any]) -> Any:
|
||||||
|
"""
|
||||||
|
发送请求并等待响应。
|
||||||
|
:param op: 操作名
|
||||||
|
:param payload: 操作参数
|
||||||
|
:return: 操作结果
|
||||||
|
"""
|
||||||
|
self._ensure_worker()
|
||||||
|
message = json.dumps({"op": op, **payload}) + "\n"
|
||||||
|
self._process.stdin.write(message.encode("utf-8"))
|
||||||
|
self._process.stdin.flush()
|
||||||
|
|
||||||
|
response = json.loads(self._read_line().decode("utf-8"))
|
||||||
|
if response.get("ok"):
|
||||||
|
return response.get("result")
|
||||||
|
# OSError(errno, strerror) 会自动映射到 FileNotFoundError 等具体子类,
|
||||||
|
# 调用方沿用原有的异常分支即可,无需感知代理的存在
|
||||||
|
raise OSError(response.get("errno") or 0, response.get("error") or "unknown error")
|
||||||
|
|
||||||
|
def _read_line(self, timeout: Optional[float] = None) -> bytes:
|
||||||
|
"""
|
||||||
|
读取一行响应,超时即强杀代理。
|
||||||
|
:param timeout: 本次读取的超时秒数,默认用单次操作超时
|
||||||
|
:return: 响应行
|
||||||
|
"""
|
||||||
|
timeout = self._timeout if timeout is None else timeout
|
||||||
|
if not self._selector.select(timeout=timeout):
|
||||||
|
logger.error(f"文件系统操作 {timeout} 秒无响应,判定挂载挂死,正在回收代理进程")
|
||||||
|
self._shutdown()
|
||||||
|
raise FileSystemTimeout(
|
||||||
|
errno_module.ETIMEDOUT,
|
||||||
|
f"文件系统操作超过 {timeout} 秒无响应,挂载可能已无响应"
|
||||||
|
)
|
||||||
|
line = self._process.stdout.readline()
|
||||||
|
if not line:
|
||||||
|
raise BrokenPipeError("文件系统代理进程已退出")
|
||||||
|
return line
|
||||||
|
|
||||||
|
def _ensure_worker(self):
|
||||||
|
"""
|
||||||
|
确保代理进程可用,不可用时重新启动。
|
||||||
|
"""
|
||||||
|
if self._process is not None and self._process.poll() is None:
|
||||||
|
return
|
||||||
|
self._shutdown()
|
||||||
|
self._process = subprocess.Popen(
|
||||||
|
[sys.executable, str(_WORKER_PATH)],
|
||||||
|
stdin=subprocess.PIPE,
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.DEVNULL,
|
||||||
|
bufsize=0,
|
||||||
|
)
|
||||||
|
self._selector = selectors.DefaultSelector()
|
||||||
|
self._selector.register(self._process.stdout, selectors.EVENT_READ)
|
||||||
|
logger.debug(f"文件系统代理进程已启动: pid={self._process.pid}")
|
||||||
|
|
||||||
|
def _shutdown(self):
|
||||||
|
"""
|
||||||
|
回收代理进程。冻在挂载上的进程用 SIGKILL,且不无限等待它消失
|
||||||
|
——否则「可放弃的代理」又变回一次不可放弃的阻塞。
|
||||||
|
"""
|
||||||
|
if self._selector is not None:
|
||||||
|
try:
|
||||||
|
self._selector.close()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
self._selector = None
|
||||||
|
process, self._process = self._process, None
|
||||||
|
if process is None:
|
||||||
|
return
|
||||||
|
for stream in (process.stdin, process.stdout):
|
||||||
|
try:
|
||||||
|
if stream:
|
||||||
|
stream.close()
|
||||||
|
except Exception: # noqa: BLE001
|
||||||
|
pass
|
||||||
|
if process.poll() is not None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
process.kill()
|
||||||
|
process.wait(timeout=_KILL_GRACE)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.warn(f"文件系统代理进程未能及时退出,交由系统回收: pid={process.pid}")
|
||||||
|
except Exception as err: # noqa: BLE001
|
||||||
|
logger.debug(f"回收文件系统代理进程失败: {err}")
|
||||||
|
|
||||||
|
|
||||||
|
# 全局单例:local 存储本身是单例,代理也只需要一个
|
||||||
|
# 不传超时参数:让它实时跟随系统设置,前端改完保存即刻生效
|
||||||
|
fsproxy = FileSystemProxy()
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
"""
|
||||||
|
文件系统操作代理 worker。
|
||||||
|
|
||||||
|
**本文件不能被 import,只能作为独立脚本执行**(fsproxy 用
|
||||||
|
`subprocess.Popen([sys.executable, <本文件绝对路径>])` 启动)。直接执行文件
|
||||||
|
路径不会触发 `app/__init__.py` 的导入链,因此这个进程只依赖标准库、启动是
|
||||||
|
毫秒级的;一旦走 import 就会把整个应用的依赖拉进来,代理被强杀后的重启成本
|
||||||
|
会高到无法接受。
|
||||||
|
|
||||||
|
存在的理由:FUSE/网络挂载进入 block 型故障时,`stat`/`listdir`/`rename` 这类
|
||||||
|
系统调用既不返回错误也不返回结果,而 Python 没有中断线程的手段——阻塞其上的
|
||||||
|
线程永远无法回收。放进独立进程后,父进程可以在超时后 SIGKILL 掉它,把
|
||||||
|
「不可处理的 block」转换成「可处理的 crash」。
|
||||||
|
|
||||||
|
协议:stdin/stdout 逐行 JSON。
|
||||||
|
请求 {"op": "stat", "path": "/mnt/cd2/x.mkv"}
|
||||||
|
成功 {"ok": true, "result": {...}}
|
||||||
|
失败 {"ok": false, "errno": 2, "error": "No such file or directory"}
|
||||||
|
"""
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import shutil
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def _stat(payload, _emit):
|
||||||
|
"""
|
||||||
|
读取路径的基本属性。
|
||||||
|
"""
|
||||||
|
path = payload["path"]
|
||||||
|
info = os.stat(path)
|
||||||
|
return {
|
||||||
|
"size": info.st_size,
|
||||||
|
"mtime": info.st_mtime,
|
||||||
|
"is_dir": os.path.isdir(path),
|
||||||
|
"is_file": os.path.isfile(path),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _exists(payload, _emit):
|
||||||
|
"""
|
||||||
|
判断路径是否存在。
|
||||||
|
|
||||||
|
用 os.stat 而不是 os.path.exists:后者会把任意 OSError 都归为「不存在」,
|
||||||
|
挂载抖动会被误判成文件消失。这里让异常原样抛出,由父进程按 errno 区分。
|
||||||
|
"""
|
||||||
|
os.stat(payload["path"])
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _listdir(payload, _emit):
|
||||||
|
"""
|
||||||
|
列出目录下的条目名。
|
||||||
|
"""
|
||||||
|
return sorted(os.listdir(payload["path"]))
|
||||||
|
|
||||||
|
|
||||||
|
def _copy(payload, emit):
|
||||||
|
"""
|
||||||
|
分块复制文件内容并周期上报进度。
|
||||||
|
|
||||||
|
进度上报同时充当心跳:复制大文件可能持续几小时,父进程无法用固定超时判断
|
||||||
|
挂死,只能看「两次上报之间隔了多久」。因此这里按固定时间间隔上报,即使
|
||||||
|
某一秒没读到数据也照常发——一旦挂载卡住,read/write 不返回,上报自然断流,
|
||||||
|
父进程据此判定并强杀本进程。
|
||||||
|
|
||||||
|
只复制内容和时间戳,不复制权限:目标目录的默认权限与继承 ACL 是媒体库的
|
||||||
|
访问策略,用源文件权限覆盖会清除已继承的 ACL。
|
||||||
|
"""
|
||||||
|
src, dst = payload["src"], payload["dst"]
|
||||||
|
chunk_size = payload.get("chunk_size") or 1024 * 1024
|
||||||
|
interval = payload.get("progress_interval") or 1.0
|
||||||
|
|
||||||
|
info = os.stat(src)
|
||||||
|
total = info.st_size
|
||||||
|
copied = 0
|
||||||
|
# 先发一次 0%:既让心跳立刻开始,也保证父进程在传输开始前就有一次检查
|
||||||
|
# 取消的机会——否则小文件会在首次定时上报之前就复制完,取消形同虚设
|
||||||
|
emit({"ok": True, "progress": {"copied": 0, "total": total}})
|
||||||
|
last_emit = time.monotonic()
|
||||||
|
with open(src, "rb") as fsrc, open(dst, "wb") as fdst:
|
||||||
|
while True:
|
||||||
|
buf = fsrc.read(chunk_size)
|
||||||
|
if not buf:
|
||||||
|
break
|
||||||
|
fdst.write(buf)
|
||||||
|
copied += len(buf)
|
||||||
|
now = time.monotonic()
|
||||||
|
if now - last_emit >= interval:
|
||||||
|
last_emit = now
|
||||||
|
emit({"ok": True, "progress": {"copied": copied, "total": total}})
|
||||||
|
os.utime(dst, ns=(info.st_atime_ns, info.st_mtime_ns))
|
||||||
|
return {"copied": copied, "total": total}
|
||||||
|
|
||||||
|
|
||||||
|
def _rename(payload, _emit):
|
||||||
|
"""
|
||||||
|
同一存储内重命名/移动。
|
||||||
|
|
||||||
|
这是第一版唯一放行的写操作:同文件系统内的 rename 由内核保证原子性,
|
||||||
|
进程被强杀后要么完全成功要么完全没发生,不存在需要清理的中间状态。
|
||||||
|
跨存储的复制+删除不走这里,它需要单独的可恢复语义。
|
||||||
|
"""
|
||||||
|
src, dst = payload["src"], payload["dst"]
|
||||||
|
if os.stat(src).st_dev != os.stat(os.path.dirname(dst) or ".").st_dev:
|
||||||
|
raise OSError(18, "Cross-device rename is not handled by the proxy")
|
||||||
|
os.rename(src, dst)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _unlink(payload, _emit):
|
||||||
|
"""
|
||||||
|
删除单个文件。unlink 是原子操作,强杀后要么删掉了要么没删,没有中间状态。
|
||||||
|
"""
|
||||||
|
os.unlink(payload["path"])
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
def _rmtree(payload, _emit):
|
||||||
|
"""
|
||||||
|
递归删除目录。
|
||||||
|
|
||||||
|
这一项不是原子的,强杀可能只删掉一部分。放行的理由是:删除被中断的后果
|
||||||
|
(残留若干文件)远轻于写入被中断(留下叫最终文件名的半成品),而且调用方
|
||||||
|
本来就以 ignore_errors 容忍部分失败、可以重复执行直到成功。
|
||||||
|
"""
|
||||||
|
shutil.rmtree(payload["path"], ignore_errors=True)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
_HANDLERS = {
|
||||||
|
"stat": _stat,
|
||||||
|
"exists": _exists,
|
||||||
|
"listdir": _listdir,
|
||||||
|
"copy": _copy,
|
||||||
|
"rename": _rename,
|
||||||
|
"unlink": _unlink,
|
||||||
|
"rmtree": _rmtree,
|
||||||
|
"ping": lambda _payload, _emit: True,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _write(message):
|
||||||
|
"""
|
||||||
|
输出一行响应。
|
||||||
|
"""
|
||||||
|
sys.stdout.write(json.dumps(message) + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
"""
|
||||||
|
请求循环:每读一行处理一个请求,直到 stdin 关闭。
|
||||||
|
|
||||||
|
一个请求可能对应多行响应:长耗时操作先流式发若干 progress 行,最后发一行
|
||||||
|
终态(result 或 error)。父进程据此区分「还在推进」和「已经挂死」。
|
||||||
|
"""
|
||||||
|
for line in sys.stdin:
|
||||||
|
line = line.strip()
|
||||||
|
if not line:
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
payload = json.loads(line)
|
||||||
|
handler = _HANDLERS.get(payload.get("op"))
|
||||||
|
if handler is None:
|
||||||
|
response = {"ok": False, "errno": 0,
|
||||||
|
"error": f"unknown op: {payload.get('op')}"}
|
||||||
|
else:
|
||||||
|
response = {"ok": True, "result": handler(payload, _write)}
|
||||||
|
except OSError as err:
|
||||||
|
response = {"ok": False, "errno": err.errno or 0,
|
||||||
|
"error": err.strerror or str(err)}
|
||||||
|
except Exception as err: # noqa: BLE001 - worker 不能因任何异常退出
|
||||||
|
response = {"ok": False, "errno": 0, "error": str(err)}
|
||||||
|
_write(response)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -8,6 +8,7 @@ from app import schemas
|
|||||||
from app.helper.progress import ProgressHelper
|
from app.helper.progress import ProgressHelper
|
||||||
from app.helper.storage import StorageHelper
|
from app.helper.storage import StorageHelper
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
|
from app.schemas.exception import StorageQueryError
|
||||||
from app.utils.crypto import HashUtils
|
from app.utils.crypto import HashUtils
|
||||||
|
|
||||||
|
|
||||||
@@ -179,9 +180,13 @@ class StorageBase(metaclass=ABCMeta):
|
|||||||
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||||
默认实现不区分「不存在」与「查询失败」,由具体存储按需覆写。
|
|
||||||
|
默认保守失败:未覆写的存储无法区分「不存在」与「查询失败」,沿用
|
||||||
|
get_item() 会让 overwrite_mode=size 的覆盖保护在查询失败时被绕过,
|
||||||
|
把「无法确认」当成「目标不存在」而放行覆盖。具体存储必须先实现
|
||||||
|
「确认不存在」的判定,再覆写本方法。
|
||||||
"""
|
"""
|
||||||
return self.get_item(path)
|
raise StorageQueryError(f"存储 {self.schema} 未实现严格查询,无法确认目标状态: {path}")
|
||||||
|
|
||||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
@@ -337,6 +342,7 @@ class StorageBase(metaclass=ABCMeta):
|
|||||||
files_info[_fileitm.path] = {
|
files_info[_fileitm.path] = {
|
||||||
'size': _fileitm.size or 0,
|
'size': _fileitm.size or 0,
|
||||||
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
'modify_time': getattr(_fileitm, 'modify_time', 0),
|
||||||
|
'fileid': getattr(_fileitm, 'fileid', None),
|
||||||
'type': _fileitm.type
|
'type': _fileitm.type
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from app.core.cache import cached
|
|||||||
from app.core.config import settings, global_vars
|
from app.core.config import settings, global_vars
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||||
from app.schemas.exception import OperationInterrupted
|
from app.schemas.exception import OperationInterrupted, StorageQueryError
|
||||||
from app.schemas.types import StorageSchema
|
from app.schemas.types import StorageSchema
|
||||||
from app.utils.http import RequestUtils
|
from app.utils.http import RequestUtils
|
||||||
from app.utils.singleton import WeakSingleton
|
from app.utils.singleton import WeakSingleton
|
||||||
@@ -471,18 +471,58 @@ class Alist(StorageBase, metaclass=WeakSingleton):
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
return self.__build_fileitem(path, result["data"])
|
||||||
|
|
||||||
|
def __build_fileitem(self, path: Path, data: dict) -> schemas.FileItem:
|
||||||
|
"""
|
||||||
|
根据接口返回数据构建文件项。
|
||||||
|
:param path: 文件路径
|
||||||
|
:param data: 接口返回的 data 字段
|
||||||
|
:return: 文件项
|
||||||
|
"""
|
||||||
return schemas.FileItem(
|
return schemas.FileItem(
|
||||||
storage=self.schema.value,
|
storage=self.schema.value,
|
||||||
type="dir" if result["data"]["is_dir"] else "file",
|
type="dir" if data["is_dir"] else "file",
|
||||||
path=path.as_posix() + ("/" if result["data"]["is_dir"] else ""),
|
path=path.as_posix() + ("/" if data["is_dir"] else ""),
|
||||||
name=result["data"]["name"],
|
name=data["name"],
|
||||||
basename=Path(result["data"]["name"]).stem,
|
basename=Path(data["name"]).stem,
|
||||||
extension=Path(result["data"]["name"]).suffix[1:],
|
extension=Path(data["name"]).suffix[1:],
|
||||||
size=result["data"]["size"],
|
size=data["size"],
|
||||||
modify_time=self.__parse_timestamp(result["data"]["modified"]),
|
modify_time=self.__parse_timestamp(data["modified"]),
|
||||||
thumbnail=result["data"]["thumb"],
|
thumbnail=data["thumb"],
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||||
|
"""
|
||||||
|
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||||
|
只有接口明确回报「对象不存在」才是确定结果,连接失败、HTTP 异常与其他
|
||||||
|
业务错误都无法确认目标状态,必须保守失败以免覆盖保护被绕过。
|
||||||
|
"""
|
||||||
|
resp = RequestUtils(headers=self.__get_header_with_token()).post_res(
|
||||||
|
self.__get_api_url("/api/fs/get"),
|
||||||
|
json={
|
||||||
|
"path": path.as_posix(),
|
||||||
|
"password": "",
|
||||||
|
"page": 1,
|
||||||
|
"per_page": 0,
|
||||||
|
"refresh": False,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
if resp is None:
|
||||||
|
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,无法连接服务")
|
||||||
|
if resp.status_code != 200:
|
||||||
|
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败,状态码:{resp.status_code}")
|
||||||
|
try:
|
||||||
|
result = resp.json()
|
||||||
|
except Exception as err:
|
||||||
|
raise StorageQueryError(f"【OpenList】解析查询结果失败: {path} - {err}") from err
|
||||||
|
if result.get("code") != 200:
|
||||||
|
message = str(result.get("message") or "")
|
||||||
|
if "not found" in message.lower() or "not exist" in message.lower():
|
||||||
|
return None
|
||||||
|
raise StorageQueryError(f"【OpenList】查询文件 {path} 失败:{message}")
|
||||||
|
return self.__build_fileitem(path, result["data"])
|
||||||
|
|
||||||
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
def get_parent(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
获取父目录
|
获取父目录
|
||||||
|
|||||||
@@ -1,5 +1,6 @@
|
|||||||
import os
|
import os
|
||||||
import shutil
|
import shutil
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Optional, List
|
from typing import Optional, List
|
||||||
|
|
||||||
@@ -7,6 +8,7 @@ from app import schemas
|
|||||||
from app.core.config import global_vars, settings
|
from app.core.config import global_vars, settings
|
||||||
from app.helper.directory import DirectoryHelper
|
from app.helper.directory import DirectoryHelper
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
|
from app.modules.filemanager.fsproxy import fsproxy
|
||||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||||
from app.schemas.exception import StorageQueryError
|
from app.schemas.exception import StorageQueryError
|
||||||
from app.schemas.types import StorageSchema
|
from app.schemas.types import StorageSchema
|
||||||
@@ -47,6 +49,10 @@ class LocalStorage(StorageBase):
|
|||||||
"""
|
"""
|
||||||
获取文件项
|
获取文件项
|
||||||
"""
|
"""
|
||||||
|
# 走代理读取:挂载挂死时这一步会在超时后抛 OSError,而不是永久悬挂线程。
|
||||||
|
# 顺带只 stat 一次——原先 size 与 modify_time 各 stat 一次,在网络挂载上
|
||||||
|
# 等于把这个热点路径的开销翻倍
|
||||||
|
info = fsproxy.stat(path)
|
||||||
return schemas.FileItem(
|
return schemas.FileItem(
|
||||||
storage=self.schema.value,
|
storage=self.schema.value,
|
||||||
type="file",
|
type="file",
|
||||||
@@ -54,8 +60,8 @@ class LocalStorage(StorageBase):
|
|||||||
name=path.name,
|
name=path.name,
|
||||||
basename=path.stem,
|
basename=path.stem,
|
||||||
extension=path.suffix[1:],
|
extension=path.suffix[1:],
|
||||||
size=path.stat().st_size,
|
size=info["size"],
|
||||||
modify_time=path.stat().st_mtime,
|
modify_time=info["mtime"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def __get_diritem(self, path: Path) -> schemas.FileItem:
|
def __get_diritem(self, path: Path) -> schemas.FileItem:
|
||||||
@@ -68,7 +74,7 @@ class LocalStorage(StorageBase):
|
|||||||
path=path.as_posix() + "/",
|
path=path.as_posix() + "/",
|
||||||
name=path.name,
|
name=path.name,
|
||||||
basename=path.stem,
|
basename=path.stem,
|
||||||
modify_time=path.stat().st_mtime,
|
modify_time=fsproxy.stat(path)["mtime"],
|
||||||
)
|
)
|
||||||
|
|
||||||
def list(self, fileitem: schemas.FileItem) -> List[schemas.FileItem]:
|
def list(self, fileitem: schemas.FileItem) -> List[schemas.FileItem]:
|
||||||
@@ -100,12 +106,14 @@ class LocalStorage(StorageBase):
|
|||||||
|
|
||||||
# 遍历目录
|
# 遍历目录
|
||||||
path_obj = Path(path)
|
path_obj = Path(path)
|
||||||
if not path_obj.exists():
|
try:
|
||||||
|
info = fsproxy.stat(path_obj)
|
||||||
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
logger.warn(f"【本地】目录不存在:{path}")
|
logger.warn(f"【本地】目录不存在:{path}")
|
||||||
return []
|
return []
|
||||||
|
|
||||||
# 如果是文件
|
# 如果是文件
|
||||||
if path_obj.is_file():
|
if info["is_file"]:
|
||||||
ret_items.append(self.__get_fileitem(path_obj))
|
ret_items.append(self.__get_fileitem(path_obj))
|
||||||
return ret_items
|
return ret_items
|
||||||
|
|
||||||
@@ -143,9 +151,11 @@ class LocalStorage(StorageBase):
|
|||||||
"""
|
"""
|
||||||
获取文件或目录,不存在返回None
|
获取文件或目录,不存在返回None
|
||||||
"""
|
"""
|
||||||
if not path.exists():
|
try:
|
||||||
|
info = fsproxy.stat(path)
|
||||||
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
return None
|
return None
|
||||||
if path.is_file():
|
if info["is_file"]:
|
||||||
return self.__get_fileitem(path)
|
return self.__get_fileitem(path)
|
||||||
return self.__get_diritem(path)
|
return self.__get_diritem(path)
|
||||||
|
|
||||||
@@ -154,9 +164,11 @@ class LocalStorage(StorageBase):
|
|||||||
获取文件或目录,无法确认状态时抛出 StorageQueryError。
|
获取文件或目录,无法确认状态时抛出 StorageQueryError。
|
||||||
Path.exists() 会把部分 errno(如 EBADF/ELOOP)归入「不存在」,
|
Path.exists() 会把部分 errno(如 EBADF/ELOOP)归入「不存在」,
|
||||||
网络/FUSE 挂载抖动时会误判,这里用 stat 显式区分。
|
网络/FUSE 挂载抖动时会误判,这里用 stat 显式区分。
|
||||||
|
挂载完全无响应时代理会超时并抛 FileSystemTimeout(OSError 子类),
|
||||||
|
同样落入下面的分支,转化成调用方能处理的查询失败。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
path.stat()
|
fsproxy.stat(path)
|
||||||
except (FileNotFoundError, NotADirectoryError):
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
return None
|
return None
|
||||||
except OSError as e:
|
except OSError as e:
|
||||||
@@ -182,13 +194,18 @@ class LocalStorage(StorageBase):
|
|||||||
if not fileitem.path:
|
if not fileitem.path:
|
||||||
return False
|
return False
|
||||||
path_obj = Path(fileitem.path)
|
path_obj = Path(fileitem.path)
|
||||||
if not path_obj.exists():
|
|
||||||
return True
|
|
||||||
try:
|
try:
|
||||||
if path_obj.is_file():
|
info = fsproxy.stat(path_obj)
|
||||||
path_obj.unlink()
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
|
return True
|
||||||
|
except OSError as e:
|
||||||
|
logger.error(f"【本地】读取待删除文件状态失败:{e}")
|
||||||
|
return False
|
||||||
|
try:
|
||||||
|
if info["is_file"]:
|
||||||
|
fsproxy.unlink(path_obj)
|
||||||
else:
|
else:
|
||||||
shutil.rmtree(path_obj, ignore_errors=True)
|
fsproxy.rmtree(path_obj)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"【本地】删除文件失败:{e}")
|
logger.error(f"【本地】删除文件失败:{e}")
|
||||||
return False
|
return False
|
||||||
@@ -199,10 +216,10 @@ class LocalStorage(StorageBase):
|
|||||||
重命名文件
|
重命名文件
|
||||||
"""
|
"""
|
||||||
path_obj = Path(fileitem.path)
|
path_obj = Path(fileitem.path)
|
||||||
if not path_obj.exists():
|
|
||||||
return False
|
|
||||||
try:
|
try:
|
||||||
path_obj.rename(path_obj.parent / name)
|
fsproxy.rename(path_obj, path_obj.parent / name)
|
||||||
|
except (FileNotFoundError, NotADirectoryError):
|
||||||
|
return False
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"【本地】重命名文件失败:{e}")
|
logger.error(f"【本地】重命名文件失败:{e}")
|
||||||
return False
|
return False
|
||||||
@@ -214,6 +231,93 @@ class LocalStorage(StorageBase):
|
|||||||
"""
|
"""
|
||||||
return Path(fileitem.path)
|
return Path(fileitem.path)
|
||||||
|
|
||||||
|
# 写入中的临时文件后缀。点开头(隐藏)+ 专用后缀双重保证:即使进程被
|
||||||
|
# SIGKILL、临时文件残留,媒体库也不会把半成品当成媒体收录
|
||||||
|
PARTIAL_SUFFIX = ".mp-partial"
|
||||||
|
# 临时文件被认定为中断残留的时长(秒)。正常失败路径会自行清理,只有被
|
||||||
|
# 强杀才会残留;阈值取得宽松,避免误删仍在写入的大文件
|
||||||
|
PARTIAL_STALE_SECONDS = 24 * 3600
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _partial_path(cls, dest: Path) -> Path:
|
||||||
|
"""
|
||||||
|
生成写入中的临时文件路径。
|
||||||
|
|
||||||
|
必须与目标同目录:os.replace 只有在同一文件系统内才是原子的,放到
|
||||||
|
/tmp 之类的地方会退化成一次完整拷贝,原子性荡然无存。带 PID 是为了
|
||||||
|
避免多进程同时写同一目标时互相踩踏。
|
||||||
|
:param dest: 目标文件路径
|
||||||
|
:return: 临时文件路径
|
||||||
|
"""
|
||||||
|
return dest.parent / f".{dest.name}.{os.getpid()}{cls.PARTIAL_SUFFIX}"
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _cleanup_stale_partials(cls, directory: Path):
|
||||||
|
"""
|
||||||
|
清理目录下中断残留的临时文件。
|
||||||
|
|
||||||
|
只做局部清理而不是全库扫描:在网络挂载上遍历整个媒体库代价不可接受,
|
||||||
|
而残留只可能出现在曾经写入过的目录里,因此每次写入时顺带清理即可。
|
||||||
|
本方法是尽力而为的旁路操作,任何失败都不影响主流程。
|
||||||
|
:param directory: 目标目录
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
threshold = time.time() - cls.PARTIAL_STALE_SECONDS
|
||||||
|
for item in directory.glob(f"*{cls.PARTIAL_SUFFIX}"):
|
||||||
|
try:
|
||||||
|
if item.stat().st_mtime < threshold:
|
||||||
|
item.unlink()
|
||||||
|
logger.info(f"【本地】已清理中断残留的临时文件:{item}")
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
except Exception as err:
|
||||||
|
logger.debug(f"【本地】清理临时文件失败:{directory} - {err}")
|
||||||
|
|
||||||
|
def _write_atomically(self, src: Path, dest: Path) -> bool:
|
||||||
|
"""
|
||||||
|
以「写临时名 → os.replace」的方式把源文件内容落到目标。
|
||||||
|
|
||||||
|
直接写目标路径的话,进程被杀(OOM、重启、宿主断电、SIGKILL)会在媒体库
|
||||||
|
里留下一个**叫最终文件名的半截文件**:媒体库会把它扫进去,后续的
|
||||||
|
「目标已存在」判断也会把它当成完成品。os.replace 在同目录内由内核保证
|
||||||
|
原子性,因此目标要么完整存在,要么根本不存在。
|
||||||
|
:param src: 源文件路径
|
||||||
|
:param dest: 目标文件路径
|
||||||
|
:return: 是否成功
|
||||||
|
"""
|
||||||
|
self._cleanup_stale_partials(dest.parent)
|
||||||
|
partial = self._partial_path(dest)
|
||||||
|
# 进度只在需要展示时才回调 UI,但代理内部始终按固定间隔上报——那是判定
|
||||||
|
# 「传输是否还在推进」的心跳,不能因为不展示进度就关掉
|
||||||
|
progress_callback = (
|
||||||
|
transfer_process(src.as_posix())
|
||||||
|
if self.__should_show_progress(src, dest) else None
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
copied = fsproxy.copy(
|
||||||
|
src, partial,
|
||||||
|
progress_cb=progress_callback,
|
||||||
|
cancel_cb=lambda: global_vars.is_transfer_stopped(src.as_posix()),
|
||||||
|
chunk_size=self.chunk_size,
|
||||||
|
)
|
||||||
|
if not copied:
|
||||||
|
logger.info(f"【本地】{src} 复制未完成")
|
||||||
|
return False
|
||||||
|
os.replace(partial, dest)
|
||||||
|
return True
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"【本地】复制文件失败:{err}")
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
if progress_callback:
|
||||||
|
progress_callback(100)
|
||||||
|
# 失败路径留下的临时文件就地清掉;成功时 replace 已经把它移走
|
||||||
|
try:
|
||||||
|
if partial.exists():
|
||||||
|
partial.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _copy_with_target_permissions(src: Path, dest: Path) -> Path:
|
def _copy_with_target_permissions(src: Path, dest: Path) -> Path:
|
||||||
"""
|
"""
|
||||||
@@ -276,12 +380,13 @@ class LocalStorage(StorageBase):
|
|||||||
try:
|
try:
|
||||||
dir_path = Path(fileitem.path)
|
dir_path = Path(fileitem.path)
|
||||||
target_path = dir_path / (new_name or path.name)
|
target_path = dir_path / (new_name or path.name)
|
||||||
if self._copy_with_progress(path, target_path):
|
# 先原子地把内容落到目标,确认完整之后才删源
|
||||||
|
if self._write_atomically(path, target_path):
|
||||||
# 上传删除源文件
|
# 上传删除源文件
|
||||||
path.unlink()
|
path.unlink()
|
||||||
return self.get_item(target_path)
|
return self.get_item(target_path)
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
logger.error(f"【本地】移动文件失败:{err}")
|
logger.error(f"【本地】上传文件失败:{err}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -304,18 +409,7 @@ class LocalStorage(StorageBase):
|
|||||||
"""
|
"""
|
||||||
复制文件(带进度)
|
复制文件(带进度)
|
||||||
"""
|
"""
|
||||||
try:
|
return self._write_atomically(Path(fileitem.path), path / new_name)
|
||||||
src = Path(fileitem.path)
|
|
||||||
dest = path / new_name
|
|
||||||
if self.__should_show_progress(src, dest):
|
|
||||||
if self._copy_with_progress(src, dest):
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
self._copy_with_target_permissions(src, dest)
|
|
||||||
return True
|
|
||||||
except Exception as err:
|
|
||||||
logger.error(f"【本地】复制文件失败:{err}")
|
|
||||||
return False
|
|
||||||
|
|
||||||
def move(
|
def move(
|
||||||
self,
|
self,
|
||||||
@@ -326,23 +420,28 @@ class LocalStorage(StorageBase):
|
|||||||
"""
|
"""
|
||||||
移动文件(带进度)
|
移动文件(带进度)
|
||||||
"""
|
"""
|
||||||
|
src = Path(fileitem.path)
|
||||||
|
dest = path / new_name
|
||||||
|
if src == dest:
|
||||||
|
# 目标和源文件相同,直接返回成功,不做任何操作
|
||||||
|
return True
|
||||||
try:
|
try:
|
||||||
src = Path(fileitem.path)
|
# 同一文件系统内 rename 是原子操作:中断后要么完全成功、要么完全
|
||||||
dest = path / new_name
|
# 没发生,既不需要临时文件也不会留下半成品。直接尝试而不预先比较
|
||||||
if src == dest:
|
# st_dev,省掉挂载上的两次 stat——跨设备会以 EXDEV 失败并落到下面
|
||||||
# 目标和源文件相同,直接返回成功,不做任何操作
|
os.replace(src, dest)
|
||||||
return True
|
return True
|
||||||
if self.__should_show_progress(src, dest):
|
except OSError as err:
|
||||||
if self._copy_with_progress(src, dest):
|
logger.debug(f"【本地】直接移动未成功,降级为复制:{src} -> {dest} - {err}")
|
||||||
# 复制成功删除源文件
|
# 跨文件系统:先原子地把内容落到目标,确认完整之后才删源。
|
||||||
src.unlink()
|
# 顺序不能反——先删源再失败就是永久丢件
|
||||||
return True
|
if not self._write_atomically(src, dest):
|
||||||
else:
|
return False
|
||||||
shutil.move(src, dest, copy_function=self._copy_with_target_permissions)
|
try:
|
||||||
return True
|
src.unlink()
|
||||||
except Exception as err:
|
except OSError as err:
|
||||||
logger.error(f"【本地】移动文件失败:{err}")
|
logger.warn(f"【本地】移动已完成但删除源文件失败:{src} - {err}")
|
||||||
return False
|
return True
|
||||||
|
|
||||||
def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool:
|
def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool:
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ from app import schemas
|
|||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.modules.filemanager.storages import StorageBase, transfer_process
|
from app.modules.filemanager.storages import StorageBase, transfer_process
|
||||||
|
from app.schemas.exception import StorageQueryError
|
||||||
from app.schemas.types import StorageSchema
|
from app.schemas.types import StorageSchema
|
||||||
from app.utils.string import StringUtils
|
from app.utils.string import StringUtils
|
||||||
from app.utils.system import SystemUtils
|
from app.utils.system import SystemUtils
|
||||||
@@ -299,6 +300,38 @@ class Rclone(StorageBase):
|
|||||||
logger.debug(f"【rclone】获取文件项失败:{err}")
|
logger.debug(f"【rclone】获取文件项失败:{err}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||||
|
"""
|
||||||
|
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||||
|
rclone 用退出码 3/4 表示目录/文件不存在,其余非零退出无法区分
|
||||||
|
「不存在」与「查询失败」,必须保守失败以免覆盖保护被绕过。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
ret = subprocess.run(
|
||||||
|
[
|
||||||
|
'rclone', 'lsjson',
|
||||||
|
f'MP:{path.parent}'
|
||||||
|
],
|
||||||
|
capture_output=True,
|
||||||
|
startupinfo=self.__get_hidden_shell()
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {err}") from err
|
||||||
|
if ret.returncode in (3, 4):
|
||||||
|
# 目录或文件不存在,是确定结果
|
||||||
|
return None
|
||||||
|
if ret.returncode != 0:
|
||||||
|
errmsg = (ret.stderr or b"").decode(errors="ignore").strip()
|
||||||
|
raise StorageQueryError(f"【rclone】查询文件项失败: {path} - {errmsg}")
|
||||||
|
try:
|
||||||
|
items = json.loads(ret.stdout)
|
||||||
|
except Exception as err:
|
||||||
|
raise StorageQueryError(f"【rclone】解析查询结果失败: {path} - {err}") from err
|
||||||
|
for item in items:
|
||||||
|
if item.get("Name") == path.name:
|
||||||
|
return self.__get_rcloneitem(item, parent=str(path.parent) + "/")
|
||||||
|
return None
|
||||||
|
|
||||||
def delete(self, fileitem: schemas.FileItem) -> bool:
|
def delete(self, fileitem: schemas.FileItem) -> bool:
|
||||||
"""
|
"""
|
||||||
删除文件
|
删除文件
|
||||||
|
|||||||
@@ -1,3 +1,4 @@
|
|||||||
|
import errno
|
||||||
import threading
|
import threading
|
||||||
import time
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -16,6 +17,7 @@ from app.core.config import settings, global_vars
|
|||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.modules.filemanager import StorageBase
|
from app.modules.filemanager import StorageBase
|
||||||
from app.modules.filemanager.storages import transfer_process
|
from app.modules.filemanager.storages import transfer_process
|
||||||
|
from app.schemas.exception import StorageQueryError
|
||||||
from app.schemas.types import StorageSchema
|
from app.schemas.types import StorageSchema
|
||||||
from app.utils.singleton import WeakSingleton
|
from app.utils.singleton import WeakSingleton
|
||||||
|
|
||||||
@@ -163,7 +165,8 @@ class SMB(StorageBase, metaclass=WeakSingleton):
|
|||||||
|
|
||||||
# 构建完整的SMB路径
|
# 构建完整的SMB路径
|
||||||
if path_str:
|
if path_str:
|
||||||
return f"{self._server_path}\\{path_str.replace('/', '\\')}"
|
normalized_path = path_str.replace("/", "\\")
|
||||||
|
return f"{self._server_path}\\{normalized_path}"
|
||||||
else:
|
else:
|
||||||
return self._server_path
|
return self._server_path
|
||||||
|
|
||||||
@@ -379,6 +382,39 @@ class SMB(StorageBase, metaclass=WeakSingleton):
|
|||||||
logger.debug(f"【SMB】获取文件项失败: {e}")
|
logger.debug(f"【SMB】获取文件项失败: {e}")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
def get_item_strict(self, path: Path) -> Optional[schemas.FileItem]:
|
||||||
|
"""
|
||||||
|
获取文件或目录,确认不存在返回None;无法确认状态时抛出 StorageQueryError。
|
||||||
|
只有 ENOENT/ENOTDIR 才是「确认不存在」,连接中断、认证失败等都无法确认
|
||||||
|
目标状态,必须保守失败以免覆盖保护被绕过。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
self._check_connection()
|
||||||
|
|
||||||
|
# 处理根目录
|
||||||
|
if str(path) == "/":
|
||||||
|
return schemas.FileItem(
|
||||||
|
storage=self.schema.value,
|
||||||
|
type="dir",
|
||||||
|
path="/",
|
||||||
|
name="",
|
||||||
|
basename="",
|
||||||
|
modify_time=int(time.time()),
|
||||||
|
)
|
||||||
|
|
||||||
|
smb_path = self._normalize_path(str(path).rstrip("/"))
|
||||||
|
try:
|
||||||
|
stat_result = smbclient.stat(smb_path)
|
||||||
|
except OSError as err:
|
||||||
|
if err.errno in (errno.ENOENT, errno.ENOTDIR):
|
||||||
|
return None
|
||||||
|
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {err}") from err
|
||||||
|
return self._create_fileitem(stat_result, smb_path, Path(path).name)
|
||||||
|
except StorageQueryError:
|
||||||
|
raise
|
||||||
|
except Exception as e:
|
||||||
|
raise StorageQueryError(f"【SMB】查询文件项失败: {path} - {e}") from e
|
||||||
|
|
||||||
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
def detail(self, fileitem: schemas.FileItem) -> Optional[schemas.FileItem]:
|
||||||
"""
|
"""
|
||||||
获取文件详情
|
获取文件详情
|
||||||
|
|||||||
@@ -545,6 +545,7 @@ class TransHandler:
|
|||||||
fail_list=[fileitem.path],
|
fail_list=[fileitem.path],
|
||||||
transfer_type=transfer_type,
|
transfer_type=transfer_type,
|
||||||
need_notify=need_notify,
|
need_notify=need_notify,
|
||||||
|
overwrite_skipped=True,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
elif overwrite_mode == "always":
|
elif overwrite_mode == "always":
|
||||||
@@ -571,6 +572,7 @@ class TransHandler:
|
|||||||
fail_list=[fileitem.path],
|
fail_list=[fileitem.path],
|
||||||
transfer_type=transfer_type,
|
transfer_type=transfer_type,
|
||||||
need_notify=need_notify,
|
need_notify=need_notify,
|
||||||
|
overwrite_skipped=True,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
else:
|
else:
|
||||||
@@ -614,6 +616,7 @@ class TransHandler:
|
|||||||
fail_list=[fileitem.path],
|
fail_list=[fileitem.path],
|
||||||
transfer_type=transfer_type,
|
transfer_type=transfer_type,
|
||||||
need_notify=need_notify,
|
need_notify=need_notify,
|
||||||
|
overwrite_skipped=True,
|
||||||
)
|
)
|
||||||
return result
|
return result
|
||||||
elif overwrite_mode == "latest":
|
elif overwrite_mode == "latest":
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ from app.modules.themoviedb.category import CategoryHelper
|
|||||||
from app.modules.themoviedb.scraper import TmdbScraper
|
from app.modules.themoviedb.scraper import TmdbScraper
|
||||||
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||||
from app.modules.themoviedb.tmdbapi import TmdbApi
|
from app.modules.themoviedb.tmdbapi import TmdbApi
|
||||||
|
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbConnectionError
|
||||||
from app.schemas.category import CategoryConfig
|
from app.schemas.category import CategoryConfig
|
||||||
from app.schemas.types import (
|
from app.schemas.types import (
|
||||||
MediaImageType,
|
MediaImageType,
|
||||||
@@ -193,16 +194,42 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
media.season = meta.begin_season
|
media.season = meta.begin_season
|
||||||
return medias
|
return medias
|
||||||
|
|
||||||
|
def _safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
|
||||||
|
"""
|
||||||
|
查询指定类型的媒体详情,将"确认TMDB连接失败"与"确认查无此项"区分开。
|
||||||
|
|
||||||
|
:param mtype: 媒体类型:电影或电视剧
|
||||||
|
:param tmdbid: TMDB的ID
|
||||||
|
:return: (媒体信息或None, 本次查询是否因TMDB连接失败而没有得到确定结果)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
|
||||||
|
except TMDbConnectionError:
|
||||||
|
return None, True
|
||||||
|
|
||||||
|
async def _async_safe_get_info_by_type(self, mtype: MediaType, tmdbid: int) -> Tuple[Optional[dict], bool]:
|
||||||
|
"""
|
||||||
|
查询指定类型的媒体详情,将"确认TMDB连接失败"与"确认查无此项"区分开(异步版本)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True), False
|
||||||
|
except TMDbConnectionError:
|
||||||
|
return None, True
|
||||||
|
|
||||||
def _get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
def _get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
||||||
meta: Optional[MetaBase]) -> Optional[dict]:
|
meta: Optional[MetaBase]) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧
|
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧
|
||||||
|
|
||||||
|
:raises TMDbConnectionError: 电影、电视剧两路查询都没有得到确定结果,且至少一路
|
||||||
|
是因TMDB连接失败导致的,此时不能断言"条目不存在",交由上层报网络故障
|
||||||
"""
|
"""
|
||||||
if mtype:
|
if mtype:
|
||||||
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid)
|
return self.tmdb.get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
|
||||||
# 类型未知,分别查询电影和电视剧
|
# 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
|
||||||
info_tv = self.tmdb.get_info(mtype=MediaType.TV, tmdbid=tmdbid)
|
# 避免一路瞬时抖动掩盖另一路已经得到的确定结果
|
||||||
info_movie = self.tmdb.get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid)
|
info_tv, tv_conn_error = self._safe_get_info_by_type(MediaType.TV, tmdbid)
|
||||||
|
info_movie, movie_conn_error = self._safe_get_info_by_type(MediaType.MOVIE, tmdbid)
|
||||||
if info_tv and info_movie:
|
if info_tv and info_movie:
|
||||||
# 同时存在,尝试通过元数据消歧
|
# 同时存在,尝试通过元数据消歧
|
||||||
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
||||||
@@ -210,18 +237,26 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
return result
|
return result
|
||||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||||
return None
|
return None
|
||||||
return info_tv or info_movie or None
|
if info_tv or info_movie:
|
||||||
|
return info_tv or info_movie
|
||||||
|
if tv_conn_error or movie_conn_error:
|
||||||
|
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
|
||||||
|
return None
|
||||||
|
|
||||||
async def _async_get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
async def _async_get_info_by_tmdbid(self, tmdbid: int, mtype: Optional[MediaType],
|
||||||
meta: Optional[MetaBase]) -> Optional[dict]:
|
meta: Optional[MetaBase]) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧(异步版本)
|
根据tmdbid查询媒体信息,当类型未知且同时存在电影和电视剧时,通过元数据消歧(异步版本)
|
||||||
|
|
||||||
|
:raises TMDbConnectionError: 电影、电视剧两路查询都没有得到确定结果,且至少一路
|
||||||
|
是因TMDB连接失败导致的,此时不能断言"条目不存在",交由上层报网络故障
|
||||||
"""
|
"""
|
||||||
if mtype:
|
if mtype:
|
||||||
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid)
|
return await self.tmdb.async_get_info(mtype=mtype, tmdbid=tmdbid, raise_on_connection_error=True)
|
||||||
# 类型未知,分别查询电影和电视剧
|
# 类型未知,分别查询电影和电视剧;每一路的连接失败要单独识别,
|
||||||
info_tv = await self.tmdb.async_get_info(mtype=MediaType.TV, tmdbid=tmdbid)
|
# 避免一路瞬时抖动掩盖另一路已经得到的确定结果
|
||||||
info_movie = await self.tmdb.async_get_info(mtype=MediaType.MOVIE, tmdbid=tmdbid)
|
info_tv, tv_conn_error = await self._async_safe_get_info_by_type(MediaType.TV, tmdbid)
|
||||||
|
info_movie, movie_conn_error = await self._async_safe_get_info_by_type(MediaType.MOVIE, tmdbid)
|
||||||
if info_tv and info_movie:
|
if info_tv and info_movie:
|
||||||
# 同时存在,尝试通过元数据消歧
|
# 同时存在,尝试通过元数据消歧
|
||||||
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
result = self._disambiguate_by_meta(info_tv, info_movie, meta)
|
||||||
@@ -229,7 +264,11 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
return result
|
return result
|
||||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||||
return None
|
return None
|
||||||
return info_tv or info_movie or None
|
if info_tv or info_movie:
|
||||||
|
return info_tv or info_movie
|
||||||
|
if tv_conn_error or movie_conn_error:
|
||||||
|
raise TMDbConnectionError(f"连接TheMovieDb失败,无法确认tmdb_id:{tmdbid} 的媒体类型")
|
||||||
|
return None
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _disambiguate_by_meta(info_tv: dict, info_movie: dict,
|
def _disambiguate_by_meta(info_tv: dict, info_movie: dict,
|
||||||
@@ -525,10 +564,15 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
# 识别匹配
|
# 识别匹配
|
||||||
if not cache_info or not cache:
|
if not cache_info or not cache:
|
||||||
info = None
|
info = None
|
||||||
|
connection_error = False
|
||||||
# 缓存没有或者强制不使用缓存
|
# 缓存没有或者强制不使用缓存
|
||||||
if tmdbid:
|
if tmdbid:
|
||||||
# 直接查询详情,支持同ID电影/电视剧消歧
|
# 直接查询详情,支持同ID电影/电视剧消歧
|
||||||
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
try:
|
||||||
|
info = self._get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(f"tmdb_id:{tmdbid} {err}")
|
||||||
|
connection_error = True
|
||||||
if not info and meta and not tmdbid:
|
if not info and meta and not tmdbid:
|
||||||
# 准备搜索名称
|
# 准备搜索名称
|
||||||
names = self._prepare_search_names(meta)
|
names = self._prepare_search_names(meta)
|
||||||
@@ -542,7 +586,11 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
info = self.tmdb.get_info(mtype=info.get("media_type"),
|
info = self.tmdb.get_info(mtype=info.get("media_type"),
|
||||||
tmdbid=info.get("id"))
|
tmdbid=info.get("id"))
|
||||||
elif not info:
|
elif not info:
|
||||||
if tmdbid:
|
if connection_error:
|
||||||
|
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
|
||||||
|
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
|
||||||
|
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
|
||||||
|
elif tmdbid:
|
||||||
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
||||||
else:
|
else:
|
||||||
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
||||||
@@ -624,10 +672,15 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
# 识别匹配
|
# 识别匹配
|
||||||
if not cache_info or not cache:
|
if not cache_info or not cache:
|
||||||
info = None
|
info = None
|
||||||
|
connection_error = False
|
||||||
# 缓存没有或者强制不使用缓存
|
# 缓存没有或者强制不使用缓存
|
||||||
if tmdbid:
|
if tmdbid:
|
||||||
# 直接查询详情,支持同ID电影/电视剧消歧
|
# 直接查询详情,支持同ID电影/电视剧消歧
|
||||||
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
try:
|
||||||
|
info = await self._async_get_info_by_tmdbid(tmdbid=tmdbid, mtype=mtype, meta=meta)
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(f"tmdb_id:{tmdbid} {err}")
|
||||||
|
connection_error = True
|
||||||
if not info and meta and not tmdbid:
|
if not info and meta and not tmdbid:
|
||||||
# 准备搜索名称
|
# 准备搜索名称
|
||||||
names = self._prepare_search_names(meta)
|
names = self._prepare_search_names(meta)
|
||||||
@@ -641,7 +694,11 @@ class TheMovieDbModule(_ModuleBase):
|
|||||||
info = await self.tmdb.async_get_info(mtype=info.get("media_type"),
|
info = await self.tmdb.async_get_info(mtype=info.get("media_type"),
|
||||||
tmdbid=info.get("id"))
|
tmdbid=info.get("id"))
|
||||||
elif not info:
|
elif not info:
|
||||||
if tmdbid:
|
if connection_error:
|
||||||
|
# 网络故障与"条目不存在"是完全不同的两类问题,不能用同一句文案掩盖,
|
||||||
|
# 否则用户无从判断该等网络恢复还是该确认条目本身是否存在
|
||||||
|
logger.error(f"tmdb_id:{tmdbid} 连接TheMovieDb失败,无法完成识别,请检查网络连接后重试")
|
||||||
|
elif tmdbid:
|
||||||
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
logger.warn(f"tmdb_id:{tmdbid} 无法确定媒体类型,识别失败")
|
||||||
else:
|
else:
|
||||||
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
logger.error("识别媒体信息时未提供元数据或唯一且有效的tmdbid")
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import traceback
|
|||||||
from math import ceil
|
from math import ceil
|
||||||
from threading import RLock
|
from threading import RLock
|
||||||
from time import time
|
from time import time
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
from app.core.cache import FileCache, TTLCache
|
from app.core.cache import FileCache, TTLCache
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
@@ -144,6 +145,30 @@ class TmdbCache(metaclass=WeakSingleton):
|
|||||||
media_id = meta.media_id if meta.media_source == MediaSource.TMDB else None
|
media_id = meta.media_id if meta.media_source == MediaSource.TMDB else None
|
||||||
return f"[{meta.type.value if meta.type else '未知'}][{settings.TMDB_LOCALE}]{media_id or meta.name}-{meta.year}-{meta.begin_season}"
|
return f"[{meta.type.value if meta.type else '未知'}][{settings.TMDB_LOCALE}]{media_id or meta.name}-{meta.year}-{meta.begin_season}"
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def __is_type_conflicted(meta: MetaBase, media_type: Any, tmdb_id: Any) -> bool:
|
||||||
|
"""
|
||||||
|
判断媒体类型是否与元数据声明的类型冲突。
|
||||||
|
|
||||||
|
只有「元数据判定为电视剧、结果却是电影」才算冲突。反向不算:名称识别在
|
||||||
|
电影分支查不到时会回退到电视剧查询,识别缓存正是用来记住这个纠正结果,
|
||||||
|
一律要求 key 与 value 类型一致会让这类条目每次都被丢弃、反复回源。而电视
|
||||||
|
剧分支恒定写入电视剧类型,`[电视剧]` 键下出现电影只可能来自 tmdbid 消歧
|
||||||
|
或共享识别回填的脏写,会让整季剧集被当成电影反复整理失败。
|
||||||
|
:param meta: 元数据
|
||||||
|
:param media_type: 待校验的媒体类型
|
||||||
|
:param tmdb_id: 对应的 TMDB ID,为空表示负缓存,不带类型信息
|
||||||
|
:return: 是否冲突
|
||||||
|
"""
|
||||||
|
if meta.type != MediaType.TV or not tmdb_id:
|
||||||
|
return False
|
||||||
|
if not isinstance(media_type, MediaType):
|
||||||
|
try:
|
||||||
|
media_type = MediaType(media_type)
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return False
|
||||||
|
return media_type == MediaType.MOVIE
|
||||||
|
|
||||||
def get(self, meta: MetaBase):
|
def get(self, meta: MetaBase):
|
||||||
"""
|
"""
|
||||||
根据KEY值获取缓存值
|
根据KEY值获取缓存值
|
||||||
@@ -154,7 +179,17 @@ class TmdbCache(metaclass=WeakSingleton):
|
|||||||
cache_data = self._cache.get(key)
|
cache_data = self._cache.get(key)
|
||||||
if not cache_data and self._expires_at.pop(key, None) is not None:
|
if not cache_data and self._expires_at.pop(key, None) is not None:
|
||||||
self._dirty = True
|
self._dirty = True
|
||||||
return cache_data or {}
|
if not cache_data or not isinstance(cache_data, dict):
|
||||||
|
return {}
|
||||||
|
if self.__is_type_conflicted(meta, cache_data.get("type"), cache_data.get("id")):
|
||||||
|
# 脏条目不丢弃就会被无限期沿用,正确的识别逻辑永远没有执行机会
|
||||||
|
logger.warn(f"识别缓存类型与元数据冲突,已丢弃并重新识别:{key} -> "
|
||||||
|
f"{cache_data.get('title')}({cache_data.get('type')})")
|
||||||
|
self._cache.delete(key)
|
||||||
|
self._expires_at.pop(key, None)
|
||||||
|
self._dirty = True
|
||||||
|
return {}
|
||||||
|
return cache_data
|
||||||
|
|
||||||
def delete(self, key: str) -> dict:
|
def delete(self, key: str) -> dict:
|
||||||
"""
|
"""
|
||||||
@@ -193,6 +228,11 @@ class TmdbCache(metaclass=WeakSingleton):
|
|||||||
"""
|
"""
|
||||||
key = self.__get_key(meta)
|
key = self.__get_key(meta)
|
||||||
if info:
|
if info:
|
||||||
|
if self.__is_type_conflicted(meta, info.get("media_type"), info.get("id")):
|
||||||
|
# 拒绝写入而不是改写键:识别结果照常返回,只是不把矛盾条目留给下一次
|
||||||
|
logger.warn(f"识别结果类型与元数据冲突,不写入识别缓存:{key} -> "
|
||||||
|
f"{info.get('title')}({info.get('media_type')})")
|
||||||
|
return
|
||||||
# 缓存标题
|
# 缓存标题
|
||||||
cache_title = info.get("title") \
|
cache_title = info.get("title") \
|
||||||
if info.get("media_type") == MediaType.MOVIE else info.get("name")
|
if info.get("media_type") == MediaType.MOVIE else info.get("name")
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ from app.schemas.types import MediaType
|
|||||||
from app.utils.string import StringUtils
|
from app.utils.string import StringUtils
|
||||||
from app.utils.zhconv import convert as zhconv_convert
|
from app.utils.zhconv import convert as zhconv_convert
|
||||||
from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
|
from .tmdbv3api import TMDb, Search, Movie, TV, Season, Episode, Discover, Trending, Person, Collection
|
||||||
from .tmdbv3api.exceptions import TMDbException
|
from .tmdbv3api.exceptions import TMDbException, TMDbConnectionError
|
||||||
|
|
||||||
|
|
||||||
class TmdbApi:
|
class TmdbApi:
|
||||||
@@ -584,11 +584,14 @@ class TmdbApi:
|
|||||||
|
|
||||||
def get_info(self,
|
def get_info(self,
|
||||||
mtype: MediaType,
|
mtype: MediaType,
|
||||||
tmdbid: int) -> dict:
|
tmdbid: int,
|
||||||
|
raise_on_connection_error: bool = False) -> dict:
|
||||||
"""
|
"""
|
||||||
给定TMDB号,查询一条媒体信息
|
给定TMDB号,查询一条媒体信息
|
||||||
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
||||||
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
||||||
|
:param raise_on_connection_error: 为True时,遇到TMDB连接失败(区别于404等业务错误)
|
||||||
|
将抛出TMDbConnectionError而不是吞掉返回None;默认False,与既有调用方行为完全一致
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __get_genre_ids(genres: list) -> list:
|
def __get_genre_ids(genres: list) -> list:
|
||||||
@@ -604,16 +607,16 @@ class TmdbApi:
|
|||||||
|
|
||||||
# 查询TMDB详情
|
# 查询TMDB详情
|
||||||
if mtype == MediaType.MOVIE:
|
if mtype == MediaType.MOVIE:
|
||||||
tmdb_info = self.__get_movie_detail(tmdbid)
|
tmdb_info = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info:
|
if tmdb_info:
|
||||||
tmdb_info['media_type'] = MediaType.MOVIE
|
tmdb_info['media_type'] = MediaType.MOVIE
|
||||||
elif mtype == MediaType.TV:
|
elif mtype == MediaType.TV:
|
||||||
tmdb_info = self.__get_tv_detail(tmdbid)
|
tmdb_info = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info:
|
if tmdb_info:
|
||||||
tmdb_info['media_type'] = MediaType.TV
|
tmdb_info['media_type'] = MediaType.TV
|
||||||
else:
|
else:
|
||||||
tmdb_info_tv = self.__get_tv_detail(tmdbid)
|
tmdb_info_tv = self.__get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
tmdb_info_movie = self.__get_movie_detail(tmdbid)
|
tmdb_info_movie = self.__get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info_tv and tmdb_info_movie:
|
if tmdb_info_tv and tmdb_info_movie:
|
||||||
tmdb_info = None
|
tmdb_info = None
|
||||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||||
@@ -797,10 +800,13 @@ class TmdbApi:
|
|||||||
"alternative_titles,"
|
"alternative_titles,"
|
||||||
"translations,"
|
"translations,"
|
||||||
"release_dates,"
|
"release_dates,"
|
||||||
"external_ids") -> Optional[dict]:
|
"external_ids",
|
||||||
|
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
获取电影的详情
|
获取电影的详情
|
||||||
:param tmdbid: TMDB ID
|
:param tmdbid: TMDB ID
|
||||||
|
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||||
|
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||||
:return: TMDB信息
|
:return: TMDB信息
|
||||||
"""
|
"""
|
||||||
"""
|
"""
|
||||||
@@ -899,6 +905,11 @@ class TmdbApi:
|
|||||||
if tmdbinfo:
|
if tmdbinfo:
|
||||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
||||||
return tmdbinfo or {}
|
return tmdbinfo or {}
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(str(err))
|
||||||
|
if raise_on_connection_error:
|
||||||
|
raise
|
||||||
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
return None
|
return None
|
||||||
@@ -911,10 +922,13 @@ class TmdbApi:
|
|||||||
"translations,"
|
"translations,"
|
||||||
"content_ratings,"
|
"content_ratings,"
|
||||||
"external_ids,"
|
"external_ids,"
|
||||||
"episode_groups") -> Optional[dict]:
|
"episode_groups",
|
||||||
|
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
获取电视剧的详情
|
获取电视剧的详情
|
||||||
:param tmdbid: TMDB ID
|
:param tmdbid: TMDB ID
|
||||||
|
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||||
|
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||||
:return: TMDB信息
|
:return: TMDB信息
|
||||||
"""
|
"""
|
||||||
"""
|
"""
|
||||||
@@ -1084,6 +1098,11 @@ class TmdbApi:
|
|||||||
if tmdbinfo:
|
if tmdbinfo:
|
||||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
||||||
return tmdbinfo or {}
|
return tmdbinfo or {}
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(str(err))
|
||||||
|
if raise_on_connection_error:
|
||||||
|
raise
|
||||||
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
return None
|
return None
|
||||||
@@ -1666,10 +1685,13 @@ class TmdbApi:
|
|||||||
"alternative_titles,"
|
"alternative_titles,"
|
||||||
"translations,"
|
"translations,"
|
||||||
"release_dates,"
|
"release_dates,"
|
||||||
"external_ids") -> Optional[dict]:
|
"external_ids",
|
||||||
|
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
获取电影的详情(异步版本)
|
获取电影的详情(异步版本)
|
||||||
:param tmdbid: TMDB ID
|
:param tmdbid: TMDB ID
|
||||||
|
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||||
|
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||||
:return: TMDB信息
|
:return: TMDB信息
|
||||||
"""
|
"""
|
||||||
if not self.movie:
|
if not self.movie:
|
||||||
@@ -1680,6 +1702,11 @@ class TmdbApi:
|
|||||||
if tmdbinfo:
|
if tmdbinfo:
|
||||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('title')}")
|
||||||
return tmdbinfo or {}
|
return tmdbinfo or {}
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(str(err))
|
||||||
|
if raise_on_connection_error:
|
||||||
|
raise
|
||||||
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
return None
|
return None
|
||||||
@@ -1692,10 +1719,13 @@ class TmdbApi:
|
|||||||
"translations,"
|
"translations,"
|
||||||
"content_ratings,"
|
"content_ratings,"
|
||||||
"external_ids,"
|
"external_ids,"
|
||||||
"episode_groups") -> Optional[dict]:
|
"episode_groups",
|
||||||
|
raise_on_connection_error: bool = False) -> Optional[dict]:
|
||||||
"""
|
"""
|
||||||
获取电视剧的详情(异步版本)
|
获取电视剧的详情(异步版本)
|
||||||
:param tmdbid: TMDB ID
|
:param tmdbid: TMDB ID
|
||||||
|
:param raise_on_connection_error: 为True时TMDB连接失败会抛出TMDbConnectionError,
|
||||||
|
而不是像默认那样吞掉返回None;404等TMDB业务错误不受影响,始终返回None
|
||||||
:return: TMDB信息
|
:return: TMDB信息
|
||||||
"""
|
"""
|
||||||
if not self.tv:
|
if not self.tv:
|
||||||
@@ -1706,6 +1736,11 @@ class TmdbApi:
|
|||||||
if tmdbinfo:
|
if tmdbinfo:
|
||||||
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
logger.debug(f"{tmdbid} 查询结果:{tmdbinfo.get('name')}")
|
||||||
return tmdbinfo or {}
|
return tmdbinfo or {}
|
||||||
|
except TMDbConnectionError as err:
|
||||||
|
logger.error(str(err))
|
||||||
|
if raise_on_connection_error:
|
||||||
|
raise
|
||||||
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(str(e))
|
logger.error(str(e))
|
||||||
return None
|
return None
|
||||||
@@ -1922,11 +1957,14 @@ class TmdbApi:
|
|||||||
|
|
||||||
async def async_get_info(self,
|
async def async_get_info(self,
|
||||||
mtype: MediaType,
|
mtype: MediaType,
|
||||||
tmdbid: int) -> dict:
|
tmdbid: int,
|
||||||
|
raise_on_connection_error: bool = False) -> dict:
|
||||||
"""
|
"""
|
||||||
给定TMDB号,查询一条媒体信息(异步版本)
|
给定TMDB号,查询一条媒体信息(异步版本)
|
||||||
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
:param mtype: 类型:电影、电视剧,为空时都查(此时用不上年份)
|
||||||
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
:param tmdbid: TMDB的ID,有tmdbid时优先使用tmdbid,否则使用年份和标题
|
||||||
|
:param raise_on_connection_error: 为True时,遇到TMDB连接失败(区别于404等业务错误)
|
||||||
|
将抛出TMDbConnectionError而不是吞掉返回None;默认False,与既有调用方行为完全一致
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __get_genre_ids(genres: list) -> list:
|
def __get_genre_ids(genres: list) -> list:
|
||||||
@@ -1942,16 +1980,16 @@ class TmdbApi:
|
|||||||
|
|
||||||
# 查询TMDB详情
|
# 查询TMDB详情
|
||||||
if mtype == MediaType.MOVIE:
|
if mtype == MediaType.MOVIE:
|
||||||
tmdb_info = await self.__async_get_movie_detail(tmdbid)
|
tmdb_info = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info:
|
if tmdb_info:
|
||||||
tmdb_info['media_type'] = MediaType.MOVIE
|
tmdb_info['media_type'] = MediaType.MOVIE
|
||||||
elif mtype == MediaType.TV:
|
elif mtype == MediaType.TV:
|
||||||
tmdb_info = await self.__async_get_tv_detail(tmdbid)
|
tmdb_info = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info:
|
if tmdb_info:
|
||||||
tmdb_info['media_type'] = MediaType.TV
|
tmdb_info['media_type'] = MediaType.TV
|
||||||
else:
|
else:
|
||||||
tmdb_info_tv = await self.__async_get_tv_detail(tmdbid)
|
tmdb_info_tv = await self.__async_get_tv_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
tmdb_info_movie = await self.__async_get_movie_detail(tmdbid)
|
tmdb_info_movie = await self.__async_get_movie_detail(tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
if tmdb_info_tv and tmdb_info_movie:
|
if tmdb_info_tv and tmdb_info_movie:
|
||||||
tmdb_info = None
|
tmdb_info = None
|
||||||
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
logger.warn(f"无法判断tmdb_id:{tmdbid} 是电影还是电视剧")
|
||||||
|
|||||||
@@ -1,2 +1,16 @@
|
|||||||
class TMDbException(Exception):
|
class TMDbException(Exception):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class TMDbConnectionError(TMDbException):
|
||||||
|
"""
|
||||||
|
TMDB连接失败异常。
|
||||||
|
|
||||||
|
仅在确认为传输层/响应格式问题(如底层HTTP请求失败、响应无法解析为JSON)时抛出,
|
||||||
|
与TMDB业务层明确返回的错误(如404条目不存在、参数错误等,仍抛出普通TMDbException)
|
||||||
|
区分开,便于上层区分"网络故障,请重试"与"条目确实不存在"两类完全不同的处理与文案。
|
||||||
|
|
||||||
|
继承自TMDbException,因此现有 `except TMDbException` 代码路径无需修改即可
|
||||||
|
继续捕获本异常,保持向后兼容。
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|||||||
@@ -12,10 +12,30 @@ import requests.exceptions
|
|||||||
from app.core.cache import cached, fresh, async_fresh
|
from app.core.cache import cached, fresh, async_fresh
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.utils.http import RequestUtils, AsyncRequestUtils
|
from app.utils.http import RequestUtils, AsyncRequestUtils
|
||||||
from .exceptions import TMDbException
|
from .exceptions import TMDbException, TMDbConnectionError
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# 单次重试前的退避等待时间(秒)。NAS+FUSE网盘等环境下TMDB连接的失败大多是数秒内
|
||||||
|
# 可自愈的瞬时抖动,零间隔重试(或异步完全不重试)基本无法穿越这类抖动窗口;
|
||||||
|
# 识别链路是同步阻塞调用,1-3秒的等待可接受,超出3秒则会让识别耗时明显变长,
|
||||||
|
# 故取区间内的经验值。
|
||||||
|
RETRY_BACKOFF_SECONDS = 2
|
||||||
|
|
||||||
|
|
||||||
|
def _is_business_failure_snapshot(snapshot) -> bool:
|
||||||
|
"""
|
||||||
|
判断响应快照是否为TMDB业务失败(success=false,如404/限流/服务端错误的合法JSON)。
|
||||||
|
|
||||||
|
这类响应若入缓存会把瞬时失败固化整个TTL周期(如12小时),期间同key请求
|
||||||
|
直接命中失败快照;跳过缓存让下次请求重新确认。真正的负缓存(条目确认
|
||||||
|
不存在)由上层 TmdbCache 负责,request 层不做失败记忆。
|
||||||
|
"""
|
||||||
|
if not isinstance(snapshot, dict):
|
||||||
|
return False
|
||||||
|
json_data = snapshot.get("json")
|
||||||
|
return isinstance(json_data, dict) and json_data.get("success") is False
|
||||||
|
|
||||||
|
|
||||||
class TMDb(object):
|
class TMDb(object):
|
||||||
_RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__"
|
_RESPONSE_SNAPSHOT_MARKER = "__mp_tmdb_response_snapshot__"
|
||||||
@@ -136,15 +156,21 @@ class TMDb(object):
|
|||||||
def wait_on_rate_limit(self, wait_on_rate_limit):
|
def wait_on_rate_limit(self, wait_on_rate_limit):
|
||||||
self._wait_on_rate_limit = bool(wait_on_rate_limit)
|
self._wait_on_rate_limit = bool(wait_on_rate_limit)
|
||||||
|
|
||||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True)
|
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||||
|
skip_if=_is_business_failure_snapshot)
|
||||||
def request(self, method, url, data, json, **kwargs):
|
def request(self, method, url, data, json, **kwargs):
|
||||||
req = self._request_once(method, url, data, json)
|
req = self._request_once(method, url, data, json)
|
||||||
if req is None and method == "GET" and self._owns_session:
|
if req is None and method == "GET" and self._owns_session:
|
||||||
logger.debug("TMDB同步请求失败,重建会话后重试一次")
|
logger.debug(f"TMDB同步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重建会话重试一次")
|
||||||
|
# 同步阻塞识别链线程;1-3秒的退避等待可接受,能显著提升对瞬时抖动的容错,
|
||||||
|
# 详见模块级常量 RETRY_BACKOFF_SECONDS 的说明。
|
||||||
|
time.sleep(RETRY_BACKOFF_SECONDS)
|
||||||
self._reset_owned_session()
|
self._reset_owned_session()
|
||||||
req = self._request_once(method, url, data, json)
|
req = self._request_once(method, url, data, json)
|
||||||
if req is None:
|
if req is None:
|
||||||
raise TMDbException("无法连接TheMovieDb,请检查网络连接!")
|
# 抛出更具体的连接异常子类,供上层(如TMDB详情查询)区分"网络故障"
|
||||||
|
# 与"TMDB业务层明确返回的错误"(如404条目不存在),两者不能混为一谈。
|
||||||
|
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
return self._snapshot_response(req)
|
return self._snapshot_response(req)
|
||||||
|
|
||||||
def _request_once(self, method, url, data, json):
|
def _request_once(self, method, url, data, json):
|
||||||
@@ -155,16 +181,28 @@ class TMDb(object):
|
|||||||
return self._req.get_res(url, params=data, json=json)
|
return self._req.get_res(url, params=data, json=json)
|
||||||
return self._req.post_res(url, data=data, json=json)
|
return self._req.post_res(url, data=data, json=json)
|
||||||
|
|
||||||
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True)
|
@cached(maxsize=settings.CONF.tmdb, ttl=settings.CONF.meta, skip_none=True,
|
||||||
|
skip_if=_is_business_failure_snapshot)
|
||||||
async def async_request(self, method, url, data, json, **kwargs):
|
async def async_request(self, method, url, data, json, **kwargs):
|
||||||
if method == "GET":
|
req = await self._async_request_once(method, url, data, json)
|
||||||
req = await self._async_req.get_res(url, params=data, json=json)
|
|
||||||
else:
|
|
||||||
req = await self._async_req.post_res(url, data=data, json=json)
|
|
||||||
if req is None:
|
if req is None:
|
||||||
raise TMDbException("无法连接TheMovieDb,请检查网络连接!")
|
logger.debug(f"TMDB异步请求失败,等待{RETRY_BACKOFF_SECONDS}秒后重试一次")
|
||||||
|
# 异步会话(AsyncRequestUtils)不像同步会话那样支持按需重建,
|
||||||
|
# 这里退化为原会话上的纯重试,同样以退避等待应对瞬时抖动。
|
||||||
|
await asyncio.sleep(RETRY_BACKOFF_SECONDS)
|
||||||
|
req = await self._async_request_once(method, url, data, json)
|
||||||
|
if req is None:
|
||||||
|
raise TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
return self._snapshot_response(req)
|
return self._snapshot_response(req)
|
||||||
|
|
||||||
|
async def _async_request_once(self, method, url, data, json):
|
||||||
|
"""
|
||||||
|
执行一次TMDB异步请求,调用方负责决定是否重试。
|
||||||
|
"""
|
||||||
|
if method == "GET":
|
||||||
|
return await self._async_req.get_res(url, params=data, json=json)
|
||||||
|
return await self._async_req.post_res(url, data=data, json=json)
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def _snapshot_response(cls, response):
|
def _snapshot_response(cls, response):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -6,6 +6,7 @@
|
|||||||
- snapshot.py 远程快照存取与比对
|
- snapshot.py 远程快照存取与比对
|
||||||
- dispatcher.py 监控事件到整理链的分发
|
- dispatcher.py 监控事件到整理链的分发
|
||||||
- poller.py 远程目录轮询监控
|
- poller.py 远程目录轮询监控
|
||||||
|
- recovery.py 触碰挂载的恢复动作的可放弃执行单元(block 型故障隔离)
|
||||||
- monitor.py Monitor 门面:装配、生命周期与健康检查
|
- monitor.py Monitor 门面:装配、生命周期与健康检查
|
||||||
"""
|
"""
|
||||||
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
|
from app.monitor.watcher import DirectoryChangeEvent, LocalDirectoryWatcher
|
||||||
|
|||||||
+229
-64
@@ -2,13 +2,16 @@ import re
|
|||||||
import traceback
|
import traceback
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from threading import Lock
|
from threading import Lock
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from app.chain.transfer import TransferChain
|
from app.chain.transfer import TransferChain
|
||||||
from app.core.cache import TTLCache
|
from app.core.cache import TTLCache
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.db.transferhistory_oper import TransferHistoryOper
|
from app.db.transferhistory_oper import TransferHistoryOper
|
||||||
from app.helper.directory import DirectoryHelper
|
from app.helper.directory import DirectoryHelper
|
||||||
|
from app.helper.transferhistory import (HistoryGateAction, describe_history_gate,
|
||||||
|
evaluate_history_gate, is_skip_action,
|
||||||
|
max_failed_retries, resolve_history)
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.schemas import FileItem
|
from app.schemas import FileItem
|
||||||
from app.schemas.types import MediaType
|
from app.schemas.types import MediaType
|
||||||
@@ -80,17 +83,57 @@ class TransferDispatcher:
|
|||||||
return f"{event_path.as_posix()}/"
|
return f"{event_path.as_posix()}/"
|
||||||
return event_path.as_posix()
|
return event_path.as_posix()
|
||||||
|
|
||||||
@staticmethod
|
@classmethod
|
||||||
def _has_transfer_history(storage: str, src_path: str) -> Optional[bool]:
|
def _should_skip_by_history(cls, storage: str, src_path: str,
|
||||||
|
file_size: Optional[float] = None,
|
||||||
|
file_modify_time: Optional[float] = None,
|
||||||
|
fileid: Optional[str] = None) -> Optional[bool]:
|
||||||
"""
|
"""
|
||||||
判断源文件是否已经存在整理记录。
|
依据整理历史判断本次是否跳过整理。
|
||||||
:return: True/False 查询成功,None 查询失败
|
|
||||||
|
判定策略由 app/helper/transferhistory.py 统一提供,整理链的计划整理段使用
|
||||||
|
同一套判定,避免此处放行的文件在下游被另一套「存在记录即拦」的策略收回。
|
||||||
|
:param storage: 存储
|
||||||
|
:param src_path: 整理记录使用的源路径
|
||||||
|
:param file_size: 当前文件大小,蓝光目录等场景可能为 None
|
||||||
|
:param file_modify_time: 当前文件修改时间
|
||||||
|
:param fileid: 当前文件唯一标识
|
||||||
|
:return: True 跳过整理,False 放行整理,None 查询失败
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
return bool(TransferHistoryOper().get_by_src(src_path, storage=storage))
|
history = resolve_history(src_path, storage=storage,
|
||||||
|
transfer_history_oper=TransferHistoryOper())
|
||||||
except Exception as err:
|
except Exception as err:
|
||||||
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
logger.error(f"查询整理历史失败: {src_path} - {err}")
|
||||||
return None
|
return None
|
||||||
|
action = evaluate_history_gate(
|
||||||
|
history,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
history_description = describe_history_gate(
|
||||||
|
history,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if action == HistoryGateAction.PASS_FAILED:
|
||||||
|
logger.debug(f"上次整理失败({history_description}),"
|
||||||
|
f"本次重新送入整理链: {src_path}")
|
||||||
|
elif action == HistoryGateAction.PASS_FAILED_VERSION_CHANGED:
|
||||||
|
logger.info(f"上次整理失败但文件版本已变化({history_description}),"
|
||||||
|
f"本次重新送入整理链: {src_path}")
|
||||||
|
elif action == HistoryGateAction.PASS_SIZE_CHANGED:
|
||||||
|
logger.info(f"已整理过但文件版本已变化({history_description}),"
|
||||||
|
f"重新送入整理链: {src_path}")
|
||||||
|
elif action == HistoryGateAction.SKIP_RETRY_EXHAUSTED:
|
||||||
|
# 放弃自动重试意味着该文件需要人介入,不能只留 debug 日志重蹈静默漏件的覆辙
|
||||||
|
logger.warn(f"整理连续失败 {max_failed_retries()} 次已达上限,不再自动重试,"
|
||||||
|
f"请手动整理或删除整理记录: {src_path}")
|
||||||
|
elif action == HistoryGateAction.SKIP:
|
||||||
|
logger.debug(f"已整理过且文件未变化,跳过: {src_path}")
|
||||||
|
return is_skip_action(action)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _pending_key(storage: str, event_path: Path) -> str:
|
def _pending_key(storage: str, event_path: Path) -> str:
|
||||||
@@ -132,12 +175,18 @@ class TransferDispatcher:
|
|||||||
)
|
)
|
||||||
return None
|
return None
|
||||||
|
|
||||||
def _register_pending(self, storage: str, event_path: Path, file_size: float = None):
|
|
||||||
|
def _register_pending(self, storage: str, event_path: Path, file_size: float = None,
|
||||||
|
file_modify_time: float = None, fileid: Optional[str] = None,
|
||||||
|
reason: str = "整理历史查询失败"):
|
||||||
"""
|
"""
|
||||||
登记历史查询失败的文件待重试,重复失败累计次数,超限后放弃。
|
登记暂时性故障的文件待重试,重复失败累计次数,超限后放弃。
|
||||||
:param storage: 存储
|
:param storage: 存储
|
||||||
:param event_path: 原始事件路径
|
:param event_path: 原始事件路径
|
||||||
:param file_size: 文件大小
|
:param file_size: 文件大小,None 表示重试时需要重新读取
|
||||||
|
:param file_modify_time: 文件修改时间
|
||||||
|
:param fileid: 文件唯一标识
|
||||||
|
:param reason: 登记原因,用于日志
|
||||||
"""
|
"""
|
||||||
key = self._pending_key(storage, event_path)
|
key = self._pending_key(storage, event_path)
|
||||||
with self._pending_guard:
|
with self._pending_guard:
|
||||||
@@ -146,7 +195,7 @@ class TransferDispatcher:
|
|||||||
entry["attempts"] += 1
|
entry["attempts"] += 1
|
||||||
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
|
if entry["attempts"] >= self.MAX_RETRY_ATTEMPTS:
|
||||||
self._pending_retries.pop(key, None)
|
self._pending_retries.pop(key, None)
|
||||||
logger.error(f"整理历史查询持续失败,已放弃重试: {key}")
|
logger.error(f"{reason}持续失败,已放弃重试: {key}")
|
||||||
return
|
return
|
||||||
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
|
if len(self._pending_retries) >= self.MAX_PENDING_RETRIES:
|
||||||
logger.error(f"整理重试队列已满,丢弃: {key}")
|
logger.error(f"整理重试队列已满,丢弃: {key}")
|
||||||
@@ -155,9 +204,39 @@ class TransferDispatcher:
|
|||||||
"storage": storage,
|
"storage": storage,
|
||||||
"event_path": event_path,
|
"event_path": event_path,
|
||||||
"file_size": file_size,
|
"file_size": file_size,
|
||||||
|
"file_modify_time": file_modify_time,
|
||||||
|
"fileid": fileid,
|
||||||
"attempts": 1
|
"attempts": 1
|
||||||
}
|
}
|
||||||
logger.warn(f"整理历史查询失败,已登记待重试: {key}")
|
logger.warn(f"{reason},已登记待重试: {key}")
|
||||||
|
|
||||||
|
def register_unreadable(self, storage: str, event_path: Path):
|
||||||
|
"""
|
||||||
|
登记读取失败的监控事件待重试。
|
||||||
|
|
||||||
|
FUSE/网络挂载抖动时 stat 会瞬时失败,直接丢弃事件就是永久漏件,
|
||||||
|
因此复用待重试队列,由健康检查周期重新读取。
|
||||||
|
:param storage: 存储
|
||||||
|
:param event_path: 事件文件路径
|
||||||
|
"""
|
||||||
|
self._register_pending(storage=storage, event_path=event_path,
|
||||||
|
file_size=None, reason="读取监控事件文件失败")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _resolve_file_state(event_path: Path) -> Tuple[Optional[int], Optional[float], bool]:
|
||||||
|
"""
|
||||||
|
重新读取本地文件指纹。
|
||||||
|
:param event_path: 文件路径
|
||||||
|
:return: (文件大小, 修改时间, 文件是否仍然存在);大小为 None 表示本次读取仍然失败
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
file_stat = Path(event_path).stat()
|
||||||
|
return file_stat.st_size, file_stat.st_mtime, True
|
||||||
|
except FileNotFoundError:
|
||||||
|
return None, None, False
|
||||||
|
except OSError as err:
|
||||||
|
logger.debug(f"重试读取文件大小失败: {event_path} - {err}")
|
||||||
|
return None, None, True
|
||||||
|
|
||||||
def _discard_pending(self, storage: str, event_path: Path):
|
def _discard_pending(self, storage: str, event_path: Path):
|
||||||
"""
|
"""
|
||||||
@@ -168,6 +247,17 @@ class TransferDispatcher:
|
|||||||
with self._pending_guard:
|
with self._pending_guard:
|
||||||
self._pending_retries.pop(self._pending_key(storage, event_path), None)
|
self._pending_retries.pop(self._pending_key(storage, event_path), None)
|
||||||
|
|
||||||
|
def clear_pending(self):
|
||||||
|
"""
|
||||||
|
清空待重试队列。监控停止或配置重载时调用,避免已移除的监控目录
|
||||||
|
在数据库恢复后仍被看门狗送入整理链。
|
||||||
|
"""
|
||||||
|
with self._pending_guard:
|
||||||
|
if not self._pending_retries:
|
||||||
|
return
|
||||||
|
logger.debug(f"清理整理重试队列,丢弃 {len(self._pending_retries)} 个待重试条目")
|
||||||
|
self._pending_retries.clear()
|
||||||
|
|
||||||
def retry_pending(self):
|
def retry_pending(self):
|
||||||
"""
|
"""
|
||||||
重试历史查询失败的文件,由健康检查周期驱动。
|
重试历史查询失败的文件,由健康检查周期驱动。
|
||||||
@@ -176,74 +266,149 @@ class TransferDispatcher:
|
|||||||
with self._pending_guard:
|
with self._pending_guard:
|
||||||
items = list(self._pending_retries.values())
|
items = list(self._pending_retries.values())
|
||||||
for item in items:
|
for item in items:
|
||||||
logger.info(f"重试整理: {item['storage']}:{item['event_path']}")
|
storage = item["storage"]
|
||||||
self.handle_file(storage=item["storage"], event_path=item["event_path"],
|
event_path = item["event_path"]
|
||||||
file_size=item["file_size"])
|
file_size = item["file_size"]
|
||||||
|
file_modify_time = item.get("file_modify_time")
|
||||||
|
fileid = item.get("fileid")
|
||||||
|
if file_size is None and storage == "local":
|
||||||
|
# 因读取失败入队的事件没有大小,重试时必须重新读取
|
||||||
|
file_size, file_modify_time, exists = self._resolve_file_state(event_path)
|
||||||
|
if not exists:
|
||||||
|
logger.debug(f"待重试文件已不存在,放弃: {storage}:{event_path}")
|
||||||
|
self._discard_pending(storage=storage, event_path=event_path)
|
||||||
|
continue
|
||||||
|
if file_size is None:
|
||||||
|
# 仍然读不到,累计失败次数后等下个周期,超限由登记逻辑放弃
|
||||||
|
self._register_pending(storage=storage, event_path=event_path,
|
||||||
|
reason="读取监控事件文件失败")
|
||||||
|
continue
|
||||||
|
logger.info(f"重试整理: {storage}:{event_path}")
|
||||||
|
self.handle_file(
|
||||||
|
storage=storage,
|
||||||
|
event_path=event_path,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
|
||||||
def handle_file(self, storage: str, event_path: Path, file_size: float = None) -> bool:
|
def handle_file(self, storage: str, event_path: Path, file_size: float = None,
|
||||||
|
file_modify_time: float = None, fileid: Optional[str] = None) -> bool:
|
||||||
"""
|
"""
|
||||||
整理一个文件。
|
整理一个文件。
|
||||||
:param storage: 存储
|
:param storage: 存储
|
||||||
:param event_path: 事件文件路径
|
:param event_path: 事件文件路径
|
||||||
:param file_size: 文件大小
|
:param file_size: 文件大小
|
||||||
|
:param file_modify_time: 文件修改时间
|
||||||
|
:param fileid: 文件唯一标识
|
||||||
:return: 是否进入整理链
|
:return: 是否进入整理链
|
||||||
"""
|
"""
|
||||||
with self._lock:
|
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
|
||||||
# 登记重试用原始事件路径,蓝光目录解析在重试时重新执行
|
origin_path = event_path
|
||||||
origin_path = event_path
|
is_bluray_folder = False
|
||||||
is_bluray_folder = False
|
# 蓝光原盘文件处理
|
||||||
# 蓝光原盘文件处理
|
if self._is_bluray_sub(event_path):
|
||||||
if self._is_bluray_sub(event_path):
|
event_path = self._get_bluray_dir(event_path)
|
||||||
event_path = self._get_bluray_dir(event_path)
|
if not event_path:
|
||||||
if not event_path:
|
|
||||||
return False
|
|
||||||
is_bluray_folder = True
|
|
||||||
elif not self.is_transfer_candidate_path(event_path):
|
|
||||||
return False
|
return False
|
||||||
|
is_bluray_folder = True
|
||||||
|
elif not self.is_transfer_candidate_path(event_path):
|
||||||
|
return False
|
||||||
|
|
||||||
# TTL缓存控重
|
# TTL 缓存控重。这是本方法唯一需要互斥的临界区,锁只保护「查缓存 + 写缓存」
|
||||||
|
# 这一步的原子性。
|
||||||
|
#
|
||||||
|
# 锁的范围绝不能扩大到下面的历史查询与整理调用:整理的规划阶段会访问挂载
|
||||||
|
# (do_transfer 内的 get_parent_item / list_files),FUSE 进入「请求永不
|
||||||
|
# 返回」状态时这些调用永远不返回,持锁线程就把这把锁永久攥在手里,连带
|
||||||
|
# 锁死所有 watcher 线程的事件派发、监控恢复后的补偿扫描和重试队列——监控层
|
||||||
|
# 即使完成自愈也送不进任何文件,漏件永远补不回来。
|
||||||
|
#
|
||||||
|
# 并发是安全的:TTL 去重保证同一路径不会并发进入;TransferChain 是单例,
|
||||||
|
# 内部用 job_lock/task_lock 保护共享状态、入队走线程安全的 queue.Queue,
|
||||||
|
# 本来就被下载完成事件、定时任务与工作流并发调用。
|
||||||
|
with self._lock:
|
||||||
if self._cache.get(str(event_path)):
|
if self._cache.get(str(event_path)):
|
||||||
return False
|
return False
|
||||||
self._cache[str(event_path)] = True
|
self._cache[str(event_path)] = True
|
||||||
|
|
||||||
src_path = self._build_transfer_src_path(
|
src_path = self._build_transfer_src_path(
|
||||||
event_path=event_path,
|
event_path=event_path,
|
||||||
is_bluray_folder=is_bluray_folder,
|
is_bluray_folder=is_bluray_folder,
|
||||||
)
|
)
|
||||||
has_transfer_history = self._has_transfer_history(
|
skip_by_history = self._should_skip_by_history(
|
||||||
|
storage=storage,
|
||||||
|
src_path=src_path,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
)
|
||||||
|
if skip_by_history is None:
|
||||||
|
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
|
||||||
|
self._register_pending(
|
||||||
storage=storage,
|
storage=storage,
|
||||||
src_path=src_path,
|
event_path=origin_path,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
)
|
)
|
||||||
if has_transfer_history is None:
|
return False
|
||||||
# 查询失败是暂时故障,登记待重试(由健康检查周期驱动),不能永久跳过
|
if skip_by_history:
|
||||||
self._register_pending(storage=storage, event_path=origin_path, file_size=file_size)
|
|
||||||
return False
|
|
||||||
self._discard_pending(storage=storage, event_path=origin_path)
|
self._discard_pending(storage=storage, event_path=origin_path)
|
||||||
if has_transfer_history:
|
return False
|
||||||
return False
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if is_bluray_folder:
|
if is_bluray_folder:
|
||||||
logger.info(f"开始整理蓝光原盘: {event_path}")
|
logger.info(f"开始整理蓝光原盘: {event_path}")
|
||||||
else:
|
else:
|
||||||
logger.info(f"开始整理文件: {event_path}")
|
logger.info(f"开始整理文件: {event_path}")
|
||||||
# 开始整理
|
# 开始整理
|
||||||
TransferChain().do_transfer(
|
TransferChain().do_transfer(
|
||||||
fileitem=FileItem(
|
fileitem=FileItem(
|
||||||
storage=storage,
|
storage=storage,
|
||||||
path=src_path,
|
path=src_path,
|
||||||
type="file" if not is_bluray_folder else "dir",
|
type="file" if not is_bluray_folder else "dir",
|
||||||
name=event_path.name,
|
name=event_path.name,
|
||||||
basename=event_path.stem,
|
basename=event_path.stem,
|
||||||
extension=event_path.suffix[1:],
|
extension=event_path.suffix[1:],
|
||||||
size=file_size
|
size=file_size,
|
||||||
),
|
modify_time=file_modify_time,
|
||||||
mtype=self._get_monitor_media_type(
|
fileid=fileid,
|
||||||
storage=storage,
|
),
|
||||||
event_path=event_path,
|
mtype=self._get_monitor_media_type(
|
||||||
),
|
storage=storage,
|
||||||
)
|
event_path=event_path,
|
||||||
return True
|
),
|
||||||
except Exception as e:
|
)
|
||||||
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
# 整理已执行完毕,此前因暂时性故障登记的重试条目到此作废
|
||||||
return False
|
self._discard_pending(storage=storage, event_path=origin_path)
|
||||||
|
return True
|
||||||
|
except Exception as e:
|
||||||
|
logger.error("目录监控整理文件发生错误:%s - %s" % (str(e), traceback.format_exc()))
|
||||||
|
# 去重缓存在入口已写入,整理抛异常时必须失效,否则 TTL 窗口内该文件的
|
||||||
|
# 后续事件会被静默吞掉,等于一次异常就丢一个文件
|
||||||
|
self._invalidate_cache(str(event_path))
|
||||||
|
# 已稳定落地的文件不会再产生任何事件,批量整理期间撞上一次 DB/网络瞬断
|
||||||
|
# 就是永久丢件,因此与历史查询失败同样登记待重试;登记用原始事件路径,
|
||||||
|
# 重试时重新解析蓝光目录并重走完整流程。异常未清空登记,重试次数会持续
|
||||||
|
# 累计,达到上限后由 _register_pending 放弃,不会无限重试
|
||||||
|
self._register_pending(storage=storage, event_path=origin_path,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
reason="整理执行异常")
|
||||||
|
return False
|
||||||
|
|
||||||
|
def _invalidate_cache(self, key: str):
|
||||||
|
"""
|
||||||
|
使去重缓存条目失效,兼容缓存后端与测试注入的字典。
|
||||||
|
:param key: 缓存键
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
delete = getattr(self._cache, "delete", None)
|
||||||
|
if callable(delete):
|
||||||
|
delete(key)
|
||||||
|
return
|
||||||
|
self._cache.pop(key, None)
|
||||||
|
except Exception as err:
|
||||||
|
logger.debug(f"清理监控去重缓存失败: {key} - {err}")
|
||||||
|
|||||||
+292
-21
@@ -1,7 +1,9 @@
|
|||||||
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
from functools import partial
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from threading import Lock
|
from threading import Lock, Thread
|
||||||
from typing import Any, Dict, List, Optional
|
from typing import Any, Callable, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
from apscheduler.schedulers.background import BackgroundScheduler
|
from apscheduler.schedulers.background import BackgroundScheduler
|
||||||
|
|
||||||
@@ -11,6 +13,7 @@ from app.helper.message import MessageHelper
|
|||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.monitor.dispatcher import TransferDispatcher
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
from app.monitor.poller import RemotePoller
|
from app.monitor.poller import RemotePoller
|
||||||
|
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
|
||||||
from app.monitor.snapshot import SnapshotStore
|
from app.monitor.snapshot import SnapshotStore
|
||||||
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
|
from app.monitor.syslimits import decide_monitor_mode, get_system_optimization_tips
|
||||||
from app.monitor.watcher import LocalDirectoryWatcher
|
from app.monitor.watcher import LocalDirectoryWatcher
|
||||||
@@ -24,11 +27,33 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
"""
|
"""
|
||||||
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
目录监控门面,单例模式:装配本地/远程监控、维护生命周期与健康检查。
|
||||||
"""
|
"""
|
||||||
CONFIG_WATCH = {SystemConfigKey.Directories.value}
|
# 除目录配置外,同时监听仅在监控线程创建时读取的环境变量:这两项经
|
||||||
|
# /system/env 保存后运行时值虽已更新,但已运行的监控不会重新决策模式,
|
||||||
|
# 必须触发 init() 全量重建才能生效(MONITOR_RESCAN_DELAYS 为实时解析,无需在列)
|
||||||
|
CONFIG_WATCH = {SystemConfigKey.Directories.value,
|
||||||
|
"MONITOR_NETWORK_FAST_MODE",
|
||||||
|
"MONITOR_POLL_DELAY_NETWORK"}
|
||||||
# 目录监控健康检查间隔(秒)
|
# 目录监控健康检查间隔(秒)
|
||||||
WATCHDOG_INTERVAL = 60
|
WATCHDOG_INTERVAL = 60
|
||||||
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
|
# 连续多少个健康检查周期无新增重启后才宣告恢复,避免反复崩溃时告警刷屏
|
||||||
RECOVERY_STABLE_CYCLES = 5
|
RECOVERY_STABLE_CYCLES = 5
|
||||||
|
# 补偿扫描的时间回溯余量(秒),覆盖心跳与文件落地之间的时间差
|
||||||
|
COMPENSATION_MARGIN = 60
|
||||||
|
# 监控内部退避重启的停摆窗口无法从外部精确观测(重启在 watcher 线程内部完成,
|
||||||
|
# 健康检查只能看到累计重启次数),保守取「健康检查周期 + 最长退避 + 余量」
|
||||||
|
RESTART_STALL_LOOKBACK = WATCHDOG_INTERVAL + max(LocalDirectoryWatcher.RESTART_BACKOFF) + COMPENSATION_MARGIN
|
||||||
|
# 单次补偿扫描最多送入整理链的文件数,避免超大目录把整理链和数据库压垮
|
||||||
|
MAX_COMPENSATION_FILES = 2000
|
||||||
|
# 恢复动作(重建监控、重试队列、挂载探测)单个健康检查周期的最长等待秒数。
|
||||||
|
# 这些动作全部会触碰挂载,一律在一次性工作线程里执行,看门狗只等待有限时间;
|
||||||
|
# 取健康检查周期的一半,保证看门狗自身永远不会被拖过下一个周期
|
||||||
|
RECOVERY_TIMEOUT = WATCHDOG_INTERVAL // 2
|
||||||
|
# 隔离目录的挂载探测超时秒数(子进程,超时可被 kill)
|
||||||
|
MOUNT_PROBE_TIMEOUT = 10
|
||||||
|
# 恢复动作的 key 前缀/常量,用于按目录隔离在途动作
|
||||||
|
REBUILD_KEY_PREFIX = "rebuild:"
|
||||||
|
PROBE_KEY = "probe"
|
||||||
|
PENDING_KEY = "pending"
|
||||||
|
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
super().__init__()
|
super().__init__()
|
||||||
@@ -38,12 +63,16 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
self._watcher_lock = Lock()
|
self._watcher_lock = Lock()
|
||||||
# 启动失败待重试的本地监控配置
|
# 启动失败待重试的本地监控配置
|
||||||
self._pending_locals: List[Dict[str, Any]] = []
|
self._pending_locals: List[Dict[str, Any]] = []
|
||||||
# 已告警的监控目录,避免重复推送
|
# 已告警的监控目录及其告警阶段,避免重复推送,同时保证故障升级能再推一次
|
||||||
self._alerted_paths: set = set()
|
self._alerted_paths: Dict[str, str] = {}
|
||||||
# 各监控目录已告警过的自动重启次数
|
# 各监控目录已告警过的自动重启次数
|
||||||
self._restart_marks: Dict[str, int] = {}
|
self._restart_marks: Dict[str, int] = {}
|
||||||
# 各监控目录连续稳定的健康检查周期数
|
# 各监控目录连续稳定的健康检查周期数
|
||||||
self._stable_cycles: Dict[str, int] = {}
|
self._stable_cycles: Dict[str, int] = {}
|
||||||
|
# 判定为挂载级故障、已暂停一切访问的监控目录(path -> 隔离状态)
|
||||||
|
self._isolated: Dict[str, Dict[str, Any]] = {}
|
||||||
|
# 触碰挂载的恢复动作执行器,把 block 型故障挡在看门狗线程之外
|
||||||
|
self._recovery = RecoveryExecutor()
|
||||||
# 定时服务
|
# 定时服务
|
||||||
self._scheduler = None
|
self._scheduler = None
|
||||||
# 整理分发器
|
# 整理分发器
|
||||||
@@ -252,7 +281,8 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||||
poll_delay_ms = None
|
poll_delay_ms = None
|
||||||
if use_polling and SystemUtils.is_network_filesystem(mon_path):
|
if use_polling and SystemUtils.is_network_filesystem(mon_path):
|
||||||
poll_delay_ms = LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS
|
poll_delay_ms = (settings.MONITOR_POLL_DELAY_NETWORK
|
||||||
|
or LocalDirectoryWatcher.POLL_DELAY_NETWORK_MS)
|
||||||
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
|
logger.info(f"检测到网络文件系统,轮询扫描间隔调整为 {poll_delay_ms}ms: {mon_path}")
|
||||||
|
|
||||||
watcher = LocalDirectoryWatcher(
|
watcher = LocalDirectoryWatcher(
|
||||||
@@ -314,23 +344,38 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
|
|
||||||
def watchdog(self):
|
def watchdog(self):
|
||||||
"""
|
"""
|
||||||
目录监控健康检查:重建崩溃或静默失效的监控线程,并重试启动失败的监控目录。
|
目录监控健康检查:检测监控线程状态并驱动恢复。
|
||||||
|
|
||||||
|
本方法是全局自愈的单点,因此**只做纯内存的检测与判定**:任何会触碰挂载
|
||||||
|
的动作(重建监控、重试启动、重试整理、挂载探测)都交给一次性工作线程,
|
||||||
|
看门狗只等待有限时间。FUSE 挂载进入「请求永不返回」状态时,内联执行这些
|
||||||
|
动作会把看门狗冻死在它自己要修复的挂载上,随后停滞检测、告警、重试驱动
|
||||||
|
全部静默失效——这正是全进程雪崩的起点。
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
self.__check_watchers()
|
broken = self.__check_watchers()
|
||||||
self.__retry_pending_locals()
|
self.__drive_recovery(broken)
|
||||||
self._dispatcher.retry_pending()
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
logger.error(f"目录监控健康检查出现错误:{e}\n{traceback.format_exc()}")
|
||||||
|
|
||||||
def __check_watchers(self):
|
def __check_watchers(self) -> List[LocalDirectoryWatcher]:
|
||||||
"""
|
"""
|
||||||
检查本地目录监控线程状态,异常时重建。
|
检查本地目录监控线程状态,返回需要重建的监控。
|
||||||
|
|
||||||
|
全程只读内存状态(线程存活标志、心跳时间戳、重启计数),不做任何文件
|
||||||
|
系统访问,确保挂载无响应时检测环节本身永远不会被阻塞。
|
||||||
|
:return: 需要重建的监控列表
|
||||||
"""
|
"""
|
||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
watchers = list(self._watchers)
|
watchers = list(self._watchers)
|
||||||
|
isolated = set(self._isolated)
|
||||||
|
broken: List[LocalDirectoryWatcher] = []
|
||||||
for watcher in watchers:
|
for watcher in watchers:
|
||||||
key = str(watcher.watch_path)
|
key = str(watcher.watch_path)
|
||||||
|
if key in isolated:
|
||||||
|
# 已判定挂载级故障:重建只会再冻死一个线程,等探测确认挂载
|
||||||
|
# 恢复应答后由 __probe_isolated 统一重建
|
||||||
|
continue
|
||||||
if watcher.is_stalled():
|
if watcher.is_stalled():
|
||||||
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
|
reason = f"监控循环超过 {LocalDirectoryWatcher.STALL_TIMEOUT} 秒无任何活动,判定为静默失效"
|
||||||
elif not watcher.is_alive():
|
elif not watcher.is_alive():
|
||||||
@@ -343,6 +388,11 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
self.__send_alert(watcher.watch_path,
|
self.__send_alert(watcher.watch_path,
|
||||||
f"目录监控发生错误并已自动重启"
|
f"目录监控发生错误并已自动重启"
|
||||||
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
|
f"(累计 {watcher.restart_count} 次): {watcher.watch_path}")
|
||||||
|
# 内部退避重启同样是停摆:轮询模式下重启会重建基线快照,停摆窗口内
|
||||||
|
# 落地的文件会被新基线静默吸收,永远不再产生事件,必须补扫。
|
||||||
|
# 重启计数只在增长时进入本分支,同一次重启不会被反复补扫
|
||||||
|
self.__start_compensation(mon_path=watcher.watch_path,
|
||||||
|
since=time.time() - self.RESTART_STALL_LOOKBACK)
|
||||||
else:
|
else:
|
||||||
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
|
# 稳定满恢复窗口才宣告恢复,避免反复崩溃时告警/恢复消息来回刷屏
|
||||||
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
|
self._stable_cycles[key] = self._stable_cycles.get(key, 0) + 1
|
||||||
@@ -352,7 +402,102 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
|
logger.error(f"目录监控异常: {watcher.watch_path} - {reason},正在重建监控线程 ...")
|
||||||
self.__send_alert(watcher.watch_path,
|
self.__send_alert(watcher.watch_path,
|
||||||
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
|
f"目录监控异常: {watcher.watch_path}\n原因: {reason}\n正在自动重建监控")
|
||||||
self.__rebuild_watcher(watcher)
|
broken.append(watcher)
|
||||||
|
return broken
|
||||||
|
|
||||||
|
def __drive_recovery(self, broken: List[LocalDirectoryWatcher]):
|
||||||
|
"""
|
||||||
|
把所有会触碰挂载的恢复动作派发到一次性工作线程,并等待有限时间。
|
||||||
|
|
||||||
|
重建动作超时(或上一轮的重建仍冻着)即判定为挂载级故障:这已经不是单个
|
||||||
|
目录的问题,继续每 60 秒重试只会不断泄漏冻死的线程,必须转入隔离,改由
|
||||||
|
可放弃的子进程探测来确认挂载何时恢复。
|
||||||
|
:param broken: 需要重建的监控列表
|
||||||
|
"""
|
||||||
|
actions: Dict[str, Callable[[], None]] = {}
|
||||||
|
rebuilds: Dict[str, LocalDirectoryWatcher] = {}
|
||||||
|
for watcher in broken:
|
||||||
|
key = f"{self.REBUILD_KEY_PREFIX}{watcher.watch_path}"
|
||||||
|
rebuilds[key] = watcher
|
||||||
|
actions[key] = partial(self.__rebuild_watcher, watcher)
|
||||||
|
if self._isolated:
|
||||||
|
actions[self.PROBE_KEY] = self.__probe_isolated
|
||||||
|
actions[self.PENDING_KEY] = self.__drive_pending
|
||||||
|
|
||||||
|
results = self._recovery.run(actions, timeout=self.RECOVERY_TIMEOUT)
|
||||||
|
|
||||||
|
for key, state in results.items():
|
||||||
|
if state is RecoveryState.COMPLETED:
|
||||||
|
continue
|
||||||
|
watcher = rebuilds.get(key)
|
||||||
|
if watcher is not None:
|
||||||
|
self.__enter_isolation(watcher)
|
||||||
|
else:
|
||||||
|
# 探测与重试驱动超时不升级为隔离:它们跨多个目录,无法归因到
|
||||||
|
# 具体挂载,下个周期由执行器的 BUSY 判定自动跳过,不会泄漏线程
|
||||||
|
logger.warn(f"目录监控恢复动作未在 {self.RECOVERY_TIMEOUT} 秒内完成"
|
||||||
|
f"({state.value}),将在后续健康检查周期重试: {key}")
|
||||||
|
|
||||||
|
def __enter_isolation(self, watcher: LocalDirectoryWatcher):
|
||||||
|
"""
|
||||||
|
将一个监控目录转入挂载级故障隔离:停止对它的一切新访问,等待探测恢复。
|
||||||
|
:param watcher: 重建未能返回的监控
|
||||||
|
"""
|
||||||
|
key = str(watcher.watch_path)
|
||||||
|
with self._watcher_lock:
|
||||||
|
if key in self._isolated:
|
||||||
|
return
|
||||||
|
self._isolated[key] = {
|
||||||
|
"watcher": watcher,
|
||||||
|
"since": time.time(),
|
||||||
|
"failures": 0,
|
||||||
|
}
|
||||||
|
logger.error(f"目录监控重建在挂载上无响应,判定为挂载级故障,"
|
||||||
|
f"已暂停对该目录的所有访问并转入周期探测: {watcher.watch_path}")
|
||||||
|
self.__send_alert(watcher.watch_path,
|
||||||
|
f"目录监控挂载无响应: {watcher.watch_path}\n"
|
||||||
|
f"已暂停对该目录的所有访问,正在周期探测挂载,恢复后将自动重建监控并补扫",
|
||||||
|
stage="isolated")
|
||||||
|
|
||||||
|
def __probe_isolated(self):
|
||||||
|
"""
|
||||||
|
对隔离中的监控目录做可放弃探测,挂载恢复应答后解除隔离并重建监控。
|
||||||
|
|
||||||
|
运行在恢复工作线程里:探测本身由子进程执行(见 recovery.probe_path),
|
||||||
|
超时可被 kill,因此本线程不会像内联 stat 那样永久冻死。探测通过后的重建
|
||||||
|
仍有极小概率再次卡住,届时本线程会被下一轮的 BUSY 判定跳过,不再泄漏。
|
||||||
|
"""
|
||||||
|
with self._watcher_lock:
|
||||||
|
keys = list(self._isolated)
|
||||||
|
for key in keys:
|
||||||
|
mon_path = Path(key)
|
||||||
|
if not probe_path(mon_path, timeout=self.MOUNT_PROBE_TIMEOUT):
|
||||||
|
with self._watcher_lock:
|
||||||
|
entry = self._isolated.get(key)
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
entry["failures"] += 1
|
||||||
|
failures = entry["failures"]
|
||||||
|
since = entry["since"]
|
||||||
|
logger.warn(f"挂载探测未通过(累计 {failures} 次,已隔离 "
|
||||||
|
f"{int(time.time() - since)} 秒),继续隔离: {mon_path}")
|
||||||
|
continue
|
||||||
|
with self._watcher_lock:
|
||||||
|
entry = self._isolated.pop(key, None)
|
||||||
|
if not entry:
|
||||||
|
continue
|
||||||
|
logger.info(f"✓ 挂载探测通过,解除隔离并重建目录监控: {mon_path}")
|
||||||
|
self.__clear_alert(mon_path, f"目录监控挂载已恢复响应,正在重建监控: {mon_path}")
|
||||||
|
# 重建内部会按 watcher 的最后心跳时间发起补偿扫描,补回隔离期间落地的文件
|
||||||
|
self.__rebuild_watcher(entry["watcher"])
|
||||||
|
|
||||||
|
def __drive_pending(self):
|
||||||
|
"""
|
||||||
|
驱动两条待重试队列。两者都会访问挂载(启动重试走目录遍历与 exists,
|
||||||
|
整理重试走 stat),必须在恢复工作线程里执行而不是看门狗线程里。
|
||||||
|
"""
|
||||||
|
self.__retry_pending_locals()
|
||||||
|
self._dispatcher.retry_pending()
|
||||||
|
|
||||||
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
def __rebuild_watcher(self, watcher: LocalDirectoryWatcher):
|
||||||
"""
|
"""
|
||||||
@@ -381,12 +526,102 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
})
|
})
|
||||||
return
|
return
|
||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
registered = any(item is watcher for item in self._watchers)
|
||||||
|
if registered:
|
||||||
|
self._watchers = [new_watcher if item is watcher else item for item in self._watchers]
|
||||||
|
if not registered:
|
||||||
|
# 卡死的重建线程可能在挂载恢复后才解冻并走到这里,而该目录此时已由
|
||||||
|
# 隔离恢复路径重建过。此处若直接放行,新建的监控既不在 _watchers 里
|
||||||
|
# (健康检查永远看不到它)、也没人调用 stop(),会变成与旧监控重复
|
||||||
|
# 派发事件的孤儿线程,必须就地停掉
|
||||||
|
logger.warn(f"目录监控已由其他路径重建,停止本次重建的冗余监控: {watcher.watch_path}")
|
||||||
|
new_watcher.stop()
|
||||||
|
return
|
||||||
# 新监控的重启计数从零开始,同步重置告警基准
|
# 新监控的重启计数从零开始,同步重置告警基准
|
||||||
self._restart_marks.pop(str(watcher.watch_path), None)
|
self._restart_marks.pop(str(watcher.watch_path), None)
|
||||||
self._stable_cycles.pop(str(watcher.watch_path), None)
|
self._stable_cycles.pop(str(watcher.watch_path), None)
|
||||||
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
|
logger.info(f"✓ 目录监控已重建: {watcher.watch_path}")
|
||||||
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
|
self.__clear_alert(watcher.watch_path, f"目录监控已自动恢复: {watcher.watch_path}")
|
||||||
|
# 重建只恢复未来的事件,停摆期间落地的文件不会再产生任何事件,必须补扫
|
||||||
|
self.__start_compensation(mon_path=watcher.watch_path,
|
||||||
|
since=watcher.last_activity_time)
|
||||||
|
|
||||||
|
def __start_compensation(self, mon_path: Path, since: float):
|
||||||
|
"""
|
||||||
|
在后台线程发起补偿扫描,避免遍历目录阻塞健康检查周期。
|
||||||
|
:param mon_path: 监控目录
|
||||||
|
:param since: 停摆起点(墙钟时间戳)
|
||||||
|
"""
|
||||||
|
if not since:
|
||||||
|
# 从未活动过说明没有可靠的停摆起点,全量补扫代价不可控,跳过
|
||||||
|
logger.debug(f"监控无活动记录,跳过补偿扫描: {mon_path}")
|
||||||
|
return
|
||||||
|
Thread(
|
||||||
|
target=self.__compensate_scan,
|
||||||
|
kwargs={"mon_path": mon_path, "since": since},
|
||||||
|
name=f"MoviePilot-MonitorCompensation-{mon_path.name}",
|
||||||
|
daemon=True
|
||||||
|
).start()
|
||||||
|
|
||||||
|
def __compensate_scan(self, mon_path: Path, since: float):
|
||||||
|
"""
|
||||||
|
补扫监控停摆期间落地的文件。
|
||||||
|
|
||||||
|
不能用 mtime 判定「停摆期间落地」:CloudDrive2/115 等网盘挂载在转存、移动
|
||||||
|
文件时保留原始 mtime(可能是几年前),按 mtime 过滤会让补偿扫描完全空转。
|
||||||
|
因此把目录内所有候选文件都送入整理链,由分发器的 TTL 去重与整理历史查重挡掉
|
||||||
|
已处理过的文件;代价是每个候选文件一次历史查询,故按 mtime 从新到旧排序并用
|
||||||
|
MAX_COMPENSATION_FILES 限制单次规模,让名额优先给最可能是新落地的文件。
|
||||||
|
:param mon_path: 监控目录
|
||||||
|
:param since: 停摆起点(墙钟时间戳),仅用于统计与日志
|
||||||
|
"""
|
||||||
|
candidates = self.__collect_compensation_files(mon_path)
|
||||||
|
if candidates is None:
|
||||||
|
return
|
||||||
|
# mtime 不再作为过滤条件,但仍是「最可能是新文件」的排序依据
|
||||||
|
candidates.sort(key=lambda item: item[1], reverse=True)
|
||||||
|
if len(candidates) > self.MAX_COMPENSATION_FILES:
|
||||||
|
logger.warn(f"补偿扫描候选文件 {len(candidates)} 个,超过单次上限 "
|
||||||
|
f"{self.MAX_COMPENSATION_FILES},本次只处理最新的一批: {mon_path}")
|
||||||
|
candidates = candidates[:self.MAX_COMPENSATION_FILES]
|
||||||
|
threshold = since - self.COMPENSATION_MARGIN
|
||||||
|
changed_count = sum(1 for candidate in candidates if candidate[1] >= threshold)
|
||||||
|
handled = 0
|
||||||
|
for file_path, file_modify_time, file_size in candidates:
|
||||||
|
if self._dispatcher.handle_file(
|
||||||
|
storage="local",
|
||||||
|
event_path=file_path,
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
):
|
||||||
|
handled += 1
|
||||||
|
logger.info(f"✓ 目录监控补偿扫描完成,{len(candidates)} 个候选文件"
|
||||||
|
f"(其中 {changed_count} 个修改时间落在停摆期间)中有 {handled} 个进入整理链: {mon_path}")
|
||||||
|
|
||||||
|
def __collect_compensation_files(self, mon_path: Path) -> Optional[List[Tuple[Path, float, int]]]:
|
||||||
|
"""
|
||||||
|
收集补偿扫描的候选文件。
|
||||||
|
:param mon_path: 监控目录
|
||||||
|
:return: (文件路径, 修改时间, 文件大小) 列表,目录遍历失败时返回 None
|
||||||
|
"""
|
||||||
|
candidates: List[Tuple[Path, float, int]] = []
|
||||||
|
try:
|
||||||
|
for file_path in mon_path.rglob("*"):
|
||||||
|
# 扩展名判断是纯字符串运算,先过滤能省掉大量 FUSE 挂载上昂贵的 stat
|
||||||
|
if not self._dispatcher.is_transfer_candidate_path(file_path):
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
if not file_path.is_file():
|
||||||
|
continue
|
||||||
|
file_stat = file_path.stat()
|
||||||
|
except OSError as err:
|
||||||
|
logger.debug(f"补偿扫描读取文件失败: {file_path} - {err}")
|
||||||
|
continue
|
||||||
|
candidates.append((file_path, file_stat.st_mtime, file_stat.st_size))
|
||||||
|
except OSError as err:
|
||||||
|
logger.error(f"补偿扫描失败: {mon_path} - {err}")
|
||||||
|
return None
|
||||||
|
return candidates
|
||||||
|
|
||||||
def __retry_pending_locals(self):
|
def __retry_pending_locals(self):
|
||||||
"""
|
"""
|
||||||
@@ -394,7 +629,12 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
"""
|
"""
|
||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
pending = list(self._pending_locals)
|
pending = list(self._pending_locals)
|
||||||
|
isolated = set(self._isolated)
|
||||||
for item in pending:
|
for item in pending:
|
||||||
|
if str(item["mon_path"]) in isolated:
|
||||||
|
# 隔离中的挂载不接受任何新访问:启动重试要走目录遍历与 exists,
|
||||||
|
# 在「请求永不返回」的挂载上会再冻死一个线程
|
||||||
|
continue
|
||||||
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
|
# 失败次数越多重试间隔越长(按健康检查周期数退避),长时间故障时不刷屏
|
||||||
if item.get("skip_cycles", 0) > 0:
|
if item.get("skip_cycles", 0) > 0:
|
||||||
item["skip_cycles"] -= 1
|
item["skip_cycles"] -= 1
|
||||||
@@ -404,17 +644,20 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
item["attempts"] = item.get("attempts", 0) + 1
|
item["attempts"] = item.get("attempts", 0) + 1
|
||||||
item["skip_cycles"] = min(item["attempts"], 10)
|
item["skip_cycles"] = min(item["attempts"], 10)
|
||||||
|
|
||||||
def __send_alert(self, mon_path: Path, message: str):
|
def __send_alert(self, mon_path: Path, message: str, stage: str = "fault"):
|
||||||
"""
|
"""
|
||||||
推送目录监控异常告警,同一目录仅在状态变化时推送一次。
|
推送目录监控异常告警,同一目录在同一阶段仅推送一次。
|
||||||
:param mon_path: 监控目录
|
:param mon_path: 监控目录
|
||||||
:param message: 告警内容
|
:param message: 告警内容
|
||||||
|
:param stage: 告警阶段。故障升级为挂载级隔离是用户必须知道的状态变化
|
||||||
|
(监控已暂停访问、等待挂载恢复),只按目录去重会把这条
|
||||||
|
关键消息吞掉,因此阶段变化时重新推送
|
||||||
"""
|
"""
|
||||||
key = str(mon_path)
|
key = str(mon_path)
|
||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
if key in self._alerted_paths:
|
if self._alerted_paths.get(key) == stage:
|
||||||
return
|
return
|
||||||
self._alerted_paths.add(key)
|
self._alerted_paths[key] = stage
|
||||||
MessageHelper().put(message, title="目录监控")
|
MessageHelper().put(message, title="目录监控")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
@@ -437,7 +680,7 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
if key not in self._alerted_paths:
|
if key not in self._alerted_paths:
|
||||||
return
|
return
|
||||||
self._alerted_paths.discard(key)
|
self._alerted_paths.pop(key, None)
|
||||||
logger.info(message)
|
logger.info(message)
|
||||||
MessageHelper().put(message, title="目录监控")
|
MessageHelper().put(message, title="目录监控")
|
||||||
|
|
||||||
@@ -474,8 +717,30 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
return
|
return
|
||||||
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
if not self._dispatcher.is_transfer_candidate_path(Path(event_path)):
|
||||||
return
|
return
|
||||||
|
file_modify_time = None
|
||||||
|
try:
|
||||||
|
file_modify_time = Path(event_path).stat().st_mtime
|
||||||
|
except OSError as err:
|
||||||
|
logger.debug(f"读取目录监控文件修改时间失败: {event_path} - {err}")
|
||||||
# 整理文件
|
# 整理文件
|
||||||
self._dispatcher.handle_file(storage="local", event_path=Path(event_path), file_size=file_size)
|
handle_kwargs = {
|
||||||
|
"storage": "local",
|
||||||
|
"event_path": Path(event_path),
|
||||||
|
"file_size": file_size,
|
||||||
|
}
|
||||||
|
if file_modify_time is not None:
|
||||||
|
handle_kwargs["file_modify_time"] = file_modify_time
|
||||||
|
self._dispatcher.handle_file(**handle_kwargs)
|
||||||
|
|
||||||
|
def event_unreadable(self, event_path: Path):
|
||||||
|
"""
|
||||||
|
处理读取失败的监控事件,登记待重试。
|
||||||
|
:param event_path: 事件文件路径
|
||||||
|
"""
|
||||||
|
event_path = Path(event_path)
|
||||||
|
if not self._dispatcher.is_transfer_candidate_path(event_path):
|
||||||
|
return
|
||||||
|
self._dispatcher.register_unreadable(storage="local", event_path=event_path)
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
"""
|
"""
|
||||||
@@ -491,13 +756,19 @@ class Monitor(ConfigReloadMixin, metaclass=SingletonClass):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"停止定时服务出现了错误:{e}")
|
logger.error(f"停止定时服务出现了错误:{e}")
|
||||||
self._scheduler = None
|
self._scheduler = None
|
||||||
|
# 待重试条目按停止前的监控范围登记,重载后范围可能变化,一并清理
|
||||||
|
self._dispatcher.clear_pending()
|
||||||
with self._watcher_lock:
|
with self._watcher_lock:
|
||||||
watchers = self._watchers
|
watchers = self._watchers
|
||||||
self._watchers = []
|
self._watchers = []
|
||||||
self._pending_locals = []
|
self._pending_locals = []
|
||||||
self._alerted_paths = set()
|
self._alerted_paths = {}
|
||||||
self._restart_marks = {}
|
self._restart_marks = {}
|
||||||
self._stable_cycles = {}
|
self._stable_cycles = {}
|
||||||
|
self._isolated = {}
|
||||||
|
# 已冻死的恢复线程无法回收,这里只是不再跟踪它们,避免重载后同名目录
|
||||||
|
# 被残留记录误判为 BUSY 而永远拿不到重建机会
|
||||||
|
self._recovery.clear()
|
||||||
if watchers:
|
if watchers:
|
||||||
logger.info("正在停止本地目录监控服务...")
|
logger.info("正在停止本地目录监控服务...")
|
||||||
for watcher in watchers:
|
for watcher in watchers:
|
||||||
|
|||||||
+45
-11
@@ -137,13 +137,17 @@ class RemotePoller:
|
|||||||
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
logger.info(f"{storage} 首次快照完成,共 {file_count} 个文件")
|
||||||
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
logger.info("*** 首次快照仅建立基准,不会处理现有文件。后续监控将处理新增和修改的文件 ***")
|
||||||
|
|
||||||
# 保存当前完整基线
|
# 保存合并后的基线。增量游标是整个存储共用的,若本轮有路径失败仍让
|
||||||
if not self._store.save(storage, current_snapshot, file_count, last_snapshot_time):
|
# 游标跟随成功路径前进,失败路径中时间落在新旧游标之间的变更会被
|
||||||
|
# 后续增量查询永久跳过,因此部分失败时把游标固定在旧值
|
||||||
|
pinned_time = last_snapshot_time if failed_paths else None
|
||||||
|
if not self._store.save(storage, current_snapshot, file_count, last_snapshot_time,
|
||||||
|
snapshot_time=pinned_time):
|
||||||
self._note_failure(storage, "保存快照基线失败")
|
self._note_failure(storage, "保存快照基线失败")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
if failed_paths:
|
if failed_paths:
|
||||||
# 部分路径失败:成功路径已合并,失败路径保留旧基线,下轮重试
|
# 部分路径失败:成功路径已合并,失败路径保留旧基线与旧游标,下轮重试
|
||||||
self._note_failure(
|
self._note_failure(
|
||||||
storage,
|
storage,
|
||||||
f"部分路径快照失败: {','.join(str(path) for path in failed_paths)}"
|
f"部分路径快照失败: {','.join(str(path) for path in failed_paths)}"
|
||||||
@@ -181,7 +185,15 @@ class RemotePoller:
|
|||||||
for new_file in added_files:
|
for new_file in added_files:
|
||||||
file_info = new_snapshot.get(new_file, {})
|
file_info = new_snapshot.get(new_file, {})
|
||||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(new_file), file_size=file_size):
|
file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
|
||||||
|
fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
|
||||||
|
if self._dispatcher.handle_file(
|
||||||
|
storage=storage,
|
||||||
|
event_path=Path(new_file),
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
):
|
||||||
handled_added_count += 1
|
handled_added_count += 1
|
||||||
|
|
||||||
# 处理修改文件
|
# 处理修改文件
|
||||||
@@ -189,7 +201,15 @@ class RemotePoller:
|
|||||||
for modified_file in modified_files:
|
for modified_file in modified_files:
|
||||||
file_info = new_snapshot.get(modified_file, {})
|
file_info = new_snapshot.get(modified_file, {})
|
||||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(modified_file), file_size=file_size):
|
file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
|
||||||
|
fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
|
||||||
|
if self._dispatcher.handle_file(
|
||||||
|
storage=storage,
|
||||||
|
event_path=Path(modified_file),
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
):
|
||||||
handled_modified_count += 1
|
handled_modified_count += 1
|
||||||
|
|
||||||
if handled_added_count or handled_modified_count:
|
if handled_added_count or handled_modified_count:
|
||||||
@@ -228,8 +248,15 @@ class RemotePoller:
|
|||||||
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
|
if not self._dispatcher.is_transfer_candidate_path(Path(file_path)):
|
||||||
continue
|
continue
|
||||||
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
file_size = file_info.get('size', 0) if isinstance(file_info, dict) else file_info
|
||||||
if self._dispatcher.handle_file(storage=storage, event_path=Path(file_path),
|
file_modify_time = file_info.get('modify_time') if isinstance(file_info, dict) else None
|
||||||
file_size=file_size):
|
fileid = file_info.get('fileid') if isinstance(file_info, dict) else None
|
||||||
|
if self._dispatcher.handle_file(
|
||||||
|
storage=storage,
|
||||||
|
event_path=Path(file_path),
|
||||||
|
file_size=file_size,
|
||||||
|
file_modify_time=file_modify_time,
|
||||||
|
fileid=fileid,
|
||||||
|
):
|
||||||
processed_count += 1
|
processed_count += 1
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.error(f"处理文件 {file_path} 失败: {e}")
|
logger.error(f"处理文件 {file_path} 失败: {e}")
|
||||||
@@ -237,11 +264,18 @@ class RemotePoller:
|
|||||||
|
|
||||||
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
logger.info(f"{storage}:{mon_path} 全量扫描完成,共处理 {processed_count}/{file_count} 个文件")
|
||||||
|
|
||||||
# 全量扫描覆盖单个路径,与已有基线合并后落盘,避免覆盖其他监控路径的基线
|
# 全量扫描只覆盖单个路径,必须与已有基线合并后落盘。读取失败时无法
|
||||||
|
# 区分「基线不存在」与「读取异常」,此时落盘会抹掉同存储下其他监控
|
||||||
|
# 路径的基线,因此直接判定失败
|
||||||
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
old_snapshot_data, load_ok = self._store.load_checked(storage)
|
||||||
old_snapshot = old_snapshot_data.get('snapshot', {}) if (load_ok and old_snapshot_data) else {}
|
if not load_ok:
|
||||||
merged_snapshot = {**old_snapshot, **new_snapshot}
|
logger.error(f"读取快照基线失败,已跳过落盘以避免覆盖其他监控路径: {storage}:{mon_path}")
|
||||||
self._store.save(storage, merged_snapshot, len(merged_snapshot))
|
return False
|
||||||
|
old_snapshot = old_snapshot_data.get('snapshot', {}) if old_snapshot_data else {}
|
||||||
|
current_snapshot = {**old_snapshot, **new_snapshot}
|
||||||
|
if not self._store.save(storage, current_snapshot, len(current_snapshot)):
|
||||||
|
logger.error(f"保存快照基线失败,全量扫描未完成: {storage}:{mon_path}")
|
||||||
|
return False
|
||||||
|
|
||||||
return True
|
return True
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,178 @@
|
|||||||
|
"""
|
||||||
|
监控恢复动作的可放弃执行单元。
|
||||||
|
|
||||||
|
FUSE/网络挂载有两种故障形态,下游程序的免疫力完全不同:
|
||||||
|
|
||||||
|
- crash 型:调用抛错(如 Transport endpoint is not connected)。异常能被捕获,
|
||||||
|
退避重启即可自愈,由 watcher 自身的重启循环覆盖。
|
||||||
|
- block 型:调用既不返回错误也不返回结果,永久悬挂。**没有任何超时参数能救
|
||||||
|
一个已经发出的 stat**,阻塞其上的线程无法被 Python 回收(线程没有强杀接口)。
|
||||||
|
|
||||||
|
本模块提供 block 型故障下唯一可行的两种自保手段:
|
||||||
|
|
||||||
|
1. RecoveryExecutor —— 把会触碰挂载的动作放进一次性守护线程执行,调用方只
|
||||||
|
等待有限时间。超时即放弃该线程(它会作为守护线程悬挂到进程退出),换取
|
||||||
|
调用方(健康检查这个全局自愈单点)永远活着。
|
||||||
|
2. probe_path —— 用子进程而非线程做挂载探测。子进程可以被 kill,因此探测
|
||||||
|
本身是可放弃的,隔离期间可以无限次周期重试而不累积不可回收的资源。
|
||||||
|
"""
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from enum import Enum
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Callable, Dict, Optional
|
||||||
|
|
||||||
|
from app.log import logger
|
||||||
|
|
||||||
|
# 探测子进程执行的脚本:只对目标路径做一次 stat,成功退出 0,失败退出非 0。
|
||||||
|
# 用 sys.executable 而不是 test/stat 等外部命令,避免依赖发行版的 coreutils 布局。
|
||||||
|
_PROBE_SCRIPT = "import os, sys; os.stat(sys.argv[1])"
|
||||||
|
# 探测子进程超时后,等待它响应 SIGKILL 的宽限秒数。挂在 FUSE 上的进程可能一时
|
||||||
|
# 收不掉,宽限期满就不再等待,残留进程由后续 Popen 自动回收,绝不能无限等待
|
||||||
|
# ——否则「可放弃的探测」又变回一次不可放弃的阻塞。
|
||||||
|
_PROBE_KILL_GRACE = 5
|
||||||
|
|
||||||
|
|
||||||
|
class RecoveryState(str, Enum):
|
||||||
|
"""
|
||||||
|
一次恢复动作的执行结论。
|
||||||
|
"""
|
||||||
|
# 动作已在限定时间内执行完毕(内部抛异常也算完成,异常已记录)
|
||||||
|
COMPLETED = "completed"
|
||||||
|
# 超时仍未返回,判定为 block 型挂载故障,线程已被放弃
|
||||||
|
TIMEOUT = "timeout"
|
||||||
|
# 同 key 的上一个动作仍未结束,本次未提交,避免持续泄漏冻死的线程
|
||||||
|
BUSY = "busy"
|
||||||
|
|
||||||
|
|
||||||
|
class RecoveryExecutor:
|
||||||
|
"""
|
||||||
|
按 key 隔离的一次性恢复线程执行器。
|
||||||
|
|
||||||
|
每个 key 同一时刻最多有一个在途动作:上一个还冻着就不再提交新的,否则每个
|
||||||
|
健康检查周期都会在同一个死挂载上多泄漏一个线程。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
# key -> 该 key 最近一次提交的执行线程
|
||||||
|
self._running: Dict[str, threading.Thread] = {}
|
||||||
|
self._lock = threading.Lock()
|
||||||
|
|
||||||
|
def run(self, actions: Dict[str, Callable[[], None]], timeout: float) -> Dict[str, RecoveryState]:
|
||||||
|
"""
|
||||||
|
并发执行一批恢复动作,整批最多等待 timeout 秒。
|
||||||
|
|
||||||
|
并发而非串行是必需的:串行等待会让总耗时随监控目录数线性增长,
|
||||||
|
13 个目录都挂死时健康检查会被拖过下一个周期,等于又一次自我冻结。
|
||||||
|
:param actions: key -> 无参恢复动作
|
||||||
|
:param timeout: 整批动作的最长等待秒数
|
||||||
|
:return: key -> 执行结论
|
||||||
|
"""
|
||||||
|
results: Dict[str, RecoveryState] = {}
|
||||||
|
started = []
|
||||||
|
for key, action in actions.items():
|
||||||
|
thread = self._start(key, action)
|
||||||
|
if thread is None:
|
||||||
|
results[key] = RecoveryState.BUSY
|
||||||
|
logger.warn(f"上一次恢复动作仍未返回,本轮跳过以避免线程泄漏: {key}")
|
||||||
|
continue
|
||||||
|
started.append((key, thread))
|
||||||
|
|
||||||
|
deadline = time.monotonic() + timeout
|
||||||
|
for key, thread in started:
|
||||||
|
thread.join(timeout=max(0.0, deadline - time.monotonic()))
|
||||||
|
if thread.is_alive():
|
||||||
|
results[key] = RecoveryState.TIMEOUT
|
||||||
|
logger.error(f"恢复动作超过 {timeout} 秒未返回,判定挂载无响应并放弃该线程: {key}")
|
||||||
|
else:
|
||||||
|
results[key] = RecoveryState.COMPLETED
|
||||||
|
return results
|
||||||
|
|
||||||
|
def discard(self, key: str):
|
||||||
|
"""
|
||||||
|
丢弃一个 key 的在途记录。监控停止或配置重载时调用,避免残留条目
|
||||||
|
让重建后的同名目录被误判为 BUSY。
|
||||||
|
:param key: 动作标识
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._running.pop(key, None)
|
||||||
|
|
||||||
|
def clear(self):
|
||||||
|
"""
|
||||||
|
清空全部在途记录。已经冻死的线程无法回收,这里只是不再跟踪它们。
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
self._running.clear()
|
||||||
|
|
||||||
|
def _start(self, key: str, action: Callable[[], None]) -> Optional[threading.Thread]:
|
||||||
|
"""
|
||||||
|
为一个 key 启动执行线程,该 key 仍有在途动作时不启动。
|
||||||
|
:param key: 动作标识
|
||||||
|
:param action: 无参恢复动作
|
||||||
|
:return: 执行线程,未启动时为 None
|
||||||
|
"""
|
||||||
|
with self._lock:
|
||||||
|
running = self._running.get(key)
|
||||||
|
if running is not None and running.is_alive():
|
||||||
|
return None
|
||||||
|
thread = threading.Thread(
|
||||||
|
target=self._execute,
|
||||||
|
args=(key, action),
|
||||||
|
name=f"MoviePilot-MonitorRecovery-{key}"[:120],
|
||||||
|
daemon=True
|
||||||
|
)
|
||||||
|
self._running[key] = thread
|
||||||
|
thread.start()
|
||||||
|
return thread
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _execute(key: str, action: Callable[[], None]):
|
||||||
|
"""
|
||||||
|
执行一个恢复动作,异常只记录不外抛,避免一个目录的失败连累整批恢复。
|
||||||
|
:param key: 动作标识
|
||||||
|
:param action: 无参恢复动作
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
action()
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"执行目录监控恢复动作失败: {key} - {err}")
|
||||||
|
|
||||||
|
|
||||||
|
def probe_path(path: Path, timeout: float) -> bool:
|
||||||
|
"""
|
||||||
|
用可放弃的子进程探测一个路径是否仍能被访问。
|
||||||
|
|
||||||
|
必须是子进程:在本线程里直接 stat,block 型故障下这个调用永不返回,探测
|
||||||
|
线程就成了又一个不可回收的悬挂线程;子进程可以在超时后被 kill,因此隔离
|
||||||
|
期间可以无限次周期探测。
|
||||||
|
:param path: 待探测路径
|
||||||
|
:param timeout: 探测超时秒数
|
||||||
|
:return: 路径是否可访问
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
[sys.executable, "-c", _PROBE_SCRIPT, str(path)],
|
||||||
|
stdout=subprocess.DEVNULL,
|
||||||
|
stderr=subprocess.DEVNULL
|
||||||
|
)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"启动挂载探测子进程失败: {path} - {err}")
|
||||||
|
return False
|
||||||
|
|
||||||
|
try:
|
||||||
|
return process.wait(timeout=timeout) == 0
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.warn(f"挂载探测 {timeout} 秒无响应,判定挂载仍未恢复: {path}")
|
||||||
|
process.kill()
|
||||||
|
try:
|
||||||
|
# 不能用无超时的 wait():进程若卡在 FUSE 上收不掉 SIGKILL,
|
||||||
|
# 这里就会替它把调用线程也一起挂住
|
||||||
|
process.wait(timeout=_PROBE_KILL_GRACE)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
logger.warn(f"挂载探测子进程未能及时退出,交由系统回收: {path}")
|
||||||
|
return False
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"挂载探测执行失败: {path} - {err}")
|
||||||
|
return False
|
||||||
+20
-6
@@ -21,20 +21,25 @@ class SnapshotStore:
|
|||||||
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
|
self._cache = cache if cache is not None else FileCache(base=settings.CACHE_PATH / "snapshots")
|
||||||
|
|
||||||
def save(self, storage: str, snapshot: Dict, file_count: int = 0,
|
def save(self, storage: str, snapshot: Dict, file_count: int = 0,
|
||||||
last_snapshot_time: Optional[float] = None) -> bool:
|
last_snapshot_time: Optional[float] = None,
|
||||||
|
snapshot_time: Optional[float] = None) -> bool:
|
||||||
"""
|
"""
|
||||||
保存快照到文件缓存。
|
保存快照到文件缓存。
|
||||||
:param storage: 存储名称
|
:param storage: 存储名称
|
||||||
:param snapshot: 快照数据
|
:param snapshot: 快照数据
|
||||||
:param file_count: 文件数量,用于调整监控间隔
|
:param file_count: 文件数量,用于调整监控间隔
|
||||||
:param last_snapshot_time: 上次快照时间戳
|
:param last_snapshot_time: 上次快照时间戳
|
||||||
|
:param snapshot_time: 强制指定的增量游标,用于部分路径失败时固定游标不前进
|
||||||
:return: 是否保存成功
|
:return: 是否保存成功
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
snapshot_time = max(
|
if snapshot_time is None:
|
||||||
last_snapshot_time or 0,
|
# 取「上次游标」与「本轮最大 mtime」的较大者:本轮全是旧文件时
|
||||||
max((item.get('modify_time', 0) for item in snapshot.values()), default=0)
|
# 游标不能回退,否则已处理过的变更会被重新判定为新增
|
||||||
)
|
snapshot_time = max(
|
||||||
|
last_snapshot_time or 0,
|
||||||
|
max((item.get('modify_time', 0) for item in snapshot.values()), default=0)
|
||||||
|
)
|
||||||
if not snapshot_time:
|
if not snapshot_time:
|
||||||
snapshot_time = time.time()
|
snapshot_time = time.time()
|
||||||
snapshot_data = {
|
snapshot_data = {
|
||||||
@@ -131,7 +136,16 @@ class SnapshotStore:
|
|||||||
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
old_time = old_info.get('modify_time', 0) if isinstance(old_info, dict) else 0
|
||||||
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
new_time = new_info.get('modify_time', 0) if isinstance(new_info, dict) else 0
|
||||||
|
|
||||||
if old_size != new_size or (old_time and new_time and old_time != new_time):
|
# 支持文件唯一标识的存储器可用它识别同大小且修改时间未变化的替换文件。
|
||||||
|
# 旧快照缺少 fileid 时保持保守,避免升级后首次补齐元数据触发全量重整。
|
||||||
|
old_fileid = old_info.get('fileid') if isinstance(old_info, dict) else None
|
||||||
|
new_fileid = new_info.get('fileid') if isinstance(new_info, dict) else None
|
||||||
|
|
||||||
|
if (
|
||||||
|
old_size != new_size
|
||||||
|
or (old_time and new_time and old_time != new_time)
|
||||||
|
or (old_fileid and new_fileid and old_fileid != new_fileid)
|
||||||
|
):
|
||||||
changes['modified'].append(file_path)
|
changes['modified'].append(file_path)
|
||||||
|
|
||||||
return changes
|
return changes
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import platform
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Any, Dict, List, Optional, Tuple
|
from typing import Any, Dict, List, Optional, Tuple
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
from app.utils.system import SystemUtils
|
from app.utils.system import SystemUtils
|
||||||
|
|
||||||
@@ -123,7 +124,10 @@ def decide_monitor_mode(directory: Path,
|
|||||||
|
|
||||||
# 检查网络文件系统
|
# 检查网络文件系统
|
||||||
if SystemUtils.is_network_filesystem(directory):
|
if SystemUtils.is_network_filesystem(directory):
|
||||||
return True, "检测到网络文件系统,建议使用兼容模式", None, None
|
if not settings.MONITOR_NETWORK_FAST_MODE:
|
||||||
|
return True, "检测到网络文件系统,建议使用兼容模式", None, None
|
||||||
|
# 用户已确认该挂载支持 inotify,继续走快速模式的系统限制检查
|
||||||
|
logger.info(f"检测到网络文件系统,但已配置允许快速模式: {directory}")
|
||||||
|
|
||||||
limits = check_system_limits()
|
limits = check_system_limits()
|
||||||
file_count, dir_count = count_directory_entries(directory)
|
file_count, dir_count = count_directory_entries(directory)
|
||||||
|
|||||||
+223
-9
@@ -7,6 +7,7 @@ from typing import Any, Optional
|
|||||||
|
|
||||||
from watchfiles import Change, DefaultFilter, watch
|
from watchfiles import Change, DefaultFilter, watch
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
from app.log import logger
|
from app.log import logger
|
||||||
|
|
||||||
|
|
||||||
@@ -35,6 +36,17 @@ class LocalDirectoryWatcher:
|
|||||||
POLL_DELAY_LOCAL_MS = 300
|
POLL_DELAY_LOCAL_MS = 300
|
||||||
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
# 网络/FUSE 挂载轮询降频,减少监控自身对挂载后端的持续 stat 压力
|
||||||
POLL_DELAY_NETWORK_MS = 5000
|
POLL_DELAY_NETWORK_MS = 5000
|
||||||
|
# 新增目录延迟重扫的间隔秒数默认值:FUSE 上目录内容的可见性有延迟,首次展开时
|
||||||
|
# 看不到的文件不会再产生任何事件,只能靠延迟重扫补回。默认在常见的 30/120 秒
|
||||||
|
# 窗口后追加两轮成本极低的长延迟轮次(600s/1800s),应对超大目录树的极端延迟;
|
||||||
|
# 实际生效值可通过 MONITOR_RESCAN_DELAYS 配置覆盖,见 DIRECTORY_RESCAN_DELAYS
|
||||||
|
DEFAULT_RESCAN_DELAYS = (30, 120, 600, 1800)
|
||||||
|
# 待重扫目录队列上限,避免大批量移入时无限增长
|
||||||
|
MAX_PENDING_RESCANS = 100
|
||||||
|
# 单个重扫条目允许的连续「整体扫描失败」次数上限:扫描失败(如 FUSE 瞬时抖动)
|
||||||
|
# 时条目会原地重试而不消耗重扫轮次,必须设置上限,避免目录被删除或长期不可
|
||||||
|
# 访问时无限重试、占满队列
|
||||||
|
MAX_RESCAN_FAILURES = 5
|
||||||
|
|
||||||
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
|
def __init__(self, mon_path: Path, callback: Any, force_polling: Optional[bool] = None,
|
||||||
poll_delay_ms: Optional[int] = None):
|
poll_delay_ms: Optional[int] = None):
|
||||||
@@ -54,8 +66,12 @@ class LocalDirectoryWatcher:
|
|||||||
self._watch_filter = DefaultFilter()
|
self._watch_filter = DefaultFilter()
|
||||||
# 最近一次监控循环活动时间(monotonic),用于检测静默失效
|
# 最近一次监控循环活动时间(monotonic),用于检测静默失效
|
||||||
self._last_activity: float = 0.0
|
self._last_activity: float = 0.0
|
||||||
|
# 最近一次活动的墙钟时间,供监控重建后的补偿扫描定位停摆起点
|
||||||
|
self._last_activity_wall: float = 0.0
|
||||||
# 累计自动重启次数
|
# 累计自动重启次数
|
||||||
self._restart_count: int = 0
|
self._restart_count: int = 0
|
||||||
|
# 待延迟重扫的新增目录
|
||||||
|
self._pending_rescans: list[dict] = []
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def watch_path(self) -> Path:
|
def watch_path(self) -> Path:
|
||||||
@@ -141,11 +157,20 @@ class LocalDirectoryWatcher:
|
|||||||
return False
|
return False
|
||||||
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
|
return (time.monotonic() - self._last_activity) > self.STALL_TIMEOUT
|
||||||
|
|
||||||
|
@property
|
||||||
|
def last_activity_time(self) -> float:
|
||||||
|
"""
|
||||||
|
获取最近一次监控循环活动的墙钟时间。
|
||||||
|
:return: Unix 时间戳,从未活动过时为 0
|
||||||
|
"""
|
||||||
|
return self._last_activity_wall
|
||||||
|
|
||||||
def _mark_activity(self):
|
def _mark_activity(self):
|
||||||
"""
|
"""
|
||||||
记录一次监控循环活动时间,作为静默失效检测的心跳。
|
记录一次监控循环活动时间,作为静默失效检测的心跳。
|
||||||
"""
|
"""
|
||||||
self._last_activity = time.monotonic()
|
self._last_activity = time.monotonic()
|
||||||
|
self._last_activity_wall = time.time()
|
||||||
|
|
||||||
def _run(self):
|
def _run(self):
|
||||||
"""
|
"""
|
||||||
@@ -199,6 +224,8 @@ class LocalDirectoryWatcher:
|
|||||||
self._mark_activity()
|
self._mark_activity()
|
||||||
if self._stop_event.is_set():
|
if self._stop_event.is_set():
|
||||||
break
|
break
|
||||||
|
# 空转周期也要推进延迟重扫,否则移入目录后没有新事件就永远不会补扫
|
||||||
|
self._process_pending_rescans()
|
||||||
if not changes:
|
if not changes:
|
||||||
continue
|
continue
|
||||||
self._handle_changes(changes)
|
self._handle_changes(changes)
|
||||||
@@ -209,7 +236,13 @@ class LocalDirectoryWatcher:
|
|||||||
将 watchfiles 原始变更转换为目录监控事件。
|
将 watchfiles 原始变更转换为目录监控事件。
|
||||||
:param changes: watchfiles 返回的变更集合
|
:param changes: watchfiles 返回的变更集合
|
||||||
"""
|
"""
|
||||||
changes = self._expand_added_directories(changes)
|
self._dispatch_changes(self._expand_added_directories(changes))
|
||||||
|
|
||||||
|
def _dispatch_changes(self, changes: set[tuple[Change, str]]):
|
||||||
|
"""
|
||||||
|
将变更集合逐个派发给回调。
|
||||||
|
:param changes: 已展开的变更集合
|
||||||
|
"""
|
||||||
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
for change_type, path_str in sorted(changes, key=lambda item: item[1]):
|
||||||
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
|
# 批量整理可能持续较久,逐个文件刷新心跳,避免被误判为静默失效
|
||||||
self._mark_activity()
|
self._mark_activity()
|
||||||
@@ -217,10 +250,17 @@ class LocalDirectoryWatcher:
|
|||||||
continue
|
continue
|
||||||
event_path = Path(path_str)
|
event_path = Path(path_str)
|
||||||
event = self._build_event(change_type=change_type, event_path=event_path)
|
event = self._build_event(change_type=change_type, event_path=event_path)
|
||||||
if not event or event.is_directory:
|
if not event:
|
||||||
|
# 「文件已消失」与「挂载抖动瞬时不可见」在此刻无法区分,静默丢弃就是
|
||||||
|
# 永久漏件,一律登记重试:真删除的文件由重试队列在下个周期确认后自动放弃
|
||||||
|
self._notify_unreadable(event_path)
|
||||||
|
continue
|
||||||
|
if event.is_directory:
|
||||||
continue
|
continue
|
||||||
file_size = self._get_file_size(event_path)
|
file_size = self._get_file_size(event_path)
|
||||||
if file_size is None:
|
if file_size is None:
|
||||||
|
# 读取失败通常是挂载抖动,直接丢弃就是永久漏件,交给回调登记重试
|
||||||
|
self._notify_unreadable(event_path)
|
||||||
continue
|
continue
|
||||||
text = self._change_text(change_type)
|
text = self._change_text(change_type)
|
||||||
try:
|
try:
|
||||||
@@ -233,6 +273,44 @@ class LocalDirectoryWatcher:
|
|||||||
except Exception as err:
|
except Exception as err:
|
||||||
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
logger.error(f"处理本地目录监控事件失败: {path_str} - {err}")
|
||||||
|
|
||||||
|
@property
|
||||||
|
def DIRECTORY_RESCAN_DELAYS(self) -> tuple[int, ...]: # noqa: N802 保持与原类常量同名,兼容既有引用/测试
|
||||||
|
"""
|
||||||
|
获取当前生效的重扫轮次延迟秒数:每次访问都会重新解析 MONITOR_RESCAN_DELAYS
|
||||||
|
配置,配置热更新后无需重建监控线程即可生效;解析失败或未配置时回退默认值。
|
||||||
|
:return: 重扫轮次延迟秒数元组
|
||||||
|
"""
|
||||||
|
return self._parse_rescan_delays(getattr(settings, "MONITOR_RESCAN_DELAYS", None))
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def _parse_rescan_delays(cls, raw: Optional[str]) -> tuple[int, ...]:
|
||||||
|
"""
|
||||||
|
解析 MONITOR_RESCAN_DELAYS 配置为重扫轮次延迟秒数元组。
|
||||||
|
:param raw: 配置原始字符串,形如 "30,120,600,1800"
|
||||||
|
:return: 重扫轮次延迟秒数元组,解析失败或为空时回退 DEFAULT_RESCAN_DELAYS
|
||||||
|
"""
|
||||||
|
if not raw or not raw.strip():
|
||||||
|
return cls.DEFAULT_RESCAN_DELAYS
|
||||||
|
try:
|
||||||
|
delays = tuple(int(part.strip()) for part in raw.split(",") if part.strip())
|
||||||
|
if not delays or any(delay <= 0 for delay in delays):
|
||||||
|
raise ValueError(f"重扫延迟必须是正整数: {raw}")
|
||||||
|
return delays
|
||||||
|
except (TypeError, ValueError) as err:
|
||||||
|
logger.warn(f"MONITOR_RESCAN_DELAYS 配置无效({raw!r}),"
|
||||||
|
f"回退默认值 {cls.DEFAULT_RESCAN_DELAYS}: {err}")
|
||||||
|
return cls.DEFAULT_RESCAN_DELAYS
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _is_descendant_of_any(path: Path, candidates: set[Path]) -> bool:
|
||||||
|
"""
|
||||||
|
判断 path 是否是 candidates 中某个目录的子孙路径(不含自身)。
|
||||||
|
:param path: 待判断路径
|
||||||
|
:param candidates: 候选祖先目录集合
|
||||||
|
:return: 是否存在祖先命中
|
||||||
|
"""
|
||||||
|
return any(candidate in path.parents for candidate in candidates)
|
||||||
|
|
||||||
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
def _expand_added_directories(self, changes: set[tuple[Change, str]]) -> set[tuple[Change, str]]:
|
||||||
"""
|
"""
|
||||||
将整体移入监控范围的新增目录展开为内部文件事件。
|
将整体移入监控范围的新增目录展开为内部文件事件。
|
||||||
@@ -240,6 +318,12 @@ class LocalDirectoryWatcher:
|
|||||||
:return: 包含目录内新增文件的变更集合
|
:return: 包含目录内新增文件的变更集合
|
||||||
"""
|
"""
|
||||||
expanded_changes = set(changes)
|
expanded_changes = set(changes)
|
||||||
|
# 大目录树整体移入监控范围时,changes 里每一层子目录都会各自产生一次
|
||||||
|
# added 事件;先收集本批全部新增目录路径,用于判断某个新增目录是否还有
|
||||||
|
# 祖先目录也在本批新增中——只有「顶层」新增目录需要登记重扫,顶层目录的
|
||||||
|
# rglob 已经递归覆盖了全部子孙目录的内容,子目录重扫条目纯属冗余,还会在
|
||||||
|
# 大目录树场景下迅速打满重扫队列
|
||||||
|
added_dirs = {Path(path_str) for change_type, path_str in changes if change_type == Change.added}
|
||||||
for change_type, path_str in changes:
|
for change_type, path_str in changes:
|
||||||
if change_type != Change.added:
|
if change_type != Change.added:
|
||||||
continue
|
continue
|
||||||
@@ -247,15 +331,145 @@ class LocalDirectoryWatcher:
|
|||||||
try:
|
try:
|
||||||
if not event_path.is_dir():
|
if not event_path.is_dir():
|
||||||
continue
|
continue
|
||||||
for nested_path in event_path.rglob("*"):
|
except OSError as err:
|
||||||
|
logger.debug(f"读取新增路径类型失败: {event_path} - {err}")
|
||||||
|
continue
|
||||||
|
nested_paths, _, _ = self._collect_directory_files(event_path, exclude=set())
|
||||||
|
for nested_path_str in nested_paths:
|
||||||
|
expanded_changes.add((Change.added, nested_path_str))
|
||||||
|
if self._is_descendant_of_any(event_path, added_dirs):
|
||||||
|
continue
|
||||||
|
# 目录内容在 FUSE 上可能延迟可见,安排延迟重扫补齐本次看不到的文件
|
||||||
|
self._schedule_rescan(event_path, seen=nested_paths)
|
||||||
|
return expanded_changes
|
||||||
|
|
||||||
|
def _collect_directory_files(self, directory: Path, exclude: set[str]) -> tuple[set[str], bool, bool]:
|
||||||
|
"""
|
||||||
|
收集目录内需要处理的文件路径。
|
||||||
|
:param directory: 目录
|
||||||
|
:param exclude: 需要排除的路径(已处理过的)
|
||||||
|
:return: 三元组 (文件路径集合, 目录是否已不存在(终态,不算失败),
|
||||||
|
目录整体扫描是否失败(如顶层 rglob 抛出 OSError,通常是 FUSE
|
||||||
|
瞬时抖动,属于可重试的暂时性失败;与单个条目读取失败区分开,
|
||||||
|
后者不影响整体遍历,不计入失败)
|
||||||
|
"""
|
||||||
|
collected: set[str] = set()
|
||||||
|
try:
|
||||||
|
if not directory.is_dir():
|
||||||
|
# 目录已被删除或从未存在,是终态而非抖动,调用方应据此让条目
|
||||||
|
# 直接出队,不再计入失败重试
|
||||||
|
return collected, True, False
|
||||||
|
except OSError as err:
|
||||||
|
logger.debug(f"读取目录状态失败,暂视为可重试的扫描失败: {directory} - {err}")
|
||||||
|
return collected, False, True
|
||||||
|
try:
|
||||||
|
for nested_path in directory.rglob("*"):
|
||||||
|
try:
|
||||||
if not nested_path.is_file():
|
if not nested_path.is_file():
|
||||||
continue
|
continue
|
||||||
nested_path_str = nested_path.as_posix()
|
except OSError as err:
|
||||||
if self._watch_filter(Change.added, nested_path_str):
|
# 单个条目读取失败不应中断整个目录的遍历,不算整体扫描失败
|
||||||
expanded_changes.add((Change.added, nested_path_str))
|
logger.debug(f"读取目录内条目失败: {nested_path} - {err}")
|
||||||
except OSError as err:
|
continue
|
||||||
logger.debug(f"扫描新增目录失败: {event_path} - {err}")
|
nested_path_str = nested_path.as_posix()
|
||||||
return expanded_changes
|
if nested_path_str in exclude:
|
||||||
|
continue
|
||||||
|
if self._watch_filter(Change.added, nested_path_str):
|
||||||
|
collected.add(nested_path_str)
|
||||||
|
except OSError as err:
|
||||||
|
# 顶层 rglob 中断代表整个目录遍历失败(如 FUSE 瞬时抖动),与「目录
|
||||||
|
# 已删除」区分开:这里应视为可重试的暂时性失败,而不是静默返回空结果
|
||||||
|
logger.debug(f"扫描新增目录失败: {directory} - {err}")
|
||||||
|
return collected, False, True
|
||||||
|
return collected, False, False
|
||||||
|
|
||||||
|
def _schedule_rescan(self, directory: Path, seen: set[str]):
|
||||||
|
"""
|
||||||
|
安排一个新增目录的延迟重扫。
|
||||||
|
:param directory: 新增目录
|
||||||
|
:param seen: 首次展开时已处理的文件路径
|
||||||
|
"""
|
||||||
|
if not self.DIRECTORY_RESCAN_DELAYS:
|
||||||
|
return
|
||||||
|
if any(item["path"] == directory for item in self._pending_rescans):
|
||||||
|
# readdir 闪断等原因可能让同一目录产生两次 added 事件,从而被重复
|
||||||
|
# 展开、重复调用到这里;重复登记只会造成队列膨胀和重复扫描——新事件
|
||||||
|
# 覆盖到的文件已经在本次展开时派发过,保留已有条目、由它继续推进
|
||||||
|
# 重扫轮次即可
|
||||||
|
logger.debug(f"目录已在待重扫队列中,跳过重复登记: {directory}")
|
||||||
|
return
|
||||||
|
if self._is_descendant_of_any(directory, {item["path"] for item in self._pending_rescans}):
|
||||||
|
# 祖先目录的重扫会用 rglob 递归覆盖到这里,无需为子孙目录单独登记
|
||||||
|
logger.debug(f"目录的祖先已在待重扫队列中,跳过重复登记: {directory}")
|
||||||
|
return
|
||||||
|
if len(self._pending_rescans) >= self.MAX_PENDING_RESCANS:
|
||||||
|
# 队列打满意味着可能有目录的重扫机会被挤掉,之后可能永久漏文件,
|
||||||
|
# 需要 warn 级别可见,而不是默默丢弃
|
||||||
|
logger.warn(f"新增目录重扫队列已满(上限 {self.MAX_PENDING_RESCANS}),跳过登记: {directory}")
|
||||||
|
return
|
||||||
|
self._pending_rescans.append({
|
||||||
|
"path": directory,
|
||||||
|
"seen": set(seen),
|
||||||
|
"round": 0,
|
||||||
|
"due": time.monotonic() + self.DIRECTORY_RESCAN_DELAYS[0],
|
||||||
|
"failures": 0,
|
||||||
|
})
|
||||||
|
|
||||||
|
def _process_pending_rescans(self):
|
||||||
|
"""
|
||||||
|
对到期的新增目录做延迟重扫,补回首次展开时尚不可见的文件。
|
||||||
|
"""
|
||||||
|
if not self._pending_rescans:
|
||||||
|
return
|
||||||
|
now = time.monotonic()
|
||||||
|
due_items = [item for item in self._pending_rescans if item["due"] <= now]
|
||||||
|
if not due_items:
|
||||||
|
return
|
||||||
|
self._pending_rescans = [item for item in self._pending_rescans if item["due"] > now]
|
||||||
|
for item in due_items:
|
||||||
|
directory = item["path"]
|
||||||
|
new_paths, is_missing, scan_failed = self._collect_directory_files(directory, exclude=item["seen"])
|
||||||
|
if is_missing:
|
||||||
|
# 目录已不存在,是终态:直接出队,不再重新入队,也不计入失败重试
|
||||||
|
logger.debug(f"待重扫目录已不存在,结束重扫: {directory}")
|
||||||
|
continue
|
||||||
|
if scan_failed:
|
||||||
|
# 整体扫描失败(通常是 FUSE 瞬时抖动):本轮不消耗重扫轮次,
|
||||||
|
# 让条目原样重新入队,下一个监控周期立即再试;但要有失败上限,
|
||||||
|
# 避免目录长期不可访问时无限重试、占满队列
|
||||||
|
item["failures"] = item.get("failures", 0) + 1
|
||||||
|
if item["failures"] >= self.MAX_RESCAN_FAILURES:
|
||||||
|
logger.warn(
|
||||||
|
f"目录延迟重扫连续失败 {item['failures']} 次,放弃重扫: {directory}")
|
||||||
|
continue
|
||||||
|
logger.debug(
|
||||||
|
f"目录延迟重扫本轮扫描失败(第 {item['failures']} 次),不消耗轮次,稍后重试: {directory}")
|
||||||
|
self._pending_rescans.append(item)
|
||||||
|
continue
|
||||||
|
# 本轮扫描成功,重置失败计数
|
||||||
|
item["failures"] = 0
|
||||||
|
if new_paths:
|
||||||
|
logger.info(f"新增目录延迟重扫发现 {len(new_paths)} 个此前不可见的文件: {directory}")
|
||||||
|
self._dispatch_changes({(Change.added, path_str) for path_str in new_paths})
|
||||||
|
item["seen"].update(new_paths)
|
||||||
|
next_round = item["round"] + 1
|
||||||
|
if next_round < len(self.DIRECTORY_RESCAN_DELAYS):
|
||||||
|
item["round"] = next_round
|
||||||
|
item["due"] = now + self.DIRECTORY_RESCAN_DELAYS[next_round]
|
||||||
|
self._pending_rescans.append(item)
|
||||||
|
|
||||||
|
def _notify_unreadable(self, event_path: Path):
|
||||||
|
"""
|
||||||
|
通知回调登记读取失败的事件,等待重试。
|
||||||
|
:param event_path: 事件文件路径
|
||||||
|
"""
|
||||||
|
handler = getattr(self._callback, "event_unreadable", None)
|
||||||
|
if not callable(handler):
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
handler(event_path=event_path)
|
||||||
|
except Exception as err:
|
||||||
|
logger.error(f"登记待重试监控事件失败: {event_path} - {err}")
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
def _build_event(change_type: Change, event_path: Path) -> Optional[DirectoryChangeEvent]:
|
||||||
|
|||||||
@@ -168,6 +168,9 @@ class TransferInfo(BaseModel):
|
|||||||
need_scrape: Optional[bool] = False
|
need_scrape: Optional[bool] = False
|
||||||
# 是否需要通知
|
# 是否需要通知
|
||||||
need_notify: Optional[bool] = False
|
need_notify: Optional[bool] = False
|
||||||
|
# 是否因覆盖模式判定「不覆盖」而放弃整理。
|
||||||
|
# 这是一次正常的策略裁决而非整理故障,调用方据此决定是否写失败历史与推送失败通知
|
||||||
|
overwrite_skipped: Optional[bool] = False
|
||||||
|
|
||||||
def to_dict(self):
|
def to_dict(self):
|
||||||
"""
|
"""
|
||||||
|
|||||||
@@ -33,6 +33,7 @@ from app.startup.scheduler_initializer import (
|
|||||||
init_scheduler,
|
init_scheduler,
|
||||||
init_plugin_scheduler,
|
init_plugin_scheduler,
|
||||||
)
|
)
|
||||||
|
from app.startup.transfer_initializer import replay_pending_transfers
|
||||||
from app.startup.workflow_initializer import init_workflow, stop_workflow
|
from app.startup.workflow_initializer import init_workflow, stop_workflow
|
||||||
from app.utils.http import aclose_shared_async_transports
|
from app.utils.http import aclose_shared_async_transports
|
||||||
|
|
||||||
@@ -91,6 +92,8 @@ async def lifespan(app: FastAPI):
|
|||||||
init_scheduler()
|
init_scheduler()
|
||||||
# 初始化监控器
|
# 初始化监控器
|
||||||
init_monitor()
|
init_monitor()
|
||||||
|
# 回放上次未整理完的文件(后台线程,不阻塞启动)
|
||||||
|
replay_pending_transfers()
|
||||||
# 初始化命令
|
# 初始化命令
|
||||||
init_command()
|
init_command()
|
||||||
# 初始化工作流
|
# 初始化工作流
|
||||||
|
|||||||
@@ -0,0 +1,13 @@
|
|||||||
|
from app.chain.transfer import TransferChain
|
||||||
|
|
||||||
|
|
||||||
|
def replay_pending_transfers():
|
||||||
|
"""
|
||||||
|
回放上次进程退出时仍未整理完的文件。
|
||||||
|
|
||||||
|
整理队列是纯内存的,挂载挂死后的人工重启、版本升级、OOM、宿主重启都会让
|
||||||
|
队列连同「这些文件还没整理」这个事实一起蒸发;而已稳定落地的文件不会再产生
|
||||||
|
任何监控事件,也不会有新的补偿扫描起点,结果就是永久漏件。
|
||||||
|
回放本身在后台线程执行,不阻塞启动流程。
|
||||||
|
"""
|
||||||
|
TransferChain().replay_pending()
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
"""3.0.2
|
||||||
|
整理历史按源存储与源路径唯一
|
||||||
|
|
||||||
|
Revision ID: 7f5c1d2e3a4b
|
||||||
|
Revises: 8a4c7e1d2f90
|
||||||
|
Create Date: 2026-08-04
|
||||||
|
"""
|
||||||
|
|
||||||
|
from collections import defaultdict
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
|
||||||
|
revision = "7f5c1d2e3a4b"
|
||||||
|
down_revision = "8a4c7e1d2f90"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
TABLE_NAME = "transferhistory"
|
||||||
|
INDEX_NAME = "ux_transferhistory_src_storage"
|
||||||
|
INDEX_COLUMNS = ["src", "src_storage"]
|
||||||
|
|
||||||
|
transferhistory = sa.table(
|
||||||
|
TABLE_NAME,
|
||||||
|
sa.column("id", sa.Integer()),
|
||||||
|
sa.column("src", sa.String()),
|
||||||
|
sa.column("src_storage", sa.String()),
|
||||||
|
sa.column("status", sa.Boolean()),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _table_exists(inspector: sa.Inspector) -> bool:
|
||||||
|
"""检查整理历史表是否存在。"""
|
||||||
|
return TABLE_NAME in inspector.get_table_names()
|
||||||
|
|
||||||
|
|
||||||
|
def _has_unique_index(inspector: sa.Inspector) -> bool:
|
||||||
|
"""检查源路径与源存储的唯一索引是否已存在。"""
|
||||||
|
return any(
|
||||||
|
tuple(index.get("column_names") or []) == tuple(INDEX_COLUMNS)
|
||||||
|
and bool(index.get("unique"))
|
||||||
|
for index in inspector.get_indexes(TABLE_NAME)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _deduplicate_rows() -> None:
|
||||||
|
"""归一化旧存储值并按现有查重语义清理重复历史。"""
|
||||||
|
bind = op.get_bind()
|
||||||
|
bind.execute(
|
||||||
|
transferhistory.update()
|
||||||
|
.where(
|
||||||
|
sa.or_(
|
||||||
|
transferhistory.c.src_storage.is_(None),
|
||||||
|
transferhistory.c.src_storage == "",
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.values(src_storage="local")
|
||||||
|
)
|
||||||
|
|
||||||
|
rows = bind.execute(
|
||||||
|
sa.select(
|
||||||
|
transferhistory.c.id,
|
||||||
|
transferhistory.c.src,
|
||||||
|
transferhistory.c.src_storage,
|
||||||
|
transferhistory.c.status,
|
||||||
|
).where(transferhistory.c.src.is_not(None))
|
||||||
|
).mappings()
|
||||||
|
grouped_rows = defaultdict(list)
|
||||||
|
for row in rows:
|
||||||
|
grouped_rows[(row["src"], row["src_storage"])].append(row)
|
||||||
|
|
||||||
|
duplicate_ids = []
|
||||||
|
for group in grouped_rows.values():
|
||||||
|
# 旧运行时在同源混有成功和失败记录时优先返回成功记录;保留其中 ID 最新的一条,
|
||||||
|
# 既延续这一保护语义,也让唯一索引能安全建立。
|
||||||
|
retained = max(
|
||||||
|
group,
|
||||||
|
key=lambda row: (bool(row["status"]), row["id"]),
|
||||||
|
)
|
||||||
|
duplicate_ids.extend(
|
||||||
|
row["id"]
|
||||||
|
for row in group
|
||||||
|
if row["id"] != retained["id"]
|
||||||
|
)
|
||||||
|
if duplicate_ids:
|
||||||
|
bind.execute(
|
||||||
|
transferhistory.delete().where(transferhistory.c.id.in_(duplicate_ids))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _make_src_storage_required(inspector: sa.Inspector) -> None:
|
||||||
|
"""将源存储设为非空,使唯一索引同样约束本地存储记录。"""
|
||||||
|
column = next(
|
||||||
|
(
|
||||||
|
current
|
||||||
|
for current in inspector.get_columns(TABLE_NAME)
|
||||||
|
if current["name"] == "src_storage"
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
if not column or not column.get("nullable"):
|
||||||
|
return
|
||||||
|
with op.batch_alter_table(TABLE_NAME) as batch_op:
|
||||||
|
batch_op.alter_column(
|
||||||
|
"src_storage",
|
||||||
|
existing_type=sa.String(),
|
||||||
|
nullable=False,
|
||||||
|
server_default="local",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""归一化并唯一化整理历史的源路径记录。"""
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if not _table_exists(inspector):
|
||||||
|
return
|
||||||
|
|
||||||
|
_deduplicate_rows()
|
||||||
|
_make_src_storage_required(sa.inspect(op.get_bind()))
|
||||||
|
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if not _has_unique_index(inspector):
|
||||||
|
op.create_index(INDEX_NAME, TABLE_NAME, INDEX_COLUMNS, unique=True)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""移除整理历史源路径唯一约束。"""
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
if not _table_exists(inspector):
|
||||||
|
return
|
||||||
|
|
||||||
|
if _has_unique_index(inspector):
|
||||||
|
op.drop_index(INDEX_NAME, table_name=TABLE_NAME)
|
||||||
|
with op.batch_alter_table(TABLE_NAME) as batch_op:
|
||||||
|
batch_op.alter_column(
|
||||||
|
"src_storage",
|
||||||
|
existing_type=sa.String(),
|
||||||
|
nullable=True,
|
||||||
|
server_default=None,
|
||||||
|
)
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
"""3.0.3
|
||||||
|
新增待整理文件登记表
|
||||||
|
|
||||||
|
整理队列是纯内存的 queue.Queue,进程重启会让队列里的任务连同「这些文件还没
|
||||||
|
整理」这个事实一起蒸发,而已稳定落地的文件不会再产生任何监控事件,等于永久
|
||||||
|
漏件。该表只登记「存储 + 源文件路径」这一最小事实,供启动时回放。
|
||||||
|
|
||||||
|
Revision ID: e3d9f4b7c806
|
||||||
|
Revises: 7f5c1d2e3a4b
|
||||||
|
Create Date: 2026-08-10
|
||||||
|
"""
|
||||||
|
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "e3d9f4b7c806"
|
||||||
|
down_revision = "7f5c1d2e3a4b"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def _has_table(table_name: str) -> bool:
|
||||||
|
"""检查数据表是否已存在。"""
|
||||||
|
inspector = sa.inspect(op.get_bind())
|
||||||
|
return table_name in inspector.get_table_names()
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
"""
|
||||||
|
创建待整理文件登记表。
|
||||||
|
"""
|
||||||
|
if _has_table("transferpending"):
|
||||||
|
return
|
||||||
|
op.create_table(
|
||||||
|
"transferpending",
|
||||||
|
sa.Column("id", sa.Integer, primary_key=True, autoincrement=True),
|
||||||
|
sa.Column("storage", sa.String, nullable=False),
|
||||||
|
sa.Column("src_path", sa.String, nullable=False),
|
||||||
|
sa.Column("created_at", sa.String),
|
||||||
|
)
|
||||||
|
# 同一个文件重复入队只保留一条,回放时不会重复送入整理链
|
||||||
|
op.create_index(
|
||||||
|
"ux_transferpending_storage_path",
|
||||||
|
"transferpending",
|
||||||
|
["storage", "src_path"],
|
||||||
|
unique=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
"""
|
||||||
|
删除待整理文件登记表。
|
||||||
|
"""
|
||||||
|
if not _has_table("transferpending"):
|
||||||
|
return
|
||||||
|
op.drop_index("ux_transferpending_storage_path", table_name="transferpending")
|
||||||
|
op.drop_table("transferpending")
|
||||||
+8
-2
@@ -98,9 +98,15 @@ FROM prepare_package AS prepare_code
|
|||||||
WORKDIR /app
|
WORKDIR /app
|
||||||
|
|
||||||
COPY . .
|
COPY . .
|
||||||
RUN FRONTEND_VERSION=$(sed -n "s/^FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/p" /app/version.py) \
|
# 构建上下文中存在 frontend-dist/ 时优先使用本地前端产物(fork 自有前端),
|
||||||
|
# 否则回退下载官方 release 的 dist.zip
|
||||||
|
RUN if [ -d /app/frontend-dist ]; then \
|
||||||
|
mv /app/frontend-dist /public; \
|
||||||
|
else \
|
||||||
|
FRONTEND_VERSION=$(sed -n "s/^FRONTEND_VERSION\s*=\s*'\([^']*\)'/\1/p" /app/version.py) \
|
||||||
&& curl -sL "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${FRONTEND_VERSION}/dist.zip" | busybox unzip -d / - \
|
&& curl -sL "https://github.com/jxxghp/MoviePilot-Frontend/releases/download/${FRONTEND_VERSION}/dist.zip" | busybox unzip -d / - \
|
||||||
&& mv /dist /public \
|
&& mv /dist /public; \
|
||||||
|
fi \
|
||||||
&& curl -sL "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \
|
&& curl -sL "https://github.com/jxxghp/MoviePilot-Plugins/archive/refs/heads/main.zip" | busybox unzip -d /tmp - \
|
||||||
&& mv -f /tmp/MoviePilot-Plugins-main/plugins.v2/* /app/app/plugins/ \
|
&& mv -f /tmp/MoviePilot-Plugins-main/plugins.v2/* /app/app/plugins/ \
|
||||||
&& cat /tmp/MoviePilot-Plugins-main/package.json | jq -r 'to_entries[] | select(.value.v2 == true) | .key' | awk '{print tolower($0)}' | \
|
&& cat /tmp/MoviePilot-Plugins-main/package.json | jq -r 'to_entries[] | select(.value.v2 == true) | .key' | awk '{print tolower($0)}' | \
|
||||||
|
|||||||
@@ -0,0 +1,224 @@
|
|||||||
|
"""
|
||||||
|
本地文件系统操作代理测试。
|
||||||
|
|
||||||
|
代理存在的唯一理由:FUSE 挂载 block 型故障下,stat/listdir 这类调用永不返回,
|
||||||
|
而 Python 既不能中断已发出的系统调用、也不能强杀线程。放进子进程后,超时可以
|
||||||
|
真正 SIGKILL 回收——block 型故障被转换成各层已能处理的 crash 型故障。
|
||||||
|
|
||||||
|
这些测试固定三项不变量:语义与直接调用一致、超时可放弃、代理死亡后能自愈。
|
||||||
|
"""
|
||||||
|
import errno
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.filemanager.fsproxy import FileSystemProxy, FileSystemTimeout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def proxy():
|
||||||
|
"""
|
||||||
|
每个用例一个独立代理,用完回收进程。
|
||||||
|
"""
|
||||||
|
instance = FileSystemProxy(timeout=30)
|
||||||
|
yield instance
|
||||||
|
instance.close()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# 语义等价:代理调用的结果必须与直接调用一致
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_stat_matches_direct_call(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
stat 的结果要与直接调用一致,调用方才能无感替换。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "Movie.2024.mkv"
|
||||||
|
media.write_bytes(b"x" * 42)
|
||||||
|
|
||||||
|
result = proxy.stat(media)
|
||||||
|
|
||||||
|
assert result["size"] == 42
|
||||||
|
assert result["is_file"] is True
|
||||||
|
assert result["is_dir"] is False
|
||||||
|
assert result["mtime"] == pytest.approx(media.stat().st_mtime, abs=1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_stat_on_directory(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
目录的 stat 要正确标记类型。
|
||||||
|
"""
|
||||||
|
result = proxy.stat(tmp_path)
|
||||||
|
|
||||||
|
assert result["is_dir"] is True
|
||||||
|
assert result["is_file"] is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_stat_missing_raises_file_not_found(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
errno 必须还原成具体的 OSError 子类,调用方的既有异常分支才能继续生效。
|
||||||
|
"""
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
proxy.stat(tmp_path / "nope.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_exists_true_and_false(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
exists 的基本语义。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "a.mkv"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
|
||||||
|
assert proxy.exists(media) is True
|
||||||
|
assert proxy.exists(tmp_path / "missing.mkv") is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_listdir_matches_direct_call(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
listdir 的结果要与直接调用一致。
|
||||||
|
"""
|
||||||
|
for name in ("b.mkv", "a.mkv", "c.srt"):
|
||||||
|
(tmp_path / name).write_bytes(b"x")
|
||||||
|
|
||||||
|
assert proxy.listdir(tmp_path) == sorted(os.listdir(tmp_path))
|
||||||
|
|
||||||
|
|
||||||
|
def test_listdir_missing_raises_file_not_found(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
目录不存在要抛 FileNotFoundError。
|
||||||
|
"""
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
proxy.listdir(tmp_path / "nodir")
|
||||||
|
|
||||||
|
|
||||||
|
def test_rename_within_same_storage(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
同存储 rename 是第一版唯一放行的写操作:内核保证原子性,
|
||||||
|
强杀后要么完全成功要么完全没发生,不需要恢复语义。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "old.mkv"
|
||||||
|
dst = tmp_path / "new.mkv"
|
||||||
|
src.write_bytes(b"data")
|
||||||
|
|
||||||
|
assert proxy.rename(src, dst) is True
|
||||||
|
assert dst.read_bytes() == b"data"
|
||||||
|
assert not src.exists()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# 核心价值:超时可放弃
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_timeout_raises_and_kills_worker(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
这是整个代理存在的意义:挂载不返回时,调用方必须在有限时间内拿到异常,
|
||||||
|
而且冻住的进程要被真正杀掉——不能像线程那样永久悬挂。
|
||||||
|
"""
|
||||||
|
import app.modules.filemanager.fsproxy as fsproxy_module
|
||||||
|
|
||||||
|
# 用一个必定挂死的 worker 替身,模拟 stat 永不返回的挂载
|
||||||
|
stuck_worker = tmp_path / "stuck_worker.py"
|
||||||
|
stuck_worker.write_text("import time\nwhile True:\n time.sleep(60)\n", encoding="utf-8")
|
||||||
|
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck_worker)
|
||||||
|
|
||||||
|
proxy = FileSystemProxy(timeout=0.5)
|
||||||
|
try:
|
||||||
|
started = time.monotonic()
|
||||||
|
with pytest.raises(FileSystemTimeout) as excinfo:
|
||||||
|
proxy.stat(tmp_path / "whatever.mkv")
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
|
||||||
|
assert elapsed < 10, "超时后没有及时放弃"
|
||||||
|
assert excinfo.value.errno == errno.ETIMEDOUT
|
||||||
|
# 冻住的代理必须已被回收,否则每次故障都会泄漏一个进程
|
||||||
|
assert proxy._process is None
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_timeout_is_an_oserror():
|
||||||
|
"""
|
||||||
|
超时必须是 OSError 子类:整理链与监控 watcher 对 OSError 已有完整的退避
|
||||||
|
重试与登记逻辑,block 型故障经此转换后可直接复用,无需改动各层调用点。
|
||||||
|
"""
|
||||||
|
assert issubclass(FileSystemTimeout, OSError)
|
||||||
|
|
||||||
|
|
||||||
|
def test_proxy_recovers_after_timeout(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
超时杀掉代理后,下一次调用要能用新代理正常工作——否则一次挂载抖动
|
||||||
|
就会让文件操作永久不可用。
|
||||||
|
"""
|
||||||
|
import app.modules.filemanager.fsproxy as fsproxy_module
|
||||||
|
|
||||||
|
stuck_worker = tmp_path / "stuck_worker.py"
|
||||||
|
stuck_worker.write_text("import time\nwhile True:\n time.sleep(60)\n", encoding="utf-8")
|
||||||
|
real_worker = fsproxy_module._WORKER_PATH
|
||||||
|
|
||||||
|
proxy = FileSystemProxy(timeout=0.5)
|
||||||
|
try:
|
||||||
|
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck_worker)
|
||||||
|
with pytest.raises(FileSystemTimeout):
|
||||||
|
proxy.stat(tmp_path)
|
||||||
|
|
||||||
|
# 挂载恢复:换回真正的 worker,代理应自动重启并正常服务
|
||||||
|
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", real_worker)
|
||||||
|
media = tmp_path / "recovered.mkv"
|
||||||
|
media.write_bytes(b"ok")
|
||||||
|
assert proxy.stat(media)["size"] == 2
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_proxy_restarts_after_worker_dies(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
代理进程被外部杀掉(OOM、误杀)后,下一次调用要能自动重启。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "a.mkv"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
assert proxy.stat(media)["size"] == 1
|
||||||
|
|
||||||
|
# 模拟代理进程被外部杀死
|
||||||
|
proxy._process.kill()
|
||||||
|
proxy._process.wait(timeout=5)
|
||||||
|
|
||||||
|
assert proxy.stat(media)["size"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_does_not_import_app_package():
|
||||||
|
"""
|
||||||
|
worker 必须只依赖标准库:一旦触发 app/__init__.py 的导入链,启动成本会从
|
||||||
|
毫秒级涨到秒级,代理被强杀后的重启就不再可行。
|
||||||
|
"""
|
||||||
|
worker = Path("app/modules/filemanager/fsworker.py").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "from app." not in worker
|
||||||
|
assert "import app" not in worker
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_runs_standalone_without_app_on_path(tmp_path):
|
||||||
|
"""
|
||||||
|
直接执行 worker 脚本必须成功,且不依赖项目根在 sys.path 上
|
||||||
|
——这是「绕开 app 导入链」这一设计前提的实证。
|
||||||
|
"""
|
||||||
|
worker_path = Path("app/modules/filemanager/fsworker.py").resolve()
|
||||||
|
media = tmp_path / "x.mkv"
|
||||||
|
media.write_bytes(b"xyz")
|
||||||
|
|
||||||
|
result = subprocess.run(
|
||||||
|
[sys.executable, str(worker_path)],
|
||||||
|
input=json.dumps({"op": "stat", "path": str(media)}) + "\n",
|
||||||
|
capture_output=True,
|
||||||
|
text=True,
|
||||||
|
timeout=30,
|
||||||
|
cwd=str(tmp_path),
|
||||||
|
)
|
||||||
|
|
||||||
|
response = json.loads(result.stdout.strip())
|
||||||
|
assert response["ok"] is True
|
||||||
|
assert response["result"]["size"] == 3
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
"""
|
||||||
|
文件系统代理的流式复制测试。
|
||||||
|
|
||||||
|
复制大文件可能持续几小时,固定超时无法区分「正常但慢」和「已经挂死」。
|
||||||
|
worker 每秒上报一次进度作为心跳,父进程判定的是**两次上报之间的间隔**:
|
||||||
|
超过 stall 阈值收不到任何一行,才认定挂载无响应并强杀 worker。
|
||||||
|
|
||||||
|
这些测试固定的不变量:内容与时间戳正确、进度可回调、慢传输不被误杀、
|
||||||
|
挂死能被判定并回收、取消可即时生效。
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.filemanager.fsproxy import FileSystemProxy, FileSystemTimeout
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def proxy():
|
||||||
|
"""
|
||||||
|
每个用例一个独立代理,用完回收进程。
|
||||||
|
"""
|
||||||
|
instance = FileSystemProxy(timeout=30, stall_timeout=30)
|
||||||
|
yield instance
|
||||||
|
instance.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_preserves_content_and_mtime(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
复制结果的内容与修改时间必须与源一致。
|
||||||
|
|
||||||
|
时间戳要保留但权限不能覆盖:目标目录的默认权限与继承 ACL 是媒体库的访问
|
||||||
|
策略,用源文件权限盖掉会清除已继承的 ACL。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "Movie.2024.mkv"
|
||||||
|
src.write_bytes(b"payload" * 5000)
|
||||||
|
old = time.time() - 86400
|
||||||
|
os.utime(src, (old, old))
|
||||||
|
dst = tmp_path / "out.mkv"
|
||||||
|
size = src.stat().st_size
|
||||||
|
|
||||||
|
assert proxy.copy(src, dst) == {"copied": size, "total": size}
|
||||||
|
|
||||||
|
assert dst.read_bytes() == src.read_bytes()
|
||||||
|
assert dst.stat().st_mtime == pytest.approx(src.stat().st_mtime, abs=1)
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_empty_file(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
空文件不能因为没有任何数据块就走进异常分支。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "empty.mkv"
|
||||||
|
src.write_bytes(b"")
|
||||||
|
dst = tmp_path / "out.mkv"
|
||||||
|
|
||||||
|
assert proxy.copy(src, dst)["total"] == 0
|
||||||
|
assert dst.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_reports_progress(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
进度必须被回调出来,且单调递增、不超过 100。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "big.mkv"
|
||||||
|
src.write_bytes(b"x" * (3 * 1024 * 1024))
|
||||||
|
dst = tmp_path / "out.mkv"
|
||||||
|
|
||||||
|
seen = []
|
||||||
|
# 用小 chunk 让传输持续足够久,确保能观察到进度上报
|
||||||
|
assert proxy.copy(src, dst, progress_cb=seen.append, chunk_size=4096)
|
||||||
|
|
||||||
|
assert seen == sorted(seen), "进度不是单调递增的"
|
||||||
|
assert all(0 <= value <= 100 for value in seen)
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_missing_source_raises_file_not_found(tmp_path, proxy):
|
||||||
|
"""
|
||||||
|
源文件不存在要还原成 FileNotFoundError,调用方的既有分支才能生效。
|
||||||
|
"""
|
||||||
|
with pytest.raises(FileNotFoundError):
|
||||||
|
proxy.copy(tmp_path / "nope.mkv", tmp_path / "out.mkv")
|
||||||
|
|
||||||
|
|
||||||
|
def test_slow_transfer_is_not_killed(tmp_path):
|
||||||
|
"""
|
||||||
|
关键区分之一:传输很慢但仍在推进时,绝不能被判定为挂死。
|
||||||
|
|
||||||
|
stall 阈值取得远小于可能的总耗时——只要心跳不断,就必须一路跑完。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "slow.mkv"
|
||||||
|
src.write_bytes(b"x" * (2 * 1024 * 1024))
|
||||||
|
dst = tmp_path / "out.mkv"
|
||||||
|
|
||||||
|
proxy = FileSystemProxy(timeout=30, stall_timeout=3)
|
||||||
|
try:
|
||||||
|
assert proxy.copy(src, dst, chunk_size=8192)
|
||||||
|
assert dst.read_bytes() == src.read_bytes()
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stalled_transfer_is_detected_and_killed(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
关键区分之二:传输彻底不推进时必须被判定并强杀,而不是永久等待。
|
||||||
|
"""
|
||||||
|
import app.modules.filemanager.fsproxy as fsproxy_module
|
||||||
|
|
||||||
|
# worker 替身:读到请求后完全不响应,模拟卡死在挂载上的传输
|
||||||
|
stuck = tmp_path / "stuck_worker.py"
|
||||||
|
stuck.write_text(
|
||||||
|
"import sys, time\nsys.stdin.readline()\nwhile True:\n time.sleep(60)\n",
|
||||||
|
encoding="utf-8"
|
||||||
|
)
|
||||||
|
monkeypatch.setattr(fsproxy_module, "_WORKER_PATH", stuck)
|
||||||
|
|
||||||
|
proxy = FileSystemProxy(timeout=30, stall_timeout=0.5)
|
||||||
|
try:
|
||||||
|
started = time.monotonic()
|
||||||
|
with pytest.raises(FileSystemTimeout):
|
||||||
|
proxy.copy(tmp_path / "a.mkv", tmp_path / "b.mkv")
|
||||||
|
assert time.monotonic() - started < 15, "挂死的传输没有被及时放弃"
|
||||||
|
assert proxy._process is None, "冻住的代理进程没有被回收"
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_can_be_cancelled_midway(tmp_path):
|
||||||
|
"""
|
||||||
|
取消要即时生效:父进程收到进度就检查取消标记,命中即杀掉 worker 中断传输,
|
||||||
|
不必等它把整个文件读完。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "big.mkv"
|
||||||
|
src.write_bytes(b"x" * (4 * 1024 * 1024))
|
||||||
|
dst = tmp_path / "out.mkv"
|
||||||
|
|
||||||
|
proxy = FileSystemProxy(timeout=30, stall_timeout=30)
|
||||||
|
try:
|
||||||
|
assert proxy.copy(src, dst, cancel_cb=lambda: True, chunk_size=4096) is False
|
||||||
|
# 取消后代理已被回收,下一次调用会重启一个干净的 worker
|
||||||
|
assert proxy._process is None
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_falls_back_to_direct_when_disabled(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
代理关闭时复制退回进程内直接执行,行为与引入代理之前一致。
|
||||||
|
"""
|
||||||
|
import app.modules.filemanager.fsproxy as fsproxy_module
|
||||||
|
|
||||||
|
monkeypatch.setattr(fsproxy_module.settings, "FS_PROXY_ENABLED", False, raising=False)
|
||||||
|
src = tmp_path / "a.mkv"
|
||||||
|
src.write_bytes(b"direct" * 100)
|
||||||
|
dst = tmp_path / "b.mkv"
|
||||||
|
|
||||||
|
proxy = FileSystemProxy()
|
||||||
|
try:
|
||||||
|
assert proxy.copy(src, dst) is True
|
||||||
|
assert dst.read_bytes() == src.read_bytes()
|
||||||
|
# 关闭时不应启动任何子进程
|
||||||
|
assert proxy._process is None
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_direct_copy_honours_cancel(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
关闭代理时取消同样要生效,否则关掉开关就丢了取消能力。
|
||||||
|
"""
|
||||||
|
import app.modules.filemanager.fsproxy as fsproxy_module
|
||||||
|
|
||||||
|
monkeypatch.setattr(fsproxy_module.settings, "FS_PROXY_ENABLED", False, raising=False)
|
||||||
|
src = tmp_path / "a.mkv"
|
||||||
|
src.write_bytes(b"x" * 8192)
|
||||||
|
dst = tmp_path / "b.mkv"
|
||||||
|
|
||||||
|
proxy = FileSystemProxy()
|
||||||
|
try:
|
||||||
|
assert proxy.copy(src, dst, cancel_cb=lambda: True, chunk_size=1024) is False
|
||||||
|
finally:
|
||||||
|
proxy.close()
|
||||||
|
|
||||||
|
|
||||||
|
def test_worker_still_standalone_after_streaming_support():
|
||||||
|
"""
|
||||||
|
加了流式协议之后 worker 仍须只依赖标准库——一旦引入 app 导入链,
|
||||||
|
强杀后的重启成本会从毫秒级涨到秒级,整个代理方案就不成立了。
|
||||||
|
"""
|
||||||
|
worker = Path("app/modules/filemanager/fsworker.py").read_text(encoding="utf-8")
|
||||||
|
|
||||||
|
assert "from app." not in worker
|
||||||
|
assert "import app" not in worker
|
||||||
@@ -0,0 +1,207 @@
|
|||||||
|
"""
|
||||||
|
本地存储写入原子性测试。
|
||||||
|
|
||||||
|
copy / 跨盘 move / upload 原先直接写目标路径,进程被杀(OOM、重启、宿主断电、
|
||||||
|
SIGKILL)时目标目录会留下一个**半截的媒体文件,而且叫最终文件名**——媒体库会
|
||||||
|
把它扫进去,后续的「目标已存在」检查也会把它当成完成品。
|
||||||
|
|
||||||
|
改成「写临时名 → os.replace」后,os.replace 在同目录内由内核保证原子性:
|
||||||
|
中断只可能留下一个带专用后缀的隐藏文件,最终文件要么完整存在,要么根本不存在。
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.filemanager.storages.local import LocalStorage
|
||||||
|
from app.schemas import FileItem
|
||||||
|
|
||||||
|
|
||||||
|
def _fileitem(path: Path) -> FileItem:
|
||||||
|
"""
|
||||||
|
构造本地文件项。
|
||||||
|
:param path: 文件路径
|
||||||
|
:return: 文件项
|
||||||
|
"""
|
||||||
|
return FileItem(
|
||||||
|
storage="local",
|
||||||
|
type="file",
|
||||||
|
path=path.as_posix(),
|
||||||
|
name=path.name,
|
||||||
|
basename=path.stem,
|
||||||
|
extension=path.suffix[1:],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def storage():
|
||||||
|
"""
|
||||||
|
本地存储实例。
|
||||||
|
"""
|
||||||
|
return LocalStorage()
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_produces_complete_file(tmp_path, storage):
|
||||||
|
"""
|
||||||
|
正常复制的结果必须与直接复制一致。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "src" / "Movie.2024.mkv"
|
||||||
|
src.parent.mkdir()
|
||||||
|
src.write_bytes(b"payload" * 100)
|
||||||
|
dest_dir = tmp_path / "library"
|
||||||
|
dest_dir.mkdir()
|
||||||
|
|
||||||
|
assert storage.copy(_fileitem(src), dest_dir, "Movie.2024.mkv") is True
|
||||||
|
|
||||||
|
assert (dest_dir / "Movie.2024.mkv").read_bytes() == b"payload" * 100
|
||||||
|
# 复制完成后不能残留任何临时文件
|
||||||
|
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_leaves_no_final_name_on_failure(tmp_path, storage, monkeypatch):
|
||||||
|
"""
|
||||||
|
复制中途失败时,目标文件名绝不能出现——这是本次修复的核心。
|
||||||
|
|
||||||
|
失败点选在内容写完、替换之前,正是原实现留下「完整文件名的半成品」的位置。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "Movie.2024.mkv"
|
||||||
|
src.write_bytes(b"x" * 1000)
|
||||||
|
dest_dir = tmp_path / "library"
|
||||||
|
dest_dir.mkdir()
|
||||||
|
|
||||||
|
def boom(*_args, **_kwargs):
|
||||||
|
"""
|
||||||
|
模拟替换阶段被打断。
|
||||||
|
"""
|
||||||
|
raise OSError(28, "No space left on device")
|
||||||
|
|
||||||
|
monkeypatch.setattr(os, "replace", boom)
|
||||||
|
|
||||||
|
assert storage.copy(_fileitem(src), dest_dir, "Movie.2024.mkv") is False
|
||||||
|
|
||||||
|
# 最终文件名不存在,媒体库不会扫到半成品
|
||||||
|
assert not (dest_dir / "Movie.2024.mkv").exists()
|
||||||
|
# 临时文件也要清掉,不留垃圾
|
||||||
|
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_partial_file_is_hidden_and_suffixed(tmp_path, storage):
|
||||||
|
"""
|
||||||
|
临时文件必须同时满足:点开头(隐藏)+ 专用后缀。
|
||||||
|
|
||||||
|
媒体库按扩展名识别媒体文件,点开头又能让多数扫描器跳过;两者叠加保证
|
||||||
|
即使进程被 SIGKILL、临时文件残留,也不会被当成媒体收录。
|
||||||
|
"""
|
||||||
|
partial = storage._partial_path(tmp_path / "Movie.2024.mkv")
|
||||||
|
|
||||||
|
assert partial.name.startswith(".")
|
||||||
|
assert partial.name.endswith(LocalStorage.PARTIAL_SUFFIX)
|
||||||
|
# 必须与目标同目录,os.replace 才是原子的(跨文件系统会退化成拷贝)
|
||||||
|
assert partial.parent == tmp_path
|
||||||
|
assert partial.suffix != ".mkv"
|
||||||
|
|
||||||
|
|
||||||
|
def test_move_produces_complete_file_and_removes_source(tmp_path, storage, monkeypatch):
|
||||||
|
"""
|
||||||
|
跨盘移动完成后目标完整、源被删除。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "src" / "Movie.2024.mkv"
|
||||||
|
src.parent.mkdir()
|
||||||
|
src.write_bytes(b"data" * 50)
|
||||||
|
dest_dir = tmp_path / "library"
|
||||||
|
dest_dir.mkdir()
|
||||||
|
|
||||||
|
# 强制走跨盘路径(copy + unlink):只让「源 → 最终目标」的直接 rename 以
|
||||||
|
# EXDEV 失败,临时文件到目标的替换仍需正常工作
|
||||||
|
real_replace = os.replace
|
||||||
|
|
||||||
|
def exdev_for_source(source, target, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
模拟源与目标不在同一文件系统。
|
||||||
|
"""
|
||||||
|
if Path(source) == src:
|
||||||
|
raise OSError(18, "Invalid cross-device link")
|
||||||
|
return real_replace(source, target, *args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(os, "replace", exdev_for_source)
|
||||||
|
|
||||||
|
assert storage.move(_fileitem(src), dest_dir, "Movie.2024.mkv") is True
|
||||||
|
|
||||||
|
assert (dest_dir / "Movie.2024.mkv").read_bytes() == b"data" * 50
|
||||||
|
assert not src.exists()
|
||||||
|
assert not list(dest_dir.glob(f"*{LocalStorage.PARTIAL_SUFFIX}"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_move_keeps_source_when_copy_fails(tmp_path, storage, monkeypatch):
|
||||||
|
"""
|
||||||
|
跨盘移动失败时源文件必须保留——先删源再失败就是永久丢件。
|
||||||
|
"""
|
||||||
|
src = tmp_path / "src" / "Movie.2024.mkv"
|
||||||
|
src.parent.mkdir()
|
||||||
|
src.write_bytes(b"data")
|
||||||
|
dest_dir = tmp_path / "library"
|
||||||
|
dest_dir.mkdir()
|
||||||
|
|
||||||
|
def boom(*_args, **_kwargs):
|
||||||
|
"""
|
||||||
|
模拟直接移动与临时文件替换都失败(磁盘写满)。
|
||||||
|
"""
|
||||||
|
raise OSError(28, "No space left on device")
|
||||||
|
|
||||||
|
monkeypatch.setattr(os, "replace", boom)
|
||||||
|
|
||||||
|
assert storage.move(_fileitem(src), dest_dir, "Movie.2024.mkv") is False
|
||||||
|
|
||||||
|
assert src.read_bytes() == b"data", "移动失败却删掉了源文件,等于永久丢件"
|
||||||
|
assert not (dest_dir / "Movie.2024.mkv").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_path_move_is_noop(tmp_path, storage):
|
||||||
|
"""
|
||||||
|
源与目标相同时应直接成功,不能把文件写没了。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "Movie.2024.mkv"
|
||||||
|
media.write_bytes(b"keep")
|
||||||
|
|
||||||
|
assert storage.move(_fileitem(media), tmp_path, "Movie.2024.mkv") is True
|
||||||
|
assert media.read_bytes() == b"keep"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_removes_stale_partials_only(tmp_path, storage):
|
||||||
|
"""
|
||||||
|
清理只针对陈旧的临时文件:正在写入的临时文件与正常媒体文件都不能被误删。
|
||||||
|
|
||||||
|
不做全库扫描,只在实际写入的目录做局部清理——全库遍历在网络挂载上代价
|
||||||
|
不可接受,而残留只会出现在曾经写入过的目录里。
|
||||||
|
"""
|
||||||
|
stale = tmp_path / f".Old.mkv.1234{LocalStorage.PARTIAL_SUFFIX}"
|
||||||
|
fresh = tmp_path / f".New.mkv.5678{LocalStorage.PARTIAL_SUFFIX}"
|
||||||
|
media = tmp_path / "Keep.mkv"
|
||||||
|
for item in (stale, fresh, media):
|
||||||
|
item.write_bytes(b"x")
|
||||||
|
|
||||||
|
# 把陈旧临时文件的 mtime 推到阈值之前
|
||||||
|
old_time = os.stat(stale).st_mtime - LocalStorage.PARTIAL_STALE_SECONDS - 60
|
||||||
|
os.utime(stale, (old_time, old_time))
|
||||||
|
|
||||||
|
storage._cleanup_stale_partials(tmp_path)
|
||||||
|
|
||||||
|
assert not stale.exists(), "陈旧临时文件没有被清理"
|
||||||
|
assert fresh.exists(), "正在写入的临时文件被误删了"
|
||||||
|
assert media.exists(), "正常媒体文件被误删了"
|
||||||
|
|
||||||
|
|
||||||
|
def test_cleanup_never_raises(tmp_path, storage, monkeypatch):
|
||||||
|
"""
|
||||||
|
清理是尽力而为的旁路操作,任何失败都不能影响主流程。
|
||||||
|
"""
|
||||||
|
def boom(*_args, **_kwargs):
|
||||||
|
"""
|
||||||
|
模拟目录不可读。
|
||||||
|
"""
|
||||||
|
raise OSError(5, "Input/output error")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "glob", boom)
|
||||||
|
|
||||||
|
# 不抛异常即为通过
|
||||||
|
storage._cleanup_stale_partials(tmp_path)
|
||||||
@@ -95,16 +95,29 @@ def test_cross_device_move_keeps_target_permissions(tmp_path: Path) -> None:
|
|||||||
source_content = source.read_bytes()
|
source_content = source.read_bytes()
|
||||||
storage = _make_storage()
|
storage = _make_storage()
|
||||||
|
|
||||||
|
real_replace = local_storage_module.os.replace
|
||||||
|
|
||||||
|
def _exdev_for_source(src_path, dst_path, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
只让「源 → 最终目标」的直接移动以 EXDEV 失败;降级路径里
|
||||||
|
「临时文件 → 目标」的替换仍需正常工作,否则整个移动都会失败。
|
||||||
|
"""
|
||||||
|
if Path(src_path) == source:
|
||||||
|
raise OSError(errno.EXDEV, "跨设备移动")
|
||||||
|
return real_replace(src_path, dst_path, *args, **kwargs)
|
||||||
|
|
||||||
with (
|
with (
|
||||||
patch.object(
|
patch.object(
|
||||||
local_storage_module.LocalStorage,
|
local_storage_module.LocalStorage,
|
||||||
"_LocalStorage__should_show_progress",
|
"_LocalStorage__should_show_progress",
|
||||||
return_value=False,
|
return_value=False,
|
||||||
),
|
),
|
||||||
|
# move 现在显式用 os.replace 做同盘原子移动,EXDEV 失败才降级为
|
||||||
|
# 「写临时名 → 替换」的复制路径,因此在这里注入跨设备错误
|
||||||
patch.object(
|
patch.object(
|
||||||
shutil.os,
|
local_storage_module.os,
|
||||||
"rename",
|
"replace",
|
||||||
side_effect=OSError(errno.EXDEV, "跨设备移动"),
|
side_effect=_exdev_for_source,
|
||||||
),
|
),
|
||||||
):
|
):
|
||||||
result = storage.move(
|
result = storage.move(
|
||||||
|
|||||||
@@ -5,7 +5,14 @@ from app.api.endpoints.transfer import (
|
|||||||
query_manual_transfer_history,
|
query_manual_transfer_history,
|
||||||
)
|
)
|
||||||
from app.chain.transfer import TransferChain
|
from app.chain.transfer import TransferChain
|
||||||
|
from app.core.config import settings
|
||||||
from app.db.transferhistory_oper import TransferHistoryOper
|
from app.db.transferhistory_oper import TransferHistoryOper
|
||||||
|
from app.helper.transferhistory import (
|
||||||
|
clear_transfer_failures,
|
||||||
|
failed_retry_count,
|
||||||
|
max_failed_retries,
|
||||||
|
record_transfer_failure,
|
||||||
|
)
|
||||||
from app.schemas import FileItem, ManualTransferItem
|
from app.schemas import FileItem, ManualTransferItem
|
||||||
from tests.test_transfer_sync_extra_files import (
|
from tests.test_transfer_sync_extra_files import (
|
||||||
FakeMeta,
|
FakeMeta,
|
||||||
@@ -14,6 +21,11 @@ from tests.test_transfer_sync_extra_files import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_failed_retries(src_path, storage=None):
|
||||||
|
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, deleted):
|
def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, deleted):
|
||||||
"""隔离整理规划依赖,并记录历史及旧目标清理动作。"""
|
"""隔离整理规划依赖,并记录历史及旧目标清理动作。"""
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
@@ -42,6 +54,7 @@ def _patch_transfer_planning(monkeypatch, chain, fileitem, history, planned, del
|
|||||||
|
|
||||||
history_oper = SimpleNamespace(
|
history_oper = SimpleNamespace(
|
||||||
get_by_src=lambda src, storage=None: history,
|
get_by_src=lambda src, storage=None: history,
|
||||||
|
get_success_by_src=lambda src, storage=None: history if history and history.status else None,
|
||||||
get_by_dest=lambda dest, storage=None: None,
|
get_by_dest=lambda dest, storage=None: None,
|
||||||
delete=lambda history_id: deleted.append(("history", history_id)),
|
delete=lambda history_id: deleted.append(("history", history_id)),
|
||||||
)
|
)
|
||||||
@@ -286,6 +299,8 @@ def test_manual_transfer_removes_failed_history_before_retry(monkeypatch):
|
|||||||
dest_fileitem=old_dest.model_dump(),
|
dest_fileitem=old_dest.model_dump(),
|
||||||
download_hash=None,
|
download_hash=None,
|
||||||
downloader=None,
|
downloader=None,
|
||||||
|
src=None,
|
||||||
|
src_storage=None,
|
||||||
)
|
)
|
||||||
planned = []
|
planned = []
|
||||||
deleted = []
|
deleted = []
|
||||||
@@ -314,8 +329,8 @@ def test_manual_transfer_removes_failed_history_before_retry(monkeypatch):
|
|||||||
assert planned == [fileitem.path]
|
assert planned == [fileitem.path]
|
||||||
|
|
||||||
|
|
||||||
def test_automatic_transfer_keeps_failed_history(monkeypatch):
|
def test_automatic_transfer_retries_failed_history_within_retry_budget(monkeypatch):
|
||||||
"""自动整理仍应保留失败历史并跳过,避免失败任务自动循环。"""
|
"""失败重试次数未达上限(默认计数为 0)时,自动整理遇到失败记录应放行重试,避免一次瞬时故障永久锁死文件。"""
|
||||||
chain = make_transfer_chain()
|
chain = make_transfer_chain()
|
||||||
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
|
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
|
||||||
history = SimpleNamespace(
|
history = SimpleNamespace(
|
||||||
@@ -327,6 +342,8 @@ def test_automatic_transfer_keeps_failed_history(monkeypatch):
|
|||||||
).model_dump(),
|
).model_dump(),
|
||||||
download_hash=None,
|
download_hash=None,
|
||||||
downloader=None,
|
downloader=None,
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
)
|
)
|
||||||
planned = []
|
planned = []
|
||||||
deleted = []
|
deleted = []
|
||||||
@@ -339,17 +356,177 @@ def test_automatic_transfer_keeps_failed_history(monkeypatch):
|
|||||||
deleted,
|
deleted,
|
||||||
)
|
)
|
||||||
|
|
||||||
state, message = TransferChain.do_transfer(
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
try:
|
||||||
|
state, message = TransferChain.do_transfer(
|
||||||
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
background=False,
|
||||||
|
manual=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert state is True
|
||||||
|
assert message == ""
|
||||||
|
assert deleted == []
|
||||||
|
assert planned == [fileitem.path]
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_automatic_transfer_skips_failed_history_when_retry_budget_exhausted(monkeypatch):
|
||||||
|
"""失败重试次数已达上限时,自动整理遇到失败记录仍应被拦截、不进入整理链,避免失败任务自动循环。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
|
||||||
|
history = SimpleNamespace(
|
||||||
|
id=13,
|
||||||
|
status=False,
|
||||||
|
mode="copy",
|
||||||
|
dest_fileitem=make_fileitem(
|
||||||
|
"/library/Test Show/Test.Show.S01E01.mkv"
|
||||||
|
).model_dump(),
|
||||||
|
download_hash=None,
|
||||||
|
downloader=None,
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
|
)
|
||||||
|
planned = []
|
||||||
|
deleted = []
|
||||||
|
_patch_transfer_planning(
|
||||||
|
monkeypatch,
|
||||||
chain,
|
chain,
|
||||||
fileitem=fileitem,
|
fileitem,
|
||||||
background=False,
|
history,
|
||||||
manual=False,
|
planned,
|
||||||
|
deleted,
|
||||||
)
|
)
|
||||||
|
|
||||||
assert state is False
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
assert message == f"{fileitem.name} 已整理过"
|
try:
|
||||||
assert deleted == []
|
record_transfer_failure(fileitem.path, fileitem.storage)
|
||||||
assert planned == []
|
|
||||||
|
state, message = TransferChain.do_transfer(
|
||||||
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
background=False,
|
||||||
|
manual=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert state is False
|
||||||
|
assert message == f"{fileitem.name} 已整理过"
|
||||||
|
assert deleted == []
|
||||||
|
assert planned == []
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_automatic_transfer_new_version_bypasses_exhausted_retry_budget(monkeypatch):
|
||||||
|
"""失败预算耗尽后同路径新版本必须进入整理,且不能给下载任务补打已整理标签。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
|
||||||
|
fileitem.size = 200
|
||||||
|
history = SimpleNamespace(
|
||||||
|
id=15,
|
||||||
|
status=False,
|
||||||
|
mode="copy",
|
||||||
|
dest_fileitem=None,
|
||||||
|
download_hash="abc123",
|
||||||
|
downloader="qbittorrent",
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
|
src_fileitem={"size": 100},
|
||||||
|
)
|
||||||
|
planned = []
|
||||||
|
deleted = []
|
||||||
|
_patch_transfer_planning(
|
||||||
|
monkeypatch,
|
||||||
|
chain,
|
||||||
|
fileitem,
|
||||||
|
history,
|
||||||
|
planned,
|
||||||
|
deleted,
|
||||||
|
)
|
||||||
|
completed = []
|
||||||
|
chain.transfer_completed = lambda download_hash, downloader: completed.append(
|
||||||
|
(download_hash, downloader)
|
||||||
|
)
|
||||||
|
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
try:
|
||||||
|
record_transfer_failure(fileitem.path, fileitem.storage, file_size=100)
|
||||||
|
|
||||||
|
state, message = TransferChain.do_transfer(
|
||||||
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
downloader="qbittorrent",
|
||||||
|
download_hash="abc123",
|
||||||
|
background=False,
|
||||||
|
manual=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert state is True
|
||||||
|
assert message == ""
|
||||||
|
assert deleted == []
|
||||||
|
assert planned == [fileitem.path]
|
||||||
|
assert completed == []
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_manual_transfer_bypasses_retry_budget_when_exhausted(monkeypatch):
|
||||||
|
"""
|
||||||
|
手动整理不受重试次数上限限制:manual=True 时查重闸根本不参与判定
|
||||||
|
(do_transfer 中 `if transferd and not manual` 已把 manual 路径排除在外),
|
||||||
|
即使失败计数已达上限(自动路径会因此拦截),仍应清理失败记录、重置计数并放行重试。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
fileitem = make_fileitem("/downloads/Test.Show.S01E01.mkv")
|
||||||
|
old_dest = make_fileitem("/library/Test Show/Test.Show.S01E01.mkv")
|
||||||
|
history = SimpleNamespace(
|
||||||
|
id=14,
|
||||||
|
status=False,
|
||||||
|
mode="copy",
|
||||||
|
dest_fileitem=old_dest.model_dump(),
|
||||||
|
download_hash=None,
|
||||||
|
downloader=None,
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
|
)
|
||||||
|
planned = []
|
||||||
|
deleted = []
|
||||||
|
_patch_transfer_planning(
|
||||||
|
monkeypatch,
|
||||||
|
chain,
|
||||||
|
fileitem,
|
||||||
|
history,
|
||||||
|
planned,
|
||||||
|
deleted,
|
||||||
|
)
|
||||||
|
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
try:
|
||||||
|
record_transfer_failure(fileitem.path, fileitem.storage)
|
||||||
|
assert failed_retry_count(fileitem.path, fileitem.storage) >= max_failed_retries()
|
||||||
|
|
||||||
|
state, message = TransferChain.do_transfer(
|
||||||
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
background=False,
|
||||||
|
manual=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert state is True
|
||||||
|
assert message == ""
|
||||||
|
assert deleted == [
|
||||||
|
("target", old_dest.path),
|
||||||
|
("history", history.id),
|
||||||
|
]
|
||||||
|
assert planned == [fileitem.path]
|
||||||
|
assert failed_retry_count(fileitem.path, fileitem.storage) == 0
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
|
||||||
def test_manual_transfer_keeps_success_history_without_confirmation(monkeypatch):
|
def test_manual_transfer_keeps_success_history_without_confirmation(monkeypatch):
|
||||||
@@ -402,6 +579,8 @@ def test_manual_reorganize_removes_success_history_and_old_target(monkeypatch):
|
|||||||
dest_fileitem=old_dest.model_dump(),
|
dest_fileitem=old_dest.model_dump(),
|
||||||
download_hash=None,
|
download_hash=None,
|
||||||
downloader=None,
|
downloader=None,
|
||||||
|
src=None,
|
||||||
|
src_storage=None,
|
||||||
)
|
)
|
||||||
planned = []
|
planned = []
|
||||||
deleted = []
|
deleted = []
|
||||||
@@ -442,6 +621,8 @@ def test_manual_reorganize_keeps_successful_move_target_as_source(monkeypatch):
|
|||||||
dest_fileitem=fileitem.model_dump(),
|
dest_fileitem=fileitem.model_dump(),
|
||||||
download_hash=None,
|
download_hash=None,
|
||||||
downloader=None,
|
downloader=None,
|
||||||
|
src=None,
|
||||||
|
src_storage=None,
|
||||||
)
|
)
|
||||||
planned = []
|
planned = []
|
||||||
deleted = []
|
deleted = []
|
||||||
@@ -480,6 +661,8 @@ def test_forced_manual_reorganize_still_removes_history(monkeypatch):
|
|||||||
dest_fileitem=old_dest.model_dump(),
|
dest_fileitem=old_dest.model_dump(),
|
||||||
download_hash=None,
|
download_hash=None,
|
||||||
downloader=None,
|
downloader=None,
|
||||||
|
src=None,
|
||||||
|
src_storage=None,
|
||||||
)
|
)
|
||||||
planned = []
|
planned = []
|
||||||
deleted = []
|
deleted = []
|
||||||
|
|||||||
@@ -0,0 +1,278 @@
|
|||||||
|
"""目录监控分发器的整理历史查重与整理异常重试测试。"""
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.helper.transferhistory import (
|
||||||
|
clear_transfer_failures,
|
||||||
|
failed_retry_count,
|
||||||
|
record_transfer_failure,
|
||||||
|
)
|
||||||
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
|
|
||||||
|
|
||||||
|
def _build_dispatcher() -> TransferDispatcher:
|
||||||
|
"""
|
||||||
|
构造测试用整理分发器,使用普通字典充当去重缓存。
|
||||||
|
:return: 整理分发器
|
||||||
|
"""
|
||||||
|
return TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
|
|
||||||
|
|
||||||
|
def _history(status: bool = True, size=None, src_fileitem=..., src=None, src_storage=None,
|
||||||
|
history_id: int = 1):
|
||||||
|
"""
|
||||||
|
构造整理历史记录替身。
|
||||||
|
:param status: 整理是否成功
|
||||||
|
:param size: 记录中的源文件大小
|
||||||
|
:param src_fileitem: 直接指定源文件项,默认按 size 生成
|
||||||
|
:param src: 记录源路径,未指定时查重闸的失败重试计数按 None 处理(恒为 0)
|
||||||
|
:param src_storage: 记录源存储
|
||||||
|
:param history_id: 记录 ID,describe_history_gate 的日志文案需要
|
||||||
|
:return: 整理历史记录替身
|
||||||
|
"""
|
||||||
|
if src_fileitem is ...:
|
||||||
|
src_fileitem = {"size": size}
|
||||||
|
return SimpleNamespace(id=history_id, status=status, src_fileitem=src_fileitem,
|
||||||
|
src=src, src_storage=src_storage)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_failed_retries(src_path, storage=None):
|
||||||
|
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_history(monkeypatch, record=None, success_record=None) -> MagicMock:
|
||||||
|
"""
|
||||||
|
替换整理历史查询,返回指定记录。
|
||||||
|
:param monkeypatch: pytest monkeypatch
|
||||||
|
:param record: get_by_src 返回的记录
|
||||||
|
:param success_record: 成功记录二次确认的返回值
|
||||||
|
:return: 整理历史操作替身
|
||||||
|
"""
|
||||||
|
oper = MagicMock()
|
||||||
|
oper.get_by_src.return_value = record
|
||||||
|
oper.get_success_by_src.return_value = success_record
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", MagicMock(return_value=oper))
|
||||||
|
return oper
|
||||||
|
|
||||||
|
|
||||||
|
def _patch_chain(monkeypatch, side_effect=None) -> MagicMock:
|
||||||
|
"""
|
||||||
|
替换整理链,记录整理调用。
|
||||||
|
:param monkeypatch: pytest monkeypatch
|
||||||
|
:param side_effect: do_transfer 的副作用
|
||||||
|
:return: 整理链替身
|
||||||
|
"""
|
||||||
|
chain = MagicMock()
|
||||||
|
if side_effect is not None:
|
||||||
|
chain.do_transfer.side_effect = side_effect
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", MagicMock(return_value=chain))
|
||||||
|
return chain
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_history_goes_to_transfer(monkeypatch):
|
||||||
|
"""没有任何整理记录的文件应直接进入整理链。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=None)
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is True
|
||||||
|
chain.do_transfer.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_history_is_retried_within_retry_budget(monkeypatch):
|
||||||
|
"""失败重试次数未达上限(默认计数为 0)时,失败的整理记录不得永久锁死文件,应放行重试。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=_history(status=False, size=100))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is True
|
||||||
|
chain.do_transfer.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_history_is_skipped_when_retry_budget_exhausted(monkeypatch):
|
||||||
|
"""失败重试次数已达上限时,失败的整理记录仍应被拦截、跳过整理。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
src_path = "/downloads/a.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch,
|
||||||
|
record=_history(status=False, size=100, src=src_path, src_storage="local"))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path(src_path), file_size=100) is False
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_history_new_version_bypasses_exhausted_retry_budget(monkeypatch):
|
||||||
|
"""失败预算耗尽后新版本仍应进入整理链,而不是被错误标记为已处理。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
src_path = "/downloads/failed-new-version.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local", file_size=100)
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(
|
||||||
|
monkeypatch,
|
||||||
|
record=_history(
|
||||||
|
status=False,
|
||||||
|
size=100,
|
||||||
|
src=src_path,
|
||||||
|
src_storage="local",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert (
|
||||||
|
dispatcher.handle_file(
|
||||||
|
storage="local",
|
||||||
|
event_path=Path(src_path),
|
||||||
|
file_size=200,
|
||||||
|
)
|
||||||
|
is True
|
||||||
|
)
|
||||||
|
chain.do_transfer.assert_called_once()
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_should_skip_by_history_returns_true_when_retry_budget_exhausted(monkeypatch):
|
||||||
|
"""直接调用 _should_skip_by_history:失败计数达到上限时应判定为跳过。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
src_path = "/downloads/b.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
_patch_history(monkeypatch,
|
||||||
|
record=_history(status=False, size=100, src=src_path, src_storage="local"))
|
||||||
|
|
||||||
|
result = TransferDispatcher._should_skip_by_history(
|
||||||
|
storage="local", src_path=src_path, file_size=100
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_history_with_changed_size_is_retried(monkeypatch):
|
||||||
|
"""同路径重新上传的不同版本应放行,由整理链的覆盖模式决断。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=_history(status=True, size=100))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=200) is True
|
||||||
|
chain.do_transfer.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_history_without_change_is_skipped(monkeypatch):
|
||||||
|
"""已成功整理且文件未变化时跳过,并留下 debug 痕迹便于排查。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=_history(status=True, size=100))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
recorder = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.logger", recorder)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
assert any("跳过" in str(call.args[0]) for call in recorder.debug.call_args_list)
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_history_without_size_info_is_skipped(monkeypatch):
|
||||||
|
"""记录中缺少源文件大小时无法比对,保守跳过而不是重复整理。"""
|
||||||
|
chain_calls = []
|
||||||
|
for src_fileitem in (None, {}, {"size": None}, {"size": "未知"}, "bad-json"):
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=_history(status=True, src_fileitem=src_fileitem))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"),
|
||||||
|
file_size=100) is False
|
||||||
|
chain_calls.append(chain.do_transfer.call_count)
|
||||||
|
assert chain_calls == [0, 0, 0, 0, 0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_bluray_folder_without_file_size_is_skipped(monkeypatch):
|
||||||
|
"""蓝光原盘目录没有文件大小可比对,已成功整理过时应跳过。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
oper = _patch_history(monkeypatch, record=_history(status=True, size=100))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local",
|
||||||
|
event_path=Path("/downloads/Movie/BDMV/STREAM/00000.m2ts"),
|
||||||
|
file_size=None) is False
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
# 蓝光目录的整理记录源路径带尾斜杠,查询必须原样传入
|
||||||
|
assert oper.get_by_src.call_args.args[0] == "/downloads/Movie/"
|
||||||
|
|
||||||
|
|
||||||
|
def test_success_history_wins_over_failed_history(monkeypatch):
|
||||||
|
"""同一源路径同时存在成功与失败记录时,以成功记录为准。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch,
|
||||||
|
record=_history(status=False, size=100),
|
||||||
|
success_record=_history(status=True, size=100))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_history_query_error_registers_pending(monkeypatch):
|
||||||
|
"""整理历史查询异常仍应登记待重试,而不是被当作已整理跳过。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
oper = MagicMock()
|
||||||
|
oper.get_by_src.side_effect = RuntimeError("数据库不可用")
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.TransferHistoryOper", MagicMock(return_value=oper))
|
||||||
|
chain = _patch_chain(monkeypatch)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
assert list(dispatcher._pending_retries) == ["local:/downloads/a.mkv"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_transfer_exception_registers_pending_and_retries(monkeypatch):
|
||||||
|
"""整理执行抛异常的文件必须登记待重试,否则已落地的文件永久丢失。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=None)
|
||||||
|
chain = _patch_chain(monkeypatch, side_effect=RuntimeError("数据库瞬断"))
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100) is False
|
||||||
|
assert list(dispatcher._pending_retries) == ["local:/downloads/a.mkv"]
|
||||||
|
|
||||||
|
# 故障恢复后由健康检查周期驱动重试
|
||||||
|
chain.do_transfer.side_effect = None
|
||||||
|
dispatcher.retry_pending()
|
||||||
|
|
||||||
|
assert chain.do_transfer.call_count == 2
|
||||||
|
assert dispatcher._pending_retries == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_transfer_exception_retry_keeps_attempt_count(monkeypatch):
|
||||||
|
"""整理持续异常时重试次数要累计,避免无限重试。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=None)
|
||||||
|
_patch_chain(monkeypatch, side_effect=RuntimeError("持续失败"))
|
||||||
|
|
||||||
|
dispatcher.handle_file(storage="local", event_path=Path("/downloads/a.mkv"), file_size=100)
|
||||||
|
dispatcher.retry_pending()
|
||||||
|
|
||||||
|
assert dispatcher._pending_retries["local:/downloads/a.mkv"]["attempts"] == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_bluray_retry_uses_origin_event_path(monkeypatch):
|
||||||
|
"""蓝光原盘整理异常后应按原始事件路径重试,重试时重新解析目录。"""
|
||||||
|
dispatcher = _build_dispatcher()
|
||||||
|
_patch_history(monkeypatch, record=None)
|
||||||
|
_patch_chain(monkeypatch, side_effect=RuntimeError("整理失败"))
|
||||||
|
event_path = Path("/downloads/Movie/BDMV/STREAM/00000.m2ts")
|
||||||
|
|
||||||
|
dispatcher.handle_file(storage="local", event_path=event_path, file_size=None)
|
||||||
|
|
||||||
|
assert list(dispatcher._pending_retries) == [f"local:{event_path.as_posix()}"]
|
||||||
@@ -0,0 +1,49 @@
|
|||||||
|
"""
|
||||||
|
分发瞬间 flap-out 事件不丢弃测试(§5.2③)。
|
||||||
|
|
||||||
|
FUSE 挂载抖动时,事件到达派发环节的瞬间文件可能恰好从视图中消失
|
||||||
|
(exists() 为 False)。原实现在路径"确认不存在"时静默丢弃事件,
|
||||||
|
而 flap-out 与真删除在这一瞬间无法区分,丢弃就是永久漏件。
|
||||||
|
正确行为:一律登记待重试,由重试队列在 60s 周期里区分
|
||||||
|
"恢复可见(继续整理)"与"确实已删除(自动放弃)"。
|
||||||
|
"""
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from tempfile import TemporaryDirectory
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
from watchfiles import Change
|
||||||
|
|
||||||
|
from app.monitor.watcher import LocalDirectoryWatcher
|
||||||
|
|
||||||
|
|
||||||
|
class FlapOutDispatchTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self._tmpdir = TemporaryDirectory()
|
||||||
|
self.mon_path = Path(self._tmpdir.name)
|
||||||
|
self.callback = Mock()
|
||||||
|
self.watcher = LocalDirectoryWatcher(
|
||||||
|
mon_path=self.mon_path, callback=self.callback)
|
||||||
|
|
||||||
|
def tearDown(self):
|
||||||
|
self._tmpdir.cleanup()
|
||||||
|
|
||||||
|
def test_vanished_file_event_registers_retry(self):
|
||||||
|
"""事件路径已不可见(flap-out 或删除)时必须登记重试,而非静默丢弃。"""
|
||||||
|
vanished = self.mon_path / "vanished.mkv"
|
||||||
|
self.watcher._dispatch_changes({(Change.added, vanished.as_posix())})
|
||||||
|
self.callback.event_unreadable.assert_called_once_with(event_path=vanished)
|
||||||
|
self.callback.event_handler.assert_not_called()
|
||||||
|
|
||||||
|
def test_existing_file_event_still_dispatched(self):
|
||||||
|
"""正常存在的文件事件照常派发,不受重试登记逻辑影响。"""
|
||||||
|
existing = self.mon_path / "existing.mkv"
|
||||||
|
existing.write_bytes(b"data")
|
||||||
|
self.watcher._dispatch_changes({(Change.added, existing.as_posix())})
|
||||||
|
self.callback.event_handler.assert_called_once()
|
||||||
|
self.callback.event_unreadable.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,259 @@
|
|||||||
|
import os
|
||||||
|
import time
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from watchfiles import Change
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||||
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
|
from app.monitor.recovery import RecoveryExecutor
|
||||||
|
from app.monitor.syslimits import decide_monitor_mode
|
||||||
|
from app.utils.system import SystemUtils
|
||||||
|
|
||||||
|
|
||||||
|
def _build_monitor(handle_file: MagicMock = None):
|
||||||
|
"""
|
||||||
|
构造带分发器的测试用 Monitor 骨架。
|
||||||
|
:param handle_file: 替换分发器 handle_file 的替身
|
||||||
|
:return: (Monitor 骨架, 分发器)
|
||||||
|
"""
|
||||||
|
from threading import Lock
|
||||||
|
monitor = object.__new__(Monitor)
|
||||||
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
|
if handle_file is not None:
|
||||||
|
dispatcher.handle_file = handle_file
|
||||||
|
monitor._dispatcher = dispatcher
|
||||||
|
monitor._watchers = []
|
||||||
|
monitor._watcher_lock = Lock()
|
||||||
|
monitor._alerted_paths = {}
|
||||||
|
monitor._restart_marks = {}
|
||||||
|
monitor._stable_cycles = {}
|
||||||
|
monitor._isolated = {}
|
||||||
|
monitor._recovery = RecoveryExecutor()
|
||||||
|
return monitor, dispatcher
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_watcher(mon_path, restart_count=0):
|
||||||
|
"""
|
||||||
|
构造存活且未静默失效的监控线程替身。
|
||||||
|
:param mon_path: 监控目录
|
||||||
|
:param restart_count: 累计自动重启次数
|
||||||
|
:return: 监控线程替身
|
||||||
|
"""
|
||||||
|
watcher = MagicMock()
|
||||||
|
watcher.watch_path = mon_path
|
||||||
|
watcher.is_alive.return_value = True
|
||||||
|
watcher.is_stalled.return_value = False
|
||||||
|
watcher.restart_count = restart_count
|
||||||
|
return watcher
|
||||||
|
|
||||||
|
|
||||||
|
def test_compensation_scan_covers_files_with_old_mtime(tmp_path):
|
||||||
|
"""
|
||||||
|
网盘挂载转存/移动文件会保留原始 mtime,补偿扫描不能因 mtime 旧就漏掉文件,
|
||||||
|
但仍应按 mtime 从新到旧优先处理。
|
||||||
|
"""
|
||||||
|
stale = tmp_path / "old.mkv"
|
||||||
|
fresh = tmp_path / "new.mkv"
|
||||||
|
stale.write_bytes(b"x")
|
||||||
|
fresh.write_bytes(b"y")
|
||||||
|
now = time.time()
|
||||||
|
os.utime(stale, (now - 86400 * 365, now - 86400 * 365))
|
||||||
|
os.utime(fresh, (now, now))
|
||||||
|
handled = MagicMock(return_value=True)
|
||||||
|
monitor, _ = _build_monitor(handle_file=handled)
|
||||||
|
|
||||||
|
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=now - 600)
|
||||||
|
|
||||||
|
handled_paths = [call.kwargs["event_path"] for call in handled.call_args_list]
|
||||||
|
assert handled_paths == [fresh, stale]
|
||||||
|
|
||||||
|
|
||||||
|
def test_compensation_scan_limits_single_batch(tmp_path, monkeypatch):
|
||||||
|
"""候选文件超过上限时只处理最新的一批,避免大目录把整理链与数据库压垮。"""
|
||||||
|
monkeypatch.setattr(Monitor, "MAX_COMPENSATION_FILES", 2)
|
||||||
|
now = time.time()
|
||||||
|
files = []
|
||||||
|
for index in range(3):
|
||||||
|
target = tmp_path / f"{index}.mkv"
|
||||||
|
target.write_bytes(b"x")
|
||||||
|
os.utime(target, (now - index * 60, now - index * 60))
|
||||||
|
files.append(target)
|
||||||
|
handled = MagicMock(return_value=True)
|
||||||
|
monitor, _ = _build_monitor(handle_file=handled)
|
||||||
|
|
||||||
|
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=now - 600)
|
||||||
|
|
||||||
|
handled_paths = [call.kwargs["event_path"] for call in handled.call_args_list]
|
||||||
|
assert handled_paths == [files[0], files[1]]
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchdog_compensates_after_internal_restart(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
watcher 内部退避重启期间落地的文件会被新基线快照静默吸收,健康检查发现
|
||||||
|
重启计数增长时应补扫一次,且同一次重启不得反复补扫。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock())
|
||||||
|
monitor, _ = _build_monitor(handle_file=MagicMock(return_value=True))
|
||||||
|
watcher = _fake_watcher(tmp_path, restart_count=1)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
started = []
|
||||||
|
setattr(monitor, "_Monitor__start_compensation", lambda **kwargs: started.append(kwargs))
|
||||||
|
|
||||||
|
monitor._Monitor__check_watchers()
|
||||||
|
assert len(started) == 1
|
||||||
|
assert started[0]["mon_path"] == tmp_path
|
||||||
|
# 停摆起点无法精确观测,应保守回溯到最坏情况
|
||||||
|
assert 0 < time.time() - started[0]["since"] <= Monitor.RESTART_STALL_LOOKBACK + 5
|
||||||
|
|
||||||
|
monitor._Monitor__check_watchers()
|
||||||
|
monitor._Monitor__check_watchers()
|
||||||
|
assert len(started) == 1
|
||||||
|
|
||||||
|
watcher.restart_count = 2
|
||||||
|
monitor._Monitor__check_watchers()
|
||||||
|
assert len(started) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_compensation_scan_skips_non_candidate_files(tmp_path):
|
||||||
|
"""补偿扫描应跳过不属于监控扩展名的文件。"""
|
||||||
|
other = tmp_path / "note.txt"
|
||||||
|
other.write_bytes(b"x")
|
||||||
|
handled = MagicMock(return_value=True)
|
||||||
|
monitor, _ = _build_monitor(handle_file=handled)
|
||||||
|
|
||||||
|
monitor._Monitor__compensate_scan(mon_path=tmp_path, since=time.time() - 60)
|
||||||
|
|
||||||
|
handled.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_compensation_skipped_without_activity_record(tmp_path, monkeypatch):
|
||||||
|
"""没有活动记录就没有可靠的停摆起点,应跳过补偿扫描。"""
|
||||||
|
monitor, _ = _build_monitor(handle_file=MagicMock())
|
||||||
|
started = []
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.Thread",
|
||||||
|
lambda **kwargs: started.append(kwargs) or MagicMock())
|
||||||
|
|
||||||
|
monitor._Monitor__start_compensation(mon_path=tmp_path, since=0)
|
||||||
|
assert started == []
|
||||||
|
|
||||||
|
monitor._Monitor__start_compensation(mon_path=tmp_path, since=time.time())
|
||||||
|
assert len(started) == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_unreadable_event_is_queued_instead_of_dropped(tmp_path, monkeypatch):
|
||||||
|
"""读取文件大小失败的事件应登记待重试,而不是被静默丢弃。"""
|
||||||
|
target = tmp_path / "a.mkv"
|
||||||
|
target.write_bytes(b"x")
|
||||||
|
monitor, dispatcher = _build_monitor(handle_file=MagicMock(return_value=True))
|
||||||
|
watcher = LocalDirectoryWatcher(tmp_path, callback=monitor, force_polling=True)
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "_get_file_size", staticmethod(lambda _p: None))
|
||||||
|
|
||||||
|
watcher._dispatch_changes({(Change.added, target.as_posix())})
|
||||||
|
|
||||||
|
assert len(dispatcher._pending_retries) == 1
|
||||||
|
entry = next(iter(dispatcher._pending_retries.values()))
|
||||||
|
assert entry["file_size"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_pending_reresolves_missing_file_size(tmp_path):
|
||||||
|
"""重试时要重新读取文件大小,再把文件送入整理链。"""
|
||||||
|
target = tmp_path / "a.mkv"
|
||||||
|
target.write_bytes(b"12345")
|
||||||
|
handled = MagicMock(return_value=True)
|
||||||
|
_, dispatcher = _build_monitor(handle_file=handled)
|
||||||
|
dispatcher.register_unreadable(storage="local", event_path=target)
|
||||||
|
|
||||||
|
dispatcher.retry_pending()
|
||||||
|
|
||||||
|
assert handled.call_args.kwargs["file_size"] == 5
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_pending_drops_vanished_file(tmp_path):
|
||||||
|
"""待重试文件已经消失时应放弃登记,不再无谓重试。"""
|
||||||
|
target = tmp_path / "gone.mkv"
|
||||||
|
handled = MagicMock(return_value=True)
|
||||||
|
_, dispatcher = _build_monitor(handle_file=handled)
|
||||||
|
dispatcher.register_unreadable(storage="local", event_path=target)
|
||||||
|
|
||||||
|
dispatcher.retry_pending()
|
||||||
|
|
||||||
|
assert dispatcher._pending_retries == {}
|
||||||
|
handled.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_transfer_failure_invalidates_dedup_cache(tmp_path, monkeypatch):
|
||||||
|
"""整理抛异常时去重缓存必须失效,否则 TTL 窗口内的后续事件会被吞掉。"""
|
||||||
|
target = tmp_path / "a.mkv"
|
||||||
|
target.write_bytes(b"x")
|
||||||
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", lambda **kwargs: False)
|
||||||
|
|
||||||
|
class _FailingChain:
|
||||||
|
"""整理时固定抛异常的整理链替身。"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def do_transfer(**kwargs):
|
||||||
|
"""模拟整理过程抛出异常。"""
|
||||||
|
raise RuntimeError("整理失败")
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", _FailingChain)
|
||||||
|
|
||||||
|
assert dispatcher.handle_file(storage="local", event_path=target, file_size=1) is False
|
||||||
|
assert dispatcher._cache == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_delayed_rescan_picks_up_late_visible_files(tmp_path, monkeypatch):
|
||||||
|
"""新增目录首次展开时不可见的文件,应由延迟重扫补回。"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
|
||||||
|
new_dir = tmp_path / "season"
|
||||||
|
new_dir.mkdir()
|
||||||
|
first = new_dir / "E03.mkv"
|
||||||
|
first.write_bytes(b"x")
|
||||||
|
recorder = MagicMock()
|
||||||
|
watcher = LocalDirectoryWatcher(tmp_path, callback=recorder, force_polling=True)
|
||||||
|
|
||||||
|
watcher._handle_changes({(Change.added, new_dir.as_posix())})
|
||||||
|
first_paths = {call.kwargs["event_path"] for call in recorder.event_handler.call_args_list}
|
||||||
|
assert first.as_posix() in first_paths
|
||||||
|
|
||||||
|
# 目录内容延迟可见:第二个文件此时才出现,不会再产生任何 watchfiles 事件
|
||||||
|
late = new_dir / "E04.mkv"
|
||||||
|
late.write_bytes(b"y")
|
||||||
|
recorder.event_handler.reset_mock()
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
|
||||||
|
rescan_paths = {call.kwargs["event_path"] for call in recorder.event_handler.call_args_list}
|
||||||
|
assert rescan_paths == {late.as_posix()}
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_filesystem_forces_polling_by_default(tmp_path, monkeypatch):
|
||||||
|
"""默认情况下网络文件系统仍应强制兼容模式。"""
|
||||||
|
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
|
||||||
|
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", False)
|
||||||
|
|
||||||
|
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "fast")
|
||||||
|
|
||||||
|
assert use_polling is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_network_filesystem_can_opt_into_fast_mode(tmp_path, monkeypatch):
|
||||||
|
"""用户确认挂载支持 inotify 后应允许快速模式。"""
|
||||||
|
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
|
||||||
|
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", True)
|
||||||
|
|
||||||
|
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "fast")
|
||||||
|
|
||||||
|
assert use_polling is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_compatibility_mode_still_wins_over_fast_mode_override(tmp_path, monkeypatch):
|
||||||
|
"""用户显式配置兼容模式时,快速模式开关不得反向覆盖。"""
|
||||||
|
monkeypatch.setattr(SystemUtils, "is_network_filesystem", staticmethod(lambda _d: True))
|
||||||
|
monkeypatch.setattr(settings, "MONITOR_NETWORK_FAST_MODE", True)
|
||||||
|
|
||||||
|
use_polling, _, _, _ = decide_monitor_mode(tmp_path, "compatibility")
|
||||||
|
|
||||||
|
assert use_polling is True
|
||||||
@@ -0,0 +1,509 @@
|
|||||||
|
"""
|
||||||
|
挂载级(block 型)故障下的监控自愈测试。
|
||||||
|
|
||||||
|
背景:CloudDrive2/115 等 FUSE 挂载会进入「请求不返回错误、永不返回」的挂死
|
||||||
|
状态。此时监控自愈体系的检测环节仍然有效,但恢复环节(重建监控线程、重试
|
||||||
|
整理队列)本身要访问挂载,一旦内联执行就会把全局自愈的单点——健康检查——
|
||||||
|
冻死在它自己要修复的挂载上,随后停滞检测、告警、重试驱动全部静默失效。
|
||||||
|
|
||||||
|
crash 型(挂载抛 Transport endpoint is not connected)的自愈由
|
||||||
|
test_monitor_resilience.py 覆盖;这些测试固定 block 型的两项不变量:
|
||||||
|
「看门狗零挂载访问」与「挂载级故障隔离/探测/恢复」。
|
||||||
|
"""
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from pathlib import Path
|
||||||
|
from threading import Lock, Thread
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||||
|
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path
|
||||||
|
|
||||||
|
|
||||||
|
def _build_monitor(monkeypatch, put_recorder=None):
|
||||||
|
"""
|
||||||
|
构造测试用 Monitor 骨架,绕过单例初始化。
|
||||||
|
:param monkeypatch: pytest monkeypatch
|
||||||
|
:param put_recorder: 消息推送记录器
|
||||||
|
:return: Monitor 骨架
|
||||||
|
"""
|
||||||
|
put_recorder = put_recorder or MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
|
||||||
|
monitor = object.__new__(Monitor)
|
||||||
|
monitor._dispatcher = MagicMock()
|
||||||
|
monitor._watchers = []
|
||||||
|
monitor._watcher_lock = Lock()
|
||||||
|
monitor._pending_locals = []
|
||||||
|
monitor._alerted_paths = {}
|
||||||
|
monitor._restart_marks = {}
|
||||||
|
monitor._stable_cycles = {}
|
||||||
|
monitor._isolated = {}
|
||||||
|
monitor._recovery = RecoveryExecutor()
|
||||||
|
return monitor
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
|
||||||
|
"""
|
||||||
|
构造测试用监控线程替身。
|
||||||
|
:param mon_path: 监控目录
|
||||||
|
:param alive: 线程是否存活
|
||||||
|
:param stalled: 是否静默失效
|
||||||
|
:param restart_count: 自动重启次数
|
||||||
|
:return: 监控线程替身
|
||||||
|
"""
|
||||||
|
watcher = MagicMock()
|
||||||
|
watcher.watch_path = mon_path
|
||||||
|
watcher.is_alive.return_value = alive
|
||||||
|
watcher.is_stalled.return_value = stalled
|
||||||
|
watcher.restart_count = restart_count
|
||||||
|
watcher.last_activity_time = time.time()
|
||||||
|
return watcher
|
||||||
|
|
||||||
|
|
||||||
|
def _run_watchdog(monitor, timeout=10.0):
|
||||||
|
"""
|
||||||
|
在独立线程里跑一次健康检查,返回它是否在限定时间内结束。
|
||||||
|
:param monitor: Monitor 骨架
|
||||||
|
:param timeout: 最长等待秒数
|
||||||
|
:return: 健康检查是否已返回
|
||||||
|
"""
|
||||||
|
done = threading.Event()
|
||||||
|
|
||||||
|
def runner():
|
||||||
|
"""
|
||||||
|
执行一次健康检查并标记结束。
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
monitor.watchdog()
|
||||||
|
finally:
|
||||||
|
done.set()
|
||||||
|
|
||||||
|
Thread(target=runner, daemon=True, name="test-watchdog").start()
|
||||||
|
return done.wait(timeout=timeout)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# P0:看门狗线程零挂载访问
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_watchdog_returns_while_rebuild_blocks_forever(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
根因回归:重建动作卡在死挂载上永不返回时,健康检查本身必须在有限时间内
|
||||||
|
返回。事故中看门狗内联执行重建,冻死后 13 个目录的停滞检测与告警全部失效。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
|
||||||
|
entered = threading.Event()
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
def blocking_rebuild(_target):
|
||||||
|
"""
|
||||||
|
模拟 block 型挂载:进入后永不返回。
|
||||||
|
"""
|
||||||
|
entered.set()
|
||||||
|
release.wait()
|
||||||
|
|
||||||
|
setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert _run_watchdog(monitor), "看门狗被重建动作冻死,未能在限定时间内返回"
|
||||||
|
assert entered.is_set(), "重建动作没有被真正发起"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchdog_returns_while_pending_retry_blocks_forever(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
重试驱动(整理重试队列的 stat)同样会卡在死挂载上,也必须移出看门狗线程。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
release = threading.Event()
|
||||||
|
monitor._dispatcher.retry_pending.side_effect = lambda: release.wait()
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert _run_watchdog(monitor), "看门狗被整理重试驱动冻死,未能在限定时间内返回"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchdog_returns_while_local_retry_blocks_forever(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
启动失败目录的重试会走 decide_monitor_mode 的 os.walk 与 watcher.start()
|
||||||
|
的 exists(),同样触碰挂载,必须移出看门狗线程。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
def blocking_start(**_kwargs):
|
||||||
|
"""
|
||||||
|
模拟启动重试卡在死挂载的目录遍历上。
|
||||||
|
"""
|
||||||
|
release.wait()
|
||||||
|
return False
|
||||||
|
|
||||||
|
setattr(monitor, "_Monitor__start_local_monitor", blocking_start)
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert _run_watchdog(monitor), "看门狗被本地监控重试冻死,未能在限定时间内返回"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_watchers_performs_no_filesystem_access(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
检测与判定必须是纯内存运算:只读线程存活标志、心跳与重启计数,
|
||||||
|
不做任何文件系统访问,否则检测环节本身也会被 block 型故障拖死。
|
||||||
|
"""
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
|
||||||
|
def forbidden(*_args, **_kwargs):
|
||||||
|
"""
|
||||||
|
任何真实的文件系统调用都应视为检测环节的缺陷。
|
||||||
|
"""
|
||||||
|
raise AssertionError("检测环节不允许访问文件系统")
|
||||||
|
|
||||||
|
for name in ("stat", "exists", "is_dir", "iterdir", "rglob"):
|
||||||
|
monkeypatch.setattr(Path, name, forbidden, raising=False)
|
||||||
|
|
||||||
|
broken = monitor._Monitor__check_watchers()
|
||||||
|
|
||||||
|
assert broken == [watcher]
|
||||||
|
|
||||||
|
|
||||||
|
def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
端到端回归:不替换任何恢复逻辑,只让 Path.exists() 永不返回(等价于对 FUSE
|
||||||
|
守护进程 kill -STOP 后挂载的表现),走真实的
|
||||||
|
__rebuild_watcher -> LocalDirectoryWatcher.start() -> exists() 调用链。
|
||||||
|
|
||||||
|
事故当天冻结的就是这一行(watcher.py 的 start() 入口校验)。修复后看门狗
|
||||||
|
必须照常返回,并把该目录转入隔离。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
|
||||||
|
# 用真实 watcher:只把底层线程换成替身,让它被判定为「存活但静默失效」
|
||||||
|
watcher = LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=True)
|
||||||
|
watcher._thread = MagicMock()
|
||||||
|
watcher._thread.is_alive.return_value = True
|
||||||
|
watcher._mark_activity()
|
||||||
|
watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
|
||||||
|
release = threading.Event()
|
||||||
|
real_exists = Path.exists
|
||||||
|
|
||||||
|
def blocking_exists(self, *args, **kwargs):
|
||||||
|
"""
|
||||||
|
模拟 block 型挂载:对监控目录的 exists() 永不返回。
|
||||||
|
"""
|
||||||
|
if self == tmp_path:
|
||||||
|
release.wait()
|
||||||
|
return real_exists(self, *args, **kwargs)
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "exists", blocking_exists)
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert _run_watchdog(monitor), "看门狗冻死在真实的重建调用链上"
|
||||||
|
assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# P0.5:挂载级故障隔离
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_rebuild_timeout_isolates_directory(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
重建在挂载上超时未返回 = 挂载级故障:该目录转入隔离,后续周期不再对它
|
||||||
|
发起任何新的挂载访问,避免每 60 秒泄漏一个冻死的线程。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.3)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
|
||||||
|
calls = []
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
def blocking_rebuild(target):
|
||||||
|
"""
|
||||||
|
模拟 block 型挂载上的重建:进入后永不返回。
|
||||||
|
"""
|
||||||
|
calls.append(target)
|
||||||
|
release.wait()
|
||||||
|
|
||||||
|
setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert _run_watchdog(monitor)
|
||||||
|
assert str(tmp_path) in monitor._isolated, "重建超时后目录没有转入隔离"
|
||||||
|
|
||||||
|
# 第二个周期:隔离中的目录不得再被提交重建
|
||||||
|
assert _run_watchdog(monitor)
|
||||||
|
assert len(calls) == 1, "隔离中的目录仍在被反复重建,会持续泄漏冻死的线程"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_isolated_directory_recovers_after_probe_succeeds(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
隔离期间用可放弃的子进程探测挂载;探测通过即解除隔离、重建监控,
|
||||||
|
并复用既有的补偿扫描补回停摆期间落地的文件。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 3}}
|
||||||
|
|
||||||
|
rebuilt = []
|
||||||
|
setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: True)
|
||||||
|
|
||||||
|
assert _run_watchdog(monitor)
|
||||||
|
|
||||||
|
assert str(tmp_path) not in monitor._isolated, "探测通过后没有解除隔离"
|
||||||
|
assert rebuilt == [watcher], "解除隔离后没有重建监控"
|
||||||
|
|
||||||
|
|
||||||
|
def test_isolated_directory_stays_isolated_while_probe_fails(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
探测未通过说明挂载仍未恢复应答,必须保持隔离并累计失败次数。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
|
monitor._watchers = [watcher]
|
||||||
|
monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 0}}
|
||||||
|
|
||||||
|
rebuilt = []
|
||||||
|
setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
|
||||||
|
monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)
|
||||||
|
|
||||||
|
assert _run_watchdog(monitor)
|
||||||
|
|
||||||
|
assert monitor._isolated[str(tmp_path)]["failures"] == 1
|
||||||
|
assert rebuilt == [], "挂载尚未恢复就重建监控,只会再冻死一个线程"
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_locals_skip_isolated_directories(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
隔离中的目录不能再走启动重试路径,否则又会在同一个挂载上冻死。
|
||||||
|
"""
|
||||||
|
monitor = _build_monitor(monkeypatch)
|
||||||
|
monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
|
||||||
|
monitor._isolated = {str(tmp_path): {"watcher": _fake_watcher(tmp_path), "since": time.time(),
|
||||||
|
"failures": 0}}
|
||||||
|
start = MagicMock(return_value=False)
|
||||||
|
setattr(monitor, "_Monitor__start_local_monitor", start)
|
||||||
|
|
||||||
|
monitor._Monitor__retry_pending_locals()
|
||||||
|
|
||||||
|
start.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_isolation_alert_is_pushed_after_fault_alert(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
故障→隔离是状态升级,必须再推一条告警:沿用「同目录只告警一次」会让
|
||||||
|
用户完全看不到「监控已暂停访问、正在等待挂载恢复」这个关键状态变化。
|
||||||
|
"""
|
||||||
|
put_recorder = MagicMock()
|
||||||
|
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||||
|
|
||||||
|
monitor._Monitor__send_alert(tmp_path, "目录监控异常")
|
||||||
|
monitor._Monitor__send_alert(tmp_path, "目录监控异常")
|
||||||
|
assert put_recorder.put.call_count == 1
|
||||||
|
|
||||||
|
monitor._Monitor__send_alert(tmp_path, "挂载无响应,已隔离", stage="isolated")
|
||||||
|
assert put_recorder.put.call_count == 2
|
||||||
|
|
||||||
|
monitor._Monitor__clear_alert(tmp_path, "已恢复")
|
||||||
|
assert put_recorder.put.call_count == 3
|
||||||
|
assert str(tmp_path) not in monitor._alerted_paths
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# 整理分发器:一个文件卡死不得锁死整条整理链
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_stuck_transfer_does_not_block_other_files(monkeypatch):
|
||||||
|
"""
|
||||||
|
根因回归:整理的规划阶段(do_transfer 里的 get_parent_item / list_files)会
|
||||||
|
访问挂载,在 block 型故障下永不返回。若分发器的互斥锁包住这段调用,这把锁
|
||||||
|
就会被永久持有,把 13 个 watcher 的事件派发、补偿扫描和重试队列一起锁死
|
||||||
|
——监控层即使自愈成功也送不进任何文件,漏件永远补不回来。
|
||||||
|
|
||||||
|
锁只应保护 TTL 去重的 check-and-set。
|
||||||
|
"""
|
||||||
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
|
|
||||||
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=False))
|
||||||
|
|
||||||
|
release = threading.Event()
|
||||||
|
stuck_entered = threading.Event()
|
||||||
|
transferred = []
|
||||||
|
|
||||||
|
class FakeChain:
|
||||||
|
"""
|
||||||
|
整理链替身:指定文件的整理进入后永不返回。
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def do_transfer(fileitem, **_kwargs):
|
||||||
|
"""
|
||||||
|
模拟规划阶段卡在死挂载的 list_files 上。
|
||||||
|
"""
|
||||||
|
transferred.append(fileitem.path)
|
||||||
|
if fileitem.path.endswith("stuck.mkv"):
|
||||||
|
stuck_entered.set()
|
||||||
|
release.wait()
|
||||||
|
|
||||||
|
monkeypatch.setattr("app.monitor.dispatcher.TransferChain", FakeChain)
|
||||||
|
|
||||||
|
def feed(name):
|
||||||
|
"""
|
||||||
|
向分发器送入一个文件。
|
||||||
|
"""
|
||||||
|
dispatcher.handle_file(storage="local", event_path=Path(f"/mnt/cd2/{name}"), file_size=1)
|
||||||
|
|
||||||
|
Thread(target=feed, args=("stuck.mkv",), daemon=True).start()
|
||||||
|
assert stuck_entered.wait(timeout=5), "卡死的整理没有真正进入"
|
||||||
|
|
||||||
|
done = threading.Event()
|
||||||
|
Thread(target=lambda: (feed("other.mkv"), done.set()), daemon=True).start()
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert done.wait(timeout=5), "一个文件卡在死挂载上就锁死了整条整理链"
|
||||||
|
assert "/mnt/cd2/other.mkv" in transferred
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# 恢复执行器与挂载探测
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
def test_recovery_executor_reports_timeout_without_blocking():
|
||||||
|
"""
|
||||||
|
永不返回的动作只应消耗一次 timeout,执行器必须放弃它并如实报告。
|
||||||
|
"""
|
||||||
|
executor = RecoveryExecutor()
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
results = executor.run({"stuck": release.wait}, timeout=0.3)
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert results == {"stuck": RecoveryState.TIMEOUT}
|
||||||
|
assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_executor_skips_key_with_running_task():
|
||||||
|
"""
|
||||||
|
同一个 key 的上一个动作还冻着时,不能再提交新线程——否则每个健康检查
|
||||||
|
周期都会在同一个死挂载上泄漏一个线程。
|
||||||
|
"""
|
||||||
|
executor = RecoveryExecutor()
|
||||||
|
release = threading.Event()
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def stuck():
|
||||||
|
"""
|
||||||
|
模拟永不返回的恢复动作。
|
||||||
|
"""
|
||||||
|
calls.append(1)
|
||||||
|
release.wait()
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.TIMEOUT}
|
||||||
|
assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.BUSY}
|
||||||
|
assert len(calls) == 1
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_executor_runs_actions_concurrently():
|
||||||
|
"""
|
||||||
|
一个周期内多个目录的恢复动作必须并发执行,否则 13 个目录串行等待会把
|
||||||
|
健康检查拖过下一个周期。
|
||||||
|
"""
|
||||||
|
executor = RecoveryExecutor()
|
||||||
|
release = threading.Event()
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
results = executor.run({f"k{i}": release.wait for i in range(5)}, timeout=0.4)
|
||||||
|
elapsed = time.monotonic() - started
|
||||||
|
|
||||||
|
try:
|
||||||
|
assert set(results.values()) == {RecoveryState.TIMEOUT}
|
||||||
|
assert elapsed < 1.5, "恢复动作是串行等待的,总耗时随目录数增长"
|
||||||
|
finally:
|
||||||
|
release.set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_recovery_executor_reports_completion_and_swallows_errors():
|
||||||
|
"""
|
||||||
|
正常完成的动作报告 COMPLETED;动作内部抛异常不能让执行器崩溃,
|
||||||
|
否则一个目录的失败会连累整批恢复。
|
||||||
|
"""
|
||||||
|
executor = RecoveryExecutor()
|
||||||
|
done = []
|
||||||
|
|
||||||
|
def boom():
|
||||||
|
"""
|
||||||
|
模拟恢复动作内部异常。
|
||||||
|
"""
|
||||||
|
raise RuntimeError("rebuild failed")
|
||||||
|
|
||||||
|
results = executor.run({"ok": lambda: done.append(1), "bad": boom}, timeout=5)
|
||||||
|
|
||||||
|
assert results == {"ok": RecoveryState.COMPLETED, "bad": RecoveryState.COMPLETED}
|
||||||
|
assert done == [1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_path_succeeds_on_existing_directory(tmp_path):
|
||||||
|
"""
|
||||||
|
挂载可用时探测应通过。
|
||||||
|
"""
|
||||||
|
assert probe_path(tmp_path, timeout=30) is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_probe_path_fails_on_missing_path(tmp_path):
|
||||||
|
"""
|
||||||
|
路径不存在时探测应失败,而不是抛异常。
|
||||||
|
"""
|
||||||
|
assert probe_path(tmp_path / "does-not-exist", timeout=30) is False
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.skipif(not Path("/bin/sh").exists(), reason="需要 POSIX 环境")
|
||||||
|
def test_probe_path_is_abandonable_on_timeout(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
探测的关键性质:卡住时可以被放弃。线程做的 stat 无法回收,只有子进程能在
|
||||||
|
超时后被 kill——这正是隔离期间能持续探测而不泄漏资源的前提。
|
||||||
|
"""
|
||||||
|
import app.monitor.recovery as recovery
|
||||||
|
|
||||||
|
# 用一个必定超时的探测脚本替换真实探测逻辑,验证超时路径的可放弃性
|
||||||
|
monkeypatch.setattr(recovery, "_PROBE_SCRIPT", "import time; time.sleep(60)")
|
||||||
|
|
||||||
|
started = time.monotonic()
|
||||||
|
assert probe_path(tmp_path, timeout=0.5) is False
|
||||||
|
assert time.monotonic() - started < 10, "探测超时后没有及时放弃子进程"
|
||||||
@@ -0,0 +1,321 @@
|
|||||||
|
"""
|
||||||
|
目录监控延迟重扫机制的边界缺陷回归测试。
|
||||||
|
|
||||||
|
覆盖点:
|
||||||
|
- 同目录重复登记去重(4.1)
|
||||||
|
- 整体扫描失败不燃烧轮次 + 失败次数上限(4.2)
|
||||||
|
- 目录删除后条目出队,不计入失败重试(4.2)
|
||||||
|
- MONITOR_RESCAN_DELAYS 配置解析(合法/非法回退)(4.3)
|
||||||
|
- 目录树整体移入时只登记顶层目录、祖先已登记时跳过子孙目录(4.4)
|
||||||
|
- 待重扫队列溢出时日志升级为 warn(4.4)
|
||||||
|
"""
|
||||||
|
import shutil
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from watchfiles import Change
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.monitor.watcher import LocalDirectoryWatcher
|
||||||
|
|
||||||
|
|
||||||
|
def _build_watcher(tmp_path, force_polling=True):
|
||||||
|
"""
|
||||||
|
构造测试用目录监控。
|
||||||
|
:param tmp_path: 监控目录
|
||||||
|
:param force_polling: 是否强制轮询
|
||||||
|
:return: 目录监控
|
||||||
|
"""
|
||||||
|
return LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=force_polling)
|
||||||
|
|
||||||
|
|
||||||
|
# ==================== 4.1 同目录重复登记去重 ====================
|
||||||
|
|
||||||
|
def test_schedule_rescan_dedups_same_directory(tmp_path):
|
||||||
|
"""
|
||||||
|
readdir 闪断可能让同一目录产生两次 added 事件,重复调用 _schedule_rescan
|
||||||
|
只应保留一条待重扫记录,不应重复登记。
|
||||||
|
"""
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
watcher._schedule_rescan(directory, seen=set())
|
||||||
|
watcher._schedule_rescan(directory, seen={"already-seen"})
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
# 保留原条目即可,不需要用新事件的 seen 覆盖
|
||||||
|
assert watcher._pending_rescans[0]["seen"] == set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_expand_added_directories_dedups_repeated_added_event_across_batches(tmp_path):
|
||||||
|
"""
|
||||||
|
同一目录在不同批次的 changes 中各出现一次 added 事件时(模拟 readdir 闪断),
|
||||||
|
整体展开流程也不应重复登记重扫。
|
||||||
|
"""
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
watcher._expand_added_directories({(Change.added, directory.as_posix())})
|
||||||
|
watcher._expand_added_directories({(Change.added, directory.as_posix())})
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ==================== 4.2 扫描失败不燃烧轮次 + 失败上限 + 目录删除终态 ====================
|
||||||
|
|
||||||
|
def test_collect_directory_files_reports_missing_directory(tmp_path):
|
||||||
|
"""
|
||||||
|
目录已不存在时应识别为终态(missing=True),而不是当作扫描失败。
|
||||||
|
"""
|
||||||
|
missing = tmp_path / "gone"
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
collected, is_missing, scan_failed = watcher._collect_directory_files(missing, exclude=set())
|
||||||
|
|
||||||
|
assert collected == set()
|
||||||
|
assert is_missing is True
|
||||||
|
assert scan_failed is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_collect_directory_files_reports_scan_failure(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
顶层 rglob 抛 OSError 应识别为整体扫描失败(scan_failed=True),
|
||||||
|
与目录已删除的终态区分开。
|
||||||
|
"""
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
|
||||||
|
def failing_rglob(self, pattern):
|
||||||
|
raise OSError("FUSE 抖动")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "rglob", failing_rglob)
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
collected, is_missing, scan_failed = watcher._collect_directory_files(directory, exclude=set())
|
||||||
|
|
||||||
|
assert collected == set()
|
||||||
|
assert is_missing is False
|
||||||
|
assert scan_failed is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_pending_rescans_does_not_burn_round_on_scan_failure(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
整体扫描失败时不应消耗重扫轮次,条目应保持在原轮次继续重试,
|
||||||
|
并累计一次失败计数。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0, 100))
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
watcher._schedule_rescan(directory, seen=set())
|
||||||
|
|
||||||
|
def failing_rglob(self, pattern):
|
||||||
|
raise OSError("FUSE 抖动")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "rglob", failing_rglob)
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
item = watcher._pending_rescans[0]
|
||||||
|
assert item["round"] == 0
|
||||||
|
assert item["failures"] == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_pending_rescans_drops_item_after_max_failures(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
连续扫描失败达到 MAX_RESCAN_FAILURES 上限后应放弃重扫并 warn,
|
||||||
|
避免目录长期不可访问时无限重试。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "MAX_RESCAN_FAILURES", 2)
|
||||||
|
logger_warn = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
watcher._schedule_rescan(directory, seen=set())
|
||||||
|
|
||||||
|
def failing_rglob(self, pattern):
|
||||||
|
raise OSError("FUSE 抖动")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "rglob", failing_rglob)
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
|
||||||
|
assert watcher._pending_rescans == []
|
||||||
|
logger_warn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_pending_rescans_resets_failure_count_after_success(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
扫描恢复成功后应重置失败计数,不应带着历史失败次数继续累积。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0, 100))
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
watcher._schedule_rescan(directory, seen=set())
|
||||||
|
watcher._pending_rescans[0]["failures"] = 3
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
assert watcher._pending_rescans[0]["failures"] == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_process_pending_rescans_drops_deleted_directory_without_failure(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
目录已被删除是终态,应直接出队,不计入失败重试次数。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "DIRECTORY_RESCAN_DELAYS", (0,))
|
||||||
|
directory = tmp_path / "season"
|
||||||
|
directory.mkdir()
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
watcher._schedule_rescan(directory, seen=set())
|
||||||
|
shutil.rmtree(directory)
|
||||||
|
|
||||||
|
watcher._process_pending_rescans()
|
||||||
|
|
||||||
|
assert watcher._pending_rescans == []
|
||||||
|
|
||||||
|
|
||||||
|
# ==================== 4.3 重扫窗口可配置 ====================
|
||||||
|
|
||||||
|
def test_parse_rescan_delays_accepts_valid_string():
|
||||||
|
"""
|
||||||
|
合法的逗号分隔正整数字符串应正确解析为元组。
|
||||||
|
"""
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays("30,120,600,1800") == (30, 120, 600, 1800)
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays(" 5 , 10 ") == (5, 10)
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_rescan_delays_falls_back_on_empty():
|
||||||
|
"""
|
||||||
|
空字符串/None 应回退到默认值。
|
||||||
|
"""
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays("") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays(None) == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_rescan_delays_falls_back_on_invalid_format(monkeypatch):
|
||||||
|
"""
|
||||||
|
非法格式(无法转换为整数)应回退默认值并记录 warn 日志。
|
||||||
|
"""
|
||||||
|
logger_warn = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
|
||||||
|
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays("abc,def") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
|
||||||
|
logger_warn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_parse_rescan_delays_rejects_non_positive_values(monkeypatch):
|
||||||
|
"""
|
||||||
|
包含非正整数(0 或负数)视为非法配置,应回退默认值并记录 warn 日志。
|
||||||
|
"""
|
||||||
|
logger_warn = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
|
||||||
|
|
||||||
|
assert LocalDirectoryWatcher._parse_rescan_delays("30,-5") == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
|
||||||
|
logger_warn.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_directory_rescan_delays_property_reads_settings(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
DIRECTORY_RESCAN_DELAYS 应实时反映 MONITOR_RESCAN_DELAYS 配置。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(settings, "MONITOR_RESCAN_DELAYS", "5,10")
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
assert watcher.DIRECTORY_RESCAN_DELAYS == (5, 10)
|
||||||
|
|
||||||
|
|
||||||
|
def test_directory_rescan_delays_property_falls_back_on_invalid_settings(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
配置非法时属性访问应回退默认值,而不是抛异常影响监控主流程。
|
||||||
|
"""
|
||||||
|
logger_warn = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
|
||||||
|
monkeypatch.setattr(settings, "MONITOR_RESCAN_DELAYS", "not-a-number")
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
assert watcher.DIRECTORY_RESCAN_DELAYS == LocalDirectoryWatcher.DEFAULT_RESCAN_DELAYS
|
||||||
|
|
||||||
|
|
||||||
|
# ==================== 4.4 目录树移入只登记顶层 + 溢出 warn ====================
|
||||||
|
|
||||||
|
def test_expand_added_directories_only_schedules_top_level(tmp_path):
|
||||||
|
"""
|
||||||
|
大目录树整体移入时,changes 里每一层子目录都会各自产生一次 added 事件,
|
||||||
|
但只有顶层目录需要登记重扫:子目录内容已被顶层目录的 rglob 覆盖。
|
||||||
|
"""
|
||||||
|
top = tmp_path / "task"
|
||||||
|
nested = top / "season1"
|
||||||
|
nested2 = nested / "sub"
|
||||||
|
nested2.mkdir(parents=True)
|
||||||
|
(top / "movie.mkv").write_bytes(b"x")
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
|
||||||
|
watcher._expand_added_directories({
|
||||||
|
(Change.added, top.as_posix()),
|
||||||
|
(Change.added, nested.as_posix()),
|
||||||
|
(Change.added, nested2.as_posix()),
|
||||||
|
})
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
assert watcher._pending_rescans[0]["path"] == top
|
||||||
|
|
||||||
|
|
||||||
|
def test_schedule_rescan_skips_descendant_of_pending_ancestor(tmp_path):
|
||||||
|
"""
|
||||||
|
某祖先目录已经在待重扫队列中时,其子孙目录不应再单独登记——
|
||||||
|
祖先条目的 rglob 会递归覆盖到子孙目录的新文件。
|
||||||
|
"""
|
||||||
|
parent = tmp_path / "task"
|
||||||
|
child = parent / "season"
|
||||||
|
child.mkdir(parents=True)
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
watcher._schedule_rescan(parent, seen=set())
|
||||||
|
|
||||||
|
watcher._schedule_rescan(child, seen=set())
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
assert watcher._pending_rescans[0]["path"] == parent
|
||||||
|
|
||||||
|
|
||||||
|
def test_is_descendant_of_any_does_not_match_sibling_with_prefix_name(tmp_path):
|
||||||
|
"""
|
||||||
|
路径前缀相同但并非真实父子关系的兄弟目录(如 task / task2)不应被
|
||||||
|
误判为祖先命中,避免字符串前缀匹配导致的误跳过。
|
||||||
|
"""
|
||||||
|
parent = tmp_path / "task"
|
||||||
|
sibling = tmp_path / "task2"
|
||||||
|
|
||||||
|
assert LocalDirectoryWatcher._is_descendant_of_any(sibling, {parent}) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_schedule_rescan_overflow_logs_warning(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
待重扫队列已满时应放弃登记,且日志级别应为 warn(原实现仅 debug,
|
||||||
|
可能导致文件永久漏扫却难以被发现)。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(LocalDirectoryWatcher, "MAX_PENDING_RESCANS", 1)
|
||||||
|
logger_warn = MagicMock()
|
||||||
|
monkeypatch.setattr("app.monitor.watcher.logger.warn", logger_warn)
|
||||||
|
watcher = _build_watcher(tmp_path)
|
||||||
|
first_dir = tmp_path / "a"
|
||||||
|
second_dir = tmp_path / "b"
|
||||||
|
first_dir.mkdir()
|
||||||
|
second_dir.mkdir()
|
||||||
|
|
||||||
|
watcher._schedule_rescan(first_dir, seen=set())
|
||||||
|
watcher._schedule_rescan(second_dir, seen=set())
|
||||||
|
|
||||||
|
assert len(watcher._pending_rescans) == 1
|
||||||
|
assert watcher._pending_rescans[0]["path"] == first_dir
|
||||||
|
logger_warn.assert_called_once()
|
||||||
@@ -2,6 +2,7 @@ from pathlib import Path
|
|||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
from app.monitor import LocalDirectoryWatcher, Monitor
|
from app.monitor import LocalDirectoryWatcher, Monitor
|
||||||
|
from app.monitor.recovery import RecoveryExecutor
|
||||||
|
|
||||||
|
|
||||||
def _build_watcher(tmp_path, force_polling):
|
def _build_watcher(tmp_path, force_polling):
|
||||||
@@ -130,12 +131,16 @@ def _build_monitor(monkeypatch, put_recorder):
|
|||||||
from threading import Lock
|
from threading import Lock
|
||||||
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
|
monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
|
||||||
monitor = object.__new__(Monitor)
|
monitor = object.__new__(Monitor)
|
||||||
|
# 自动重启后健康检查会发起补偿扫描,骨架需要一个分发器替身
|
||||||
|
monitor._dispatcher = MagicMock()
|
||||||
monitor._watchers = []
|
monitor._watchers = []
|
||||||
monitor._watcher_lock = Lock()
|
monitor._watcher_lock = Lock()
|
||||||
monitor._pending_locals = []
|
monitor._pending_locals = []
|
||||||
monitor._alerted_paths = set()
|
monitor._alerted_paths = {}
|
||||||
monitor._restart_marks = {}
|
monitor._restart_marks = {}
|
||||||
monitor._stable_cycles = {}
|
monitor._stable_cycles = {}
|
||||||
|
monitor._isolated = {}
|
||||||
|
monitor._recovery = RecoveryExecutor()
|
||||||
return monitor
|
return monitor
|
||||||
|
|
||||||
|
|
||||||
@@ -158,35 +163,31 @@ def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
|
|||||||
|
|
||||||
def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch):
|
def test_watchdog_rebuilds_dead_watcher(tmp_path, monkeypatch):
|
||||||
"""
|
"""
|
||||||
监控线程退出后健康检查应重建线程并告警。
|
监控线程退出后健康检查应判定为待重建并告警。
|
||||||
|
|
||||||
|
重建本身会触碰挂载,已移出看门狗线程,因此检测环节只负责判定与告警,
|
||||||
|
真正的重建由 __drive_recovery 派发到一次性工作线程执行。
|
||||||
"""
|
"""
|
||||||
put_recorder = MagicMock()
|
put_recorder = MagicMock()
|
||||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||||
watcher = _fake_watcher(tmp_path, alive=False)
|
watcher = _fake_watcher(tmp_path, alive=False)
|
||||||
monitor._watchers = [watcher]
|
monitor._watchers = [watcher]
|
||||||
rebuild = MagicMock()
|
|
||||||
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
|
|
||||||
|
|
||||||
monitor._Monitor__check_watchers()
|
assert monitor._Monitor__check_watchers() == [watcher]
|
||||||
|
|
||||||
rebuild.assert_called_once_with(watcher)
|
|
||||||
put_recorder.put.assert_called_once()
|
put_recorder.put.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch):
|
def test_watchdog_rebuilds_stalled_watcher(tmp_path, monkeypatch):
|
||||||
"""
|
"""
|
||||||
静默失效的监控线程也应被健康检查重建。
|
静默失效的监控线程也应被健康检查判定为待重建。
|
||||||
"""
|
"""
|
||||||
put_recorder = MagicMock()
|
put_recorder = MagicMock()
|
||||||
monitor = _build_monitor(monkeypatch, put_recorder)
|
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||||
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
|
||||||
monitor._watchers = [watcher]
|
monitor._watchers = [watcher]
|
||||||
rebuild = MagicMock()
|
|
||||||
setattr(monitor, "_Monitor__rebuild_watcher", rebuild)
|
|
||||||
|
|
||||||
monitor._Monitor__check_watchers()
|
assert monitor._Monitor__check_watchers() == [watcher]
|
||||||
|
|
||||||
rebuild.assert_called_once_with(watcher)
|
|
||||||
|
|
||||||
|
|
||||||
def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch):
|
def test_watchdog_alerts_on_restart_and_recovers_after_stable_window(tmp_path, monkeypatch):
|
||||||
@@ -235,7 +236,7 @@ def test_dispatcher_retries_after_history_query_failure(monkeypatch):
|
|||||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
event_path = Path("/downloads/movie.mkv")
|
event_path = Path("/downloads/movie.mkv")
|
||||||
history = MagicMock(side_effect=[None, False])
|
history = MagicMock(side_effect=[None, False])
|
||||||
monkeypatch.setattr(dispatcher, "_has_transfer_history", history)
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", history)
|
||||||
transfer_chain_instance = MagicMock()
|
transfer_chain_instance = MagicMock()
|
||||||
monkeypatch.setattr("app.monitor.dispatcher.TransferChain",
|
monkeypatch.setattr("app.monitor.dispatcher.TransferChain",
|
||||||
MagicMock(return_value=transfer_chain_instance))
|
MagicMock(return_value=transfer_chain_instance))
|
||||||
@@ -253,6 +254,38 @@ def test_dispatcher_retries_after_history_query_failure(monkeypatch):
|
|||||||
assert dispatcher._pending_retries == {}
|
assert dispatcher._pending_retries == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_dispatcher_clear_pending_drops_stale_entries(monkeypatch):
|
||||||
|
"""
|
||||||
|
停止/配置重载时应清空待重试队列,避免已移除的监控目录在数据库恢复后
|
||||||
|
仍被送入整理链。
|
||||||
|
"""
|
||||||
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=None))
|
||||||
|
dispatcher.handle_file(storage="local", event_path=Path("/removed/movie.mkv"), file_size=1)
|
||||||
|
assert dispatcher._pending_retries
|
||||||
|
|
||||||
|
dispatcher.clear_pending()
|
||||||
|
|
||||||
|
assert dispatcher._pending_retries == {}
|
||||||
|
dispatcher.retry_pending()
|
||||||
|
assert dispatcher._pending_retries == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_monitor_stop_clears_dispatcher_pending(monkeypatch):
|
||||||
|
"""
|
||||||
|
Monitor.stop 应连带清理分发器的待重试队列。
|
||||||
|
"""
|
||||||
|
put_recorder = MagicMock()
|
||||||
|
monitor = _build_monitor(monkeypatch, put_recorder)
|
||||||
|
monitor._scheduler = None
|
||||||
|
monitor._dispatcher = MagicMock()
|
||||||
|
|
||||||
|
monitor.stop()
|
||||||
|
|
||||||
|
monitor._dispatcher.clear_pending.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
|
def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
|
||||||
"""
|
"""
|
||||||
历史查询持续失败达到上限后应放弃重试,避免队列无限累积。
|
历史查询持续失败达到上限后应放弃重试,避免队列无限累积。
|
||||||
@@ -260,7 +293,7 @@ def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
|
|||||||
from app.monitor.dispatcher import TransferDispatcher
|
from app.monitor.dispatcher import TransferDispatcher
|
||||||
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
|
||||||
event_path = Path("/downloads/movie.mkv")
|
event_path = Path("/downloads/movie.mkv")
|
||||||
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=None))
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=None))
|
||||||
|
|
||||||
dispatcher.handle_file(storage="local", event_path=event_path, file_size=1)
|
dispatcher.handle_file(storage="local", event_path=event_path, file_size=1)
|
||||||
key = f"local:{event_path.as_posix()}"
|
key = f"local:{event_path.as_posix()}"
|
||||||
@@ -271,3 +304,14 @@ def test_dispatcher_drops_pending_after_max_attempts(monkeypatch):
|
|||||||
dispatcher.retry_pending()
|
dispatcher.retry_pending()
|
||||||
|
|
||||||
assert dispatcher._pending_retries == {}
|
assert dispatcher._pending_retries == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_monitor_watches_mode_env_keys_for_hot_reload():
|
||||||
|
"""
|
||||||
|
快速模式/轮询间隔只在监控线程创建时读取,必须监听对应 env 变更触发
|
||||||
|
init() 重建才能热生效;防止上游合并时丢失监听键。
|
||||||
|
"""
|
||||||
|
from app.schemas.types import SystemConfigKey
|
||||||
|
assert SystemConfigKey.Directories.value in Monitor.CONFIG_WATCH
|
||||||
|
assert "MONITOR_NETWORK_FAST_MODE" in Monitor.CONFIG_WATCH
|
||||||
|
assert "MONITOR_POLL_DELAY_NETWORK" in Monitor.CONFIG_WATCH
|
||||||
|
|||||||
@@ -254,6 +254,71 @@ def test_poll_failure_alert_threshold_and_recovery(monkeypatch):
|
|||||||
assert "已恢复" in alert_cb.call_args.args[1]
|
assert "已恢复" in alert_cb.call_args.args[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_poll_partial_failure_pins_incremental_cursor(monkeypatch):
|
||||||
|
"""
|
||||||
|
部分路径失败时增量游标必须固定在旧值,否则失败路径中时间落在新旧游标
|
||||||
|
之间的变更会被后续增量查询永久跳过。
|
||||||
|
"""
|
||||||
|
poller, store, _ = _build_poller()
|
||||||
|
store.load_checked.return_value = (dict(BASELINE), True)
|
||||||
|
_mock_storage_chain(monkeypatch, [None, {'/mon2/b.mkv': {'size': 2, 'modify_time': 999}}])
|
||||||
|
|
||||||
|
poller.poll("u115", [Path("/mon"), Path("/mon2")])
|
||||||
|
|
||||||
|
# 游标固定为旧基线时间 100,而不是成功路径产生的 999
|
||||||
|
assert store.save.call_args.kwargs["snapshot_time"] == 100
|
||||||
|
|
||||||
|
|
||||||
|
def test_poll_full_success_advances_cursor(monkeypatch):
|
||||||
|
"""
|
||||||
|
全部路径成功时不固定游标,由快照内容推进增量游标。
|
||||||
|
"""
|
||||||
|
poller, store, _ = _build_poller()
|
||||||
|
store.load_checked.return_value = (dict(BASELINE), True)
|
||||||
|
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 999}}])
|
||||||
|
|
||||||
|
poller.poll("u115", [Path("/mon")])
|
||||||
|
|
||||||
|
assert store.save.call_args.kwargs["snapshot_time"] is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_full_scan_aborts_on_baseline_load_error(monkeypatch):
|
||||||
|
"""
|
||||||
|
全量扫描读取基线失败时必须放弃落盘,否则会抹掉同存储其他监控路径的基线。
|
||||||
|
"""
|
||||||
|
poller, store, _ = _build_poller()
|
||||||
|
store.load_checked.return_value = (None, False)
|
||||||
|
_mock_storage_chain(monkeypatch, [{'/mon/a.mkv': {'size': 1, 'modify_time': 100}}])
|
||||||
|
|
||||||
|
assert poller.force_full_scan("u115", Path("/mon")) is False
|
||||||
|
store.save.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_full_scan_reports_save_failure(monkeypatch):
|
||||||
|
"""
|
||||||
|
全量扫描持久化失败必须传播为失败,不能返回成功掩盖基线未更新。
|
||||||
|
"""
|
||||||
|
poller, store, _ = _build_poller()
|
||||||
|
store.load_checked.return_value = (dict(BASELINE), True)
|
||||||
|
store.save.return_value = False
|
||||||
|
_mock_storage_chain(monkeypatch, [{'/mon/b.mkv': {'size': 2, 'modify_time': 200}}])
|
||||||
|
|
||||||
|
assert poller.force_full_scan("u115", Path("/mon")) is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_force_full_scan_merges_with_existing_baseline(monkeypatch):
|
||||||
|
"""
|
||||||
|
全量扫描只覆盖单个路径,应与已有基线合并后落盘。
|
||||||
|
"""
|
||||||
|
poller, store, _ = _build_poller()
|
||||||
|
store.load_checked.return_value = (dict(BASELINE), True)
|
||||||
|
_mock_storage_chain(monkeypatch, [{'/mon2/b.mkv': {'size': 2, 'modify_time': 200}}])
|
||||||
|
|
||||||
|
assert poller.force_full_scan("u115", Path("/mon2")) is True
|
||||||
|
saved_snapshot = store.save.call_args.args[1]
|
||||||
|
assert set(saved_snapshot.keys()) == {'/mon/a.mkv', '/mon2/b.mkv'}
|
||||||
|
|
||||||
|
|
||||||
def test_watcher_poll_delay_defaults_and_override(tmp_path):
|
def test_watcher_poll_delay_defaults_and_override(tmp_path):
|
||||||
"""
|
"""
|
||||||
轮询扫描间隔默认取本地值,显式传入网络值时生效。
|
轮询扫描间隔默认取本地值,显式传入网络值时生效。
|
||||||
|
|||||||
@@ -326,7 +326,9 @@ def test_handle_file_prefers_music_type_from_monitor_directory(monkeypatch):
|
|||||||
]
|
]
|
||||||
transfer_chain_instance = MagicMock()
|
transfer_chain_instance = MagicMock()
|
||||||
|
|
||||||
monkeypatch.setattr(dispatcher, "_has_transfer_history", MagicMock(return_value=False))
|
# 历史查重已由 _should_skip_by_history 统一承担(含失败重试预算与版本变化判定),
|
||||||
|
# 这里放行以便验证 mtype 的传递
|
||||||
|
monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=False))
|
||||||
monkeypatch.setattr(
|
monkeypatch.setattr(
|
||||||
"app.monitor.dispatcher.DirectoryHelper",
|
"app.monitor.dispatcher.DirectoryHelper",
|
||||||
MagicMock(return_value=MagicMock(get_download_dirs=MagicMock(return_value=directories))),
|
MagicMock(return_value=MagicMock(get_download_dirs=MagicMock(return_value=directories))),
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
import importlib
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.filemanager.storages import StorageBase
|
||||||
|
from app.schemas.exception import StorageQueryError
|
||||||
|
|
||||||
|
# 在树存储模块,全部必须实现严格查询
|
||||||
|
BUILTIN_STORAGE_MODULES = ["alipan", "alist", "local", "rclone", "smb", "u115"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_base_get_item_strict_fails_conservatively():
|
||||||
|
"""未覆写严格查询的存储必须保守失败,而不是回退到宽松查询。"""
|
||||||
|
stub = SimpleNamespace(schema="dummy")
|
||||||
|
|
||||||
|
with pytest.raises(StorageQueryError):
|
||||||
|
StorageBase.get_item_strict(stub, Path("/media/示例.mkv"))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("module_name", BUILTIN_STORAGE_MODULES)
|
||||||
|
def test_builtin_storage_overrides_strict_query(module_name):
|
||||||
|
"""每个在树存储都必须覆写严格查询,否则整理会被基类保守拒绝。"""
|
||||||
|
module = importlib.import_module(f"app.modules.filemanager.storages.{module_name}")
|
||||||
|
storage_classes = [
|
||||||
|
obj for obj in vars(module).values()
|
||||||
|
if isinstance(obj, type)
|
||||||
|
and issubclass(obj, StorageBase)
|
||||||
|
and obj is not StorageBase
|
||||||
|
and obj.__module__ == module.__name__
|
||||||
|
]
|
||||||
|
|
||||||
|
assert storage_classes, f"{module_name} 未定义存储类"
|
||||||
|
for storage_class in storage_classes:
|
||||||
|
assert "get_item_strict" in vars(storage_class), \
|
||||||
|
f"{storage_class.__name__} 未实现 get_item_strict"
|
||||||
@@ -0,0 +1,122 @@
|
|||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||||
|
from app.schemas.types import MediaSource, MediaType
|
||||||
|
|
||||||
|
|
||||||
|
class _MemoryCacheStub:
|
||||||
|
"""TMDB 识别缓存测试用的最小内存后端。"""
|
||||||
|
|
||||||
|
def __init__(self, data: dict = None):
|
||||||
|
"""使用给定字典初始化测试缓存。"""
|
||||||
|
self.data = data if data is not None else {}
|
||||||
|
self.ttls = {}
|
||||||
|
|
||||||
|
def get(self, key: str):
|
||||||
|
"""读取指定缓存条目。"""
|
||||||
|
return self.data.get(key)
|
||||||
|
|
||||||
|
def set(self, key: str, value, ttl=None):
|
||||||
|
"""写入缓存条目并记录使用的 TTL。"""
|
||||||
|
self.data[key] = value
|
||||||
|
self.ttls[key] = ttl
|
||||||
|
|
||||||
|
def delete(self, key: str):
|
||||||
|
"""删除指定缓存条目。"""
|
||||||
|
self.data.pop(key, None)
|
||||||
|
|
||||||
|
def items(self):
|
||||||
|
"""返回全部缓存条目。"""
|
||||||
|
return self.data.items()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def is_redis() -> bool:
|
||||||
|
"""测试后端不是 Redis。"""
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
|
def _build_cache(data: dict = None) -> TmdbCache:
|
||||||
|
"""构造绕过单例初始化的 TMDB 缓存实例。"""
|
||||||
|
cache = object.__new__(TmdbCache)
|
||||||
|
cache._cache = _MemoryCacheStub(data)
|
||||||
|
cache.ttl = 43200
|
||||||
|
cache._expires_at = {}
|
||||||
|
cache._dirty = False
|
||||||
|
cache.save = lambda force=False: None
|
||||||
|
return cache
|
||||||
|
|
||||||
|
|
||||||
|
def _build_meta(mtype: MediaType, begin_season=1) -> SimpleNamespace:
|
||||||
|
"""构造识别缓存所需的最小元数据。"""
|
||||||
|
# v3 起缓存键改用 media_source/media_id 统一标识媒体来源,
|
||||||
|
# 非 TMDB 来源不参与 TMDB 缓存键构造
|
||||||
|
return SimpleNamespace(type=mtype, tmdbid=329809, year="2022",
|
||||||
|
begin_season=begin_season, name="死神",
|
||||||
|
media_source=MediaSource.TMDB, media_id="329809")
|
||||||
|
|
||||||
|
|
||||||
|
def _key(type_name: str, begin_season=1) -> str:
|
||||||
|
"""按缓存键格式构造测试用键。"""
|
||||||
|
return f"[{type_name}][{settings.TMDB_LOCALE}]329809-2022-{begin_season}"
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_discards_movie_value_cached_under_tv_key():
|
||||||
|
"""电视剧元数据命中电影缓存值属于脏条目,应丢弃并回源。"""
|
||||||
|
meta = _build_meta(MediaType.TV)
|
||||||
|
key = _key("电视剧")
|
||||||
|
cache = _build_cache({key: {"id": 329809, "type": MediaType.MOVIE,
|
||||||
|
"title": "白鼬", "year": "2015"}})
|
||||||
|
|
||||||
|
assert cache.get(meta) == {}
|
||||||
|
assert cache._cache.get(key) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_discards_movie_value_stored_as_plain_string():
|
||||||
|
"""缓存值经序列化后类型退化为字符串时,同样要识别出冲突。"""
|
||||||
|
meta = _build_meta(MediaType.TV)
|
||||||
|
key = _key("电视剧")
|
||||||
|
cache = _build_cache({key: {"id": 329809, "type": "电影", "title": "白鼬"}})
|
||||||
|
|
||||||
|
assert cache.get(meta) == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_keeps_tv_value_cached_under_movie_key():
|
||||||
|
"""电影元数据命中电视剧缓存值是名称识别的正常纠正结果,必须保留。"""
|
||||||
|
meta = _build_meta(MediaType.MOVIE, begin_season=None)
|
||||||
|
key = _key("电影", begin_season=None)
|
||||||
|
cache = _build_cache({key: {"id": 1, "type": MediaType.TV, "title": "某剧"}})
|
||||||
|
|
||||||
|
assert cache.get(meta)["type"] == MediaType.TV
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_keeps_negative_cache_entry():
|
||||||
|
"""负缓存不带类型信息,不应被类型校验误删。"""
|
||||||
|
meta = _build_meta(MediaType.TV)
|
||||||
|
cache = _build_cache({_key("电视剧"): {"id": 0}})
|
||||||
|
|
||||||
|
assert cache.get(meta) == {"id": 0}
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_refuses_to_cache_movie_result_for_tv_meta():
|
||||||
|
"""电视剧元数据得到电影识别结果时不得写入缓存,避免固化脏条目。"""
|
||||||
|
meta = _build_meta(MediaType.TV)
|
||||||
|
cache = _build_cache()
|
||||||
|
|
||||||
|
cache.update(meta, {"id": 329809, "media_type": MediaType.MOVIE,
|
||||||
|
"title": "白鼬", "release_date": "2015-01-01"})
|
||||||
|
|
||||||
|
assert cache._cache.data == {}
|
||||||
|
|
||||||
|
|
||||||
|
def test_update_caches_tv_result_for_movie_meta():
|
||||||
|
"""电影元数据得到电视剧结果是正常纠正,应照常写入缓存。"""
|
||||||
|
meta = _build_meta(MediaType.MOVIE, begin_season=None)
|
||||||
|
cache = _build_cache()
|
||||||
|
|
||||||
|
cache.update(meta, {"id": 1, "media_type": MediaType.TV,
|
||||||
|
"name": "某剧", "first_air_date": "2022-01-01"})
|
||||||
|
|
||||||
|
stored = cache._cache.get(_key("电影", begin_season=None))
|
||||||
|
assert stored["type"] == MediaType.TV
|
||||||
|
assert stored["title"] == "某剧"
|
||||||
@@ -0,0 +1,72 @@
|
|||||||
|
"""
|
||||||
|
TMDB request 层业务失败响应不缓存测试(§6.4)。
|
||||||
|
|
||||||
|
TMDB 对 404 等业务失败返回合法 JSON(success=false),原实现会随快照缓存
|
||||||
|
12 小时;瞬时的服务端错误也会被同样固化,期间同 key 请求直接命中失败快照。
|
||||||
|
业务失败响应必须跳过缓存,允许下次请求重新确认。
|
||||||
|
"""
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from app.core.cache import cached
|
||||||
|
from app.modules.themoviedb.tmdbv3api.tmdb import TMDb
|
||||||
|
|
||||||
|
from tests.test_tmdb_response_cache import _FakeResponse
|
||||||
|
|
||||||
|
_NOT_FOUND_PAYLOAD = {
|
||||||
|
"success": False,
|
||||||
|
"status_code": 34,
|
||||||
|
"status_message": "The resource you requested could not be found.",
|
||||||
|
}
|
||||||
|
_HEADERS = {"Content-Type": "application/json"}
|
||||||
|
|
||||||
|
|
||||||
|
class CachedSkipIfTest(unittest.TestCase):
|
||||||
|
|
||||||
|
def test_skip_if_prevents_caching_matching_values(self):
|
||||||
|
"""skip_if 命中的返回值不入缓存,后续调用重新执行函数。"""
|
||||||
|
calls = {"bad": 0, "good": 0}
|
||||||
|
|
||||||
|
@cached(region="test_skip_if", ttl=60,
|
||||||
|
skip_if=lambda value: value.get("bad"))
|
||||||
|
def fetch(kind: str) -> dict:
|
||||||
|
calls[kind] += 1
|
||||||
|
return {"bad": kind == "bad"}
|
||||||
|
|
||||||
|
fetch("bad")
|
||||||
|
fetch("bad")
|
||||||
|
self.assertEqual(calls["bad"], 2)
|
||||||
|
fetch("good")
|
||||||
|
fetch("good")
|
||||||
|
self.assertEqual(calls["good"], 1)
|
||||||
|
|
||||||
|
|
||||||
|
class TmdbFailureSnapshotCacheTest(unittest.TestCase):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _make_tmdb() -> TMDb:
|
||||||
|
tmdb = TMDb()
|
||||||
|
tmdb.api_key = "test-key"
|
||||||
|
return tmdb
|
||||||
|
|
||||||
|
def test_business_failure_response_is_not_cached(self):
|
||||||
|
"""404 业务失败 JSON 不入缓存,同参数再次请求会重新访问 TMDB。"""
|
||||||
|
tmdb = self._make_tmdb()
|
||||||
|
fake = _FakeResponse(_NOT_FOUND_PAYLOAD, _HEADERS, status_code=404)
|
||||||
|
with patch.object(TMDb, "_request_once", return_value=fake) as req:
|
||||||
|
tmdb.request("GET", "https://api.tmdb.test/failure-not-cached", None, None)
|
||||||
|
tmdb.request("GET", "https://api.tmdb.test/failure-not-cached", None, None)
|
||||||
|
self.assertEqual(req.call_count, 2)
|
||||||
|
|
||||||
|
def test_success_response_is_still_cached(self):
|
||||||
|
"""成功响应保持缓存,同参数第二次请求命中快照。"""
|
||||||
|
tmdb = self._make_tmdb()
|
||||||
|
fake = _FakeResponse({"id": 98865, "title": "Test"}, _HEADERS)
|
||||||
|
with patch.object(TMDb, "_request_once", return_value=fake) as req:
|
||||||
|
tmdb.request("GET", "https://api.tmdb.test/success-cached", None, None)
|
||||||
|
tmdb.request("GET", "https://api.tmdb.test/success-cached", None, None)
|
||||||
|
self.assertEqual(req.call_count, 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,503 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
"""
|
||||||
|
覆盖TMDB识别链路两个已核实缺陷的回归测试。
|
||||||
|
|
||||||
|
§6.1 零退避重试:
|
||||||
|
同步 `TMDb.request` 在连接失败重建会话后立即重试,两次尝试间隔为0;
|
||||||
|
异步 `TMDb.async_request` 遇到连接失败时完全没有重试。
|
||||||
|
NAS+FUSE网盘环境下TMDB连接偶发的都是数秒内可自愈的瞬时抖动,零退避/不重试
|
||||||
|
基本无法覆盖这类抖动窗口。
|
||||||
|
|
||||||
|
§6.2 网络故障与"条目不存在"文案混淆:
|
||||||
|
`TmdbApi.__get_movie_detail`/`__get_tv_detail` 对所有异常(含真实网络故障与
|
||||||
|
TMDB返回的404"资源不存在")一律 `except Exception: return None`,两类完全不同
|
||||||
|
性质的失败被折叠成同一个 None;`TheMovieDbModule._get_info_by_tmdbid` 又用
|
||||||
|
`info_tv or info_movie or None` 把结果进一步折叠,最终识别失败统一报
|
||||||
|
"无法确定媒体类型,识别失败",用户无法判断该等网络恢复还是该确认条目不存在。
|
||||||
|
"""
|
||||||
|
import asyncio
|
||||||
|
from unittest.mock import AsyncMock, Mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
import app.modules.themoviedb as themoviedb_module
|
||||||
|
from app.core.metainfo import MetaInfo
|
||||||
|
from app.modules.themoviedb import TheMovieDbModule
|
||||||
|
from app.modules.themoviedb.tmdb_cache import TmdbCache
|
||||||
|
from app.modules.themoviedb.tmdbapi import TmdbApi
|
||||||
|
from app.modules.themoviedb.tmdbv3api import tmdb as tmdb_module
|
||||||
|
from app.modules.themoviedb.tmdbv3api.exceptions import TMDbConnectionError, TMDbException
|
||||||
|
from app.modules.themoviedb.tmdbv3api.tmdb import TMDb
|
||||||
|
from app.schemas.types import MediaSource, MediaType
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeResponse:
|
||||||
|
"""测试用响应对象,模拟 requests/httpx 响应的最小接口。"""
|
||||||
|
|
||||||
|
def __init__(self, payload, headers: dict = None):
|
||||||
|
"""初始化响应内容。"""
|
||||||
|
self._payload = payload
|
||||||
|
self.headers = headers or {}
|
||||||
|
self.status_code = 200
|
||||||
|
self.text = ""
|
||||||
|
|
||||||
|
def json(self):
|
||||||
|
"""返回预置JSON内容。"""
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# §6.1 零退避重试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_tmdb_connection_error_is_tmdb_exception_subclass():
|
||||||
|
"""
|
||||||
|
TMDbConnectionError必须是TMDbException的子类,
|
||||||
|
否则现有 `except TMDbException` 代码路径会因新增异常类型而失效。
|
||||||
|
"""
|
||||||
|
assert issubclass(TMDbConnectionError, TMDbException)
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_sleeps_before_retry_after_connection_failure(monkeypatch):
|
||||||
|
"""
|
||||||
|
同步请求失败后,重建会话重试前应等待 RETRY_BACKOFF_SECONDS 秒,
|
||||||
|
而不是零间隔立即重试。
|
||||||
|
"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
response = _FakeResponse(payload={"id": 1})
|
||||||
|
request_results = [None, response]
|
||||||
|
tmdb._request_once = lambda method, url, data, json: request_results.pop(0)
|
||||||
|
|
||||||
|
sleep_calls = []
|
||||||
|
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: sleep_calls.append(seconds))
|
||||||
|
|
||||||
|
result = TMDb.request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
|
||||||
|
|
||||||
|
assert result["json"] == {"id": 1}
|
||||||
|
assert sleep_calls == [tmdb_module.RETRY_BACKOFF_SECONDS]
|
||||||
|
assert 1 <= tmdb_module.RETRY_BACKOFF_SECONDS <= 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_raises_connection_error_after_retry_exhausted(monkeypatch):
|
||||||
|
"""
|
||||||
|
重试后依旧失败时,应抛出更具体的 TMDbConnectionError(而不仅是笼统的 TMDbException),
|
||||||
|
供上层区分"网络故障"与TMDB业务层错误;对外仍保持"最终失败抛异常"的语义不变。
|
||||||
|
"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
tmdb._request_once = lambda method, url, data, json: None
|
||||||
|
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: None)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError, match="无法连接TheMovieDb"):
|
||||||
|
TMDb.request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_request_retries_once_after_connection_failure(monkeypatch):
|
||||||
|
"""
|
||||||
|
异步请求当前完全没有重试;修复后首次失败应等待 RETRY_BACKOFF_SECONDS 秒
|
||||||
|
后重试一次,重试成功则返回正常结果。
|
||||||
|
"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
response = _FakeResponse(payload={"id": 2})
|
||||||
|
outcomes = [None, response]
|
||||||
|
|
||||||
|
async def _fake_once(method, url, data, json):
|
||||||
|
return outcomes.pop(0)
|
||||||
|
|
||||||
|
tmdb._async_request_once = _fake_once
|
||||||
|
|
||||||
|
sleep_calls = []
|
||||||
|
|
||||||
|
async def _fake_sleep(seconds):
|
||||||
|
sleep_calls.append(seconds)
|
||||||
|
|
||||||
|
monkeypatch.setattr(tmdb_module.asyncio, "sleep", _fake_sleep)
|
||||||
|
|
||||||
|
result = asyncio.run(
|
||||||
|
TMDb.async_request.__wrapped__(tmdb, "GET", "https://example.com", None, None)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result["json"] == {"id": 2}
|
||||||
|
assert sleep_calls == [tmdb_module.RETRY_BACKOFF_SECONDS]
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_request_raises_connection_error_after_retry_exhausted(monkeypatch):
|
||||||
|
"""异步请求重试一次后仍失败,应抛出 TMDbConnectionError,保持最终失败语义。"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
|
||||||
|
async def _fake_once(method, url, data, json):
|
||||||
|
return None
|
||||||
|
|
||||||
|
tmdb._async_request_once = _fake_once
|
||||||
|
|
||||||
|
async def _fake_sleep(seconds):
|
||||||
|
return None
|
||||||
|
|
||||||
|
monkeypatch.setattr(tmdb_module.asyncio, "sleep", _fake_sleep)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError, match="无法连接TheMovieDb"):
|
||||||
|
asyncio.run(TMDb.async_request.__wrapped__(tmdb, "GET", "https://example.com", None, None))
|
||||||
|
|
||||||
|
|
||||||
|
def test_request_exception_path_is_not_cached(monkeypatch):
|
||||||
|
"""
|
||||||
|
`request`/`async_request` 带 `@cached(skip_none=True)`;抛异常时函数没有
|
||||||
|
正常返回值,缓存装饰器的 `cache_backend.set` 分支不会被执行到,天然不会缓存。
|
||||||
|
这里通过真实缓存路径(不使用 __wrapped__)验证:连续两次同参数请求都会真实
|
||||||
|
触发底层请求,而不是第二次直接命中一个"异常快照"。
|
||||||
|
"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
call_count = {"n": 0}
|
||||||
|
|
||||||
|
def _always_fail(method, url, data, json):
|
||||||
|
call_count["n"] += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
tmdb._request_once = _always_fail
|
||||||
|
tmdb._reset_owned_session = lambda: None
|
||||||
|
monkeypatch.setattr(tmdb_module.time, "sleep", lambda seconds: None)
|
||||||
|
|
||||||
|
for _ in range(2):
|
||||||
|
with pytest.raises(TMDbConnectionError):
|
||||||
|
tmdb.request("GET", "https://example.com/exc-not-cached", None, None)
|
||||||
|
|
||||||
|
# 两次调用都真实触发了底层请求(各自都经历了一次重试),说明异常没有被当作
|
||||||
|
# 缓存命中而跳过;即 2 次调用 × 2 次尝试 = 4 次底层请求。
|
||||||
|
assert call_count["n"] == 4
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# §6.2 网络故障与"条目不存在"文案混淆
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
def test_request_obj_raises_plain_tmdb_exception_for_not_found_status():
|
||||||
|
"""
|
||||||
|
调研结论:TMDB对404"资源不存在"的响应体形如
|
||||||
|
`{"success": false, "status_code": 34, "status_message": "..."}`,
|
||||||
|
`TMDb._handle_errors` 据此抛出的是普通 TMDbException,而不是表示传输层失败的
|
||||||
|
TMDbConnectionError。这与`request`/`async_request`在请求彻底失败(返回None)
|
||||||
|
时抛出的 TMDbConnectionError是两类不同的异常,不应混淆处理。
|
||||||
|
"""
|
||||||
|
tmdb = TMDb()
|
||||||
|
snapshot = {
|
||||||
|
TMDb._RESPONSE_SNAPSHOT_MARKER: True,
|
||||||
|
"headers": {},
|
||||||
|
"json": {
|
||||||
|
"success": False,
|
||||||
|
"status_code": 34,
|
||||||
|
"status_message": "The resource you requested could not be found.",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
tmdb.request = lambda *args, **kwargs: snapshot
|
||||||
|
|
||||||
|
with pytest.raises(TMDbException) as exc_info:
|
||||||
|
tmdb._request_obj("/movie/999999999")
|
||||||
|
assert not isinstance(exc_info.value, TMDbConnectionError)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_movie_detail_propagates_connection_error_when_requested():
|
||||||
|
"""
|
||||||
|
`TmdbApi.get_info(..., raise_on_connection_error=True)` 遇到TMDB连接失败时
|
||||||
|
应该把 TMDbConnectionError 传播出去,而不是像默认那样吞掉返回 None。
|
||||||
|
"""
|
||||||
|
tmdb_api = TmdbApi()
|
||||||
|
tmdb_api.movie.details = Mock(
|
||||||
|
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError):
|
||||||
|
tmdb_api.get_info(mtype=MediaType.MOVIE, tmdbid=1, raise_on_connection_error=True)
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_movie_detail_swallows_connection_error_by_default():
|
||||||
|
"""
|
||||||
|
不传 `raise_on_connection_error` 时(默认False),行为必须与修复前完全一致:
|
||||||
|
任何异常都吞掉返回 None,不破坏现有调用方。
|
||||||
|
"""
|
||||||
|
tmdb_api = TmdbApi()
|
||||||
|
tmdb_api.movie.details = Mock(
|
||||||
|
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
)
|
||||||
|
|
||||||
|
assert tmdb_api.get_info(mtype=MediaType.MOVIE, tmdbid=1) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_tv_detail_not_found_error_still_returns_none_even_with_flag():
|
||||||
|
"""
|
||||||
|
即使显式要求 `raise_on_connection_error=True`,TMDB业务层返回的"资源不存在"
|
||||||
|
(普通TMDbException,非TMDbConnectionError)也不应被当作连接失败传播出去,
|
||||||
|
应继续按现有语义返回 None。
|
||||||
|
"""
|
||||||
|
tmdb_api = TmdbApi()
|
||||||
|
tmdb_api.tv.details = Mock(
|
||||||
|
side_effect=TMDbException("The resource you requested could not be found.")
|
||||||
|
)
|
||||||
|
|
||||||
|
result = tmdb_api.get_info(mtype=MediaType.TV, tmdbid=1, raise_on_connection_error=True)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_get_info_propagates_connection_error_when_requested():
|
||||||
|
"""异步版本的详情查询同样要支持连接失败的显式传播。"""
|
||||||
|
tmdb_api = TmdbApi()
|
||||||
|
tmdb_api.movie.async_details = AsyncMock(
|
||||||
|
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError):
|
||||||
|
asyncio.run(
|
||||||
|
tmdb_api.async_get_info(mtype=MediaType.MOVIE, tmdbid=1, raise_on_connection_error=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_get_info_swallows_connection_error_by_default():
|
||||||
|
"""异步默认行为同样必须保持向后兼容。"""
|
||||||
|
tmdb_api = TmdbApi()
|
||||||
|
tmdb_api.movie.async_details = AsyncMock(
|
||||||
|
side_effect=TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
)
|
||||||
|
|
||||||
|
result = asyncio.run(tmdb_api.async_get_info(mtype=MediaType.MOVIE, tmdbid=1))
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
class _FakeTmdbApi:
|
||||||
|
"""
|
||||||
|
模拟 TmdbApi.get_info/async_get_info,用给定结果驱动
|
||||||
|
`TheMovieDbModule._get_info_by_tmdbid` 的三种分支:确定命中、确认不存在、连接失败。
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, tv_outcome, movie_outcome):
|
||||||
|
"""使用电视剧、电影两路各自的结果初始化。"""
|
||||||
|
self.tv_outcome = tv_outcome
|
||||||
|
self.movie_outcome = movie_outcome
|
||||||
|
self.calls = []
|
||||||
|
|
||||||
|
def _resolve(self, outcome, raise_on_connection_error):
|
||||||
|
if isinstance(outcome, Exception):
|
||||||
|
if raise_on_connection_error and isinstance(outcome, TMDbConnectionError):
|
||||||
|
raise outcome
|
||||||
|
return None
|
||||||
|
return outcome
|
||||||
|
|
||||||
|
def match_multi(self, name: str):
|
||||||
|
"""
|
||||||
|
多类型名称匹配。真实实现(tmdbapi.match_multi)会吞掉所有异常并返回 None,
|
||||||
|
连接失败与「未找到」在这条路径上不可区分,fake 必须保持同样语义。
|
||||||
|
"""
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def async_match_multi(self, name: str):
|
||||||
|
"""异步版多类型匹配,语义同上。"""
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def async_match_multi(self, name: str):
|
||||||
|
"""异步版多类型匹配。"""
|
||||||
|
return self._multi_outcome()
|
||||||
|
|
||||||
|
def get_info(self, mtype, tmdbid, raise_on_connection_error=False):
|
||||||
|
"""同步查询:按mtype返回预置结果,忠实模拟 raise_on_connection_error 语义。"""
|
||||||
|
self.calls.append((mtype, raise_on_connection_error))
|
||||||
|
outcome = self.tv_outcome if mtype == MediaType.TV else self.movie_outcome
|
||||||
|
return self._resolve(outcome, raise_on_connection_error)
|
||||||
|
|
||||||
|
async def async_get_info(self, mtype, tmdbid, raise_on_connection_error=False):
|
||||||
|
"""异步查询:委托同步实现。"""
|
||||||
|
return self.get_info(mtype, tmdbid, raise_on_connection_error=raise_on_connection_error)
|
||||||
|
|
||||||
|
|
||||||
|
class _CacheProbe:
|
||||||
|
"""记录 update 调用参数,get 恒返回未命中,用于验证网络故障路径不会触碰缓存。"""
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
"""初始化调用记录列表。"""
|
||||||
|
self.update_calls = []
|
||||||
|
|
||||||
|
def get(self, meta):
|
||||||
|
"""始终返回缓存未命中。"""
|
||||||
|
return {}
|
||||||
|
|
||||||
|
def update(self, meta, info):
|
||||||
|
"""记录写入尝试,供测试断言从未被调用或调用了哪些内容。"""
|
||||||
|
self.update_calls.append(info)
|
||||||
|
|
||||||
|
|
||||||
|
def _build_module(tv_outcome, movie_outcome) -> TheMovieDbModule:
|
||||||
|
"""构造绕开真实HTTP/缓存的 TheMovieDbModule 测试实例。"""
|
||||||
|
module = TheMovieDbModule()
|
||||||
|
module.tmdb = _FakeTmdbApi(tv_outcome, movie_outcome)
|
||||||
|
module.cache = _CacheProbe()
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def _build_meta(tmdbid: int = 98865) -> MetaInfo:
|
||||||
|
"""构造识别所需的最小元数据。"""
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
meta.tmdbid = tmdbid
|
||||||
|
return meta
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_info_by_tmdbid_raises_connection_error_when_both_types_fail_to_connect():
|
||||||
|
"""
|
||||||
|
电影、电视剧两路查询都因TMDB连接失败而没有得到确定结果时,
|
||||||
|
不能断言"条目不存在",应向上抛出 TMDbConnectionError。
|
||||||
|
"""
|
||||||
|
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError):
|
||||||
|
module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_info_by_tmdbid_returns_none_when_both_confirmed_not_found():
|
||||||
|
"""
|
||||||
|
电影、电视剧两路查询都明确返回"未查询到"(空dict,非连接异常)时,
|
||||||
|
应保持原有"无法确定媒体类型"语义,返回 None 且不抛异常。
|
||||||
|
"""
|
||||||
|
module = _build_module(tv_outcome={}, movie_outcome={})
|
||||||
|
|
||||||
|
result = module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_info_by_tmdbid_prefers_positive_result_over_partial_connection_error():
|
||||||
|
"""
|
||||||
|
电视剧一路连接失败,但电影一路查到了确定结果时,应直接返回电影结果,
|
||||||
|
不能因为另一路的瞬时抖动就误判为整体失败。
|
||||||
|
"""
|
||||||
|
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
movie_info = {
|
||||||
|
"id": 98865,
|
||||||
|
"media_type": MediaType.MOVIE,
|
||||||
|
"title": "测试电影",
|
||||||
|
"release_date": "2020-01-01",
|
||||||
|
"genres": [{"id": 28, "name": "动作"}],
|
||||||
|
}
|
||||||
|
module = _build_module(tv_outcome=conn_err, movie_outcome=movie_info)
|
||||||
|
|
||||||
|
result = module._get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
|
||||||
|
|
||||||
|
assert result is movie_info
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_get_info_by_tmdbid_raises_connection_error_when_both_types_fail_to_connect():
|
||||||
|
"""异步版本同样要在两路都判定为连接失败时抛出 TMDbConnectionError。"""
|
||||||
|
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
|
||||||
|
|
||||||
|
with pytest.raises(TMDbConnectionError):
|
||||||
|
asyncio.run(
|
||||||
|
module._async_get_info_by_tmdbid(tmdbid=98865, mtype=None, meta=_build_meta())
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_recognize_media_reports_network_error_message_and_skips_cache_on_connection_failure(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""
|
||||||
|
端到端回归:复现因果链中的场景(tmdb_id:98865 双路查询均连接失败)。
|
||||||
|
识别应返回 None,日志应给出明确的网络故障文案(而不是"无法确定媒体类型"),
|
||||||
|
且绝不能写入任何缓存(正缓存、负缓存都不能写)。
|
||||||
|
"""
|
||||||
|
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
|
||||||
|
|
||||||
|
mock_logger = Mock()
|
||||||
|
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
|
||||||
|
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
result = module.recognize_media(meta=meta, media_source=MediaSource.TMDB,
|
||||||
|
media_id="98865", cache=True)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
# 网络故障场景不写入任何缓存条目(正缓存/负缓存皆不写)
|
||||||
|
assert module.cache.update_calls == []
|
||||||
|
|
||||||
|
logged_messages = " ".join(
|
||||||
|
str(call.args[0]) if call.args else "" for call in mock_logger.error.call_args_list
|
||||||
|
)
|
||||||
|
assert "连接TheMovieDb失败" in logged_messages or "连接 TheMovieDb 失败" in logged_messages
|
||||||
|
assert "无法确定媒体类型" not in logged_messages
|
||||||
|
|
||||||
|
|
||||||
|
def test_recognize_media_keeps_not_found_message_when_both_types_confirmed_absent(monkeypatch):
|
||||||
|
"""
|
||||||
|
电影、电视剧都确认查无此项(非连接失败)时,应维持原有的
|
||||||
|
"无法确定媒体类型,识别失败"文案,不能被网络错误文案顶替。
|
||||||
|
"""
|
||||||
|
module = _build_module(tv_outcome={}, movie_outcome={})
|
||||||
|
|
||||||
|
mock_logger = Mock()
|
||||||
|
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
|
||||||
|
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
result = module.recognize_media(meta=meta, media_source=MediaSource.TMDB,
|
||||||
|
media_id="98865", cache=True)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
assert module.cache.update_calls == []
|
||||||
|
|
||||||
|
logged_messages = " ".join(
|
||||||
|
str(call.args[0]) if call.args else "" for call in mock_logger.warn.call_args_list
|
||||||
|
)
|
||||||
|
assert "无法确定媒体类型" in logged_messages
|
||||||
|
|
||||||
|
|
||||||
|
def test_async_recognize_media_reports_network_error_message_and_skips_cache_on_connection_failure(
|
||||||
|
monkeypatch,
|
||||||
|
):
|
||||||
|
"""异步识别路径同样要区分网络故障文案,且不得写入缓存。"""
|
||||||
|
conn_err = TMDbConnectionError("无法连接TheMovieDb,请检查网络连接!")
|
||||||
|
module = _build_module(tv_outcome=conn_err, movie_outcome=conn_err)
|
||||||
|
|
||||||
|
mock_logger = Mock()
|
||||||
|
monkeypatch.setattr(themoviedb_module, "logger", mock_logger)
|
||||||
|
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
result = asyncio.run(
|
||||||
|
module.async_recognize_media(meta=meta, media_source=MediaSource.TMDB,
|
||||||
|
media_id="98865", cache=True)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
assert module.cache.update_calls == []
|
||||||
|
|
||||||
|
logged_messages = " ".join(
|
||||||
|
str(call.args[0]) if call.args else "" for call in mock_logger.error.call_args_list
|
||||||
|
)
|
||||||
|
assert "连接TheMovieDb失败" in logged_messages or "连接 TheMovieDb 失败" in logged_messages
|
||||||
|
assert "无法确定媒体类型" not in logged_messages
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_update_writes_nothing_for_connection_error_sentinel():
|
||||||
|
"""
|
||||||
|
回归锁定 `TmdbCache.update` 的既有安全语义:info=None(网络错误场景对应的
|
||||||
|
取值)不应写入任何缓存条目,正、负缓存都不写。
|
||||||
|
"""
|
||||||
|
cache = object.__new__(TmdbCache)
|
||||||
|
cache._cache = Mock()
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
meta.tmdbid = 98865
|
||||||
|
|
||||||
|
cache.update(meta, None)
|
||||||
|
|
||||||
|
cache._cache.set.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_update_still_writes_negative_cache_for_confirmed_empty_result():
|
||||||
|
"""
|
||||||
|
对照组:info={}(确认查无此项)时仍应写入负缓存 `{"id": 0}`,
|
||||||
|
与 info=None(网络错误)的行为形成对照,证明二者语义未被本次修复混淆。
|
||||||
|
"""
|
||||||
|
cache = object.__new__(TmdbCache)
|
||||||
|
cache._cache = Mock()
|
||||||
|
meta = MetaInfo(title="测试标题")
|
||||||
|
meta.tmdbid = 98865
|
||||||
|
|
||||||
|
cache.update(meta, {})
|
||||||
|
|
||||||
|
cache._cache.set.assert_called_once()
|
||||||
|
args, _kwargs = cache._cache.set.call_args
|
||||||
|
assert args[1] == {"id": 0}
|
||||||
@@ -0,0 +1,134 @@
|
|||||||
|
"""
|
||||||
|
有界重试预算的端到端行为验证。
|
||||||
|
|
||||||
|
app/helper/transferhistory.py 的查重闸真值表与计数器 API 已在
|
||||||
|
tests/test_transfer_history_gate.py 逐项覆盖,本文件换一个角度:把「同一源路径
|
||||||
|
连续多个监控事件」串成一条时间线,验证瞬时故障能在预算内自愈、耗尽预算后被拦、
|
||||||
|
以及删除整理记录会让预算重新满额,贴近真实使用场景。
|
||||||
|
"""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
from app import schemas
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.helper.transferhistory import (
|
||||||
|
HistoryGateAction,
|
||||||
|
clear_transfer_failures,
|
||||||
|
evaluate_history_gate,
|
||||||
|
failed_retry_count,
|
||||||
|
record_transfer_failure,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_failed_retries(src_path, storage=None):
|
||||||
|
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
|
def _failed_history(history_id: int, src_path: str, storage: str):
|
||||||
|
"""构造一条持续失败的整理记录替身,模拟同一源路径屡次整理失败后落库的状态。"""
|
||||||
|
return SimpleNamespace(id=history_id, status=False, src_fileitem=None,
|
||||||
|
src=src_path, src_storage=storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_transient_failures_self_heal_within_retry_budget(monkeypatch):
|
||||||
|
"""
|
||||||
|
瞬时故障自愈:上限为 3 时,连续 2 次失败后第 3 个事件仍应放行重试;
|
||||||
|
第 3 次也失败后计数达到上限,第 4 个事件应被拦截。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
|
||||||
|
src_path = "/downloads/retry-budget-self-heal.mkv"
|
||||||
|
storage = "local"
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
try:
|
||||||
|
history = _failed_history(1, src_path, storage)
|
||||||
|
|
||||||
|
# 事件 1:整理失败,登记第 1 次失败
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
assert failed_retry_count(src_path, storage) == 1
|
||||||
|
|
||||||
|
# 事件 2:计数为 1,未达上限,放行重试;本次也失败,登记第 2 次失败
|
||||||
|
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
assert failed_retry_count(src_path, storage) == 2
|
||||||
|
|
||||||
|
# 事件 3:连续失败 2 次后,计数为 2 仍未达上限(3),第 3 个事件仍应放行重试
|
||||||
|
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
|
||||||
|
|
||||||
|
# 第 3 次尝试也失败,登记第 3 次失败,计数达到上限
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
assert failed_retry_count(src_path, storage) == 3
|
||||||
|
|
||||||
|
# 事件 4:计数达到上限(3),应被拦截,不再自动重试
|
||||||
|
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_clearing_transfer_history_restores_full_retry_budget(monkeypatch):
|
||||||
|
"""删除整理记录清零:耗尽重试预算后调用 clear_transfer_failures,应重新获得放行资格。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
|
||||||
|
src_path = "/downloads/retry-budget-cleared-by-delete.mkv"
|
||||||
|
storage = "local"
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
try:
|
||||||
|
history = _failed_history(2, src_path, storage)
|
||||||
|
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
assert failed_retry_count(src_path, storage) == 2
|
||||||
|
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
|
||||||
|
# 用户删除整理记录,显式要求重来
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
assert failed_retry_count(src_path, storage) == 0
|
||||||
|
assert evaluate_history_gate(history, file_size=None) == HistoryGateAction.PASS_FAILED
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
|
def test_delete_transfer_history_endpoint_clears_retry_count(monkeypatch):
|
||||||
|
"""
|
||||||
|
app/api/endpoints/history.py::delete_transfer_history 是用户删除整理记录的入口,
|
||||||
|
删除时应连带清空失败重试计数,否则重整仍会受上一轮次数限制。
|
||||||
|
|
||||||
|
该端点依赖 SQLAlchemy Session 与鉴权依赖,这里按仓库内既有做法(参见
|
||||||
|
tests/test_manual_transfer_history.py 对 app.api.endpoints.transfer 端点的用法)
|
||||||
|
直接以关键字参数调用端点函数本身,绕开 FastAPI 的依赖注入,只替换端点内部
|
||||||
|
实际用到的 TransferHistory.get / TransferHistory.delete 两个类方法。
|
||||||
|
"""
|
||||||
|
from app.api.endpoints.history import delete_transfer_history
|
||||||
|
|
||||||
|
src_path = "/downloads/retry-budget-delete-endpoint.mkv"
|
||||||
|
storage = "local"
|
||||||
|
history = SimpleNamespace(
|
||||||
|
id=101,
|
||||||
|
src=src_path,
|
||||||
|
src_storage=storage,
|
||||||
|
dest_fileitem=None,
|
||||||
|
src_fileitem=None,
|
||||||
|
download_hash=None,
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("app.api.endpoints.history.TransferHistory.get",
|
||||||
|
lambda db, history_id: history)
|
||||||
|
monkeypatch.setattr("app.api.endpoints.history.TransferHistory.delete",
|
||||||
|
lambda db, history_id: None)
|
||||||
|
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
record_transfer_failure(src_path, storage)
|
||||||
|
assert failed_retry_count(src_path, storage) == 2
|
||||||
|
|
||||||
|
response = delete_transfer_history(
|
||||||
|
history_in=schemas.TransferHistory(id=101),
|
||||||
|
deletesrc=False,
|
||||||
|
deletedest=False,
|
||||||
|
db=object(),
|
||||||
|
_="token",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert response.success is True
|
||||||
|
assert failed_retry_count(src_path, storage) == 0
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
@@ -0,0 +1,549 @@
|
|||||||
|
"""
|
||||||
|
覆盖 app/helper/transferhistory.py 的整理历史查重闸。
|
||||||
|
|
||||||
|
监控分发(app/monitor/dispatcher.py)与整理链计划整理段(app/chain/transfer.py)
|
||||||
|
共用这套判定,本文件只测判定本身的真值表与查询辅助函数,不涉及调用方。
|
||||||
|
"""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from app.core.config import settings
|
||||||
|
from app.helper import transferhistory as transferhistory_helper
|
||||||
|
from app.helper.transferhistory import (
|
||||||
|
HistoryGateAction,
|
||||||
|
clear_transfer_failures,
|
||||||
|
coerce_size,
|
||||||
|
describe_history_gate,
|
||||||
|
evaluate_history_gate,
|
||||||
|
failed_retry_count,
|
||||||
|
history_src_size,
|
||||||
|
is_skip_action,
|
||||||
|
max_failed_retries,
|
||||||
|
record_transfer_failure,
|
||||||
|
resolve_history,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def make_history(status: bool, size=1024, has_src_fileitem: bool = True,
|
||||||
|
history_id: int = 1, src_fileitem_override=None,
|
||||||
|
src=None, src_storage=None, modify_time=None, fileid=None):
|
||||||
|
"""构造用于查重闸判定的整理记录替身。"""
|
||||||
|
if src_fileitem_override is not None:
|
||||||
|
src_fileitem = src_fileitem_override
|
||||||
|
elif has_src_fileitem:
|
||||||
|
src_fileitem = {
|
||||||
|
"size": size,
|
||||||
|
"modify_time": modify_time,
|
||||||
|
"fileid": fileid,
|
||||||
|
}
|
||||||
|
else:
|
||||||
|
src_fileitem = None
|
||||||
|
return SimpleNamespace(id=history_id, status=status, src_fileitem=src_fileitem,
|
||||||
|
src=src, src_storage=src_storage)
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_failed_retries(src_path, storage=None):
|
||||||
|
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# evaluate_history_gate 真值表:无记录 / 成功记录
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_passes_when_no_record():
|
||||||
|
"""没有整理记录时应放行整理。"""
|
||||||
|
action = evaluate_history_gate(None, file_size=1024)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.PASS_NO_RECORD
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_passes_when_success_size_changed():
|
||||||
|
"""成功记录但源文件大小已变化时应放行,交由 overwrite_mode 决断。"""
|
||||||
|
history = make_history(status=True, size=1024)
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=2048)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.PASS_SIZE_CHANGED
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_skips_when_success_size_unchanged():
|
||||||
|
"""成功记录且源文件大小未变化时应跳过。"""
|
||||||
|
history = make_history(status=True, size=1024)
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=1024)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.SKIP
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_skips_when_recorded_size_missing():
|
||||||
|
"""成功记录缺少大小信息(如蓝光目录)时无法比对,保守跳过。"""
|
||||||
|
history = make_history(status=True, has_src_fileitem=False)
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=1024)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.SKIP
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_skips_when_current_size_is_none():
|
||||||
|
"""当前文件大小取不到时无法比对,保守跳过。"""
|
||||||
|
history = make_history(status=True, size=1024)
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=None)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.SKIP
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_skips_when_src_fileitem_is_not_dict():
|
||||||
|
"""src_fileitem 历史数据异常(不是字典)时无法比对,保守跳过。"""
|
||||||
|
history = make_history(status=True, src_fileitem_override="not-a-dict")
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=1024)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.SKIP
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_size_changed_ignores_failed_retry_budget(monkeypatch):
|
||||||
|
"""
|
||||||
|
重试次数上限只影响失败记录的判定:即便同路径的失败计数早已超过上限,
|
||||||
|
成功记录 + 源文件大小变化时仍应放行,交由 overwrite_mode 决断。
|
||||||
|
"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
|
||||||
|
src_path = "/downloads/gate-test-size-changed-ignores-budget.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
for _ in range(5):
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
assert failed_retry_count(src_path, "local") > max_failed_retries()
|
||||||
|
|
||||||
|
history = make_history(status=True, size=1024, src=src_path, src_storage="local")
|
||||||
|
action = evaluate_history_gate(history, file_size=2048)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.PASS_SIZE_CHANGED
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# evaluate_history_gate 真值表:失败记录 —— 有界重试
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("retry_count, expected", [
|
||||||
|
(0, HistoryGateAction.PASS_FAILED),
|
||||||
|
(1, HistoryGateAction.PASS_FAILED),
|
||||||
|
(2, HistoryGateAction.PASS_FAILED),
|
||||||
|
(3, HistoryGateAction.SKIP_RETRY_EXHAUSTED),
|
||||||
|
(4, HistoryGateAction.SKIP_RETRY_EXHAUSTED),
|
||||||
|
])
|
||||||
|
def test_evaluate_history_gate_failed_record_retry_budget_truth_table(monkeypatch, retry_count, expected):
|
||||||
|
"""失败记录按有界重试判定:未达上限(含 0)放行重试,达到或超过上限后跳过。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
|
||||||
|
history = make_history(status=False, size=1024)
|
||||||
|
|
||||||
|
action = evaluate_history_gate(history, file_size=1024, retry_count=retry_count)
|
||||||
|
|
||||||
|
assert action == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_failed_record_queries_realtime_count_when_omitted(monkeypatch):
|
||||||
|
"""retry_count 省略(为 None)时应按 history.src / history.src_storage 实时查询失败计数。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
|
||||||
|
src_path = "/downloads/gate-test-realtime-count.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
history = make_history(status=False, size=1024, src=src_path, src_storage="local")
|
||||||
|
|
||||||
|
# 计数为 0:未达上限(2),放行重试
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.PASS_FAILED
|
||||||
|
|
||||||
|
# 累计一次失败,计数为 1:仍未达上限
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.PASS_FAILED
|
||||||
|
|
||||||
|
# 累计第二次失败,计数为 2:达到上限,跳过
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_evaluate_history_gate_explicit_retry_count_overrides_realtime_lookup(monkeypatch):
|
||||||
|
"""显式传入 retry_count 时不应再触发实时查询,不受计数器实际状态影响。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
|
||||||
|
src_path = "/downloads/gate-test-explicit-overrides-realtime.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
for _ in range(10):
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
history = make_history(status=False, size=1024, src=src_path, src_storage="local")
|
||||||
|
|
||||||
|
# 即使实时计数早已超限,显式传入的低 retry_count 仍应放行
|
||||||
|
action = evaluate_history_gate(history, file_size=1024, retry_count=0)
|
||||||
|
|
||||||
|
assert action == HistoryGateAction.PASS_FAILED
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_history_new_size_passes_and_resets_retry_budget(monkeypatch):
|
||||||
|
"""失败预算耗尽后同路径文件大小变化时,应放行新版本并从第 1 次失败重新计数。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
|
||||||
|
src_path = "/downloads/gate-test-failed-new-size.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
history = make_history(
|
||||||
|
status=False,
|
||||||
|
size=1024,
|
||||||
|
src=src_path,
|
||||||
|
src_storage="local",
|
||||||
|
)
|
||||||
|
record_transfer_failure(src_path, "local", file_size=1024)
|
||||||
|
record_transfer_failure(src_path, "local", file_size=1024)
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
|
||||||
|
assert (
|
||||||
|
evaluate_history_gate(history, file_size=2048)
|
||||||
|
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
|
||||||
|
)
|
||||||
|
assert record_transfer_failure(src_path, "local", file_size=2048) == 1
|
||||||
|
assert failed_retry_count(src_path, "local", file_size=2048) == 1
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
"current_fields",
|
||||||
|
[
|
||||||
|
{"file_modify_time": 200.0, "fileid": "same-id"},
|
||||||
|
{"file_modify_time": 100.0, "fileid": "new-id"},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_failed_history_same_size_new_fingerprint_passes(current_fields, monkeypatch):
|
||||||
|
"""大小相同但修改时间或文件 ID 改变时,也应视为失败文件的新版本。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 1)
|
||||||
|
src_path = "/downloads/gate-test-failed-new-fingerprint.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
history = make_history(
|
||||||
|
status=False,
|
||||||
|
size=1024,
|
||||||
|
modify_time=100.0,
|
||||||
|
fileid="same-id",
|
||||||
|
src=src_path,
|
||||||
|
src_storage="local",
|
||||||
|
)
|
||||||
|
record_transfer_failure(
|
||||||
|
src_path,
|
||||||
|
"local",
|
||||||
|
file_size=1024,
|
||||||
|
file_modify_time=100.0,
|
||||||
|
fileid="same-id",
|
||||||
|
)
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
|
||||||
|
assert (
|
||||||
|
evaluate_history_gate(history, file_size=1024, **current_fields)
|
||||||
|
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_legacy_integer_retry_count_is_upgraded_after_new_version_failure(monkeypatch):
|
||||||
|
"""Redis 中遗留的整数计数不得阻断新版本,并应在下次失败时升级为指纹状态。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 2)
|
||||||
|
src_path = "/downloads/gate-test-legacy-retry-state.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
key = transferhistory_helper.failed_retry_key(src_path, "local")
|
||||||
|
transferhistory_helper._failed_retry_counts[key] = 2
|
||||||
|
history = make_history(
|
||||||
|
status=False,
|
||||||
|
size=1024,
|
||||||
|
src=src_path,
|
||||||
|
src_storage="local",
|
||||||
|
)
|
||||||
|
assert evaluate_history_gate(history, file_size=1024) == HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||||
|
assert (
|
||||||
|
evaluate_history_gate(history, file_size=2048)
|
||||||
|
== HistoryGateAction.PASS_FAILED_VERSION_CHANGED
|
||||||
|
)
|
||||||
|
|
||||||
|
assert record_transfer_failure(src_path, "local", file_size=2048) == 1
|
||||||
|
assert failed_retry_count(src_path, "local", file_size=2048) == 1
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# max_failed_retries 钳制
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("raw, expected", [
|
||||||
|
(-1, 1),
|
||||||
|
(0, 1),
|
||||||
|
(1, 1),
|
||||||
|
(3, 3),
|
||||||
|
(10, 10),
|
||||||
|
(11, 10),
|
||||||
|
])
|
||||||
|
def test_max_failed_retries_clamps_numeric_values(monkeypatch, raw, expected):
|
||||||
|
"""合法区间外的配置值应被钳制到 [1, 10],区间内的值原样返回。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", raw)
|
||||||
|
|
||||||
|
assert max_failed_retries() == expected
|
||||||
|
|
||||||
|
|
||||||
|
def test_max_failed_retries_falls_back_when_non_integer(monkeypatch):
|
||||||
|
"""非整数配置(如解析失败的字符串)应回退为下界 1。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", "abc")
|
||||||
|
|
||||||
|
assert max_failed_retries() == 1
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 失败计数器:record_transfer_failure / failed_retry_count / clear_transfer_failures
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_record_transfer_failure_returns_incrementing_count():
|
||||||
|
"""连续记录失败应返回递增的累计次数。"""
|
||||||
|
src_path = "/downloads/gate-test-incrementing-count.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
assert record_transfer_failure(src_path, "local") == 1
|
||||||
|
assert record_transfer_failure(src_path, "local") == 2
|
||||||
|
assert record_transfer_failure(src_path, "local") == 3
|
||||||
|
assert failed_retry_count(src_path, "local") == 3
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_transfer_failures_resets_count_to_zero():
|
||||||
|
"""清空后应归零,且不再影响后续查询。"""
|
||||||
|
src_path = "/downloads/gate-test-clear-resets.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
assert failed_retry_count(src_path, "local") == 2
|
||||||
|
|
||||||
|
clear_transfer_failures(src_path, "local")
|
||||||
|
|
||||||
|
assert failed_retry_count(src_path, "local") == 0
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_retry_count_isolates_different_storages_for_same_path():
|
||||||
|
"""相同源路径但不同 storage 的失败计数应互不影响。"""
|
||||||
|
src_path = "/downloads/gate-test-storage-isolation.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
_reset_failed_retries(src_path, "alist")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
record_transfer_failure(src_path, "alist")
|
||||||
|
|
||||||
|
assert failed_retry_count(src_path, "local") == 2
|
||||||
|
assert failed_retry_count(src_path, "alist") == 1
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
_reset_failed_retries(src_path, "alist")
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_retry_count_defaults_to_zero_without_record():
|
||||||
|
"""未记录过失败的路径应返回 0。"""
|
||||||
|
src_path = "/downloads/gate-test-no-record-yet.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
assert failed_retry_count(src_path, "local") == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# is_skip_action
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("action, expected", [
|
||||||
|
(HistoryGateAction.PASS_NO_RECORD, False),
|
||||||
|
(HistoryGateAction.PASS_FAILED, False),
|
||||||
|
(HistoryGateAction.PASS_FAILED_VERSION_CHANGED, False),
|
||||||
|
(HistoryGateAction.PASS_SIZE_CHANGED, False),
|
||||||
|
(HistoryGateAction.SKIP_RETRY_EXHAUSTED, True),
|
||||||
|
(HistoryGateAction.SKIP, True),
|
||||||
|
])
|
||||||
|
def test_is_skip_action(action, expected):
|
||||||
|
"""跳过整理的判定应只对 SKIP 与 SKIP_RETRY_EXHAUSTED 返回 True。"""
|
||||||
|
assert is_skip_action(action) is expected
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# coerce_size
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_coerce_size_returns_none_for_none():
|
||||||
|
"""None 应原样返回 None,表示不可比对。"""
|
||||||
|
assert coerce_size(None) is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_coerce_size_returns_none_for_non_numeric_string():
|
||||||
|
"""非数字字符串无法转换,应返回 None。"""
|
||||||
|
assert coerce_size("not-a-number") is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_coerce_size_truncates_float():
|
||||||
|
"""浮点数应按 int() 截断转换。"""
|
||||||
|
assert coerce_size(1024.9) == 1024
|
||||||
|
|
||||||
|
|
||||||
|
def test_coerce_size_parses_numeric_string():
|
||||||
|
"""数字字符串应正确转换为整数。"""
|
||||||
|
assert coerce_size("2048") == 2048
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# resolve_history
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_history_upgrades_failed_hit_to_success_record():
|
||||||
|
"""get_by_src 命中失败记录且存在成功记录时,应返回成功记录。"""
|
||||||
|
failed_history = make_history(status=False, history_id=1)
|
||||||
|
success_history = make_history(status=True, history_id=2)
|
||||||
|
oper = SimpleNamespace(
|
||||||
|
get_by_src=lambda src, storage=None: failed_history,
|
||||||
|
get_success_by_src=lambda src, storage=None: success_history,
|
||||||
|
)
|
||||||
|
|
||||||
|
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
|
||||||
|
|
||||||
|
assert history is success_history
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_history_keeps_failed_hit_when_no_success_record():
|
||||||
|
"""get_by_src 命中失败记录但不存在成功记录时,应返回原失败记录。"""
|
||||||
|
failed_history = make_history(status=False, history_id=1)
|
||||||
|
oper = SimpleNamespace(
|
||||||
|
get_by_src=lambda src, storage=None: failed_history,
|
||||||
|
get_success_by_src=lambda src, storage=None: None,
|
||||||
|
)
|
||||||
|
|
||||||
|
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
|
||||||
|
|
||||||
|
assert history is failed_history
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_history_does_not_query_success_when_already_successful():
|
||||||
|
"""get_by_src 直接命中成功记录时,不应再查询 get_success_by_src。"""
|
||||||
|
success_history = make_history(status=True, history_id=3)
|
||||||
|
success_query_calls = []
|
||||||
|
|
||||||
|
def get_success_by_src(src, storage=None):
|
||||||
|
success_query_calls.append(src)
|
||||||
|
return success_history
|
||||||
|
|
||||||
|
oper = SimpleNamespace(
|
||||||
|
get_by_src=lambda src, storage=None: success_history,
|
||||||
|
get_success_by_src=get_success_by_src,
|
||||||
|
)
|
||||||
|
|
||||||
|
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
|
||||||
|
|
||||||
|
assert history is success_history
|
||||||
|
assert success_query_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_resolve_history_returns_none_when_no_record():
|
||||||
|
"""没有命中任何记录时应返回 None,不触发额外查询。"""
|
||||||
|
success_query_calls = []
|
||||||
|
|
||||||
|
def get_success_by_src(src, storage=None):
|
||||||
|
success_query_calls.append(src)
|
||||||
|
return None
|
||||||
|
|
||||||
|
oper = SimpleNamespace(
|
||||||
|
get_by_src=lambda src, storage=None: None,
|
||||||
|
get_success_by_src=get_success_by_src,
|
||||||
|
)
|
||||||
|
|
||||||
|
history = resolve_history("/downloads/a.mkv", storage="local", transfer_history_oper=oper)
|
||||||
|
|
||||||
|
assert history is None
|
||||||
|
assert success_query_calls == []
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# describe_history_gate
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_describe_history_gate_reports_no_record():
|
||||||
|
"""没有记录时应给出明确的无记录说明。"""
|
||||||
|
assert describe_history_gate(None, file_size=1024) == "无整理记录"
|
||||||
|
|
||||||
|
|
||||||
|
def test_describe_history_gate_includes_status_and_sizes():
|
||||||
|
"""说明文案应包含记录状态、记录中的大小与当前大小两个数值。"""
|
||||||
|
history = make_history(status=True, size=1024, history_id=9)
|
||||||
|
|
||||||
|
description = describe_history_gate(history, file_size=2048)
|
||||||
|
|
||||||
|
assert "成功记录 #9" in description
|
||||||
|
assert str(history_src_size(history)) in description
|
||||||
|
assert str(coerce_size(2048)) in description
|
||||||
|
assert "1024" in description
|
||||||
|
assert "2048" in description
|
||||||
|
|
||||||
|
|
||||||
|
def test_describe_history_gate_reports_failed_status_with_retry_progress(monkeypatch):
|
||||||
|
"""失败记录的说明文案应标注记录号,并包含「已重试 n/max 次」的实时计数。"""
|
||||||
|
monkeypatch.setattr(settings, "TRANSFER_MAX_FAILED_RETRIES", 3)
|
||||||
|
src_path = "/downloads/gate-test-describe-failed.mkv"
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
try:
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
record_transfer_failure(src_path, "local")
|
||||||
|
history = make_history(status=False, size=1024, history_id=5,
|
||||||
|
src=src_path, src_storage="local")
|
||||||
|
|
||||||
|
description = describe_history_gate(history, file_size=1024)
|
||||||
|
|
||||||
|
assert "失败记录 #5" in description
|
||||||
|
assert "已重试" in description
|
||||||
|
assert "2/3" in description
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, "local")
|
||||||
|
|
||||||
|
|
||||||
|
def test_describe_history_gate_reports_incomparable_sizes():
|
||||||
|
"""两侧大小都取不到时应说明大小不可比对。"""
|
||||||
|
history = make_history(status=True, has_src_fileitem=False, history_id=7)
|
||||||
|
|
||||||
|
description = describe_history_gate(history, file_size=None)
|
||||||
|
|
||||||
|
assert description == "成功记录 #7,大小不可比对"
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_transfer_failures_is_safe_when_no_count_recorded():
|
||||||
|
"""
|
||||||
|
清空从未失败过的源路径不得抛异常。
|
||||||
|
|
||||||
|
整理成功回调会对每个文件无条件清零,而绝大多数文件从未失败过;
|
||||||
|
底层 CacheBackend.pop 把「default 为 None」当成「未提供 default」,
|
||||||
|
键不存在时会抛 KeyError,一旦回归就会让每一次首次成功整理都失败。
|
||||||
|
"""
|
||||||
|
clear_transfer_failures("/downloads/gate-test-never-failed.mkv", "local")
|
||||||
|
|
||||||
|
assert failed_retry_count("/downloads/gate-test-never-failed.mkv", "local") == 0
|
||||||
|
|
||||||
|
|
||||||
|
def test_clear_transfer_failures_is_safe_for_empty_path():
|
||||||
|
"""源路径为空时清零应静默返回,不得抛异常。"""
|
||||||
|
clear_transfer_failures(None, None)
|
||||||
|
clear_transfer_failures("", "local")
|
||||||
@@ -7,11 +7,21 @@ from app.core.config import settings
|
|||||||
from app.core.context import MediaInfo
|
from app.core.context import MediaInfo
|
||||||
from app.core.meta import MetaVideo
|
from app.core.meta import MetaVideo
|
||||||
from app.chain.transfer import JobManager, TransferChain
|
from app.chain.transfer import JobManager, TransferChain
|
||||||
|
from app.helper.transferhistory import (
|
||||||
|
clear_transfer_failures,
|
||||||
|
failed_retry_count,
|
||||||
|
record_transfer_failure,
|
||||||
|
)
|
||||||
from app.modules.filemanager.transhandler import TransHandler
|
from app.modules.filemanager.transhandler import TransHandler
|
||||||
from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask
|
from app.schemas import EpisodeFormat, FileItem, TransferInfo, TransferTask
|
||||||
from app.schemas.types import EventType, MediaSource, MediaType
|
from app.schemas.types import EventType, MediaSource, MediaType
|
||||||
|
|
||||||
|
|
||||||
|
def _reset_failed_retries(src_path, storage=None):
|
||||||
|
"""清空失败重试计数,隔离用例之间共享的模块级计数缓存。"""
|
||||||
|
clear_transfer_failures(src_path, storage)
|
||||||
|
|
||||||
|
|
||||||
class FakeMeta:
|
class FakeMeta:
|
||||||
def __init__(self, episode: int, season: int = 1):
|
def __init__(self, episode: int, season: int = 1):
|
||||||
self.name = "Test Show"
|
self.name = "Test Show"
|
||||||
@@ -600,12 +610,14 @@ class TransferJobManagerTest(unittest.TestCase):
|
|||||||
|
|
||||||
fileitem = make_task(1).fileitem
|
fileitem = make_task(1).fileitem
|
||||||
history = SimpleNamespace(
|
history = SimpleNamespace(
|
||||||
|
id=1,
|
||||||
status=True,
|
status=True,
|
||||||
download_hash="abc123",
|
download_hash="abc123",
|
||||||
downloader="qbittorrent",
|
downloader="qbittorrent",
|
||||||
)
|
)
|
||||||
transfer_history_oper = SimpleNamespace(
|
transfer_history_oper = SimpleNamespace(
|
||||||
get_by_src=lambda src, storage=None: history
|
get_by_src=lambda src, storage=None: history,
|
||||||
|
get_success_by_src=lambda src, storage=None: history,
|
||||||
)
|
)
|
||||||
system_config_oper = SimpleNamespace(get=lambda key: None)
|
system_config_oper = SimpleNamespace(get=lambda key: None)
|
||||||
|
|
||||||
@@ -628,7 +640,11 @@ class TransferJobManagerTest(unittest.TestCase):
|
|||||||
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
|
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
|
||||||
self.assertEqual([("abc123", "qbittorrent")], completed)
|
self.assertEqual([("abc123", "qbittorrent")], completed)
|
||||||
|
|
||||||
def test_failed_history_skip_still_marks_downloader_hash_completed(self):
|
def test_failed_history_is_retried_within_retry_budget(self):
|
||||||
|
"""
|
||||||
|
失败重试次数未达上限(默认计数为 0)时,失败记录不再跳过整理,
|
||||||
|
会放行重新送入整理链;由于种子还没有真正整理完成,也不能连带把种子标记为已整理。
|
||||||
|
"""
|
||||||
chain = make_transfer_chain()
|
chain = make_transfer_chain()
|
||||||
completed = []
|
completed = []
|
||||||
|
|
||||||
@@ -641,35 +657,222 @@ class TransferJobManagerTest(unittest.TestCase):
|
|||||||
(fileitem, False)
|
(fileitem, False)
|
||||||
]
|
]
|
||||||
|
|
||||||
|
planned = []
|
||||||
|
|
||||||
|
def fake_handle_transfer(task, callback=None):
|
||||||
|
"""记录放行重试后实际进入整理执行阶段的文件。"""
|
||||||
|
planned.append(task.fileitem.path)
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
chain._TransferChain__handle_transfer = fake_handle_transfer
|
||||||
|
|
||||||
fileitem = make_task(1).fileitem
|
fileitem = make_task(1).fileitem
|
||||||
history = SimpleNamespace(
|
history = SimpleNamespace(
|
||||||
|
id=2,
|
||||||
status=False,
|
status=False,
|
||||||
download_hash="abc123",
|
download_hash="abc123",
|
||||||
downloader="qbittorrent",
|
downloader="qbittorrent",
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
)
|
)
|
||||||
transfer_history_oper = SimpleNamespace(
|
transfer_history_oper = SimpleNamespace(
|
||||||
get_by_src=lambda src, storage=None: history
|
get_by_src=lambda src, storage=None: history,
|
||||||
|
get_success_by_src=lambda src, storage=None: None,
|
||||||
|
)
|
||||||
|
download_history_oper = SimpleNamespace(
|
||||||
|
get_by_hash=lambda download_hash: None,
|
||||||
|
get_file_by_fullpath=lambda fullpath: None,
|
||||||
|
get_files_by_savepath=lambda savepath: [],
|
||||||
|
get_by_path=lambda path: None,
|
||||||
)
|
)
|
||||||
system_config_oper = SimpleNamespace(get=lambda key: None)
|
system_config_oper = SimpleNamespace(get=lambda key: None)
|
||||||
|
|
||||||
with patch(
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
"app.chain.transfer.TransferHistoryOper",
|
try:
|
||||||
return_value=transfer_history_oper,
|
with patch(
|
||||||
), patch(
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
"app.chain.transfer.SystemConfigOper",
|
return_value=transfer_history_oper,
|
||||||
return_value=system_config_oper,
|
), patch(
|
||||||
):
|
"app.chain.transfer.DownloadHistoryOper",
|
||||||
state, errmsg = TransferChain.do_transfer(
|
return_value=download_history_oper,
|
||||||
chain,
|
), patch(
|
||||||
fileitem=fileitem,
|
"app.chain.transfer.SystemConfigOper",
|
||||||
downloader="qbittorrent",
|
return_value=system_config_oper,
|
||||||
download_hash="abc123",
|
):
|
||||||
background=False,
|
state, errmsg = TransferChain.do_transfer(
|
||||||
)
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
downloader="qbittorrent",
|
||||||
|
download_hash="abc123",
|
||||||
|
background=False,
|
||||||
|
)
|
||||||
|
|
||||||
self.assertFalse(state)
|
self.assertTrue(state)
|
||||||
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
|
self.assertEqual("", errmsg)
|
||||||
self.assertEqual([("abc123", "qbittorrent")], completed)
|
self.assertEqual([fileitem.path], planned)
|
||||||
|
self.assertEqual([], completed)
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
def test_failed_history_skip_marks_downloader_hash_when_retry_budget_exhausted(self):
|
||||||
|
"""
|
||||||
|
失败重试次数已达上限时,失败记录仍会拦截整理;但拦截意味着不再重试,
|
||||||
|
此时仍要给种子打已整理标签,让种子退出下载器轮询,否则下载器每一轮都会
|
||||||
|
重新扫描到同一个失败记录,空转且刷屏。
|
||||||
|
"""
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
completed = []
|
||||||
|
|
||||||
|
def fake_transfer_completed(hashs, downloader):
|
||||||
|
completed.append((hashs, downloader))
|
||||||
|
|
||||||
|
chain.transfer_completed = fake_transfer_completed
|
||||||
|
chain.list_torrents = lambda **kwargs: [SimpleNamespace(progress=100)]
|
||||||
|
chain._TransferChain__get_trans_fileitems = lambda fileitem, predicate: [
|
||||||
|
(fileitem, False)
|
||||||
|
]
|
||||||
|
|
||||||
|
planned = []
|
||||||
|
|
||||||
|
def fake_handle_transfer(task, callback=None):
|
||||||
|
"""达到重试上限时不应有文件进入实际整理执行阶段。"""
|
||||||
|
planned.append(task.fileitem.path)
|
||||||
|
return True, ""
|
||||||
|
|
||||||
|
chain._TransferChain__handle_transfer = fake_handle_transfer
|
||||||
|
|
||||||
|
fileitem = make_task(1).fileitem
|
||||||
|
history = SimpleNamespace(
|
||||||
|
id=3,
|
||||||
|
status=False,
|
||||||
|
download_hash="abc123",
|
||||||
|
downloader="qbittorrent",
|
||||||
|
src=fileitem.path,
|
||||||
|
src_storage=fileitem.storage,
|
||||||
|
)
|
||||||
|
transfer_history_oper = SimpleNamespace(
|
||||||
|
get_by_src=lambda src, storage=None: history,
|
||||||
|
get_success_by_src=lambda src, storage=None: None,
|
||||||
|
)
|
||||||
|
download_history_oper = SimpleNamespace(
|
||||||
|
get_by_hash=lambda download_hash: None,
|
||||||
|
get_file_by_fullpath=lambda fullpath: None,
|
||||||
|
get_files_by_savepath=lambda savepath: [],
|
||||||
|
get_by_path=lambda path: None,
|
||||||
|
)
|
||||||
|
system_config_oper = SimpleNamespace(get=lambda key: None)
|
||||||
|
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
try:
|
||||||
|
with patch.object(
|
||||||
|
settings, "TRANSFER_MAX_FAILED_RETRIES", 1,
|
||||||
|
):
|
||||||
|
record_transfer_failure(fileitem.path, fileitem.storage)
|
||||||
|
with patch(
|
||||||
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
|
return_value=transfer_history_oper,
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.DownloadHistoryOper",
|
||||||
|
return_value=download_history_oper,
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.SystemConfigOper",
|
||||||
|
return_value=system_config_oper,
|
||||||
|
):
|
||||||
|
state, errmsg = TransferChain.do_transfer(
|
||||||
|
chain,
|
||||||
|
fileitem=fileitem,
|
||||||
|
downloader="qbittorrent",
|
||||||
|
download_hash="abc123",
|
||||||
|
background=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
self.assertFalse(state)
|
||||||
|
self.assertEqual("Test.Show.S01E01.mkv 已整理过", errmsg)
|
||||||
|
self.assertEqual([], planned)
|
||||||
|
self.assertEqual([("abc123", "qbittorrent")], completed)
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(fileitem.path, fileitem.storage)
|
||||||
|
|
||||||
|
def test_default_callback_failure_and_success_track_retry_counter(self):
|
||||||
|
"""
|
||||||
|
__default_callback 应在整理失败时累计连续失败次数、整理成功时清零,
|
||||||
|
避免瞬时故障与长期失败混用同一份计数导致误判上限。
|
||||||
|
"""
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
chain.eventmanager = MagicMock()
|
||||||
|
chain.post_message = MagicMock()
|
||||||
|
chain.transfer_completed = lambda *args, **kwargs: None
|
||||||
|
|
||||||
|
task = make_task(1)
|
||||||
|
task.mediainfo = FakeMedia()
|
||||||
|
# __default_callback 失败通知路径需要读取海报图,FakeMedia 本身不提供该接口
|
||||||
|
task.mediainfo.get_message_image = lambda: "poster.jpg"
|
||||||
|
task.background = False
|
||||||
|
task.manual = True
|
||||||
|
|
||||||
|
src_path = task.fileitem.path
|
||||||
|
storage = task.fileitem.storage
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
try:
|
||||||
|
self.assertEqual(0, failed_retry_count(src_path, storage))
|
||||||
|
|
||||||
|
failed_transferinfo = TransferInfo(
|
||||||
|
success=False,
|
||||||
|
fileitem=task.fileitem,
|
||||||
|
message="整理失败测试",
|
||||||
|
transfer_type="copy",
|
||||||
|
need_notify=False,
|
||||||
|
)
|
||||||
|
failed_history_oper = SimpleNamespace(
|
||||||
|
add_fail=lambda **kwargs: SimpleNamespace(id=1),
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
|
return_value=failed_history_oper,
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
|
||||||
|
):
|
||||||
|
state, _ = chain._TransferChain__default_callback(task, failed_transferinfo)
|
||||||
|
|
||||||
|
self.assertFalse(state)
|
||||||
|
self.assertEqual(1, failed_retry_count(src_path, storage))
|
||||||
|
|
||||||
|
self.assertTrue(chain._TransferChain__put_to_jobview(task))
|
||||||
|
success_transferinfo = TransferInfo(
|
||||||
|
success=True,
|
||||||
|
fileitem=task.fileitem,
|
||||||
|
target_item=FileItem(
|
||||||
|
storage=storage,
|
||||||
|
path="/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv",
|
||||||
|
type="file",
|
||||||
|
name="Test.Show.S01E01.mkv",
|
||||||
|
extension="mkv",
|
||||||
|
),
|
||||||
|
target_diritem=FileItem(
|
||||||
|
storage=storage,
|
||||||
|
path="/library/Test Show (2026)/Season 1/",
|
||||||
|
type="dir",
|
||||||
|
name="Season 1",
|
||||||
|
),
|
||||||
|
file_list_new=[
|
||||||
|
"/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv"
|
||||||
|
],
|
||||||
|
transfer_type="copy",
|
||||||
|
need_scrape=False,
|
||||||
|
need_notify=False,
|
||||||
|
)
|
||||||
|
with patch(
|
||||||
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
|
return_value=SimpleNamespace(add_success=lambda **kwargs: SimpleNamespace(id=2)),
|
||||||
|
):
|
||||||
|
state, _ = chain._TransferChain__default_callback(task, success_transferinfo)
|
||||||
|
|
||||||
|
self.assertTrue(state)
|
||||||
|
self.assertEqual(0, failed_retry_count(src_path, storage))
|
||||||
|
finally:
|
||||||
|
_reset_failed_retries(src_path, storage)
|
||||||
|
|
||||||
def test_unrecognized_task_marks_downloader_hash_completed(self):
|
def test_unrecognized_task_marks_downloader_hash_completed(self):
|
||||||
chain = make_transfer_chain()
|
chain = make_transfer_chain()
|
||||||
|
|||||||
@@ -0,0 +1,218 @@
|
|||||||
|
"""
|
||||||
|
覆盖「不覆盖裁决不应降级已有成功记录」的行为。
|
||||||
|
|
||||||
|
查重闸放行同路径新版本后,若 overwrite_mode 最终裁定不覆盖,媒体库中原有的
|
||||||
|
成功版本仍然在位——这是一次正常策略裁决而非整理故障。TransferChain 内部的
|
||||||
|
__is_overwrite_declined 用于识别这一场景,__default_callback 失败分支据此
|
||||||
|
决定是否写失败历史、发送失败事件与失败通知。本文件覆盖两者。
|
||||||
|
"""
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
|
from app.chain.transfer import TransferChain
|
||||||
|
from app.schemas import TransferInfo
|
||||||
|
from app.schemas.types import EventType
|
||||||
|
from tests.test_transfer_job_manager import FakeMedia, make_task, make_transfer_chain
|
||||||
|
|
||||||
|
|
||||||
|
def make_history_oper(history=None, success_history=None, raise_on_query: bool = False,
|
||||||
|
add_fail_calls=None):
|
||||||
|
"""构造 __is_overwrite_declined / __default_callback 查询与写入整理历史使用的替身。"""
|
||||||
|
|
||||||
|
def get_by_src(src, storage=None):
|
||||||
|
if raise_on_query:
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
return history
|
||||||
|
|
||||||
|
def get_success_by_src(src, storage=None):
|
||||||
|
return success_history
|
||||||
|
|
||||||
|
def add_fail(**kwargs):
|
||||||
|
if add_fail_calls is not None:
|
||||||
|
add_fail_calls.append(kwargs)
|
||||||
|
return SimpleNamespace(id=1)
|
||||||
|
|
||||||
|
return SimpleNamespace(
|
||||||
|
get_by_src=get_by_src,
|
||||||
|
get_success_by_src=get_success_by_src,
|
||||||
|
add_fail=add_fail,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# TransferChain.__is_overwrite_declined
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def test_overwrite_declined_false_when_flag_not_set():
|
||||||
|
"""overwrite_skipped 为假时直接判定为 False,且不应触发历史查询。"""
|
||||||
|
task = make_task(1)
|
||||||
|
transferinfo = TransferInfo(success=False, overwrite_skipped=False)
|
||||||
|
transferhis = make_history_oper(raise_on_query=True)
|
||||||
|
|
||||||
|
result = TransferChain._TransferChain__is_overwrite_declined(
|
||||||
|
task, transferinfo, transferhis
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_overwrite_declined_true_when_success_history_exists():
|
||||||
|
"""overwrite_skipped 为真且同源已有成功记录时,应判定为保护场景。"""
|
||||||
|
task = make_task(1)
|
||||||
|
success_history = SimpleNamespace(id=1, status=True)
|
||||||
|
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
|
||||||
|
transferhis = make_history_oper(history=success_history)
|
||||||
|
|
||||||
|
result = TransferChain._TransferChain__is_overwrite_declined(
|
||||||
|
task, transferinfo, transferhis
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_overwrite_declined_false_when_no_history():
|
||||||
|
"""overwrite_skipped 为真但没有任何整理记录时,不应判定为保护场景。"""
|
||||||
|
task = make_task(1)
|
||||||
|
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
|
||||||
|
transferhis = make_history_oper(history=None)
|
||||||
|
|
||||||
|
result = TransferChain._TransferChain__is_overwrite_declined(
|
||||||
|
task, transferinfo, transferhis
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_overwrite_declined_false_when_only_failed_history():
|
||||||
|
"""overwrite_skipped 为真但只有失败记录时,不应判定为保护场景。"""
|
||||||
|
task = make_task(1)
|
||||||
|
failed_history = SimpleNamespace(id=2, status=False)
|
||||||
|
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
|
||||||
|
transferhis = make_history_oper(history=failed_history, success_history=None)
|
||||||
|
|
||||||
|
result = TransferChain._TransferChain__is_overwrite_declined(
|
||||||
|
task, transferinfo, transferhis
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_overwrite_declined_false_when_query_raises():
|
||||||
|
"""查询整理历史异常时应保守返回 False,不阻断原有失败语义。"""
|
||||||
|
task = make_task(1)
|
||||||
|
transferinfo = TransferInfo(success=False, overwrite_skipped=True)
|
||||||
|
transferhis = make_history_oper(raise_on_query=True)
|
||||||
|
|
||||||
|
result = TransferChain._TransferChain__is_overwrite_declined(
|
||||||
|
task, transferinfo, transferhis
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is False
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# __default_callback 失败分支
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
def _make_failed_task():
|
||||||
|
"""构造一个失败回调测试所需的最小整理任务。"""
|
||||||
|
task = make_task(1)
|
||||||
|
task.mediainfo = FakeMedia()
|
||||||
|
# __default_callback 失败通知路径需要读取海报图,FakeMedia 本身不提供该接口
|
||||||
|
task.mediainfo.get_message_image = lambda: "poster.jpg"
|
||||||
|
task.background = False
|
||||||
|
task.manual = True
|
||||||
|
return task
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_callback_skips_history_and_notification_when_overwrite_declined():
|
||||||
|
"""
|
||||||
|
同源已有成功记录时,覆盖裁决不覆盖不应写失败历史、不应发送失败事件与通知。
|
||||||
|
"""
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
chain.eventmanager = MagicMock()
|
||||||
|
chain.post_message = MagicMock()
|
||||||
|
|
||||||
|
task = _make_failed_task()
|
||||||
|
success_history = SimpleNamespace(id=99, status=True)
|
||||||
|
add_fail_calls = []
|
||||||
|
transfer_history_oper = make_history_oper(
|
||||||
|
history=success_history, add_fail_calls=add_fail_calls
|
||||||
|
)
|
||||||
|
|
||||||
|
transferinfo = TransferInfo(
|
||||||
|
success=False,
|
||||||
|
fileitem=task.fileitem,
|
||||||
|
message="目标已存在,按覆盖策略跳过覆盖",
|
||||||
|
transfer_type="copy",
|
||||||
|
overwrite_skipped=True,
|
||||||
|
need_notify=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
|
return_value=transfer_history_oper,
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
|
||||||
|
):
|
||||||
|
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
|
||||||
|
|
||||||
|
assert state is False
|
||||||
|
assert errmsg == transferinfo.message
|
||||||
|
assert add_fail_calls == []
|
||||||
|
assert chain.post_message.call_count == 0
|
||||||
|
transfer_failed_events = [
|
||||||
|
call
|
||||||
|
for call in chain.eventmanager.send_event.call_args_list
|
||||||
|
if call.args[0] == EventType.TransferFailed
|
||||||
|
]
|
||||||
|
assert transfer_failed_events == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_callback_keeps_original_failure_semantics_without_success_history():
|
||||||
|
"""
|
||||||
|
没有已有成功记录时(即使 overwrite_skipped 为真),仍应按原有语义写失败历史并通知。
|
||||||
|
"""
|
||||||
|
chain = make_transfer_chain()
|
||||||
|
chain.eventmanager = MagicMock()
|
||||||
|
chain.post_message = MagicMock()
|
||||||
|
|
||||||
|
task = _make_failed_task()
|
||||||
|
add_fail_calls = []
|
||||||
|
transfer_history_oper = make_history_oper(
|
||||||
|
history=None, add_fail_calls=add_fail_calls
|
||||||
|
)
|
||||||
|
|
||||||
|
transferinfo = TransferInfo(
|
||||||
|
success=False,
|
||||||
|
fileitem=task.fileitem,
|
||||||
|
message="目标已存在,按覆盖策略跳过覆盖",
|
||||||
|
transfer_type="copy",
|
||||||
|
overwrite_skipped=True,
|
||||||
|
need_notify=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch(
|
||||||
|
"app.chain.transfer.TransferHistoryOper",
|
||||||
|
return_value=transfer_history_oper,
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_ENABLE", False
|
||||||
|
), patch(
|
||||||
|
"app.chain.transfer.settings.AI_AGENT_RETRY_TRANSFER", False
|
||||||
|
):
|
||||||
|
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
|
||||||
|
|
||||||
|
assert state is False
|
||||||
|
assert errmsg == transferinfo.message
|
||||||
|
assert len(add_fail_calls) == 1
|
||||||
|
assert chain.post_message.call_count == 1
|
||||||
|
transfer_failed_events = [
|
||||||
|
call
|
||||||
|
for call in chain.eventmanager.send_event.call_args_list
|
||||||
|
if call.args[0] == EventType.TransferFailed
|
||||||
|
]
|
||||||
|
assert len(transfer_failed_events) == 1
|
||||||
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
from app.modules.filemanager.storages import StorageBase
|
||||||
from app.modules.filemanager.storages.alipan import AliPan
|
from app.modules.filemanager.storages.alipan import AliPan
|
||||||
from app.modules.filemanager.storages.local import LocalStorage
|
from app.modules.filemanager.storages.local import LocalStorage
|
||||||
from app.modules.filemanager.storages.rclone import Rclone
|
from app.modules.filemanager.storages.rclone import Rclone
|
||||||
@@ -87,13 +88,17 @@ def test_local_strict_raises_on_stat_error(tmp_path, monkeypatch):
|
|||||||
"""
|
"""
|
||||||
target = tmp_path / "movie.mkv"
|
target = tmp_path / "movie.mkv"
|
||||||
|
|
||||||
def raise_stat_error(self, *args, **kwargs):
|
def raise_stat_error(*_args, **_kwargs):
|
||||||
"""
|
"""
|
||||||
模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE。
|
模拟 CloudDrive FUSE 挂载返回 ENOTRECOVERABLE。
|
||||||
"""
|
"""
|
||||||
raise OSError(131, "State not recoverable")
|
raise OSError(131, "State not recoverable")
|
||||||
|
|
||||||
monkeypatch.setattr(Path, "stat", raise_stat_error)
|
# 文件系统边界已下移到代理子进程,patch Path.stat 影响不到那里,
|
||||||
|
# 必须在代理这一层注入故障
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"app.modules.filemanager.storages.local.fsproxy.stat", raise_stat_error
|
||||||
|
)
|
||||||
|
|
||||||
with pytest.raises(StorageQueryError):
|
with pytest.raises(StorageQueryError):
|
||||||
_local().get_item_strict(target)
|
_local().get_item_strict(target)
|
||||||
@@ -229,12 +234,14 @@ def test_alipan_strict_returns_item(monkeypatch):
|
|||||||
assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM"
|
assert storage.get_item_strict(Path("/movie.mkv")) == "ITEM"
|
||||||
|
|
||||||
|
|
||||||
def test_storage_base_strict_defaults_to_get_item():
|
def test_storage_base_strict_fails_conservatively_without_override():
|
||||||
"""
|
"""
|
||||||
未覆写的存储沿用 get_item 判定,行为不变。
|
未覆写严格查询的存储必须保守失败:沿用 get_item 会把「查询失败」当成
|
||||||
|
「目标不存在」,让 overwrite_mode=size 的覆盖保护被绕过。
|
||||||
"""
|
"""
|
||||||
storage = object.__new__(Rclone)
|
storage = object.__new__(Rclone)
|
||||||
storage.get_item = MagicMock(return_value=None)
|
storage.get_item = MagicMock(return_value=None)
|
||||||
|
|
||||||
assert storage.get_item_strict(Path("/movie.mkv")) is None
|
with pytest.raises(StorageQueryError):
|
||||||
storage.get_item.assert_called_once()
|
StorageBase.get_item_strict(storage, Path("/movie.mkv"))
|
||||||
|
storage.get_item.assert_not_called()
|
||||||
|
|||||||
@@ -0,0 +1,232 @@
|
|||||||
|
"""
|
||||||
|
整理队列持久化与重启回放测试。
|
||||||
|
|
||||||
|
整理队列是纯内存的 queue.Queue:挂载挂死后的人工重启、版本升级、OOM、宿主
|
||||||
|
重启都会让队列连同「这些文件还没整理」这个事实一起蒸发,而已稳定落地的文件
|
||||||
|
不会再产生任何监控事件,也不会有新的补偿扫描起点——结果就是永久漏件。
|
||||||
|
|
||||||
|
这些测试固定三项不变量:入队即落盘登记、终态即注销、重启能回放。
|
||||||
|
"""
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
|
from app.chain.transfer import TransferChain
|
||||||
|
from app.schemas import FileItem, TransferTask
|
||||||
|
|
||||||
|
|
||||||
|
def _build_chain(pendingoper) -> TransferChain:
|
||||||
|
"""
|
||||||
|
构造绕过单例初始化的 TransferChain 骨架。
|
||||||
|
:param pendingoper: 待整理登记管理替身
|
||||||
|
:return: TransferChain 骨架
|
||||||
|
"""
|
||||||
|
chain = object.__new__(TransferChain)
|
||||||
|
chain._pendingoper = pendingoper
|
||||||
|
return chain
|
||||||
|
|
||||||
|
|
||||||
|
def _task(path: str, storage: str = "local") -> TransferTask:
|
||||||
|
"""
|
||||||
|
构造测试用整理任务。
|
||||||
|
:param path: 源文件路径
|
||||||
|
:param storage: 存储
|
||||||
|
:return: 整理任务
|
||||||
|
"""
|
||||||
|
file_path = Path(path)
|
||||||
|
return TransferTask(fileitem=FileItem(
|
||||||
|
storage=storage,
|
||||||
|
path=path,
|
||||||
|
type="file",
|
||||||
|
name=file_path.name,
|
||||||
|
basename=file_path.stem,
|
||||||
|
extension=file_path.suffix[1:],
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def test_register_pending_records_storage_and_path():
|
||||||
|
"""
|
||||||
|
入队时必须落盘登记「存储 + 源路径」这一最小事实。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
chain._TransferChain__register_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
|
||||||
|
|
||||||
|
pendingoper.register.assert_called_once_with(
|
||||||
|
storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_register_pending_failure_does_not_break_enqueue():
|
||||||
|
"""
|
||||||
|
落盘登记只是重启后的补救手段,登记失败绝不能阻断正常整理。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.register.side_effect = RuntimeError("db locked")
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
# 不抛异常即为通过
|
||||||
|
chain._TransferChain__register_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_discard_pending_on_terminal_state():
|
||||||
|
"""
|
||||||
|
整理到达终态后必须注销登记,否则每次重启都会重复回放。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
chain._TransferChain__discard_pending(_task("/mnt/cd2/downloads/Movie.2024.mkv"))
|
||||||
|
|
||||||
|
pendingoper.discard.assert_called_once_with(
|
||||||
|
storage="local", src_path="/mnt/cd2/downloads/Movie.2024.mkv"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_resends_pending_files_to_transfer(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
重启回放:登记过的文件要重新送入整理链,恢复被内存队列蒸发的任务。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "Movie.2024.mkv"
|
||||||
|
media.write_bytes(b"x" * 10)
|
||||||
|
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.return_value = [("local", str(media))]
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
transferred = []
|
||||||
|
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
assert len(transferred) == 1
|
||||||
|
item = transferred[0]
|
||||||
|
assert item.path == media.as_posix()
|
||||||
|
assert item.storage == "local"
|
||||||
|
assert item.type == "file"
|
||||||
|
# 回放时重新读取当前大小,不依赖登记时的陈旧信息
|
||||||
|
assert item.size == 10
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_discards_vanished_files(tmp_path):
|
||||||
|
"""
|
||||||
|
源文件已消失的登记要注销,否则每次启动都会重复回放一个不存在的文件。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
missing = tmp_path / "gone.mkv"
|
||||||
|
pendingoper.list_all.return_value = [("local", str(missing))]
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
chain.do_transfer = MagicMock()
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
pendingoper.discard.assert_called_once_with(storage="local", src_path=str(missing))
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_keeps_registration_when_mount_unreadable(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
挂载未就绪时读取失败属于暂时性故障,登记必须保留,等下次启动或人工整理。
|
||||||
|
|
||||||
|
这与「文件已消失」必须区别对待:把挂载抖动误判成文件消失就等于主动丢件。
|
||||||
|
"""
|
||||||
|
media = tmp_path / "Movie.2024.mkv"
|
||||||
|
media.write_bytes(b"x")
|
||||||
|
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.return_value = [("local", str(media))]
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
chain.do_transfer = MagicMock()
|
||||||
|
|
||||||
|
def unreadable(self, *_args, **_kwargs):
|
||||||
|
"""
|
||||||
|
模拟挂载未就绪时的 stat 失败。
|
||||||
|
"""
|
||||||
|
raise OSError(107, "Transport endpoint is not connected")
|
||||||
|
|
||||||
|
monkeypatch.setattr(Path, "stat", unreadable)
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
pendingoper.discard.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_restores_bluray_directory_type(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
蓝光原盘登记时保留尾部斜杠,回放必须还原成目录类型,否则会被当成单文件整理。
|
||||||
|
"""
|
||||||
|
bluray = tmp_path / "Movie.2024.BluRay"
|
||||||
|
bluray.mkdir()
|
||||||
|
src_path = f"{bluray.as_posix()}/"
|
||||||
|
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.return_value = [("local", src_path)]
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
transferred = []
|
||||||
|
monkeypatch.setattr(chain, "do_transfer", lambda **kw: transferred.append(kw["fileitem"]))
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
assert len(transferred) == 1
|
||||||
|
assert transferred[0].type == "dir"
|
||||||
|
assert transferred[0].path == src_path
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_is_noop_without_registrations():
|
||||||
|
"""
|
||||||
|
没有登记时回放不应触碰整理链。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.return_value = []
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
chain.do_transfer = MagicMock()
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_survives_db_failure():
|
||||||
|
"""
|
||||||
|
读取登记失败不能让启动流程报错。
|
||||||
|
"""
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.side_effect = RuntimeError("db gone")
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
chain.do_transfer = MagicMock()
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
chain.do_transfer.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_replay_continues_after_single_file_failure(tmp_path, monkeypatch):
|
||||||
|
"""
|
||||||
|
单个文件回放失败不能中断整批回放,否则一个坏文件会拖住所有漏件的恢复。
|
||||||
|
"""
|
||||||
|
first = tmp_path / "A.mkv"
|
||||||
|
second = tmp_path / "B.mkv"
|
||||||
|
for item in (first, second):
|
||||||
|
item.write_bytes(b"x")
|
||||||
|
|
||||||
|
pendingoper = MagicMock()
|
||||||
|
pendingoper.list_all.return_value = [("local", str(first)), ("local", str(second))]
|
||||||
|
chain = _build_chain(pendingoper)
|
||||||
|
|
||||||
|
handled = []
|
||||||
|
|
||||||
|
def flaky(**kw):
|
||||||
|
"""
|
||||||
|
第一个文件整理抛异常,第二个正常。
|
||||||
|
"""
|
||||||
|
if kw["fileitem"].name == "A.mkv":
|
||||||
|
raise RuntimeError("boom")
|
||||||
|
handled.append(kw["fileitem"].name)
|
||||||
|
|
||||||
|
monkeypatch.setattr(chain, "do_transfer", flaky)
|
||||||
|
|
||||||
|
chain._TransferChain__replay_pending()
|
||||||
|
|
||||||
|
assert handled == ["B.mkv"]
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
"""
|
||||||
|
整理队列批次计数口径测试。
|
||||||
|
|
||||||
|
背景:作业视图中的任务完成后仅标记终态,作业要等关联任务全部终态才整体移除,
|
||||||
|
因此追更/分批场景下已完成任务会跨批次残留。批次开始日志与进度分母若用全量
|
||||||
|
total(),会把历史任务计入「当前共 N 个文件」(如实际只处理 2 个却显示 8 个),
|
||||||
|
且进度百分比永远走不满。批次统计必须只数未终态任务。
|
||||||
|
"""
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from app.chain.transfer import JobManager
|
||||||
|
|
||||||
|
from tests.test_transfer_job_manager import make_task
|
||||||
|
|
||||||
|
|
||||||
|
class TransferQueueCountTest(unittest.TestCase):
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _build_jobview_with_stale_tasks() -> JobManager:
|
||||||
|
"""
|
||||||
|
构造用户实测的 8→2 场景:同一作业 8 个任务,6 个已完成(作业因
|
||||||
|
仍有未终态任务不会被移除),2 个等待处理。
|
||||||
|
"""
|
||||||
|
jobview = JobManager()
|
||||||
|
completed = [make_task(episode) for episode in range(1, 7)]
|
||||||
|
waiting = [make_task(episode) for episode in range(7, 9)]
|
||||||
|
for task in completed + waiting:
|
||||||
|
assert jobview.add_task(task)
|
||||||
|
for task in completed:
|
||||||
|
jobview.finish_task(task)
|
||||||
|
# 作业尚有未终态任务,不会被移除,已完成任务随作业残留
|
||||||
|
jobview.try_remove_job(task)
|
||||||
|
return jobview
|
||||||
|
|
||||||
|
def test_total_still_counts_terminal_tasks(self):
|
||||||
|
"""total() 保持全量语义(供作业视图展示),包含已完成任务。"""
|
||||||
|
jobview = self._build_jobview_with_stale_tasks()
|
||||||
|
self.assertEqual(jobview.total(), 8)
|
||||||
|
|
||||||
|
def test_pending_total_excludes_terminal_tasks(self):
|
||||||
|
"""pending_total() 只数未终态任务,不受跨批次残留影响。"""
|
||||||
|
jobview = self._build_jobview_with_stale_tasks()
|
||||||
|
self.assertEqual(jobview.pending_total(), 2)
|
||||||
|
|
||||||
|
def test_pending_total_excludes_failed_tasks(self):
|
||||||
|
"""失败任务同为终态,不应计入待处理数。"""
|
||||||
|
jobview = JobManager()
|
||||||
|
failed_task = make_task(1)
|
||||||
|
waiting_task = make_task(2)
|
||||||
|
assert jobview.add_task(failed_task)
|
||||||
|
assert jobview.add_task(waiting_task)
|
||||||
|
jobview.fail_task(failed_task)
|
||||||
|
self.assertEqual(jobview.pending_total(), 1)
|
||||||
|
|
||||||
|
def test_pending_total_counts_running_tasks(self):
|
||||||
|
"""运行中的任务属于本批,必须计入。"""
|
||||||
|
jobview = JobManager()
|
||||||
|
running_task = make_task(1)
|
||||||
|
assert jobview.add_task(running_task)
|
||||||
|
jobview.running_task(running_task)
|
||||||
|
self.assertEqual(jobview.pending_total(), 1)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user