"""
挂载级(block 型)故障下的监控自愈测试。

背景：CloudDrive2/115 等 FUSE 挂载会进入「请求不返回错误、永不返回」的挂死
状态。此时监控自愈体系的检测环节仍然有效，但恢复环节（重建监控线程、重试
整理队列）本身要访问挂载，一旦内联执行就会把全局自愈的单点——健康检查——
冻死在它自己要修复的挂载上，随后停滞检测、告警、重试驱动全部静默失效。

crash 型（挂载抛 Transport endpoint is not connected）的自愈由
test_monitor_resilience.py 覆盖；这些测试固定 block 型的两项不变量：
「看门狗零挂载访问」与「挂载级故障隔离/探测/恢复」。
"""
import threading
import time
from pathlib import Path
from threading import Lock, Thread
from unittest.mock import MagicMock

import pytest

from app.monitor import LocalDirectoryWatcher, Monitor
from app.monitor.recovery import RecoveryExecutor, RecoveryState, probe_path


def _build_monitor(monkeypatch, put_recorder=None):
    """
    构造测试用 Monitor 骨架，绕过单例初始化。
    :param monkeypatch: pytest monkeypatch
    :param put_recorder: 消息推送记录器
    :return: Monitor 骨架
    """
    put_recorder = put_recorder or MagicMock()
    monkeypatch.setattr("app.monitor.monitor.MessageHelper", MagicMock(return_value=put_recorder))
    monitor = object.__new__(Monitor)
    monitor._dispatcher = MagicMock()
    monitor._watchers = []
    monitor._watcher_lock = Lock()
    monitor._pending_locals = []
    monitor._alerted_paths = {}
    monitor._restart_marks = {}
    monitor._stable_cycles = {}
    monitor._isolated = {}
    monitor._pending_rebuild = {}
    monitor._recovery = RecoveryExecutor()
    return monitor


def _fake_watcher(mon_path, alive=True, stalled=False, restart_count=0):
    """
    构造测试用监控线程替身。
    :param mon_path: 监控目录
    :param alive: 线程是否存活
    :param stalled: 是否静默失效
    :param restart_count: 自动重启次数
    :return: 监控线程替身
    """
    watcher = MagicMock()
    watcher.watch_path = mon_path
    watcher.is_alive.return_value = alive
    watcher.is_stalled.return_value = stalled
    watcher.restart_count = restart_count
    watcher.last_activity_time = time.time()
    return watcher


def _run_watchdog(monitor, timeout=10.0):
    """
    在独立线程里跑一次健康检查，返回它是否在限定时间内结束。
    :param monitor: Monitor 骨架
    :param timeout: 最长等待秒数
    :return: 健康检查是否已返回
    """
    done = threading.Event()

    def runner():
        """
        执行一次健康检查并标记结束。
        """
        try:
            monitor.watchdog()
        finally:
            done.set()

    thread = Thread(target=runner, daemon=True, name="test-watchdog")
    thread.start()
    finished = done.wait(timeout=timeout)
    if finished:
        thread.join(timeout=timeout)
    return finished and not thread.is_alive()


def _release_and_join(release, threads, timeout=10.0):
    """
    释放测试注入的阻塞，并等待受控工作线程结束。

    真实故障下的恢复线程允许随进程退出，但测试主动解除阻塞后必须完成回收，
    避免仍在执行应用代码的守护线程与解释器关闭过程并发。
    :param release: 控制阻塞动作的事件
    :param threads: 需要等待的线程集合
    :param timeout: 每个线程的最长等待秒数
    """
    release.set()
    for thread in threads:
        thread.join(timeout=timeout)
        assert not thread.is_alive(), f"受控线程未能结束: {thread.name}"


def _stop_monitor_threads(monitor, timeout=10.0):
    """
    停止测试期间重建的目录监控，并等待其派生任务结束。

    目录 watcher 会进入 watchfiles 原生扩展，必须在解释器关闭前退出；补偿扫描
    由重建流程派生，也要在测试边界内完成。
    :param monitor: Monitor 骨架
    :param timeout: 每个线程的最长等待秒数
    """
    for watcher in monitor._watchers:
        if isinstance(watcher, LocalDirectoryWatcher):
            watcher.stop()
            watcher.join(timeout=timeout)
            assert not watcher.is_alive(), f"目录监控线程未能结束: {watcher.watch_path}"
    for thread in threading.enumerate():
        if thread.name.startswith("MoviePilot-MonitorCompensation-"):
            thread.join(timeout=timeout)
            assert not thread.is_alive(), f"补偿扫描线程未能结束: {thread.name}"


# --------------------------------------------------------------------------- #
# P0：看门狗线程零挂载访问
# --------------------------------------------------------------------------- #

def test_watchdog_returns_while_rebuild_blocks_forever(tmp_path, monkeypatch):
    """
    根因回归：重建动作卡在死挂载上永不返回时，健康检查本身必须在有限时间内
    返回。事故中看门狗内联执行重建，冻死后 13 个目录的停滞检测与告警全部失效。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
    monitor = _build_monitor(monkeypatch)
    watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
    monitor._watchers = [watcher]

    entered = threading.Event()
    release = threading.Event()

    def blocking_rebuild(_target):
        """
        模拟 block 型挂载：进入后永不返回。
        """
        entered.set()
        release.wait()

    setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)

    try:
        assert _run_watchdog(monitor), "看门狗被重建动作冻死，未能在限定时间内返回"
        assert entered.is_set(), "重建动作没有被真正发起"
    finally:
        _release_and_join(release, monitor._recovery._running.values())


def test_watchdog_returns_while_pending_retry_blocks_forever(tmp_path, monkeypatch):
    """
    重试驱动（整理重试队列的 stat）同样会卡在死挂载上，也必须移出看门狗线程。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
    monitor = _build_monitor(monkeypatch)
    release = threading.Event()
    monitor._dispatcher.retry_pending.side_effect = lambda: release.wait()

    try:
        assert _run_watchdog(monitor), "看门狗被整理重试驱动冻死，未能在限定时间内返回"
    finally:
        _release_and_join(release, monitor._recovery._running.values())


def test_watchdog_returns_while_local_retry_blocks_forever(tmp_path, monkeypatch):
    """
    启动失败目录的重试会走 decide_monitor_mode 的 os.walk 与 watcher.start()
    的 exists()，同样触碰挂载，必须移出看门狗线程。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
    monitor = _build_monitor(monkeypatch)
    monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
    release = threading.Event()

    def blocking_start(**_kwargs):
        """
        模拟启动重试卡在死挂载的目录遍历上。
        """
        release.wait()
        return False

    setattr(monitor, "_Monitor__start_local_monitor", blocking_start)

    try:
        assert _run_watchdog(monitor), "看门狗被本地监控重试冻死，未能在限定时间内返回"
    finally:
        _release_and_join(release, monitor._recovery._running.values())


def test_check_watchers_performs_no_filesystem_access(tmp_path, monkeypatch):
    """
    检测与判定必须是纯内存运算：只读线程存活标志、心跳与重启计数，
    不做任何文件系统访问，否则检测环节本身也会被 block 型故障拖死。
    """
    monitor = _build_monitor(monkeypatch)
    watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
    monitor._watchers = [watcher]

    def forbidden(*_args, **_kwargs):
        """
        任何真实的文件系统调用都应视为检测环节的缺陷。
        """
        raise AssertionError("检测环节不允许访问文件系统")

    for name in ("stat", "exists", "is_dir", "iterdir", "rglob"):
        monkeypatch.setattr(Path, name, forbidden, raising=False)

    broken = monitor._Monitor__check_watchers()

    assert broken == [watcher]


def test_watchdog_survives_blocking_exists_in_real_rebuild(tmp_path, monkeypatch):
    """
    端到端回归：不替换任何恢复逻辑，只让 Path.exists() 永不返回（等价于对 FUSE
    守护进程 kill -STOP 后挂载的表现），走真实的
    __rebuild_watcher -> LocalDirectoryWatcher.start() -> exists() 调用链。

    事故当天冻结的就是这一行（watcher.py 的 start() 入口校验）。修复后看门狗
    必须照常返回，并把该目录转入隔离。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.5)
    monitor = _build_monitor(monkeypatch)

    # 用真实 watcher：只把底层线程换成替身，让它被判定为「存活但静默失效」
    watcher = LocalDirectoryWatcher(tmp_path, callback=MagicMock(), force_polling=True)
    watcher._thread = MagicMock()
    watcher._thread.is_alive.return_value = True
    watcher._mark_activity()
    watcher._last_activity -= LocalDirectoryWatcher.STALL_TIMEOUT + 1
    monitor._watchers = [watcher]

    release = threading.Event()

    def blocking_stat(path, *_args, **_kwargs):
        """
        模拟 block 型挂载：监控目录的属性读取永不返回。
        入口校验已改走子进程代理，因此在代理这一层注入阻塞。
        """
        if Path(path) == tmp_path:
            release.wait()
        return {"size": 0, "mtime": 0.0, "is_dir": True, "is_file": False}

    monkeypatch.setattr(
        "app.modules.filemanager.fsproxy.fsproxy.stat", blocking_stat
    )
    monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)

    try:
        assert _run_watchdog(monitor), "看门狗冻死在真实的重建调用链上"
        assert str(tmp_path) in monitor._isolated, "重建无响应后目录没有转入隔离"
    finally:
        _release_and_join(release, monitor._recovery._running.values())
        _stop_monitor_threads(monitor)


# --------------------------------------------------------------------------- #
# P0.5：挂载级故障隔离
# --------------------------------------------------------------------------- #

def test_rebuild_timeout_isolates_directory(tmp_path, monkeypatch):
    """
    重建在挂载上超时未返回 = 挂载级故障：该目录转入隔离，后续周期不再对它
    发起任何新的挂载访问，避免每 60 秒泄漏一个冻死的线程。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 0.3)
    monitor = _build_monitor(monkeypatch)
    watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
    monitor._watchers = [watcher]

    calls = []
    release = threading.Event()

    def blocking_rebuild(target):
        """
        模拟 block 型挂载上的重建：进入后永不返回。
        """
        calls.append(target)
        release.wait()

    setattr(monitor, "_Monitor__rebuild_watcher", blocking_rebuild)
    monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)

    try:
        assert _run_watchdog(monitor)
        assert str(tmp_path) in monitor._isolated, "重建超时后目录没有转入隔离"

        # 第二个周期：隔离中的目录不得再被提交重建
        assert _run_watchdog(monitor)
        assert len(calls) == 1, "隔离中的目录仍在被反复重建，会持续泄漏冻死的线程"
    finally:
        _release_and_join(release, monitor._recovery._running.values())


def test_isolated_directory_recovers_after_probe_succeeds(tmp_path, monkeypatch):
    """
    隔离期间用可放弃的子进程探测挂载；探测通过即解除隔离、重建监控，
    并复用既有的补偿扫描补回停摆期间落地的文件。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
    monitor = _build_monitor(monkeypatch)
    watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
    monitor._watchers = [watcher]
    monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 3}}

    rebuilt = []
    setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
    monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: True)

    assert _run_watchdog(monitor)

    assert str(tmp_path) not in monitor._isolated, "探测通过后没有解除隔离"
    # 不得在探测线程里内联重建：重建会触碰挂载，若此时再次挂死，全局 PROBE_KEY
    # 将永久 BUSY，其余隔离目录连探测机会都没有
    assert rebuilt == [], "探测线程内联执行了重建，会饿死其他隔离目录"
    assert str(tmp_path) in monitor._pending_rebuild, "没有登记待重建"

    # 下一周期应把它送去按目录隔离的重建路径
    assert _run_watchdog(monitor)
    assert rebuilt == [watcher], "下一周期没有重建该监控"
    assert not monitor._pending_rebuild, "重建后未清空待重建登记"


def test_isolated_directory_stays_isolated_while_probe_fails(tmp_path, monkeypatch):
    """
    探测未通过说明挂载仍未恢复应答，必须保持隔离并累计失败次数。
    """
    monkeypatch.setattr(Monitor, "RECOVERY_TIMEOUT", 5.0)
    monitor = _build_monitor(monkeypatch)
    watcher = _fake_watcher(tmp_path, alive=True, stalled=True)
    monitor._watchers = [watcher]
    monitor._isolated = {str(tmp_path): {"watcher": watcher, "since": time.time(), "failures": 0}}

    rebuilt = []
    setattr(monitor, "_Monitor__rebuild_watcher", rebuilt.append)
    monkeypatch.setattr("app.monitor.monitor.probe_path", lambda *_a, **_kw: False)

    assert _run_watchdog(monitor)

    assert monitor._isolated[str(tmp_path)]["failures"] == 1
    assert rebuilt == [], "挂载尚未恢复就重建监控，只会再冻死一个线程"


def test_pending_locals_skip_isolated_directories(tmp_path, monkeypatch):
    """
    隔离中的目录不能再走启动重试路径，否则又会在同一个挂载上冻死。
    """
    monitor = _build_monitor(monkeypatch)
    monitor._pending_locals = [{"mon_path": tmp_path, "monitor_mode": "compatibility"}]
    monitor._isolated = {str(tmp_path): {"watcher": _fake_watcher(tmp_path), "since": time.time(),
                                         "failures": 0}}
    start = MagicMock(return_value=False)
    setattr(monitor, "_Monitor__start_local_monitor", start)

    monitor._Monitor__retry_pending_locals()

    start.assert_not_called()


def test_isolation_alert_is_pushed_after_fault_alert(tmp_path, monkeypatch):
    """
    故障→隔离是状态升级，必须再推一条告警：沿用「同目录只告警一次」会让
    用户完全看不到「监控已暂停访问、正在等待挂载恢复」这个关键状态变化。
    """
    put_recorder = MagicMock()
    monitor = _build_monitor(monkeypatch, put_recorder)

    monitor._Monitor__send_alert(tmp_path, "目录监控异常")
    monitor._Monitor__send_alert(tmp_path, "目录监控异常")
    assert put_recorder.put.call_count == 1

    monitor._Monitor__send_alert(tmp_path, "挂载无响应，已隔离", stage="isolated")
    assert put_recorder.put.call_count == 2

    monitor._Monitor__clear_alert(tmp_path, "已恢复")
    assert put_recorder.put.call_count == 3
    assert str(tmp_path) not in monitor._alerted_paths


# --------------------------------------------------------------------------- #
# 整理分发器：一个文件卡死不得锁死整条整理链
# --------------------------------------------------------------------------- #

def test_stuck_transfer_does_not_block_other_files(monkeypatch):
    """
    根因回归：整理的规划阶段（do_transfer 里的 get_parent_item / list_files）会
    访问挂载，在 block 型故障下永不返回。若分发器的互斥锁包住这段调用，这把锁
    就会被永久持有，把 13 个 watcher 的事件派发、补偿扫描和重试队列一起锁死
    ——监控层即使自愈成功也送不进任何文件，漏件永远补不回来。

    锁只应保护 TTL 去重的 check-and-set。
    """
    from app.monitor.dispatcher import TransferDispatcher

    dispatcher = TransferDispatcher(all_exts=[".mkv"], cache={})
    monkeypatch.setattr(dispatcher, "_should_skip_by_history", MagicMock(return_value=False))

    release = threading.Event()
    stuck_entered = threading.Event()
    transferred = []

    class FakeChain:
        """
        整理链替身：指定文件的整理进入后永不返回。
        """

        @staticmethod
        def do_transfer(fileitem, **_kwargs):
            """
            模拟规划阶段卡在死挂载的 list_files 上。
            """
            transferred.append(fileitem.path)
            if fileitem.path.endswith("stuck.mkv"):
                stuck_entered.set()
                release.wait()

    monkeypatch.setattr("app.monitor.dispatcher.TransferChain", FakeChain)

    def feed(name):
        """
        向分发器送入一个文件。
        """
        dispatcher.handle_file(storage="local", event_path=Path(f"/mnt/cd2/{name}"), file_size=1)

    stuck_thread = Thread(target=feed, args=("stuck.mkv",), daemon=True)
    stuck_thread.start()
    assert stuck_entered.wait(timeout=5), "卡死的整理没有真正进入"

    done = threading.Event()
    other_thread = Thread(target=lambda: (feed("other.mkv"), done.set()), daemon=True)
    other_thread.start()

    try:
        assert done.wait(timeout=5), "一个文件卡在死挂载上就锁死了整条整理链"
        assert "/mnt/cd2/other.mkv" in transferred
    finally:
        _release_and_join(release, (stuck_thread, other_thread))


# --------------------------------------------------------------------------- #
# 恢复执行器与挂载探测
# --------------------------------------------------------------------------- #

def test_recovery_executor_reports_timeout_without_blocking():
    """
    永不返回的动作只应消耗一次 timeout，执行器必须放弃它并如实报告。
    """
    executor = RecoveryExecutor()
    release = threading.Event()

    started = time.monotonic()
    results = executor.run({"stuck": release.wait}, timeout=0.3)
    elapsed = time.monotonic() - started

    try:
        assert results == {"stuck": RecoveryState.TIMEOUT}
        assert elapsed < 3.0, "执行器没有在超时后放弃冻死的动作"
    finally:
        _release_and_join(release, executor._running.values())


def test_recovery_executor_skips_key_with_running_task():
    """
    同一个 key 的上一个动作还冻着时，不能再提交新线程——否则每个健康检查
    周期都会在同一个死挂载上泄漏一个线程。
    """
    executor = RecoveryExecutor()
    release = threading.Event()
    calls = []

    def stuck():
        """
        模拟永不返回的恢复动作。
        """
        calls.append(1)
        release.wait()

    try:
        assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.TIMEOUT}
        assert executor.run({"k": stuck}, timeout=0.2) == {"k": RecoveryState.BUSY}
        assert len(calls) == 1
    finally:
        _release_and_join(release, executor._running.values())


def test_recovery_executor_runs_actions_concurrently():
    """
    一个周期内多个目录的恢复动作必须并发执行，否则 13 个目录串行等待会把
    健康检查拖过下一个周期。
    """
    executor = RecoveryExecutor()
    release = threading.Event()

    started = time.monotonic()
    results = executor.run({f"k{i}": release.wait for i in range(5)}, timeout=0.4)
    elapsed = time.monotonic() - started

    try:
        assert set(results.values()) == {RecoveryState.TIMEOUT}
        assert elapsed < 1.5, "恢复动作是串行等待的，总耗时随目录数增长"
    finally:
        _release_and_join(release, executor._running.values())


def test_recovery_executor_reports_completion_and_swallows_errors():
    """
    正常完成的动作报告 COMPLETED；动作内部抛异常不能让执行器崩溃，
    否则一个目录的失败会连累整批恢复。
    """
    executor = RecoveryExecutor()
    done = []

    def boom():
        """
        模拟恢复动作内部异常。
        """
        raise RuntimeError("rebuild failed")

    results = executor.run({"ok": lambda: done.append(1), "bad": boom}, timeout=5)

    assert results == {"ok": RecoveryState.COMPLETED, "bad": RecoveryState.COMPLETED}
    assert done == [1]


def test_probe_path_succeeds_on_existing_directory(tmp_path):
    """
    挂载可用时探测应通过。
    """
    assert probe_path(tmp_path, timeout=30) is True


def test_probe_path_fails_on_missing_path(tmp_path):
    """
    路径不存在时探测应失败，而不是抛异常。
    """
    assert probe_path(tmp_path / "does-not-exist", timeout=30) is False


@pytest.mark.skipif(not Path("/bin/sh").exists(), reason="需要 POSIX 环境")
def test_probe_path_is_abandonable_on_timeout(tmp_path, monkeypatch):
    """
    探测的关键性质：卡住时可以被放弃。线程做的 stat 无法回收，只有子进程能在
    超时后被 kill——这正是隔离期间能持续探测而不泄漏资源的前提。
    """
    import app.monitor.recovery as recovery

    # 用一个必定超时的探测脚本替换真实探测逻辑，验证超时路径的可放弃性
    monkeypatch.setattr(recovery, "_PROBE_SCRIPT", "import time; time.sleep(60)")

    started = time.monotonic()
    assert probe_path(tmp_path, timeout=0.5) is False
    assert time.monotonic() - started < 10, "探测超时后没有及时放弃子进程"