fix(qqbot): converge heartbeat lifecycle

This commit is contained in:
jxxghp
2026-08-26 22:40:01 +08:00
parent b1c4150895
commit 50b85235d8
5 changed files with 526 additions and 74 deletions
+130 -35
View File
@@ -39,27 +39,116 @@ def run_gateway(
:param ws_holder: 调用方持有的单元素列表,存放当前 WebSocketApp,供 stop() 时 close 以打断 run_forever :param ws_holder: 调用方持有的单元素列表,存放当前 WebSocketApp,供 stop() 时 close 以打断 run_forever
""" """
last_seq: Optional[int] = None last_seq: Optional[int] = None
heartbeat_interval_ms: Optional[int] = None heartbeat_thread: Optional[threading.Thread] = None
heartbeat_timer: Optional[threading.Timer] = None heartbeat_stop_event: Optional[threading.Event] = None
heartbeat_epoch = 0
heartbeat_state_lock = threading.Lock()
heartbeat_replace_lock = threading.RLock()
def send_heartbeat(): def heartbeat_loop(
nonlocal heartbeat_timer generation_epoch: int,
generation_stop_event: threading.Event,
interval_seconds: float,
ws: websocket.WebSocketApp,
) -> None:
"""在当前连接 generation 内串行发送心跳,停止后不再派生新线程。"""
while not generation_stop_event.wait(interval_seconds):
if stop_event.is_set(): if stop_event.is_set():
return return
with heartbeat_state_lock:
if (
generation_epoch != heartbeat_epoch
or heartbeat_stop_event is not generation_stop_event
):
return
sequence = last_seq
try: try:
if ws_holder and ws_holder[0]: payload = {"op": 1, "d": sequence}
payload = {"op": 1, "d": last_seq} ws.send(json.dumps(payload))
ws_holder[0].send(json.dumps(payload)) logger.debug(
logger.debug(f"[QQ Gateway:{config_name}] Heartbeat sent, seq={last_seq}") f"[QQ Gateway:{config_name}] Heartbeat sent, seq={sequence}"
)
except Exception as err: except Exception as err:
logger.debug(f"[QQ Gateway:{config_name}] Heartbeat error: {err}") logger.debug(f"[QQ Gateway:{config_name}] Heartbeat error: {err}")
if heartbeat_interval_ms and not stop_event.is_set():
heartbeat_timer = threading.Timer(heartbeat_interval_ms / 1000.0, send_heartbeat)
heartbeat_timer.daemon = True
heartbeat_timer.start()
def on_ws_message(_, message): def invalidate_heartbeat_generation() -> tuple[Optional[threading.Thread], int]:
nonlocal last_seq, heartbeat_interval_ms, heartbeat_timer """锁内失效当前 generation,并返回待等待的 owner 与新 epoch。"""
nonlocal heartbeat_epoch
with heartbeat_state_lock:
heartbeat_epoch += 1
current_thread = heartbeat_thread
if heartbeat_stop_event is not None:
heartbeat_stop_event.set()
return current_thread, heartbeat_epoch
def clear_heartbeat_generation(current_thread: Optional[threading.Thread]) -> None:
"""仅在快照仍是当前 owner 且已经终止时清除 generation 引用。"""
nonlocal heartbeat_thread, heartbeat_stop_event
if current_thread is not None and current_thread.is_alive():
return
with heartbeat_state_lock:
if heartbeat_thread is current_thread:
heartbeat_thread = None
heartbeat_stop_event = None
def stop_heartbeat(*, wait: bool) -> None:
"""停止当前心跳;等待阶段不持有状态锁,避免阻塞 close 回调。"""
if wait:
heartbeat_replace_lock.acquire()
try:
current_thread, _ = invalidate_heartbeat_generation()
if (
wait
and current_thread is not None
and current_thread.is_alive()
and current_thread is not threading.current_thread()
):
current_thread.join()
clear_heartbeat_generation(current_thread)
finally:
if wait:
heartbeat_replace_lock.release()
def start_heartbeat(ws: websocket.WebSocketApp, interval_ms: int) -> None:
"""停止旧 generation 后启动一个由 Gateway 聚合的新心跳 owner。"""
nonlocal heartbeat_thread, heartbeat_stop_event
with heartbeat_replace_lock:
current_thread, generation_epoch = invalidate_heartbeat_generation()
if (
current_thread is not None
and current_thread.is_alive()
and current_thread is not threading.current_thread()
):
current_thread.join()
clear_heartbeat_generation(current_thread)
with heartbeat_state_lock:
if (
generation_epoch != heartbeat_epoch
or stop_event.is_set()
or not ws_holder
or ws_holder[0] is not ws
):
return
generation_stop_event = threading.Event()
# 心跳是 Gateway 的常驻子 owner,不占用进程共享 ThreadHelperGateway 退出前统一 join。
generation_thread = threading.Thread(
target=heartbeat_loop,
args=(
generation_epoch,
generation_stop_event,
interval_ms / 1000.0,
ws,
),
daemon=True,
name=f"qq-gateway-heartbeat-{config_name}",
)
heartbeat_stop_event = generation_stop_event
heartbeat_thread = generation_thread
generation_thread.start()
def on_ws_message(ws, message):
"""处理当前 WebSocket 连接收到的 QQ Gateway 消息。"""
nonlocal last_seq
try: try:
payload = json.loads(message) payload = json.loads(message)
except json.JSONDecodeError as err: except json.JSONDecodeError as err:
@@ -72,6 +161,7 @@ def run_gateway(
t = payload.get("t") t = payload.get("t")
if s is not None: if s is not None:
with heartbeat_state_lock:
last_seq = s last_seq = s
logger.debug(f"[QQ Gateway:{config_name}] op={op} t={t}") logger.debug(f"[QQ Gateway:{config_name}] op={op} t={t}")
@@ -89,15 +179,10 @@ def run_gateway(
"shard": [0, 1], "shard": [0, 1],
}, },
} }
ws_holder[0].send(json.dumps(identify)) ws.send(json.dumps(identify))
logger.info(f"[QQ Gateway:{config_name}] Identify sent") logger.info(f"[QQ Gateway:{config_name}] Identify sent")
# 启动心跳 start_heartbeat(ws, heartbeat_interval_ms)
if heartbeat_timer:
heartbeat_timer.cancel()
heartbeat_timer = threading.Timer(heartbeat_interval_ms / 1000.0, send_heartbeat)
heartbeat_timer.daemon = True
heartbeat_timer.start()
elif op == 0: # Dispatch elif op == 0: # Dispatch
if t == "READY": if t == "READY":
@@ -147,21 +232,27 @@ def run_gateway(
elif op == 9: # Invalid Session elif op == 9: # Invalid Session
logger.warning(f"[QQ Gateway:{config_name}] Invalid session") logger.warning(f"[QQ Gateway:{config_name}] Invalid session")
if ws_holder and ws_holder[0]: ws.close()
ws_holder[0].close()
def on_ws_error(_, error): def on_ws_error(_, error):
"""记录当前 WebSocket 连接上报的错误。"""
logger.error(f"[QQ Gateway:{config_name}] WebSocket error: {error}") logger.error(f"[QQ Gateway:{config_name}] WebSocket error: {error}")
def on_ws_close(_, close_status_code, close_msg): def on_ws_close(ws, close_status_code, close_msg):
"""失效当前连接及其心跳 generation,但不在回调线程等待。"""
logger.info(f"[QQ Gateway:{config_name}] WebSocket closed: {close_status_code} {close_msg}") logger.info(f"[QQ Gateway:{config_name}] WebSocket closed: {close_status_code} {close_msg}")
if heartbeat_timer: # close 回调可能由 stop() 调用线程触发,只发信号,统一由 Gateway 线程等待终态。
heartbeat_timer.cancel() with heartbeat_state_lock:
is_current_connection = bool(ws_holder and ws_holder[0] is ws)
if is_current_connection:
ws_holder.clear() ws_holder.clear()
if is_current_connection:
stop_heartbeat(wait=False)
reconnect_delays = [1, 2, 5, 10, 30, 60] reconnect_delays = [1, 2, 5, 10, 30, 60]
attempt = 0 attempt = 0
try:
while not stop_event.is_set(): while not stop_event.is_set():
try: try:
token = get_token_fn(app_id, app_secret) token = get_token_fn(app_id, app_secret)
@@ -174,32 +265,36 @@ def run_gateway(
on_error=on_ws_error, on_error=on_ws_error,
on_close=on_ws_close, on_close=on_ws_close,
) )
with heartbeat_state_lock:
ws_holder.clear() ws_holder.clear()
ws_holder.append(ws) ws_holder.append(ws)
# run_forever 会阻塞,需要传入 stop_event 的检查 # websocket-client 的 run_forever 会阻塞;QQ 协议使用自定义心跳,不启用 ping。
# websocket-client 的 run_forever 支持 ping_interval, ping_timeout try:
# 我们使用自定义心跳,所以不设置 ping
ws.run_forever( ws.run_forever(
ping_interval=None, ping_interval=None,
ping_timeout=None, ping_timeout=None,
skip_utf8_validation=True, skip_utf8_validation=True,
) )
finally:
# 旧连接的心跳必须先终止,下一轮 reconnect 才能取得 owner。
stop_heartbeat(wait=True)
except Exception as e: except Exception as err:
logger.error(f"[QQ Gateway:{config_name}] Connection error: {e}") logger.error(f"[QQ Gateway:{config_name}] Connection error: {err}")
if stop_event.is_set(): if stop_event.is_set():
break break
delay = reconnect_delays[min(attempt, len(reconnect_delays) - 1)] delay = reconnect_delays[min(attempt, len(reconnect_delays) - 1)]
attempt += 1 attempt += 1
logger.info(f"[QQ Gateway:{config_name}] Reconnecting in {delay}s (attempt {attempt})") logger.info(
f"[QQ Gateway:{config_name}] Reconnecting in {delay}s (attempt {attempt})"
)
for _ in range(delay * 10): for _ in range(delay * 10):
if stop_event.is_set(): if stop_event.is_set():
break break
time.sleep(0.1) time.sleep(0.1)
finally:
if heartbeat_timer: stop_heartbeat(wait=True)
heartbeat_timer.cancel()
logger.info(f"[QQ Gateway:{config_name}] Gateway thread stopped") logger.info(f"[QQ Gateway:{config_name}] Gateway thread stopped")
+29 -6
View File
@@ -85,9 +85,9 @@
|---|---|---|---| |---|---|---|---|
| 0 | 历史任务清账、现行架构图、外部契约核对和宿主基线对齐 | 已推送 | `d234c7132`;远端同 SHAahead/behind `0/0`;架构契约 `71 passed` | | 0 | 历史任务清账、现行架构图、外部契约核对和宿主基线对齐 | 已推送 | `d234c7132`;远端同 SHAahead/behind `0/0`;架构契约 `71 passed` |
| 1 | Mypy fail-closed,并把 Ruff/Mypy 已下降债务固化为真实低水位 | 已推送 | `6062b0661`;远端同 SHAahead/behind `0/0`Mypy 11994、Ruff 976 | | 1 | Mypy fail-closed,并把 Ruff/Mypy 已下降债务固化为真实低水位 | 已推送 | `6062b0661`;远端同 SHAahead/behind `0/0`Mypy 11994、Ruff 976 |
| 2 | 用全量串行测试初始化非零 Coverage 低水位,并补齐 CI/文档防回退契约 | 已本地验证 | Ubuntu canonicalApplication `9292/11949`77.76%Domain `3390/4278`79.24%);专项 `26 passed` | | 2 | 用全量串行测试初始化非零 Coverage 低水位,并补齐 CI/文档防回退契约 | 已推送 | `265d3c6d1`;远端同 SHAahead/behind `0/0`Application 77.76%Domain 79.24% |
| 3 | 收口阶段 62 遗留的 QQ Gateway heartbeat Timer 所有权 | 待批次 2 | Timer 只 cancel 不 joinGateway 主线程可能在 heartbeat 仍执行时报告停止成功 | | 3 | 收口阶段 62 遗留的 QQ Gateway heartbeat Timer 所有权 | 已全量验证 | generation owner 已实现;全量 `6391 passed, 6 skipped`;等待提交推送 |
| Final | 全仓回归、插件兼容复核、台账定稿和远端一致性验证 | 待前置批次 | 所有准入项已推送;全量测试和适用门禁通过;本地/远端 0/0 | | Final | 全仓回归、插件兼容复核、台账定稿和远端一致性验证 | 进行中 | 本地门禁通过;等待批次 3 推送及远端 0/0 复核 |
### 批次 0:审计与基线对齐 ### 批次 0:审计与基线对齐
@@ -162,7 +162,8 @@
最新专项覆盖,质量/CI 专项 `26 passed`Ruff 通过,Pylint `10.00/10`,宿主、Schema、 最新专项覆盖,质量/CI 专项 `26 passed`Ruff 通过,Pylint `10.00/10`,宿主、Schema、
Coverage 和锁文件门禁均通过。 Coverage 和锁文件门禁均通过。
待完成:提交推送并记录远端证据。 交付证据:提交 `265d3c6d1` 已推送到 `origin/v3``git ls-remote` 返回同一 SHA
`HEAD...origin/v3``0/0`,提交严格包含批次 2 的 8 个主仓路径。
### 批次 3QQ Gateway heartbeat owner ### 批次 3QQ Gateway heartbeat owner
@@ -171,11 +172,29 @@
WebSocket 和 Gateway 退出路径只调用 `cancel()`,而 `QQBot.stop()` 只等待 Gateway 主线程。 WebSocket 和 Gateway 退出路径只调用 `cancel()`,而 `QQBot.stop()` 只等待 Gateway 主线程。
当 heartbeat callback 已进入 `send()` 时,`cancel()` 不能终止它,外层因此可能误报收敛。 当 heartbeat callback 已进入 `send()` 时,`cancel()` 不能终止它,外层因此可能误报收敛。
停止条件:Gateway 最终退出路径在现有外层 20 秒硬预算内等待当前 Timer 真正终止; 停止条件:Gateway 最终退出路径等待当前 heartbeat owner 真正终止,不为 heartbeat 叠加
Timer 未终止时 Gateway 线程保持存活,`QQBot` 保留 owner 并允许再次停止;故障注入证明首次 第二份调用方等待预算;heartbeat 未终止时 Gateway 线程保持存活,`QQBot` 在现有 20 秒
Gateway join 预算后保留 owner 并允许再次停止;故障注入证明首次
停止返回 `False`、释放 callback 后第二次返回 `True`;生命周期专项和 task ownership 门禁通过; 停止返回 `False`、释放 callback 后第二次返回 `True`;生命周期专项和 task ownership 门禁通过;
批次独立提交推送。 批次独立提交推送。
本地实现与专项验收:
* 递归 Timer 已替换为一个由 Gateway 聚合、按 connection generation 捕获精确 WebSocket 的
常驻心跳线程;新 Hello 必须先失效并等待旧 generation,旧连接 close 不能终止新连接心跳;
* 状态锁只保护 epoch、owner 和 sequence 快照,任何 `join()`、网络发送及 close 回调都不持锁;
Gateway 的每次连接和最终退出都等待子 owner 终止,所以未收敛时外层 Gateway 线程保持存活;
* 回调中的 Identify、Invalid Session 均使用回调携带的精确 WebSocket,避免 `ws_holder`
被并发 close 清空或替换后误发到别的连接;公开 `QQBot.stop()` 合同和等待预算未改变;
* 三条阻塞发送、重复 Hello 和双连接重连故障注入测试并行重复 4 轮均通过;完整生命周期专项
`96 passed`Ruff 通过,Pylint `10.00/10`task ownership、复杂度、async blocking、
宿主基线及 Ruff/Mypy ratchet 均通过;Mypy 低水位从 11994 净减至 11993。
* 双连接用例直接证明重连前必须等待旧心跳终止,且迟到的旧连接 close 不会终止新 generation
独立代码审查未发现生产线程生命周期、锁序、generation 或 reconnect 合同缺陷;
* 4 分片全量回归 `6391 passed, 6 skipped`;锁文件、Schema、Coverage、service locator 和
官方插件 ABI 语义门禁均通过,架构/质量专项 `50 passed`。插件参考仓只有不参与语义判定的
schema/provenance 漂移,且本地分支落后其远端 2 个提交,本轮保持参考仓只读且不刷新指纹。
## 五、验证矩阵 ## 五、验证矩阵
每个批次按改动范围选择下列命令,Final 全部执行: 每个批次按改动范围选择下列命令,Final 全部执行:
@@ -217,6 +236,10 @@ git rev-list --left-right --count HEAD...origin/v3
| 2026-08-26 | 批次 1 交付 | `6062b0661` 已推送;远端同 SHAahead/behind `0/0`;显式变更 10 个路径 | | 2026-08-26 | 批次 1 交付 | `6062b0661` 已推送;远端同 SHAahead/behind `0/0`;显式变更 10 个路径 |
| 2026-08-26 | 批次 2 canonical Coverage | run `32977180133` 与本机计数一致;Application 77.76%Domain 79.24%;零 fixture 已初始化 | | 2026-08-26 | 批次 2 canonical Coverage | run `32977180133` 与本机计数一致;Application 77.76%Domain 79.24%;零 fixture 已初始化 |
| 2026-08-26 | 批次 2 本地验收 | 串行全量 `6379 passed, 6 skipped`;最新专项 `26 passed`Coverage/宿主/Schema/锁文件门禁通过 | | 2026-08-26 | 批次 2 本地验收 | 串行全量 `6379 passed, 6 skipped`;最新专项 `26 passed`Coverage/宿主/Schema/锁文件门禁通过 |
| 2026-08-26 | 批次 2 交付 | `265d3c6d1` 已推送;远端同 SHAahead/behind `0/0`;显式变更 8 个路径 |
| 2026-08-26 | 批次 3 启动 | 仅收口阶段 62 已记录的 QQ heartbeat owner,不扩张公开 `QQBot.stop()` 合同 |
| 2026-08-26 | 批次 3 专项验收 | 三条故障注入并行重复 4 轮稳定;生命周期 `96 passed`;静态、所有权和架构门禁通过 |
| 2026-08-26 | 批次 3 全量验收 | 最终快照 4 分片 `6391 passed, 6 skipped`;全部适用主仓门禁通过;插件 ABI 语义无变化 |
## 七、本轮停止条件 ## 七、本轮停止条件
+4 -1
View File
@@ -35,7 +35,10 @@
- Prefer `async def` for I/O-bound operations (network requests, database queries, file operations). - Prefer `async def` for I/O-bound operations (network requests, database queries, file operations).
- Use `await` consistently; do not mix sync and async code paths in the same function without using `run_in_threadpool` from FastAPI or `asyncio.to_thread`. - Use `await` consistently; do not mix sync and async code paths in the same function without using `run_in_threadpool` from FastAPI or `asyncio.to_thread`.
- For CPU-bound work that must not block the event loop, submit to `ThreadHelper` (see `app/runtime/thread.py`). - For CPU-bound work that must not block the event loop, submit to `ThreadHelper` (see `app/runtime/thread.py`).
- Do not use bare `threading.Thread` in new code; use `ThreadHelper.submit()`. - Use `ThreadHelper.submit()` for finite background work. A long-lived protocol loop may use a
dedicated `threading.Thread` only when its existing lifecycle owner signals and joins that child,
while the external caller bounded-waits and retains the parent owner on non-convergence; do not
create a dedicated thread pool for that exception.
--- ---
+1 -1
View File
@@ -2580,7 +2580,7 @@
"type-arg": 13 "type-arg": 13
}, },
"app/modules/qqbot/gateway.py": { "app/modules/qqbot/gateway.py": {
"no-untyped-def": 4, "no-untyped-def": 3,
"type-arg": 2 "type-arg": 2
}, },
"app/modules/qqbot/module.py": { "app/modules/qqbot/module.py": {
+331
View File
@@ -1,4 +1,5 @@
import asyncio import asyncio
import json
import threading import threading
import time import time
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
@@ -13,6 +14,7 @@ from app.modules.feishu import FeishuModule
from app.modules.feishu.feishu import Feishu from app.modules.feishu.feishu import Feishu
from app.modules.filter import FilterModule from app.modules.filter import FilterModule
from app.modules.plex import PlexModule from app.modules.plex import PlexModule
from app.modules.qqbot import gateway as qq_gateway
from app.modules.qqbot.module import QQBotModule from app.modules.qqbot.module import QQBotModule
from app.modules.qqbot.qqbot import QQBot from app.modules.qqbot.qqbot import QQBot
from app.modules.slack import SlackModule from app.modules.slack import SlackModule
@@ -328,6 +330,335 @@ def test_qqbot_stop_retains_gateway_thread_until_retry() -> None:
assert client.stop() is True assert client.stop() is True
def test_qqbot_stop_retains_gateway_until_inflight_heartbeat_finishes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""心跳发送仍阻塞时 Gateway 必须保持存活,释放后才允许关闭成功。"""
heartbeat_started = threading.Event()
release_heartbeat = threading.Event()
connection_closed = threading.Event()
callbacks = {}
fake_ws = Mock()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
def build_websocket(_url, **kwargs):
"""保存 Gateway 回调并返回隔离的 WebSocket 桩。"""
callbacks.update(kwargs)
return fake_ws
def send(payload: str) -> None:
"""Identify 立即完成,首个心跳保持在发送中的故障状态。"""
if json.loads(payload).get("op") == 1:
heartbeat_started.set()
release_heartbeat.wait()
def close() -> None:
"""同步触发 close 回调,复现 QQBot.stop() 的真实调用线程。"""
callbacks["on_close"](fake_ws, 1000, "test close")
connection_closed.set()
def run_forever(**_kwargs) -> None:
"""发送 Hello 后保持连接,直到 stop() 主动关闭。"""
callbacks["on_message"](fake_ws, hello_payload)
connection_closed.wait()
fake_ws.send.side_effect = send
fake_ws.close.side_effect = close
fake_ws.run_forever.side_effect = run_forever
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
client = QQBot.__new__(QQBot)
client._gateway_stop = threading.Event()
client._gateway_ws_holder = []
client._gateway_join_timeout_seconds = 0.02
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": client._gateway_stop,
"ws_holder": client._gateway_ws_holder,
},
daemon=True,
)
client._gateway_thread = gateway_thread
gateway_thread.start()
try:
assert heartbeat_started.wait(0.5)
started_at = time.monotonic()
assert client.stop() is False
assert time.monotonic() - started_at < 0.2
assert gateway_thread.is_alive()
release_heartbeat.set()
gateway_thread.join(timeout=1.0)
assert client.stop() is True
assert not gateway_thread.is_alive()
finally:
client._gateway_stop.set()
release_heartbeat.set()
connection_closed.set()
gateway_thread.join(timeout=1.0)
def test_qq_gateway_replaces_heartbeat_generation_without_overlap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""重复 Hello 必须等待旧心跳终止,任何时刻只能有一个发送 generation。"""
first_heartbeat_started = threading.Event()
release_first_heartbeat = threading.Event()
second_heartbeat_started = threading.Event()
release_second_heartbeat = threading.Event()
third_heartbeat_started = threading.Event()
second_hello_entered = threading.Event()
third_hello_entered = threading.Event()
connection_ready = threading.Event()
connection_closed = threading.Event()
counters_lock = threading.Lock()
callbacks = {}
fake_ws = Mock()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
heartbeat_count = 0
active_heartbeats = 0
max_active_heartbeats = 0
def build_websocket(_url, **kwargs):
"""保存 Gateway 回调并返回隔离的 WebSocket 桩。"""
callbacks.update(kwargs)
return fake_ws
def send(payload: str) -> None:
"""阻塞第一代心跳,并记录是否出现跨 generation 并发发送。"""
nonlocal heartbeat_count, active_heartbeats, max_active_heartbeats
if json.loads(payload).get("op") != 1:
return
with counters_lock:
heartbeat_count += 1
current_heartbeat = heartbeat_count
active_heartbeats += 1
max_active_heartbeats = max(
max_active_heartbeats,
active_heartbeats,
)
try:
if current_heartbeat == 1:
first_heartbeat_started.set()
release_first_heartbeat.wait()
elif current_heartbeat == 2:
second_heartbeat_started.set()
release_second_heartbeat.wait()
elif current_heartbeat == 3:
third_heartbeat_started.set()
finally:
with counters_lock:
active_heartbeats -= 1
def run_forever(**_kwargs) -> None:
"""发送首个 Hello 后保持连接,第二个 Hello 由测试线程注入。"""
callbacks["on_message"](fake_ws, hello_payload)
connection_ready.set()
connection_closed.wait()
def send_second_hello() -> None:
"""从独立调用线程注入重复 Hello,以观测 generation 屏障。"""
second_hello_entered.set()
callbacks["on_message"](fake_ws, hello_payload)
def send_third_hello() -> None:
"""在第二代发送中再次注入 Hello,供 close 失效待发布 generation。"""
third_hello_entered.set()
callbacks["on_message"](fake_ws, hello_payload)
fake_ws.send.side_effect = send
fake_ws.run_forever.side_effect = run_forever
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
stop_event = threading.Event()
ws_holder = []
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "generation-test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": stop_event,
"ws_holder": ws_holder,
},
daemon=True,
)
gateway_thread.start()
hello_thread = threading.Thread(target=send_second_hello, daemon=True)
hello_thread_started = False
third_hello_thread = threading.Thread(target=send_third_hello, daemon=True)
third_hello_thread_started = False
try:
assert connection_ready.wait(0.5)
assert first_heartbeat_started.wait(0.5)
hello_thread.start()
hello_thread_started = True
assert second_hello_entered.wait(0.2)
assert not second_heartbeat_started.wait(0.03)
assert hello_thread.is_alive()
assert max_active_heartbeats == 1
release_first_heartbeat.set()
hello_thread.join(timeout=1.0)
assert not hello_thread.is_alive()
assert second_heartbeat_started.wait(0.5)
assert max_active_heartbeats == 1
third_hello_thread.start()
third_hello_thread_started = True
assert third_hello_entered.wait(0.2)
assert third_hello_thread.is_alive()
close_started_at = time.monotonic()
callbacks["on_close"](fake_ws, 1000, "close during replacement")
assert time.monotonic() - close_started_at < 0.2
release_second_heartbeat.set()
third_hello_thread.join(timeout=1.0)
assert not third_hello_thread.is_alive()
assert not third_heartbeat_started.wait(0.05)
assert max_active_heartbeats == 1
stop_event.set()
connection_closed.set()
gateway_thread.join(timeout=1.0)
assert not gateway_thread.is_alive()
finally:
stop_event.set()
release_first_heartbeat.set()
release_second_heartbeat.set()
connection_closed.set()
if hello_thread_started:
hello_thread.join(timeout=1.0)
if third_hello_thread_started:
third_hello_thread.join(timeout=1.0)
gateway_thread.join(timeout=1.0)
def test_qq_gateway_reconnect_joins_old_heartbeat_and_ignores_stale_close(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""重连必须先回收旧心跳,迟到的旧连接 close 不得终止新 generation。"""
first_callbacks = {}
second_callbacks = {}
first_ws = Mock()
second_ws = Mock()
first_heartbeat_started = threading.Event()
release_first_heartbeat = threading.Event()
return_first_connection = threading.Event()
second_connection_created = threading.Event()
second_heartbeat_started = threading.Event()
release_second_heartbeat = threading.Event()
second_followup_started = threading.Event()
return_second_connection = threading.Event()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
websocket_count = 0
second_heartbeat_count = 0
def build_websocket(_url, **kwargs):
"""依次创建两条连接,并保留各自回调以注入迟到 close。"""
nonlocal websocket_count
websocket_count += 1
if websocket_count == 1:
first_callbacks.update(kwargs)
return first_ws
if websocket_count == 2:
second_callbacks.update(kwargs)
second_connection_created.set()
return second_ws
raise AssertionError("Gateway 在停止后不应建立第三条测试连接")
def send_first(payload: str) -> None:
"""阻塞第一条连接的心跳,供重连屏障检查。"""
if json.loads(payload).get("op") == 1:
first_heartbeat_started.set()
release_first_heartbeat.wait()
def send_second(payload: str) -> None:
"""阻塞新连接首个心跳,并记录迟到 close 后是否继续工作。"""
nonlocal second_heartbeat_count
if json.loads(payload).get("op") != 1:
return
second_heartbeat_count += 1
if second_heartbeat_count == 1:
second_heartbeat_started.set()
release_second_heartbeat.wait()
else:
second_followup_started.set()
def run_first(**_kwargs) -> None:
"""第一条连接握手后按测试信号返回,触发真实 reconnect 路径。"""
first_callbacks["on_message"](first_ws, hello_payload)
return_first_connection.wait()
def run_second(**_kwargs) -> None:
"""第二条连接保持运行,直到测试完成迟到 close 校验。"""
second_callbacks["on_message"](second_ws, hello_payload)
return_second_connection.wait()
first_ws.send.side_effect = send_first
first_ws.run_forever.side_effect = run_first
second_ws.send.side_effect = send_second
second_ws.run_forever.side_effect = run_second
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
monkeypatch.setattr(qq_gateway.time, "sleep", lambda _seconds: None)
stop_event = threading.Event()
ws_holder = []
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "reconnect-test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": stop_event,
"ws_holder": ws_holder,
},
daemon=True,
)
gateway_thread.start()
try:
assert first_heartbeat_started.wait(0.5)
return_first_connection.set()
assert not second_connection_created.wait(0.05)
release_first_heartbeat.set()
assert second_connection_created.wait(0.5)
assert second_heartbeat_started.wait(0.5)
first_callbacks["on_close"](first_ws, 1000, "stale close")
release_second_heartbeat.set()
assert second_followup_started.wait(0.5)
stop_event.set()
second_callbacks["on_close"](second_ws, 1000, "current close")
return_second_connection.set()
gateway_thread.join(timeout=1.0)
assert not gateway_thread.is_alive()
finally:
stop_event.set()
release_first_heartbeat.set()
release_second_heartbeat.set()
return_first_connection.set()
return_second_connection.set()
gateway_thread.join(timeout=1.0)
def test_wechat_bot_stop_reports_each_live_thread_until_retry() -> None: def test_wechat_bot_stop_reports_each_live_thread_until_retry() -> None:
"""企业微信网关或心跳任一存活时都必须返回未收敛。""" """企业微信网关或心跳任一存活时都必须返回未收敛。"""
client = WeChatBot.__new__(WeChatBot) client = WeChatBot.__new__(WeChatBot)