fix(qqbot): converge heartbeat lifecycle

This commit is contained in:
jxxghp
2026-08-26 22:40:01 +08:00
parent b1c4150895
commit 50b85235d8
5 changed files with 526 additions and 74 deletions
+161 -66
View File
@@ -39,27 +39,116 @@ def run_gateway(
:param ws_holder: 调用方持有的单元素列表,存放当前 WebSocketApp,供 stop() 时 close 以打断 run_forever
"""
last_seq: Optional[int] = None
heartbeat_interval_ms: Optional[int] = None
heartbeat_timer: Optional[threading.Timer] = None
heartbeat_thread: Optional[threading.Thread] = None
heartbeat_stop_event: Optional[threading.Event] = None
heartbeat_epoch = 0
heartbeat_state_lock = threading.Lock()
heartbeat_replace_lock = threading.RLock()
def send_heartbeat():
nonlocal heartbeat_timer
if stop_event.is_set():
def heartbeat_loop(
generation_epoch: int,
generation_stop_event: threading.Event,
interval_seconds: float,
ws: websocket.WebSocketApp,
) -> None:
"""在当前连接 generation 内串行发送心跳,停止后不再派生新线程。"""
while not generation_stop_event.wait(interval_seconds):
if stop_event.is_set():
return
with heartbeat_state_lock:
if (
generation_epoch != heartbeat_epoch
or heartbeat_stop_event is not generation_stop_event
):
return
sequence = last_seq
try:
payload = {"op": 1, "d": sequence}
ws.send(json.dumps(payload))
logger.debug(
f"[QQ Gateway:{config_name}] Heartbeat sent, seq={sequence}"
)
except Exception as err:
logger.debug(f"[QQ Gateway:{config_name}] Heartbeat error: {err}")
def invalidate_heartbeat_generation() -> tuple[Optional[threading.Thread], int]:
"""锁内失效当前 generation,并返回待等待的 owner 与新 epoch。"""
nonlocal heartbeat_epoch
with heartbeat_state_lock:
heartbeat_epoch += 1
current_thread = heartbeat_thread
if heartbeat_stop_event is not None:
heartbeat_stop_event.set()
return current_thread, heartbeat_epoch
def clear_heartbeat_generation(current_thread: Optional[threading.Thread]) -> None:
"""仅在快照仍是当前 owner 且已经终止时清除 generation 引用。"""
nonlocal heartbeat_thread, heartbeat_stop_event
if current_thread is not None and current_thread.is_alive():
return
try:
if ws_holder and ws_holder[0]:
payload = {"op": 1, "d": last_seq}
ws_holder[0].send(json.dumps(payload))
logger.debug(f"[QQ Gateway:{config_name}] Heartbeat sent, seq={last_seq}")
except Exception as err:
logger.debug(f"[QQ Gateway:{config_name}] Heartbeat error: {err}")
if heartbeat_interval_ms and not stop_event.is_set():
heartbeat_timer = threading.Timer(heartbeat_interval_ms / 1000.0, send_heartbeat)
heartbeat_timer.daemon = True
heartbeat_timer.start()
with heartbeat_state_lock:
if heartbeat_thread is current_thread:
heartbeat_thread = None
heartbeat_stop_event = None
def on_ws_message(_, message):
nonlocal last_seq, heartbeat_interval_ms, heartbeat_timer
def stop_heartbeat(*, wait: bool) -> None:
"""停止当前心跳;等待阶段不持有状态锁,避免阻塞 close 回调。"""
if wait:
heartbeat_replace_lock.acquire()
try:
current_thread, _ = invalidate_heartbeat_generation()
if (
wait
and current_thread is not None
and current_thread.is_alive()
and current_thread is not threading.current_thread()
):
current_thread.join()
clear_heartbeat_generation(current_thread)
finally:
if wait:
heartbeat_replace_lock.release()
def start_heartbeat(ws: websocket.WebSocketApp, interval_ms: int) -> None:
"""停止旧 generation 后启动一个由 Gateway 聚合的新心跳 owner。"""
nonlocal heartbeat_thread, heartbeat_stop_event
with heartbeat_replace_lock:
current_thread, generation_epoch = invalidate_heartbeat_generation()
if (
current_thread is not None
and current_thread.is_alive()
and current_thread is not threading.current_thread()
):
current_thread.join()
clear_heartbeat_generation(current_thread)
with heartbeat_state_lock:
if (
generation_epoch != heartbeat_epoch
or stop_event.is_set()
or not ws_holder
or ws_holder[0] is not ws
):
return
generation_stop_event = threading.Event()
# 心跳是 Gateway 的常驻子 owner,不占用进程共享 ThreadHelperGateway 退出前统一 join。
generation_thread = threading.Thread(
target=heartbeat_loop,
args=(
generation_epoch,
generation_stop_event,
interval_ms / 1000.0,
ws,
),
daemon=True,
name=f"qq-gateway-heartbeat-{config_name}",
)
heartbeat_stop_event = generation_stop_event
heartbeat_thread = generation_thread
generation_thread.start()
def on_ws_message(ws, message):
"""处理当前 WebSocket 连接收到的 QQ Gateway 消息。"""
nonlocal last_seq
try:
payload = json.loads(message)
except json.JSONDecodeError as err:
@@ -72,7 +161,8 @@ def run_gateway(
t = payload.get("t")
if s is not None:
last_seq = s
with heartbeat_state_lock:
last_seq = s
logger.debug(f"[QQ Gateway:{config_name}] op={op} t={t}")
@@ -89,15 +179,10 @@ def run_gateway(
"shard": [0, 1],
},
}
ws_holder[0].send(json.dumps(identify))
ws.send(json.dumps(identify))
logger.info(f"[QQ Gateway:{config_name}] Identify sent")
# 启动心跳
if heartbeat_timer:
heartbeat_timer.cancel()
heartbeat_timer = threading.Timer(heartbeat_interval_ms / 1000.0, send_heartbeat)
heartbeat_timer.daemon = True
heartbeat_timer.start()
start_heartbeat(ws, heartbeat_interval_ms)
elif op == 0: # Dispatch
if t == "READY":
@@ -147,59 +232,69 @@ def run_gateway(
elif op == 9: # Invalid Session
logger.warning(f"[QQ Gateway:{config_name}] Invalid session")
if ws_holder and ws_holder[0]:
ws_holder[0].close()
ws.close()
def on_ws_error(_, error):
"""记录当前 WebSocket 连接上报的错误。"""
logger.error(f"[QQ Gateway:{config_name}] WebSocket error: {error}")
def on_ws_close(_, close_status_code, close_msg):
def on_ws_close(ws, close_status_code, close_msg):
"""失效当前连接及其心跳 generation,但不在回调线程等待。"""
logger.info(f"[QQ Gateway:{config_name}] WebSocket closed: {close_status_code} {close_msg}")
if heartbeat_timer:
heartbeat_timer.cancel()
ws_holder.clear()
# close 回调可能由 stop() 调用线程触发,只发信号,统一由 Gateway 线程等待终态。
with heartbeat_state_lock:
is_current_connection = bool(ws_holder and ws_holder[0] is ws)
if is_current_connection:
ws_holder.clear()
if is_current_connection:
stop_heartbeat(wait=False)
reconnect_delays = [1, 2, 5, 10, 30, 60]
attempt = 0
while not stop_event.is_set():
try:
token = get_token_fn(app_id, app_secret)
gateway_url = get_gateway_url_fn(token)
logger.info(f"[QQ Gateway:{config_name}] Connecting to {gateway_url[:60]}...")
try:
while not stop_event.is_set():
try:
token = get_token_fn(app_id, app_secret)
gateway_url = get_gateway_url_fn(token)
logger.info(f"[QQ Gateway:{config_name}] Connecting to {gateway_url[:60]}...")
ws = websocket.WebSocketApp(
gateway_url,
on_message=on_ws_message,
on_error=on_ws_error,
on_close=on_ws_close,
)
ws_holder.clear()
ws_holder.append(ws)
ws = websocket.WebSocketApp(
gateway_url,
on_message=on_ws_message,
on_error=on_ws_error,
on_close=on_ws_close,
)
with heartbeat_state_lock:
ws_holder.clear()
ws_holder.append(ws)
# run_forever 会阻塞,需要传入 stop_event 的检查
# websocket-client 的 run_forever 支持 ping_interval, ping_timeout
# 我们使用自定义心跳,所以不设置 ping
ws.run_forever(
ping_interval=None,
ping_timeout=None,
skip_utf8_validation=True,
)
# websocket-client 的 run_forever 会阻塞;QQ 协议使用自定义心跳,不启用 ping。
try:
ws.run_forever(
ping_interval=None,
ping_timeout=None,
skip_utf8_validation=True,
)
finally:
# 旧连接的心跳必须先终止,下一轮 reconnect 才能取得 owner。
stop_heartbeat(wait=True)
except Exception as e:
logger.error(f"[QQ Gateway:{config_name}] Connection error: {e}")
except Exception as err:
logger.error(f"[QQ Gateway:{config_name}] Connection error: {err}")
if stop_event.is_set():
break
delay = reconnect_delays[min(attempt, len(reconnect_delays) - 1)]
attempt += 1
logger.info(f"[QQ Gateway:{config_name}] Reconnecting in {delay}s (attempt {attempt})")
for _ in range(delay * 10):
if stop_event.is_set():
break
time.sleep(0.1)
if heartbeat_timer:
heartbeat_timer.cancel()
logger.info(f"[QQ Gateway:{config_name}] Gateway thread stopped")
delay = reconnect_delays[min(attempt, len(reconnect_delays) - 1)]
attempt += 1
logger.info(
f"[QQ Gateway:{config_name}] Reconnecting in {delay}s (attempt {attempt})"
)
for _ in range(delay * 10):
if stop_event.is_set():
break
time.sleep(0.1)
finally:
stop_heartbeat(wait=True)
logger.info(f"[QQ Gateway:{config_name}] Gateway thread stopped")
+29 -6
View File
@@ -85,9 +85,9 @@
|---|---|---|---|
| 0 | 历史任务清账、现行架构图、外部契约核对和宿主基线对齐 | 已推送 | `d234c7132`;远端同 SHAahead/behind `0/0`;架构契约 `71 passed` |
| 1 | Mypy fail-closed,并把 Ruff/Mypy 已下降债务固化为真实低水位 | 已推送 | `6062b0661`;远端同 SHAahead/behind `0/0`Mypy 11994、Ruff 976 |
| 2 | 用全量串行测试初始化非零 Coverage 低水位,并补齐 CI/文档防回退契约 | 已本地验证 | Ubuntu canonicalApplication `9292/11949`77.76%Domain `3390/4278`79.24%);专项 `26 passed` |
| 3 | 收口阶段 62 遗留的 QQ Gateway heartbeat Timer 所有权 | 待批次 2 | Timer 只 cancel 不 joinGateway 主线程可能在 heartbeat 仍执行时报告停止成功 |
| Final | 全仓回归、插件兼容复核、台账定稿和远端一致性验证 | 待前置批次 | 所有准入项已推送;全量测试和适用门禁通过;本地/远端 0/0 |
| 2 | 用全量串行测试初始化非零 Coverage 低水位,并补齐 CI/文档防回退契约 | 已推送 | `265d3c6d1`;远端同 SHAahead/behind `0/0`Application 77.76%Domain 79.24% |
| 3 | 收口阶段 62 遗留的 QQ Gateway heartbeat Timer 所有权 | 已全量验证 | generation owner 已实现;全量 `6391 passed, 6 skipped`;等待提交推送 |
| Final | 全仓回归、插件兼容复核、台账定稿和远端一致性验证 | 进行中 | 本地门禁通过;等待批次 3 推送及远端 0/0 复核 |
### 批次 0:审计与基线对齐
@@ -162,7 +162,8 @@
最新专项覆盖,质量/CI 专项 `26 passed`Ruff 通过,Pylint `10.00/10`,宿主、Schema、
Coverage 和锁文件门禁均通过。
待完成:提交推送并记录远端证据。
交付证据:提交 `265d3c6d1` 已推送到 `origin/v3``git ls-remote` 返回同一 SHA
`HEAD...origin/v3``0/0`,提交严格包含批次 2 的 8 个主仓路径。
### 批次 3QQ Gateway heartbeat owner
@@ -171,11 +172,29 @@
WebSocket 和 Gateway 退出路径只调用 `cancel()`,而 `QQBot.stop()` 只等待 Gateway 主线程。
当 heartbeat callback 已进入 `send()` 时,`cancel()` 不能终止它,外层因此可能误报收敛。
停止条件:Gateway 最终退出路径在现有外层 20 秒硬预算内等待当前 Timer 真正终止;
Timer 未终止时 Gateway 线程保持存活,`QQBot` 保留 owner 并允许再次停止;故障注入证明首次
停止条件:Gateway 最终退出路径等待当前 heartbeat owner 真正终止,不为 heartbeat 叠加
第二份调用方等待预算;heartbeat 未终止时 Gateway 线程保持存活,`QQBot` 在现有 20 秒
Gateway join 预算后保留 owner 并允许再次停止;故障注入证明首次
停止返回 `False`、释放 callback 后第二次返回 `True`;生命周期专项和 task ownership 门禁通过;
批次独立提交推送。
本地实现与专项验收:
* 递归 Timer 已替换为一个由 Gateway 聚合、按 connection generation 捕获精确 WebSocket 的
常驻心跳线程;新 Hello 必须先失效并等待旧 generation,旧连接 close 不能终止新连接心跳;
* 状态锁只保护 epoch、owner 和 sequence 快照,任何 `join()`、网络发送及 close 回调都不持锁;
Gateway 的每次连接和最终退出都等待子 owner 终止,所以未收敛时外层 Gateway 线程保持存活;
* 回调中的 Identify、Invalid Session 均使用回调携带的精确 WebSocket,避免 `ws_holder`
被并发 close 清空或替换后误发到别的连接;公开 `QQBot.stop()` 合同和等待预算未改变;
* 三条阻塞发送、重复 Hello 和双连接重连故障注入测试并行重复 4 轮均通过;完整生命周期专项
`96 passed`Ruff 通过,Pylint `10.00/10`task ownership、复杂度、async blocking、
宿主基线及 Ruff/Mypy ratchet 均通过;Mypy 低水位从 11994 净减至 11993。
* 双连接用例直接证明重连前必须等待旧心跳终止,且迟到的旧连接 close 不会终止新 generation
独立代码审查未发现生产线程生命周期、锁序、generation 或 reconnect 合同缺陷;
* 4 分片全量回归 `6391 passed, 6 skipped`;锁文件、Schema、Coverage、service locator 和
官方插件 ABI 语义门禁均通过,架构/质量专项 `50 passed`。插件参考仓只有不参与语义判定的
schema/provenance 漂移,且本地分支落后其远端 2 个提交,本轮保持参考仓只读且不刷新指纹。
## 五、验证矩阵
每个批次按改动范围选择下列命令,Final 全部执行:
@@ -217,6 +236,10 @@ git rev-list --left-right --count HEAD...origin/v3
| 2026-08-26 | 批次 1 交付 | `6062b0661` 已推送;远端同 SHAahead/behind `0/0`;显式变更 10 个路径 |
| 2026-08-26 | 批次 2 canonical Coverage | run `32977180133` 与本机计数一致;Application 77.76%Domain 79.24%;零 fixture 已初始化 |
| 2026-08-26 | 批次 2 本地验收 | 串行全量 `6379 passed, 6 skipped`;最新专项 `26 passed`Coverage/宿主/Schema/锁文件门禁通过 |
| 2026-08-26 | 批次 2 交付 | `265d3c6d1` 已推送;远端同 SHAahead/behind `0/0`;显式变更 8 个路径 |
| 2026-08-26 | 批次 3 启动 | 仅收口阶段 62 已记录的 QQ heartbeat owner,不扩张公开 `QQBot.stop()` 合同 |
| 2026-08-26 | 批次 3 专项验收 | 三条故障注入并行重复 4 轮稳定;生命周期 `96 passed`;静态、所有权和架构门禁通过 |
| 2026-08-26 | 批次 3 全量验收 | 最终快照 4 分片 `6391 passed, 6 skipped`;全部适用主仓门禁通过;插件 ABI 语义无变化 |
## 七、本轮停止条件
+4 -1
View File
@@ -35,7 +35,10 @@
- Prefer `async def` for I/O-bound operations (network requests, database queries, file operations).
- Use `await` consistently; do not mix sync and async code paths in the same function without using `run_in_threadpool` from FastAPI or `asyncio.to_thread`.
- For CPU-bound work that must not block the event loop, submit to `ThreadHelper` (see `app/runtime/thread.py`).
- Do not use bare `threading.Thread` in new code; use `ThreadHelper.submit()`.
- Use `ThreadHelper.submit()` for finite background work. A long-lived protocol loop may use a
dedicated `threading.Thread` only when its existing lifecycle owner signals and joins that child,
while the external caller bounded-waits and retains the parent owner on non-convergence; do not
create a dedicated thread pool for that exception.
---
+1 -1
View File
@@ -2580,7 +2580,7 @@
"type-arg": 13
},
"app/modules/qqbot/gateway.py": {
"no-untyped-def": 4,
"no-untyped-def": 3,
"type-arg": 2
},
"app/modules/qqbot/module.py": {
+331
View File
@@ -1,4 +1,5 @@
import asyncio
import json
import threading
import time
from unittest.mock import Mock, patch
@@ -13,6 +14,7 @@ from app.modules.feishu import FeishuModule
from app.modules.feishu.feishu import Feishu
from app.modules.filter import FilterModule
from app.modules.plex import PlexModule
from app.modules.qqbot import gateway as qq_gateway
from app.modules.qqbot.module import QQBotModule
from app.modules.qqbot.qqbot import QQBot
from app.modules.slack import SlackModule
@@ -328,6 +330,335 @@ def test_qqbot_stop_retains_gateway_thread_until_retry() -> None:
assert client.stop() is True
def test_qqbot_stop_retains_gateway_until_inflight_heartbeat_finishes(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""心跳发送仍阻塞时 Gateway 必须保持存活,释放后才允许关闭成功。"""
heartbeat_started = threading.Event()
release_heartbeat = threading.Event()
connection_closed = threading.Event()
callbacks = {}
fake_ws = Mock()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
def build_websocket(_url, **kwargs):
"""保存 Gateway 回调并返回隔离的 WebSocket 桩。"""
callbacks.update(kwargs)
return fake_ws
def send(payload: str) -> None:
"""Identify 立即完成,首个心跳保持在发送中的故障状态。"""
if json.loads(payload).get("op") == 1:
heartbeat_started.set()
release_heartbeat.wait()
def close() -> None:
"""同步触发 close 回调,复现 QQBot.stop() 的真实调用线程。"""
callbacks["on_close"](fake_ws, 1000, "test close")
connection_closed.set()
def run_forever(**_kwargs) -> None:
"""发送 Hello 后保持连接,直到 stop() 主动关闭。"""
callbacks["on_message"](fake_ws, hello_payload)
connection_closed.wait()
fake_ws.send.side_effect = send
fake_ws.close.side_effect = close
fake_ws.run_forever.side_effect = run_forever
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
client = QQBot.__new__(QQBot)
client._gateway_stop = threading.Event()
client._gateway_ws_holder = []
client._gateway_join_timeout_seconds = 0.02
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": client._gateway_stop,
"ws_holder": client._gateway_ws_holder,
},
daemon=True,
)
client._gateway_thread = gateway_thread
gateway_thread.start()
try:
assert heartbeat_started.wait(0.5)
started_at = time.monotonic()
assert client.stop() is False
assert time.monotonic() - started_at < 0.2
assert gateway_thread.is_alive()
release_heartbeat.set()
gateway_thread.join(timeout=1.0)
assert client.stop() is True
assert not gateway_thread.is_alive()
finally:
client._gateway_stop.set()
release_heartbeat.set()
connection_closed.set()
gateway_thread.join(timeout=1.0)
def test_qq_gateway_replaces_heartbeat_generation_without_overlap(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""重复 Hello 必须等待旧心跳终止,任何时刻只能有一个发送 generation。"""
first_heartbeat_started = threading.Event()
release_first_heartbeat = threading.Event()
second_heartbeat_started = threading.Event()
release_second_heartbeat = threading.Event()
third_heartbeat_started = threading.Event()
second_hello_entered = threading.Event()
third_hello_entered = threading.Event()
connection_ready = threading.Event()
connection_closed = threading.Event()
counters_lock = threading.Lock()
callbacks = {}
fake_ws = Mock()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
heartbeat_count = 0
active_heartbeats = 0
max_active_heartbeats = 0
def build_websocket(_url, **kwargs):
"""保存 Gateway 回调并返回隔离的 WebSocket 桩。"""
callbacks.update(kwargs)
return fake_ws
def send(payload: str) -> None:
"""阻塞第一代心跳,并记录是否出现跨 generation 并发发送。"""
nonlocal heartbeat_count, active_heartbeats, max_active_heartbeats
if json.loads(payload).get("op") != 1:
return
with counters_lock:
heartbeat_count += 1
current_heartbeat = heartbeat_count
active_heartbeats += 1
max_active_heartbeats = max(
max_active_heartbeats,
active_heartbeats,
)
try:
if current_heartbeat == 1:
first_heartbeat_started.set()
release_first_heartbeat.wait()
elif current_heartbeat == 2:
second_heartbeat_started.set()
release_second_heartbeat.wait()
elif current_heartbeat == 3:
third_heartbeat_started.set()
finally:
with counters_lock:
active_heartbeats -= 1
def run_forever(**_kwargs) -> None:
"""发送首个 Hello 后保持连接,第二个 Hello 由测试线程注入。"""
callbacks["on_message"](fake_ws, hello_payload)
connection_ready.set()
connection_closed.wait()
def send_second_hello() -> None:
"""从独立调用线程注入重复 Hello,以观测 generation 屏障。"""
second_hello_entered.set()
callbacks["on_message"](fake_ws, hello_payload)
def send_third_hello() -> None:
"""在第二代发送中再次注入 Hello,供 close 失效待发布 generation。"""
third_hello_entered.set()
callbacks["on_message"](fake_ws, hello_payload)
fake_ws.send.side_effect = send
fake_ws.run_forever.side_effect = run_forever
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
stop_event = threading.Event()
ws_holder = []
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "generation-test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": stop_event,
"ws_holder": ws_holder,
},
daemon=True,
)
gateway_thread.start()
hello_thread = threading.Thread(target=send_second_hello, daemon=True)
hello_thread_started = False
third_hello_thread = threading.Thread(target=send_third_hello, daemon=True)
third_hello_thread_started = False
try:
assert connection_ready.wait(0.5)
assert first_heartbeat_started.wait(0.5)
hello_thread.start()
hello_thread_started = True
assert second_hello_entered.wait(0.2)
assert not second_heartbeat_started.wait(0.03)
assert hello_thread.is_alive()
assert max_active_heartbeats == 1
release_first_heartbeat.set()
hello_thread.join(timeout=1.0)
assert not hello_thread.is_alive()
assert second_heartbeat_started.wait(0.5)
assert max_active_heartbeats == 1
third_hello_thread.start()
third_hello_thread_started = True
assert third_hello_entered.wait(0.2)
assert third_hello_thread.is_alive()
close_started_at = time.monotonic()
callbacks["on_close"](fake_ws, 1000, "close during replacement")
assert time.monotonic() - close_started_at < 0.2
release_second_heartbeat.set()
third_hello_thread.join(timeout=1.0)
assert not third_hello_thread.is_alive()
assert not third_heartbeat_started.wait(0.05)
assert max_active_heartbeats == 1
stop_event.set()
connection_closed.set()
gateway_thread.join(timeout=1.0)
assert not gateway_thread.is_alive()
finally:
stop_event.set()
release_first_heartbeat.set()
release_second_heartbeat.set()
connection_closed.set()
if hello_thread_started:
hello_thread.join(timeout=1.0)
if third_hello_thread_started:
third_hello_thread.join(timeout=1.0)
gateway_thread.join(timeout=1.0)
def test_qq_gateway_reconnect_joins_old_heartbeat_and_ignores_stale_close(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""重连必须先回收旧心跳,迟到的旧连接 close 不得终止新 generation。"""
first_callbacks = {}
second_callbacks = {}
first_ws = Mock()
second_ws = Mock()
first_heartbeat_started = threading.Event()
release_first_heartbeat = threading.Event()
return_first_connection = threading.Event()
second_connection_created = threading.Event()
second_heartbeat_started = threading.Event()
release_second_heartbeat = threading.Event()
second_followup_started = threading.Event()
return_second_connection = threading.Event()
hello_payload = json.dumps({"op": 10, "d": {"heartbeat_interval": 1}})
websocket_count = 0
second_heartbeat_count = 0
def build_websocket(_url, **kwargs):
"""依次创建两条连接,并保留各自回调以注入迟到 close。"""
nonlocal websocket_count
websocket_count += 1
if websocket_count == 1:
first_callbacks.update(kwargs)
return first_ws
if websocket_count == 2:
second_callbacks.update(kwargs)
second_connection_created.set()
return second_ws
raise AssertionError("Gateway 在停止后不应建立第三条测试连接")
def send_first(payload: str) -> None:
"""阻塞第一条连接的心跳,供重连屏障检查。"""
if json.loads(payload).get("op") == 1:
first_heartbeat_started.set()
release_first_heartbeat.wait()
def send_second(payload: str) -> None:
"""阻塞新连接首个心跳,并记录迟到 close 后是否继续工作。"""
nonlocal second_heartbeat_count
if json.loads(payload).get("op") != 1:
return
second_heartbeat_count += 1
if second_heartbeat_count == 1:
second_heartbeat_started.set()
release_second_heartbeat.wait()
else:
second_followup_started.set()
def run_first(**_kwargs) -> None:
"""第一条连接握手后按测试信号返回,触发真实 reconnect 路径。"""
first_callbacks["on_message"](first_ws, hello_payload)
return_first_connection.wait()
def run_second(**_kwargs) -> None:
"""第二条连接保持运行,直到测试完成迟到 close 校验。"""
second_callbacks["on_message"](second_ws, hello_payload)
return_second_connection.wait()
first_ws.send.side_effect = send_first
first_ws.run_forever.side_effect = run_first
second_ws.send.side_effect = send_second
second_ws.run_forever.side_effect = run_second
monkeypatch.setattr(qq_gateway.websocket, "WebSocketApp", build_websocket)
monkeypatch.setattr(qq_gateway.time, "sleep", lambda _seconds: None)
stop_event = threading.Event()
ws_holder = []
gateway_thread = threading.Thread(
target=qq_gateway.run_gateway,
kwargs={
"app_id": "app-id",
"app_secret": "secret",
"config_name": "reconnect-test",
"get_token_fn": lambda _app_id, _secret: "token",
"get_gateway_url_fn": lambda _token: "wss://gateway.test",
"on_message_fn": lambda _payload: None,
"stop_event": stop_event,
"ws_holder": ws_holder,
},
daemon=True,
)
gateway_thread.start()
try:
assert first_heartbeat_started.wait(0.5)
return_first_connection.set()
assert not second_connection_created.wait(0.05)
release_first_heartbeat.set()
assert second_connection_created.wait(0.5)
assert second_heartbeat_started.wait(0.5)
first_callbacks["on_close"](first_ws, 1000, "stale close")
release_second_heartbeat.set()
assert second_followup_started.wait(0.5)
stop_event.set()
second_callbacks["on_close"](second_ws, 1000, "current close")
return_second_connection.set()
gateway_thread.join(timeout=1.0)
assert not gateway_thread.is_alive()
finally:
stop_event.set()
release_first_heartbeat.set()
release_second_heartbeat.set()
return_first_connection.set()
return_second_connection.set()
gateway_thread.join(timeout=1.0)
def test_wechat_bot_stop_reports_each_live_thread_until_retry() -> None:
"""企业微信网关或心跳任一存活时都必须返回未收敛。"""
client = WeChatBot.__new__(WeChatBot)