mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-09-07 00:16:57 +08:00
fix(outbox): serialize subscription completion delivery
This commit is contained in:
@@ -20,6 +20,7 @@ SUBSCRIBE_DELETED_TOPIC = "subscribe.deleted"
|
|||||||
DOWNLOAD_ADDED_TOPIC = "download.added"
|
DOWNLOAD_ADDED_TOPIC = "download.added"
|
||||||
TRANSFER_COMPLETED_TOPIC = "transfer.completed"
|
TRANSFER_COMPLETED_TOPIC = "transfer.completed"
|
||||||
TRANSFER_FAILED_TOPIC = "transfer.failed"
|
TRANSFER_FAILED_TOPIC = "transfer.failed"
|
||||||
|
OUTBOX_LEASE_SECONDS = 60
|
||||||
|
|
||||||
DURABLE_EVENT_TOPICS: Mapping[EventType, str] = MappingProxyType({
|
DURABLE_EVENT_TOPICS: Mapping[EventType, str] = MappingProxyType({
|
||||||
EventType.SubscribeAdded: SUBSCRIBE_ADDED_TOPIC,
|
EventType.SubscribeAdded: SUBSCRIBE_ADDED_TOPIC,
|
||||||
@@ -126,6 +127,14 @@ class SyncOutboxTransaction(Protocol):
|
|||||||
def stage(self, intent: OutboxIntent, now: datetime) -> None:
|
def stage(self, intent: OutboxIntent, now: datetime) -> None:
|
||||||
"""把 intent 加入调用方事务,但不自行提交。"""
|
"""把 intent 加入调用方事务,但不自行提交。"""
|
||||||
|
|
||||||
|
def claim_by_event_key(
|
||||||
|
self,
|
||||||
|
event_key: str,
|
||||||
|
now: datetime,
|
||||||
|
lease_until: datetime,
|
||||||
|
) -> bool:
|
||||||
|
"""在同步副作用前原子认领 intent,已被其他投递者持有时返回 False。"""
|
||||||
|
|
||||||
def complete_by_event_key(
|
def complete_by_event_key(
|
||||||
self,
|
self,
|
||||||
event_key: str,
|
event_key: str,
|
||||||
@@ -183,7 +192,7 @@ class OutboxDispatcher:
|
|||||||
handlers: dict[str, Callable[[ClaimedOutboxMessage], None]],
|
handlers: dict[str, Callable[[ClaimedOutboxMessage], None]],
|
||||||
*,
|
*,
|
||||||
max_attempts: int = 5,
|
max_attempts: int = 5,
|
||||||
lease_seconds: int = 60,
|
lease_seconds: int = OUTBOX_LEASE_SECONDS,
|
||||||
clock: Callable[[], datetime] | None = None,
|
clock: Callable[[], datetime] | None = None,
|
||||||
close: Callable[[], None] | None = None,
|
close: Callable[[], None] | None = None,
|
||||||
failure_observer: Callable[[bool], None] | None = None,
|
failure_observer: Callable[[bool], None] | None = None,
|
||||||
|
|||||||
@@ -4,10 +4,15 @@ from __future__ import annotations
|
|||||||
|
|
||||||
from collections.abc import Callable, Mapping
|
from collections.abc import Callable, Mapping
|
||||||
from contextlib import AbstractContextManager
|
from contextlib import AbstractContextManager
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from typing import Any, Protocol
|
from typing import Any, Protocol
|
||||||
|
|
||||||
from app.application.outbox import OutboxIntent, SyncOutboxTransaction, SyncUnitOfWork
|
from app.application.outbox import (
|
||||||
|
OUTBOX_LEASE_SECONDS,
|
||||||
|
OutboxIntent,
|
||||||
|
SyncOutboxTransaction,
|
||||||
|
SyncUnitOfWork,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class SubscriptionCompletionRepository(Protocol):
|
class SubscriptionCompletionRepository(Protocol):
|
||||||
@@ -101,19 +106,35 @@ class CompleteSubscriptionCommand:
|
|||||||
self._unit_of_work.rollback()
|
self._unit_of_work.rollback()
|
||||||
raise
|
raise
|
||||||
|
|
||||||
|
if notification:
|
||||||
|
if self._claim_sync_delivery(notification_key):
|
||||||
notify()
|
notify()
|
||||||
if self._outbox and notification:
|
self._complete_sync_delivery(notification_key)
|
||||||
self._outbox.complete_by_event_key(
|
else:
|
||||||
notification_key,
|
notify()
|
||||||
datetime.now(timezone.utc),
|
if self._claim_sync_delivery(event_key):
|
||||||
)
|
|
||||||
self._publish(event_payload)
|
self._publish(event_payload)
|
||||||
if self._outbox:
|
self._complete_sync_delivery(event_key)
|
||||||
self._outbox.complete_by_event_key(event_key, datetime.now(timezone.utc))
|
if self._claim_sync_delivery(report_key):
|
||||||
if report(report_payload["subscribe_info"]) is False:
|
if report(report_payload["subscribe_info"]) is False:
|
||||||
raise RuntimeError("订阅完成统计上报未确认")
|
raise RuntimeError("订阅完成统计上报未确认")
|
||||||
|
self._complete_sync_delivery(report_key)
|
||||||
|
|
||||||
|
def _claim_sync_delivery(self, event_key: str) -> bool:
|
||||||
|
"""在同步副作用前取得 lease,已由恢复投递接管时跳过直投。"""
|
||||||
|
if self._outbox is None:
|
||||||
|
return True
|
||||||
|
now = datetime.now(timezone.utc)
|
||||||
|
return self._outbox.claim_by_event_key(
|
||||||
|
event_key,
|
||||||
|
now,
|
||||||
|
now + timedelta(seconds=OUTBOX_LEASE_SECONDS),
|
||||||
|
)
|
||||||
|
|
||||||
|
def _complete_sync_delivery(self, event_key: str) -> None:
|
||||||
|
"""收口当前同步投递持有的 durable intent。"""
|
||||||
if self._outbox:
|
if self._outbox:
|
||||||
self._outbox.complete_by_event_key(report_key, datetime.now(timezone.utc))
|
self._outbox.complete_by_event_key(event_key, datetime.now(timezone.utc))
|
||||||
|
|
||||||
|
|
||||||
def completion_event_key(subscribe_id: int, subscribe_info: Mapping[str, Any]) -> str:
|
def completion_event_key(subscribe_id: int, subscribe_info: Mapping[str, Any]) -> str:
|
||||||
|
|||||||
@@ -91,6 +91,52 @@ class SqlAlchemyOutboxRepository:
|
|||||||
attempt=next_attempt,
|
attempt=next_attempt,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def claim_by_event_key(
|
||||||
|
self,
|
||||||
|
event_key: str,
|
||||||
|
now: datetime,
|
||||||
|
lease_until: datetime,
|
||||||
|
) -> bool:
|
||||||
|
"""按事件键原子认领同步投递,避免与 dispatcher 并发重复发送。"""
|
||||||
|
now_text = _iso(now)
|
||||||
|
candidate = self._session.execute(
|
||||||
|
select(OutboxMessage)
|
||||||
|
.where(
|
||||||
|
OutboxMessage.event_key == event_key,
|
||||||
|
OutboxMessage.status.in_(("pending", "processing")),
|
||||||
|
OutboxMessage.next_retry_at <= now_text,
|
||||||
|
or_(
|
||||||
|
OutboxMessage.lease_until.is_(None),
|
||||||
|
OutboxMessage.lease_until <= now_text,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.limit(1)
|
||||||
|
).scalars().first()
|
||||||
|
if candidate is None:
|
||||||
|
return False
|
||||||
|
claimed = execute_dml(
|
||||||
|
self._session,
|
||||||
|
update(OutboxMessage)
|
||||||
|
.where(
|
||||||
|
OutboxMessage.id == candidate.id,
|
||||||
|
OutboxMessage.attempt == candidate.attempt,
|
||||||
|
OutboxMessage.event_key == event_key,
|
||||||
|
OutboxMessage.status.in_(("pending", "processing")),
|
||||||
|
OutboxMessage.next_retry_at <= now_text,
|
||||||
|
or_(
|
||||||
|
OutboxMessage.lease_until.is_(None),
|
||||||
|
OutboxMessage.lease_until <= now_text,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.values(
|
||||||
|
status="processing",
|
||||||
|
attempt=OutboxMessage.attempt + 1,
|
||||||
|
lease_until=_iso(lease_until),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
self._session.commit()
|
||||||
|
return bool(claimed)
|
||||||
|
|
||||||
def complete(self, message_id: int, completed_at: datetime) -> None:
|
def complete(self, message_id: int, completed_at: datetime) -> None:
|
||||||
"""持久化完成终态并释放 lease。"""
|
"""持久化完成终态并释放 lease。"""
|
||||||
self._session.execute(
|
self._session.execute(
|
||||||
|
|||||||
+39
-2
@@ -1,12 +1,17 @@
|
|||||||
"""durable side-effect outbox 原子性、认领、重试与幂等测试。"""
|
"""durable side-effect outbox 原子性、认领、重试与幂等测试。"""
|
||||||
|
|
||||||
from datetime import datetime, timezone
|
from datetime import datetime, timedelta, timezone
|
||||||
from unittest.mock import MagicMock
|
from unittest.mock import MagicMock
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
from sqlalchemy import create_engine, select
|
||||||
|
from sqlalchemy.orm import sessionmaker
|
||||||
|
|
||||||
from app.application.outbox import ClaimedOutboxMessage, OutboxDispatcher
|
from app.application.outbox import ClaimedOutboxMessage, OutboxDispatcher, OutboxIntent
|
||||||
from app.application.subscription.write import CreateSubscriptionCommand
|
from app.application.subscription.write import CreateSubscriptionCommand
|
||||||
|
from app.db.adapters.outbox import SqlAlchemyOutboxRepository
|
||||||
|
from app.db.base import Base
|
||||||
|
from app.db.models.outbox import OutboxMessage
|
||||||
|
|
||||||
|
|
||||||
class _Staged:
|
class _Staged:
|
||||||
@@ -134,3 +139,35 @@ def test_dispatcher_marks_success_and_closes_owned_resource() -> None:
|
|||||||
repository.complete.assert_called_once_with(7, now)
|
repository.complete.assert_called_once_with(7, now)
|
||||||
dispatcher.close()
|
dispatcher.close()
|
||||||
close.assert_called_once_with()
|
close.assert_called_once_with()
|
||||||
|
|
||||||
|
|
||||||
|
def test_sync_outbox_claim_is_exclusive_for_event_key() -> None:
|
||||||
|
"""同步投递与恢复投递竞争同一 intent 时只允许一个取得 lease。"""
|
||||||
|
engine = create_engine("sqlite+pysqlite:///:memory:")
|
||||||
|
Base.metadata.create_all(engine)
|
||||||
|
factory = sessionmaker(bind=engine)
|
||||||
|
now = datetime(2026, 8, 24, tzinfo=timezone.utc)
|
||||||
|
lease_until = now + timedelta(seconds=60)
|
||||||
|
event_key = "subscribe.complete:7:tmdb:123:v1"
|
||||||
|
|
||||||
|
with factory() as session:
|
||||||
|
repository = SqlAlchemyOutboxRepository(session)
|
||||||
|
repository.stage(
|
||||||
|
OutboxIntent(event_key=event_key, topic="subscribe.complete", payload={}),
|
||||||
|
now,
|
||||||
|
)
|
||||||
|
session.commit()
|
||||||
|
|
||||||
|
with factory() as owner, factory() as competitor:
|
||||||
|
assert SqlAlchemyOutboxRepository(owner).claim_by_event_key(
|
||||||
|
event_key, now, lease_until
|
||||||
|
) is True
|
||||||
|
assert SqlAlchemyOutboxRepository(competitor).claim_by_event_key(
|
||||||
|
event_key, now, lease_until
|
||||||
|
) is False
|
||||||
|
|
||||||
|
with factory() as session:
|
||||||
|
message = session.execute(select(OutboxMessage)).scalar_one()
|
||||||
|
assert message.status == "processing"
|
||||||
|
assert message.attempt == 1
|
||||||
|
assert message.lease_until == lease_until.isoformat()
|
||||||
|
|||||||
@@ -45,20 +45,33 @@ class _UnitOfWork:
|
|||||||
class _Outbox:
|
class _Outbox:
|
||||||
"""记录 intent 暂存与即时收口。"""
|
"""记录 intent 暂存与即时收口。"""
|
||||||
|
|
||||||
def __init__(self, calls: list[tuple]) -> None:
|
def __init__(self, calls: list[tuple], claim_result: bool = True) -> None:
|
||||||
"""保存共享调用序列。"""
|
"""保存共享调用序列。"""
|
||||||
self.calls = calls
|
self.calls = calls
|
||||||
|
self.claim_result = claim_result
|
||||||
|
|
||||||
def stage(self, intent, _now: datetime) -> None:
|
def stage(self, intent, _now: datetime) -> None:
|
||||||
"""记录 durable intent。"""
|
"""记录 durable intent。"""
|
||||||
self.calls.append(("stage", intent))
|
self.calls.append(("stage", intent))
|
||||||
|
|
||||||
|
def claim_by_event_key(self, event_key: str, _now: datetime, _lease_until: datetime) -> bool:
|
||||||
|
"""记录同步投递认领结果。"""
|
||||||
|
self.calls.append(("claim", event_key))
|
||||||
|
return self.claim_result
|
||||||
|
|
||||||
def complete_by_event_key(self, event_key: str, _now: datetime) -> None:
|
def complete_by_event_key(self, event_key: str, _now: datetime) -> None:
|
||||||
"""记录成功副作用对应的 intent 收口。"""
|
"""记录成功副作用对应的 intent 收口。"""
|
||||||
self.calls.append(("complete", event_key))
|
self.calls.append(("complete", event_key))
|
||||||
|
|
||||||
|
|
||||||
def _command(calls: list[tuple], *, publish_error=None, report_result=True, notify_error=None):
|
def _command(
|
||||||
|
calls: list[tuple],
|
||||||
|
*,
|
||||||
|
publish_error=None,
|
||||||
|
report_result=True,
|
||||||
|
notify_error=None,
|
||||||
|
claim_result=True,
|
||||||
|
):
|
||||||
"""构造可注入失败的完成命令。"""
|
"""构造可注入失败的完成命令。"""
|
||||||
def notify() -> None:
|
def notify() -> None:
|
||||||
"""记录通知。"""
|
"""记录通知。"""
|
||||||
@@ -80,7 +93,7 @@ def _command(calls: list[tuple], *, publish_error=None, report_result=True, noti
|
|||||||
return CompleteSubscriptionCommand(
|
return CompleteSubscriptionCommand(
|
||||||
repository=_Repository(calls),
|
repository=_Repository(calls),
|
||||||
unit_of_work=_UnitOfWork(calls),
|
unit_of_work=_UnitOfWork(calls),
|
||||||
outbox=_Outbox(calls),
|
outbox=_Outbox(calls, claim_result),
|
||||||
publish=publish,
|
publish=publish,
|
||||||
), notify, report
|
), notify, report
|
||||||
|
|
||||||
@@ -113,10 +126,10 @@ def test_completion_stages_business_and_independent_intents_before_commit(failur
|
|||||||
if failure == "notify":
|
if failure == "notify":
|
||||||
assert [call[0] for call in calls[5:]] == ["notify"]
|
assert [call[0] for call in calls[5:]] == ["notify"]
|
||||||
elif failure == "event":
|
elif failure == "event":
|
||||||
assert [call[0] for call in calls[5:]] == ["notify", "event"]
|
assert [call[0] for call in calls[5:]] == ["notify", "claim", "event"]
|
||||||
else:
|
else:
|
||||||
assert [call[0] for call in calls[5:]] == [
|
assert [call[0] for call in calls[5:]] == [
|
||||||
"notify", "event", "complete", "report",
|
"notify", "claim", "event", "complete", "claim", "report",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
@@ -135,10 +148,11 @@ def test_completion_success_closes_event_then_report_intent():
|
|||||||
|
|
||||||
assert [call[0] for call in calls] == [
|
assert [call[0] for call in calls] == [
|
||||||
"history", "delete", "stage", "stage", "commit",
|
"history", "delete", "stage", "stage", "commit",
|
||||||
"notify", "event", "complete", "report", "complete",
|
"notify", "claim", "event", "complete",
|
||||||
|
"claim", "report", "complete",
|
||||||
]
|
]
|
||||||
assert calls[6][1]["idempotency_key"] == calls[2][1].event_key
|
assert calls[7][1]["idempotency_key"] == calls[2][1].event_key
|
||||||
assert calls[8][1]["idempotency_key"] == calls[3][1].event_key
|
assert calls[10][1]["idempotency_key"] == calls[3][1].event_key
|
||||||
|
|
||||||
|
|
||||||
def test_completion_stages_and_closes_notification_snapshot() -> None:
|
def test_completion_stages_and_closes_notification_snapshot() -> None:
|
||||||
@@ -164,3 +178,23 @@ def test_completion_stages_and_closes_notification_snapshot() -> None:
|
|||||||
assert staged[1].payload["message"]["title"] == "完成"
|
assert staged[1].payload["message"]["title"] == "完成"
|
||||||
completed = [call[1] for call in calls if call[0] == "complete"]
|
completed = [call[1] for call in calls if call[0] == "complete"]
|
||||||
assert completed[0].endswith(":notification")
|
assert completed[0].endswith(":notification")
|
||||||
|
|
||||||
|
|
||||||
|
def test_completion_skips_sync_delivery_owned_by_outbox_dispatcher() -> None:
|
||||||
|
"""后台已认领 intent 时同步路径不得再次发送相同副作用。"""
|
||||||
|
calls = []
|
||||||
|
command, notify, report = _command(calls, claim_result=False)
|
||||||
|
|
||||||
|
command.execute(
|
||||||
|
7,
|
||||||
|
{"id": 7, "media_source": "tmdb", "media_id": "123", "season": 2},
|
||||||
|
{"title": "Test"},
|
||||||
|
notify=notify,
|
||||||
|
report=report,
|
||||||
|
notification={"title": "完成", "text": "Test"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert [call[0] for call in calls] == [
|
||||||
|
"history", "delete", "stage", "stage", "stage", "commit",
|
||||||
|
"claim", "claim", "claim",
|
||||||
|
]
|
||||||
|
|||||||
Reference in New Issue
Block a user