GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-06-11 18:50:59 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(image-proxy): 阻断日志输出诊断原因并合并重复告警 (#5858)

Browse Source
This commit is contained in:
InfinityPacer
2026-05-29 14:14:16 +08:00
committed by GitHub
parent b45956f850
commit f4ca4120bc
6 changed files with 1248 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

201
tests/test_coalesce.py Normal file
View File

@@ -0,0 +1,201 @@
"""
`EventCoalescer` 基础设施单元测试。
测试策略：用极短窗口（默认 0.05s）驱动真实事件循环触发 flush，避免引入
对时间 mock 的复杂度；同时通过 `asyncio.sleep` 让出控制权以保证 flush
回调被调度执行。
"""
import asyncio
from typing import List
from unittest import IsolatedAsyncioTestCase
from app.utils.coalesce import (
CoalesceDecision,
CoalesceSummary,
EventCoalescer,
)
# 窗口尽量短，但要大于事件循环单次 tick 的开销，避免 flush 在 record 仍持锁时触发
_TEST_WINDOW = 0.05
# 等待窗口到期 + flush 任务完成所需的额外余量
_TEST_WAIT = _TEST_WINDOW * 4
class EventCoalescerTest(IsolatedAsyncioTestCase):
"""
覆盖 EventCoalescer 的核心契约：首条 EMIT、窗口内 SUPPRESS、count>1
时 flush 摘要、不同 key 互不影响、close() 立即 flush、on_flush 异常
被吞、同步/async on_flush 都可用。
"""
async def test_first_record_returns_emit(self):
"""
某 key 在新窗口内的首次出现必须返回 EMIT，确保调用方按原样输出。
"""
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(_TEST_WINDOW, summaries.append)
decision = await coalescer.record(("host", "reason"), payload={"i": 1})
self.assertIs(decision, CoalesceDecision.EMIT)
await coalescer.close()
async def test_subsequent_same_key_records_are_suppressed(self):
"""
同一 key 在窗口内连续命中，第 2 次起返回 SUPPRESS。
"""
coalescer = EventCoalescer(_TEST_WINDOW, lambda _s: None)
await coalescer.record("k", payload="first")
for _ in range(3):
self.assertIs(
await coalescer.record("k", payload="ignored"),
CoalesceDecision.SUPPRESS,
)
await coalescer.close()
async def test_window_expiry_flushes_summary_when_count_gt_one(self):
"""
窗口到期且 count>1 时，on_flush 收到包含 count、first_payload、window 的摘要。
"""
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(_TEST_WINDOW, summaries.append, source="test")
key = ("h", "r")
await coalescer.record(key, payload={"url": "u1"})
await coalescer.record(key, payload={"url": "u2"})
await coalescer.record(key, payload={"url": "u3"})
await asyncio.sleep(_TEST_WAIT)
self.assertEqual(len(summaries), 1)
summary = summaries[0]
self.assertEqual(summary.key, key)
self.assertEqual(summary.count, 3)
self.assertEqual(summary.first_payload, {"url": "u1"})
self.assertEqual(summary.window_seconds, _TEST_WINDOW)
async def test_window_expiry_does_not_flush_when_count_is_one(self):
"""
窗口内只出现一次时，首条 EMIT 已表达完整事件，不再补发聚合摘要。
"""
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(_TEST_WINDOW, summaries.append)
await coalescer.record("solo", payload=None)
await asyncio.sleep(_TEST_WAIT)
self.assertEqual(summaries, [])
async def test_different_keys_do_not_collapse(self):
"""
不同 key 各自独立计数与 flush，互不吞并。
"""
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(_TEST_WINDOW, summaries.append)
await coalescer.record("a", payload="a1")
await coalescer.record("b", payload="b1")
await coalescer.record("a", payload="a2")
await coalescer.record("b", payload="b2")
await coalescer.record("a", payload="a3")
await asyncio.sleep(_TEST_WAIT)
by_key = {s.key: s for s in summaries}
self.assertEqual(set(by_key.keys()), {"a", "b"})
self.assertEqual(by_key["a"].count, 3)
self.assertEqual(by_key["a"].first_payload, "a1")
self.assertEqual(by_key["b"].count, 2)
self.assertEqual(by_key["b"].first_payload, "b1")
async def test_new_window_after_flush_emits_again(self):
"""
窗口结束后下一条同 key 事件应被视为新窗口的首条，返回 EMIT。
"""
coalescer = EventCoalescer(_TEST_WINDOW, lambda _s: None)
await coalescer.record("k", payload=1)
await coalescer.record("k", payload=2)
await asyncio.sleep(_TEST_WAIT)
decision = await coalescer.record("k", payload=3)
self.assertIs(decision, CoalesceDecision.EMIT)
await coalescer.close()
async def test_close_flushes_pending_buckets_immediately(self):
"""
close() 必须取消未到期 timer 并立即触发 count>1 的 bucket flush，
用于进程退出路径。
"""
# 使用一个足够长的窗口，确保自然到期不会先于 close 触发
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(1.0, summaries.append)
await coalescer.record("k", payload="first")
await coalescer.record("k", payload="second")
await coalescer.close()
self.assertEqual(len(summaries), 1)
self.assertEqual(summaries[0].count, 2)
self.assertEqual(summaries[0].first_payload, "first")
async def test_close_does_not_emit_when_count_is_one(self):
"""
close() 与正常窗口到期一致，count==1 时不输出摘要。
"""
summaries: List[CoalesceSummary] = []
coalescer = EventCoalescer(1.0, summaries.append)
await coalescer.record("k", payload="only")
await coalescer.close()
self.assertEqual(summaries, [])
async def test_async_on_flush_is_awaited(self):
"""
on_flush 为 async 函数时应被正确 await，而不是被丢弃成协程对象。
"""
awaited: List[CoalesceSummary] = []
async def on_flush(summary: CoalesceSummary) -> None:
await asyncio.sleep(0)
awaited.append(summary)
coalescer = EventCoalescer(_TEST_WINDOW, on_flush)
await coalescer.record("k", payload="a")
await coalescer.record("k", payload="b")
await asyncio.sleep(_TEST_WAIT)
self.assertEqual(len(awaited), 1)
self.assertEqual(awaited[0].count, 2)
async def test_on_flush_exception_is_swallowed(self):
"""
on_flush 抛异常不能影响 coalescer 自身或上层调用方，仅 debug 记录。
"""
def on_flush(_summary: CoalesceSummary) -> None:
raise RuntimeError("boom")
coalescer = EventCoalescer(_TEST_WINDOW, on_flush)
await coalescer.record("k", payload="x")
await coalescer.record("k", payload="y")
await asyncio.sleep(_TEST_WAIT)
# 异常被吞，新窗口可以继续接受 record
self.assertIs(
await coalescer.record("k", payload="z"),
CoalesceDecision.EMIT,
)
await coalescer.close()
async def test_invalid_window_raises(self):
"""
非正数窗口值在构造期即拒绝，避免运行期出现 0 或负窗口的死循环 flush。
"""
with self.assertRaises(ValueError):
EventCoalescer(0, lambda _s: None)
with self.assertRaises(ValueError):
EventCoalescer(-1.0, lambda _s: None)

289
tests/test_security_image_url_log.py Normal file
View File

@@ -0,0 +1,289 @@
"""
覆盖 `SecurityUtils.is_safe_image_url_async` 的阻断分支与日志聚合接线：
- 各 `UrlSafetyReason` 分支落入正确的 warning 字段；
- `NON_GLOBAL_DNS_RESULT` 且未配置允许网段时附 fake-ip 提示；
- 签名 URL 校验通过时静默放行；URL 携带签名但失败时附 `invalid_signature` 标记；
- 同 (host, reason) 高频拦截只输出首条 warning，窗口结束输出聚合摘要；
- 不同 (host, reason) 互不吞并。
"""
import asyncio
from typing import List, Optional
from unittest import IsolatedAsyncioTestCase
from unittest.mock import patch
from app.utils import security as security_module
from app.utils.coalesce import EventCoalescer
from app.utils.security import (
SecurityUtils,
UrlSafetyDiagnosis,
UrlSafetyReason,
)
_TEST_WINDOW = 0.05
_TEST_WAIT = _TEST_WINDOW * 4
def _diag(
reason: UrlSafetyReason,
*,
host: Optional[str] = "image.tmdb.org",
ips: Optional[List[str]] = None,
) -> UrlSafetyDiagnosis:
"""
构造测试用 `UrlSafetyDiagnosis`：DOMAIN_NOT_ALLOWED 强制清空 host，保持与
`evaluate_url_safety_async` 真实输出的字段约束一致。
"""
if reason is UrlSafetyReason.DOMAIN_NOT_ALLOWED:
host = None
return UrlSafetyDiagnosis(
allowed=False,
reason=reason,
host=host,
ips=ips or [],
)
class IsSafeImageUrlLogTest(IsolatedAsyncioTestCase):
"""
`is_safe_image_url_async` 阻断路径的结构化日志 + 聚合行为校验。
"""
async def asyncSetUp(self) -> None:
# 用短窗口实例临时替换模块级 coalescer，便于在测试内驱动窗口到期 flush
self._original_coalescer = security_module._image_proxy_block_log_coalescer
self._coalescer = EventCoalescer(
window_seconds=_TEST_WINDOW,
on_flush=security_module._log_image_proxy_block_summary,
source="image_proxy_test",
)
security_module._image_proxy_block_log_coalescer = self._coalescer
self._allowed_domains = {"image.tmdb.org"}
async def asyncTearDown(self) -> None:
await self._coalescer.close()
security_module._image_proxy_block_log_coalescer = self._original_coalescer
async def _invoke(
self,
diagnosis: UrlSafetyDiagnosis,
*,
url: str = "https://image.tmdb.org/t/p/w500/x.jpg",
signed_clean_url: Optional[str] = None,
allowed_private_ranges: Optional[List[str]] = None,
):
"""
以指定诊断结果与签名校验返回值驱动 `is_safe_image_url_async`，捕获 warning。
"""
async def fake_evaluate(*_args, **_kwargs):
return diagnosis
warns: List[str] = []
with patch.object(
SecurityUtils,
"evaluate_url_safety_async",
side_effect=fake_evaluate,
), patch.object(
SecurityUtils,
"verify_signed_url",
return_value=signed_clean_url,
), patch.object(
security_module.logger,
"warn",
side_effect=warns.append,
):
allowed = await SecurityUtils.is_safe_image_url_async(
url,
self._allowed_domains,
allowed_private_ranges=allowed_private_ranges,
)
return allowed, warns
async def test_domain_not_allowed_emits_clean_reason_label(self):
"""
普通外链（未携带 mp_sig）撞 allowlist 失败时，warning 标记
DOMAIN_NOT_ALLOWED，不附 fake-ip 提示，也不挂签名失败标记，
避免误导未签名调用方以为必须签名。
"""
allowed, warns = await self._invoke(
_diag(UrlSafetyReason.DOMAIN_NOT_ALLOWED),
)
self.assertFalse(allowed)
self.assertEqual(len(warns), 1)
self.assertIn("reason=domain_not_allowed", warns[0])
self.assertIn("Blocked unsafe image URL", warns[0])
self.assertNotIn("fake-ip", warns[0])
self.assertNotIn("invalid_signature", warns[0])
async def test_invalid_signature_tag_only_when_url_signed(self):
"""
URL 显式携带 `#mp_sig=...` 但校验失败时，reason 末尾追加
`invalid_signature`，便于区分"签名失效"与"未签名外链拦截"。
"""
allowed, warns = await self._invoke(
_diag(UrlSafetyReason.DOMAIN_NOT_ALLOWED),
url="https://attacker.example.com/x.jpg#mp_sig=deadbeef&mp_purpose=image-proxy",
)
self.assertFalse(allowed)
self.assertEqual(len(warns), 1)
self.assertIn(
"reason=domain_not_allowed+invalid_signature", warns[0]
)
async def test_non_global_dns_result_lists_ips_with_hint(self):
"""
DNS 解析到非公网且未配置允许网段时，warning 列出解析 IP 并附 fake-ip 提示。
"""
allowed, warns = await self._invoke(
_diag(
UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
ips=["198.18.16.96", "198.18.16.97"],
),
)
self.assertFalse(allowed)
self.assertEqual(len(warns), 1)
warning = warns[0]
self.assertIn("reason=non_global_dns_result", warning)
self.assertIn("host=image.tmdb.org", warning)
self.assertIn("ips=198.18.16.96,198.18.16.97", warning)
self.assertIn("IMAGE_PROXY_ALLOWED_PRIVATE_RANGES", warning)
self.assertIn("198.18.0.0/15", warning)
async def test_configured_ranges_skip_fakeip_hint(self):
"""
已配置 allowed_private_ranges 时不再追加 fake-ip 提示，避免重复引导。
warning 同时把已生效的网段列在字段里供运维对照。
"""
_, warns = await self._invoke(
_diag(
UrlSafetyReason.MIXED_OR_DISALLOWED_PRIVATE_RESULT,
ips=["10.0.0.8"],
),
allowed_private_ranges=["198.18.0.0/15"],
)
self.assertEqual(len(warns), 1)
warning = warns[0]
self.assertIn("reason=mixed_or_disallowed_private_result", warning)
self.assertIn("allowed_private_ranges=198.18.0.0/15", warning)
self.assertNotIn("提示", warning)
async def test_dns_resolution_failed_carries_empty_ips(self):
"""
DNS 解析失败的 warning 携带空 ips 字段，便于运维直接定位 DNS 路径。
"""
_, warns = await self._invoke(
_diag(UrlSafetyReason.DNS_RESOLUTION_FAILED, ips=[]),
)
self.assertEqual(len(warns), 1)
self.assertIn("reason=dns_resolution_failed", warns[0])
self.assertIn("ips=,", warns[0])
async def test_signed_url_success_silently_allows(self):
"""
标准校验失败但签名 URL 校验通过时返回 True，且不输出 warning，
避免运维误判后端预签名路径是异常拦截。
"""
allowed, warns = await self._invoke(
_diag(UrlSafetyReason.DOMAIN_NOT_ALLOWED),
signed_clean_url="https://image.tmdb.org/t/p/w500/x.jpg",
)
self.assertTrue(allowed)
self.assertEqual(warns, [])
async def test_repeated_block_in_window_emits_only_first_warning(self):
"""
同 (host, reason) 在窗口内的多次命中只输出首条 warning；窗口到期后
补一条聚合摘要，count 等于窗口内总命中数，sample_url 来自首条事件。
"""
diag = _diag(
UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
ips=["198.18.16.96"],
)
async def fake_evaluate(*_args, **_kwargs):
return diag
warns: List[str] = []
with patch.object(
SecurityUtils,
"evaluate_url_safety_async",
side_effect=fake_evaluate,
), patch.object(
SecurityUtils,
"verify_signed_url",
return_value=None,
), patch.object(
security_module.logger,
"warn",
side_effect=warns.append,
):
for i in range(5):
await SecurityUtils.is_safe_image_url_async(
f"https://image.tmdb.org/t/p/w500/{i}.jpg",
self._allowed_domains,
)
self.assertEqual(len(warns), 1)
self.assertIn("/0.jpg", warns[0])
await asyncio.sleep(_TEST_WAIT)
self.assertEqual(len(warns), 2)
summary = warns[1]
self.assertIn("aggregated", summary)
self.assertIn("count=5", summary)
self.assertIn("/0.jpg", summary)
self.assertNotIn("/1.jpg", summary)
self.assertNotIn("/4.jpg", summary)
# 摘要附带首条样例的解析 IP，便于直接锁定批量拦截的网络成因
self.assertIn("sample_ips=198.18.16.96", summary)
async def test_different_keys_do_not_collapse(self):
"""
不同 (host, reason) 各自计数与输出，互不吞并。
"""
warns: List[str] = []
sequence = {
"evil": _diag(UrlSafetyReason.DOMAIN_NOT_ALLOWED, host=None),
"tmdb": _diag(
UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
host="image.tmdb.org",
ips=["198.18.16.96"],
),
}
async def fake_evaluate(url, *_args, **_kwargs):
return sequence["evil"] if "evil" in url else sequence["tmdb"]
with patch.object(
SecurityUtils,
"evaluate_url_safety_async",
side_effect=fake_evaluate,
), patch.object(
SecurityUtils,
"verify_signed_url",
return_value=None,
), patch.object(
security_module.logger,
"warn",
side_effect=warns.append,
):
await SecurityUtils.is_safe_image_url_async(
"https://evil.example.com/x.jpg",
self._allowed_domains,
)
await SecurityUtils.is_safe_image_url_async(
"https://image.tmdb.org/t/p/w500/a.jpg",
self._allowed_domains,
)
self.assertEqual(len(warns), 2)
self.assertIn("reason=domain_not_allowed", warns[0])
self.assertIn("reason=non_global_dns_result", warns[1])

170
tests/test_security_utils.py
View File

@@ -4,6 +4,8 @@ from unittest.mock import patch
from app.utils.security import (
SecurityUtils,
UrlSafetyDiagnosis,
UrlSafetyReason,
_dns_inflight_locks,
_dns_negative_cache,
_dns_positive_cache,
@@ -681,3 +683,171 @@ class SecurityUtilsTest(TestCase):
_dns_inflight_locks,
"并发等待者全部退出后必须释放 in-flight 锁字典条目",
)
class UrlSafetyDiagnosisTest(TestCase):
"""
覆盖 `evaluate_url_safety(_async)` 的结构化诊断结果，确保每条
`UrlSafetyReason` 分支返回的字段满足日志渲染契约。
"""
def setUp(self) -> None:
_dns_positive_cache.clear()
_dns_negative_cache.clear()
_dns_inflight_locks.clear()
def test_domain_not_allowed_returns_reason_and_no_host(self):
"""
协议或 allowlist 校验未通过时，诊断返回 DOMAIN_NOT_ALLOWED，
且不暴露 host/ips 字段。
"""
diag = SecurityUtils.evaluate_url_safety(
"https://attacker.example.com/x.jpg",
{"image.tmdb.org"},
)
self.assertIsInstance(diag, UrlSafetyDiagnosis)
self.assertFalse(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.DOMAIN_NOT_ALLOWED)
self.assertIsNone(diag.host)
self.assertEqual(diag.ips, [])
self.assertEqual(diag.matched_private_ranges, [])
def test_allowed_without_block_private_skips_dns(self):
"""
未启用 block_private 时直接放行，不发起 DNS 解析，ips 保持为空。
"""
with patch(
"app.utils.security.socket.getaddrinfo",
side_effect=AssertionError("不应触发 DNS 解析"),
):
diag = SecurityUtils.evaluate_url_safety(
"https://image.tmdb.org/t/p/w500/x.jpg",
{"image.tmdb.org"},
)
self.assertTrue(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.ALLOWED)
self.assertEqual(diag.host, "image.tmdb.org")
self.assertEqual(diag.ips, [])
def test_dns_resolution_failed_carries_host_without_ips(self):
"""
`block_private=True` 下 DNS 抛错时返回 DNS_RESOLUTION_FAILED，
附带 host 便于排查但不携带 ips。
"""
with patch(
"app.utils.security.socket.getaddrinfo",
side_effect=socket.gaierror,
):
diag = SecurityUtils.evaluate_url_safety(
"https://image.tmdb.org/t/p/w500/x.jpg",
{"image.tmdb.org"},
block_private=True,
)
self.assertFalse(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.DNS_RESOLUTION_FAILED)
self.assertEqual(diag.host, "image.tmdb.org")
self.assertEqual(diag.ips, [])
def test_non_global_dns_result_lists_resolved_ips(self):
"""
命中 allowlist 但 DNS 解析到非公网且未配置允许网段时，诊断标记
NON_GLOBAL_DNS_RESULT 并把解析到的 IP 列出来，供日志附带 fake-ip 提示。
"""
with patch(
"app.utils.security.socket.getaddrinfo",
return_value=[
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("198.18.16.96", 0)),
],
):
diag = SecurityUtils.evaluate_url_safety(
"https://image.tmdb.org/t/p/w500/x.jpg",
{"image.tmdb.org"},
block_private=True,
)
self.assertFalse(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.NON_GLOBAL_DNS_RESULT)
self.assertEqual(diag.host, "image.tmdb.org")
self.assertEqual(diag.ips, ["198.18.16.96"])
self.assertEqual(diag.matched_private_ranges, [])
def test_mixed_private_and_public_with_ranges_reports_mixed_reason(self):
"""
配置了 allowed_private_ranges 但解析结果存在公网或不在允许网段内的私网
地址时，诊断必须标记 MIXED_OR_DISALLOWED_PRIVATE_RESULT，避免与"未配置
允许网段"场景混淆。
"""
with patch(
"app.utils.security.socket.getaddrinfo",
return_value=[
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("198.18.16.96", 0)),
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("10.0.0.8", 0)),
],
):
diag = SecurityUtils.evaluate_url_safety(
"https://image.tmdb.org/t/p/w500/x.jpg",
{"image.tmdb.org"},
block_private=True,
allowed_private_ranges=["198.18.0.0/15"],
)
self.assertFalse(diag.allowed)
self.assertIs(
diag.reason, UrlSafetyReason.MIXED_OR_DISALLOWED_PRIVATE_RESULT
)
self.assertEqual(diag.ips, ["198.18.16.96", "10.0.0.8"])
def test_allowed_via_configured_private_range_reports_matched_networks(self):
"""
通过 allowed_private_ranges 放行时返回 ALLOWED，同时把命中的 IP 与
网段填入诊断对象，便于排查日志确认放行依据。
"""
with patch(
"app.utils.security.socket.getaddrinfo",
return_value=[
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("198.18.16.96", 0)),
],
):
diag = SecurityUtils.evaluate_url_safety(
"https://image.tmdb.org/t/p/w500/x.jpg",
{"image.tmdb.org"},
block_private=True,
allowed_private_ranges=["198.18.0.0/15"],
)
self.assertTrue(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.ALLOWED)
self.assertEqual(diag.ips, ["198.18.16.96"])
self.assertEqual(diag.matched_private_ranges, ["198.18.0.0/15"])
def test_async_evaluation_returns_same_diagnosis(self):
"""
异步版本走事件循环线程池但应保持与同步版本一致的诊断结果。
"""
import asyncio
async def fake_getaddrinfo(host, *_args, **_kwargs):
return [
(socket.AF_INET, socket.SOCK_STREAM, 0, "", ("198.18.16.96", 0)),
]
async def run():
with patch.object(
asyncio.get_running_loop(),
"getaddrinfo",
side_effect=fake_getaddrinfo,
create=True,
):
return await SecurityUtils.evaluate_url_safety_async(
"https://image.tmdb.org/x.jpg",
{"image.tmdb.org"},
block_private=True,
)
diag = asyncio.run(run())
self.assertFalse(diag.allowed)
self.assertIs(diag.reason, UrlSafetyReason.NON_GLOBAL_DNS_RESULT)
self.assertEqual(diag.ips, ["198.18.16.96"])