GithubBackup/MoviePilot
1
0
Fork 0
mirror of https://github.com/jxxghp/MoviePilot.git synced 2026-09-05 23:47:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor: reorganize backend module boundaries

Browse Source
This commit is contained in:
jxxghp
2026-08-14 19:53:33 +08:00
parent fe0b444ceb
commit 369d7d6448
584 changed files with 2423 additions and 2275 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/application/security/__init__.py
+1
View File
@@ -0,0 +1 @@
"""认证授权、URL 安全、OTP、Cookie、Passkey 和双因素认证能力。"""
app/application/security/access.py
+442
View File
@@ -0,0 +1,442 @@
import base64
import datetime
import hashlib
import hmac
import json
import os
import traceback
from datetime import timedelta
from typing import Any, Union, Annotated, Optional, Callable
import jwt
from Crypto.Cipher import AES
from Crypto.Util.Padding import pad
from cryptography.fernet import Fernet
from fastapi import HTTPException, status, Security, Request, Response
from fastapi.security import OAuth2PasswordBearer, APIKeyHeader, APIKeyQuery, APIKeyCookie, HTTPBearer
from passlib.context import CryptContext
from app import schemas
from app.runtime.cache import cached
from app.runtime.config import settings
from app.runtime.log import logger
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
ALGORITHM = "HS256"
SuperuserTokenPayloadProvider = Callable[[], schemas.TokenPayload]
_superuser_token_payload_provider: Optional[SuperuserTokenPayloadProvider] = None
def set_superuser_token_payload_provider(
provider: SuperuserTokenPayloadProvider,
) -> None:
"""注入 API 密钥认证所需的超级用户载荷提供器。"""
global _superuser_token_payload_provider
_superuser_token_payload_provider = provider
# OAuth2PasswordBearer 用于 JWT Token 认证
oauth2_scheme_manual_error = OAuth2PasswordBearer(
auto_error=False, # 禁用自动错误处理，用以支持API令牌鉴权
tokenUrl=f"{settings.API_V1_STR}/login/access-token"
)
# RESOURCE TOKEN 通过 Cookie 认证
resource_token_cookie = APIKeyCookie(name=settings.PROJECT_NAME, auto_error=False, scheme_name="resource_token_cookie")
# API TOKEN 通过 QUERY 认证
api_token_query = APIKeyQuery(name="token", auto_error=False, scheme_name="api_token_query")
# API KEY 通过 Header 认证
api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False, scheme_name="api_key_header")
# API KEY 通过 QUERY 认证
api_key_query = APIKeyQuery(name="apikey", auto_error=False, scheme_name="api_key_query")
# OpenAI compatible Bearer Token 认证
openai_bearer_scheme = HTTPBearer(auto_error=False)
# Anthropic compatible API Key 认证
anthropic_api_key_header = APIKeyHeader(name="x-api-key", auto_error=False, scheme_name="anthropic_api_key_header")
def __get_api_token(
token_query: Annotated[str | None, Security(api_token_query)] = None
) -> str | None:
"""
从 URL 查询参数中获取 API Token
:param token_query: 从 URL 中的 `token` 查询参数获取 API Token
:return: 返回获取到的 API Token，若无则返回 None
"""
return token_query
def __get_api_key(
key_query: Annotated[str | None, Security(api_key_query)] = None,
key_header: Annotated[str | None, Security(api_key_header)] = None
) -> str | None:
"""
从 URL 查询参数或请求头部获取 API Key，优先使用请求头
:param key_query: URL 中的 `apikey` 查询参数
:param key_header: 请求头中的 `X-API-KEY` 参数
:return: 返回从 URL 或请求头中获取的 API Key，若无则返回 None
"""
return key_header or key_query # 首选请求头
@cached(maxsize=1, ttl=600)
def __create_superuser_token_payload() -> schemas.TokenPayload:
"""
创建管理员用户的TokenPayload
:return: 管理员TokenPayload
"""
if not _superuser_token_payload_provider:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="认证服务尚未初始化",
)
return _superuser_token_payload_provider()
def create_access_token(
userid: Union[str, Any],
username: str,
super_user: Optional[bool] = False,
expires_delta: Optional[timedelta] = None,
level: Optional[int] = 1,
purpose: Optional[str] = "authentication"
) -> str:
"""
创建 JWT 访问令牌，包含用户 ID、用户名、是否为超级用户以及权限等级
:param userid: 用户的唯一标识符，通常是字符串或整数
:param username: 用户名，用于标识用户的账户名
:param super_user: 是否为超级用户，默认值为 False
:param expires_delta: 令牌的有效期时长，如果不提供则根据用途使用默认过期时间
:param level: 用户的权限级别，默认为 1
:param purpose: 令牌的用途，"authentication" 或 "resource"
:return: 编码后的 JWT 令牌字符串
:raises ValueError: 如果 expires_delta 为负数
"""
if purpose == "resource":
default_expire = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
secret_key = settings.RESOURCE_SECRET_KEY
else:
default_expire = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
secret_key = settings.SECRET_KEY
if expires_delta is not None:
if expires_delta.total_seconds() <= 0:
raise ValueError("过期时间必须为正数")
expire = datetime.datetime.now(datetime.UTC) + expires_delta
else:
expire = datetime.datetime.now(datetime.UTC) + default_expire
to_encode = {
"exp": expire,
"iat": datetime.datetime.now(datetime.UTC),
"sub": str(userid),
"username": username,
"super_user": super_user,
"level": level,
"purpose": purpose
}
encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM)
return encoded_jwt
def set_or_refresh_resource_token_cookie(
request: Request, response: Response, payload: schemas.TokenPayload
) -> None:
"""
设置资源令牌 Cookie
:param request: 包含请求相关的上下文数据
:param response: 用于在服务器响应时设置 Cookie
:param payload: 已通过身份验证的 TokenPayload 对象
"""
resource_token = request.cookies.get(settings.PROJECT_NAME)
if resource_token:
# 检查令牌剩余时间
try:
decoded_token = jwt.decode(resource_token, settings.RESOURCE_SECRET_KEY, algorithms=[ALGORITHM])
exp = decoded_token.get("exp")
if exp:
remaining_time = datetime.datetime.fromtimestamp(exp, tz=datetime.UTC) - datetime.datetime.now(datetime.UTC)
# 根据剩余时长提前刷新令牌
if remaining_time < timedelta(seconds=(settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS / 3)):
raise jwt.ExpiredSignatureError
expected_claims = {
"sub": str(payload.sub),
"username": payload.username,
"super_user": payload.super_user,
"level": payload.level,
"purpose": "resource",
}
if any(decoded_token.get(claim) != value for claim, value in expected_claims.items()):
raise jwt.InvalidTokenError("资源令牌身份或权限上下文不匹配")
except jwt.PyJWTError:
logger.debug(f"Token error occurred. refreshing token")
except Exception as e:
logger.debug(f"Unexpected error occurred while decoding token: {e}")
else:
# 如果令牌有效且没有即将过期，则不需要刷新
return
# 创建新的资源访问令牌
resource_token_expires = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
resource_token = create_access_token(
userid=payload.sub,
username=payload.username,
super_user=payload.super_user,
expires_delta=resource_token_expires,
level=payload.level,
purpose="resource"
)
# 判断请求是否为 HTTPS：直连协议为 https，或经反向代理转发时携带 X-Forwarded-Proto: https。
# 无法确认为明文 HTTP 时按 fail-safe 默认设置 secure=True，避免代理终止 HTTPS 后以 HTTP 转发导致 Cookie 明文传输。
is_https = (
request.url.scheme == "https"
or request.headers.get("x-forwarded-proto", "").lower() == "https"
)
# 设置会话级别的 HttpOnly Cookie
response.set_cookie(
key=settings.PROJECT_NAME,
value=resource_token,
httponly=True,
secure=is_https, # 根据当前请求协议（含反向代理转发标识）设置 secure 属性
samesite="lax" # 不同浏览器对 "Strict" 的处理可能不同，设置 SameSite 为 "Lax"，以平衡安全性和兼容性
)
def __verify_token(token: str, purpose: Optional[str] = "authentication") -> schemas.TokenPayload:
"""
使用 JWT Token 进行身份认证并解析 Token 的内容
:param token: JWT 令牌
:param purpose: 期望的令牌用途，默认为 "authentication"
:return: 包含用户身份信息的 Token 负载数据
:raises HTTPException: 如果令牌无效或用途不匹配
"""
try:
if purpose == "resource":
secret_key = settings.RESOURCE_SECRET_KEY
else:
secret_key = settings.SECRET_KEY
if not token:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"{purpose} token not found"
)
payload = jwt.decode(
token, secret_key, algorithms=[ALGORITHM]
)
token_payload = schemas.TokenPayload(**payload)
if token_payload.purpose != purpose:
raise jwt.InvalidTokenError("令牌用途不匹配")
return schemas.TokenPayload(**payload)
except (jwt.DecodeError, jwt.InvalidTokenError, jwt.ImmatureSignatureError):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="token校验不通过",
)
def verify_token(
request: Request,
response: Response,
jwt_token: Annotated[str | None, Security(oauth2_scheme_manual_error)],
api_key: Annotated[str | None, Security(__get_api_key)],
api_token: Annotated[str | None, Security(__get_api_token)],
) -> schemas.TokenPayload:
"""
验证 JWT 令牌并自动处理 resource_token 写入
如果缺少JWT令牌再尝试用API令牌鉴权
:param request: 请求对象，用于访问 Cookie 和请求信息
:param response: 响应对象，用于设置 Cookie
:param jwt_token: 从 Authorization 头部获取的 JWT 令牌
:param api_key: 从 查询参数`apikey` 或 请求头`X-API-KEY` 获取 API Token
:param api_token: 从 查询参数`token` 获取 API Token
:return: 解析后的 TokenPayload
:raises HTTPException: 如果令牌无效或用途不匹配
"""
if jwt_token:
# 验证并解析 JWT 认证令牌
payload = __verify_token(token=jwt_token, purpose="authentication")
# 如果没有 resource_token，生成并写入到 Cookie
set_or_refresh_resource_token_cookie(request, response, payload)
return payload
elif api_key:
verify_apikey(api_key)
return __create_superuser_token_payload()
elif api_token:
verify_apitoken(api_token)
return __create_superuser_token_payload()
else:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
def verify_resource_token(
resource_token: Annotated[str, Security(resource_token_cookie)]
) -> schemas.TokenPayload:
"""
验证资源访问令牌（从 Cookie 中获取）
:param resource_token: 从 Cookie 中获取的资源访问令牌
:return: 解析后的 TokenPayload
:raises HTTPException: 如果资源访问令牌无效
"""
# 验证并解析资源访问令牌
return __verify_token(token=resource_token, purpose="resource")
def __verify_key(key: str | None, expected_key: str, key_type: str) -> str:
"""
通用的 API Key 或 Token 验证函数
:param key: 从请求中获取的 API Key 或 Token
:param expected_key: 系统配置中的期望值，用于验证的 API Key 或 Token
:param key_type: 键的类型（例如 "API_KEY" 或 "API_TOKEN"），用于错误消息
:return: 返回校验通过的 API Key 或 Token
:raises HTTPException: 如果校验不通过，抛出 401 错误
"""
if not key or key != expected_key:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=f"{key_type} 校验不通过"
)
return key
def verify_apitoken(token: Annotated[str | None, Security(__get_api_token)]) -> str:
"""
使用 API Token 进行受信第三方集成认证。
校验值来自 settings.API_TOKEN；通过后只确认集成凭据有效，不生成 per-user 权限上下文。
:param token: API Token，从 URL 查询参数中获取 token=xxx
:return: 返回校验通过的 API Token
"""
return __verify_key(token, settings.API_TOKEN, "token")
def verify_apikey(apikey: Annotated[str | None, Security(__get_api_key)]) -> str:
"""
使用 API Key 形式进行受信第三方集成认证。
请求字段名兼容 API Key，实际校验值来自 settings.API_TOKEN，不生成 per-user 权限上下文。
:param apikey: API Key，从 URL 查询参数中获取 apikey=xxx，或请求头中获取 X-API-KEY=xxx
:return: 返回校验通过的 API Key
"""
return __verify_key(apikey, settings.API_TOKEN, "apikey")
def verify_password(plain_password: str, hashed_password: str) -> bool:
"""校验明文密码是否匹配已保存的密码摘要。"""
return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password: str) -> str:
"""生成适合持久化保存的密码摘要。"""
return pwd_context.hash(password)
def decrypt(data: bytes, key: bytes) -> Optional[bytes]:
"""
解密二进制数据
"""
fernet = Fernet(key)
try:
return fernet.decrypt(data)
except Exception as e:
logger.error(f"解密失败：{str(e)} - {traceback.format_exc()}")
return None
def encrypt_message(message: str, key: bytes) -> str:
"""
使用给定的key对消息进行加密，并返回加密后的字符串
"""
f = Fernet(key)
encrypted_message = f.encrypt(message.encode())
return encrypted_message.decode()
def hash_sha256(message: str) -> str:
"""
对字符串做hash运算
"""
return hashlib.sha256(message.encode()).hexdigest()
def aes_decrypt(data: str, key: str) -> str:
"""
AES解密
"""
if not data:
return ""
data = base64.b64decode(data)
iv = data[:16]
encrypted = data[16:]
# 使用AES-256-CBC解密
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv)
result = cipher.decrypt(encrypted)
# 去除填充
padding = result[-1]
if padding < 1 or padding > AES.block_size:
return ""
result = result[:-padding]
return result.decode('utf-8')
def aes_encrypt(data: str, key: str) -> str:
"""
AES加密
"""
if not data:
return ""
# 使用AES-256-CBC加密
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC)
# 填充
padding = AES.block_size - len(data) % AES.block_size
data += chr(padding) * padding
result = cipher.encrypt(data.encode('utf-8'))
# 使用base64编码
return base64.b64encode(cipher.iv + result).decode('utf-8')
def nexusphp_encrypt(data_str: str, key: bytes) -> str:
"""
NexusPHP加密
"""
# 生成16字节长的随机字符串
iv = os.urandom(16)
# 对向量进行 Base64 编码
iv_base64 = base64.b64encode(iv)
# 加密数据
cipher = AES.new(key, AES.MODE_CBC, iv)
ciphertext = cipher.encrypt(pad(data_str.encode(), AES.block_size))
ciphertext_base64 = base64.b64encode(ciphertext)
# 对向量的字符串表示进行签名
mac = hmac.new(key, msg=iv_base64 + ciphertext_base64, digestmod=hashlib.sha256).hexdigest()
# 构造 JSON 字符串
json_str = json.dumps({
'iv': iv_base64.decode(),
'value': ciphertext_base64.decode(),
'mac': mac,
'tag': ''
})
# 对 JSON 字符串进行 Base64 编码
return base64.b64encode(json_str.encode()).decode()
app/application/security/auth.py
+166
View File
@@ -0,0 +1,166 @@
import secrets
import threading
import time
from datetime import timedelta
from typing import Any, Optional
from fastapi import HTTPException, status
from app import schemas
from app.application.security import access as security
from app.runtime.config import settings
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import UserOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.schemas.types import SystemConfigKey
from app.foundation.singleton import Singleton
class AuthTicketStore(metaclass=Singleton):
"""
插件认证一次性票据存储。
"""
_ttl_seconds = 120
_max_items = 1024
def __init__(self):
"""
初始化内存票据缓存。
"""
self._tickets: dict[str, dict[str, Any]] = {}
self._lock = threading.RLock()
def create(self, user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
"""
创建短时一次性登录票据。
:param user_id: 已通过插件认证的本地用户 ID
:param provider_id: 认证提供方 ID
:param metadata: 插件侧附加信息
:return: 一次性票据字符串
"""
ticket = secrets.token_urlsafe(32)
now = time.time()
with self._lock:
self._cleanup(now)
self._tickets[ticket] = {
"user_id": int(user_id),
"provider_id": provider_id,
"metadata": metadata or {},
"created_at": now,
}
return ticket
def consume(self, ticket: str) -> Optional[dict[str, Any]]:
"""
消费并删除一次性登录票据。
:param ticket: 登录票据
:return: 票据数据，票据不存在或过期时返回 None
"""
if not ticket:
return None
now = time.time()
with self._lock:
data = self._tickets.pop(ticket, None)
self._cleanup(now)
if not data:
return None
if now - float(data.get("created_at") or 0) > self._ttl_seconds:
return None
return data
def _cleanup(self, now: Optional[float] = None) -> None:
"""
清理过期或过量的票据缓存。
:param now: 当前时间戳，未传入时自动读取
"""
current = now or time.time()
expired = [
key
for key, value in self._tickets.items()
if current - float(value.get("created_at") or 0) > self._ttl_seconds
]
for key in expired:
self._tickets.pop(key, None)
if len(self._tickets) <= self._max_items:
return
ordered = sorted(
self._tickets.items(),
key=lambda item: float(item[1].get("created_at") or 0),
)
for key, _ in ordered[: len(self._tickets) - self._max_items]:
self._tickets.pop(key, None)
def create_plugin_auth_ticket(user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
"""
为插件认证成功的用户创建一次性登录票据。
:param user_id: 本地用户 ID
:param provider_id: 认证提供方 ID
:param metadata: 插件侧附加信息
:return: 一次性票据字符串
"""
return AuthTicketStore().create(user_id=user_id, provider_id=provider_id, metadata=metadata)
def consume_plugin_auth_ticket(ticket: str) -> Optional[dict[str, Any]]:
"""
消费插件认证登录票据。
:param ticket: 登录票据
:return: 票据数据，票据不存在或过期时返回 None
"""
return AuthTicketStore().consume(ticket)
def build_superuser_token_payload() -> schemas.TokenPayload:
"""从持久化用户和站点认证状态构造超级用户令牌载荷。"""
user = UserOper().get_by_name(settings.SUPERUSER)
if not user or not user.is_superuser:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="用户权限不足",
)
return schemas.TokenPayload(
sub=user.id,
username=user.name,
super_user=user.is_superuser,
level=SitesHelper().auth_level,
purpose="authentication",
)
def build_token_response(user: User) -> schemas.Token:
"""
使用系统统一逻辑构造登录 Token 响应。
:param user: 已认证的本地用户
:return: 标准 Token 响应
"""
level = SitesHelper().auth_level
show_wizard = (
not SystemConfigOper().get(SystemConfigKey.SetupWizardState)
and not settings.ADVANCED_MODE
)
return schemas.Token(
access_token=security.create_access_token(
userid=user.id,
username=user.name,
super_user=user.is_superuser,
expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES),
level=level,
),
token_type="bearer",
super_user=user.is_superuser,
user_id=user.id,
user_name=user.name,
avatar=user.avatar,
level=level,
permissions=user.permissions or {},
wizard=show_wizard,
)
app/application/security/cookie.py
+358
View File
@@ -0,0 +1,358 @@
import base64
import time
from typing import Tuple, Optional
from lxml import etree
from app.adapters.network.browser import BrowserPage, PlaywrightHelper
from app.adapters.external.ocr import OcrHelper
from app.application.security.twofactor import TwoFactorAuth
from app.runtime.log import logger
from app.adapters.network.http import RequestUtils
from app.domain.site import SiteUtils
from app.domain.string import StringUtils
class CookieHelper:
"""处理站点登录表单、验证码和 Cookie 获取流程。"""
# 站点登录界面元素XPATH
_SITE_LOGIN_XPATH = {
"username": [
'//input[@name="username"]',
'//input[@id="form_item_username"]',
'//input[@id="username"]',
'//input[contains(@placeholder,"用户名")]',
],
"password": [
'//input[@name="password"]',
'//input[@id="form_item_password"]',
'//input[@id="password"]',
'//input[@type="password"]',
],
"captcha": [
'//input[@name="imagestring"]',
'//input[@name="captcha"]',
'//input[@id="form_item_captcha"]',
'//input[@placeholder="驗證碼"]',
],
"captcha_img": [
'//img[@alt="captcha"]/@src',
'//img[@alt="CAPTCHA"]/@src',
'//img[@alt="SECURITY CODE"]/@src',
'//img[@id="LAY-user-get-vercode"]/@src',
'//img[contains(@src,"/api/getCaptcha")]/@src',
],
"submit": [
'//input[@type="submit"]',
'//button[@type="submit"]',
'//button[@lay-filter="login"]',
'//button[@lay-filter="formLogin"]',
'//input[@type="button"][@value="登录"]',
'//input[@id="submit-btn"]',
],
"error": [
"//table[@class='main']//td[@class='text']/text()",
],
"remember": [
'//input[@type="checkbox"][contains(@name,"remember") or contains(@id,"remember")]',
'//*[@role="checkbox"][contains(.,"保持登录") or contains(.,"记住我") or contains(.,"自动登录")]',
],
"twostep": [
'//input[@name="two_step_code"]',
'//input[@name="2fa_secret"]',
'//input[@name="otp"]',
]
}
@staticmethod
def get_page_content(page: BrowserPage, retries: int = 3, interval: float = 1.0) -> Optional[str]:
"""
获取页面源码，页面跳转中（如登录前后的重定向）会导致 page.content() 抛出
"Unable to retrieve content because the page is navigating" 异常，等待加载完成后重试
:param page: 浏览器页面
:param retries: 最大重试次数
:param interval: 重试间隔（秒）
:return: 页面源码
"""
for i in range(retries):
# 等待加载失败不代表源码不可读取，最后一次等待失败时仍尝试直接获取源码
try:
page.wait_for_load_state("domcontentloaded", timeout=10 * 1000)
except Exception as e:
if i < retries - 1:
logger.warning(f"等待页面加载完成失败：{str(e)}，{interval}秒后重试 ({i + 1}/{retries - 1})")
time.sleep(interval)
continue
logger.warning(f"等待页面加载完成失败：{str(e)}，尝试直接获取源码")
try:
return page.content()
except Exception as e:
if i >= retries - 1:
logger.error(f"获取页面源码失败：{str(e)}")
return None
logger.warning(f"获取页面源码失败：{str(e)}，{interval}秒后重试 ({i + 1}/{retries - 1})")
time.sleep(interval)
return None
@staticmethod
def parse_cookies(cookies: list) -> str:
"""
将浏览器返回的cookies转化为字符串
"""
if not cookies:
return ""
cookie_str = ""
for cookie in cookies:
cookie_str += f"{cookie['name']}={cookie['value']}; "
return cookie_str
def get_site_cookie_ua(self,
url: str,
username: str,
password: str,
two_step_code: Optional[str] = None,
proxies: Optional[dict] = None,
timeout: int = None) -> Tuple[Optional[str], Optional[str], str]:
"""
获取站点cookie和ua
:param url: 站点地址
:param username: 用户名
:param password: 密码
:param two_step_code: 二步验证码或密钥
:param proxies: 代理
:param timeout: 超时时间
:return: cookie、ua、message
"""
def __page_handler(page: BrowserPage) -> Tuple[Optional[str], Optional[str], str]:
"""
页面处理
:return: Cookie和UA
"""
# 登录页面代码
html_text = self.get_page_content(page)
if not html_text:
return None, None, "获取源码失败"
# 查找用户名输入框
html = etree.HTML(html_text)
if html is None:
return None, None, "解析网页源码失败"
try:
username_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("username"):
if html.xpath(xpath):
username_xpath = xpath
break
if not username_xpath:
# 登录页可能为JS动态渲染（如SPA），等待用户名输入框出现后重试
try:
username_union_xpath = " | ".join(self._SITE_LOGIN_XPATH.get("username"))
page.wait_for_selector(f"xpath={username_union_xpath}", timeout=5000)
except Exception:
pass
html_text = self.get_page_content(page)
html = etree.HTML(html_text) if html_text else None
if html is None:
return None, None, "解析网页源码失败"
for xpath in self._SITE_LOGIN_XPATH.get("username"):
if html.xpath(xpath):
username_xpath = xpath
break
if not username_xpath:
return None, None, "未找到用户名输入框"
# 查找密码输入框
password_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("password"):
if html.xpath(xpath):
password_xpath = xpath
break
if not password_xpath:
return None, None, "未找到密码输入框"
# 处理二步验证码
otp_code = TwoFactorAuth(two_step_code).get_code()
# 查找二步验证码输入框
twostep_xpath = None
if otp_code:
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
if html.xpath(xpath):
twostep_xpath = xpath
break
# 查找验证码输入框
captcha_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("captcha"):
if html.xpath(xpath):
captcha_xpath = xpath
break
# 查找验证码图片
captcha_img_url = None
if captcha_xpath:
for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"):
if html.xpath(xpath):
captcha_img_url = html.xpath(xpath)[0]
break
if not captcha_img_url:
return None, None, "未找到验证码图片"
# 查找登录按钮
submit_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("submit"):
if html.xpath(xpath):
submit_xpath = xpath
break
if not submit_xpath:
return None, None, "未找到登录按钮"
# 点击登录按钮
try:
# 等待登录按钮准备好
page.wait_for_selector(submit_xpath)
# 输入用户名
page.fill(username_xpath, username)
# 输入密码
page.fill(password_xpath, password)
# 勾选“记住我/保持登录”等选项，获取长期会话（部分站点默认发放短期会话）
for xpath in self._SITE_LOGIN_XPATH.get("remember"):
remember_element = page.query_selector(xpath)
if not remember_element:
continue
try:
checked = remember_element.get_attribute("aria-checked")
if checked is None:
checked = "true" if remember_element.is_checked() else "false"
if checked != "true":
remember_element.click(timeout=3000)
break
except Exception as e:
# 当前候选不可操作（如隐藏元素）时继续尝试后续候选
logger.warning(f"勾选记住登录选项失败：{str(e)}，尝试下一候选")
continue
# 输入二步验证码
if twostep_xpath:
page.fill(twostep_xpath, otp_code)
# 识别验证码
if captcha_xpath and captcha_img_url:
captcha_element = page.query_selector(captcha_xpath)
if captcha_element.is_visible():
# 验证码图片地址
code_url = self.__get_captcha_url(url, captcha_img_url)
# 获取当前的cookie和ua
cookie = self.parse_cookies(page.context.cookies())
ua = page.evaluate("() => window.navigator.userAgent")
# 自动OCR识别验证码
captcha = self.__get_captcha_text(cookie=cookie, ua=ua, code_url=code_url)
if captcha:
logger.info("验证码地址为：%s，识别结果：%s" % (code_url, captcha))
else:
return None, None, "验证码识别失败"
# 输入验证码
captcha_element.fill(captcha)
else:
# 不可见元素不处理
pass
# 点击登录按钮
page.click(submit_xpath)
page.wait_for_load_state("networkidle", timeout=30 * 1000)
except Exception as e:
logger.error(f"仿真登录失败：{str(e)}")
return None, None, f"仿真登录失败：{str(e)}"
# 对于某二次验证码为单页面的站点，输入二次验证码
if "verify" in page.url:
if not otp_code:
return None, None, "需要二次验证码"
html_text = self.get_page_content(page)
if not html_text:
return None, None, "获取网页源码失败"
html = etree.HTML(html_text)
if html is None:
return None, None, "解析网页源码失败"
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
if html.xpath(xpath):
try:
# 刷新一下 2fa code
otp_code = TwoFactorAuth(two_step_code).get_code()
page.fill(xpath, otp_code)
# 登录按钮 xpath 理论上相同，不再重复查找
page.click(submit_xpath)
page.wait_for_load_state("networkidle", timeout=30 * 1000)
except Exception as e:
logger.error(f"二次验证码输入失败：{str(e)}")
return None, None, f"二次验证码输入失败：{str(e)}"
break
# 登录后的源码（部分站点登录成功后由前端脚本延迟跳转，等待并重试判定）
html_text = None
for i in range(3):
if i:
time.sleep(2)
latest_text = self.get_page_content(page)
if not latest_text:
continue
if SiteUtils.is_logged_in(latest_text):
return self.parse_cookies(page.context.cookies()), \
page.evaluate("() => window.navigator.userAgent"), ""
# 保留首个快照用于失败时解析错误信息，避免提示被后续跳转或自动消失覆盖
if html_text is None:
html_text = latest_text
# 页面已出现明确的登录错误信息时，以该快照为准并提前结束重试
latest_html = etree.HTML(latest_text)
if latest_html is not None and \
any(latest_html.xpath(x) for x in self._SITE_LOGIN_XPATH.get("error")):
html_text = latest_text
break
if not html_text:
return None, None, "获取网页源码失败"
else:
# 从登录后的页面读取错误信息
html = etree.HTML(html_text)
if html is None:
return None, None, "登录失败"
error_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("error"):
if html.xpath(xpath):
error_xpath = xpath
break
if not error_xpath:
return None, None, "登录失败"
else:
error_msg = html.xpath(error_xpath)[0]
return None, None, error_msg
finally:
if html:
del html
if not url or not username or not password:
return None, None, "参数错误"
return PlaywrightHelper().action(url=url,
callback=__page_handler,
proxies=proxies,
timeout=timeout)
@staticmethod
def __get_captcha_text(cookie: str, ua: str, code_url: str) -> str:
"""
识别验证码图片的内容
"""
if not code_url:
return ""
ret = RequestUtils(ua=ua, cookies=cookie).get_res(code_url)
if ret:
if not ret.content:
return ""
return OcrHelper().get_captcha_text(
image_b64=base64.b64encode(ret.content).decode()
)
else:
return ""
@staticmethod
def __get_captcha_url(siteurl: str, imageurl: str) -> str:
"""
获取验证码图片的URL
"""
if not siteurl or not imageurl:
return ""
if imageurl.startswith("/"):
imageurl = imageurl[1:]
return "%s/%s" % (StringUtils.get_base_url(siteurl), imageurl)
app/application/security/otp.py
+53
View File
@@ -0,0 +1,53 @@
from typing import Tuple
import pyotp
class OtpUtils:
"""提供基于 TOTP 的二次验证辅助能力。"""
@staticmethod
def generate_secret_key(username: str) -> Tuple[str, str]:
"""生成 TOTP 密钥及其配置 URI。"""
try:
secret = pyotp.random_base32()
uri = pyotp.totp.TOTP(secret).provisioning_uri(name='MoviePilot',
issuer_name='MoviePilot(' + username + ')')
return secret, uri
except Exception as err:
print(str(err))
return "", ""
@staticmethod
def is_legal(otp_uri: str, password: str) -> bool:
"""
校验二次验证是否正确
"""
try:
return pyotp.TOTP(pyotp.parse_uri(otp_uri).secret).verify(password)
except Exception as err:
print(str(err))
return False
@staticmethod
def check(secret: str, password: str) -> bool:
"""
校验二次验证是否正确
"""
try:
totp = pyotp.TOTP(secret)
return totp.verify(password)
except Exception as err:
print(str(err))
return False
@staticmethod
def get_secret(otp_uri: str) -> str:
"""
获取uri中的secret
"""
try:
return pyotp.parse_uri(otp_uri).secret
except Exception as err:
print(str(err))
return ""
app/application/security/passkey.py
+451
View File
@@ -0,0 +1,451 @@
"""
PassKey WebAuthn 辅助工具类
"""
import base64
import binascii
import json
import secrets
import threading
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Tuple
from urllib.parse import urlparse
from webauthn import (
generate_registration_options,
verify_registration_response,
generate_authentication_options,
verify_authentication_response,
options_to_json
)
from webauthn.helpers import (
parse_registration_credential_json,
parse_authentication_credential_json
)
from webauthn.helpers.structs import (
PublicKeyCredentialDescriptor,
AuthenticatorTransport,
UserVerificationRequirement,
ResidentKeyRequirement,
AuthenticatorSelectionCriteria
)
from webauthn.helpers.cose import COSEAlgorithmIdentifier
from webauthn.helpers.exceptions import InvalidRegistrationResponse
from app.runtime.cache import TTLCache
from app.runtime.config import settings
from app.adapters.cache.redis import RedisHelper
from app.runtime.log import logger
PASSKEY_CHALLENGE_TTL_SECONDS = 5 * 60
PasskeyChallengePurpose = Literal["authentication", "registration"]
@dataclass(frozen=True)
class PasskeyChallenge:
"""服务端保存的一次性 Passkey challenge 及其认证边界。"""
challenge: str
purpose: PasskeyChallengePurpose
user_id: Optional[int]
class PasskeyChallengeStore:
"""使用当前缓存后端签发并原子消费短时 Passkey challenge。"""
_cache = TTLCache(
region="passkey_challenge",
maxsize=4096,
ttl=PASSKEY_CHALLENGE_TTL_SECONDS,
)
_memory_consume_lock = threading.Lock()
@classmethod
def issue(
cls,
*,
challenge: str,
purpose: PasskeyChallengePurpose,
user_id: Optional[int],
) -> str:
"""保存 challenge 并返回不携带认证事实的随机事务 token。"""
transaction_token = secrets.token_urlsafe(32)
cls._cache.set(
transaction_token,
PasskeyChallenge(
challenge=challenge,
purpose=purpose,
user_id=user_id,
),
)
return transaction_token
@classmethod
def consume(
cls,
*,
transaction_token: str,
purpose: PasskeyChallengePurpose,
) -> Optional[PasskeyChallenge]:
"""原子领取 challenge；任何完成尝试都会使事务失效。"""
if not transaction_token:
return None
if cls._cache.is_redis():
challenge = RedisHelper().pop(
transaction_token,
region="passkey_challenge",
)
else:
with cls._memory_consume_lock:
try:
challenge = cls._cache.pop(transaction_token)
except KeyError:
challenge = None
if not isinstance(challenge, PasskeyChallenge):
return None
if challenge.purpose != purpose:
return None
return challenge
class PassKeyRegistrationVerificationError(Exception):
"""Passkey 注册响应未通过 WebAuthn 安全校验。"""
class PassKeyRegistrationOriginMismatchError(PassKeyRegistrationVerificationError):
"""浏览器来源与系统配置的 Passkey 注册来源不一致。"""
class PassKeyHelper:
"""
PassKey WebAuthn 辅助类
"""
@staticmethod
def get_rp_id() -> str:
"""
获取 Relying Party ID
"""
if settings.APP_DOMAIN:
app_domain = settings.APP_DOMAIN.strip()
# 确保存在协议前缀，以便 urlparse 正确解析主机和端口
if not app_domain.startswith(('http://', 'https://')):
app_domain = f'https://{app_domain}'
parsed = urlparse(app_domain)
host = parsed.hostname
if host:
return host
# 从 APP_DOMAIN 中提取域名
host = settings.APP_DOMAIN.replace('https://', '').replace('http://', '')
# 移除端口号
if ':' in host:
host = host.split(':')[0]
return host
# 只有在未配置 APP_DOMAIN 时，才默认为 localhost
return 'localhost'
@staticmethod
def get_rp_name() -> str:
"""
获取 Relying Party 名称
"""
return "MoviePilot"
@staticmethod
def get_origin() -> str:
"""
获取源地址
"""
if settings.APP_DOMAIN:
return settings.APP_DOMAIN.rstrip('/')
# 如果未配置APP_DOMAIN，使用默认的localhost地址
return f'http://localhost:{settings.NGINX_PORT}'
@staticmethod
def standardize_credential_id(credential_id: str) -> str:
"""
标准化凭证ID（Base64 URL Safe）
"""
try:
# Base64解码并重新编码以标准化格式
decoded = base64.urlsafe_b64decode(credential_id + '==')
return base64.urlsafe_b64encode(decoded).decode('utf-8').rstrip('=')
except (binascii.Error, TypeError, ValueError) as e:
logger.error(f"标准化凭证ID失败: {e}")
return credential_id
@staticmethod
def _base64_encode_urlsafe(data: bytes) -> str:
"""
Base64 URL Safe 编码（不带填充）
:param data: 要编码的字节数据
:return: Base64 URL Safe 编码的字符串
"""
return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')
@staticmethod
def _base64_decode_urlsafe(data: str) -> bytes:
"""
Base64 URL Safe 解码（自动添加填充）
:param data: Base64 URL Safe 编码的字符串
:return: 解码后的字节数据
"""
return base64.urlsafe_b64decode(data + '==')
@staticmethod
def _parse_credential_list(credentials: List[Dict[str, Any]]) -> List[PublicKeyCredentialDescriptor]:
"""
解析凭证列表为 PublicKeyCredentialDescriptor 列表
:param credentials: 凭证字典列表
:return: PublicKeyCredentialDescriptor 列表
"""
result = []
for cred in credentials:
try:
result.append(
PublicKeyCredentialDescriptor(
id=PassKeyHelper._base64_decode_urlsafe(cred['credential_id']),
transports=[
AuthenticatorTransport(t) for t in cred.get('transports', '').split(',') if t
] if cred.get('transports') else None
)
)
except Exception as e:
logger.warning(f"解析凭证失败: {e}")
continue
return result
@staticmethod
def _get_user_verification_requirement(user_verification: Optional[str] = None) -> UserVerificationRequirement:
"""
获取用户验证要求
:param user_verification: 指定的用户验证要求，如果不指定则从配置中读取
:return: UserVerificationRequirement
"""
if user_verification:
return UserVerificationRequirement(user_verification)
return UserVerificationRequirement.REQUIRED if settings.PASSKEY_REQUIRE_UV \
else UserVerificationRequirement.PREFERRED
@staticmethod
def _get_verification_params(
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[str, str]:
"""
获取验证参数（origin 和 rp_id）
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (origin, rp_id)
"""
origin = expected_origin or PassKeyHelper.get_origin()
rp_id = expected_rp_id or PassKeyHelper.get_rp_id()
return origin, rp_id
@staticmethod
def generate_registration_options(
user_id: int,
username: str,
display_name: Optional[str] = None,
existing_credentials: Optional[List[Dict[str, Any]]] = None
) -> Tuple[str, str]:
"""
生成注册选项
:param user_id: 用户ID
:param username: 用户名
:param display_name: 显示名称
:param existing_credentials: 已存在的凭证列表
:return: (options_json, challenge)
"""
try:
# 用户信息
user_id_bytes = str(user_id).encode('utf-8')
# 排除已有的凭证
exclude_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
if existing_credentials else None
# 用户验证要求
uv_requirement = PassKeyHelper._get_user_verification_requirement()
# 生成注册选项
options = generate_registration_options(
rp_id=PassKeyHelper.get_rp_id(),
rp_name=PassKeyHelper.get_rp_name(),
user_id=user_id_bytes,
user_name=username,
user_display_name=display_name or username,
exclude_credentials=exclude_credentials,
authenticator_selection=AuthenticatorSelectionCriteria(
authenticator_attachment=None,
resident_key=ResidentKeyRequirement.REQUIRED,
user_verification=uv_requirement,
),
supported_pub_key_algs=[
COSEAlgorithmIdentifier.ECDSA_SHA_256,
COSEAlgorithmIdentifier.RSASSA_PKCS1_v1_5_SHA_256,
]
)
# 转换为JSON
options_json = options_to_json(options)
# 提取challenge（用于后续验证）
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
return options_json, challenge
except Exception as e:
logger.error(f"生成注册选项失败: {e}")
raise
@staticmethod
def verify_registration_response(
credential: Dict[str, Any],
expected_challenge: str,
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[str, str, int, Optional[str]]:
"""
验证注册响应
:param credential: 客户端返回的凭证
:param expected_challenge: 期望的challenge
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (credential_id, public_key, sign_count, aaguid)
"""
try:
# 准备验证参数
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
# 解码challenge
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
# 构建RegistrationCredential对象
registration_credential = parse_registration_credential_json(json.dumps(credential))
# 验证注册响应
verification = verify_registration_response(
credential=registration_credential,
expected_challenge=challenge_bytes,
expected_rp_id=rp_id,
expected_origin=origin,
require_user_verification=settings.PASSKEY_REQUIRE_UV
)
# 提取信息
credential_id = PassKeyHelper._base64_encode_urlsafe(verification.credential_id)
public_key = PassKeyHelper._base64_encode_urlsafe(verification.credential_public_key)
sign_count = verification.sign_count
# aaguid 可能已经是字符串格式，也可能是bytes
if verification.aaguid:
if isinstance(verification.aaguid, bytes):
aaguid = verification.aaguid.hex()
else:
aaguid = str(verification.aaguid)
else:
aaguid = None
return credential_id, public_key, sign_count, aaguid
except InvalidRegistrationResponse as e:
logger.error(f"验证注册响应失败: {e}")
if str(e).startswith("Unexpected client data origin "):
raise PassKeyRegistrationOriginMismatchError() from e
raise PassKeyRegistrationVerificationError() from e
except Exception as e:
logger.error(f"验证注册响应失败: {e}")
raise
@staticmethod
def generate_authentication_options(
existing_credentials: Optional[List[Dict[str, Any]]] = None,
user_verification: Optional[str] = None
) -> Tuple[str, str]:
"""
生成认证选项
:param existing_credentials: 已存在的凭证列表（用于限制可用凭证）
:param user_verification: 用户验证要求，如果不指定则从配置中读取
:return: (options_json, challenge)
"""
try:
# 允许的凭证
allow_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
if existing_credentials else None
# 用户验证要求
uv_requirement = PassKeyHelper._get_user_verification_requirement(user_verification)
# 生成认证选项
options = generate_authentication_options(
rp_id=PassKeyHelper.get_rp_id(),
allow_credentials=allow_credentials,
user_verification=uv_requirement
)
# 转换为JSON
options_json = options_to_json(options)
# 提取challenge
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
return options_json, challenge
except Exception as e:
logger.error(f"生成认证选项失败: {e}")
raise
@staticmethod
def verify_authentication_response(
credential: Dict[str, Any],
expected_challenge: str,
credential_public_key: str,
credential_current_sign_count: int,
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[bool, int]:
"""
验证认证响应
:param credential: 客户端返回的凭证
:param expected_challenge: 期望的challenge
:param credential_public_key: 凭证公钥
:param credential_current_sign_count: 当前签名计数
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (验证成功, 新的签名计数)
"""
try:
# 准备验证参数
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
# 解码
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
public_key_bytes = PassKeyHelper._base64_decode_urlsafe(credential_public_key)
# 构建AuthenticationCredential对象
authentication_credential = parse_authentication_credential_json(json.dumps(credential))
# 验证认证响应
verification = verify_authentication_response(
credential=authentication_credential,
expected_challenge=challenge_bytes,
expected_rp_id=rp_id,
expected_origin=origin,
credential_public_key=public_key_bytes,
credential_current_sign_count=credential_current_sign_count,
require_user_verification=settings.PASSKEY_REQUIRE_UV
)
return True, verification.new_sign_count
except Exception as e:
logger.error(f"验证认证响应失败: {e}")
return False, credential_current_sign_count
app/application/security/twofactor.py
+48
View File
@@ -0,0 +1,48 @@
import base64
import hashlib
import hmac
import struct
import sys
import time
from app.runtime.log import logger
class TwoFactorAuth:
"""解析已有验证码或根据共享密钥生成 TOTP 验证码。"""
def __init__(self, code_or_secret: str):
"""按长度区分用户验证码和 TOTP 共享密钥。"""
if code_or_secret and len(code_or_secret) >= 16:
self.code = None
self.secret = code_or_secret
else:
self.code = code_or_secret
self.secret = None
@staticmethod
def __calc(secret_key: str) -> str:
"""按 30 秒时间窗计算六位 TOTP 验证码。"""
if not secret_key:
return ""
try:
input_time = int(time.time()) // 30
key = base64.b32decode(secret_key)
msg = struct.pack(">Q", input_time)
google_code = hmac.new(key, msg, hashlib.sha1).digest()
o = (
google_code[19] & 15
if sys.version_info > (2, 7)
else ord(str(google_code[19])) & 15
)
google_code = str(
(struct.unpack(">I", google_code[o: o + 4])[0] & 0x7FFFFFFF) % 1000000
)
return f"0{google_code}" if len(google_code) == 5 else google_code
except Exception as e:
logger.error(f"计算动态验证码失败：{str(e)}")
return ""
def get_code(self) -> str:
"""返回显式验证码，或从共享密钥实时计算。"""
return self.code or self.__calc(self.secret)
app/application/security/url.py
+983
View File
@@ -0,0 +1,983 @@
import asyncio
import hmac
import ipaddress
import socket
import threading
from dataclasses import dataclass, field
from enum import Enum
from hashlib import sha256
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Set, Union
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
from anyio import Path as AsyncPath
from cachetools import TTLCache
from app.runtime.config import settings
from app.runtime.log import logger
from app.runtime.coalesce import (
CoalesceDecision,
CoalesceSummary,
EventCoalescer,
)
# DNS 解析结果缓存。
# 正向缓存 TTL 选择 120s，短于常见 CDN / fake-ip 的 DNS TTL，避免长期持有失效 IP；
# 负向缓存 TTL 选择 15s，避免临时解析失败把目标长时间拉黑。
_DNS_CACHE_MAXSIZE = 1024
_DNS_CACHE_TTL_POSITIVE = 120
_DNS_CACHE_TTL_NEGATIVE = 15
_dns_positive_cache: "TTLCache[str, List[ipaddress._BaseAddress]]" = TTLCache(
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_POSITIVE
)
_dns_negative_cache: "TTLCache[str, bool]" = TTLCache(
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_NEGATIVE
)
# 同步路径下保护 TTLCache 读写：`cachetools.TTLCache` 本身非线程安全。
# 锁只覆盖缓存读写，不包 `getaddrinfo`，避免把 DNS 查询本身串行化。
_dns_cache_lock = threading.Lock()
# 同 hostname 的并发异步解析去重：同一 hostname 首次未命中时建立锁，
# 后续并发请求 await 同一把锁，避免对同一目标重复发起 `getaddrinfo`。
_dns_inflight_locks: Dict[str, asyncio.Lock] = {}
_dns_inflight_meta_lock = threading.Lock()
class UrlSafetyReason(str, Enum):
"""
`evaluate_url_safety` 返回的诊断原因枚举。
成员值为稳定的小写蛇形字符串，可直接作为日志字段或告警标签使用，
扩展枚举时保留既有成员的取值，避免破坏下游聚合系统对原因的归类。
"""
# 通过全部校验，URL 可被请求
ALLOWED = "allowed"
# 协议非 http/https，或 netloc 无效，或域名不在允许列表内
DOMAIN_NOT_ALLOWED = "domain_not_allowed"
# 已通过域名 allowlist，但 DNS 解析失败（无返回或抛错）
DNS_RESOLUTION_FAILED = "dns_resolution_failed"
# DNS 解析到至少一个非公网地址，且未配置 `allowed_private_ranges`
NON_GLOBAL_DNS_RESULT = "non_global_dns_result"
# 配置了 `allowed_private_ranges`，但仍存在不在允许网段内的解析结果
MIXED_OR_DISALLOWED_PRIVATE_RESULT = "mixed_or_disallowed_private_result"
@dataclass(frozen=True)
class UrlSafetyDiagnosis:
"""
URL 安全校验的结构化诊断结果，由 `evaluate_url_safety(_async)` 返回。
`is_safe_url` 仅使用 `allowed` 字段；日志、告警、运维诊断需要细分原因或
解析 IP 时通过本对象消费。字段约束：
- `host` 仅在通过域名 allowlist 后才被填充；DOMAIN_NOT_ALLOWED 场景为 None。
- `ips` 仅在执行过 DNS 阶段后才可能非空；不含纯字符串协议失败场景。
- `matched_private_ranges` 仅在通过 `allowed_private_ranges` 放行时填充。
"""
# 是否放行
allowed: bool
# 放行/拦截的具体原因
reason: UrlSafetyReason
# 通过 allowlist 后从 URL 解析出的 hostname，未通过时为 None
host: Optional[str] = None
# DNS 解析结果（含命中或未命中私网放行的 IP），格式化为字符串
ips: List[str] = field(default_factory=list)
# 命中允许放行的非公网网段，仅 `ALLOWED` 且走私网放行分支时非空
matched_private_ranges: List[str] = field(default_factory=list)
def _resolve_addrinfo_to_ips(
address_infos: Iterable,
) -> Optional[List[ipaddress._BaseAddress]]:
"""
将 `socket.getaddrinfo` 返回的结果归一化为 IP 列表。
任一条目无法解析为 IP 即视为异常情况，整体返回 None 让上层按"不安全目标"
处理，避免出现"部分 IP 漏校验"的情况。
"""
addresses: List[ipaddress._BaseAddress] = []
for address_info in address_infos:
try:
addresses.append(ipaddress.ip_address(address_info[4][0]))
except ValueError:
return None
return addresses or None
class SecurityUtils:
"""提供路径、URL、签名和网络目标安全校验能力。"""
_SIGNED_URL_PURPOSE = "image-proxy"
_SUBTITLE_DOWNLOAD_PURPOSE_PREFIX = "subtitle-download"
@staticmethod
def is_safe_path(base_path: Path, user_path: Path,
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
"""
验证用户提供的路径是否在基准目录内，并检查文件类型是否合法，防止目录遍历攻击
:param base_path: 基准目录，允许访问的根目录
:param user_path: 用户提供的路径，需检查其是否位于基准目录内
:param allowed_suffixes: 允许的文件后缀名集合，用于验证文件类型
:return: 如果用户路径安全且位于基准目录内，且文件类型合法，返回 True；否则返回 False
:raises Exception: 如果解析路径时发生错误，则捕获并记录异常
"""
try:
# resolve() 将相对路径转换为绝对路径，并处理符号链接和'..'
base_path_resolved = base_path.resolve()
user_path_resolved = user_path.resolve()
# 检查用户路径是否在基准目录或基准目录的子目录内
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
return False
if allowed_suffixes is not None:
allowed_suffixes = set(allowed_suffixes)
if user_path.suffix.lower() not in allowed_suffixes:
return False
return True
except Exception as e:
logger.debug(f"Error occurred while validating paths: {e}")
return False
@staticmethod
async def async_is_safe_path(base_path: AsyncPath, user_path: AsyncPath,
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
"""
异步验证用户提供的路径是否在基准目录内，并检查文件类型是否合法，防止目录遍历攻击
:param base_path: 基准目录，允许访问的根目录
:param user_path: 用户提供的路径，需检查其是否位于基准目录内
:param allowed_suffixes: 允许的文件后缀名集合，用于验证文件类型
:return: 如果用户路径安全且位于基准目录内，且文件类型合法，返回 True；否则返回 False
:raises Exception: 如果解析路径时发生错误，则捕获并记录异常
"""
try:
# resolve() 将相对路径转换为绝对路径，并处理符号链接和'..'
base_path_resolved = await base_path.resolve()
user_path_resolved = await user_path.resolve()
# 检查用户路径是否在基准目录或基准目录的子目录内
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
return False
if allowed_suffixes is not None:
allowed_suffixes = set(allowed_suffixes)
if user_path.suffix.lower() not in allowed_suffixes:
return False
return True
except Exception as e:
logger.debug(f"Error occurred while validating paths: {e}")
return False
@staticmethod
def _literal_ip(hostname: str) -> Optional[ipaddress._BaseAddress]:
"""
若 hostname 是字面量 IP（含 IPv6 的 `[::1]` 形式）则返回 IP 对象，否则 None。
"""
if not hostname:
return None
candidate = hostname
if candidate.startswith("[") and candidate.endswith("]"):
candidate = candidate[1:-1]
try:
return ipaddress.ip_address(candidate)
except ValueError:
return None
@staticmethod
def _cache_lookup(hostname: str) -> tuple[bool, Optional[List[ipaddress._BaseAddress]]]:
"""
在 TTL 缓存中查找 hostname，返回 (是否命中, 命中值)。
命中值为 `None` 表示命中负向缓存（先前解析失败）。
"""
with _dns_cache_lock:
cached = _dns_positive_cache.get(hostname)
if cached is not None:
return True, cached
if hostname in _dns_negative_cache:
return True, None
return False, None
@staticmethod
def _cache_store(
hostname: str, addresses: Optional[List[ipaddress._BaseAddress]]
) -> None:
"""
将解析结果写入对应的正向/负向缓存。
"""
with _dns_cache_lock:
if addresses is None:
_dns_negative_cache[hostname] = True
else:
_dns_positive_cache[hostname] = addresses
@staticmethod
def _hostname_addresses(hostname: str) -> Optional[List[ipaddress._BaseAddress]]:
"""
同步解析主机名并返回全部 IP 地址，结果走 TTL 缓存。
字面量 IP 直接返回自身；DNS 解析失败或结果异常时返回 None，由上层按
不安全目标处理。async 调用方应使用 `_hostname_addresses_async`。
"""
if not hostname:
return None
literal = SecurityUtils._literal_ip(hostname)
if literal is not None:
return [literal]
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
try:
address_infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
except socket.gaierror:
SecurityUtils._cache_store(hostname, None)
return None
addresses = _resolve_addrinfo_to_ips(address_infos)
SecurityUtils._cache_store(hostname, addresses)
return addresses
@staticmethod
def _get_inflight_lock(hostname: str) -> asyncio.Lock:
"""
取得 hostname 对应的 in-flight 锁，不存在则按需创建。
用 `threading.Lock` 保护字典写入，避免多个事件循环线程并发创建出多把锁
破坏去重语义；锁本身是 `asyncio.Lock`，归属当前事件循环。
"""
with _dns_inflight_meta_lock:
lock = _dns_inflight_locks.get(hostname)
if lock is None:
lock = asyncio.Lock()
_dns_inflight_locks[hostname] = lock
return lock
@staticmethod
def _release_inflight_lock(hostname: str, lock: asyncio.Lock) -> None:
"""
请求结束后清理 in-flight 锁，避免长期持有大量已闲置的 `asyncio.Lock`。
仅当字典中登记的仍是当前 lock，且 `lock.locked()` 为 False 时才删除。
`asyncio.Lock` 公平 FIFO：持有者释放后若仍有等待者，锁会立刻被下一个
等待者接走、`locked()` 重新变为 True，因此该守卫可同时排除"仍有持有者"
与"刚被等待者接走"两种情况，避免误删后续协程仍在使用的字典条目。
"""
with _dns_inflight_meta_lock:
current = _dns_inflight_locks.get(hostname)
if current is lock and not lock.locked():
_dns_inflight_locks.pop(hostname, None)
@staticmethod
async def _hostname_addresses_async(
hostname: str,
) -> Optional[List[ipaddress._BaseAddress]]:
"""
异步解析主机名并返回全部 IP 地址，与同步版本共用同一份 TTL 缓存。
通过事件循环的默认线程池执行 `getaddrinfo`，不阻塞 asyncio 事件循环；
同 hostname 的并发未命中请求通过 in-flight 锁去重，只发起一次 DNS 查询。
"""
if not hostname:
return None
literal = SecurityUtils._literal_ip(hostname)
if literal is not None:
return [literal]
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
lock = SecurityUtils._get_inflight_lock(hostname)
try:
async with lock:
# 等到锁后再查一次缓存，前一个持锁者可能已经回填结果
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
loop = asyncio.get_running_loop()
try:
address_infos = await loop.getaddrinfo(
hostname, None, type=socket.SOCK_STREAM
)
except socket.gaierror:
SecurityUtils._cache_store(hostname, None)
return None
addresses = _resolve_addrinfo_to_ips(address_infos)
SecurityUtils._cache_store(hostname, addresses)
return addresses
finally:
# 必须在 `async with` 释放锁之后再清理字典：`_release_inflight_lock`
# 以 `not lock.locked()` 为清理守卫，持锁状态下调用会跳过 pop。
SecurityUtils._release_inflight_lock(hostname, lock)
@staticmethod
def _addresses_all_global(
addresses: Optional[List[ipaddress._BaseAddress]],
) -> bool:
"""
判断解析结果是否全部为公网地址（空列表/None 视为非公网）。
"""
if not addresses:
return False
return all(address.is_global for address in addresses)
@staticmethod
def _is_global_hostname(hostname: str) -> bool:
"""
判断主机名解析结果是否全部为公网地址（同步版本）。
图片代理会访问用户可控的 URL，这里必须在 allowlist 命中前后都排除
私有、回环、链路本地、保留地址等非公网目标，避免通过 DNS 或字面量 IP
绕过域名白名单访问内网服务。
"""
return SecurityUtils._addresses_all_global(
SecurityUtils._hostname_addresses(hostname)
)
@staticmethod
async def _is_global_hostname_async(hostname: str) -> bool:
"""
判断主机名解析结果是否全部为公网地址（异步版本）。语义与 `_is_global_hostname` 一致。
"""
return SecurityUtils._addresses_all_global(
await SecurityUtils._hostname_addresses_async(hostname)
)
@staticmethod
def _parse_ip_networks(ranges: Optional[Iterable[str]]) -> List[ipaddress._BaseNetwork]:
"""
解析用户配置的 IP/CIDR 网段。
配置错误的条目会被忽略并写入 debug 日志，避免单个无效值导致所有图片代理
校验失败。调用方仍然需要先完成域名白名单匹配，不能单独依赖该网段放行。
"""
networks = []
for value in ranges or []:
if not value:
continue
try:
networks.append(ipaddress.ip_network(str(value).strip(), strict=False))
except ValueError:
logger.debug(f"忽略无效的图片代理允许网段配置: {value}")
return networks
@staticmethod
def _match_private_addresses(
addresses: Optional[List[ipaddress._BaseAddress]],
networks: List[ipaddress._BaseNetwork],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
在已解析出的地址列表中匹配显式允许的非公网网段。
所有解析地址都必须命中至少一个允许网段才放行；只要有一个 IP 落在允许
网段外（或解析结果是全公网），就视为不匹配私网放行规则。
"""
if not addresses or not networks:
return None
if all(address.is_global for address in addresses):
return None
matched_networks: List[ipaddress._BaseNetwork] = []
for address in addresses:
matched_for_address = [
network for network in networks if address in network
]
if not matched_for_address:
return None
matched_networks.extend(matched_for_address)
return addresses, list(dict.fromkeys(matched_networks))
@staticmethod
def _is_allowed_private_hostname(
hostname: str,
allowed_private_ranges: Optional[Iterable[str]],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
返回主机名命中的显式允许非公网地址和网段（同步版本）。
该能力只用于图片代理的受控例外，例如 TUN fake-ip 或内网 CDN。必须由
`is_safe_url` 先完成域名 allowlist 校验后再调用，避免把任意用户 URL
变成 SSRF 绕过入口。
"""
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return None
return SecurityUtils._match_private_addresses(
SecurityUtils._hostname_addresses(hostname), networks
)
@staticmethod
async def _is_allowed_private_hostname_async(
hostname: str,
allowed_private_ranges: Optional[Iterable[str]],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
`_is_allowed_private_hostname` 的异步版本，语义保持一致。
"""
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return None
return SecurityUtils._match_private_addresses(
await SecurityUtils._hostname_addresses_async(hostname), networks
)
@staticmethod
def _url_signature_payload(url: str, purpose: str) -> bytes:
"""
构造 URL 签名载荷。
签名覆盖用途与完整 URL，确保同一个签名不能挪用到其它代理用途或其它 URL。
"""
return f"{purpose}\n{url}".encode("utf-8")
@staticmethod
def _sign_url_payload(url: str, purpose: str) -> str:
"""
使用 RESOURCE_SECRET_KEY 对 URL 签名载荷生成 HMAC。
相同 `(url, purpose, RESOURCE_SECRET_KEY)` 组合在进程生命周期内输出
完全一致；签名的失效边界绑定在 `RESOURCE_SECRET_KEY` 上，进程重启
或显式轮换密钥时所有旧签名一起作废。
"""
return hmac.new(
settings.RESOURCE_SECRET_KEY.encode("utf-8"),
SecurityUtils._url_signature_payload(url, purpose),
sha256,
).hexdigest()
@staticmethod
def strip_url_signature(url: str) -> str:
"""
移除 URL fragment 中的资源签名信息，得到真正要请求的地址。
签名放在 fragment 中，浏览器会把它传给 MoviePilot，但 HTTP 客户端
请求外部资源前不能把这些内部参数带过去。
"""
if not url:
return url
parsed_url = urlparse(url)
return urlunparse(parsed_url._replace(fragment=""))
@staticmethod
def subtitle_download_purpose(site_id: int) -> str:
"""
构造字幕下载 URL 签名用途，签名必须绑定站点 ID，避免跨站点复用。
"""
return f"{SecurityUtils._SUBTITLE_DOWNLOAD_PURPOSE_PREFIX}:{site_id}"
@staticmethod
def sign_url(
url: str,
purpose: str = _SIGNED_URL_PURPOSE,
) -> str:
"""
给服务端返回的资源 URL 添加稳定签名。
签名作为后端资源能力凭证：外部请求边界可以用不同 `purpose` 绑定
具体业务语义，避免一个场景签出的 URL 被挪用到另一个场景。
签名为 `(url, purpose, RESOURCE_SECRET_KEY)` 的确定性 HMAC，**不带
过期时间**：相同 URL 多次调用结果完全一致，让浏览器与 Service Worker
的缓存能稳定命中；失效边界由 `RESOURCE_SECRET_KEY` 控制——进程重启
自动重生成、或者运维显式轮换后所有历史签名一起作废。
"""
if not url:
return url
parsed_url = urlparse(url)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
return url
clean_url = SecurityUtils.strip_url_signature(url)
signature = SecurityUtils._sign_url_payload(clean_url, purpose)
fragment = urlencode(
{
"mp_sig": signature,
"mp_purpose": purpose,
}
)
return urlunparse(urlparse(clean_url)._replace(fragment=fragment))
@staticmethod
def verify_signed_url(
url: str,
purpose: str = _SIGNED_URL_PURPOSE,
) -> Optional[str]:
"""
验证 URL fragment 中的资源签名，成功时返回去签名后的真实 URL。
签名只校验 `(url, purpose, RESOURCE_SECRET_KEY)`，密钥轮换/进程重启
后旧签名自动失效。
"""
if not url:
return None
parsed_url = urlparse(url)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
return None
fragment_params = dict(parse_qsl(parsed_url.fragment, keep_blank_values=True))
signature = fragment_params.get("mp_sig")
signed_purpose = fragment_params.get("mp_purpose")
if not signature or signed_purpose != purpose:
return None
clean_url = SecurityUtils.strip_url_signature(url)
expected_signature = SecurityUtils._sign_url_payload(clean_url, purpose)
if not hmac.compare_digest(signature, expected_signature):
return None
return clean_url
@staticmethod
def _check_url_allowlist(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool,
) -> Optional[str]:
"""
执行"协议 + netloc + 域名白名单"前置校验，命中返回 hostname，未命中返回 None。
DNS 校验（SSRF 防御）由调用方自行接续，本方法不发起 DNS 查询。
"""
try:
parsed_url = urlparse(url)
except Exception as e: # noqa: BLE001 - 任何解析异常都视为不安全 URL
logger.debug(f"Error occurred while validating URL: {e}")
return None
# 如果 URL 没有包含有效的 scheme，或者无法从中提取到有效的 netloc，则认为该 URL 是无效的
if not parsed_url.scheme or not parsed_url.netloc:
return None
# 仅允许 http 或 https 协议
if parsed_url.scheme not in {"http", "https"}:
return None
# 获取完整的 netloc（包括 IP 和端口）并转换为小写
netloc = parsed_url.netloc.lower()
if not netloc:
return None
# 检查每个允许的域名
normalized_allowed = {d.lower() for d in allowed_domains}
domain_allowed = False
for domain in normalized_allowed:
parsed_allowed_url = urlparse(domain)
allowed_netloc = parsed_allowed_url.netloc or parsed_allowed_url.path
if strict:
# 严格模式下，要求完全匹配域名和端口
if netloc == allowed_netloc:
domain_allowed = True
break
else:
# 非严格模式下，允许子域名匹配
if netloc == allowed_netloc or netloc.endswith("." + allowed_netloc):
domain_allowed = True
break
if not domain_allowed:
return None
return parsed_url.hostname or ""
@staticmethod
def _log_private_range_allowed(
url: str,
match: tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]],
) -> None:
"""
记录"图片代理允许访问配置的非公网网段"放行日志，便于运维排查。
"""
addresses, matched_networks = match
logger.debug(
"图片代理允许访问配置的非公网网段: "
f"url={url}, ips={','.join(map(str, addresses))}, "
f"ranges={','.join(map(str, matched_networks))}"
)
@staticmethod
def is_safe_url(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
验证 URL 是否在允许的域名列表中，包括带有端口的域名（同步版本）。
:param url: 需要验证的 URL
:param allowed_domains: 允许的域名集合，域名可以包含端口
:param strict: 是否严格匹配一级域名（默认 False，允许多级域名）
:param block_private: 是否拦截解析到非公网地址的 URL，防止 SSRF
:param allowed_private_ranges: 域名命中后额外允许的非公网 IP/CIDR 网段
:return: URL 合法且通过安全校验时返回 True，否则返回 False
校验细节与失败原因由 `evaluate_url_safety` 返回；本方法只暴露布尔结果，
作为只关心通过/拒绝判断的调用方的最薄入口。`block_private=True` 时会
同步调用 `getaddrinfo`；async 上下文请改用 `is_safe_url_async`。
"""
return SecurityUtils.evaluate_url_safety(
url,
allowed_domains,
strict=strict,
block_private=block_private,
allowed_private_ranges=allowed_private_ranges,
).allowed
@staticmethod
async def is_safe_url_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
判定 URL 是否在允许的域名列表中，包括带有端口的域名。
DNS 解析通过事件循环线程池执行，并复用 TTL 缓存，不阻塞调用方所在的
事件循环。参数与返回值含义同 `is_safe_url`；需要失败原因/解析 IP
等结构化信息时调用 `evaluate_url_safety_async`。
"""
diagnosis = await SecurityUtils.evaluate_url_safety_async(
url,
allowed_domains,
strict=strict,
block_private=block_private,
allowed_private_ranges=allowed_private_ranges,
)
return diagnosis.allowed
@staticmethod
def evaluate_url_safety(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> "UrlSafetyDiagnosis":
"""
在 `is_safe_url` 的判定路径上输出结构化诊断结果（同步版本）。
与 `is_safe_url` 共用同一套校验顺序：协议/域名 allowlist → 可选 DNS 解析
→ 可选非公网放行匹配；本方法额外返回失败原因、解析到的 IP 列表和命中的
私网网段，供日志与告警渲染消费。校验中遇到未预期异常时按默认拒绝原则
归类为 `DOMAIN_NOT_ALLOWED`，避免任何解析路径漏过 SSRF 校验。
"""
try:
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
if hostname is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
if not block_private:
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
)
addresses = SecurityUtils._hostname_addresses(hostname)
return SecurityUtils._diagnose_resolved_addresses(
url, hostname, addresses, allowed_private_ranges
)
except Exception as e: # noqa: BLE001 - 默认拒绝，避免漏过 SSRF 校验
logger.debug(f"Error occurred while validating URL: {e}")
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
@staticmethod
async def evaluate_url_safety_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> "UrlSafetyDiagnosis":
"""
输出与 `evaluate_url_safety` 完全一致的结构化诊断结果。
DNS 解析通过事件循环线程池执行，并复用 TTL 缓存，不阻塞调用方所在的
事件循环；校验顺序、字段含义、异常归类均与同步版本相同。
"""
try:
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
if hostname is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
if not block_private:
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
)
addresses = await SecurityUtils._hostname_addresses_async(hostname)
return SecurityUtils._diagnose_resolved_addresses(
url, hostname, addresses, allowed_private_ranges
)
except Exception as e: # noqa: BLE001 - 默认拒绝，避免漏过 SSRF 校验
logger.debug(f"Error occurred while validating URL: {e}")
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
@staticmethod
async def is_safe_image_url_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
判定 URL 是否可作为图片代理请求目标。
校验顺序：协议 + 域名 allowlist + DNS SSRF 拦截 + 非公网放行匹配；标准
校验失败时再用 `verify_signed_url` 兜底，允许后端预签名的媒体服务器
URL 跳过私网拦截。两者皆失败才视为拒绝。
拒绝路径会输出结构化阻断日志：单次拦截立即打印一条 warning，同
`(host, reason)` 的连续命中在 `_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS`
窗口内合并为一条聚合摘要，避免媒体详情页一次请求把日志刷爆。日志字段
范围严格限定为 URL、host、reason、解析 IP 与允许网段配置；cookies、
签名串、token、请求头等敏感材料一律不进入日志。
"""
diagnosis = await SecurityUtils.evaluate_url_safety_async(
url,
allowed_domains,
block_private=True,
allowed_private_ranges=allowed_private_ranges,
)
if diagnosis.allowed:
return True
if SecurityUtils.verify_signed_url(url) is not None:
return True
await _emit_image_proxy_block_warning(
url=url,
diagnosis=diagnosis,
signature_carried=_url_carries_signature(url),
allowed_private_ranges=allowed_private_ranges,
)
return False
@staticmethod
def _diagnose_resolved_addresses(
url: str,
hostname: str,
addresses: Optional[List[ipaddress._BaseAddress]],
allowed_private_ranges: Optional[Iterable[str]],
) -> "UrlSafetyDiagnosis":
"""
对已完成 DNS 解析的地址列表执行非公网放行判断，并归一化诊断结果。
- 地址列表为空/None：视为 DNS 不可信，拒绝并标记 `DNS_RESOLUTION_FAILED`。
- 全部公网地址：直接放行。
- 存在非公网地址且未配置允许网段：拒绝并标记 `NON_GLOBAL_DNS_RESULT`，
供日志附带"如使用 fake-ip 需要配置 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
的提示。
- 存在非公网地址且配置了允许网段但未全部命中：拒绝并标记
`MIXED_OR_DISALLOWED_PRIVATE_RESULT`，提示存在不允许的解析结果。
- 全部命中允许网段：放行并附带命中的 IP 与网段，由
`_log_private_range_allowed` 输出排查日志。
"""
if not addresses:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DNS_RESOLUTION_FAILED,
host=hostname,
)
if SecurityUtils._addresses_all_global(addresses):
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
ips=[str(addr) for addr in addresses],
)
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
host=hostname,
ips=[str(addr) for addr in addresses],
)
match = SecurityUtils._match_private_addresses(addresses, networks)
if match is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.MIXED_OR_DISALLOWED_PRIVATE_RESULT,
host=hostname,
ips=[str(addr) for addr in addresses],
)
matched_addresses, matched_networks = match
SecurityUtils._log_private_range_allowed(url, match)
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
ips=[str(addr) for addr in matched_addresses],
matched_private_ranges=[str(net) for net in matched_networks],
)
@staticmethod
def sanitize_url_path(url: str, max_length: int = 120) -> str:
"""
将 URL 的路径部分进行编码，确保合法字符，并对路径长度进行压缩处理（如果超出最大长度）
:param url: 需要处理的 URL
:param max_length: 路径允许的最大长度，超出时进行压缩
:return: 处理后的路径字符串
"""
# 解析 URL，获取路径部分
parsed_url = urlparse(url)
path = parsed_url.path.lstrip("/")
# 对路径中的特殊字符进行编码
safe_path = quote(path)
# 如果路径过长，进行压缩处理
if len(safe_path) > max_length:
# 使用 SHA-256 对路径进行哈希，取前 16 位作为压缩后的路径
hash_value = sha256(safe_path.encode()).hexdigest()[:16]
# 使用哈希值代替过长的路径，同时保留文件扩展名
file_extension = Path(safe_path).suffix.lower() if Path(safe_path).suffix else ""
safe_path = f"compressed_{hash_value}{file_extension}"
return safe_path
# 图片代理阻断日志聚合窗口（秒）。媒体详情页一次请求会批量触发同 host/同原因的拦截，
# 按 (host, reason) 合并后只输出首条 warning + 窗口结束的聚合摘要，避免日志刷屏。
_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS = 60.0
# fake-ip / 旁路 DNS 用户最常因 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES 未配置而踩坑，
# 在 reason=NON_GLOBAL_DNS_RESULT 且当前未配置允许网段时随 warning 一起输出，指向正确的修复开关。
_IMAGE_PROXY_FAKEIP_HINT = (
"提示：若使用 fake-ip / 旁路 DNS（常见网段 198.18.0.0/15、100.64.0.0/10），"
"请将对应网段加入 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
)
# URL fragment 中实际携带代理签名但校验失败时附在 reason 末尾的标记。
# 仅起标识作用，签名串本身不写入日志，避免泄露签名材料。
_INVALID_SIGNATURE_TAG = "invalid_signature"
def _url_carries_signature(url: str) -> bool:
"""
判断 URL 是否在 fragment 中显式携带代理签名参数 `mp_sig`。
仅做轻量字符串匹配，避免对普通图片 URL 跑完整签名校验路径；未携带签名
的外链不会触发 `invalid_signature` 标记，避免阻断日志误导未签名调用方。
"""
if not url:
return False
fragment_start = url.find("#")
if fragment_start < 0:
return False
return "mp_sig=" in url[fragment_start + 1:]
def _format_image_proxy_block_warning(
*,
url: str,
reason: str,
host: Optional[str],
ips: List[str],
allowed_private_ranges: List[str],
hint: Optional[str],
) -> str:
"""
渲染图片代理首条阻断 warning 文案。
字段范围严格限定为 URL、host、reason、IP 与允许网段配置；hint 仅在
reason 与配置缺失同时满足时由调用方填充。其余敏感材料（cookies、签名
串、token、请求头）不允许进入该日志路径。
"""
fields = [
f"url={url}",
f"reason={reason}",
f"host={host or ''}",
f"ips={','.join(ips)}",
f"allowed_private_ranges={','.join(allowed_private_ranges)}",
]
line = "Blocked unsafe image URL: " + ", ".join(fields)
if hint:
line = f"{line} | {hint}"
return line
def _log_image_proxy_block_summary(summary: CoalesceSummary) -> None:
"""
图片代理阻断日志聚合窗口到期回调，输出窗口内的命中计数与首条样例。
summary.key 由 `_emit_image_proxy_block_warning` 固定构造为
`(host, reason_label)` 二元组；摘要保留首条事件的 URL 与解析 IP，
避免运维只看到 count 而无法定位是哪批请求被合并。
"""
host, reason = summary.key
payload = summary.first_payload or {}
sample_ips = ",".join(payload.get("ips") or [])
logger.warn(
"Blocked unsafe image URL (aggregated): "
f"host={host or ''}, reason={reason}, "
f"count={summary.count}, window={summary.window_seconds:g}s, "
f"sample_url={payload.get('url', '')}, sample_ips={sample_ips}"
)
# 图片代理阻断日志聚合器。同 (host, reason) 高频拦截在窗口内合并为一条聚合摘要，避免媒体详情页一次请求把日志刷爆；
# 放行 debug 日志与诊断布尔结果不受聚合影响。
_image_proxy_block_log_coalescer = EventCoalescer(
window_seconds=_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS,
on_flush=_log_image_proxy_block_summary,
source="image_proxy",
)
async def _emit_image_proxy_block_warning(
*,
url: str,
diagnosis: "UrlSafetyDiagnosis",
signature_carried: bool,
allowed_private_ranges: Optional[Iterable[str]],
) -> None:
"""
把诊断结果转写为结构化阻断 warning，并交由 coalescer 决定是否实际输出。
`signature_carried=True` 表示请求 URL 在 fragment 里实际携带了代理签名但
校验失败，此时在 reason 末尾追加 `invalid_signature` 标记，便于区分
"未签名外链直接撞 allowlist"与"签名 URL 已失效"两种排查路径。
"""
# reason_label 既作为 warning 字段，也作为 coalescer 桶键的一部分；签名
# 标记拼接到同一字符串里是为了让"带签名失败"的命中与"裸 URL 失败"分桶，
# 各自独立计数与摘要，不要在不引入新桶维度的情况下拆开。
reason_label = diagnosis.reason.value
if signature_carried:
reason_label = f"{reason_label}+{_INVALID_SIGNATURE_TAG}"
allowed_ranges = [str(r) for r in (allowed_private_ranges or [])]
hint = (
_IMAGE_PROXY_FAKEIP_HINT
if diagnosis.reason is UrlSafetyReason.NON_GLOBAL_DNS_RESULT
and not allowed_ranges
else None
)
key = (diagnosis.host or "", reason_label)
payload = {"url": url, "ips": list(diagnosis.ips)}
decision = await _image_proxy_block_log_coalescer.record(key=key, payload=payload)
if decision is CoalesceDecision.EMIT:
logger.warn(
_format_image_proxy_block_warning(
url=url,
reason=reason_label,
host=diagnosis.host,
ips=list(diagnosis.ips),
allowed_private_ranges=allowed_ranges,
hint=hint,
)
)