refactor: reorganize backend module boundaries

This commit is contained in:
jxxghp
2026-08-14 19:53:33 +08:00
parent fe0b444ceb
commit 369d7d6448
584 changed files with 2423 additions and 2275 deletions

View File

@@ -0,0 +1 @@
"""编排领域对象和基础设施的应用服务。"""

317
app/application/audio.py Normal file
View File

@@ -0,0 +1,317 @@
from pathlib import Path
from typing import Any, Optional, Union
from uuid import UUID
from mutagen import File as MutagenFile
from mutagen.flac import FLAC, Picture
from mutagen.id3 import APIC
from mutagen.mp4 import MP4, MP4Cover
from app.domain.context import MusicInfo
from app.domain.meta.metamusic import MetaMusic
from app.runtime.log import logger
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource
class AudioMetadataHelper:
"""读取和写入音频标签,并转换为标准音乐元数据。"""
@classmethod
def read(cls, path: Path) -> MetaMusic:
"""读取本地音频标签,并以完整文件名模式和目录线索补充缺失字段。"""
tag_meta = cls.read_tags(path)
if tag_meta:
return tag_meta.apply_path_context(path)
return cls.read_filename(path)
@classmethod
def read_evidence(
cls,
path: Path,
) -> tuple[MetaMusic, Optional[MetaMusic], MetaMusic]:
"""分别返回合并元数据、纯标签元数据和纯文件名元数据。"""
filename_meta = cls.read_filename(path)
tag_meta = cls.read_tags(path) if path.exists() and path.is_file() else None
if not tag_meta:
return filename_meta, None, filename_meta
merged_meta = MetaMusic.from_dict(tag_meta.to_dict()).apply_path_context(path)
return merged_meta, tag_meta, filename_meta
@classmethod
def read_many(cls, paths: list[Path]) -> list[MetaMusic]:
"""批量读取一组音频路径的标签与文件名元数据。"""
return [cls.read(path) for path in paths]
@classmethod
def read_tags(cls, path: Path) -> Optional[MetaMusic]:
"""只读取本地音频标签和流参数,不使用文件名或目录补齐。"""
try:
audio = MutagenFile(path, easy=True)
except Exception as err:
logger.warning(f"读取音频标签失败:{path} - {err}")
return None
if not audio:
return None
tags = audio.tags or {}
track_number, total_tracks = cls._number_pair(cls._first(tags, "tracknumber"))
disc_number, total_discs = cls._number_pair(cls._first(tags, "discnumber"))
musicbrainz_id = cls._normalize_musicbrainz_id(
cls._first_of(
tags,
"musicbrainz_trackid",
"musicbrainz_recordingid",
)
)
info = getattr(audio, "info", None)
return MetaMusic(
org_string=path.name,
title=cls._first(tags, "title"),
artists=cls._values(tags, "artist"),
album=cls._first(tags, "album"),
album_artist=cls._first(tags, "albumartist"),
year=cls._year(cls._first(tags, "date") or cls._first(tags, "originaldate")),
disc_number=disc_number,
track_number=track_number,
total_discs=total_discs,
total_tracks=total_tracks,
version=cls._first(tags, "version") or cls._first(tags, "subtitle"),
audio_format=cls._audio_format(path, info),
bit_depth=cls._optional_int(getattr(info, "bits_per_sample", None)),
sample_rate=cls._optional_int(getattr(info, "sample_rate", None)),
bitrate=cls._optional_int(getattr(info, "bitrate", None)),
duration=round(info.length) if info and getattr(info, "length", None) else None,
isrc=cls._first(tags, "isrc"),
media_source=MediaSource.MusicBrainz if musicbrainz_id else None,
media_id=musicbrainz_id,
)
@staticmethod
def read_filename(path: Path) -> MetaMusic:
"""只从文件名和目录结构解析音乐元数据。"""
return MetaMusic(
org_string=path.name,
title=path.stem,
audio_format=path.suffix.lstrip(".").upper() or None,
).apply_path_context(path)
@classmethod
def write(
cls,
path: Path,
music: Union[MetaMusic, MusicInfo],
cover_data: Optional[bytes] = None,
cover_mime: str = "image/jpeg",
overwrite: bool = True,
write_tags: bool = True,
cover_overwrite: Optional[bool] = None,
) -> bool:
"""按独立策略写入标准音乐标签,并为常见格式嵌入专辑封面。"""
try:
audio = MutagenFile(path, easy=True)
if not audio:
logger.warning(f"无法写入音频标签:{path}")
return False
if write_tags:
if audio.tags is None:
audio.add_tags()
for key, value in cls._tag_values(music).items():
if value in (None, "", []):
continue
if not overwrite and audio.tags.get(key):
continue
try:
audio[key] = value if isinstance(value, list) else [str(value)]
except (KeyError, TypeError, ValueError) as err:
logger.debug(f"音频格式不支持标签 {key}{path} - {err}")
audio.save()
if cover_data:
cls._write_cover(
path=path,
cover_data=cover_data,
cover_mime=cover_mime,
overwrite=(
overwrite
if cover_overwrite is None
else cover_overwrite
),
)
return True
except Exception as err:
logger.warning(f"写入音频标签失败:{path} - {err}")
return False
@classmethod
def _tag_values(cls, music: Union[MetaMusic, MusicInfo]) -> dict[str, Any]:
"""把标准音乐对象转换为 Mutagen Easy 标签字典。"""
track_number = cls._number_text(
getattr(music, "track_number", None),
getattr(music, "total_tracks", None),
)
disc_number = cls._number_text(
getattr(music, "disc_number", None),
getattr(music, "total_discs", None),
)
return {
"title": getattr(music, "title", None),
"artist": list(getattr(music, "artists", None) or []),
"album": getattr(music, "album", None),
"albumartist": getattr(music, "album_artist", None),
"date": getattr(music, "year", None),
"tracknumber": track_number,
"discnumber": disc_number,
"isrc": getattr(music, "isrc", None),
"musicbrainz_trackid": cls._musicbrainz_recording_id(music),
}
@staticmethod
def _musicbrainz_recording_id(
music: Union[MetaMusic, MusicInfo],
) -> Optional[str]:
"""仅将 MusicBrainz 单曲身份写入 recording 标签,避免误写专辑 ID。"""
if (
getattr(music, "media_source", None) == MediaSource.MusicBrainz
and getattr(music, "music_type", MUSIC_ENTITY_RECORDING)
== MUSIC_ENTITY_RECORDING
):
media_id = getattr(music, "media_id", None)
return str(media_id) if media_id else None
return None
@staticmethod
def _number_text(current: Optional[int], total: Optional[int]) -> Optional[str]:
"""把曲序或碟号转换为常见的 current/total 标签文本。"""
if current is None:
return None
return f"{current}/{total}" if total else str(current)
@staticmethod
def _write_cover(
path: Path,
cover_data: bytes,
cover_mime: str,
overwrite: bool,
) -> None:
"""为 MP3、FLAC 和 MP4/M4A 写入内嵌封面,其它格式保留标签写入结果。"""
audio = MutagenFile(path)
if isinstance(audio, FLAC):
if audio.pictures and not overwrite:
return
picture = Picture()
picture.type = 3
picture.mime = cover_mime
picture.desc = "Cover"
picture.data = cover_data
if overwrite:
audio.clear_pictures()
audio.add_picture(picture)
audio.save()
return
if isinstance(audio, MP4):
if audio.tags is None:
audio.add_tags()
if audio.tags.get("covr") and not overwrite:
return
image_format = (
MP4Cover.FORMAT_PNG
if cover_mime == "image/png"
else MP4Cover.FORMAT_JPEG
)
audio.tags["covr"] = [MP4Cover(cover_data, imageformat=image_format)]
audio.save()
return
tags = getattr(audio, "tags", None)
if tags is not None and hasattr(tags, "add"):
if tags.getall("APIC") and not overwrite:
return
if overwrite:
tags.delall("APIC")
tags.add(
APIC(
encoding=3,
mime=cover_mime,
type=3,
desc="Cover",
data=cover_data,
)
)
audio.save()
@staticmethod
def _values(tags: Any, key: str) -> list[str]:
"""从 Mutagen Easy 标签中提取非空字符串列表。"""
value = tags.get(key) if hasattr(tags, "get") else None
if value is None:
return []
if isinstance(value, (list, tuple)):
return [str(item).strip() for item in value if str(item).strip()]
return [str(value).strip()] if str(value).strip() else []
@classmethod
def _first(cls, tags: Any, key: str) -> Optional[str]:
"""返回指定音频标签的第一个非空值。"""
values = cls._values(tags, key)
return values[0] if values else None
@classmethod
def _first_of(cls, tags: Any, *keys: str) -> Optional[str]:
"""按顺序返回多个音频标签中的第一个非空值。"""
for key in keys:
if value := cls._first(tags, key):
return value
return None
@staticmethod
def _normalize_musicbrainz_id(value: Optional[str]) -> Optional[str]:
"""校验并规范化音频标签中的 MusicBrainz UUID。"""
try:
return str(UUID(str(value)))
except (AttributeError, TypeError, ValueError):
return None
@staticmethod
def _number_pair(value: Optional[str]) -> tuple[Optional[int], Optional[int]]:
"""解析 track/disc 标签中的当前编号和总数。"""
if not value:
return None, None
parts = str(value).split("/", 1)
current = AudioMetadataHelper._optional_int(parts[0])
total = AudioMetadataHelper._optional_int(parts[1]) if len(parts) > 1 else None
return current, total
@staticmethod
def _year(value: Optional[str]) -> Optional[int]:
"""从完整或不完整日期标签中提取四位年份。"""
if not value:
return None
return AudioMetadataHelper._optional_int(str(value)[:4])
@staticmethod
def _audio_format(path: Path, info: Any) -> Optional[str]:
"""结合扩展名与流编码识别音频格式,区分同为 M4A 容器的 AAC 和 ALAC。"""
codec_text = " ".join(
str(value or "")
for value in (
getattr(info, "codec", None),
getattr(info, "codec_description", None),
)
).casefold()
codec_formats = (
(("alac", "apple lossless"), "ALAC"),
(("aac", "mp4a"), "AAC"),
(("opus",), "OPUS"),
(("vorbis",), "OGG"),
(("flac",), "FLAC"),
)
for markers, audio_format in codec_formats:
if any(marker in codec_text for marker in markers):
return audio_format
return path.suffix.lstrip(".").upper() or None
@staticmethod
def _optional_int(value: Any) -> Optional[int]:
"""将音频技术参数安全转换为整数。"""
try:
return int(value) if value is not None and str(value).strip() else None
except (TypeError, ValueError):
return None

View File

@@ -0,0 +1,354 @@
import re
from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
from typing import List, Optional, Tuple
from app import schemas
from app.domain.context import MediaInfo
from app.db.systemconfig_oper import SystemConfigOper
from app.runtime.log import logger
from app.schemas.types import MediaType, StorageSchema, SystemConfigKey
from app.adapters.system.host import SystemUtils
JINJA2_VAR_PATTERN = re.compile(r"\{\{.*?}}", re.DOTALL)
WINDOWS_DRIVE_PATTERN = re.compile(r"^[A-Za-z]:[\\/]")
WINDOWS_DRIVE_PREFIX_PATTERN = re.compile(r"^[A-Za-z]:")
class DirectoryHelper:
"""
下载目录/媒体库目录帮助类
"""
@staticmethod
def get_dirs() -> List[schemas.TransferDirectoryConf]:
"""
获取所有下载目录
"""
dir_confs: List[dict] = SystemConfigOper().get(SystemConfigKey.Directories)
if not dir_confs:
return []
return [schemas.TransferDirectoryConf(**d) for d in dir_confs]
def get_download_dirs(self) -> List[schemas.TransferDirectoryConf]:
"""
获取所有下载目录
"""
return sorted([d for d in self.get_dirs() if d.download_path], key=lambda x: x.priority)
def get_local_download_dirs(self) -> List[schemas.TransferDirectoryConf]:
"""
获取所有本地的可下载目录
"""
return [d for d in self.get_download_dirs() if d.storage == "local"]
def get_download_dir_by_save_path(
self,
media: Optional[MediaInfo],
save_path: str,
) -> Optional[schemas.TransferDirectoryConf]:
"""
按媒体信息和精确保存根路径匹配下载目录配置。
仅配置根目录本身继承自动分类规则;根目录下的自定义子目录保持调用方指定的完整路径。
:param media: 媒体信息
:param save_path: 已选择的下载保存目录,支持本地路径或远端 FileURI
:return: 匹配的下载目录配置
"""
value = str(save_path or "").strip()
try:
storage, raw_path = _split_file_uri(value)
target_style, target_path = _normalize_download_path(raw_path, storage)
except ValueError:
return None
media_type = media.type.value if media else None
for dir_info in self.get_download_dirs():
root = _normalize_download_root(dir_info)
if not root:
continue
root_storage, root_style, root_path = root
if storage != root_storage or target_style != root_style or target_path != root_path:
continue
if not media_type or not dir_info.media_type:
return dir_info
if dir_info.media_type == media_type and not dir_info.media_category:
return dir_info
if dir_info.media_type == media_type and dir_info.media_category == media.category:
return dir_info
return None
def get_library_dirs(self) -> List[schemas.TransferDirectoryConf]:
"""
获取所有媒体库目录
"""
return sorted([d for d in self.get_dirs() if d.library_path], key=lambda x: x.priority)
def get_local_library_dirs(self) -> List[schemas.TransferDirectoryConf]:
"""
获取所有本地的媒体库目录
"""
return [d for d in self.get_library_dirs() if d.library_storage == "local"]
def get_dir(self, media: Optional[MediaInfo], include_unsorted: Optional[bool] = False,
storage: Optional[str] = None, src_path: Path = None,
target_storage: Optional[str] = None, dest_path: Path = None
) -> Optional[schemas.TransferDirectoryConf]:
"""
根据媒体信息获取下载目录、媒体库目录配置
:param media: 媒体信息
:param include_unsorted: 包含不整理目录
:param storage: 源存储类型
:param target_storage: 目标存储类型
:param src_path: 源目录,有值时直接匹配
:param dest_path: 目标目录,有值时直接匹配
"""
# 电影/电视剧
media_type = media.type.value if media else None
dirs = self.get_dirs()
# 如果存在源目录,并源目录为任一下载目录的子目录时,则进行源目录匹配,否则,允许源目录按同盘优先的逻辑匹配
matching_dirs = [d for d in dirs if src_path.is_relative_to(d.download_path)] if src_path else []
# 根据是否有匹配的源目录,决定要考虑的目录集合
dirs_to_consider = matching_dirs if matching_dirs else dirs
# 已匹配的目录
matched_dirs: List[schemas.TransferDirectoryConf] = []
# 按照配置顺序查找
for d in dirs_to_consider:
# 没有启用整理的目录
if not d.monitor_type and not include_unsorted:
continue
# 源存储类型不匹配
if storage and d.storage != storage:
continue
# 目标存储类型不匹配
if target_storage and d.library_storage != target_storage:
continue
# 有目标目录时,目标目录不匹配媒体库目录
if dest_path and dest_path != Path(d.library_path):
continue
# 目录类型为全部的,符合条件
if not media_type or not d.media_type:
matched_dirs.append(d)
continue
# 目录类型相等,目录类别为全部,符合条件
if d.media_type == media_type and not d.media_category:
matched_dirs.append(d)
continue
# 目录类型相等,目录类别相等,符合条件
if d.media_type == media_type and d.media_category == media.category:
matched_dirs.append(d)
continue
if matched_dirs:
if src_path:
# 优先源目录同盘
for matched_dir in matched_dirs:
matched_path = Path(matched_dir.download_path)
if self._is_same_source((src_path, storage or "local"), (matched_path, matched_dir.library_storage)):
return matched_dir
return matched_dirs[0]
return None
@staticmethod
def _is_same_source(src: Tuple[Path, str], tar: Tuple[Path, str]) -> bool:
"""
判断源目录和目标目录是否在同一存储盘
:param src: 源目录路径和存储类型
:param tar: 目标目录路径和存储类型
:return: 是否在同一存储盘
"""
src_path, src_storage = src
tar_path, tar_storage = tar
if "local" == tar_storage == src_storage:
return SystemUtils.is_same_disk(src_path, tar_path)
# 网络存储,直接比较类型
return src_storage == tar_storage
@staticmethod
def get_media_root_path(
rename_format: str,
rename_path: Path,
media_type: Optional[MediaType] = None,
) -> Optional[Path]:
"""
获取重命名后的媒体文件根路径
:param rename_format: 重命名格式
:param rename_path: 重命名后的路径
:param media_type: 媒体类型;音乐需要避开可选碟片目录并返回专辑目录
:return: 媒体文件根路径
"""
if not rename_format:
logger.error("重命名格式不能为空")
return None
if media_type == MediaType.MUSIC:
# 音乐模板允许按多碟动态增加 Disc 子目录,不能按静态模板层数反推。
# 文件的直接父目录通常就是专辑目录;命中碟片目录时再上移一级。
media_root = rename_path.parent
if re.fullmatch(
r"(?:cd|disc|disk)\s*0*\d+",
media_root.name,
re.IGNORECASE,
):
media_root = media_root.parent
return media_root
# 计算重命名中的文件夹层数
rename_list = rename_format.split("/")
rename_format_level = len(rename_list) - 1
# 反向查找标题参数所在层
for level, name in enumerate(reversed(rename_list)):
if level == 0:
# 跳过文件名的标题参数
continue
matchs = JINJA2_VAR_PATTERN.findall(name)
if not matchs:
continue
# 处理特例,有的人重命名的第一层是年份、分辨率
if (any("title" in m for m in matchs)
and not any("season" in m for m in matchs)):
# 找出最后一层含有标题且不含季参数的目录作为媒体根目录
rename_format_level = level
break
else:
# 假定第一层目录是媒体根目录
logger.warn(f"重命名格式 {rename_format} 缺少标题目录")
if rename_format_level > len(rename_path.parents):
# 通常因为路径以/结尾被Path规范化删除了
logger.error(f"路径 {rename_path} 不匹配重命名格式 {rename_format}")
return None
if rename_format_level <= 0:
# 所有媒体文件都存在一个目录内的特殊需求
rename_format_level = 1
# 媒体根路径
media_root = rename_path.parents[rename_format_level - 1]
return media_root
def _split_file_uri(value: str) -> Tuple[str, str]:
"""
拆分 FileURI 字符串,保留原始路径用于安全校验。
"""
for storage in StorageSchema:
protocol = f"{storage.value}:"
if value.startswith(protocol):
return storage.value, value[len(protocol):]
return "local", value
def _normalize_safe_posix_path(raw_path: str) -> PurePosixPath:
"""
规范化保存目录路径,并拒绝跨目录或跨平台歧义写法。
"""
if not raw_path:
raise ValueError("保存路径不能为空")
if "\\" in raw_path:
raise ValueError("保存路径不能包含反斜杠")
if raw_path.startswith("//"):
raise ValueError("保存路径不能使用 UNC 路径")
if WINDOWS_DRIVE_PATTERN.match(raw_path):
raise ValueError("保存路径不能使用 Windows 盘符路径")
if not raw_path.startswith("/"):
raise ValueError("保存路径必须是绝对路径")
path = PurePosixPath(raw_path)
parts = [part for part in path.parts if part != "/"]
if ".." in parts:
raise ValueError("保存路径不能包含上级目录")
if parts and re.fullmatch(r"[A-Za-z]:", parts[0]):
raise ValueError("保存路径不能使用 Windows 盘符路径")
return path
def _normalize_safe_windows_path(raw_path: str) -> PureWindowsPath:
"""
规范化已配置的 Windows 盘符路径UNC 与反斜杠写法不参与下载目录 allowlist。
"""
if not raw_path:
raise ValueError("保存路径不能为空")
if "\\" in raw_path:
raise ValueError("保存路径不能包含反斜杠")
if raw_path.startswith("//"):
raise ValueError("保存路径不能使用 UNC 路径")
if not WINDOWS_DRIVE_PATTERN.match(raw_path):
raise ValueError("保存路径必须是 Windows 绝对路径")
path = PureWindowsPath(raw_path)
if ".." in path.parts:
raise ValueError("保存路径不能包含上级目录")
return path
def _normalize_download_path(raw_path: str, storage: str) -> Tuple[str, PurePath]:
"""
按存储类型解析下载路径,本地允许 POSIX 或已配置的 Windows drive远端保持 FileURI POSIX 语义。
"""
path_value = str(raw_path or "").strip()
if storage == "local" and WINDOWS_DRIVE_PREFIX_PATTERN.match(path_value):
return "windows", _normalize_safe_windows_path(path_value)
return "posix", _normalize_safe_posix_path(path_value)
def _download_path_uri(storage: str, path: PurePath) -> str:
"""
生成可传给下载器的 save_path保持 /download/paths 暴露的本地和远端路径风格。
"""
path_value = path.as_posix()
if storage == "local":
return path_value
return schemas.FileURI(storage=storage, path=path_value).uri
def _normalize_download_root(dir_info: schemas.TransferDirectoryConf) -> Optional[Tuple[str, str, PurePath]]:
"""
读取下载目录配置中的根路径;无效配置不参与用户 save_path allowlist。
"""
if not dir_info.download_path:
return None
storage = dir_info.storage or "local"
try:
path_style, root_path = _normalize_download_path(dir_info.download_path, storage)
return storage, path_style, root_path
except ValueError as err:
logger.warn(f"跳过无效下载目录配置:{str(err)}")
return None
def validate_download_save_path(save_path: str) -> str:
"""
校验用户传入的下载保存目录,/download/paths 暴露的下载目录配置是允许写入的公共合同。
:param save_path: 下载保存目录,支持本地 /path、远端 <storage>:/path 和旧版订阅中的无前缀远程路径
:return: 可直接传给下载接口的规范化保存目录
"""
value = str(save_path or "").strip()
has_storage_prefix = any(value.startswith(f"{item.value}:") for item in StorageSchema)
storage, raw_path = _split_file_uri(value)
target_style, target_path = _normalize_download_path(raw_path, storage)
download_roots = []
for dir_info in DirectoryHelper().get_download_dirs():
root = _normalize_download_root(dir_info)
if root:
download_roots.append(root)
for root_storage, root_style, root_path in download_roots:
if storage != root_storage:
continue
if target_style != root_style:
continue
if target_path == root_path or target_path.is_relative_to(root_path):
return _download_path_uri(storage, target_path)
# 旧版订阅界面只持久化 download_path需要从已配置根目录恢复远程存储类型。
if (not has_storage_prefix
and storage == StorageSchema.Local.value
and target_style == "posix"):
for root_storage, root_style, root_path in download_roots:
if root_storage == StorageSchema.Local.value or target_style != root_style:
continue
if target_path == root_path or target_path.is_relative_to(root_path):
return _download_path_uri(root_storage, target_path)
raise ValueError("保存路径不在允许的下载目录范围内")

View File

@@ -0,0 +1,38 @@
from typing import Optional
from app.runtime.extensions.service_registry import ServiceBaseHelper
from app.schemas import DownloaderConf, ServiceInfo
from app.schemas.types import SystemConfigKey, ModuleType
class DownloaderHelper(ServiceBaseHelper[DownloaderConf]):
"""
下载器帮助类
"""
def __init__(self):
"""绑定下载器配置和下载器模块类型。"""
super().__init__(
config_key=SystemConfigKey.Downloaders,
conf_type=DownloaderConf,
module_type=ModuleType.Downloader
)
def is_downloader(
self,
service_type: Optional[str] = None,
service: Optional[ServiceInfo] = None,
name: Optional[str] = None,
) -> bool:
"""
通用的下载器类型判断方法
:param service_type: 下载器的类型名称(如 'qbittorrent', 'transmission', 'rtorrent'
:param service: 要判断的服务信息
:param name: 服务的名称
:return: 如果服务类型或实例为指定类型,返回 True否则返回 False
"""
# 如果未提供 service 则通过 name 获取服务
service = service or self.get_service(name=name)
# 判断服务类型是否为指定类型
return bool(service and service.type == service_type)

66
app/application/filter.py Normal file
View File

@@ -0,0 +1,66 @@
from typing import List, Optional
from app.db.systemconfig_oper import SystemConfigOper
from app.domain.context import MediaInfo
from app.schemas import CustomRule, FilterRuleGroup
from app.schemas.types import SystemConfigKey
class RuleHelper:
"""读取用户过滤规则配置,并按媒体上下文选择适用规则组。"""
@staticmethod
def get_rule_groups() -> List[FilterRuleGroup]:
"""返回用户配置的全部过滤规则组。"""
rule_groups: List[dict] = SystemConfigOper().get(
SystemConfigKey.UserFilterRuleGroups
)
if not rule_groups:
return []
return [FilterRuleGroup(**group) for group in rule_groups]
def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]:
"""按名称返回过滤规则组。"""
return next(
(group for group in self.get_rule_groups() if group.name == group_name),
None,
)
def get_rule_group_by_media(
self,
media: Optional[MediaInfo] = None,
group_names: Optional[list] = None,
) -> List[FilterRuleGroup]:
"""按媒体类型、分类和候选名称筛选适用规则组。"""
rule_groups = self.get_rule_groups()
if group_names:
rule_groups = [
group for group in rule_groups if group.name in group_names
]
return [
group
for group in rule_groups
if not group.media_type
or (
media
and (
(not group.category and group.media_type == media.type.value)
or group.category == media.category
)
)
]
@staticmethod
def get_custom_rules() -> List[CustomRule]:
"""返回用户配置的全部自定义过滤规则。"""
rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules)
if not rules:
return []
return [CustomRule(**rule) for rule in rules]
def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]:
"""按 ID 返回一条自定义过滤规则。"""
return next(
(rule for rule in self.get_custom_rules() if rule.id == rule_id),
None,
)

File diff suppressed because it is too large Load Diff

422
app/application/history.py Normal file
View File

@@ -0,0 +1,422 @@
from typing import Any, Dict, Optional
from app.runtime.cache import TTLCache
from app.runtime.config import settings
from app.db.models.transferhistory import TransferHistory
from app.db.transferhistory_oper import TransferHistoryOper
from app.runtime.log import logger
# 失败重试次数的合法区间。下界为 1一次瞬时故障网络抖动、TMDB 瞬断、移动失败)
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10永远识别不出的文件重试再多
# 也不会成功,只会重复推送失败通知,批量导入场景下会刷屏,所以不允许无限重试
MIN_FAILED_RETRIES = 1
MAX_FAILED_RETRIES = 10
# 同一源路径的连续整理失败状态。整理链在写失败历史时累计、整理成功或删除历史时清零,
# 查重闸只读不写,避免监控层与整理链对同一个事件重复计数。缓存值会同时保存文件指纹,
# 因此同一路径的新版本天然获得独立预算内存缓存会随进程重启清空Redis 后端则保留到 TTL 到期。
FAILED_RETRY_TTL = 24 * 3600
_failed_retry_counts = TTLCache(region="transfer_failed_retry", maxsize=5000, ttl=FAILED_RETRY_TTL)
class HistoryGateAction:
"""
整理历史查重闸的判定结果。
监控分发app/monitor/dispatcher.py与整理链计划整理段app/chain/transfer.py
共用本模块,避免两处各写一套去重策略后互相对冲:上游放行的文件被下游按
「存在记录即拦」全额收回,等于放行逻辑完全失效。
"""
# 没有整理记录
PASS_NO_RECORD = "pass_no_record"
# 上次整理失败且重试次数未用尽,放行重试
PASS_FAILED = "pass_failed"
# 上次整理失败但源文件已变为新版本,放行并重置该版本的重试预算
PASS_FAILED_VERSION_CHANGED = "pass_failed_version_changed"
# 已整理成功但源文件已变化,放行交由 overwrite_mode 决断
PASS_SIZE_CHANGED = "pass_size_changed"
# 上次整理失败且重试次数已用尽,跳过
SKIP_RETRY_EXHAUSTED = "skip_retry_exhausted"
# 已整理成功且源文件未变化,跳过
SKIP = "skip"
def is_skip_action(action: str) -> bool:
"""
判断查重闸判定是否为跳过整理。
:param action: HistoryGateAction 之一
:return: True 表示跳过
"""
return action in (HistoryGateAction.SKIP, HistoryGateAction.SKIP_RETRY_EXHAUSTED)
def max_failed_retries() -> int:
"""
读取失败重试上限并钳制到合法区间。
配置为负数、0 或超过上界时都会被钳制并记录 warn关闭重试会让瞬时故障造成
永久漏件,无限重试会让永久失败的文件反复刷通知,两端都不接受。
:return: 合法的最大重试次数
"""
raw = settings.TRANSFER_MAX_FAILED_RETRIES
try:
value = int(raw)
except (TypeError, ValueError):
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 配置非法({raw!r}"
f"已回退为 {MIN_FAILED_RETRIES}")
return MIN_FAILED_RETRIES
if value < MIN_FAILED_RETRIES:
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能小于 {MIN_FAILED_RETRIES}"
f"(当前 {value}),已按 {MIN_FAILED_RETRIES} 处理")
return MIN_FAILED_RETRIES
if value > MAX_FAILED_RETRIES:
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能大于 {MAX_FAILED_RETRIES}"
f"(当前 {value}),已按 {MAX_FAILED_RETRIES} 处理")
return MAX_FAILED_RETRIES
return value
def failed_retry_key(src_path: Optional[str], storage: Optional[str] = None) -> Optional[str]:
"""
生成失败重试计数的缓存键。
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:return: 缓存键,源路径为空时返回 None
"""
if not src_path:
return None
return f"{storage or 'local'}:{src_path}"
def coerce_modify_time(modify_time: Any) -> Optional[float]:
"""
统一转换文件修改时间,无法转换时返回 None。
:param modify_time: 原始修改时间值
:return: 文件修改时间
"""
if modify_time is None:
return None
try:
return float(modify_time)
except (TypeError, ValueError):
return None
def coerce_fileid(fileid: Any) -> Optional[str]:
"""
统一转换文件唯一标识,空值视为不可比对。
:param fileid: 原始文件唯一标识
:return: 非空文件唯一标识
"""
if fileid is None:
return None
value = str(fileid).strip()
return value or None
def file_fingerprint(
file_size: Any = None,
file_modify_time: Any = None,
fileid: Any = None,
) -> Dict[str, Any]:
"""
生成用于区分同一路径文件版本的稳定指纹。
大小是所有存储器都尽量提供的最小指纹;两端均有数据时,修改时间和文件 ID 还可
识别“同大小替换”。只保留可比较字段,避免缺失元数据把同一文件误判成新版本。
:param file_size: 文件大小
:param file_modify_time: 文件修改时间
:param fileid: 存储器文件唯一标识
:return: 非空且可比较的指纹字段
"""
fingerprint = {}
size = coerce_size(file_size)
if size is not None:
fingerprint["size"] = size
modify_time = coerce_modify_time(file_modify_time)
if modify_time is not None:
fingerprint["modify_time"] = modify_time
normalized_fileid = coerce_fileid(fileid)
if normalized_fileid is not None:
fingerprint["fileid"] = normalized_fileid
return fingerprint
def _retry_state(value: Any) -> tuple[int, Dict[str, Any]]:
"""将新旧缓存值统一转换为失败次数与文件指纹。"""
if isinstance(value, dict):
raw_count = value.get("count", 0)
raw_fingerprint = value.get("fingerprint")
else:
# 兼容已写入 Redis 或内存的旧整数计数;下次带指纹写入时会自动升级结构。
raw_count = value
raw_fingerprint = None
try:
count = max(int(raw_count or 0), 0)
except (TypeError, ValueError):
count = 0
fingerprint = (
file_fingerprint(
file_size=raw_fingerprint.get("size"),
file_modify_time=raw_fingerprint.get("modify_time"),
fileid=raw_fingerprint.get("fileid"),
)
if isinstance(raw_fingerprint, dict)
else {}
)
return count, fingerprint
def _is_file_version_changed(
recorded_fingerprint: Dict[str, Any],
current_fingerprint: Dict[str, Any],
) -> bool:
"""判断两个可比文件指纹是否指向不同版本。"""
for field in ("fileid", "modify_time", "size"):
recorded_value = recorded_fingerprint.get(field)
current_value = current_fingerprint.get(field)
if (
recorded_value is not None
and current_value is not None
and recorded_value != current_value
):
return True
return False
def failed_retry_count(src_path: Optional[str], storage: Optional[str] = None,
file_size: Any = None, file_modify_time: Any = None,
fileid: Any = None) -> int:
"""
读取同一源路径已累计的连续整理失败次数。
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 当前文件版本已失败次数,无记录时为 0
"""
key = failed_retry_key(src_path, storage)
if not key:
return 0
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if (
recorded_fingerprint
and current_fingerprint
and _is_file_version_changed(recorded_fingerprint, current_fingerprint)
):
return 0
return count
def record_transfer_failure(src_path: Optional[str], storage: Optional[str] = None,
file_size: Any = None, file_modify_time: Any = None,
fileid: Any = None) -> int:
"""
累计一次整理失败。
:param src_path: 整理记录使用的源路径
:param storage: 源存储
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 当前文件版本累计后的失败次数
"""
key = failed_retry_key(src_path, storage)
if not key:
return 0
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if current_fingerprint and (
not recorded_fingerprint
or _is_file_version_changed(recorded_fingerprint, current_fingerprint)
):
count = 0
count += 1
if current_fingerprint:
_failed_retry_counts[key] = {
"count": count,
"fingerprint": current_fingerprint,
}
elif recorded_fingerprint:
_failed_retry_counts[key] = {
"count": count,
"fingerprint": recorded_fingerprint,
}
else:
_failed_retry_counts[key] = count
return count
def clear_transfer_failures(src_path: Optional[str], storage: Optional[str] = None) -> None:
"""
清空同一源路径的失败计数。整理成功、或用户删除整理记录(显式要求重来)时调用。
:param src_path: 整理记录使用的源路径
:param storage: 源存储
"""
key = failed_retry_key(src_path, storage)
if key:
# 缺省值必须是 0 而不是 NoneCacheBackend.pop 把「default 为 None」当成「未提供
# default」键不存在时会抛 KeyError。整理成功路径上绝大多数文件从未失败过
# 传 None 会让每一次首次成功整理都炸掉成功回调
_failed_retry_counts.pop(key, 0)
def coerce_size(size: Any) -> Optional[int]:
"""
统一转换文件大小,无法转换时返回 None视为不可比对
:param size: 原始大小值
:return: 文件大小
"""
if size is None:
return None
try:
return int(size)
except (TypeError, ValueError):
return None
def history_src_size(history: TransferHistory) -> Optional[int]:
"""
读取整理记录中的源文件大小。
src_fileitem 是 JSON 列,历史数据可能为空、缺 size 键甚至不是字典,
取不到时统一返回 None 交由调用方保守处理。
:param history: 整理记录
:return: 源文件大小,取不到时为 None
"""
return history_src_fingerprint(history).get("size")
def history_src_fingerprint(history: TransferHistory) -> Dict[str, Any]:
"""
读取整理记录中的源文件版本指纹。
:param history: 整理记录
:return: 源文件的可比较指纹字段
"""
src_fileitem = getattr(history, "src_fileitem", None)
if not isinstance(src_fileitem, dict):
return {}
return file_fingerprint(
file_size=src_fileitem.get("size"),
file_modify_time=src_fileitem.get("modify_time"),
fileid=src_fileitem.get("fileid"),
)
def resolve_history(src_path: str, storage: Optional[str] = None,
transfer_history_oper: Optional[TransferHistoryOper] = None
) -> Optional[TransferHistory]:
"""
查询源路径对应的整理记录。
新表通过 (src, src_storage) 唯一索引保证单条记录;仍保留对成功记录的二次确认,
兼容升级前可能残留的重复数据,避免把已整理成功的文件重复整理。查询异常不在
此处吞掉,由调用方按各自的重试策略处理。
:param src_path: 整理记录使用的源路径
:param storage: 存储
:param transfer_history_oper: 复用的历史操作对象,未传时新建
:return: 命中的整理记录,未命中时为 None
"""
oper = transfer_history_oper or TransferHistoryOper()
history = oper.get_by_src(src_path, storage=storage)
if history is not None and not history.status:
history = oper.get_success_by_src(src_path, storage=storage) or history
return history
def evaluate_history_gate(history: Optional[TransferHistory],
file_size: Optional[float] = None,
file_modify_time: Optional[float] = None,
fileid: Optional[str] = None,
retry_count: Optional[int] = None) -> str:
"""
依据整理历史判断本次是否跳过整理。
成功记录不能简单地「存在即跳过」:同路径重新上传的新版本会因此没有机会走到
整理链的 overwrite_mode 判定,升级永远无法入库,故任一可比文件指纹变化时一律放行。
失败记录按文件版本使用有界重试:新版本先放行并在下一次失败时从 1 重新计数,
同一版本未达上限时继续重试,让瞬时故障(网络/识别/移动)自愈;达到上限后跳过,
避免永久失败的文件反复刷失败通知。
:param history: 整理记录,未命中时为 None
:param file_size: 当前文件大小,蓝光目录等场景可能为 None
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:param retry_count: 已累计的失败次数None 表示按记录源路径实时查询
:return: HistoryGateAction 之一
"""
if history is None:
return HistoryGateAction.PASS_NO_RECORD
recorded_fingerprint = history_src_fingerprint(history)
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if not history.status:
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
return HistoryGateAction.PASS_FAILED_VERSION_CHANGED
if retry_count is None:
retry_count = failed_retry_count(
getattr(history, "src", None),
getattr(history, "src_storage", None),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if retry_count >= max_failed_retries():
return HistoryGateAction.SKIP_RETRY_EXHAUSTED
# 监控事件是稀疏驱动的(落地事件/延迟重扫/补偿扫描),入口还有 TTL 去重兜底,
# 配合失败次数上限,重试频率与总量都可控
return HistoryGateAction.PASS_FAILED
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
# 同路径换成了另一个版本(如升级为更高码率),是否覆盖交给整理链的
# overwrite_mode 决断,查重闸不做替代判断
return HistoryGateAction.PASS_SIZE_CHANGED
# 无法比对大小(蓝光目录、历史记录缺 size时保守跳过避免重复整理
return HistoryGateAction.SKIP
def describe_history_gate(history: Optional[TransferHistory],
file_size: Optional[float] = None,
file_modify_time: Optional[float] = None,
fileid: Optional[str] = None) -> str:
"""
生成查重闸判定的可读说明,供日志定位「到底是哪条记录在拦」。
:param history: 整理记录
:param file_size: 当前文件大小
:param file_modify_time: 当前文件修改时间
:param fileid: 当前文件唯一标识
:return: 说明文本
"""
if history is None:
return "无整理记录"
recorded_fingerprint = history_src_fingerprint(history)
current_fingerprint = file_fingerprint(
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if not history.status:
count = failed_retry_count(
getattr(history, "src", None),
getattr(history, "src_storage", None),
file_size=file_size,
file_modify_time=file_modify_time,
fileid=fileid,
)
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
return f"失败记录 #{history.id},文件版本已变化,重试预算将重置"
return f"失败记录 #{history.id},已重试 {count}/{max_failed_retries()}"
recorded_size = recorded_fingerprint.get("size")
current_size = current_fingerprint.get("size")
if recorded_size is None and current_size is None:
return f"成功记录 #{history.id},大小不可比对"
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"

389
app/application/image.py Normal file
View File

@@ -0,0 +1,389 @@
import io
from pathlib import Path
from typing import Callable, Optional, List
from PIL import Image
from app.runtime.cache import cached, FileCache, AsyncFileCache
from app.runtime.config import settings
from app.runtime.log import logger
from app.adapters.network.http import RequestUtils, AsyncRequestUtils
from app.adapters.network.ip import IpUtils
from app.application.security.url import SecurityUtils
from app.foundation.singleton import Singleton
WallpaperProvider = Callable[[], Optional[str]]
WallpaperListProvider = Callable[[int], List[str]]
def _empty_wallpaper_provider() -> Optional[str]:
"""在启动组合根尚未装配壁纸来源时返回空结果。"""
return None
def _empty_wallpaper_list_provider(_count: int) -> List[str]:
"""在启动组合根尚未装配壁纸来源时返回空列表。"""
return []
_tmdb_wallpaper_provider: WallpaperProvider = _empty_wallpaper_provider
_tmdb_wallpaper_list_provider: WallpaperListProvider = (
_empty_wallpaper_list_provider
)
_mediaserver_wallpaper_provider: WallpaperProvider = _empty_wallpaper_provider
_mediaserver_wallpaper_list_provider: WallpaperListProvider = (
_empty_wallpaper_list_provider
)
def configure_wallpaper_providers(
*,
tmdb_wallpaper: WallpaperProvider,
tmdb_wallpapers: WallpaperListProvider,
mediaserver_wallpaper: WallpaperProvider,
mediaserver_wallpapers: WallpaperListProvider,
) -> None:
"""由启动组合根注入需要业务 Chain 才能提供的壁纸来源。"""
global _tmdb_wallpaper_provider
global _tmdb_wallpaper_list_provider
global _mediaserver_wallpaper_provider
global _mediaserver_wallpaper_list_provider
_tmdb_wallpaper_provider = tmdb_wallpaper
_tmdb_wallpaper_list_provider = tmdb_wallpapers
_mediaserver_wallpaper_provider = mediaserver_wallpaper
_mediaserver_wallpaper_list_provider = mediaserver_wallpapers
class WallpaperHelper(metaclass=Singleton):
"""
壁纸帮助类
"""
def get_wallpaper(self) -> Optional[str]:
"""
获取登录页面壁纸
"""
if settings.WALLPAPER == "bing":
return self.get_bing_wallpaper()
elif settings.WALLPAPER == "mediaserver":
return self.get_mediaserver_wallpaper()
elif settings.WALLPAPER == "customize":
return self.get_customize_wallpaper()
elif settings.WALLPAPER == "tmdb":
return self.get_tmdb_wallpaper()
return ''
def get_wallpapers(self, num: int = 10) -> List[str]:
"""
获取登录页面壁纸列表
"""
if settings.WALLPAPER == "bing":
return self.get_bing_wallpapers(num)
elif settings.WALLPAPER == "mediaserver":
return self.get_mediaserver_wallpapers(num)
elif settings.WALLPAPER == "customize":
return self.get_customize_wallpapers()
elif settings.WALLPAPER == "tmdb":
return self.get_tmdb_wallpapers(num)
return []
@cached(maxsize=1, ttl=3600)
def get_tmdb_wallpaper(self) -> Optional[str]:
"""
获取TMDB每日壁纸
"""
return _tmdb_wallpaper_provider()
@cached(maxsize=1, ttl=3600, skip_empty=True)
def get_tmdb_wallpapers(self, num: int = 10) -> List[str]:
"""
获取7天的TMDB每日壁纸
"""
return _tmdb_wallpaper_list_provider(num)
@cached(maxsize=1, ttl=3600)
def get_bing_wallpaper(self) -> Optional[str]:
"""
获取Bing每日壁纸
"""
url = "https://cn.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1"
resp = RequestUtils(timeout=5).get_res(url)
if resp and resp.status_code == 200:
try:
result = resp.json()
if isinstance(result, dict):
for image in result.get('images') or []:
return f"https://cn.bing.com{image.get('url')}" if 'url' in image else ''
except Exception as err:
print(str(err))
return None
@cached(maxsize=1, ttl=3600, skip_empty=True)
def get_bing_wallpapers(self, num: int = 7) -> List[str]:
"""
获取7天的Bing每日壁纸
"""
url = f"https://cn.bing.com/HPImageArchive.aspx?format=js&idx=0&n={num}"
resp = RequestUtils(timeout=5).get_res(url)
if resp and resp.status_code == 200:
try:
result = resp.json()
if isinstance(result, dict):
return [f"https://cn.bing.com{image.get('url')}" for image in result.get('images') or []]
except Exception as err:
print(str(err))
return []
@cached(maxsize=1, ttl=3600)
def get_mediaserver_wallpaper(self) -> Optional[str]:
"""
获取媒体服务器壁纸
"""
return _mediaserver_wallpaper_provider()
@cached(maxsize=1, ttl=3600, skip_empty=True)
def get_mediaserver_wallpapers(self, num: int = 10) -> List[str]:
"""
获取媒体服务器壁纸列表
"""
return _mediaserver_wallpaper_list_provider(num)
@cached(maxsize=1, ttl=3600)
def get_customize_wallpaper(self) -> Optional[str]:
"""
获取自定义壁纸api壁纸
"""
wallpaper_list = self.get_customize_wallpapers()
if wallpaper_list:
return wallpaper_list[0]
return None
@cached(maxsize=1, ttl=3600, skip_empty=True)
def get_customize_wallpapers(self) -> List[str]:
"""
获取自定义壁纸api壁纸
"""
def find_files_with_suffixes(obj, suffixes: List[str]) -> List[str]:
"""
递归查找对象中所有包含特定后缀的文件,返回匹配的字符串列表
支持输入:字典、列表、字符串
"""
_result = []
# 处理字符串
if isinstance(obj, str):
if obj.endswith(tuple(suffixes)):
_result.append(obj)
# 处理字典
elif isinstance(obj, dict):
for value in obj.values():
_result.extend(find_files_with_suffixes(value, suffixes))
# 处理列表
elif isinstance(obj, list):
for item in obj:
_result.extend(find_files_with_suffixes(item, suffixes))
return _result
# 判断是否存在自定义壁纸api
if settings.CUSTOMIZE_WALLPAPER_API_URL:
wallpaper_list = []
resp = RequestUtils(timeout=15).get_res(settings.CUSTOMIZE_WALLPAPER_API_URL)
if resp and resp.status_code == 200:
# 如果返回的是图片格式
content_type = resp.headers.get('Content-Type')
if content_type and content_type.lower().startswith('image/'):
wallpaper_list.append(settings.CUSTOMIZE_WALLPAPER_API_URL)
else:
try:
result = resp.json()
if isinstance(result, list) or isinstance(result, dict) or isinstance(result, str):
wallpaper_list = find_files_with_suffixes(result, settings.SECURITY_IMAGE_SUFFIXES)
except Exception as err:
print(str(err))
return wallpaper_list
else:
return []
class ImageHelper(metaclass=Singleton):
"""统一管理同步和异步图片缓存。"""
def __init__(self):
"""按全局图片缓存天数初始化文件缓存。"""
_base_path = settings.CACHE_PATH
_ttl = settings.GLOBAL_IMAGE_CACHE_DAYS * 24 * 3600
self.file_cache = FileCache(base=_base_path, ttl=_ttl)
self.async_file_cache = AsyncFileCache(base=_base_path, ttl=_ttl)
@staticmethod
def _prepare_cache_path(url: str) -> str:
"""缓存路径"""
sanitized_path = SecurityUtils.sanitize_url_path(url)
cache_path = Path(sanitized_path)
if not cache_path.suffix:
cache_path = cache_path.with_suffix(".jpg")
return cache_path.as_posix()
@staticmethod
def get_image_mime_type(content: bytes, verify: bool = True) -> Optional[str]:
"""
根据图片内容返回 Pillow 识别的图片 MIME 类型。
外部响应在写入缓存前需要完整校验;已校验的缓存只需读取格式头。
非图片或可脚本化的 MIME 类型不作为图片代理响应。
"""
if not content:
return None
try:
with Image.open(io.BytesIO(content)) as image:
image_format = (image.format or "").upper()
if verify:
image.verify()
mime_type = Image.MIME.get(image_format)
if (
not mime_type
or not mime_type.startswith("image/")
or mime_type == "image/svg+xml"
):
return None
return mime_type
except Exception as err:
logger.warning(f"Invalid image format: {err}")
return None
@staticmethod
def _get_request_params(url: str, proxy: Optional[bool], cookies: Optional[str | dict]) -> dict:
"""获取参数"""
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
if proxy is None:
proxies = settings.PROXY if not (referer or IpUtils.is_internal(url)) else None
else:
proxies = settings.PROXY if proxy else None
return {
"ua": settings.NORMAL_USER_AGENT,
"proxies": proxies,
"referer": referer,
"cookies": cookies,
"accept_type": "image/avif,image/webp,image/apng,*/*",
}
def fetch_image(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None) -> Optional[bytes]:
"""
获取图片(同步版本)
"""
result = self.fetch_image_with_mime_type(
url=url,
proxy=proxy,
use_cache=use_cache,
cookies=cookies,
)
return result[0] if result else None
def fetch_image_with_mime_type(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None,
) -> Optional[tuple[bytes, str]]:
"""
同步获取图片及其内容识别 MIME 类型。
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
"""
if not url:
return None
cache_path = self._prepare_cache_path(url)
# 检查缓存
if use_cache:
content = self.file_cache.get(cache_path, region="images")
if content:
mime_type = self.get_image_mime_type(content, verify=False)
if mime_type:
return content, mime_type
# 请求远程图片
params = self._get_request_params(url, proxy, cookies)
response = RequestUtils(**params).get_res(url=url)
if response is None or response.status_code != 200:
logger.warn(f"Failed to fetch image from URL: {url}")
return None
content = response.content
mime_type = self.get_image_mime_type(content)
if not mime_type:
return None
# 保存缓存
self.file_cache.set(cache_path, content, region="images")
return content, mime_type
async def async_fetch_image(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None) -> Optional[bytes]:
"""
获取图片(异步版本)
"""
result = await self.async_fetch_image_with_mime_type(
url=url,
proxy=proxy,
use_cache=use_cache,
cookies=cookies,
)
return result[0] if result else None
async def async_fetch_image_with_mime_type(
self,
url: str,
proxy: Optional[bool] = None,
use_cache: bool = True,
cookies: Optional[str | dict] = None,
) -> Optional[tuple[bytes, str]]:
"""
异步获取图片及其内容识别 MIME 类型。
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
"""
if not url:
return None
cache_path = self._prepare_cache_path(url)
# 检查缓存
if use_cache:
content = await self.async_file_cache.get(cache_path, region="images")
if content:
mime_type = self.get_image_mime_type(content, verify=False)
if mime_type:
return content, mime_type
# 请求远程图片
params = self._get_request_params(url, proxy, cookies)
response = await AsyncRequestUtils(**params).get_res(url=url)
if response is None or response.status_code != 200:
logger.warn(f"Failed to fetch image from URL: {url}")
return None
content = response.content
mime_type = self.get_image_mime_type(content)
if not mime_type:
return None
# 保存缓存
await self.async_file_cache.set(cache_path, content, region="images")
return content, mime_type

View File

@@ -0,0 +1,268 @@
import re
from collections.abc import Iterable, Mapping
from typing import Any, Optional
from app import schemas
from app.domain.context import MusicInfo
from app.domain.media import normalize_media_source, resolve_media_identity
from app.runtime.extensions.service_registry import ServiceBaseHelper
from app.schemas import MediaServerConf, ServiceInfo
from app.schemas.types import (
MUSIC_ENTITY_ALBUM,
MediaSource,
ModuleType,
SystemConfigKey,
)
class MediaServerIdentityHelper:
"""将媒体服务器专有 ProviderIds 适配为统一媒体身份。"""
_provider_keys = (
(MediaSource.TMDB, ("Tmdb", "TMDB", "tmdb", "tmdb_id")),
(MediaSource.Douban, ("Douban", "douban", "douban_id")),
(MediaSource.Bangumi, ("Bangumi", "bangumi", "bangumi_id")),
(MediaSource.AniList, ("AniList", "Anilist", "anilist", "anilist_id")),
(MediaSource.IMDb, ("Imdb", "IMDb", "imdb", "imdb_id")),
(MediaSource.TVDB, ("Tvdb", "TVDB", "tvdb", "tvdb_id")),
(MediaSource.MusicBrainz, ("MusicBrainz", "musicbrainz", "musicbrainz_id")),
)
@classmethod
def from_provider_ids(
cls,
provider_ids: Optional[Mapping[str, Any]],
) -> tuple[Optional[MediaSource], Optional[str]]:
"""按固定优先级从外部 ProviderIds 选择一个规范媒体身份。"""
if not isinstance(provider_ids, Mapping):
return None, None
for media_source, keys in cls._provider_keys:
for key in keys:
value = provider_ids.get(key)
if value is not None and str(value).strip():
return media_source, str(value).strip()
return None, None
@staticmethod
def are_compatible(
left_source: Optional[MediaSource | str],
left_id: Optional[str],
right_source: Optional[MediaSource | str],
right_id: Optional[str],
) -> bool:
"""判断两组身份是否没有可证实的同来源 ID 冲突。"""
left_source, left_id = resolve_media_identity(
media_source=left_source,
media_id=left_id,
)
right_source, right_id = resolve_media_identity(
media_source=right_source,
media_id=right_id,
)
if not left_source or not right_source:
return True
if normalize_media_source(left_source) != normalize_media_source(right_source):
return True
return left_id == right_id
@classmethod
def is_compatible(
cls,
item: schemas.MediaServerItem,
media_source: Optional[MediaSource | str],
media_id: Optional[str],
) -> bool:
"""判断目标与媒体库条目是否无明确身份冲突。"""
item_source, item_id = resolve_media_identity(media=item)
return cls.are_compatible(item_source, item_id, media_source, media_id)
class MusicMediaServerHelper:
"""统一音乐媒体库条目的字段转换、精确匹配和整专完整性判断。"""
_name_pattern = re.compile(r"[\W_]+", re.UNICODE)
@classmethod
def normalize_name(cls, value: Optional[str]) -> str:
"""忽略大小写、空白和标点,生成用于音乐名称精确比较的稳定文本。"""
return cls._name_pattern.sub("", str(value or "").casefold())
@classmethod
def same_name(cls, left: Optional[str], right: Optional[str]) -> bool:
"""判断两个非空音乐名称在规范化后是否完全一致。"""
normalized_left = cls.normalize_name(left)
normalized_right = cls.normalize_name(right)
return bool(normalized_left) and normalized_left == normalized_right
@staticmethod
def _first_value(data: Mapping[str, Any], *keys: str) -> Any:
"""按候选键顺序返回第一个非空字段,兼容不同媒体服务器命名。"""
for key in keys:
value = data.get(key)
if value not in (None, "", []):
return value
return None
@classmethod
def _extract_names(cls, value: Any) -> list[str]:
"""从字符串、对象列表或名称列表中提取非空名称。"""
if isinstance(value, str):
return [value] if value.strip() else []
if isinstance(value, Mapping):
name = cls._first_value(value, "Name", "name", "Title", "title")
return [str(name)] if name and str(name).strip() else []
if not isinstance(value, Iterable) or isinstance(value, bytes):
return []
names: list[str] = []
for item in value:
if isinstance(item, Mapping):
name = cls._first_value(item, "Name", "name", "Title", "title")
else:
name = item
if name and str(name).strip():
names.append(str(name))
return names
@classmethod
def build_note(cls, item: Mapping[str, Any]) -> dict[str, Any]:
"""把 Emby 系和 NAS 搜索结果中的音乐字段转换为统一备注结构。"""
artists = cls._extract_names(
cls._first_value(item, "Artists", "artists", "ArtistItems", "artist_items")
)
album_artists = cls._extract_names(
cls._first_value(item, "AlbumArtists", "album_artists")
)
artist = cls._first_value(
item,
"AlbumArtist",
"album_artist",
"Artist",
"artist",
"artist_name",
"singer",
)
explicit_artists = cls._extract_names(artist)
if explicit_artists:
artist = explicit_artists[0]
if not artist:
artist = next(iter(album_artists or artists), None)
item_type = cls.normalize_name(
cls._first_value(item, "Type", "type", "item_type")
)
album = cls._first_value(item, "Album", "album", "album_name")
if not album and item_type in {"musicalbum", "album"}:
album = cls._first_value(item, "Name", "name", "Title", "title")
song_count = cls._first_value(
item,
"ChildCount",
"child_count",
"SongCount",
"songCount",
"song_count",
"TrackCount",
"trackCount",
"track_count",
"LeafCount",
"leafCount",
)
return {
"artist": str(artist) if artist is not None else None,
"artists": artists or album_artists,
"album": str(album) if album is not None else None,
"song_count": song_count,
}
@staticmethod
def search_params(mediainfo: MusicInfo) -> dict[str, Optional[str]]:
"""按单曲或专辑实体构造媒体服务器音乐搜索参数。"""
is_album = getattr(mediainfo, "music_type", None) == MUSIC_ENTITY_ALBUM
artists = getattr(mediainfo, "artists", None) or []
artist = (
getattr(mediainfo, "album_artist", None)
or next(iter(artists), None)
or getattr(mediainfo, "artist", None)
)
title = getattr(mediainfo, "title", None)
album = getattr(mediainfo, "album", None) or title
return {
"title": None if is_album else title,
"artist": artist,
"album": album if is_album else None,
}
@classmethod
def item_matches(
cls,
mediainfo: MusicInfo,
item: schemas.MediaServerItem,
) -> bool:
"""校验媒体库条目是否精确对应单曲,或完整覆盖目标专辑。"""
note = item.note if isinstance(item.note, Mapping) else {}
is_album = getattr(mediainfo, "music_type", None) == MUSIC_ENTITY_ALBUM
target_title = getattr(mediainfo, "title", None)
actual_title = item.title
if is_album:
target_title = getattr(mediainfo, "album", None) or target_title
actual_title = note.get("album") or actual_title
if not cls.same_name(actual_title, target_title):
return False
target_artists = [
getattr(mediainfo, "artist", None),
getattr(mediainfo, "album_artist", None),
*(getattr(mediainfo, "artists", None) or []),
]
target_artists = [artist for artist in target_artists if artist]
actual_artists = [note.get("artist"), *cls._extract_names(note.get("artists"))]
actual_artists = [artist for artist in actual_artists if artist]
if target_artists and not any(
cls.same_name(actual, target)
for actual in actual_artists
for target in target_artists
):
return False
if not is_album:
return True
try:
expected_tracks = int(getattr(mediainfo, "total_tracks", None) or 0)
actual_tracks = int(note.get("song_count") or 0)
except (TypeError, ValueError):
return False
return expected_tracks > 0 and actual_tracks >= expected_tracks
@classmethod
def find_match(
cls,
mediainfo: MusicInfo,
items: Optional[Iterable[schemas.MediaServerItem]],
) -> Optional[schemas.MediaServerItem]:
"""返回首个满足单曲精确匹配或整专完整性要求的媒体库条目。"""
return next(
(item for item in items or [] if item and cls.item_matches(mediainfo, item)),
None,
)
class MediaServerHelper(ServiceBaseHelper[MediaServerConf]):
"""管理媒体服务器配置,并按类型发现已启用的服务实例。"""
def __init__(self) -> None:
"""绑定媒体服务器配置键、配置模型和模块类型。"""
super().__init__(
config_key=SystemConfigKey.MediaServers,
conf_type=MediaServerConf,
module_type=ModuleType.MediaServer,
)
def is_media_server(
self,
service_type: Optional[str] = None,
service: Optional[ServiceInfo] = None,
name: Optional[str] = None,
) -> bool:
"""判断给定服务或服务名称是否属于指定媒体服务器类型。"""
service = service or self.get_service(name=name)
return bool(service and service.type == service_type)

View File

@@ -0,0 +1 @@
"""Agent 消息桥接、消息模板、交互、通知和推送能力。"""

View File

@@ -0,0 +1,246 @@
from queue import Queue
from threading import Lock
from typing import Callable, Iterable, Optional, Union
from app.schemas.types import MessageChannel
_WEB_AGENT_EDIT_QUEUES: dict[str, list[Queue[dict]]] = {}
_WEB_AGENT_EDIT_LOCK = Lock()
_ChannelAdminResolver = Callable[[Optional[dict]], Iterable[Union[str, int]]]
_CHANNEL_ADMIN_RESOLVERS: dict[str, _ChannelAdminResolver] = {}
def register_channel_admin_resolver(
channel: Union[MessageChannel, str],
resolver: _ChannelAdminResolver,
) -> None:
"""
注册消息渠道的管理员主体 ID 解析器。
:param channel: 消息渠道
:param resolver: 由渠道配置解析全部管理员主体 ID 的函数
"""
channel_value = channel.value if isinstance(channel, MessageChannel) else str(channel)
_CHANNEL_ADMIN_RESOLVERS[channel_value] = resolver
def resolve_config_principal_ids(
config: Optional[dict],
*config_keys: str,
) -> set[str]:
"""
从渠道自行声明的配置键中解析主体 ID。
:param config: 当前消息渠道配置
:param config_keys: 由渠道模块维护的主体 ID 配置键
:return: 去空白后的主体 ID 集合
"""
principal_ids = set()
for config_key in config_keys:
principal_ids.update(
item.strip()
for item in str((config or {}).get(config_key) or "").split(",")
if item.strip()
)
return principal_ids
def matches_channel_admin(
channel: Union[MessageChannel, str],
config: Optional[dict],
*principal_ids: Optional[Union[str, int]],
) -> bool:
"""
按渠道配置中的稳定主体 ID 判断管理员身份。
:param channel: 消息渠道
:param config: 当前消息渠道配置
:param principal_ids: 消息渠道提供的稳定用户主体 ID
:return: 任一用户主体 ID 命中渠道注册的管理员集合时返回 True
"""
channel_value = channel.value if isinstance(channel, MessageChannel) else str(channel)
resolver = _CHANNEL_ADMIN_RESOLVERS.get(channel_value)
if not resolver:
return False
authorized_ids = {
str(principal_id).strip()
for principal_id in resolver(config)
if principal_id is not None and str(principal_id).strip()
}
if not authorized_ids:
return False
candidates = {
str(principal_id).strip()
for principal_id in principal_ids
if principal_id is not None and str(principal_id).strip()
}
return bool(authorized_ids.intersection(candidates))
def normalize_web_agent_button_rows(buttons: Optional[list[list[dict]]]) -> list[list[dict]]:
"""
将消息按钮转换为 WebAgent 前端可识别的按钮行。
:param buttons: 传统消息模块返回的按钮二维数组
:return: WebAgent 前端选项按钮二维数组
"""
button_rows: list[list[dict]] = []
for row in buttons or []:
normalized_row = []
for button in row or []:
label = str(button.get("text") or button.get("label") or "").strip()
callback_data = str(button.get("callback_data") or "").strip()
if not label or not callback_data:
continue
normalized_button = {
"label": label,
"callback_data": callback_data,
}
if button.get("description"):
normalized_button["description"] = str(button.get("description"))
normalized_row.append(normalized_button)
if normalized_row:
button_rows.append(normalized_row)
return button_rows
def _resolve_web_agent_choice_id(
message_id: Union[str, int],
button_rows: list[list[dict]],
) -> str:
"""
从按钮回调中提取稳定的 WebAgent 选项 ID。
:param message_id: 前端助手消息 ID
:param button_rows: 已规范化的按钮行
:return: 选项卡片 ID
"""
for row in button_rows:
for button in row:
callback_data = str(button.get("callback_data") or "").strip()
if not callback_data:
continue
parts = callback_data.split(":")
if len(parts) >= 2 and parts[1]:
return parts[1]
return callback_data
return str(message_id)
def build_web_agent_message_update_event(
*,
message_id: Union[str, int],
title: Optional[str],
text: str,
buttons: Optional[list[list[dict]]],
) -> dict:
"""
构造 WebAgent 原消息更新事件。
:param message_id: 前端助手消息 ID
:param title: 更新后的标题
:param text: 更新后的正文
:param buttons: 更新后的按钮
:return: 前端可应用到原消息的 SSE 事件
"""
button_rows = normalize_web_agent_button_rows(buttons)
content_parts = [part for part in (title, text) if part]
target_message = {
"id": str(message_id),
"content": "" if button_rows else "\n\n".join(content_parts),
"choices": [],
"attachments": [],
"tools": [],
"status": "done",
}
if button_rows:
target_message["choices"].append({
"id": _resolve_web_agent_choice_id(message_id, button_rows),
"title": title,
"prompt": text or "",
"buttons": [button for row in button_rows for button in row],
"button_rows": button_rows,
"status": "pending",
})
return {
"type": "message_update",
"target_message": target_message,
}
def attach_web_agent_edit_queue(user_id: str, edit_queue: Queue[dict]) -> None:
"""
为当前 WebAgent 请求挂载原消息编辑事件队列。
:param user_id: 当前用户 ID
:param edit_queue: 用于接收编辑事件的队列
"""
with _WEB_AGENT_EDIT_LOCK:
_WEB_AGENT_EDIT_QUEUES.setdefault(str(user_id), []).append(edit_queue)
def detach_web_agent_edit_queue(user_id: str, edit_queue: Queue[dict]) -> None:
"""
移除当前 WebAgent 请求的原消息编辑事件队列。
:param user_id: 当前用户 ID
:param edit_queue: 需要移除的队列
"""
with _WEB_AGENT_EDIT_LOCK:
queues = _WEB_AGENT_EDIT_QUEUES.get(str(user_id))
if not queues:
return
_WEB_AGENT_EDIT_QUEUES[str(user_id)] = [
item for item in queues if item is not edit_queue
]
if not _WEB_AGENT_EDIT_QUEUES[str(user_id)]:
_WEB_AGENT_EDIT_QUEUES.pop(str(user_id), None)
def dispatch_web_agent_edit_event(
*,
user_id: str,
event: dict,
) -> bool:
"""
将 WebAgent 原消息编辑事件分发给正在等待的请求队列。
:param user_id: 当前用户 ID
:param event: 前端可应用的 SSE 事件
:return: 是否存在接收本次编辑事件的请求队列
"""
with _WEB_AGENT_EDIT_LOCK:
queues = list(_WEB_AGENT_EDIT_QUEUES.get(str(user_id)) or [])
for edit_queue in queues:
edit_queue.put(event)
return bool(queues)
def edit_web_agent_message(
*,
user_id: str,
message_id: Union[str, int],
title: Optional[str],
text: str,
buttons: Optional[list[list[dict]]] = None,
) -> bool:
"""
原地更新 WebAgent 前端消息卡片。
:param user_id: 当前用户 ID
:param message_id: 前端助手消息 ID
:param title: 更新后的标题
:param text: 更新后的正文
:param buttons: 更新后的按钮
:return: 是否已投递编辑事件
"""
if not user_id:
return False
event = build_web_agent_message_update_event(
message_id=message_id,
title=title,
text=text,
buttons=buttons,
)
return dispatch_web_agent_edit_event(user_id=user_id, event=event)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,34 @@
from typing import Optional
from app.runtime.extensions.service_registry import ServiceBaseHelper
from app.schemas import NotificationConf, ServiceInfo
from app.schemas.types import ModuleType, SystemConfigKey
class NotificationHelper(ServiceBaseHelper[NotificationConf]):
"""提供按持久化配置发现通知服务的能力。"""
def __init__(self):
"""绑定通知配置和通知模块类型。"""
super().__init__(
config_key=SystemConfigKey.Notifications,
conf_type=NotificationConf,
module_type=ModuleType.Notification,
)
def is_notification(
self,
service_type: Optional[str] = None,
service: Optional[ServiceInfo] = None,
name: Optional[str] = None,
) -> bool:
"""
判断通知服务是否属于指定类型。
:param service_type: 消息通知服务的类型名称
:param service: 要判断的服务信息
:param name: 未传入服务信息时用于查询的服务名称
:return: 服务存在且类型匹配时返回 True
"""
service = service or self.get_service(name=name)
return bool(service and service.type == service_type)

View File

@@ -0,0 +1,24 @@
from typing import Optional
from app.db.systemconfig_oper import SystemConfigOper
from app.schemas.types import SystemConfigKey
class RecognitionRuleService:
"""集中读取用户持久化的媒体识别规则,供启动层注入纯领域匹配器。"""
def __init__(self, systemconfig: Optional[SystemConfigOper] = None) -> None:
"""绑定系统配置访问器,测试可传入隔离替身。"""
self._systemconfig = systemconfig or SystemConfigOper()
def get_customization(self) -> object:
"""返回当前自定义占位符配置。"""
return self._systemconfig.get(SystemConfigKey.Customization)
def get_release_groups(self) -> object:
"""返回当前用户自定义制作组配置。"""
return self._systemconfig.get(SystemConfigKey.CustomReleaseGroups)
def get_custom_words(self) -> object:
"""返回当前自定义识别词配置。"""
return self._systemconfig.get(SystemConfigKey.CustomIdentifiers)

530
app/application/rss.py Normal file
View File

@@ -0,0 +1,530 @@
import re
import traceback
from typing import List, Tuple, Union, Optional
from urllib.parse import urljoin, urlparse
import dateutil.parser
from lxml import etree
from app.runtime.config import settings
from app.adapters.network.browser import PlaywrightHelper
from app.runtime.log import logger
from app.adapters.system import rust as rust_accel
from app.adapters.network.http import RequestUtils
class RssHelper:
"""
RSS帮助类解析RSS报文、获取RSS地址等
"""
# RSS解析限制配置
MAX_RSS_SIZE = 50 * 1024 * 1024 # 50MB最大RSS文件大小
MAX_RSS_ITEMS = 1000 # 最大解析条目数
# 各站点RSS链接获取配置
rss_link_conf = {
"default": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"hares.top": {
"xpath": "//*[@id='layui-layer100001']/div[2]/div/p[4]/a/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"et8.org": {
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/a[2]/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"pttime.org": {
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/text()[5]",
"url": "getrss.php",
"params": {
"showrows": 10,
"inclbookmarked": 0,
"itemsmalldescr": 1
}
},
"ourbits.club": {
"xpath": "//a[@class='gen_rsslink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"totheglory.im": {
"xpath": "//textarea/text()",
"url": "rsstools.php?c51=51&c52=52&c53=53&c54=54&c108=108&c109=109&c62=62&c63=63&c67=67&c69=69&c70=70&c73=73&c76=76&c75=75&c74=74&c87=87&c88=88&c99=99&c90=90&c58=58&c103=103&c101=101&c60=60",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"monikadesign.uk": {
"xpath": "//a/@href",
"url": "rss",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"zhuque.in": {
"xpath": "//a/@href",
"url": "user/rss",
"render": True,
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
}
},
"hdchina.org": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"rsscart": 0
}
},
"audiences.me": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"torrent_type": 1,
"exp": 180
}
},
"shadowflow.org": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"paid": 0,
"search_mode": 0,
"showrows": 30
}
},
"hddolby.com": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"exp": 180
}
},
"hdhome.org": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"exp": 180
}
},
"pthome.net": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"exp": 180
}
},
"ptsbao.club": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"size": 0
}
},
"leaves.red": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 0,
"paid": 2
}
},
"hdtime.org": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 0,
}
},
"m-team.io": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"showrows": 50,
"inclbookmarked": 0,
"itemsmalldescr": 1,
"https": 1
}
},
"u2.dmhy.org": {
"xpath": "//a[@class='faqlink']/@href",
"url": "getrss.php",
"params": {
"inclbookmarked": 0,
"itemsmalldescr": 1,
"showrows": 50,
"search_mode": 1,
"inclautochecked": 1,
"trackerssl": 1
}
},
}
@classmethod
def _get_site_domain(cls, url: str) -> str:
"""按 RSS 站点配置匹配域名,未命中时回退到最后两级域名。"""
hostname = (urlparse(url).hostname or "").lower()
for domain in cls.rss_link_conf:
if domain == "default":
continue
if hostname == domain or hostname.endswith(f".{domain}"):
return domain
parts = hostname.split(".")
return ".".join(parts[-2:]) if len(parts) >= 2 else hostname
@staticmethod
def _parse_publish_time(value: str):
"""将 RSS 常见日期表达解析为 datetime无法解析时返回 None。"""
try:
return dateutil.parser.parse(value)
except dateutil.parser.ParserError:
return None
def __parse_with_rust(self, ret_xml: Optional[str]) -> Optional[list]:
"""
调用 Rust RSS 解析器,并统一处理基础 XML 校验和最大条目限制。
"""
if not ret_xml or not ret_xml.strip():
return None
ret_xml_stripped = ret_xml.strip()
if not ret_xml_stripped.startswith('<'):
return None
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
if rust_items is None:
return None
if len(rust_items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}")
return rust_items[:self.MAX_RSS_ITEMS]
def parse(self, url, proxy: bool = False,
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
"""
解析RSS订阅URL获取RSS中的种子信息
:param url: RSS地址
:param proxy: 是否使用代理
:param timeout: 请求超时
:param headers: 自定义请求头
:param ua: 自定义User-Agent
:return: 种子信息列表如为None代表Rss过期如果为False则为错误
"""
# 开始处理
ret_array: list = []
if not url:
return False
try:
ret = RequestUtils(ua=ua,
proxies=settings.PROXY if proxy else None,
timeout=timeout or 30, headers=headers).get_res(url)
if not ret:
logger.error(f"获取RSS失败请求返回空值URL: {url}")
return False
except Exception as err:
logger.error(f"获取RSS失败{str(err)} - {traceback.format_exc()}")
return False
if ret:
# 检查HTTP状态码
if ret.status_code != 200:
logger.error(f"RSS请求失败状态码: {ret.status_code}, URL: {url}")
return False
ret_xml = None
root = None
try:
# 检查响应大小避免处理过大的RSS文件
raw_data = ret.content
if raw_data and len(raw_data) > self.MAX_RSS_SIZE:
logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB跳过解析")
return False
if raw_data:
ret_xml = RequestUtils.get_decoded_xml_content(
ret,
performance_mode=settings.ENCODING_DETECTION_PERFORMANCE_MODE,
confidence_threshold=settings.ENCODING_DETECTION_MIN_CONFIDENCE
)
rust_items = self.__parse_with_rust(ret_xml)
if rust_items is not None:
return rust_items
if not ret_xml:
ret_xml = ret.text
# 验证RSS内容是否有效
if not ret_xml or not ret_xml.strip():
logger.error("RSS内容为空")
return False
# 检查是否包含基本的RSS/XML结构
ret_xml_stripped = ret_xml.strip()
if not ret_xml_stripped.startswith('<'):
logger.error("RSS内容不是有效的XML格式")
return False
rust_items = self.__parse_with_rust(ret_xml)
if rust_items is not None:
return rust_items
# 使用lxml.etree解析XML
parser = None
try:
# 创建解析器,禁用网络访问以提高安全性和性能
parser = etree.XMLParser(
recover=True, # 容错模式
strip_cdata=False, # 保留CDATA
resolve_entities=False, # 禁用外部实体解析
no_network=True, # 禁用网络访问
huge_tree=False # 禁用大文档解析,避免内存问题
)
root = etree.fromstring(ret_xml.encode('utf-8'), parser=parser)
except etree.XMLSyntaxError as xml_error:
logger.debug(f"XML解析失败{str(xml_error)}尝试HTML解析")
# 如果XML解析失败尝试作为HTML解析
try:
root = etree.HTML(ret_xml)
if root is not None:
# 查找RSS根节点
rss_root = root.xpath('//rss | //feed')
if rss_root:
root = rss_root[0]
except Exception as e:
logger.error(f"HTML解析也失败{str(e)}")
return False
except Exception as general_error:
logger.error(f"解析RSS时发生未预期错误{str(general_error)}")
return False
finally:
if parser is not None:
try:
parser.close()
except Exception as close_error:
logger.debug(f"关闭解析器时出错:{str(close_error)}")
del parser
if root is None:
logger.error("无法解析RSS内容")
return False
# 查找所有item或entry节点
items = root.xpath('.//item | .//entry')
# 限制处理的条目数量
items_count = min(len(items), self.MAX_RSS_ITEMS)
if len(items) > self.MAX_RSS_ITEMS:
logger.warning(f"RSS条目过多: {len(items)},仅处理前{self.MAX_RSS_ITEMS}")
try:
for item in items[:items_count]:
try:
# 使用xpath提取信息更高效
title_nodes = item.xpath('.//title')
title = title_nodes[0].text if title_nodes and title_nodes[0].text else ""
if not title:
continue
# 描述
desc_nodes = item.xpath('.//description | .//summary')
description = desc_nodes[0].text if desc_nodes and desc_nodes[0].text else ""
# 种子页面
link_nodes = item.xpath('.//link')
if link_nodes:
link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else link_nodes[0].get('href', '')
else:
link = ""
# 种子链接
enclosure_nodes = item.xpath('.//enclosure')
enclosure = enclosure_nodes[0].get('url', '') if enclosure_nodes else ""
if not enclosure and not link:
continue
# 部分RSS只有link没有enclosure
if not enclosure and link:
enclosure = link
# 大小
size = 0
if enclosure_nodes:
size_attr = enclosure_nodes[0].get('length', '0')
if size_attr and str(size_attr).isdigit():
size = int(size_attr)
# 发布日期
pubdate_nodes = item.xpath('./pubDate | ./published | ./updated')
if not pubdate_nodes:
pubdate_nodes = item.xpath('.//*[local-name()="pubDate"] | .//*[local-name()="published"] | .//*[local-name()="updated"]')
pubdate = ""
if pubdate_nodes and pubdate_nodes[0].text:
pubdate = self._parse_publish_time(pubdate_nodes[0].text)
if pubdate is not None:
# 转为本地时区
pubdate = pubdate.astimezone(tz=None)
# 获取豆瓣昵称
nickname_nodes = item.xpath('.//*[local-name()="creator"]')
nickname = nickname_nodes[0].text if nickname_nodes and nickname_nodes[0].text else ""
# 返回对象
tmp_dict = {
'title': title,
'enclosure': enclosure,
'size': size,
'description': description,
'link': link,
'pubdate': pubdate
}
# 如果豆瓣昵称不为空返回数据增加豆瓣昵称供doubansync插件获取
if nickname:
tmp_dict['nickname'] = nickname
ret_array.append(tmp_dict)
except Exception as e1:
logger.debug(f"解析RSS条目失败{str(e1)} - {traceback.format_exc()}")
continue
finally:
items.clear()
del items
except Exception as e2:
logger.error(f"解析RSS失败{str(e2)} - {traceback.format_exc()}")
# RSS过期检查
_rss_expired_msg = [
"RSS 链接已过期, 您需要获得一个新的!",
"RSS Link has expired, You need to get a new one!",
"RSS Link has expired, You need to get new!"
]
if ret_xml in _rss_expired_msg:
return None
return False
finally:
if root is not None:
del root
if ret_xml is not None:
del ret_xml
return ret_array
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False, timeout: int = None) -> Tuple[str, str]:
"""
获取站点rss地址
:param url: 站点地址
:param cookie: 站点cookie
:param ua: 站点ua
:param proxy: 是否使用代理
:param timeout: 请求超时时间
:return: rss地址、错误信息
"""
try:
# 获取站点域名
domain = self._get_site_domain(url)
# 获取配置
site_conf = self.rss_link_conf.get(domain) or self.rss_link_conf.get("default")
# RSS地址
rss_url = urljoin(url, site_conf.get("url"))
# RSS请求参数
rss_params = site_conf.get("params")
# 请求RSS页面
if site_conf.get("render"):
html_text = PlaywrightHelper().get_page_source(
url=rss_url,
cookies=cookie,
ua=ua,
proxies=settings.PROXY_SERVER if proxy else None,
timeout=timeout or 60
)
else:
res = RequestUtils(
cookies=cookie,
timeout=timeout or 30,
ua=ua,
proxies=settings.PROXY if proxy else None
).post_res(url=rss_url, data=rss_params)
if res:
html_text = res.text
elif res is not None:
return "", f"获取 {url} RSS链接失败错误码{res.status_code},错误原因:{res.reason}"
else:
return "", f"获取RSS链接失败无法连接 {url} "
# 解析HTML
if html_text:
html = None
try:
html = etree.HTML(html_text)
if html is not None and len(html) > 0:
rss_link = html.xpath(site_conf.get("xpath"))
if rss_link:
return str(rss_link[-1]), ""
finally:
if html is not None:
del html
return "", f"获取RSS链接失败{url}"
except Exception as e:
return "", f"获取 {url} RSS链接失败{str(e)}"

View File

@@ -0,0 +1 @@
"""认证授权、URL 安全、OTP、Cookie、Passkey 和双因素认证能力。"""

View File

@@ -0,0 +1,442 @@
import base64
import datetime
import hashlib
import hmac
import json
import os
import traceback
from datetime import timedelta
from typing import Any, Union, Annotated, Optional, Callable
import jwt
from Crypto.Cipher import AES
from Crypto.Util.Padding import pad
from cryptography.fernet import Fernet
from fastapi import HTTPException, status, Security, Request, Response
from fastapi.security import OAuth2PasswordBearer, APIKeyHeader, APIKeyQuery, APIKeyCookie, HTTPBearer
from passlib.context import CryptContext
from app import schemas
from app.runtime.cache import cached
from app.runtime.config import settings
from app.runtime.log import logger
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
ALGORITHM = "HS256"
SuperuserTokenPayloadProvider = Callable[[], schemas.TokenPayload]
_superuser_token_payload_provider: Optional[SuperuserTokenPayloadProvider] = None
def set_superuser_token_payload_provider(
provider: SuperuserTokenPayloadProvider,
) -> None:
"""注入 API 密钥认证所需的超级用户载荷提供器。"""
global _superuser_token_payload_provider
_superuser_token_payload_provider = provider
# OAuth2PasswordBearer 用于 JWT Token 认证
oauth2_scheme_manual_error = OAuth2PasswordBearer(
auto_error=False, # 禁用自动错误处理用以支持API令牌鉴权
tokenUrl=f"{settings.API_V1_STR}/login/access-token"
)
# RESOURCE TOKEN 通过 Cookie 认证
resource_token_cookie = APIKeyCookie(name=settings.PROJECT_NAME, auto_error=False, scheme_name="resource_token_cookie")
# API TOKEN 通过 QUERY 认证
api_token_query = APIKeyQuery(name="token", auto_error=False, scheme_name="api_token_query")
# API KEY 通过 Header 认证
api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False, scheme_name="api_key_header")
# API KEY 通过 QUERY 认证
api_key_query = APIKeyQuery(name="apikey", auto_error=False, scheme_name="api_key_query")
# OpenAI compatible Bearer Token 认证
openai_bearer_scheme = HTTPBearer(auto_error=False)
# Anthropic compatible API Key 认证
anthropic_api_key_header = APIKeyHeader(name="x-api-key", auto_error=False, scheme_name="anthropic_api_key_header")
def __get_api_token(
token_query: Annotated[str | None, Security(api_token_query)] = None
) -> str | None:
"""
从 URL 查询参数中获取 API Token
:param token_query: 从 URL 中的 `token` 查询参数获取 API Token
:return: 返回获取到的 API Token若无则返回 None
"""
return token_query
def __get_api_key(
key_query: Annotated[str | None, Security(api_key_query)] = None,
key_header: Annotated[str | None, Security(api_key_header)] = None
) -> str | None:
"""
从 URL 查询参数或请求头部获取 API Key优先使用请求头
:param key_query: URL 中的 `apikey` 查询参数
:param key_header: 请求头中的 `X-API-KEY` 参数
:return: 返回从 URL 或请求头中获取的 API Key若无则返回 None
"""
return key_header or key_query # 首选请求头
@cached(maxsize=1, ttl=600)
def __create_superuser_token_payload() -> schemas.TokenPayload:
"""
创建管理员用户的TokenPayload
:return: 管理员TokenPayload
"""
if not _superuser_token_payload_provider:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="认证服务尚未初始化",
)
return _superuser_token_payload_provider()
def create_access_token(
userid: Union[str, Any],
username: str,
super_user: Optional[bool] = False,
expires_delta: Optional[timedelta] = None,
level: Optional[int] = 1,
purpose: Optional[str] = "authentication"
) -> str:
"""
创建 JWT 访问令牌,包含用户 ID、用户名、是否为超级用户以及权限等级
:param userid: 用户的唯一标识符,通常是字符串或整数
:param username: 用户名,用于标识用户的账户名
:param super_user: 是否为超级用户,默认值为 False
:param expires_delta: 令牌的有效期时长,如果不提供则根据用途使用默认过期时间
:param level: 用户的权限级别,默认为 1
:param purpose: 令牌的用途,"authentication""resource"
:return: 编码后的 JWT 令牌字符串
:raises ValueError: 如果 expires_delta 为负数
"""
if purpose == "resource":
default_expire = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
secret_key = settings.RESOURCE_SECRET_KEY
else:
default_expire = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
secret_key = settings.SECRET_KEY
if expires_delta is not None:
if expires_delta.total_seconds() <= 0:
raise ValueError("过期时间必须为正数")
expire = datetime.datetime.now(datetime.UTC) + expires_delta
else:
expire = datetime.datetime.now(datetime.UTC) + default_expire
to_encode = {
"exp": expire,
"iat": datetime.datetime.now(datetime.UTC),
"sub": str(userid),
"username": username,
"super_user": super_user,
"level": level,
"purpose": purpose
}
encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM)
return encoded_jwt
def set_or_refresh_resource_token_cookie(
request: Request, response: Response, payload: schemas.TokenPayload
) -> None:
"""
设置资源令牌 Cookie
:param request: 包含请求相关的上下文数据
:param response: 用于在服务器响应时设置 Cookie
:param payload: 已通过身份验证的 TokenPayload 对象
"""
resource_token = request.cookies.get(settings.PROJECT_NAME)
if resource_token:
# 检查令牌剩余时间
try:
decoded_token = jwt.decode(resource_token, settings.RESOURCE_SECRET_KEY, algorithms=[ALGORITHM])
exp = decoded_token.get("exp")
if exp:
remaining_time = datetime.datetime.fromtimestamp(exp, tz=datetime.UTC) - datetime.datetime.now(datetime.UTC)
# 根据剩余时长提前刷新令牌
if remaining_time < timedelta(seconds=(settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS / 3)):
raise jwt.ExpiredSignatureError
expected_claims = {
"sub": str(payload.sub),
"username": payload.username,
"super_user": payload.super_user,
"level": payload.level,
"purpose": "resource",
}
if any(decoded_token.get(claim) != value for claim, value in expected_claims.items()):
raise jwt.InvalidTokenError("资源令牌身份或权限上下文不匹配")
except jwt.PyJWTError:
logger.debug(f"Token error occurred. refreshing token")
except Exception as e:
logger.debug(f"Unexpected error occurred while decoding token: {e}")
else:
# 如果令牌有效且没有即将过期,则不需要刷新
return
# 创建新的资源访问令牌
resource_token_expires = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
resource_token = create_access_token(
userid=payload.sub,
username=payload.username,
super_user=payload.super_user,
expires_delta=resource_token_expires,
level=payload.level,
purpose="resource"
)
# 判断请求是否为 HTTPS直连协议为 https或经反向代理转发时携带 X-Forwarded-Proto: https。
# 无法确认为明文 HTTP 时按 fail-safe 默认设置 secure=True避免代理终止 HTTPS 后以 HTTP 转发导致 Cookie 明文传输。
is_https = (
request.url.scheme == "https"
or request.headers.get("x-forwarded-proto", "").lower() == "https"
)
# 设置会话级别的 HttpOnly Cookie
response.set_cookie(
key=settings.PROJECT_NAME,
value=resource_token,
httponly=True,
secure=is_https, # 根据当前请求协议(含反向代理转发标识)设置 secure 属性
samesite="lax" # 不同浏览器对 "Strict" 的处理可能不同,设置 SameSite 为 "Lax",以平衡安全性和兼容性
)
def __verify_token(token: str, purpose: Optional[str] = "authentication") -> schemas.TokenPayload:
"""
使用 JWT Token 进行身份认证并解析 Token 的内容
:param token: JWT 令牌
:param purpose: 期望的令牌用途,默认为 "authentication"
:return: 包含用户身份信息的 Token 负载数据
:raises HTTPException: 如果令牌无效或用途不匹配
"""
try:
if purpose == "resource":
secret_key = settings.RESOURCE_SECRET_KEY
else:
secret_key = settings.SECRET_KEY
if not token:
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"{purpose} token not found"
)
payload = jwt.decode(
token, secret_key, algorithms=[ALGORITHM]
)
token_payload = schemas.TokenPayload(**payload)
if token_payload.purpose != purpose:
raise jwt.InvalidTokenError("令牌用途不匹配")
return schemas.TokenPayload(**payload)
except (jwt.DecodeError, jwt.InvalidTokenError, jwt.ImmatureSignatureError):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="token校验不通过",
)
def verify_token(
request: Request,
response: Response,
jwt_token: Annotated[str | None, Security(oauth2_scheme_manual_error)],
api_key: Annotated[str | None, Security(__get_api_key)],
api_token: Annotated[str | None, Security(__get_api_token)],
) -> schemas.TokenPayload:
"""
验证 JWT 令牌并自动处理 resource_token 写入
如果缺少JWT令牌再尝试用API令牌鉴权
:param request: 请求对象,用于访问 Cookie 和请求信息
:param response: 响应对象,用于设置 Cookie
:param jwt_token: 从 Authorization 头部获取的 JWT 令牌
:param api_key: 从 查询参数`apikey` 或 请求头`X-API-KEY` 获取 API Token
:param api_token: 从 查询参数`token` 获取 API Token
:return: 解析后的 TokenPayload
:raises HTTPException: 如果令牌无效或用途不匹配
"""
if jwt_token:
# 验证并解析 JWT 认证令牌
payload = __verify_token(token=jwt_token, purpose="authentication")
# 如果没有 resource_token生成并写入到 Cookie
set_or_refresh_resource_token_cookie(request, response, payload)
return payload
elif api_key:
verify_apikey(api_key)
return __create_superuser_token_payload()
elif api_token:
verify_apitoken(api_token)
return __create_superuser_token_payload()
else:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Not authenticated",
headers={"WWW-Authenticate": "Bearer"},
)
def verify_resource_token(
resource_token: Annotated[str, Security(resource_token_cookie)]
) -> schemas.TokenPayload:
"""
验证资源访问令牌(从 Cookie 中获取)
:param resource_token: 从 Cookie 中获取的资源访问令牌
:return: 解析后的 TokenPayload
:raises HTTPException: 如果资源访问令牌无效
"""
# 验证并解析资源访问令牌
return __verify_token(token=resource_token, purpose="resource")
def __verify_key(key: str | None, expected_key: str, key_type: str) -> str:
"""
通用的 API Key 或 Token 验证函数
:param key: 从请求中获取的 API Key 或 Token
:param expected_key: 系统配置中的期望值,用于验证的 API Key 或 Token
:param key_type: 键的类型(例如 "API_KEY""API_TOKEN"),用于错误消息
:return: 返回校验通过的 API Key 或 Token
:raises HTTPException: 如果校验不通过,抛出 401 错误
"""
if not key or key != expected_key:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail=f"{key_type} 校验不通过"
)
return key
def verify_apitoken(token: Annotated[str | None, Security(__get_api_token)]) -> str:
"""
使用 API Token 进行受信第三方集成认证。
校验值来自 settings.API_TOKEN通过后只确认集成凭据有效不生成 per-user 权限上下文。
:param token: API Token从 URL 查询参数中获取 token=xxx
:return: 返回校验通过的 API Token
"""
return __verify_key(token, settings.API_TOKEN, "token")
def verify_apikey(apikey: Annotated[str | None, Security(__get_api_key)]) -> str:
"""
使用 API Key 形式进行受信第三方集成认证。
请求字段名兼容 API Key实际校验值来自 settings.API_TOKEN不生成 per-user 权限上下文。
:param apikey: API Key从 URL 查询参数中获取 apikey=xxx或请求头中获取 X-API-KEY=xxx
:return: 返回校验通过的 API Key
"""
return __verify_key(apikey, settings.API_TOKEN, "apikey")
def verify_password(plain_password: str, hashed_password: str) -> bool:
"""校验明文密码是否匹配已保存的密码摘要。"""
return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password: str) -> str:
"""生成适合持久化保存的密码摘要。"""
return pwd_context.hash(password)
def decrypt(data: bytes, key: bytes) -> Optional[bytes]:
"""
解密二进制数据
"""
fernet = Fernet(key)
try:
return fernet.decrypt(data)
except Exception as e:
logger.error(f"解密失败:{str(e)} - {traceback.format_exc()}")
return None
def encrypt_message(message: str, key: bytes) -> str:
"""
使用给定的key对消息进行加密并返回加密后的字符串
"""
f = Fernet(key)
encrypted_message = f.encrypt(message.encode())
return encrypted_message.decode()
def hash_sha256(message: str) -> str:
"""
对字符串做hash运算
"""
return hashlib.sha256(message.encode()).hexdigest()
def aes_decrypt(data: str, key: str) -> str:
"""
AES解密
"""
if not data:
return ""
data = base64.b64decode(data)
iv = data[:16]
encrypted = data[16:]
# 使用AES-256-CBC解密
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv)
result = cipher.decrypt(encrypted)
# 去除填充
padding = result[-1]
if padding < 1 or padding > AES.block_size:
return ""
result = result[:-padding]
return result.decode('utf-8')
def aes_encrypt(data: str, key: str) -> str:
"""
AES加密
"""
if not data:
return ""
# 使用AES-256-CBC加密
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC)
# 填充
padding = AES.block_size - len(data) % AES.block_size
data += chr(padding) * padding
result = cipher.encrypt(data.encode('utf-8'))
# 使用base64编码
return base64.b64encode(cipher.iv + result).decode('utf-8')
def nexusphp_encrypt(data_str: str, key: bytes) -> str:
"""
NexusPHP加密
"""
# 生成16字节长的随机字符串
iv = os.urandom(16)
# 对向量进行 Base64 编码
iv_base64 = base64.b64encode(iv)
# 加密数据
cipher = AES.new(key, AES.MODE_CBC, iv)
ciphertext = cipher.encrypt(pad(data_str.encode(), AES.block_size))
ciphertext_base64 = base64.b64encode(ciphertext)
# 对向量的字符串表示进行签名
mac = hmac.new(key, msg=iv_base64 + ciphertext_base64, digestmod=hashlib.sha256).hexdigest()
# 构造 JSON 字符串
json_str = json.dumps({
'iv': iv_base64.decode(),
'value': ciphertext_base64.decode(),
'mac': mac,
'tag': ''
})
# 对 JSON 字符串进行 Base64 编码
return base64.b64encode(json_str.encode()).decode()

View File

@@ -0,0 +1,166 @@
import secrets
import threading
import time
from datetime import timedelta
from typing import Any, Optional
from fastapi import HTTPException, status
from app import schemas
from app.application.security import access as security
from app.runtime.config import settings
from app.db.models.user import User
from app.db.systemconfig_oper import SystemConfigOper
from app.db.user_oper import UserOper
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
from app.schemas.types import SystemConfigKey
from app.foundation.singleton import Singleton
class AuthTicketStore(metaclass=Singleton):
"""
插件认证一次性票据存储。
"""
_ttl_seconds = 120
_max_items = 1024
def __init__(self):
"""
初始化内存票据缓存。
"""
self._tickets: dict[str, dict[str, Any]] = {}
self._lock = threading.RLock()
def create(self, user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
"""
创建短时一次性登录票据。
:param user_id: 已通过插件认证的本地用户 ID
:param provider_id: 认证提供方 ID
:param metadata: 插件侧附加信息
:return: 一次性票据字符串
"""
ticket = secrets.token_urlsafe(32)
now = time.time()
with self._lock:
self._cleanup(now)
self._tickets[ticket] = {
"user_id": int(user_id),
"provider_id": provider_id,
"metadata": metadata or {},
"created_at": now,
}
return ticket
def consume(self, ticket: str) -> Optional[dict[str, Any]]:
"""
消费并删除一次性登录票据。
:param ticket: 登录票据
:return: 票据数据,票据不存在或过期时返回 None
"""
if not ticket:
return None
now = time.time()
with self._lock:
data = self._tickets.pop(ticket, None)
self._cleanup(now)
if not data:
return None
if now - float(data.get("created_at") or 0) > self._ttl_seconds:
return None
return data
def _cleanup(self, now: Optional[float] = None) -> None:
"""
清理过期或过量的票据缓存。
:param now: 当前时间戳,未传入时自动读取
"""
current = now or time.time()
expired = [
key
for key, value in self._tickets.items()
if current - float(value.get("created_at") or 0) > self._ttl_seconds
]
for key in expired:
self._tickets.pop(key, None)
if len(self._tickets) <= self._max_items:
return
ordered = sorted(
self._tickets.items(),
key=lambda item: float(item[1].get("created_at") or 0),
)
for key, _ in ordered[: len(self._tickets) - self._max_items]:
self._tickets.pop(key, None)
def create_plugin_auth_ticket(user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
"""
为插件认证成功的用户创建一次性登录票据。
:param user_id: 本地用户 ID
:param provider_id: 认证提供方 ID
:param metadata: 插件侧附加信息
:return: 一次性票据字符串
"""
return AuthTicketStore().create(user_id=user_id, provider_id=provider_id, metadata=metadata)
def consume_plugin_auth_ticket(ticket: str) -> Optional[dict[str, Any]]:
"""
消费插件认证登录票据。
:param ticket: 登录票据
:return: 票据数据,票据不存在或过期时返回 None
"""
return AuthTicketStore().consume(ticket)
def build_superuser_token_payload() -> schemas.TokenPayload:
"""从持久化用户和站点认证状态构造超级用户令牌载荷。"""
user = UserOper().get_by_name(settings.SUPERUSER)
if not user or not user.is_superuser:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="用户权限不足",
)
return schemas.TokenPayload(
sub=user.id,
username=user.name,
super_user=user.is_superuser,
level=SitesHelper().auth_level,
purpose="authentication",
)
def build_token_response(user: User) -> schemas.Token:
"""
使用系统统一逻辑构造登录 Token 响应。
:param user: 已认证的本地用户
:return: 标准 Token 响应
"""
level = SitesHelper().auth_level
show_wizard = (
not SystemConfigOper().get(SystemConfigKey.SetupWizardState)
and not settings.ADVANCED_MODE
)
return schemas.Token(
access_token=security.create_access_token(
userid=user.id,
username=user.name,
super_user=user.is_superuser,
expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES),
level=level,
),
token_type="bearer",
super_user=user.is_superuser,
user_id=user.id,
user_name=user.name,
avatar=user.avatar,
level=level,
permissions=user.permissions or {},
wizard=show_wizard,
)

View File

@@ -0,0 +1,358 @@
import base64
import time
from typing import Tuple, Optional
from lxml import etree
from app.adapters.network.browser import BrowserPage, PlaywrightHelper
from app.adapters.external.ocr import OcrHelper
from app.application.security.twofactor import TwoFactorAuth
from app.runtime.log import logger
from app.adapters.network.http import RequestUtils
from app.domain.site import SiteUtils
from app.domain.string import StringUtils
class CookieHelper:
"""处理站点登录表单、验证码和 Cookie 获取流程。"""
# 站点登录界面元素XPATH
_SITE_LOGIN_XPATH = {
"username": [
'//input[@name="username"]',
'//input[@id="form_item_username"]',
'//input[@id="username"]',
'//input[contains(@placeholder,"用户名")]',
],
"password": [
'//input[@name="password"]',
'//input[@id="form_item_password"]',
'//input[@id="password"]',
'//input[@type="password"]',
],
"captcha": [
'//input[@name="imagestring"]',
'//input[@name="captcha"]',
'//input[@id="form_item_captcha"]',
'//input[@placeholder="驗證碼"]',
],
"captcha_img": [
'//img[@alt="captcha"]/@src',
'//img[@alt="CAPTCHA"]/@src',
'//img[@alt="SECURITY CODE"]/@src',
'//img[@id="LAY-user-get-vercode"]/@src',
'//img[contains(@src,"/api/getCaptcha")]/@src',
],
"submit": [
'//input[@type="submit"]',
'//button[@type="submit"]',
'//button[@lay-filter="login"]',
'//button[@lay-filter="formLogin"]',
'//input[@type="button"][@value="登录"]',
'//input[@id="submit-btn"]',
],
"error": [
"//table[@class='main']//td[@class='text']/text()",
],
"remember": [
'//input[@type="checkbox"][contains(@name,"remember") or contains(@id,"remember")]',
'//*[@role="checkbox"][contains(.,"保持登录") or contains(.,"记住我") or contains(.,"自动登录")]',
],
"twostep": [
'//input[@name="two_step_code"]',
'//input[@name="2fa_secret"]',
'//input[@name="otp"]',
]
}
@staticmethod
def get_page_content(page: BrowserPage, retries: int = 3, interval: float = 1.0) -> Optional[str]:
"""
获取页面源码,页面跳转中(如登录前后的重定向)会导致 page.content() 抛出
"Unable to retrieve content because the page is navigating" 异常,等待加载完成后重试
:param page: 浏览器页面
:param retries: 最大重试次数
:param interval: 重试间隔(秒)
:return: 页面源码
"""
for i in range(retries):
# 等待加载失败不代表源码不可读取,最后一次等待失败时仍尝试直接获取源码
try:
page.wait_for_load_state("domcontentloaded", timeout=10 * 1000)
except Exception as e:
if i < retries - 1:
logger.warning(f"等待页面加载完成失败:{str(e)}{interval}秒后重试 ({i + 1}/{retries - 1})")
time.sleep(interval)
continue
logger.warning(f"等待页面加载完成失败:{str(e)},尝试直接获取源码")
try:
return page.content()
except Exception as e:
if i >= retries - 1:
logger.error(f"获取页面源码失败:{str(e)}")
return None
logger.warning(f"获取页面源码失败:{str(e)}{interval}秒后重试 ({i + 1}/{retries - 1})")
time.sleep(interval)
return None
@staticmethod
def parse_cookies(cookies: list) -> str:
"""
将浏览器返回的cookies转化为字符串
"""
if not cookies:
return ""
cookie_str = ""
for cookie in cookies:
cookie_str += f"{cookie['name']}={cookie['value']}; "
return cookie_str
def get_site_cookie_ua(self,
url: str,
username: str,
password: str,
two_step_code: Optional[str] = None,
proxies: Optional[dict] = None,
timeout: int = None) -> Tuple[Optional[str], Optional[str], str]:
"""
获取站点cookie和ua
:param url: 站点地址
:param username: 用户名
:param password: 密码
:param two_step_code: 二步验证码或密钥
:param proxies: 代理
:param timeout: 超时时间
:return: cookie、ua、message
"""
def __page_handler(page: BrowserPage) -> Tuple[Optional[str], Optional[str], str]:
"""
页面处理
:return: Cookie和UA
"""
# 登录页面代码
html_text = self.get_page_content(page)
if not html_text:
return None, None, "获取源码失败"
# 查找用户名输入框
html = etree.HTML(html_text)
if html is None:
return None, None, "解析网页源码失败"
try:
username_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("username"):
if html.xpath(xpath):
username_xpath = xpath
break
if not username_xpath:
# 登录页可能为JS动态渲染如SPA等待用户名输入框出现后重试
try:
username_union_xpath = " | ".join(self._SITE_LOGIN_XPATH.get("username"))
page.wait_for_selector(f"xpath={username_union_xpath}", timeout=5000)
except Exception:
pass
html_text = self.get_page_content(page)
html = etree.HTML(html_text) if html_text else None
if html is None:
return None, None, "解析网页源码失败"
for xpath in self._SITE_LOGIN_XPATH.get("username"):
if html.xpath(xpath):
username_xpath = xpath
break
if not username_xpath:
return None, None, "未找到用户名输入框"
# 查找密码输入框
password_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("password"):
if html.xpath(xpath):
password_xpath = xpath
break
if not password_xpath:
return None, None, "未找到密码输入框"
# 处理二步验证码
otp_code = TwoFactorAuth(two_step_code).get_code()
# 查找二步验证码输入框
twostep_xpath = None
if otp_code:
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
if html.xpath(xpath):
twostep_xpath = xpath
break
# 查找验证码输入框
captcha_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("captcha"):
if html.xpath(xpath):
captcha_xpath = xpath
break
# 查找验证码图片
captcha_img_url = None
if captcha_xpath:
for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"):
if html.xpath(xpath):
captcha_img_url = html.xpath(xpath)[0]
break
if not captcha_img_url:
return None, None, "未找到验证码图片"
# 查找登录按钮
submit_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("submit"):
if html.xpath(xpath):
submit_xpath = xpath
break
if not submit_xpath:
return None, None, "未找到登录按钮"
# 点击登录按钮
try:
# 等待登录按钮准备好
page.wait_for_selector(submit_xpath)
# 输入用户名
page.fill(username_xpath, username)
# 输入密码
page.fill(password_xpath, password)
# 勾选“记住我/保持登录”等选项,获取长期会话(部分站点默认发放短期会话)
for xpath in self._SITE_LOGIN_XPATH.get("remember"):
remember_element = page.query_selector(xpath)
if not remember_element:
continue
try:
checked = remember_element.get_attribute("aria-checked")
if checked is None:
checked = "true" if remember_element.is_checked() else "false"
if checked != "true":
remember_element.click(timeout=3000)
break
except Exception as e:
# 当前候选不可操作(如隐藏元素)时继续尝试后续候选
logger.warning(f"勾选记住登录选项失败:{str(e)},尝试下一候选")
continue
# 输入二步验证码
if twostep_xpath:
page.fill(twostep_xpath, otp_code)
# 识别验证码
if captcha_xpath and captcha_img_url:
captcha_element = page.query_selector(captcha_xpath)
if captcha_element.is_visible():
# 验证码图片地址
code_url = self.__get_captcha_url(url, captcha_img_url)
# 获取当前的cookie和ua
cookie = self.parse_cookies(page.context.cookies())
ua = page.evaluate("() => window.navigator.userAgent")
# 自动OCR识别验证码
captcha = self.__get_captcha_text(cookie=cookie, ua=ua, code_url=code_url)
if captcha:
logger.info("验证码地址为:%s,识别结果:%s" % (code_url, captcha))
else:
return None, None, "验证码识别失败"
# 输入验证码
captcha_element.fill(captcha)
else:
# 不可见元素不处理
pass
# 点击登录按钮
page.click(submit_xpath)
page.wait_for_load_state("networkidle", timeout=30 * 1000)
except Exception as e:
logger.error(f"仿真登录失败:{str(e)}")
return None, None, f"仿真登录失败:{str(e)}"
# 对于某二次验证码为单页面的站点,输入二次验证码
if "verify" in page.url:
if not otp_code:
return None, None, "需要二次验证码"
html_text = self.get_page_content(page)
if not html_text:
return None, None, "获取网页源码失败"
html = etree.HTML(html_text)
if html is None:
return None, None, "解析网页源码失败"
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
if html.xpath(xpath):
try:
# 刷新一下 2fa code
otp_code = TwoFactorAuth(two_step_code).get_code()
page.fill(xpath, otp_code)
# 登录按钮 xpath 理论上相同,不再重复查找
page.click(submit_xpath)
page.wait_for_load_state("networkidle", timeout=30 * 1000)
except Exception as e:
logger.error(f"二次验证码输入失败:{str(e)}")
return None, None, f"二次验证码输入失败:{str(e)}"
break
# 登录后的源码(部分站点登录成功后由前端脚本延迟跳转,等待并重试判定)
html_text = None
for i in range(3):
if i:
time.sleep(2)
latest_text = self.get_page_content(page)
if not latest_text:
continue
if SiteUtils.is_logged_in(latest_text):
return self.parse_cookies(page.context.cookies()), \
page.evaluate("() => window.navigator.userAgent"), ""
# 保留首个快照用于失败时解析错误信息,避免提示被后续跳转或自动消失覆盖
if html_text is None:
html_text = latest_text
# 页面已出现明确的登录错误信息时,以该快照为准并提前结束重试
latest_html = etree.HTML(latest_text)
if latest_html is not None and \
any(latest_html.xpath(x) for x in self._SITE_LOGIN_XPATH.get("error")):
html_text = latest_text
break
if not html_text:
return None, None, "获取网页源码失败"
else:
# 从登录后的页面读取错误信息
html = etree.HTML(html_text)
if html is None:
return None, None, "登录失败"
error_xpath = None
for xpath in self._SITE_LOGIN_XPATH.get("error"):
if html.xpath(xpath):
error_xpath = xpath
break
if not error_xpath:
return None, None, "登录失败"
else:
error_msg = html.xpath(error_xpath)[0]
return None, None, error_msg
finally:
if html:
del html
if not url or not username or not password:
return None, None, "参数错误"
return PlaywrightHelper().action(url=url,
callback=__page_handler,
proxies=proxies,
timeout=timeout)
@staticmethod
def __get_captcha_text(cookie: str, ua: str, code_url: str) -> str:
"""
识别验证码图片的内容
"""
if not code_url:
return ""
ret = RequestUtils(ua=ua, cookies=cookie).get_res(code_url)
if ret:
if not ret.content:
return ""
return OcrHelper().get_captcha_text(
image_b64=base64.b64encode(ret.content).decode()
)
else:
return ""
@staticmethod
def __get_captcha_url(siteurl: str, imageurl: str) -> str:
"""
获取验证码图片的URL
"""
if not siteurl or not imageurl:
return ""
if imageurl.startswith("/"):
imageurl = imageurl[1:]
return "%s/%s" % (StringUtils.get_base_url(siteurl), imageurl)

View File

@@ -0,0 +1,53 @@
from typing import Tuple
import pyotp
class OtpUtils:
"""提供基于 TOTP 的二次验证辅助能力。"""
@staticmethod
def generate_secret_key(username: str) -> Tuple[str, str]:
"""生成 TOTP 密钥及其配置 URI。"""
try:
secret = pyotp.random_base32()
uri = pyotp.totp.TOTP(secret).provisioning_uri(name='MoviePilot',
issuer_name='MoviePilot(' + username + ')')
return secret, uri
except Exception as err:
print(str(err))
return "", ""
@staticmethod
def is_legal(otp_uri: str, password: str) -> bool:
"""
校验二次验证是否正确
"""
try:
return pyotp.TOTP(pyotp.parse_uri(otp_uri).secret).verify(password)
except Exception as err:
print(str(err))
return False
@staticmethod
def check(secret: str, password: str) -> bool:
"""
校验二次验证是否正确
"""
try:
totp = pyotp.TOTP(secret)
return totp.verify(password)
except Exception as err:
print(str(err))
return False
@staticmethod
def get_secret(otp_uri: str) -> str:
"""
获取uri中的secret
"""
try:
return pyotp.parse_uri(otp_uri).secret
except Exception as err:
print(str(err))
return ""

View File

@@ -0,0 +1,451 @@
"""
PassKey WebAuthn 辅助工具类
"""
import base64
import binascii
import json
import secrets
import threading
from dataclasses import dataclass
from typing import Any, Dict, List, Literal, Optional, Tuple
from urllib.parse import urlparse
from webauthn import (
generate_registration_options,
verify_registration_response,
generate_authentication_options,
verify_authentication_response,
options_to_json
)
from webauthn.helpers import (
parse_registration_credential_json,
parse_authentication_credential_json
)
from webauthn.helpers.structs import (
PublicKeyCredentialDescriptor,
AuthenticatorTransport,
UserVerificationRequirement,
ResidentKeyRequirement,
AuthenticatorSelectionCriteria
)
from webauthn.helpers.cose import COSEAlgorithmIdentifier
from webauthn.helpers.exceptions import InvalidRegistrationResponse
from app.runtime.cache import TTLCache
from app.runtime.config import settings
from app.adapters.cache.redis import RedisHelper
from app.runtime.log import logger
PASSKEY_CHALLENGE_TTL_SECONDS = 5 * 60
PasskeyChallengePurpose = Literal["authentication", "registration"]
@dataclass(frozen=True)
class PasskeyChallenge:
"""服务端保存的一次性 Passkey challenge 及其认证边界。"""
challenge: str
purpose: PasskeyChallengePurpose
user_id: Optional[int]
class PasskeyChallengeStore:
"""使用当前缓存后端签发并原子消费短时 Passkey challenge。"""
_cache = TTLCache(
region="passkey_challenge",
maxsize=4096,
ttl=PASSKEY_CHALLENGE_TTL_SECONDS,
)
_memory_consume_lock = threading.Lock()
@classmethod
def issue(
cls,
*,
challenge: str,
purpose: PasskeyChallengePurpose,
user_id: Optional[int],
) -> str:
"""保存 challenge 并返回不携带认证事实的随机事务 token。"""
transaction_token = secrets.token_urlsafe(32)
cls._cache.set(
transaction_token,
PasskeyChallenge(
challenge=challenge,
purpose=purpose,
user_id=user_id,
),
)
return transaction_token
@classmethod
def consume(
cls,
*,
transaction_token: str,
purpose: PasskeyChallengePurpose,
) -> Optional[PasskeyChallenge]:
"""原子领取 challenge任何完成尝试都会使事务失效。"""
if not transaction_token:
return None
if cls._cache.is_redis():
challenge = RedisHelper().pop(
transaction_token,
region="passkey_challenge",
)
else:
with cls._memory_consume_lock:
try:
challenge = cls._cache.pop(transaction_token)
except KeyError:
challenge = None
if not isinstance(challenge, PasskeyChallenge):
return None
if challenge.purpose != purpose:
return None
return challenge
class PassKeyRegistrationVerificationError(Exception):
"""Passkey 注册响应未通过 WebAuthn 安全校验。"""
class PassKeyRegistrationOriginMismatchError(PassKeyRegistrationVerificationError):
"""浏览器来源与系统配置的 Passkey 注册来源不一致。"""
class PassKeyHelper:
"""
PassKey WebAuthn 辅助类
"""
@staticmethod
def get_rp_id() -> str:
"""
获取 Relying Party ID
"""
if settings.APP_DOMAIN:
app_domain = settings.APP_DOMAIN.strip()
# 确保存在协议前缀,以便 urlparse 正确解析主机和端口
if not app_domain.startswith(('http://', 'https://')):
app_domain = f'https://{app_domain}'
parsed = urlparse(app_domain)
host = parsed.hostname
if host:
return host
# 从 APP_DOMAIN 中提取域名
host = settings.APP_DOMAIN.replace('https://', '').replace('http://', '')
# 移除端口号
if ':' in host:
host = host.split(':')[0]
return host
# 只有在未配置 APP_DOMAIN 时,才默认为 localhost
return 'localhost'
@staticmethod
def get_rp_name() -> str:
"""
获取 Relying Party 名称
"""
return "MoviePilot"
@staticmethod
def get_origin() -> str:
"""
获取源地址
"""
if settings.APP_DOMAIN:
return settings.APP_DOMAIN.rstrip('/')
# 如果未配置APP_DOMAIN使用默认的localhost地址
return f'http://localhost:{settings.NGINX_PORT}'
@staticmethod
def standardize_credential_id(credential_id: str) -> str:
"""
标准化凭证IDBase64 URL Safe
"""
try:
# Base64解码并重新编码以标准化格式
decoded = base64.urlsafe_b64decode(credential_id + '==')
return base64.urlsafe_b64encode(decoded).decode('utf-8').rstrip('=')
except (binascii.Error, TypeError, ValueError) as e:
logger.error(f"标准化凭证ID失败: {e}")
return credential_id
@staticmethod
def _base64_encode_urlsafe(data: bytes) -> str:
"""
Base64 URL Safe 编码(不带填充)
:param data: 要编码的字节数据
:return: Base64 URL Safe 编码的字符串
"""
return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')
@staticmethod
def _base64_decode_urlsafe(data: str) -> bytes:
"""
Base64 URL Safe 解码(自动添加填充)
:param data: Base64 URL Safe 编码的字符串
:return: 解码后的字节数据
"""
return base64.urlsafe_b64decode(data + '==')
@staticmethod
def _parse_credential_list(credentials: List[Dict[str, Any]]) -> List[PublicKeyCredentialDescriptor]:
"""
解析凭证列表为 PublicKeyCredentialDescriptor 列表
:param credentials: 凭证字典列表
:return: PublicKeyCredentialDescriptor 列表
"""
result = []
for cred in credentials:
try:
result.append(
PublicKeyCredentialDescriptor(
id=PassKeyHelper._base64_decode_urlsafe(cred['credential_id']),
transports=[
AuthenticatorTransport(t) for t in cred.get('transports', '').split(',') if t
] if cred.get('transports') else None
)
)
except Exception as e:
logger.warning(f"解析凭证失败: {e}")
continue
return result
@staticmethod
def _get_user_verification_requirement(user_verification: Optional[str] = None) -> UserVerificationRequirement:
"""
获取用户验证要求
:param user_verification: 指定的用户验证要求,如果不指定则从配置中读取
:return: UserVerificationRequirement
"""
if user_verification:
return UserVerificationRequirement(user_verification)
return UserVerificationRequirement.REQUIRED if settings.PASSKEY_REQUIRE_UV \
else UserVerificationRequirement.PREFERRED
@staticmethod
def _get_verification_params(
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[str, str]:
"""
获取验证参数origin 和 rp_id
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (origin, rp_id)
"""
origin = expected_origin or PassKeyHelper.get_origin()
rp_id = expected_rp_id or PassKeyHelper.get_rp_id()
return origin, rp_id
@staticmethod
def generate_registration_options(
user_id: int,
username: str,
display_name: Optional[str] = None,
existing_credentials: Optional[List[Dict[str, Any]]] = None
) -> Tuple[str, str]:
"""
生成注册选项
:param user_id: 用户ID
:param username: 用户名
:param display_name: 显示名称
:param existing_credentials: 已存在的凭证列表
:return: (options_json, challenge)
"""
try:
# 用户信息
user_id_bytes = str(user_id).encode('utf-8')
# 排除已有的凭证
exclude_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
if existing_credentials else None
# 用户验证要求
uv_requirement = PassKeyHelper._get_user_verification_requirement()
# 生成注册选项
options = generate_registration_options(
rp_id=PassKeyHelper.get_rp_id(),
rp_name=PassKeyHelper.get_rp_name(),
user_id=user_id_bytes,
user_name=username,
user_display_name=display_name or username,
exclude_credentials=exclude_credentials,
authenticator_selection=AuthenticatorSelectionCriteria(
authenticator_attachment=None,
resident_key=ResidentKeyRequirement.REQUIRED,
user_verification=uv_requirement,
),
supported_pub_key_algs=[
COSEAlgorithmIdentifier.ECDSA_SHA_256,
COSEAlgorithmIdentifier.RSASSA_PKCS1_v1_5_SHA_256,
]
)
# 转换为JSON
options_json = options_to_json(options)
# 提取challenge用于后续验证
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
return options_json, challenge
except Exception as e:
logger.error(f"生成注册选项失败: {e}")
raise
@staticmethod
def verify_registration_response(
credential: Dict[str, Any],
expected_challenge: str,
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[str, str, int, Optional[str]]:
"""
验证注册响应
:param credential: 客户端返回的凭证
:param expected_challenge: 期望的challenge
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (credential_id, public_key, sign_count, aaguid)
"""
try:
# 准备验证参数
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
# 解码challenge
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
# 构建RegistrationCredential对象
registration_credential = parse_registration_credential_json(json.dumps(credential))
# 验证注册响应
verification = verify_registration_response(
credential=registration_credential,
expected_challenge=challenge_bytes,
expected_rp_id=rp_id,
expected_origin=origin,
require_user_verification=settings.PASSKEY_REQUIRE_UV
)
# 提取信息
credential_id = PassKeyHelper._base64_encode_urlsafe(verification.credential_id)
public_key = PassKeyHelper._base64_encode_urlsafe(verification.credential_public_key)
sign_count = verification.sign_count
# aaguid 可能已经是字符串格式也可能是bytes
if verification.aaguid:
if isinstance(verification.aaguid, bytes):
aaguid = verification.aaguid.hex()
else:
aaguid = str(verification.aaguid)
else:
aaguid = None
return credential_id, public_key, sign_count, aaguid
except InvalidRegistrationResponse as e:
logger.error(f"验证注册响应失败: {e}")
if str(e).startswith("Unexpected client data origin "):
raise PassKeyRegistrationOriginMismatchError() from e
raise PassKeyRegistrationVerificationError() from e
except Exception as e:
logger.error(f"验证注册响应失败: {e}")
raise
@staticmethod
def generate_authentication_options(
existing_credentials: Optional[List[Dict[str, Any]]] = None,
user_verification: Optional[str] = None
) -> Tuple[str, str]:
"""
生成认证选项
:param existing_credentials: 已存在的凭证列表(用于限制可用凭证)
:param user_verification: 用户验证要求,如果不指定则从配置中读取
:return: (options_json, challenge)
"""
try:
# 允许的凭证
allow_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
if existing_credentials else None
# 用户验证要求
uv_requirement = PassKeyHelper._get_user_verification_requirement(user_verification)
# 生成认证选项
options = generate_authentication_options(
rp_id=PassKeyHelper.get_rp_id(),
allow_credentials=allow_credentials,
user_verification=uv_requirement
)
# 转换为JSON
options_json = options_to_json(options)
# 提取challenge
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
return options_json, challenge
except Exception as e:
logger.error(f"生成认证选项失败: {e}")
raise
@staticmethod
def verify_authentication_response(
credential: Dict[str, Any],
expected_challenge: str,
credential_public_key: str,
credential_current_sign_count: int,
expected_origin: Optional[str] = None,
expected_rp_id: Optional[str] = None
) -> Tuple[bool, int]:
"""
验证认证响应
:param credential: 客户端返回的凭证
:param expected_challenge: 期望的challenge
:param credential_public_key: 凭证公钥
:param credential_current_sign_count: 当前签名计数
:param expected_origin: 期望的源地址
:param expected_rp_id: 期望的RP ID
:return: (验证成功, 新的签名计数)
"""
try:
# 准备验证参数
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
# 解码
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
public_key_bytes = PassKeyHelper._base64_decode_urlsafe(credential_public_key)
# 构建AuthenticationCredential对象
authentication_credential = parse_authentication_credential_json(json.dumps(credential))
# 验证认证响应
verification = verify_authentication_response(
credential=authentication_credential,
expected_challenge=challenge_bytes,
expected_rp_id=rp_id,
expected_origin=origin,
credential_public_key=public_key_bytes,
credential_current_sign_count=credential_current_sign_count,
require_user_verification=settings.PASSKEY_REQUIRE_UV
)
return True, verification.new_sign_count
except Exception as e:
logger.error(f"验证认证响应失败: {e}")
return False, credential_current_sign_count

View File

@@ -0,0 +1,48 @@
import base64
import hashlib
import hmac
import struct
import sys
import time
from app.runtime.log import logger
class TwoFactorAuth:
"""解析已有验证码或根据共享密钥生成 TOTP 验证码。"""
def __init__(self, code_or_secret: str):
"""按长度区分用户验证码和 TOTP 共享密钥。"""
if code_or_secret and len(code_or_secret) >= 16:
self.code = None
self.secret = code_or_secret
else:
self.code = code_or_secret
self.secret = None
@staticmethod
def __calc(secret_key: str) -> str:
"""按 30 秒时间窗计算六位 TOTP 验证码。"""
if not secret_key:
return ""
try:
input_time = int(time.time()) // 30
key = base64.b32decode(secret_key)
msg = struct.pack(">Q", input_time)
google_code = hmac.new(key, msg, hashlib.sha1).digest()
o = (
google_code[19] & 15
if sys.version_info > (2, 7)
else ord(str(google_code[19])) & 15
)
google_code = str(
(struct.unpack(">I", google_code[o: o + 4])[0] & 0x7FFFFFFF) % 1000000
)
return f"0{google_code}" if len(google_code) == 5 else google_code
except Exception as e:
logger.error(f"计算动态验证码失败:{str(e)}")
return ""
def get_code(self) -> str:
"""返回显式验证码,或从共享密钥实时计算。"""
return self.code or self.__calc(self.secret)

View File

@@ -0,0 +1,983 @@
import asyncio
import hmac
import ipaddress
import socket
import threading
from dataclasses import dataclass, field
from enum import Enum
from hashlib import sha256
from pathlib import Path
from typing import Dict, Iterable, List, Optional, Set, Union
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
from anyio import Path as AsyncPath
from cachetools import TTLCache
from app.runtime.config import settings
from app.runtime.log import logger
from app.runtime.coalesce import (
CoalesceDecision,
CoalesceSummary,
EventCoalescer,
)
# DNS 解析结果缓存。
# 正向缓存 TTL 选择 120s短于常见 CDN / fake-ip 的 DNS TTL避免长期持有失效 IP
# 负向缓存 TTL 选择 15s避免临时解析失败把目标长时间拉黑。
_DNS_CACHE_MAXSIZE = 1024
_DNS_CACHE_TTL_POSITIVE = 120
_DNS_CACHE_TTL_NEGATIVE = 15
_dns_positive_cache: "TTLCache[str, List[ipaddress._BaseAddress]]" = TTLCache(
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_POSITIVE
)
_dns_negative_cache: "TTLCache[str, bool]" = TTLCache(
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_NEGATIVE
)
# 同步路径下保护 TTLCache 读写:`cachetools.TTLCache` 本身非线程安全。
# 锁只覆盖缓存读写,不包 `getaddrinfo`,避免把 DNS 查询本身串行化。
_dns_cache_lock = threading.Lock()
# 同 hostname 的并发异步解析去重:同一 hostname 首次未命中时建立锁,
# 后续并发请求 await 同一把锁,避免对同一目标重复发起 `getaddrinfo`。
_dns_inflight_locks: Dict[str, asyncio.Lock] = {}
_dns_inflight_meta_lock = threading.Lock()
class UrlSafetyReason(str, Enum):
"""
`evaluate_url_safety` 返回的诊断原因枚举。
成员值为稳定的小写蛇形字符串,可直接作为日志字段或告警标签使用,
扩展枚举时保留既有成员的取值,避免破坏下游聚合系统对原因的归类。
"""
# 通过全部校验URL 可被请求
ALLOWED = "allowed"
# 协议非 http/https或 netloc 无效,或域名不在允许列表内
DOMAIN_NOT_ALLOWED = "domain_not_allowed"
# 已通过域名 allowlist但 DNS 解析失败(无返回或抛错)
DNS_RESOLUTION_FAILED = "dns_resolution_failed"
# DNS 解析到至少一个非公网地址,且未配置 `allowed_private_ranges`
NON_GLOBAL_DNS_RESULT = "non_global_dns_result"
# 配置了 `allowed_private_ranges`,但仍存在不在允许网段内的解析结果
MIXED_OR_DISALLOWED_PRIVATE_RESULT = "mixed_or_disallowed_private_result"
@dataclass(frozen=True)
class UrlSafetyDiagnosis:
"""
URL 安全校验的结构化诊断结果,由 `evaluate_url_safety(_async)` 返回。
`is_safe_url` 仅使用 `allowed` 字段;日志、告警、运维诊断需要细分原因或
解析 IP 时通过本对象消费。字段约束:
- `host` 仅在通过域名 allowlist 后才被填充DOMAIN_NOT_ALLOWED 场景为 None。
- `ips` 仅在执行过 DNS 阶段后才可能非空;不含纯字符串协议失败场景。
- `matched_private_ranges` 仅在通过 `allowed_private_ranges` 放行时填充。
"""
# 是否放行
allowed: bool
# 放行/拦截的具体原因
reason: UrlSafetyReason
# 通过 allowlist 后从 URL 解析出的 hostname未通过时为 None
host: Optional[str] = None
# DNS 解析结果(含命中或未命中私网放行的 IP格式化为字符串
ips: List[str] = field(default_factory=list)
# 命中允许放行的非公网网段,仅 `ALLOWED` 且走私网放行分支时非空
matched_private_ranges: List[str] = field(default_factory=list)
def _resolve_addrinfo_to_ips(
address_infos: Iterable,
) -> Optional[List[ipaddress._BaseAddress]]:
"""
将 `socket.getaddrinfo` 返回的结果归一化为 IP 列表。
任一条目无法解析为 IP 即视为异常情况,整体返回 None 让上层按"不安全目标"
处理,避免出现"部分 IP 漏校验"的情况。
"""
addresses: List[ipaddress._BaseAddress] = []
for address_info in address_infos:
try:
addresses.append(ipaddress.ip_address(address_info[4][0]))
except ValueError:
return None
return addresses or None
class SecurityUtils:
"""提供路径、URL、签名和网络目标安全校验能力。"""
_SIGNED_URL_PURPOSE = "image-proxy"
_SUBTITLE_DOWNLOAD_PURPOSE_PREFIX = "subtitle-download"
@staticmethod
def is_safe_path(base_path: Path, user_path: Path,
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
"""
验证用户提供的路径是否在基准目录内,并检查文件类型是否合法,防止目录遍历攻击
:param base_path: 基准目录,允许访问的根目录
:param user_path: 用户提供的路径,需检查其是否位于基准目录内
:param allowed_suffixes: 允许的文件后缀名集合,用于验证文件类型
:return: 如果用户路径安全且位于基准目录内,且文件类型合法,返回 True否则返回 False
:raises Exception: 如果解析路径时发生错误,则捕获并记录异常
"""
try:
# resolve() 将相对路径转换为绝对路径,并处理符号链接和'..'
base_path_resolved = base_path.resolve()
user_path_resolved = user_path.resolve()
# 检查用户路径是否在基准目录或基准目录的子目录内
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
return False
if allowed_suffixes is not None:
allowed_suffixes = set(allowed_suffixes)
if user_path.suffix.lower() not in allowed_suffixes:
return False
return True
except Exception as e:
logger.debug(f"Error occurred while validating paths: {e}")
return False
@staticmethod
async def async_is_safe_path(base_path: AsyncPath, user_path: AsyncPath,
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
"""
异步验证用户提供的路径是否在基准目录内,并检查文件类型是否合法,防止目录遍历攻击
:param base_path: 基准目录,允许访问的根目录
:param user_path: 用户提供的路径,需检查其是否位于基准目录内
:param allowed_suffixes: 允许的文件后缀名集合,用于验证文件类型
:return: 如果用户路径安全且位于基准目录内,且文件类型合法,返回 True否则返回 False
:raises Exception: 如果解析路径时发生错误,则捕获并记录异常
"""
try:
# resolve() 将相对路径转换为绝对路径,并处理符号链接和'..'
base_path_resolved = await base_path.resolve()
user_path_resolved = await user_path.resolve()
# 检查用户路径是否在基准目录或基准目录的子目录内
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
return False
if allowed_suffixes is not None:
allowed_suffixes = set(allowed_suffixes)
if user_path.suffix.lower() not in allowed_suffixes:
return False
return True
except Exception as e:
logger.debug(f"Error occurred while validating paths: {e}")
return False
@staticmethod
def _literal_ip(hostname: str) -> Optional[ipaddress._BaseAddress]:
"""
若 hostname 是字面量 IP含 IPv6 的 `[::1]` 形式)则返回 IP 对象,否则 None。
"""
if not hostname:
return None
candidate = hostname
if candidate.startswith("[") and candidate.endswith("]"):
candidate = candidate[1:-1]
try:
return ipaddress.ip_address(candidate)
except ValueError:
return None
@staticmethod
def _cache_lookup(hostname: str) -> tuple[bool, Optional[List[ipaddress._BaseAddress]]]:
"""
在 TTL 缓存中查找 hostname返回 (是否命中, 命中值)。
命中值为 `None` 表示命中负向缓存(先前解析失败)。
"""
with _dns_cache_lock:
cached = _dns_positive_cache.get(hostname)
if cached is not None:
return True, cached
if hostname in _dns_negative_cache:
return True, None
return False, None
@staticmethod
def _cache_store(
hostname: str, addresses: Optional[List[ipaddress._BaseAddress]]
) -> None:
"""
将解析结果写入对应的正向/负向缓存。
"""
with _dns_cache_lock:
if addresses is None:
_dns_negative_cache[hostname] = True
else:
_dns_positive_cache[hostname] = addresses
@staticmethod
def _hostname_addresses(hostname: str) -> Optional[List[ipaddress._BaseAddress]]:
"""
同步解析主机名并返回全部 IP 地址,结果走 TTL 缓存。
字面量 IP 直接返回自身DNS 解析失败或结果异常时返回 None由上层按
不安全目标处理。async 调用方应使用 `_hostname_addresses_async`。
"""
if not hostname:
return None
literal = SecurityUtils._literal_ip(hostname)
if literal is not None:
return [literal]
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
try:
address_infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
except socket.gaierror:
SecurityUtils._cache_store(hostname, None)
return None
addresses = _resolve_addrinfo_to_ips(address_infos)
SecurityUtils._cache_store(hostname, addresses)
return addresses
@staticmethod
def _get_inflight_lock(hostname: str) -> asyncio.Lock:
"""
取得 hostname 对应的 in-flight 锁,不存在则按需创建。
用 `threading.Lock` 保护字典写入,避免多个事件循环线程并发创建出多把锁
破坏去重语义;锁本身是 `asyncio.Lock`,归属当前事件循环。
"""
with _dns_inflight_meta_lock:
lock = _dns_inflight_locks.get(hostname)
if lock is None:
lock = asyncio.Lock()
_dns_inflight_locks[hostname] = lock
return lock
@staticmethod
def _release_inflight_lock(hostname: str, lock: asyncio.Lock) -> None:
"""
请求结束后清理 in-flight 锁,避免长期持有大量已闲置的 `asyncio.Lock`。
仅当字典中登记的仍是当前 lock且 `lock.locked()` 为 False 时才删除。
`asyncio.Lock` 公平 FIFO持有者释放后若仍有等待者锁会立刻被下一个
等待者接走、`locked()` 重新变为 True因此该守卫可同时排除"仍有持有者"
"刚被等待者接走"两种情况,避免误删后续协程仍在使用的字典条目。
"""
with _dns_inflight_meta_lock:
current = _dns_inflight_locks.get(hostname)
if current is lock and not lock.locked():
_dns_inflight_locks.pop(hostname, None)
@staticmethod
async def _hostname_addresses_async(
hostname: str,
) -> Optional[List[ipaddress._BaseAddress]]:
"""
异步解析主机名并返回全部 IP 地址,与同步版本共用同一份 TTL 缓存。
通过事件循环的默认线程池执行 `getaddrinfo`,不阻塞 asyncio 事件循环;
同 hostname 的并发未命中请求通过 in-flight 锁去重,只发起一次 DNS 查询。
"""
if not hostname:
return None
literal = SecurityUtils._literal_ip(hostname)
if literal is not None:
return [literal]
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
lock = SecurityUtils._get_inflight_lock(hostname)
try:
async with lock:
# 等到锁后再查一次缓存,前一个持锁者可能已经回填结果
hit, value = SecurityUtils._cache_lookup(hostname)
if hit:
return value
loop = asyncio.get_running_loop()
try:
address_infos = await loop.getaddrinfo(
hostname, None, type=socket.SOCK_STREAM
)
except socket.gaierror:
SecurityUtils._cache_store(hostname, None)
return None
addresses = _resolve_addrinfo_to_ips(address_infos)
SecurityUtils._cache_store(hostname, addresses)
return addresses
finally:
# 必须在 `async with` 释放锁之后再清理字典:`_release_inflight_lock`
# 以 `not lock.locked()` 为清理守卫,持锁状态下调用会跳过 pop。
SecurityUtils._release_inflight_lock(hostname, lock)
@staticmethod
def _addresses_all_global(
addresses: Optional[List[ipaddress._BaseAddress]],
) -> bool:
"""
判断解析结果是否全部为公网地址(空列表/None 视为非公网)。
"""
if not addresses:
return False
return all(address.is_global for address in addresses)
@staticmethod
def _is_global_hostname(hostname: str) -> bool:
"""
判断主机名解析结果是否全部为公网地址(同步版本)。
图片代理会访问用户可控的 URL这里必须在 allowlist 命中前后都排除
私有、回环、链路本地、保留地址等非公网目标,避免通过 DNS 或字面量 IP
绕过域名白名单访问内网服务。
"""
return SecurityUtils._addresses_all_global(
SecurityUtils._hostname_addresses(hostname)
)
@staticmethod
async def _is_global_hostname_async(hostname: str) -> bool:
"""
判断主机名解析结果是否全部为公网地址(异步版本)。语义与 `_is_global_hostname` 一致。
"""
return SecurityUtils._addresses_all_global(
await SecurityUtils._hostname_addresses_async(hostname)
)
@staticmethod
def _parse_ip_networks(ranges: Optional[Iterable[str]]) -> List[ipaddress._BaseNetwork]:
"""
解析用户配置的 IP/CIDR 网段。
配置错误的条目会被忽略并写入 debug 日志,避免单个无效值导致所有图片代理
校验失败。调用方仍然需要先完成域名白名单匹配,不能单独依赖该网段放行。
"""
networks = []
for value in ranges or []:
if not value:
continue
try:
networks.append(ipaddress.ip_network(str(value).strip(), strict=False))
except ValueError:
logger.debug(f"忽略无效的图片代理允许网段配置: {value}")
return networks
@staticmethod
def _match_private_addresses(
addresses: Optional[List[ipaddress._BaseAddress]],
networks: List[ipaddress._BaseNetwork],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
在已解析出的地址列表中匹配显式允许的非公网网段。
所有解析地址都必须命中至少一个允许网段才放行;只要有一个 IP 落在允许
网段外(或解析结果是全公网),就视为不匹配私网放行规则。
"""
if not addresses or not networks:
return None
if all(address.is_global for address in addresses):
return None
matched_networks: List[ipaddress._BaseNetwork] = []
for address in addresses:
matched_for_address = [
network for network in networks if address in network
]
if not matched_for_address:
return None
matched_networks.extend(matched_for_address)
return addresses, list(dict.fromkeys(matched_networks))
@staticmethod
def _is_allowed_private_hostname(
hostname: str,
allowed_private_ranges: Optional[Iterable[str]],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
返回主机名命中的显式允许非公网地址和网段(同步版本)。
该能力只用于图片代理的受控例外,例如 TUN fake-ip 或内网 CDN。必须由
`is_safe_url` 先完成域名 allowlist 校验后再调用,避免把任意用户 URL
变成 SSRF 绕过入口。
"""
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return None
return SecurityUtils._match_private_addresses(
SecurityUtils._hostname_addresses(hostname), networks
)
@staticmethod
async def _is_allowed_private_hostname_async(
hostname: str,
allowed_private_ranges: Optional[Iterable[str]],
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
"""
`_is_allowed_private_hostname` 的异步版本,语义保持一致。
"""
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return None
return SecurityUtils._match_private_addresses(
await SecurityUtils._hostname_addresses_async(hostname), networks
)
@staticmethod
def _url_signature_payload(url: str, purpose: str) -> bytes:
"""
构造 URL 签名载荷。
签名覆盖用途与完整 URL确保同一个签名不能挪用到其它代理用途或其它 URL。
"""
return f"{purpose}\n{url}".encode("utf-8")
@staticmethod
def _sign_url_payload(url: str, purpose: str) -> str:
"""
使用 RESOURCE_SECRET_KEY 对 URL 签名载荷生成 HMAC。
相同 `(url, purpose, RESOURCE_SECRET_KEY)` 组合在进程生命周期内输出
完全一致;签名的失效边界绑定在 `RESOURCE_SECRET_KEY` 上,进程重启
或显式轮换密钥时所有旧签名一起作废。
"""
return hmac.new(
settings.RESOURCE_SECRET_KEY.encode("utf-8"),
SecurityUtils._url_signature_payload(url, purpose),
sha256,
).hexdigest()
@staticmethod
def strip_url_signature(url: str) -> str:
"""
移除 URL fragment 中的资源签名信息,得到真正要请求的地址。
签名放在 fragment 中,浏览器会把它传给 MoviePilot但 HTTP 客户端
请求外部资源前不能把这些内部参数带过去。
"""
if not url:
return url
parsed_url = urlparse(url)
return urlunparse(parsed_url._replace(fragment=""))
@staticmethod
def subtitle_download_purpose(site_id: int) -> str:
"""
构造字幕下载 URL 签名用途,签名必须绑定站点 ID避免跨站点复用。
"""
return f"{SecurityUtils._SUBTITLE_DOWNLOAD_PURPOSE_PREFIX}:{site_id}"
@staticmethod
def sign_url(
url: str,
purpose: str = _SIGNED_URL_PURPOSE,
) -> str:
"""
给服务端返回的资源 URL 添加稳定签名。
签名作为后端资源能力凭证:外部请求边界可以用不同 `purpose` 绑定
具体业务语义,避免一个场景签出的 URL 被挪用到另一个场景。
签名为 `(url, purpose, RESOURCE_SECRET_KEY)` 的确定性 HMAC**不带
过期时间**:相同 URL 多次调用结果完全一致,让浏览器与 Service Worker
的缓存能稳定命中;失效边界由 `RESOURCE_SECRET_KEY` 控制——进程重启
自动重生成、或者运维显式轮换后所有历史签名一起作废。
"""
if not url:
return url
parsed_url = urlparse(url)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
return url
clean_url = SecurityUtils.strip_url_signature(url)
signature = SecurityUtils._sign_url_payload(clean_url, purpose)
fragment = urlencode(
{
"mp_sig": signature,
"mp_purpose": purpose,
}
)
return urlunparse(urlparse(clean_url)._replace(fragment=fragment))
@staticmethod
def verify_signed_url(
url: str,
purpose: str = _SIGNED_URL_PURPOSE,
) -> Optional[str]:
"""
验证 URL fragment 中的资源签名,成功时返回去签名后的真实 URL。
签名只校验 `(url, purpose, RESOURCE_SECRET_KEY)`,密钥轮换/进程重启
后旧签名自动失效。
"""
if not url:
return None
parsed_url = urlparse(url)
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
return None
fragment_params = dict(parse_qsl(parsed_url.fragment, keep_blank_values=True))
signature = fragment_params.get("mp_sig")
signed_purpose = fragment_params.get("mp_purpose")
if not signature or signed_purpose != purpose:
return None
clean_url = SecurityUtils.strip_url_signature(url)
expected_signature = SecurityUtils._sign_url_payload(clean_url, purpose)
if not hmac.compare_digest(signature, expected_signature):
return None
return clean_url
@staticmethod
def _check_url_allowlist(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool,
) -> Optional[str]:
"""
执行"协议 + netloc + 域名白名单"前置校验,命中返回 hostname未命中返回 None。
DNS 校验SSRF 防御)由调用方自行接续,本方法不发起 DNS 查询。
"""
try:
parsed_url = urlparse(url)
except Exception as e: # noqa: BLE001 - 任何解析异常都视为不安全 URL
logger.debug(f"Error occurred while validating URL: {e}")
return None
# 如果 URL 没有包含有效的 scheme或者无法从中提取到有效的 netloc则认为该 URL 是无效的
if not parsed_url.scheme or not parsed_url.netloc:
return None
# 仅允许 http 或 https 协议
if parsed_url.scheme not in {"http", "https"}:
return None
# 获取完整的 netloc包括 IP 和端口)并转换为小写
netloc = parsed_url.netloc.lower()
if not netloc:
return None
# 检查每个允许的域名
normalized_allowed = {d.lower() for d in allowed_domains}
domain_allowed = False
for domain in normalized_allowed:
parsed_allowed_url = urlparse(domain)
allowed_netloc = parsed_allowed_url.netloc or parsed_allowed_url.path
if strict:
# 严格模式下,要求完全匹配域名和端口
if netloc == allowed_netloc:
domain_allowed = True
break
else:
# 非严格模式下,允许子域名匹配
if netloc == allowed_netloc or netloc.endswith("." + allowed_netloc):
domain_allowed = True
break
if not domain_allowed:
return None
return parsed_url.hostname or ""
@staticmethod
def _log_private_range_allowed(
url: str,
match: tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]],
) -> None:
"""
记录"图片代理允许访问配置的非公网网段"放行日志,便于运维排查。
"""
addresses, matched_networks = match
logger.debug(
"图片代理允许访问配置的非公网网段: "
f"url={url}, ips={','.join(map(str, addresses))}, "
f"ranges={','.join(map(str, matched_networks))}"
)
@staticmethod
def is_safe_url(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
验证 URL 是否在允许的域名列表中,包括带有端口的域名(同步版本)。
:param url: 需要验证的 URL
:param allowed_domains: 允许的域名集合,域名可以包含端口
:param strict: 是否严格匹配一级域名(默认 False允许多级域名
:param block_private: 是否拦截解析到非公网地址的 URL防止 SSRF
:param allowed_private_ranges: 域名命中后额外允许的非公网 IP/CIDR 网段
:return: URL 合法且通过安全校验时返回 True否则返回 False
校验细节与失败原因由 `evaluate_url_safety` 返回;本方法只暴露布尔结果,
作为只关心通过/拒绝判断的调用方的最薄入口。`block_private=True` 时会
同步调用 `getaddrinfo`async 上下文请改用 `is_safe_url_async`。
"""
return SecurityUtils.evaluate_url_safety(
url,
allowed_domains,
strict=strict,
block_private=block_private,
allowed_private_ranges=allowed_private_ranges,
).allowed
@staticmethod
async def is_safe_url_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
判定 URL 是否在允许的域名列表中,包括带有端口的域名。
DNS 解析通过事件循环线程池执行,并复用 TTL 缓存,不阻塞调用方所在的
事件循环。参数与返回值含义同 `is_safe_url`;需要失败原因/解析 IP
等结构化信息时调用 `evaluate_url_safety_async`。
"""
diagnosis = await SecurityUtils.evaluate_url_safety_async(
url,
allowed_domains,
strict=strict,
block_private=block_private,
allowed_private_ranges=allowed_private_ranges,
)
return diagnosis.allowed
@staticmethod
def evaluate_url_safety(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> "UrlSafetyDiagnosis":
"""
在 `is_safe_url` 的判定路径上输出结构化诊断结果(同步版本)。
与 `is_safe_url` 共用同一套校验顺序:协议/域名 allowlist → 可选 DNS 解析
→ 可选非公网放行匹配;本方法额外返回失败原因、解析到的 IP 列表和命中的
私网网段,供日志与告警渲染消费。校验中遇到未预期异常时按默认拒绝原则
归类为 `DOMAIN_NOT_ALLOWED`,避免任何解析路径漏过 SSRF 校验。
"""
try:
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
if hostname is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
if not block_private:
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
)
addresses = SecurityUtils._hostname_addresses(hostname)
return SecurityUtils._diagnose_resolved_addresses(
url, hostname, addresses, allowed_private_ranges
)
except Exception as e: # noqa: BLE001 - 默认拒绝,避免漏过 SSRF 校验
logger.debug(f"Error occurred while validating URL: {e}")
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
@staticmethod
async def evaluate_url_safety_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
strict: bool = False,
block_private: bool = False,
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> "UrlSafetyDiagnosis":
"""
输出与 `evaluate_url_safety` 完全一致的结构化诊断结果。
DNS 解析通过事件循环线程池执行,并复用 TTL 缓存,不阻塞调用方所在的
事件循环;校验顺序、字段含义、异常归类均与同步版本相同。
"""
try:
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
if hostname is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
if not block_private:
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
)
addresses = await SecurityUtils._hostname_addresses_async(hostname)
return SecurityUtils._diagnose_resolved_addresses(
url, hostname, addresses, allowed_private_ranges
)
except Exception as e: # noqa: BLE001 - 默认拒绝,避免漏过 SSRF 校验
logger.debug(f"Error occurred while validating URL: {e}")
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
)
@staticmethod
async def is_safe_image_url_async(
url: str,
allowed_domains: Union[Set[str], List[str]],
allowed_private_ranges: Optional[Iterable[str]] = None,
) -> bool:
"""
判定 URL 是否可作为图片代理请求目标。
校验顺序:协议 + 域名 allowlist + DNS SSRF 拦截 + 非公网放行匹配;标准
校验失败时再用 `verify_signed_url` 兜底,允许后端预签名的媒体服务器
URL 跳过私网拦截。两者皆失败才视为拒绝。
拒绝路径会输出结构化阻断日志:单次拦截立即打印一条 warning
`(host, reason)` 的连续命中在 `_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS`
窗口内合并为一条聚合摘要,避免媒体详情页一次请求把日志刷爆。日志字段
范围严格限定为 URL、host、reason、解析 IP 与允许网段配置cookies、
签名串、token、请求头等敏感材料一律不进入日志。
"""
diagnosis = await SecurityUtils.evaluate_url_safety_async(
url,
allowed_domains,
block_private=True,
allowed_private_ranges=allowed_private_ranges,
)
if diagnosis.allowed:
return True
if SecurityUtils.verify_signed_url(url) is not None:
return True
await _emit_image_proxy_block_warning(
url=url,
diagnosis=diagnosis,
signature_carried=_url_carries_signature(url),
allowed_private_ranges=allowed_private_ranges,
)
return False
@staticmethod
def _diagnose_resolved_addresses(
url: str,
hostname: str,
addresses: Optional[List[ipaddress._BaseAddress]],
allowed_private_ranges: Optional[Iterable[str]],
) -> "UrlSafetyDiagnosis":
"""
对已完成 DNS 解析的地址列表执行非公网放行判断,并归一化诊断结果。
- 地址列表为空/None视为 DNS 不可信,拒绝并标记 `DNS_RESOLUTION_FAILED`。
- 全部公网地址:直接放行。
- 存在非公网地址且未配置允许网段:拒绝并标记 `NON_GLOBAL_DNS_RESULT`
供日志附带"如使用 fake-ip 需要配置 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
的提示。
- 存在非公网地址且配置了允许网段但未全部命中:拒绝并标记
`MIXED_OR_DISALLOWED_PRIVATE_RESULT`,提示存在不允许的解析结果。
- 全部命中允许网段:放行并附带命中的 IP 与网段,由
`_log_private_range_allowed` 输出排查日志。
"""
if not addresses:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.DNS_RESOLUTION_FAILED,
host=hostname,
)
if SecurityUtils._addresses_all_global(addresses):
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
ips=[str(addr) for addr in addresses],
)
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
if not networks:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
host=hostname,
ips=[str(addr) for addr in addresses],
)
match = SecurityUtils._match_private_addresses(addresses, networks)
if match is None:
return UrlSafetyDiagnosis(
allowed=False,
reason=UrlSafetyReason.MIXED_OR_DISALLOWED_PRIVATE_RESULT,
host=hostname,
ips=[str(addr) for addr in addresses],
)
matched_addresses, matched_networks = match
SecurityUtils._log_private_range_allowed(url, match)
return UrlSafetyDiagnosis(
allowed=True,
reason=UrlSafetyReason.ALLOWED,
host=hostname,
ips=[str(addr) for addr in matched_addresses],
matched_private_ranges=[str(net) for net in matched_networks],
)
@staticmethod
def sanitize_url_path(url: str, max_length: int = 120) -> str:
"""
将 URL 的路径部分进行编码,确保合法字符,并对路径长度进行压缩处理(如果超出最大长度)
:param url: 需要处理的 URL
:param max_length: 路径允许的最大长度,超出时进行压缩
:return: 处理后的路径字符串
"""
# 解析 URL获取路径部分
parsed_url = urlparse(url)
path = parsed_url.path.lstrip("/")
# 对路径中的特殊字符进行编码
safe_path = quote(path)
# 如果路径过长,进行压缩处理
if len(safe_path) > max_length:
# 使用 SHA-256 对路径进行哈希,取前 16 位作为压缩后的路径
hash_value = sha256(safe_path.encode()).hexdigest()[:16]
# 使用哈希值代替过长的路径,同时保留文件扩展名
file_extension = Path(safe_path).suffix.lower() if Path(safe_path).suffix else ""
safe_path = f"compressed_{hash_value}{file_extension}"
return safe_path
# 图片代理阻断日志聚合窗口(秒)。媒体详情页一次请求会批量触发同 host/同原因的拦截,
# 按 (host, reason) 合并后只输出首条 warning + 窗口结束的聚合摘要,避免日志刷屏。
_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS = 60.0
# fake-ip / 旁路 DNS 用户最常因 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES 未配置而踩坑,
# 在 reason=NON_GLOBAL_DNS_RESULT 且当前未配置允许网段时随 warning 一起输出,指向正确的修复开关。
_IMAGE_PROXY_FAKEIP_HINT = (
"提示:若使用 fake-ip / 旁路 DNS常见网段 198.18.0.0/15、100.64.0.0/10"
"请将对应网段加入 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
)
# URL fragment 中实际携带代理签名但校验失败时附在 reason 末尾的标记。
# 仅起标识作用,签名串本身不写入日志,避免泄露签名材料。
_INVALID_SIGNATURE_TAG = "invalid_signature"
def _url_carries_signature(url: str) -> bool:
"""
判断 URL 是否在 fragment 中显式携带代理签名参数 `mp_sig`。
仅做轻量字符串匹配,避免对普通图片 URL 跑完整签名校验路径;未携带签名
的外链不会触发 `invalid_signature` 标记,避免阻断日志误导未签名调用方。
"""
if not url:
return False
fragment_start = url.find("#")
if fragment_start < 0:
return False
return "mp_sig=" in url[fragment_start + 1:]
def _format_image_proxy_block_warning(
*,
url: str,
reason: str,
host: Optional[str],
ips: List[str],
allowed_private_ranges: List[str],
hint: Optional[str],
) -> str:
"""
渲染图片代理首条阻断 warning 文案。
字段范围严格限定为 URL、host、reason、IP 与允许网段配置hint 仅在
reason 与配置缺失同时满足时由调用方填充。其余敏感材料cookies、签名
串、token、请求头不允许进入该日志路径。
"""
fields = [
f"url={url}",
f"reason={reason}",
f"host={host or ''}",
f"ips={','.join(ips)}",
f"allowed_private_ranges={','.join(allowed_private_ranges)}",
]
line = "Blocked unsafe image URL: " + ", ".join(fields)
if hint:
line = f"{line} | {hint}"
return line
def _log_image_proxy_block_summary(summary: CoalesceSummary) -> None:
"""
图片代理阻断日志聚合窗口到期回调,输出窗口内的命中计数与首条样例。
summary.key 由 `_emit_image_proxy_block_warning` 固定构造为
`(host, reason_label)` 二元组;摘要保留首条事件的 URL 与解析 IP
避免运维只看到 count 而无法定位是哪批请求被合并。
"""
host, reason = summary.key
payload = summary.first_payload or {}
sample_ips = ",".join(payload.get("ips") or [])
logger.warn(
"Blocked unsafe image URL (aggregated): "
f"host={host or ''}, reason={reason}, "
f"count={summary.count}, window={summary.window_seconds:g}s, "
f"sample_url={payload.get('url', '')}, sample_ips={sample_ips}"
)
# 图片代理阻断日志聚合器。同 (host, reason) 高频拦截在窗口内合并为一条聚合摘要,避免媒体详情页一次请求把日志刷爆;
# 放行 debug 日志与诊断布尔结果不受聚合影响。
_image_proxy_block_log_coalescer = EventCoalescer(
window_seconds=_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS,
on_flush=_log_image_proxy_block_summary,
source="image_proxy",
)
async def _emit_image_proxy_block_warning(
*,
url: str,
diagnosis: "UrlSafetyDiagnosis",
signature_carried: bool,
allowed_private_ranges: Optional[Iterable[str]],
) -> None:
"""
把诊断结果转写为结构化阻断 warning并交由 coalescer 决定是否实际输出。
`signature_carried=True` 表示请求 URL 在 fragment 里实际携带了代理签名但
校验失败,此时在 reason 末尾追加 `invalid_signature` 标记,便于区分
"未签名外链直接撞 allowlist""签名 URL 已失效"两种排查路径。
"""
# reason_label 既作为 warning 字段,也作为 coalescer 桶键的一部分;签名
# 标记拼接到同一字符串里是为了让"带签名失败"的命中与"裸 URL 失败"分桶,
# 各自独立计数与摘要,不要在不引入新桶维度的情况下拆开。
reason_label = diagnosis.reason.value
if signature_carried:
reason_label = f"{reason_label}+{_INVALID_SIGNATURE_TAG}"
allowed_ranges = [str(r) for r in (allowed_private_ranges or [])]
hint = (
_IMAGE_PROXY_FAKEIP_HINT
if diagnosis.reason is UrlSafetyReason.NON_GLOBAL_DNS_RESULT
and not allowed_ranges
else None
)
key = (diagnosis.host or "", reason_label)
payload = {"url": url, "ips": list(diagnosis.ips)}
decision = await _image_proxy_block_log_coalescer.record(key=key, payload=payload)
if decision is CoalesceDecision.EMIT:
logger.warn(
_format_image_proxy_block_warning(
url=url,
reason=reason_label,
host=diagnosis.host,
ips=list(diagnosis.ips),
allowed_private_ranges=allowed_ranges,
hint=hint,
)
)

View File

@@ -0,0 +1 @@
"""站点目录、认证与索引资源的应用能力包。"""

View File

@@ -0,0 +1,60 @@
from typing import List, Optional, Tuple
class SitesHelper:
"""声明 Cython 站点认证与索引扩展的宿主接口。"""
def __init__(self) -> None:
"""返回进程内共享的站点助手实例。"""
...
@property
def auth_version(self) -> str:
"""返回认证资源版本。"""
...
@property
def indexer_version(self) -> str:
"""返回站点索引资源版本。"""
...
@property
def auth_level(self) -> int:
"""返回当前用户认证等级。"""
...
def check(self, domain: str) -> Tuple[bool, str]:
"""检查站点域名是否触发访问频率限制。"""
...
def get_indexers(self) -> List[dict]:
"""返回全部可用站点索引配置。"""
...
async def async_get_indexers(self) -> List[dict]:
"""异步返回全部可用站点索引配置。"""
...
def get_indexer(self, domain: str) -> Optional[dict]:
"""按域名返回单个站点索引配置。"""
...
async def async_get_indexer(self, domain: str) -> Optional[dict]:
"""异步按域名返回单个站点索引配置。"""
...
def get_authsites(self) -> dict:
"""返回认证站点配置。"""
...
def get_indexsites(self) -> dict:
"""返回内置站点索引配置。"""
...
def check_user(
self,
site: Optional[str] = None,
params: Optional[dict] = None,
) -> Tuple[bool, str]:
"""校验用户站点认证信息并返回状态与消息。"""
...

View File

@@ -0,0 +1,82 @@
from typing import List, Optional
from app import schemas
from app.db.systemconfig_oper import SystemConfigOper
from app.schemas.types import SystemConfigKey
class StorageHelper:
"""
存储帮助类
"""
@staticmethod
def get_storagies() -> List[schemas.StorageConf]:
"""
获取所有存储设置
"""
storage_confs: List[dict] = SystemConfigOper().get(SystemConfigKey.Storages)
if not storage_confs:
return []
return [schemas.StorageConf(**s) for s in storage_confs]
def get_storage(self, storage: str) -> Optional[schemas.StorageConf]:
"""
获取指定存储配置
"""
storagies = self.get_storagies()
for s in storagies:
if s.type == storage:
return s
return None
def set_storage(self, storage: str, conf: dict):
"""
设置存储配置
"""
storagies = self.get_storagies()
if not storagies:
storagies = [
schemas.StorageConf(
type=storage,
config=conf
)
]
else:
for s in storagies:
if s.type == storage:
s.config = conf
break
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
def add_storage(self, storage: str, name: str, conf: dict):
"""
添加存储配置
"""
storagies = self.get_storagies()
if not storagies:
storagies = [
schemas.StorageConf(
type=storage,
name=name,
config=conf
)
]
else:
storagies.append(schemas.StorageConf(
type=storage,
name=name,
config=conf
))
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
def reset_storage(self, storage: str):
"""
重置存储配置
"""
storagies = self.get_storagies()
for s in storagies:
if s.type == storage:
s.config = {}
break
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])

627
app/application/torrent.py Normal file
View File

@@ -0,0 +1,627 @@
import datetime
import re
from functools import lru_cache
from pathlib import Path
from typing import Tuple, Optional, List, Union, Dict, Any
from urllib.parse import unquote
from torrentool.api import Torrent
from app.runtime.cache import TTLCache, FileCache
from app.runtime.config import settings
from app.domain.context import Context, TorrentInfo, MediaInfo
from app.domain.meta.metabase import MetaBase
from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format, parse_audio_quality
from app.domain.metainfo import MetaInfo
from app.db.site_oper import SiteOper
from app.db.systemconfig_oper import SystemConfigOper
from app.runtime.log import logger
from app.schemas.types import MediaType, SystemConfigKey
from app.adapters.network.http import RequestUtils
from app.domain.media import resolve_media_identity
from app.domain.string import StringUtils
_SIZE_UNIT = 1024 * 1024
@lru_cache(maxsize=512)
def _compile_filter_pattern(pattern: str) -> re.Pattern:
"""
编译订阅/工作流附加过滤正则。
用户输入沿用原本的正则语义,缓存只减少同一规则反复匹配大量种子时的编译成本。
"""
return re.compile(r"%s" % pattern, re.I)
def _filter_pattern_search(pattern: Union[str, int, float], content: str) -> bool:
"""
按原有字符串插值语义执行过滤正则匹配。
"""
return bool(_compile_filter_pattern(str(pattern)).search(content))
@lru_cache(maxsize=256)
def _parse_filter_size_range(size_range: str) -> Tuple[str, float, Optional[float]]:
"""
解析附加过滤的大小范围,单位为 MB。
"""
if size_range.find("-") != -1:
size_min, size_max = size_range.split("-")
return "between", float(size_min.strip()) * _SIZE_UNIT, float(size_max.strip()) * _SIZE_UNIT
if size_range.startswith(">"):
return "gte", float(size_range[1:].strip()) * _SIZE_UNIT, None
if size_range.startswith("<"):
return "lte", 0, float(size_range[1:].strip()) * _SIZE_UNIT
return "unknown", 0, None
class TorrentHelper:
"""
种子帮助类
"""
def __init__(self):
"""初始化种子失败地址缓存"""
self._invalid_torrents = TTLCache(region="invalid_torrents", maxsize=128, ttl=3600 * 24)
def download_torrent(self, url: str,
cookie: Optional[str] = None,
ua: Optional[str] = None,
referer: Optional[str] = None,
proxy: Optional[bool] = False,
cache_invalid: bool = True) \
-> Tuple[Optional[Path], Optional[Union[str, bytes]], Optional[str], Optional[list], Optional[str]]:
"""
把种子下载到本地
:param url: 种子下载地址
:param cookie: 站点 Cookie
:param ua: 请求 User-Agent
:param referer: 请求来源地址
:param proxy: 是否使用系统代理
:param cache_invalid: 是否缓存失败地址;短时凭证地址必须关闭
:return: 种子缓存相对路径【用于索引缓存】, 种子内容、种子主目录、种子文件清单、错误信息
"""
if url.startswith("magnet:"):
return None, url, "", [], f"磁力链接"
# 构建 torrent 种子文件的缓存路径
cache_path = Path(StringUtils.md5_hash(url)).with_suffix(".torrent")
# 缓存处理器
cache_backend = FileCache()
# 读取缓存的种子文件
torrent_content = cache_backend.get(cache_path.as_posix(), region="torrents")
if torrent_content:
# 缓存已存在
try:
# 获取种子目录和文件清单
folder_name, file_list = self.get_fileinfo_from_torrent_content(torrent_content)
# 无法获取信息,则认为缓存文件无效
if not folder_name and not file_list:
raise ValueError("无效的缓存种子文件")
# 成功拿到种子数据
return cache_path, torrent_content, folder_name, file_list, ""
except Exception as err:
logger.error(f"处理缓存的种子文件 {cache_path} 时出错: {err},将重新下载")
# 下载种子文件
req = RequestUtils(
ua=ua,
cookies=cookie,
referer=referer,
proxies=settings.PROXY if proxy else None
).get_res(url=url, allow_redirects=False)
while req and req.status_code in [301, 302]:
url = req.headers['Location']
if url and url.startswith("magnet:"):
return None, url, "", [], f"获取到磁力链接"
req = RequestUtils(
ua=ua,
cookies=cookie,
referer=referer,
proxies=settings.PROXY if proxy else None
).get_res(url=url, allow_redirects=False)
if req and req.status_code == 200:
if not req.content:
return cache_path, None, "", [], "未下载到种子数据"
# 解析内容格式
if req.content.startswith(b"magnet:"):
# 磁力链接
return cache_path, req.text, "", [], f"获取到磁力链接"
if "下载种子文件".encode("utf-8") in req.content:
# 首次下载提示页面
skip_flag = False
try:
forms = re.findall(r'<form.*?action="(.*?)".*?>(.*?)</form>', req.text, re.S)
for form in forms:
action = form[0]
if action != "?":
continue
action = url
inputs = re.findall(r'<input.*?name="(.*?)".*?value="(.*?)".*?>', form[1], re.S)
if inputs:
data = {}
for item in inputs:
data[item[0]] = item[1]
# 改写req
req = RequestUtils(
ua=ua,
cookies=cookie,
referer=referer,
proxies=settings.PROXY if proxy else None
).post_res(url=action, data=data)
if req and req.status_code == 200:
# 检查是不是种子文件,如果不是抛出异常
Torrent.from_string(req.content)
# 跳过成功
logger.info("触发了站点首次种子下载,已自动跳过")
skip_flag = True
elif req is not None:
logger.warn(f"触发了站点首次种子下载,且无法自动跳过,"
f"返回码:{req.status_code},错误原因:{req.reason}")
else:
logger.warn("触发了站点首次种子下载,且无法自动跳过")
break
except Exception as err:
logger.warn(f"触发了站点首次种子下载,尝试自动跳过时出现错误:{str(err)}")
if not skip_flag:
return cache_path, None, "", [], "种子数据有误请确认链接是否正确如为PT站点则需手工在站点下载一次种子"
# 种子内容
if req.content:
# 检查是不是种子文件,如果不是仍然抛出异常
try:
# 获取种子目录和文件清单
folder_name, file_list = self.get_fileinfo_from_torrent_content(req.content)
if file_list:
# 保存到缓存
cache_backend.set(cache_path.as_posix(), req.content, region="torrents")
# 成功拿到种子数据
return cache_path, req.content, folder_name, file_list, ""
except Exception as err:
logger.error(f"种子文件解析失败:{str(err)}")
# 种子数据仍然错误
return cache_path, None, "", [], "种子数据有误,请确认链接是否正确"
# 返回失败
return cache_path, None, "", [], ""
elif req is None:
return cache_path, None, "", [], "无法打开链接"
elif req.status_code == 429:
return cache_path, None, "", [], "触发站点流控,请稍后重试"
else:
# 把错误的种子记下来,避免重复使用
if cache_invalid:
self.add_invalid(url)
return cache_path, None, "", [], f"下载种子出错,状态码:{req.status_code}"
def get_torrent_info(self, torrent_path: Path) -> Tuple[str, List[str]]:
"""
获取种子文件的文件夹名和文件清单
:param torrent_path: 种子文件路径
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
"""
if not torrent_path or not torrent_path.exists():
return "", []
try:
torrentinfo = Torrent.from_file(torrent_path)
# 获取文件清单
return self.get_fileinfo_from_torrent(torrentinfo)
except Exception as err:
logger.error(f"种子文件解析失败:{str(err)}")
return "", []
@staticmethod
def get_fileinfo_from_torrent(torrent: Torrent) -> Tuple[str, List[str]]:
"""
从种子文件中获取文件清单
:param torrent: 种子文件对象
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
"""
if not torrent or not torrent.files:
return "", []
# 获取文件清单
if len(torrent.files) == 1 and torrent.files[0].name == torrent.name:
# 单文件种子目录名返回空
folder_name = ""
# 单文件种子
file_list = [torrent.name]
else:
# 目录名
folder_name = torrent.name
# 文件清单,如果一级目录与种子名相同则去掉
file_list = []
for fileinfo in torrent.files:
file_path = Path(fileinfo.name)
# 根路径
root_path = file_path.parts[0]
if root_path == folder_name:
file_list.append(str(file_path.relative_to(root_path)))
else:
file_list.append(fileinfo.name)
logger.debug(f"解析种子:{torrent.name} => 目录:{folder_name},文件清单:{file_list}")
return folder_name, file_list
def get_fileinfo_from_torrent_content(self, torrent_content: Union[str, bytes]) -> Tuple[str, List[str]]:
"""
从种子内容中获取文件夹名和文件清单
:param torrent_content: 种子内容
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
"""
if not torrent_content:
return "", []
# 检查是否为磁力链接
if StringUtils.is_magnet_link(torrent_content):
return "", []
try:
# 解析种子内容
torrentinfo = Torrent.from_string(torrent_content)
# 获取文件清单
return self.get_fileinfo_from_torrent(torrentinfo)
except Exception as err:
logger.error(f"种子内容解析失败:{str(err)}")
return "", []
@staticmethod
def get_url_filename(req: Any, url: str) -> str:
"""
从下载请求中获取种子文件名
"""
if not req:
return ""
disposition = req.headers.get('content-disposition') or ""
file_name = re.findall(r"filename=\"?(.+)\"?", disposition)
if file_name:
file_name = unquote(str(file_name[0].encode('ISO-8859-1').decode()).split(";")[0].strip())
if file_name.endswith('"'):
file_name = file_name[:-1]
elif url and url.endswith(".torrent"):
file_name = unquote(url.split("/")[-1])
else:
file_name = str(datetime.datetime.now())
return file_name
@staticmethod
def sort_torrents(torrent_list: List[Context]) -> List[Context]:
"""
对种子对行排序torrent、site、upload、seeder
"""
if not torrent_list:
return []
# 下载规则
priority_rule: List[str] = SystemConfigOper().get(
SystemConfigKey.TorrentsPriority) or ["torrent", "upload", "seeder"]
# 站点上传量
site_uploads = {
site.name: site.upload for site in SiteOper().get_userdata_latest()
}
def get_sort_str(_context):
"""
拼装排序字段
"""
_meta = _context.meta_info
_torrent = _context.torrent_info
_media = _context.media_info
# 标题
_title = str(_media.title).ljust(200, ' ')
# 站点优先级
_site_order = str(999 - (_torrent.site_order or 0)).rjust(3, '0')
# 站点上传量
_site_upload = str(site_uploads.get(_torrent.site_name) or 0).rjust(30, '0')
# 资源优先级
_torrent_order = str(_torrent.pri_order or 0).rjust(3, '0')
# 资源做种数
_torrent_seeders = str(_torrent.seeders or 0).rjust(10, '0')
# 季集
if not _meta.episode_list:
# 无集数的排最前面
_season_episode = "%s%s" % (str(len(_meta.season_list)).rjust(3, '0'), "9999")
else:
# 集数越多的排越前面
_season_episode = "%s%s" % (str(len(_meta.season_list)).rjust(3, '0'),
str(len(_meta.episode_list)).rjust(4, '0'))
# 根据下载规则的顺序拼装排序字符串
_sort_str = _title
for rule in priority_rule:
if rule == "torrent":
_sort_str += _torrent_order
elif rule == "site":
_sort_str += _site_order
elif rule == "upload":
_sort_str += _site_upload
elif rule == "seeder":
_sort_str += _torrent_seeders
_sort_str += _season_episode
return _sort_str
# 排序
return sorted(torrent_list, key=lambda x: get_sort_str(x), reverse=True)
def sort_group_torrents(self, torrent_list: List[Context]) -> List[Context]:
"""
对媒体信息进行排序、去重
"""
if not torrent_list:
return []
# 排序
torrent_list = self.sort_torrents(torrent_list)
# 控重
result = []
_added = []
# 排序后重新加入数组,按真实名称控重,即只取每个名称的第一个
for context in torrent_list:
# 控重的主链是名称、年份、季、集
meta = context.meta_info
media = context.media_info
if media.type == MediaType.TV:
media_name = "%s%s" % (media.title_year,
meta.season_episode)
else:
media_name = media.title_year
if media_name not in _added:
_added.append(media_name)
result.append(context)
return result
@staticmethod
def get_torrent_episodes(files: list, custom_words: Optional[List[str]] = None) -> list:
"""
从种子的文件清单中获取所有集数
:param files: 种子文件清单
:param custom_words: 当前下载来源的临时自定义识别词
:return: 识别到的全部集数
"""
episodes = []
for file in files:
if not file:
continue
file_path = Path(file)
if not file_path.suffix or file_path.suffix.lower() not in settings.RMT_MEDIAEXT:
continue
# 只使用文件名识别
meta = MetaInfo(file_path.name, custom_words=custom_words)
if not meta.begin_episode:
continue
episodes = list(set(episodes).union(set(meta.episode_list)))
return episodes
def is_invalid(self, url: Optional[str]) -> bool:
"""
判断种子是否是无效种子
"""
return url in self._invalid_torrents if url else True
def add_invalid(self, url: str):
"""
添加无效种子
"""
if url not in self._invalid_torrents:
self._invalid_torrents[url] = True
@staticmethod
def match_torrent(mediainfo: MediaInfo, torrent_meta: MetaBase, torrent: TorrentInfo) -> bool:
"""
检查种子是否匹配媒体信息
:param mediainfo: 需要匹配的媒体信息
:param torrent_meta: 种子识别信息
:param torrent: 种子信息
"""
# 显式标题标签与识别结果使用同一主身份比较。
torrent_identity = resolve_media_identity(media=torrent_meta)
media_identity = resolve_media_identity(media=mediainfo)
if all(torrent_identity) and torrent_identity == media_identity:
logger.info(
f'{mediainfo.title} 通过词表指定媒体身份 '
f'{torrent_identity[0]}:{torrent_identity[1]} 匹配到资源:'
f'{torrent.site_name} - {torrent.title}'
)
return True
# 要匹配的媒体标题、原标题
media_titles = {
StringUtils.clear_upper(mediainfo.title),
StringUtils.clear_upper(mediainfo.original_title)
} - {""}
# 要匹配的媒体别名、译名
media_names = {StringUtils.clear_upper(name) for name in mediainfo.names if name}
# 识别的种子中英文名
meta_names = {
StringUtils.clear_upper(torrent_meta.cn_name),
StringUtils.clear_upper(torrent_meta.en_name)
} - {""}
# 比对种子识别类型
if torrent_meta.type == MediaType.TV and mediainfo.type != MediaType.TV:
logger.debug(f'{torrent.site_name} - {torrent.title} 种子标题类型为 {torrent_meta.type.value}'
f'不匹配 {mediainfo.type.value}')
return False
# 比对种子在站点中的类型
if torrent.category == MediaType.TV.value and mediainfo.type != MediaType.TV:
logger.debug(f'{torrent.site_name} - {torrent.title} 种子在站点中归类为 {torrent.category}'
f'不匹配 {mediainfo.type.value}')
return False
# 比对年份
if mediainfo.year:
if mediainfo.type == MediaType.TV:
# 剧集年份,每季的年份可能不同,没年份时不比较年份(很多剧集种子不带年份)
if torrent_meta.year and torrent_meta.year not in [year for year in
mediainfo.season_years.values()]:
logger.debug(f'{torrent.site_name} - {torrent.title} 年份不匹配 {mediainfo.season_years}')
return False
else:
# 电影年份上下浮动1年没年份时不通过
if not torrent_meta.year or torrent_meta.year not in [str(int(mediainfo.year) - 1),
mediainfo.year,
str(int(mediainfo.year) + 1)]:
logger.debug(f'{torrent.site_name} - {torrent.title} 年份不匹配 {mediainfo.year}')
return False
# 比对标题和原语种标题
if meta_names.intersection(media_titles):
logger.info(f'{mediainfo.title} 通过标题匹配到资源:{torrent.site_name} - {torrent.title}')
return True
# 比对别名和译名
if media_names:
if meta_names.intersection(media_names):
logger.info(f'{mediainfo.title} 通过别名或译名匹配到资源:{torrent.site_name} - {torrent.title}')
return True
# 标题拆分
if torrent_meta.org_string:
# 只拆分出标题中的非英文单词进行匹配,英文单词容易误匹配(带空格的多个单词组合除外)
titles = [StringUtils.clear_upper(t) for t in re.split(
r'[\s/【】.\[\]\-]+',
torrent_meta.org_string
) if not StringUtils.is_english_word(t)]
# 在标题中判断是否存在标题、原语种标题
if media_titles.intersection(titles):
logger.info(f'{mediainfo.title} 通过标题匹配到资源:{torrent.site_name} - {torrent.title}')
return True
# 在副标题中(非英文单词)判断是否存在标题、原语种标题、别名、译名
if torrent.description:
subtitles = {StringUtils.clear_upper(t) for t in re.split(
r'[\s/【】|]+',
torrent.description) if not StringUtils.is_english_word(t)}
if media_titles.intersection(subtitles) or media_names.intersection(subtitles):
logger.info(f'{mediainfo.title} 通过副标题匹配到资源:{torrent.site_name} - {torrent.title}'
f'副标题:{torrent.description}')
return True
# 未匹配
logger.debug(f'{torrent.site_name} - {torrent.title} 标题不匹配,识别名称:{meta_names}')
return False
@staticmethod
def filter_torrent(torrent_info: TorrentInfo,
filter_params: Dict[str, Any]) -> bool:
"""
检查种子是否匹配订阅过滤规则
"""
if not filter_params:
return True
# 匹配内容
content = (f"{torrent_info.title} "
f"{torrent_info.description} "
f"{' '.join(torrent_info.labels or [])} "
f"{torrent_info.volume_factor}")
# 包含
include = filter_params.get("include")
if include:
if not _filter_pattern_search(include, content):
logger.info(f"{content} 不匹配包含规则 {include}")
return False
# 排除
exclude = filter_params.get("exclude")
if exclude:
if _filter_pattern_search(exclude, content):
logger.info(f"{content} 匹配排除规则 {exclude}")
return False
# 质量
quality = filter_params.get("quality")
if quality:
if not _filter_pattern_search(quality, torrent_info.title):
logger.info(f"{torrent_info.title} 不匹配质量规则 {quality}")
return False
# 分辨率
resolution = filter_params.get("resolution")
if resolution:
if not _filter_pattern_search(resolution, torrent_info.title):
logger.info(f"{torrent_info.title} 不匹配分辨率规则 {resolution}")
return False
# 特效
effect = filter_params.get("effect")
if effect:
if not _filter_pattern_search(effect, torrent_info.title):
logger.info(f"{torrent_info.title} 不匹配特效规则 {effect}")
return False
# 音乐音质。技术参数从标题、副标题和标签统一解析,避免只靠用户正则筛选。
audio_filters = {
key: filter_params.get(key)
for key in ("audio_quality", "audio_format", "min_bitrate", "min_bit_depth", "min_sample_rate")
if filter_params.get(key) not in (None, "")
}
if audio_filters:
specs = parse_audio_quality(content)
tier = audio_quality_tier(**specs)
audio_quality = audio_filters.get("audio_quality")
quality_pattern = "hires|lossless" \
if str(audio_quality).casefold() == "lossless" else audio_quality
if audio_quality and (not tier or not _filter_pattern_search(quality_pattern, tier)):
logger.info(f"{torrent_info.title} 不匹配音乐音质规则 {audio_quality}")
return False
audio_format = audio_filters.get("audio_format")
normalized_format = normalize_audio_format(specs.get("audio_format"))
if audio_format and (
not normalized_format or not _filter_pattern_search(audio_format, normalized_format)
):
logger.info(f"{torrent_info.title} 不匹配音频格式规则 {audio_format}")
return False
for key, spec_key, label in (
("min_bitrate", "bitrate", "码率"),
("min_bit_depth", "bit_depth", "位深"),
("min_sample_rate", "sample_rate", "采样率"),
):
minimum = audio_filters.get(key)
actual = specs.get(spec_key)
if minimum is not None and (actual is None or int(actual) < int(minimum)):
logger.info(f"{torrent_info.title} 不满足最低{label} {minimum}")
return False
# 大小
size_range = filter_params.get("size")
if size_range:
size_rule, size_min, size_max = _parse_filter_size_range(size_range)
if size_rule == "between":
# 区间
if torrent_info.size < size_min or torrent_info.size > size_max:
return False
elif size_rule == "gte":
# 大于
if torrent_info.size < size_min:
return False
elif size_rule == "lte":
# 小于
if torrent_info.size > size_max:
return False
return True
@staticmethod
def match_season_episodes(torrent: TorrentInfo, meta: MetaBase, season_episodes: Dict[int, list]) -> bool:
"""
判断种子是否匹配季集数
:param torrent: 种子信息
:param meta: 种子元数据
:param season_episodes: 季集数 {season:[episodes]}
"""
# 匹配季
seasons = season_episodes.keys()
seasons_set = set(seasons)
# 种子季
torrent_seasons = meta.season_list
if not torrent_seasons:
# 按第一季处理
torrent_seasons = [1]
# 种子集
torrent_episodes = meta.episode_list
if not set(torrent_seasons).issubset(seasons_set):
# 种子季不在过滤季中
logger.debug(
f"种子 {torrent.site_name} - {torrent.title} 包含季 {torrent_seasons} 不是需要的季 {list(seasons)}")
return False
if not torrent_episodes:
# 整季按匹配处理
return True
if len(torrent_seasons) == 1:
need_episodes = season_episodes.get(torrent_seasons[0])
if need_episodes \
and not set(torrent_episodes).intersection(need_episodes):
# 单季集没有交集的不要
logger.debug(f"种子 {torrent.site_name} - {torrent.title} "
f"{torrent_episodes} 没有需要的集:{need_episodes}")
return False
return True