mirror of
https://github.com/jxxghp/MoviePilot.git
synced 2026-08-19 05:03:57 +08:00
refactor: reorganize backend module boundaries
This commit is contained in:
1
app/application/__init__.py
Normal file
1
app/application/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""编排领域对象和基础设施的应用服务。"""
|
||||
317
app/application/audio.py
Normal file
317
app/application/audio.py
Normal file
@@ -0,0 +1,317 @@
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Union
|
||||
from uuid import UUID
|
||||
|
||||
from mutagen import File as MutagenFile
|
||||
from mutagen.flac import FLAC, Picture
|
||||
from mutagen.id3 import APIC
|
||||
from mutagen.mp4 import MP4, MP4Cover
|
||||
|
||||
from app.domain.context import MusicInfo
|
||||
from app.domain.meta.metamusic import MetaMusic
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import MUSIC_ENTITY_RECORDING, MediaSource
|
||||
|
||||
|
||||
class AudioMetadataHelper:
|
||||
"""读取和写入音频标签,并转换为标准音乐元数据。"""
|
||||
|
||||
@classmethod
|
||||
def read(cls, path: Path) -> MetaMusic:
|
||||
"""读取本地音频标签,并以完整文件名模式和目录线索补充缺失字段。"""
|
||||
tag_meta = cls.read_tags(path)
|
||||
if tag_meta:
|
||||
return tag_meta.apply_path_context(path)
|
||||
return cls.read_filename(path)
|
||||
|
||||
@classmethod
|
||||
def read_evidence(
|
||||
cls,
|
||||
path: Path,
|
||||
) -> tuple[MetaMusic, Optional[MetaMusic], MetaMusic]:
|
||||
"""分别返回合并元数据、纯标签元数据和纯文件名元数据。"""
|
||||
filename_meta = cls.read_filename(path)
|
||||
tag_meta = cls.read_tags(path) if path.exists() and path.is_file() else None
|
||||
if not tag_meta:
|
||||
return filename_meta, None, filename_meta
|
||||
merged_meta = MetaMusic.from_dict(tag_meta.to_dict()).apply_path_context(path)
|
||||
return merged_meta, tag_meta, filename_meta
|
||||
|
||||
@classmethod
|
||||
def read_many(cls, paths: list[Path]) -> list[MetaMusic]:
|
||||
"""批量读取一组音频路径的标签与文件名元数据。"""
|
||||
return [cls.read(path) for path in paths]
|
||||
|
||||
@classmethod
|
||||
def read_tags(cls, path: Path) -> Optional[MetaMusic]:
|
||||
"""只读取本地音频标签和流参数,不使用文件名或目录补齐。"""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
except Exception as err:
|
||||
logger.warning(f"读取音频标签失败:{path} - {err}")
|
||||
return None
|
||||
if not audio:
|
||||
return None
|
||||
|
||||
tags = audio.tags or {}
|
||||
track_number, total_tracks = cls._number_pair(cls._first(tags, "tracknumber"))
|
||||
disc_number, total_discs = cls._number_pair(cls._first(tags, "discnumber"))
|
||||
musicbrainz_id = cls._normalize_musicbrainz_id(
|
||||
cls._first_of(
|
||||
tags,
|
||||
"musicbrainz_trackid",
|
||||
"musicbrainz_recordingid",
|
||||
)
|
||||
)
|
||||
info = getattr(audio, "info", None)
|
||||
return MetaMusic(
|
||||
org_string=path.name,
|
||||
title=cls._first(tags, "title"),
|
||||
artists=cls._values(tags, "artist"),
|
||||
album=cls._first(tags, "album"),
|
||||
album_artist=cls._first(tags, "albumartist"),
|
||||
year=cls._year(cls._first(tags, "date") or cls._first(tags, "originaldate")),
|
||||
disc_number=disc_number,
|
||||
track_number=track_number,
|
||||
total_discs=total_discs,
|
||||
total_tracks=total_tracks,
|
||||
version=cls._first(tags, "version") or cls._first(tags, "subtitle"),
|
||||
audio_format=cls._audio_format(path, info),
|
||||
bit_depth=cls._optional_int(getattr(info, "bits_per_sample", None)),
|
||||
sample_rate=cls._optional_int(getattr(info, "sample_rate", None)),
|
||||
bitrate=cls._optional_int(getattr(info, "bitrate", None)),
|
||||
duration=round(info.length) if info and getattr(info, "length", None) else None,
|
||||
isrc=cls._first(tags, "isrc"),
|
||||
media_source=MediaSource.MusicBrainz if musicbrainz_id else None,
|
||||
media_id=musicbrainz_id,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def read_filename(path: Path) -> MetaMusic:
|
||||
"""只从文件名和目录结构解析音乐元数据。"""
|
||||
return MetaMusic(
|
||||
org_string=path.name,
|
||||
title=path.stem,
|
||||
audio_format=path.suffix.lstrip(".").upper() or None,
|
||||
).apply_path_context(path)
|
||||
|
||||
@classmethod
|
||||
def write(
|
||||
cls,
|
||||
path: Path,
|
||||
music: Union[MetaMusic, MusicInfo],
|
||||
cover_data: Optional[bytes] = None,
|
||||
cover_mime: str = "image/jpeg",
|
||||
overwrite: bool = True,
|
||||
write_tags: bool = True,
|
||||
cover_overwrite: Optional[bool] = None,
|
||||
) -> bool:
|
||||
"""按独立策略写入标准音乐标签,并为常见格式嵌入专辑封面。"""
|
||||
try:
|
||||
audio = MutagenFile(path, easy=True)
|
||||
if not audio:
|
||||
logger.warning(f"无法写入音频标签:{path}")
|
||||
return False
|
||||
if write_tags:
|
||||
if audio.tags is None:
|
||||
audio.add_tags()
|
||||
for key, value in cls._tag_values(music).items():
|
||||
if value in (None, "", []):
|
||||
continue
|
||||
if not overwrite and audio.tags.get(key):
|
||||
continue
|
||||
try:
|
||||
audio[key] = value if isinstance(value, list) else [str(value)]
|
||||
except (KeyError, TypeError, ValueError) as err:
|
||||
logger.debug(f"音频格式不支持标签 {key}:{path} - {err}")
|
||||
audio.save()
|
||||
if cover_data:
|
||||
cls._write_cover(
|
||||
path=path,
|
||||
cover_data=cover_data,
|
||||
cover_mime=cover_mime,
|
||||
overwrite=(
|
||||
overwrite
|
||||
if cover_overwrite is None
|
||||
else cover_overwrite
|
||||
),
|
||||
)
|
||||
return True
|
||||
except Exception as err:
|
||||
logger.warning(f"写入音频标签失败:{path} - {err}")
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _tag_values(cls, music: Union[MetaMusic, MusicInfo]) -> dict[str, Any]:
|
||||
"""把标准音乐对象转换为 Mutagen Easy 标签字典。"""
|
||||
track_number = cls._number_text(
|
||||
getattr(music, "track_number", None),
|
||||
getattr(music, "total_tracks", None),
|
||||
)
|
||||
disc_number = cls._number_text(
|
||||
getattr(music, "disc_number", None),
|
||||
getattr(music, "total_discs", None),
|
||||
)
|
||||
return {
|
||||
"title": getattr(music, "title", None),
|
||||
"artist": list(getattr(music, "artists", None) or []),
|
||||
"album": getattr(music, "album", None),
|
||||
"albumartist": getattr(music, "album_artist", None),
|
||||
"date": getattr(music, "year", None),
|
||||
"tracknumber": track_number,
|
||||
"discnumber": disc_number,
|
||||
"isrc": getattr(music, "isrc", None),
|
||||
"musicbrainz_trackid": cls._musicbrainz_recording_id(music),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _musicbrainz_recording_id(
|
||||
music: Union[MetaMusic, MusicInfo],
|
||||
) -> Optional[str]:
|
||||
"""仅将 MusicBrainz 单曲身份写入 recording 标签,避免误写专辑 ID。"""
|
||||
if (
|
||||
getattr(music, "media_source", None) == MediaSource.MusicBrainz
|
||||
and getattr(music, "music_type", MUSIC_ENTITY_RECORDING)
|
||||
== MUSIC_ENTITY_RECORDING
|
||||
):
|
||||
media_id = getattr(music, "media_id", None)
|
||||
return str(media_id) if media_id else None
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _number_text(current: Optional[int], total: Optional[int]) -> Optional[str]:
|
||||
"""把曲序或碟号转换为常见的 current/total 标签文本。"""
|
||||
if current is None:
|
||||
return None
|
||||
return f"{current}/{total}" if total else str(current)
|
||||
|
||||
@staticmethod
|
||||
def _write_cover(
|
||||
path: Path,
|
||||
cover_data: bytes,
|
||||
cover_mime: str,
|
||||
overwrite: bool,
|
||||
) -> None:
|
||||
"""为 MP3、FLAC 和 MP4/M4A 写入内嵌封面,其它格式保留标签写入结果。"""
|
||||
audio = MutagenFile(path)
|
||||
if isinstance(audio, FLAC):
|
||||
if audio.pictures and not overwrite:
|
||||
return
|
||||
picture = Picture()
|
||||
picture.type = 3
|
||||
picture.mime = cover_mime
|
||||
picture.desc = "Cover"
|
||||
picture.data = cover_data
|
||||
if overwrite:
|
||||
audio.clear_pictures()
|
||||
audio.add_picture(picture)
|
||||
audio.save()
|
||||
return
|
||||
if isinstance(audio, MP4):
|
||||
if audio.tags is None:
|
||||
audio.add_tags()
|
||||
if audio.tags.get("covr") and not overwrite:
|
||||
return
|
||||
image_format = (
|
||||
MP4Cover.FORMAT_PNG
|
||||
if cover_mime == "image/png"
|
||||
else MP4Cover.FORMAT_JPEG
|
||||
)
|
||||
audio.tags["covr"] = [MP4Cover(cover_data, imageformat=image_format)]
|
||||
audio.save()
|
||||
return
|
||||
tags = getattr(audio, "tags", None)
|
||||
if tags is not None and hasattr(tags, "add"):
|
||||
if tags.getall("APIC") and not overwrite:
|
||||
return
|
||||
if overwrite:
|
||||
tags.delall("APIC")
|
||||
tags.add(
|
||||
APIC(
|
||||
encoding=3,
|
||||
mime=cover_mime,
|
||||
type=3,
|
||||
desc="Cover",
|
||||
data=cover_data,
|
||||
)
|
||||
)
|
||||
audio.save()
|
||||
|
||||
@staticmethod
|
||||
def _values(tags: Any, key: str) -> list[str]:
|
||||
"""从 Mutagen Easy 标签中提取非空字符串列表。"""
|
||||
value = tags.get(key) if hasattr(tags, "get") else None
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, (list, tuple)):
|
||||
return [str(item).strip() for item in value if str(item).strip()]
|
||||
return [str(value).strip()] if str(value).strip() else []
|
||||
|
||||
@classmethod
|
||||
def _first(cls, tags: Any, key: str) -> Optional[str]:
|
||||
"""返回指定音频标签的第一个非空值。"""
|
||||
values = cls._values(tags, key)
|
||||
return values[0] if values else None
|
||||
|
||||
@classmethod
|
||||
def _first_of(cls, tags: Any, *keys: str) -> Optional[str]:
|
||||
"""按顺序返回多个音频标签中的第一个非空值。"""
|
||||
for key in keys:
|
||||
if value := cls._first(tags, key):
|
||||
return value
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_musicbrainz_id(value: Optional[str]) -> Optional[str]:
|
||||
"""校验并规范化音频标签中的 MusicBrainz UUID。"""
|
||||
try:
|
||||
return str(UUID(str(value)))
|
||||
except (AttributeError, TypeError, ValueError):
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _number_pair(value: Optional[str]) -> tuple[Optional[int], Optional[int]]:
|
||||
"""解析 track/disc 标签中的当前编号和总数。"""
|
||||
if not value:
|
||||
return None, None
|
||||
parts = str(value).split("/", 1)
|
||||
current = AudioMetadataHelper._optional_int(parts[0])
|
||||
total = AudioMetadataHelper._optional_int(parts[1]) if len(parts) > 1 else None
|
||||
return current, total
|
||||
|
||||
@staticmethod
|
||||
def _year(value: Optional[str]) -> Optional[int]:
|
||||
"""从完整或不完整日期标签中提取四位年份。"""
|
||||
if not value:
|
||||
return None
|
||||
return AudioMetadataHelper._optional_int(str(value)[:4])
|
||||
|
||||
@staticmethod
|
||||
def _audio_format(path: Path, info: Any) -> Optional[str]:
|
||||
"""结合扩展名与流编码识别音频格式,区分同为 M4A 容器的 AAC 和 ALAC。"""
|
||||
codec_text = " ".join(
|
||||
str(value or "")
|
||||
for value in (
|
||||
getattr(info, "codec", None),
|
||||
getattr(info, "codec_description", None),
|
||||
)
|
||||
).casefold()
|
||||
codec_formats = (
|
||||
(("alac", "apple lossless"), "ALAC"),
|
||||
(("aac", "mp4a"), "AAC"),
|
||||
(("opus",), "OPUS"),
|
||||
(("vorbis",), "OGG"),
|
||||
(("flac",), "FLAC"),
|
||||
)
|
||||
for markers, audio_format in codec_formats:
|
||||
if any(marker in codec_text for marker in markers):
|
||||
return audio_format
|
||||
return path.suffix.lstrip(".").upper() or None
|
||||
|
||||
@staticmethod
|
||||
def _optional_int(value: Any) -> Optional[int]:
|
||||
"""将音频技术参数安全转换为整数。"""
|
||||
try:
|
||||
return int(value) if value is not None and str(value).strip() else None
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
354
app/application/directory.py
Normal file
354
app/application/directory.py
Normal file
@@ -0,0 +1,354 @@
|
||||
import re
|
||||
from pathlib import Path, PurePath, PurePosixPath, PureWindowsPath
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
from app import schemas
|
||||
from app.domain.context import MediaInfo
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import MediaType, StorageSchema, SystemConfigKey
|
||||
from app.adapters.system.host import SystemUtils
|
||||
|
||||
JINJA2_VAR_PATTERN = re.compile(r"\{\{.*?}}", re.DOTALL)
|
||||
WINDOWS_DRIVE_PATTERN = re.compile(r"^[A-Za-z]:[\\/]")
|
||||
WINDOWS_DRIVE_PREFIX_PATTERN = re.compile(r"^[A-Za-z]:")
|
||||
|
||||
|
||||
class DirectoryHelper:
|
||||
"""
|
||||
下载目录/媒体库目录帮助类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_dirs() -> List[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
获取所有下载目录
|
||||
"""
|
||||
dir_confs: List[dict] = SystemConfigOper().get(SystemConfigKey.Directories)
|
||||
if not dir_confs:
|
||||
return []
|
||||
return [schemas.TransferDirectoryConf(**d) for d in dir_confs]
|
||||
|
||||
def get_download_dirs(self) -> List[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
获取所有下载目录
|
||||
"""
|
||||
return sorted([d for d in self.get_dirs() if d.download_path], key=lambda x: x.priority)
|
||||
|
||||
def get_local_download_dirs(self) -> List[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
获取所有本地的可下载目录
|
||||
"""
|
||||
return [d for d in self.get_download_dirs() if d.storage == "local"]
|
||||
|
||||
def get_download_dir_by_save_path(
|
||||
self,
|
||||
media: Optional[MediaInfo],
|
||||
save_path: str,
|
||||
) -> Optional[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
按媒体信息和精确保存根路径匹配下载目录配置。
|
||||
|
||||
仅配置根目录本身继承自动分类规则;根目录下的自定义子目录保持调用方指定的完整路径。
|
||||
|
||||
:param media: 媒体信息
|
||||
:param save_path: 已选择的下载保存目录,支持本地路径或远端 FileURI
|
||||
:return: 匹配的下载目录配置
|
||||
"""
|
||||
value = str(save_path or "").strip()
|
||||
try:
|
||||
storage, raw_path = _split_file_uri(value)
|
||||
target_style, target_path = _normalize_download_path(raw_path, storage)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
media_type = media.type.value if media else None
|
||||
for dir_info in self.get_download_dirs():
|
||||
root = _normalize_download_root(dir_info)
|
||||
if not root:
|
||||
continue
|
||||
root_storage, root_style, root_path = root
|
||||
if storage != root_storage or target_style != root_style or target_path != root_path:
|
||||
continue
|
||||
if not media_type or not dir_info.media_type:
|
||||
return dir_info
|
||||
if dir_info.media_type == media_type and not dir_info.media_category:
|
||||
return dir_info
|
||||
if dir_info.media_type == media_type and dir_info.media_category == media.category:
|
||||
return dir_info
|
||||
return None
|
||||
|
||||
def get_library_dirs(self) -> List[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
获取所有媒体库目录
|
||||
"""
|
||||
return sorted([d for d in self.get_dirs() if d.library_path], key=lambda x: x.priority)
|
||||
|
||||
def get_local_library_dirs(self) -> List[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
获取所有本地的媒体库目录
|
||||
"""
|
||||
return [d for d in self.get_library_dirs() if d.library_storage == "local"]
|
||||
|
||||
def get_dir(self, media: Optional[MediaInfo], include_unsorted: Optional[bool] = False,
|
||||
storage: Optional[str] = None, src_path: Path = None,
|
||||
target_storage: Optional[str] = None, dest_path: Path = None
|
||||
) -> Optional[schemas.TransferDirectoryConf]:
|
||||
"""
|
||||
根据媒体信息获取下载目录、媒体库目录配置
|
||||
:param media: 媒体信息
|
||||
:param include_unsorted: 包含不整理目录
|
||||
:param storage: 源存储类型
|
||||
:param target_storage: 目标存储类型
|
||||
:param src_path: 源目录,有值时直接匹配
|
||||
:param dest_path: 目标目录,有值时直接匹配
|
||||
"""
|
||||
# 电影/电视剧
|
||||
media_type = media.type.value if media else None
|
||||
dirs = self.get_dirs()
|
||||
|
||||
# 如果存在源目录,并源目录为任一下载目录的子目录时,则进行源目录匹配,否则,允许源目录按同盘优先的逻辑匹配
|
||||
matching_dirs = [d for d in dirs if src_path.is_relative_to(d.download_path)] if src_path else []
|
||||
# 根据是否有匹配的源目录,决定要考虑的目录集合
|
||||
dirs_to_consider = matching_dirs if matching_dirs else dirs
|
||||
|
||||
# 已匹配的目录
|
||||
matched_dirs: List[schemas.TransferDirectoryConf] = []
|
||||
# 按照配置顺序查找
|
||||
for d in dirs_to_consider:
|
||||
# 没有启用整理的目录
|
||||
if not d.monitor_type and not include_unsorted:
|
||||
continue
|
||||
# 源存储类型不匹配
|
||||
if storage and d.storage != storage:
|
||||
continue
|
||||
# 目标存储类型不匹配
|
||||
if target_storage and d.library_storage != target_storage:
|
||||
continue
|
||||
# 有目标目录时,目标目录不匹配媒体库目录
|
||||
if dest_path and dest_path != Path(d.library_path):
|
||||
continue
|
||||
# 目录类型为全部的,符合条件
|
||||
if not media_type or not d.media_type:
|
||||
matched_dirs.append(d)
|
||||
continue
|
||||
# 目录类型相等,目录类别为全部,符合条件
|
||||
if d.media_type == media_type and not d.media_category:
|
||||
matched_dirs.append(d)
|
||||
continue
|
||||
# 目录类型相等,目录类别相等,符合条件
|
||||
if d.media_type == media_type and d.media_category == media.category:
|
||||
matched_dirs.append(d)
|
||||
continue
|
||||
if matched_dirs:
|
||||
if src_path:
|
||||
# 优先源目录同盘
|
||||
for matched_dir in matched_dirs:
|
||||
matched_path = Path(matched_dir.download_path)
|
||||
if self._is_same_source((src_path, storage or "local"), (matched_path, matched_dir.library_storage)):
|
||||
return matched_dir
|
||||
return matched_dirs[0]
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _is_same_source(src: Tuple[Path, str], tar: Tuple[Path, str]) -> bool:
|
||||
"""
|
||||
判断源目录和目标目录是否在同一存储盘
|
||||
|
||||
:param src: 源目录路径和存储类型
|
||||
:param tar: 目标目录路径和存储类型
|
||||
:return: 是否在同一存储盘
|
||||
"""
|
||||
src_path, src_storage = src
|
||||
tar_path, tar_storage = tar
|
||||
if "local" == tar_storage == src_storage:
|
||||
return SystemUtils.is_same_disk(src_path, tar_path)
|
||||
# 网络存储,直接比较类型
|
||||
return src_storage == tar_storage
|
||||
|
||||
@staticmethod
|
||||
def get_media_root_path(
|
||||
rename_format: str,
|
||||
rename_path: Path,
|
||||
media_type: Optional[MediaType] = None,
|
||||
) -> Optional[Path]:
|
||||
"""
|
||||
获取重命名后的媒体文件根路径
|
||||
|
||||
:param rename_format: 重命名格式
|
||||
:param rename_path: 重命名后的路径
|
||||
:param media_type: 媒体类型;音乐需要避开可选碟片目录并返回专辑目录
|
||||
:return: 媒体文件根路径
|
||||
"""
|
||||
if not rename_format:
|
||||
logger.error("重命名格式不能为空")
|
||||
return None
|
||||
if media_type == MediaType.MUSIC:
|
||||
# 音乐模板允许按多碟动态增加 Disc 子目录,不能按静态模板层数反推。
|
||||
# 文件的直接父目录通常就是专辑目录;命中碟片目录时再上移一级。
|
||||
media_root = rename_path.parent
|
||||
if re.fullmatch(
|
||||
r"(?:cd|disc|disk)\s*0*\d+",
|
||||
media_root.name,
|
||||
re.IGNORECASE,
|
||||
):
|
||||
media_root = media_root.parent
|
||||
return media_root
|
||||
# 计算重命名中的文件夹层数
|
||||
rename_list = rename_format.split("/")
|
||||
rename_format_level = len(rename_list) - 1
|
||||
# 反向查找标题参数所在层
|
||||
for level, name in enumerate(reversed(rename_list)):
|
||||
if level == 0:
|
||||
# 跳过文件名的标题参数
|
||||
continue
|
||||
matchs = JINJA2_VAR_PATTERN.findall(name)
|
||||
if not matchs:
|
||||
continue
|
||||
# 处理特例,有的人重命名的第一层是年份、分辨率
|
||||
if (any("title" in m for m in matchs)
|
||||
and not any("season" in m for m in matchs)):
|
||||
# 找出最后一层含有标题且不含季参数的目录作为媒体根目录
|
||||
rename_format_level = level
|
||||
break
|
||||
else:
|
||||
# 假定第一层目录是媒体根目录
|
||||
logger.warn(f"重命名格式 {rename_format} 缺少标题目录")
|
||||
if rename_format_level > len(rename_path.parents):
|
||||
# 通常因为路径以/结尾,被Path规范化删除了
|
||||
logger.error(f"路径 {rename_path} 不匹配重命名格式 {rename_format}")
|
||||
return None
|
||||
if rename_format_level <= 0:
|
||||
# 所有媒体文件都存在一个目录内的特殊需求
|
||||
rename_format_level = 1
|
||||
# 媒体根路径
|
||||
media_root = rename_path.parents[rename_format_level - 1]
|
||||
return media_root
|
||||
|
||||
|
||||
def _split_file_uri(value: str) -> Tuple[str, str]:
|
||||
"""
|
||||
拆分 FileURI 字符串,保留原始路径用于安全校验。
|
||||
"""
|
||||
for storage in StorageSchema:
|
||||
protocol = f"{storage.value}:"
|
||||
if value.startswith(protocol):
|
||||
return storage.value, value[len(protocol):]
|
||||
return "local", value
|
||||
|
||||
|
||||
def _normalize_safe_posix_path(raw_path: str) -> PurePosixPath:
|
||||
"""
|
||||
规范化保存目录路径,并拒绝跨目录或跨平台歧义写法。
|
||||
"""
|
||||
if not raw_path:
|
||||
raise ValueError("保存路径不能为空")
|
||||
if "\\" in raw_path:
|
||||
raise ValueError("保存路径不能包含反斜杠")
|
||||
if raw_path.startswith("//"):
|
||||
raise ValueError("保存路径不能使用 UNC 路径")
|
||||
if WINDOWS_DRIVE_PATTERN.match(raw_path):
|
||||
raise ValueError("保存路径不能使用 Windows 盘符路径")
|
||||
if not raw_path.startswith("/"):
|
||||
raise ValueError("保存路径必须是绝对路径")
|
||||
|
||||
path = PurePosixPath(raw_path)
|
||||
parts = [part for part in path.parts if part != "/"]
|
||||
if ".." in parts:
|
||||
raise ValueError("保存路径不能包含上级目录")
|
||||
if parts and re.fullmatch(r"[A-Za-z]:", parts[0]):
|
||||
raise ValueError("保存路径不能使用 Windows 盘符路径")
|
||||
return path
|
||||
|
||||
|
||||
def _normalize_safe_windows_path(raw_path: str) -> PureWindowsPath:
|
||||
"""
|
||||
规范化已配置的 Windows 盘符路径;UNC 与反斜杠写法不参与下载目录 allowlist。
|
||||
"""
|
||||
if not raw_path:
|
||||
raise ValueError("保存路径不能为空")
|
||||
if "\\" in raw_path:
|
||||
raise ValueError("保存路径不能包含反斜杠")
|
||||
if raw_path.startswith("//"):
|
||||
raise ValueError("保存路径不能使用 UNC 路径")
|
||||
if not WINDOWS_DRIVE_PATTERN.match(raw_path):
|
||||
raise ValueError("保存路径必须是 Windows 绝对路径")
|
||||
|
||||
path = PureWindowsPath(raw_path)
|
||||
if ".." in path.parts:
|
||||
raise ValueError("保存路径不能包含上级目录")
|
||||
return path
|
||||
|
||||
|
||||
def _normalize_download_path(raw_path: str, storage: str) -> Tuple[str, PurePath]:
|
||||
"""
|
||||
按存储类型解析下载路径,本地允许 POSIX 或已配置的 Windows drive,远端保持 FileURI POSIX 语义。
|
||||
"""
|
||||
path_value = str(raw_path or "").strip()
|
||||
if storage == "local" and WINDOWS_DRIVE_PREFIX_PATTERN.match(path_value):
|
||||
return "windows", _normalize_safe_windows_path(path_value)
|
||||
return "posix", _normalize_safe_posix_path(path_value)
|
||||
|
||||
|
||||
def _download_path_uri(storage: str, path: PurePath) -> str:
|
||||
"""
|
||||
生成可传给下载器的 save_path,保持 /download/paths 暴露的本地和远端路径风格。
|
||||
"""
|
||||
path_value = path.as_posix()
|
||||
if storage == "local":
|
||||
return path_value
|
||||
return schemas.FileURI(storage=storage, path=path_value).uri
|
||||
|
||||
|
||||
def _normalize_download_root(dir_info: schemas.TransferDirectoryConf) -> Optional[Tuple[str, str, PurePath]]:
|
||||
"""
|
||||
读取下载目录配置中的根路径;无效配置不参与用户 save_path allowlist。
|
||||
"""
|
||||
if not dir_info.download_path:
|
||||
return None
|
||||
storage = dir_info.storage or "local"
|
||||
try:
|
||||
path_style, root_path = _normalize_download_path(dir_info.download_path, storage)
|
||||
return storage, path_style, root_path
|
||||
except ValueError as err:
|
||||
logger.warn(f"跳过无效下载目录配置:{str(err)}")
|
||||
return None
|
||||
|
||||
|
||||
def validate_download_save_path(save_path: str) -> str:
|
||||
"""
|
||||
校验用户传入的下载保存目录,/download/paths 暴露的下载目录配置是允许写入的公共合同。
|
||||
|
||||
:param save_path: 下载保存目录,支持本地 /path、远端 <storage>:/path 和旧版订阅中的无前缀远程路径
|
||||
:return: 可直接传给下载接口的规范化保存目录
|
||||
"""
|
||||
value = str(save_path or "").strip()
|
||||
has_storage_prefix = any(value.startswith(f"{item.value}:") for item in StorageSchema)
|
||||
storage, raw_path = _split_file_uri(value)
|
||||
target_style, target_path = _normalize_download_path(raw_path, storage)
|
||||
|
||||
download_roots = []
|
||||
for dir_info in DirectoryHelper().get_download_dirs():
|
||||
root = _normalize_download_root(dir_info)
|
||||
if root:
|
||||
download_roots.append(root)
|
||||
|
||||
for root_storage, root_style, root_path in download_roots:
|
||||
if storage != root_storage:
|
||||
continue
|
||||
if target_style != root_style:
|
||||
continue
|
||||
if target_path == root_path or target_path.is_relative_to(root_path):
|
||||
return _download_path_uri(storage, target_path)
|
||||
|
||||
# 旧版订阅界面只持久化 download_path,需要从已配置根目录恢复远程存储类型。
|
||||
if (not has_storage_prefix
|
||||
and storage == StorageSchema.Local.value
|
||||
and target_style == "posix"):
|
||||
for root_storage, root_style, root_path in download_roots:
|
||||
if root_storage == StorageSchema.Local.value or target_style != root_style:
|
||||
continue
|
||||
if target_path == root_path or target_path.is_relative_to(root_path):
|
||||
return _download_path_uri(root_storage, target_path)
|
||||
|
||||
raise ValueError("保存路径不在允许的下载目录范围内")
|
||||
38
app/application/downloader.py
Normal file
38
app/application/downloader.py
Normal file
@@ -0,0 +1,38 @@
|
||||
from typing import Optional
|
||||
|
||||
from app.runtime.extensions.service_registry import ServiceBaseHelper
|
||||
from app.schemas import DownloaderConf, ServiceInfo
|
||||
from app.schemas.types import SystemConfigKey, ModuleType
|
||||
|
||||
|
||||
class DownloaderHelper(ServiceBaseHelper[DownloaderConf]):
|
||||
"""
|
||||
下载器帮助类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""绑定下载器配置和下载器模块类型。"""
|
||||
super().__init__(
|
||||
config_key=SystemConfigKey.Downloaders,
|
||||
conf_type=DownloaderConf,
|
||||
module_type=ModuleType.Downloader
|
||||
)
|
||||
|
||||
def is_downloader(
|
||||
self,
|
||||
service_type: Optional[str] = None,
|
||||
service: Optional[ServiceInfo] = None,
|
||||
name: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
通用的下载器类型判断方法
|
||||
:param service_type: 下载器的类型名称(如 'qbittorrent', 'transmission', 'rtorrent')
|
||||
:param service: 要判断的服务信息
|
||||
:param name: 服务的名称
|
||||
:return: 如果服务类型或实例为指定类型,返回 True;否则返回 False
|
||||
"""
|
||||
# 如果未提供 service 则通过 name 获取服务
|
||||
service = service or self.get_service(name=name)
|
||||
|
||||
# 判断服务类型是否为指定类型
|
||||
return bool(service and service.type == service_type)
|
||||
66
app/application/filter.py
Normal file
66
app/application/filter.py
Normal file
@@ -0,0 +1,66 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.domain.context import MediaInfo
|
||||
from app.schemas import CustomRule, FilterRuleGroup
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class RuleHelper:
|
||||
"""读取用户过滤规则配置,并按媒体上下文选择适用规则组。"""
|
||||
|
||||
@staticmethod
|
||||
def get_rule_groups() -> List[FilterRuleGroup]:
|
||||
"""返回用户配置的全部过滤规则组。"""
|
||||
rule_groups: List[dict] = SystemConfigOper().get(
|
||||
SystemConfigKey.UserFilterRuleGroups
|
||||
)
|
||||
if not rule_groups:
|
||||
return []
|
||||
return [FilterRuleGroup(**group) for group in rule_groups]
|
||||
|
||||
def get_rule_group(self, group_name: str) -> Optional[FilterRuleGroup]:
|
||||
"""按名称返回过滤规则组。"""
|
||||
return next(
|
||||
(group for group in self.get_rule_groups() if group.name == group_name),
|
||||
None,
|
||||
)
|
||||
|
||||
def get_rule_group_by_media(
|
||||
self,
|
||||
media: Optional[MediaInfo] = None,
|
||||
group_names: Optional[list] = None,
|
||||
) -> List[FilterRuleGroup]:
|
||||
"""按媒体类型、分类和候选名称筛选适用规则组。"""
|
||||
rule_groups = self.get_rule_groups()
|
||||
if group_names:
|
||||
rule_groups = [
|
||||
group for group in rule_groups if group.name in group_names
|
||||
]
|
||||
return [
|
||||
group
|
||||
for group in rule_groups
|
||||
if not group.media_type
|
||||
or (
|
||||
media
|
||||
and (
|
||||
(not group.category and group.media_type == media.type.value)
|
||||
or group.category == media.category
|
||||
)
|
||||
)
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def get_custom_rules() -> List[CustomRule]:
|
||||
"""返回用户配置的全部自定义过滤规则。"""
|
||||
rules: List[dict] = SystemConfigOper().get(SystemConfigKey.CustomFilterRules)
|
||||
if not rules:
|
||||
return []
|
||||
return [CustomRule(**rule) for rule in rules]
|
||||
|
||||
def get_custom_rule(self, rule_id: str) -> Optional[CustomRule]:
|
||||
"""按 ID 返回一条自定义过滤规则。"""
|
||||
return next(
|
||||
(rule for rule in self.get_custom_rules() if rule.id == rule_id),
|
||||
None,
|
||||
)
|
||||
1521
app/application/formatting.py
Normal file
1521
app/application/formatting.py
Normal file
File diff suppressed because it is too large
Load Diff
422
app/application/history.py
Normal file
422
app/application/history.py
Normal file
@@ -0,0 +1,422 @@
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from app.runtime.cache import TTLCache
|
||||
from app.runtime.config import settings
|
||||
from app.db.models.transferhistory import TransferHistory
|
||||
from app.db.transferhistory_oper import TransferHistoryOper
|
||||
from app.runtime.log import logger
|
||||
|
||||
# 失败重试次数的合法区间。下界为 1:一次瞬时故障(网络抖动、TMDB 瞬断、移动失败)
|
||||
# 不该让文件永久漏整理,所以不允许关闭重试;上界为 10:永远识别不出的文件重试再多
|
||||
# 也不会成功,只会重复推送失败通知,批量导入场景下会刷屏,所以不允许无限重试
|
||||
MIN_FAILED_RETRIES = 1
|
||||
MAX_FAILED_RETRIES = 10
|
||||
|
||||
# 同一源路径的连续整理失败状态。整理链在写失败历史时累计、整理成功或删除历史时清零,
|
||||
# 查重闸只读不写,避免监控层与整理链对同一个事件重复计数。缓存值会同时保存文件指纹,
|
||||
# 因此同一路径的新版本天然获得独立预算;内存缓存会随进程重启清空,Redis 后端则保留到 TTL 到期。
|
||||
FAILED_RETRY_TTL = 24 * 3600
|
||||
_failed_retry_counts = TTLCache(region="transfer_failed_retry", maxsize=5000, ttl=FAILED_RETRY_TTL)
|
||||
|
||||
|
||||
class HistoryGateAction:
|
||||
"""
|
||||
整理历史查重闸的判定结果。
|
||||
|
||||
监控分发(app/monitor/dispatcher.py)与整理链计划整理段(app/chain/transfer.py)
|
||||
共用本模块,避免两处各写一套去重策略后互相对冲:上游放行的文件被下游按
|
||||
「存在记录即拦」全额收回,等于放行逻辑完全失效。
|
||||
"""
|
||||
# 没有整理记录
|
||||
PASS_NO_RECORD = "pass_no_record"
|
||||
# 上次整理失败且重试次数未用尽,放行重试
|
||||
PASS_FAILED = "pass_failed"
|
||||
# 上次整理失败但源文件已变为新版本,放行并重置该版本的重试预算
|
||||
PASS_FAILED_VERSION_CHANGED = "pass_failed_version_changed"
|
||||
# 已整理成功但源文件已变化,放行交由 overwrite_mode 决断
|
||||
PASS_SIZE_CHANGED = "pass_size_changed"
|
||||
# 上次整理失败且重试次数已用尽,跳过
|
||||
SKIP_RETRY_EXHAUSTED = "skip_retry_exhausted"
|
||||
# 已整理成功且源文件未变化,跳过
|
||||
SKIP = "skip"
|
||||
|
||||
|
||||
def is_skip_action(action: str) -> bool:
|
||||
"""
|
||||
判断查重闸判定是否为跳过整理。
|
||||
:param action: HistoryGateAction 之一
|
||||
:return: True 表示跳过
|
||||
"""
|
||||
return action in (HistoryGateAction.SKIP, HistoryGateAction.SKIP_RETRY_EXHAUSTED)
|
||||
|
||||
|
||||
def max_failed_retries() -> int:
|
||||
"""
|
||||
读取失败重试上限并钳制到合法区间。
|
||||
|
||||
配置为负数、0 或超过上界时都会被钳制并记录 warn:关闭重试会让瞬时故障造成
|
||||
永久漏件,无限重试会让永久失败的文件反复刷通知,两端都不接受。
|
||||
:return: 合法的最大重试次数
|
||||
"""
|
||||
raw = settings.TRANSFER_MAX_FAILED_RETRIES
|
||||
try:
|
||||
value = int(raw)
|
||||
except (TypeError, ValueError):
|
||||
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 配置非法({raw!r}),"
|
||||
f"已回退为 {MIN_FAILED_RETRIES}")
|
||||
return MIN_FAILED_RETRIES
|
||||
if value < MIN_FAILED_RETRIES:
|
||||
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能小于 {MIN_FAILED_RETRIES}"
|
||||
f"(当前 {value}),已按 {MIN_FAILED_RETRIES} 处理")
|
||||
return MIN_FAILED_RETRIES
|
||||
if value > MAX_FAILED_RETRIES:
|
||||
logger.warn(f"TRANSFER_MAX_FAILED_RETRIES 不能大于 {MAX_FAILED_RETRIES}"
|
||||
f"(当前 {value}),已按 {MAX_FAILED_RETRIES} 处理")
|
||||
return MAX_FAILED_RETRIES
|
||||
return value
|
||||
|
||||
|
||||
def failed_retry_key(src_path: Optional[str], storage: Optional[str] = None) -> Optional[str]:
|
||||
"""
|
||||
生成失败重试计数的缓存键。
|
||||
:param src_path: 整理记录使用的源路径
|
||||
:param storage: 源存储
|
||||
:return: 缓存键,源路径为空时返回 None
|
||||
"""
|
||||
if not src_path:
|
||||
return None
|
||||
return f"{storage or 'local'}:{src_path}"
|
||||
|
||||
|
||||
def coerce_modify_time(modify_time: Any) -> Optional[float]:
|
||||
"""
|
||||
统一转换文件修改时间,无法转换时返回 None。
|
||||
:param modify_time: 原始修改时间值
|
||||
:return: 文件修改时间
|
||||
"""
|
||||
if modify_time is None:
|
||||
return None
|
||||
try:
|
||||
return float(modify_time)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def coerce_fileid(fileid: Any) -> Optional[str]:
|
||||
"""
|
||||
统一转换文件唯一标识,空值视为不可比对。
|
||||
:param fileid: 原始文件唯一标识
|
||||
:return: 非空文件唯一标识
|
||||
"""
|
||||
if fileid is None:
|
||||
return None
|
||||
value = str(fileid).strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def file_fingerprint(
|
||||
file_size: Any = None,
|
||||
file_modify_time: Any = None,
|
||||
fileid: Any = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""
|
||||
生成用于区分同一路径文件版本的稳定指纹。
|
||||
|
||||
大小是所有存储器都尽量提供的最小指纹;两端均有数据时,修改时间和文件 ID 还可
|
||||
识别“同大小替换”。只保留可比较字段,避免缺失元数据把同一文件误判成新版本。
|
||||
:param file_size: 文件大小
|
||||
:param file_modify_time: 文件修改时间
|
||||
:param fileid: 存储器文件唯一标识
|
||||
:return: 非空且可比较的指纹字段
|
||||
"""
|
||||
fingerprint = {}
|
||||
size = coerce_size(file_size)
|
||||
if size is not None:
|
||||
fingerprint["size"] = size
|
||||
modify_time = coerce_modify_time(file_modify_time)
|
||||
if modify_time is not None:
|
||||
fingerprint["modify_time"] = modify_time
|
||||
normalized_fileid = coerce_fileid(fileid)
|
||||
if normalized_fileid is not None:
|
||||
fingerprint["fileid"] = normalized_fileid
|
||||
return fingerprint
|
||||
|
||||
|
||||
def _retry_state(value: Any) -> tuple[int, Dict[str, Any]]:
|
||||
"""将新旧缓存值统一转换为失败次数与文件指纹。"""
|
||||
if isinstance(value, dict):
|
||||
raw_count = value.get("count", 0)
|
||||
raw_fingerprint = value.get("fingerprint")
|
||||
else:
|
||||
# 兼容已写入 Redis 或内存的旧整数计数;下次带指纹写入时会自动升级结构。
|
||||
raw_count = value
|
||||
raw_fingerprint = None
|
||||
try:
|
||||
count = max(int(raw_count or 0), 0)
|
||||
except (TypeError, ValueError):
|
||||
count = 0
|
||||
fingerprint = (
|
||||
file_fingerprint(
|
||||
file_size=raw_fingerprint.get("size"),
|
||||
file_modify_time=raw_fingerprint.get("modify_time"),
|
||||
fileid=raw_fingerprint.get("fileid"),
|
||||
)
|
||||
if isinstance(raw_fingerprint, dict)
|
||||
else {}
|
||||
)
|
||||
return count, fingerprint
|
||||
|
||||
|
||||
def _is_file_version_changed(
|
||||
recorded_fingerprint: Dict[str, Any],
|
||||
current_fingerprint: Dict[str, Any],
|
||||
) -> bool:
|
||||
"""判断两个可比文件指纹是否指向不同版本。"""
|
||||
for field in ("fileid", "modify_time", "size"):
|
||||
recorded_value = recorded_fingerprint.get(field)
|
||||
current_value = current_fingerprint.get(field)
|
||||
if (
|
||||
recorded_value is not None
|
||||
and current_value is not None
|
||||
and recorded_value != current_value
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def failed_retry_count(src_path: Optional[str], storage: Optional[str] = None,
|
||||
file_size: Any = None, file_modify_time: Any = None,
|
||||
fileid: Any = None) -> int:
|
||||
"""
|
||||
读取同一源路径已累计的连续整理失败次数。
|
||||
:param src_path: 整理记录使用的源路径
|
||||
:param storage: 源存储
|
||||
:param file_size: 当前文件大小
|
||||
:param file_modify_time: 当前文件修改时间
|
||||
:param fileid: 当前文件唯一标识
|
||||
:return: 当前文件版本已失败次数,无记录时为 0
|
||||
"""
|
||||
key = failed_retry_key(src_path, storage)
|
||||
if not key:
|
||||
return 0
|
||||
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
|
||||
current_fingerprint = file_fingerprint(
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if (
|
||||
recorded_fingerprint
|
||||
and current_fingerprint
|
||||
and _is_file_version_changed(recorded_fingerprint, current_fingerprint)
|
||||
):
|
||||
return 0
|
||||
return count
|
||||
|
||||
|
||||
def record_transfer_failure(src_path: Optional[str], storage: Optional[str] = None,
|
||||
file_size: Any = None, file_modify_time: Any = None,
|
||||
fileid: Any = None) -> int:
|
||||
"""
|
||||
累计一次整理失败。
|
||||
:param src_path: 整理记录使用的源路径
|
||||
:param storage: 源存储
|
||||
:param file_size: 当前文件大小
|
||||
:param file_modify_time: 当前文件修改时间
|
||||
:param fileid: 当前文件唯一标识
|
||||
:return: 当前文件版本累计后的失败次数
|
||||
"""
|
||||
key = failed_retry_key(src_path, storage)
|
||||
if not key:
|
||||
return 0
|
||||
count, recorded_fingerprint = _retry_state(_failed_retry_counts.get(key))
|
||||
current_fingerprint = file_fingerprint(
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if current_fingerprint and (
|
||||
not recorded_fingerprint
|
||||
or _is_file_version_changed(recorded_fingerprint, current_fingerprint)
|
||||
):
|
||||
count = 0
|
||||
count += 1
|
||||
if current_fingerprint:
|
||||
_failed_retry_counts[key] = {
|
||||
"count": count,
|
||||
"fingerprint": current_fingerprint,
|
||||
}
|
||||
elif recorded_fingerprint:
|
||||
_failed_retry_counts[key] = {
|
||||
"count": count,
|
||||
"fingerprint": recorded_fingerprint,
|
||||
}
|
||||
else:
|
||||
_failed_retry_counts[key] = count
|
||||
return count
|
||||
|
||||
|
||||
def clear_transfer_failures(src_path: Optional[str], storage: Optional[str] = None) -> None:
|
||||
"""
|
||||
清空同一源路径的失败计数。整理成功、或用户删除整理记录(显式要求重来)时调用。
|
||||
:param src_path: 整理记录使用的源路径
|
||||
:param storage: 源存储
|
||||
"""
|
||||
key = failed_retry_key(src_path, storage)
|
||||
if key:
|
||||
# 缺省值必须是 0 而不是 None:CacheBackend.pop 把「default 为 None」当成「未提供
|
||||
# default」,键不存在时会抛 KeyError。整理成功路径上绝大多数文件从未失败过,
|
||||
# 传 None 会让每一次首次成功整理都炸掉成功回调
|
||||
_failed_retry_counts.pop(key, 0)
|
||||
|
||||
|
||||
def coerce_size(size: Any) -> Optional[int]:
|
||||
"""
|
||||
统一转换文件大小,无法转换时返回 None(视为不可比对)。
|
||||
:param size: 原始大小值
|
||||
:return: 文件大小
|
||||
"""
|
||||
if size is None:
|
||||
return None
|
||||
try:
|
||||
return int(size)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def history_src_size(history: TransferHistory) -> Optional[int]:
|
||||
"""
|
||||
读取整理记录中的源文件大小。
|
||||
src_fileitem 是 JSON 列,历史数据可能为空、缺 size 键甚至不是字典,
|
||||
取不到时统一返回 None 交由调用方保守处理。
|
||||
:param history: 整理记录
|
||||
:return: 源文件大小,取不到时为 None
|
||||
"""
|
||||
return history_src_fingerprint(history).get("size")
|
||||
|
||||
|
||||
def history_src_fingerprint(history: TransferHistory) -> Dict[str, Any]:
|
||||
"""
|
||||
读取整理记录中的源文件版本指纹。
|
||||
:param history: 整理记录
|
||||
:return: 源文件的可比较指纹字段
|
||||
"""
|
||||
src_fileitem = getattr(history, "src_fileitem", None)
|
||||
if not isinstance(src_fileitem, dict):
|
||||
return {}
|
||||
return file_fingerprint(
|
||||
file_size=src_fileitem.get("size"),
|
||||
file_modify_time=src_fileitem.get("modify_time"),
|
||||
fileid=src_fileitem.get("fileid"),
|
||||
)
|
||||
|
||||
|
||||
def resolve_history(src_path: str, storage: Optional[str] = None,
|
||||
transfer_history_oper: Optional[TransferHistoryOper] = None
|
||||
) -> Optional[TransferHistory]:
|
||||
"""
|
||||
查询源路径对应的整理记录。
|
||||
|
||||
新表通过 (src, src_storage) 唯一索引保证单条记录;仍保留对成功记录的二次确认,
|
||||
兼容升级前可能残留的重复数据,避免把已整理成功的文件重复整理。查询异常不在
|
||||
此处吞掉,由调用方按各自的重试策略处理。
|
||||
:param src_path: 整理记录使用的源路径
|
||||
:param storage: 存储
|
||||
:param transfer_history_oper: 复用的历史操作对象,未传时新建
|
||||
:return: 命中的整理记录,未命中时为 None
|
||||
"""
|
||||
oper = transfer_history_oper or TransferHistoryOper()
|
||||
history = oper.get_by_src(src_path, storage=storage)
|
||||
if history is not None and not history.status:
|
||||
history = oper.get_success_by_src(src_path, storage=storage) or history
|
||||
return history
|
||||
|
||||
|
||||
def evaluate_history_gate(history: Optional[TransferHistory],
|
||||
file_size: Optional[float] = None,
|
||||
file_modify_time: Optional[float] = None,
|
||||
fileid: Optional[str] = None,
|
||||
retry_count: Optional[int] = None) -> str:
|
||||
"""
|
||||
依据整理历史判断本次是否跳过整理。
|
||||
|
||||
成功记录不能简单地「存在即跳过」:同路径重新上传的新版本会因此没有机会走到
|
||||
整理链的 overwrite_mode 判定,升级永远无法入库,故任一可比文件指纹变化时一律放行。
|
||||
失败记录按文件版本使用有界重试:新版本先放行并在下一次失败时从 1 重新计数,
|
||||
同一版本未达上限时继续重试,让瞬时故障(网络/识别/移动)自愈;达到上限后跳过,
|
||||
避免永久失败的文件反复刷失败通知。
|
||||
:param history: 整理记录,未命中时为 None
|
||||
:param file_size: 当前文件大小,蓝光目录等场景可能为 None
|
||||
:param file_modify_time: 当前文件修改时间
|
||||
:param fileid: 当前文件唯一标识
|
||||
:param retry_count: 已累计的失败次数,None 表示按记录源路径实时查询
|
||||
:return: HistoryGateAction 之一
|
||||
"""
|
||||
if history is None:
|
||||
return HistoryGateAction.PASS_NO_RECORD
|
||||
recorded_fingerprint = history_src_fingerprint(history)
|
||||
current_fingerprint = file_fingerprint(
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if not history.status:
|
||||
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||
return HistoryGateAction.PASS_FAILED_VERSION_CHANGED
|
||||
if retry_count is None:
|
||||
retry_count = failed_retry_count(
|
||||
getattr(history, "src", None),
|
||||
getattr(history, "src_storage", None),
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if retry_count >= max_failed_retries():
|
||||
return HistoryGateAction.SKIP_RETRY_EXHAUSTED
|
||||
# 监控事件是稀疏驱动的(落地事件/延迟重扫/补偿扫描),入口还有 TTL 去重兜底,
|
||||
# 配合失败次数上限,重试频率与总量都可控
|
||||
return HistoryGateAction.PASS_FAILED
|
||||
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||
# 同路径换成了另一个版本(如升级为更高码率),是否覆盖交给整理链的
|
||||
# overwrite_mode 决断,查重闸不做替代判断
|
||||
return HistoryGateAction.PASS_SIZE_CHANGED
|
||||
# 无法比对大小(蓝光目录、历史记录缺 size)时保守跳过,避免重复整理
|
||||
return HistoryGateAction.SKIP
|
||||
|
||||
|
||||
def describe_history_gate(history: Optional[TransferHistory],
|
||||
file_size: Optional[float] = None,
|
||||
file_modify_time: Optional[float] = None,
|
||||
fileid: Optional[str] = None) -> str:
|
||||
"""
|
||||
生成查重闸判定的可读说明,供日志定位「到底是哪条记录在拦」。
|
||||
:param history: 整理记录
|
||||
:param file_size: 当前文件大小
|
||||
:param file_modify_time: 当前文件修改时间
|
||||
:param fileid: 当前文件唯一标识
|
||||
:return: 说明文本
|
||||
"""
|
||||
if history is None:
|
||||
return "无整理记录"
|
||||
recorded_fingerprint = history_src_fingerprint(history)
|
||||
current_fingerprint = file_fingerprint(
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if not history.status:
|
||||
count = failed_retry_count(
|
||||
getattr(history, "src", None),
|
||||
getattr(history, "src_storage", None),
|
||||
file_size=file_size,
|
||||
file_modify_time=file_modify_time,
|
||||
fileid=fileid,
|
||||
)
|
||||
if _is_file_version_changed(recorded_fingerprint, current_fingerprint):
|
||||
return f"失败记录 #{history.id},文件版本已变化,重试预算将重置"
|
||||
return f"失败记录 #{history.id},已重试 {count}/{max_failed_retries()} 次"
|
||||
recorded_size = recorded_fingerprint.get("size")
|
||||
current_size = current_fingerprint.get("size")
|
||||
if recorded_size is None and current_size is None:
|
||||
return f"成功记录 #{history.id},大小不可比对"
|
||||
return f"成功记录 #{history.id},大小 {recorded_size} -> {current_size}"
|
||||
389
app/application/image.py
Normal file
389
app/application/image.py
Normal file
@@ -0,0 +1,389 @@
|
||||
import io
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional, List
|
||||
|
||||
from PIL import Image
|
||||
|
||||
from app.runtime.cache import cached, FileCache, AsyncFileCache
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.log import logger
|
||||
from app.adapters.network.http import RequestUtils, AsyncRequestUtils
|
||||
from app.adapters.network.ip import IpUtils
|
||||
from app.application.security.url import SecurityUtils
|
||||
from app.foundation.singleton import Singleton
|
||||
|
||||
|
||||
WallpaperProvider = Callable[[], Optional[str]]
|
||||
WallpaperListProvider = Callable[[int], List[str]]
|
||||
|
||||
|
||||
def _empty_wallpaper_provider() -> Optional[str]:
|
||||
"""在启动组合根尚未装配壁纸来源时返回空结果。"""
|
||||
return None
|
||||
|
||||
|
||||
def _empty_wallpaper_list_provider(_count: int) -> List[str]:
|
||||
"""在启动组合根尚未装配壁纸来源时返回空列表。"""
|
||||
return []
|
||||
|
||||
|
||||
_tmdb_wallpaper_provider: WallpaperProvider = _empty_wallpaper_provider
|
||||
_tmdb_wallpaper_list_provider: WallpaperListProvider = (
|
||||
_empty_wallpaper_list_provider
|
||||
)
|
||||
_mediaserver_wallpaper_provider: WallpaperProvider = _empty_wallpaper_provider
|
||||
_mediaserver_wallpaper_list_provider: WallpaperListProvider = (
|
||||
_empty_wallpaper_list_provider
|
||||
)
|
||||
|
||||
|
||||
def configure_wallpaper_providers(
|
||||
*,
|
||||
tmdb_wallpaper: WallpaperProvider,
|
||||
tmdb_wallpapers: WallpaperListProvider,
|
||||
mediaserver_wallpaper: WallpaperProvider,
|
||||
mediaserver_wallpapers: WallpaperListProvider,
|
||||
) -> None:
|
||||
"""由启动组合根注入需要业务 Chain 才能提供的壁纸来源。"""
|
||||
global _tmdb_wallpaper_provider
|
||||
global _tmdb_wallpaper_list_provider
|
||||
global _mediaserver_wallpaper_provider
|
||||
global _mediaserver_wallpaper_list_provider
|
||||
_tmdb_wallpaper_provider = tmdb_wallpaper
|
||||
_tmdb_wallpaper_list_provider = tmdb_wallpapers
|
||||
_mediaserver_wallpaper_provider = mediaserver_wallpaper
|
||||
_mediaserver_wallpaper_list_provider = mediaserver_wallpapers
|
||||
|
||||
|
||||
class WallpaperHelper(metaclass=Singleton):
|
||||
"""
|
||||
壁纸帮助类
|
||||
"""
|
||||
|
||||
def get_wallpaper(self) -> Optional[str]:
|
||||
"""
|
||||
获取登录页面壁纸
|
||||
"""
|
||||
if settings.WALLPAPER == "bing":
|
||||
return self.get_bing_wallpaper()
|
||||
elif settings.WALLPAPER == "mediaserver":
|
||||
return self.get_mediaserver_wallpaper()
|
||||
elif settings.WALLPAPER == "customize":
|
||||
return self.get_customize_wallpaper()
|
||||
elif settings.WALLPAPER == "tmdb":
|
||||
return self.get_tmdb_wallpaper()
|
||||
return ''
|
||||
|
||||
def get_wallpapers(self, num: int = 10) -> List[str]:
|
||||
"""
|
||||
获取登录页面壁纸列表
|
||||
"""
|
||||
if settings.WALLPAPER == "bing":
|
||||
return self.get_bing_wallpapers(num)
|
||||
elif settings.WALLPAPER == "mediaserver":
|
||||
return self.get_mediaserver_wallpapers(num)
|
||||
elif settings.WALLPAPER == "customize":
|
||||
return self.get_customize_wallpapers()
|
||||
elif settings.WALLPAPER == "tmdb":
|
||||
return self.get_tmdb_wallpapers(num)
|
||||
return []
|
||||
|
||||
@cached(maxsize=1, ttl=3600)
|
||||
def get_tmdb_wallpaper(self) -> Optional[str]:
|
||||
"""
|
||||
获取TMDB每日壁纸
|
||||
"""
|
||||
return _tmdb_wallpaper_provider()
|
||||
|
||||
@cached(maxsize=1, ttl=3600, skip_empty=True)
|
||||
def get_tmdb_wallpapers(self, num: int = 10) -> List[str]:
|
||||
"""
|
||||
获取7天的TMDB每日壁纸
|
||||
"""
|
||||
return _tmdb_wallpaper_list_provider(num)
|
||||
|
||||
@cached(maxsize=1, ttl=3600)
|
||||
def get_bing_wallpaper(self) -> Optional[str]:
|
||||
"""
|
||||
获取Bing每日壁纸
|
||||
"""
|
||||
url = "https://cn.bing.com/HPImageArchive.aspx?format=js&idx=0&n=1"
|
||||
resp = RequestUtils(timeout=5).get_res(url)
|
||||
if resp and resp.status_code == 200:
|
||||
try:
|
||||
result = resp.json()
|
||||
if isinstance(result, dict):
|
||||
for image in result.get('images') or []:
|
||||
return f"https://cn.bing.com{image.get('url')}" if 'url' in image else ''
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return None
|
||||
|
||||
@cached(maxsize=1, ttl=3600, skip_empty=True)
|
||||
def get_bing_wallpapers(self, num: int = 7) -> List[str]:
|
||||
"""
|
||||
获取7天的Bing每日壁纸
|
||||
"""
|
||||
url = f"https://cn.bing.com/HPImageArchive.aspx?format=js&idx=0&n={num}"
|
||||
resp = RequestUtils(timeout=5).get_res(url)
|
||||
if resp and resp.status_code == 200:
|
||||
try:
|
||||
result = resp.json()
|
||||
if isinstance(result, dict):
|
||||
return [f"https://cn.bing.com{image.get('url')}" for image in result.get('images') or []]
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return []
|
||||
|
||||
@cached(maxsize=1, ttl=3600)
|
||||
def get_mediaserver_wallpaper(self) -> Optional[str]:
|
||||
"""
|
||||
获取媒体服务器壁纸
|
||||
"""
|
||||
return _mediaserver_wallpaper_provider()
|
||||
|
||||
@cached(maxsize=1, ttl=3600, skip_empty=True)
|
||||
def get_mediaserver_wallpapers(self, num: int = 10) -> List[str]:
|
||||
"""
|
||||
获取媒体服务器壁纸列表
|
||||
"""
|
||||
return _mediaserver_wallpaper_list_provider(num)
|
||||
|
||||
@cached(maxsize=1, ttl=3600)
|
||||
def get_customize_wallpaper(self) -> Optional[str]:
|
||||
"""
|
||||
获取自定义壁纸api壁纸
|
||||
"""
|
||||
wallpaper_list = self.get_customize_wallpapers()
|
||||
if wallpaper_list:
|
||||
return wallpaper_list[0]
|
||||
return None
|
||||
|
||||
@cached(maxsize=1, ttl=3600, skip_empty=True)
|
||||
def get_customize_wallpapers(self) -> List[str]:
|
||||
"""
|
||||
获取自定义壁纸api壁纸
|
||||
"""
|
||||
|
||||
def find_files_with_suffixes(obj, suffixes: List[str]) -> List[str]:
|
||||
"""
|
||||
递归查找对象中所有包含特定后缀的文件,返回匹配的字符串列表
|
||||
支持输入:字典、列表、字符串
|
||||
"""
|
||||
_result = []
|
||||
|
||||
# 处理字符串
|
||||
if isinstance(obj, str):
|
||||
if obj.endswith(tuple(suffixes)):
|
||||
_result.append(obj)
|
||||
|
||||
# 处理字典
|
||||
elif isinstance(obj, dict):
|
||||
for value in obj.values():
|
||||
_result.extend(find_files_with_suffixes(value, suffixes))
|
||||
|
||||
# 处理列表
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
_result.extend(find_files_with_suffixes(item, suffixes))
|
||||
|
||||
return _result
|
||||
|
||||
# 判断是否存在自定义壁纸api
|
||||
if settings.CUSTOMIZE_WALLPAPER_API_URL:
|
||||
wallpaper_list = []
|
||||
resp = RequestUtils(timeout=15).get_res(settings.CUSTOMIZE_WALLPAPER_API_URL)
|
||||
if resp and resp.status_code == 200:
|
||||
# 如果返回的是图片格式
|
||||
content_type = resp.headers.get('Content-Type')
|
||||
if content_type and content_type.lower().startswith('image/'):
|
||||
wallpaper_list.append(settings.CUSTOMIZE_WALLPAPER_API_URL)
|
||||
else:
|
||||
try:
|
||||
result = resp.json()
|
||||
if isinstance(result, list) or isinstance(result, dict) or isinstance(result, str):
|
||||
wallpaper_list = find_files_with_suffixes(result, settings.SECURITY_IMAGE_SUFFIXES)
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return wallpaper_list
|
||||
else:
|
||||
return []
|
||||
|
||||
|
||||
class ImageHelper(metaclass=Singleton):
|
||||
"""统一管理同步和异步图片缓存。"""
|
||||
|
||||
def __init__(self):
|
||||
"""按全局图片缓存天数初始化文件缓存。"""
|
||||
_base_path = settings.CACHE_PATH
|
||||
_ttl = settings.GLOBAL_IMAGE_CACHE_DAYS * 24 * 3600
|
||||
self.file_cache = FileCache(base=_base_path, ttl=_ttl)
|
||||
self.async_file_cache = AsyncFileCache(base=_base_path, ttl=_ttl)
|
||||
|
||||
@staticmethod
|
||||
def _prepare_cache_path(url: str) -> str:
|
||||
"""缓存路径"""
|
||||
sanitized_path = SecurityUtils.sanitize_url_path(url)
|
||||
cache_path = Path(sanitized_path)
|
||||
if not cache_path.suffix:
|
||||
cache_path = cache_path.with_suffix(".jpg")
|
||||
return cache_path.as_posix()
|
||||
|
||||
@staticmethod
|
||||
def get_image_mime_type(content: bytes, verify: bool = True) -> Optional[str]:
|
||||
"""
|
||||
根据图片内容返回 Pillow 识别的图片 MIME 类型。
|
||||
|
||||
外部响应在写入缓存前需要完整校验;已校验的缓存只需读取格式头。
|
||||
非图片或可脚本化的 MIME 类型不作为图片代理响应。
|
||||
"""
|
||||
if not content:
|
||||
return None
|
||||
try:
|
||||
with Image.open(io.BytesIO(content)) as image:
|
||||
image_format = (image.format or "").upper()
|
||||
if verify:
|
||||
image.verify()
|
||||
mime_type = Image.MIME.get(image_format)
|
||||
if (
|
||||
not mime_type
|
||||
or not mime_type.startswith("image/")
|
||||
or mime_type == "image/svg+xml"
|
||||
):
|
||||
return None
|
||||
return mime_type
|
||||
except Exception as err:
|
||||
logger.warning(f"Invalid image format: {err}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_request_params(url: str, proxy: Optional[bool], cookies: Optional[str | dict]) -> dict:
|
||||
"""获取参数"""
|
||||
referer = "https://movie.douban.com/" if "doubanio.com" in url else None
|
||||
if proxy is None:
|
||||
proxies = settings.PROXY if not (referer or IpUtils.is_internal(url)) else None
|
||||
else:
|
||||
proxies = settings.PROXY if proxy else None
|
||||
return {
|
||||
"ua": settings.NORMAL_USER_AGENT,
|
||||
"proxies": proxies,
|
||||
"referer": referer,
|
||||
"cookies": cookies,
|
||||
"accept_type": "image/avif,image/webp,image/apng,*/*",
|
||||
}
|
||||
|
||||
def fetch_image(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None) -> Optional[bytes]:
|
||||
"""
|
||||
获取图片(同步版本)
|
||||
"""
|
||||
result = self.fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
def fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
同步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
cache_path = self._prepare_cache_path(url)
|
||||
|
||||
# 检查缓存
|
||||
if use_cache:
|
||||
content = self.file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
response = RequestUtils(**params).get_res(url=url)
|
||||
if response is None or response.status_code != 200:
|
||||
logger.warn(f"Failed to fetch image from URL: {url}")
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
self.file_cache.set(cache_path, content, region="images")
|
||||
return content, mime_type
|
||||
|
||||
async def async_fetch_image(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None) -> Optional[bytes]:
|
||||
"""
|
||||
获取图片(异步版本)
|
||||
"""
|
||||
result = await self.async_fetch_image_with_mime_type(
|
||||
url=url,
|
||||
proxy=proxy,
|
||||
use_cache=use_cache,
|
||||
cookies=cookies,
|
||||
)
|
||||
return result[0] if result else None
|
||||
|
||||
async def async_fetch_image_with_mime_type(
|
||||
self,
|
||||
url: str,
|
||||
proxy: Optional[bool] = None,
|
||||
use_cache: bool = True,
|
||||
cookies: Optional[str | dict] = None,
|
||||
) -> Optional[tuple[bytes, str]]:
|
||||
"""
|
||||
异步获取图片及其内容识别 MIME 类型。
|
||||
|
||||
网络响应在写入缓存前完整验证一次;缓存命中仅重新识别格式头。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
|
||||
cache_path = self._prepare_cache_path(url)
|
||||
|
||||
# 检查缓存
|
||||
if use_cache:
|
||||
content = await self.async_file_cache.get(cache_path, region="images")
|
||||
if content:
|
||||
mime_type = self.get_image_mime_type(content, verify=False)
|
||||
if mime_type:
|
||||
return content, mime_type
|
||||
|
||||
# 请求远程图片
|
||||
params = self._get_request_params(url, proxy, cookies)
|
||||
response = await AsyncRequestUtils(**params).get_res(url=url)
|
||||
if response is None or response.status_code != 200:
|
||||
logger.warn(f"Failed to fetch image from URL: {url}")
|
||||
return None
|
||||
|
||||
content = response.content
|
||||
mime_type = self.get_image_mime_type(content)
|
||||
if not mime_type:
|
||||
return None
|
||||
|
||||
# 保存缓存
|
||||
await self.async_file_cache.set(cache_path, content, region="images")
|
||||
return content, mime_type
|
||||
268
app/application/mediaserver.py
Normal file
268
app/application/mediaserver.py
Normal file
@@ -0,0 +1,268 @@
|
||||
import re
|
||||
from collections.abc import Iterable, Mapping
|
||||
from typing import Any, Optional
|
||||
|
||||
from app import schemas
|
||||
from app.domain.context import MusicInfo
|
||||
from app.domain.media import normalize_media_source, resolve_media_identity
|
||||
from app.runtime.extensions.service_registry import ServiceBaseHelper
|
||||
from app.schemas import MediaServerConf, ServiceInfo
|
||||
from app.schemas.types import (
|
||||
MUSIC_ENTITY_ALBUM,
|
||||
MediaSource,
|
||||
ModuleType,
|
||||
SystemConfigKey,
|
||||
)
|
||||
|
||||
|
||||
class MediaServerIdentityHelper:
|
||||
"""将媒体服务器专有 ProviderIds 适配为统一媒体身份。"""
|
||||
|
||||
_provider_keys = (
|
||||
(MediaSource.TMDB, ("Tmdb", "TMDB", "tmdb", "tmdb_id")),
|
||||
(MediaSource.Douban, ("Douban", "douban", "douban_id")),
|
||||
(MediaSource.Bangumi, ("Bangumi", "bangumi", "bangumi_id")),
|
||||
(MediaSource.AniList, ("AniList", "Anilist", "anilist", "anilist_id")),
|
||||
(MediaSource.IMDb, ("Imdb", "IMDb", "imdb", "imdb_id")),
|
||||
(MediaSource.TVDB, ("Tvdb", "TVDB", "tvdb", "tvdb_id")),
|
||||
(MediaSource.MusicBrainz, ("MusicBrainz", "musicbrainz", "musicbrainz_id")),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def from_provider_ids(
|
||||
cls,
|
||||
provider_ids: Optional[Mapping[str, Any]],
|
||||
) -> tuple[Optional[MediaSource], Optional[str]]:
|
||||
"""按固定优先级从外部 ProviderIds 选择一个规范媒体身份。"""
|
||||
if not isinstance(provider_ids, Mapping):
|
||||
return None, None
|
||||
for media_source, keys in cls._provider_keys:
|
||||
for key in keys:
|
||||
value = provider_ids.get(key)
|
||||
if value is not None and str(value).strip():
|
||||
return media_source, str(value).strip()
|
||||
return None, None
|
||||
|
||||
@staticmethod
|
||||
def are_compatible(
|
||||
left_source: Optional[MediaSource | str],
|
||||
left_id: Optional[str],
|
||||
right_source: Optional[MediaSource | str],
|
||||
right_id: Optional[str],
|
||||
) -> bool:
|
||||
"""判断两组身份是否没有可证实的同来源 ID 冲突。"""
|
||||
left_source, left_id = resolve_media_identity(
|
||||
media_source=left_source,
|
||||
media_id=left_id,
|
||||
)
|
||||
right_source, right_id = resolve_media_identity(
|
||||
media_source=right_source,
|
||||
media_id=right_id,
|
||||
)
|
||||
if not left_source or not right_source:
|
||||
return True
|
||||
if normalize_media_source(left_source) != normalize_media_source(right_source):
|
||||
return True
|
||||
return left_id == right_id
|
||||
|
||||
@classmethod
|
||||
def is_compatible(
|
||||
cls,
|
||||
item: schemas.MediaServerItem,
|
||||
media_source: Optional[MediaSource | str],
|
||||
media_id: Optional[str],
|
||||
) -> bool:
|
||||
"""判断目标与媒体库条目是否无明确身份冲突。"""
|
||||
item_source, item_id = resolve_media_identity(media=item)
|
||||
return cls.are_compatible(item_source, item_id, media_source, media_id)
|
||||
|
||||
|
||||
class MusicMediaServerHelper:
|
||||
"""统一音乐媒体库条目的字段转换、精确匹配和整专完整性判断。"""
|
||||
|
||||
_name_pattern = re.compile(r"[\W_]+", re.UNICODE)
|
||||
|
||||
@classmethod
|
||||
def normalize_name(cls, value: Optional[str]) -> str:
|
||||
"""忽略大小写、空白和标点,生成用于音乐名称精确比较的稳定文本。"""
|
||||
return cls._name_pattern.sub("", str(value or "").casefold())
|
||||
|
||||
@classmethod
|
||||
def same_name(cls, left: Optional[str], right: Optional[str]) -> bool:
|
||||
"""判断两个非空音乐名称在规范化后是否完全一致。"""
|
||||
normalized_left = cls.normalize_name(left)
|
||||
normalized_right = cls.normalize_name(right)
|
||||
return bool(normalized_left) and normalized_left == normalized_right
|
||||
|
||||
@staticmethod
|
||||
def _first_value(data: Mapping[str, Any], *keys: str) -> Any:
|
||||
"""按候选键顺序返回第一个非空字段,兼容不同媒体服务器命名。"""
|
||||
for key in keys:
|
||||
value = data.get(key)
|
||||
if value not in (None, "", []):
|
||||
return value
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _extract_names(cls, value: Any) -> list[str]:
|
||||
"""从字符串、对象列表或名称列表中提取非空名称。"""
|
||||
if isinstance(value, str):
|
||||
return [value] if value.strip() else []
|
||||
if isinstance(value, Mapping):
|
||||
name = cls._first_value(value, "Name", "name", "Title", "title")
|
||||
return [str(name)] if name and str(name).strip() else []
|
||||
if not isinstance(value, Iterable) or isinstance(value, bytes):
|
||||
return []
|
||||
names: list[str] = []
|
||||
for item in value:
|
||||
if isinstance(item, Mapping):
|
||||
name = cls._first_value(item, "Name", "name", "Title", "title")
|
||||
else:
|
||||
name = item
|
||||
if name and str(name).strip():
|
||||
names.append(str(name))
|
||||
return names
|
||||
|
||||
@classmethod
|
||||
def build_note(cls, item: Mapping[str, Any]) -> dict[str, Any]:
|
||||
"""把 Emby 系和 NAS 搜索结果中的音乐字段转换为统一备注结构。"""
|
||||
artists = cls._extract_names(
|
||||
cls._first_value(item, "Artists", "artists", "ArtistItems", "artist_items")
|
||||
)
|
||||
album_artists = cls._extract_names(
|
||||
cls._first_value(item, "AlbumArtists", "album_artists")
|
||||
)
|
||||
artist = cls._first_value(
|
||||
item,
|
||||
"AlbumArtist",
|
||||
"album_artist",
|
||||
"Artist",
|
||||
"artist",
|
||||
"artist_name",
|
||||
"singer",
|
||||
)
|
||||
explicit_artists = cls._extract_names(artist)
|
||||
if explicit_artists:
|
||||
artist = explicit_artists[0]
|
||||
if not artist:
|
||||
artist = next(iter(album_artists or artists), None)
|
||||
|
||||
item_type = cls.normalize_name(
|
||||
cls._first_value(item, "Type", "type", "item_type")
|
||||
)
|
||||
album = cls._first_value(item, "Album", "album", "album_name")
|
||||
if not album and item_type in {"musicalbum", "album"}:
|
||||
album = cls._first_value(item, "Name", "name", "Title", "title")
|
||||
|
||||
song_count = cls._first_value(
|
||||
item,
|
||||
"ChildCount",
|
||||
"child_count",
|
||||
"SongCount",
|
||||
"songCount",
|
||||
"song_count",
|
||||
"TrackCount",
|
||||
"trackCount",
|
||||
"track_count",
|
||||
"LeafCount",
|
||||
"leafCount",
|
||||
)
|
||||
return {
|
||||
"artist": str(artist) if artist is not None else None,
|
||||
"artists": artists or album_artists,
|
||||
"album": str(album) if album is not None else None,
|
||||
"song_count": song_count,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def search_params(mediainfo: MusicInfo) -> dict[str, Optional[str]]:
|
||||
"""按单曲或专辑实体构造媒体服务器音乐搜索参数。"""
|
||||
is_album = getattr(mediainfo, "music_type", None) == MUSIC_ENTITY_ALBUM
|
||||
artists = getattr(mediainfo, "artists", None) or []
|
||||
artist = (
|
||||
getattr(mediainfo, "album_artist", None)
|
||||
or next(iter(artists), None)
|
||||
or getattr(mediainfo, "artist", None)
|
||||
)
|
||||
title = getattr(mediainfo, "title", None)
|
||||
album = getattr(mediainfo, "album", None) or title
|
||||
return {
|
||||
"title": None if is_album else title,
|
||||
"artist": artist,
|
||||
"album": album if is_album else None,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def item_matches(
|
||||
cls,
|
||||
mediainfo: MusicInfo,
|
||||
item: schemas.MediaServerItem,
|
||||
) -> bool:
|
||||
"""校验媒体库条目是否精确对应单曲,或完整覆盖目标专辑。"""
|
||||
note = item.note if isinstance(item.note, Mapping) else {}
|
||||
is_album = getattr(mediainfo, "music_type", None) == MUSIC_ENTITY_ALBUM
|
||||
target_title = getattr(mediainfo, "title", None)
|
||||
actual_title = item.title
|
||||
if is_album:
|
||||
target_title = getattr(mediainfo, "album", None) or target_title
|
||||
actual_title = note.get("album") or actual_title
|
||||
if not cls.same_name(actual_title, target_title):
|
||||
return False
|
||||
|
||||
target_artists = [
|
||||
getattr(mediainfo, "artist", None),
|
||||
getattr(mediainfo, "album_artist", None),
|
||||
*(getattr(mediainfo, "artists", None) or []),
|
||||
]
|
||||
target_artists = [artist for artist in target_artists if artist]
|
||||
actual_artists = [note.get("artist"), *cls._extract_names(note.get("artists"))]
|
||||
actual_artists = [artist for artist in actual_artists if artist]
|
||||
if target_artists and not any(
|
||||
cls.same_name(actual, target)
|
||||
for actual in actual_artists
|
||||
for target in target_artists
|
||||
):
|
||||
return False
|
||||
|
||||
if not is_album:
|
||||
return True
|
||||
try:
|
||||
expected_tracks = int(getattr(mediainfo, "total_tracks", None) or 0)
|
||||
actual_tracks = int(note.get("song_count") or 0)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return expected_tracks > 0 and actual_tracks >= expected_tracks
|
||||
|
||||
@classmethod
|
||||
def find_match(
|
||||
cls,
|
||||
mediainfo: MusicInfo,
|
||||
items: Optional[Iterable[schemas.MediaServerItem]],
|
||||
) -> Optional[schemas.MediaServerItem]:
|
||||
"""返回首个满足单曲精确匹配或整专完整性要求的媒体库条目。"""
|
||||
return next(
|
||||
(item for item in items or [] if item and cls.item_matches(mediainfo, item)),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
class MediaServerHelper(ServiceBaseHelper[MediaServerConf]):
|
||||
"""管理媒体服务器配置,并按类型发现已启用的服务实例。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""绑定媒体服务器配置键、配置模型和模块类型。"""
|
||||
super().__init__(
|
||||
config_key=SystemConfigKey.MediaServers,
|
||||
conf_type=MediaServerConf,
|
||||
module_type=ModuleType.MediaServer,
|
||||
)
|
||||
|
||||
def is_media_server(
|
||||
self,
|
||||
service_type: Optional[str] = None,
|
||||
service: Optional[ServiceInfo] = None,
|
||||
name: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""判断给定服务或服务名称是否属于指定媒体服务器类型。"""
|
||||
service = service or self.get_service(name=name)
|
||||
return bool(service and service.type == service_type)
|
||||
1
app/application/messaging/__init__.py
Normal file
1
app/application/messaging/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""Agent 消息桥接、消息模板、交互、通知和推送能力。"""
|
||||
246
app/application/messaging/agent.py
Normal file
246
app/application/messaging/agent.py
Normal file
@@ -0,0 +1,246 @@
|
||||
from queue import Queue
|
||||
from threading import Lock
|
||||
from typing import Callable, Iterable, Optional, Union
|
||||
|
||||
from app.schemas.types import MessageChannel
|
||||
|
||||
|
||||
_WEB_AGENT_EDIT_QUEUES: dict[str, list[Queue[dict]]] = {}
|
||||
_WEB_AGENT_EDIT_LOCK = Lock()
|
||||
_ChannelAdminResolver = Callable[[Optional[dict]], Iterable[Union[str, int]]]
|
||||
_CHANNEL_ADMIN_RESOLVERS: dict[str, _ChannelAdminResolver] = {}
|
||||
|
||||
|
||||
def register_channel_admin_resolver(
|
||||
channel: Union[MessageChannel, str],
|
||||
resolver: _ChannelAdminResolver,
|
||||
) -> None:
|
||||
"""
|
||||
注册消息渠道的管理员主体 ID 解析器。
|
||||
|
||||
:param channel: 消息渠道
|
||||
:param resolver: 由渠道配置解析全部管理员主体 ID 的函数
|
||||
"""
|
||||
channel_value = channel.value if isinstance(channel, MessageChannel) else str(channel)
|
||||
_CHANNEL_ADMIN_RESOLVERS[channel_value] = resolver
|
||||
|
||||
|
||||
def resolve_config_principal_ids(
|
||||
config: Optional[dict],
|
||||
*config_keys: str,
|
||||
) -> set[str]:
|
||||
"""
|
||||
从渠道自行声明的配置键中解析主体 ID。
|
||||
|
||||
:param config: 当前消息渠道配置
|
||||
:param config_keys: 由渠道模块维护的主体 ID 配置键
|
||||
:return: 去空白后的主体 ID 集合
|
||||
"""
|
||||
principal_ids = set()
|
||||
for config_key in config_keys:
|
||||
principal_ids.update(
|
||||
item.strip()
|
||||
for item in str((config or {}).get(config_key) or "").split(",")
|
||||
if item.strip()
|
||||
)
|
||||
return principal_ids
|
||||
|
||||
|
||||
def matches_channel_admin(
|
||||
channel: Union[MessageChannel, str],
|
||||
config: Optional[dict],
|
||||
*principal_ids: Optional[Union[str, int]],
|
||||
) -> bool:
|
||||
"""
|
||||
按渠道配置中的稳定主体 ID 判断管理员身份。
|
||||
|
||||
:param channel: 消息渠道
|
||||
:param config: 当前消息渠道配置
|
||||
:param principal_ids: 消息渠道提供的稳定用户主体 ID
|
||||
:return: 任一用户主体 ID 命中渠道注册的管理员集合时返回 True
|
||||
"""
|
||||
channel_value = channel.value if isinstance(channel, MessageChannel) else str(channel)
|
||||
resolver = _CHANNEL_ADMIN_RESOLVERS.get(channel_value)
|
||||
if not resolver:
|
||||
return False
|
||||
authorized_ids = {
|
||||
str(principal_id).strip()
|
||||
for principal_id in resolver(config)
|
||||
if principal_id is not None and str(principal_id).strip()
|
||||
}
|
||||
if not authorized_ids:
|
||||
return False
|
||||
candidates = {
|
||||
str(principal_id).strip()
|
||||
for principal_id in principal_ids
|
||||
if principal_id is not None and str(principal_id).strip()
|
||||
}
|
||||
return bool(authorized_ids.intersection(candidates))
|
||||
|
||||
|
||||
def normalize_web_agent_button_rows(buttons: Optional[list[list[dict]]]) -> list[list[dict]]:
|
||||
"""
|
||||
将消息按钮转换为 WebAgent 前端可识别的按钮行。
|
||||
|
||||
:param buttons: 传统消息模块返回的按钮二维数组
|
||||
:return: WebAgent 前端选项按钮二维数组
|
||||
"""
|
||||
button_rows: list[list[dict]] = []
|
||||
for row in buttons or []:
|
||||
normalized_row = []
|
||||
for button in row or []:
|
||||
label = str(button.get("text") or button.get("label") or "").strip()
|
||||
callback_data = str(button.get("callback_data") or "").strip()
|
||||
if not label or not callback_data:
|
||||
continue
|
||||
normalized_button = {
|
||||
"label": label,
|
||||
"callback_data": callback_data,
|
||||
}
|
||||
if button.get("description"):
|
||||
normalized_button["description"] = str(button.get("description"))
|
||||
normalized_row.append(normalized_button)
|
||||
if normalized_row:
|
||||
button_rows.append(normalized_row)
|
||||
return button_rows
|
||||
|
||||
|
||||
def _resolve_web_agent_choice_id(
|
||||
message_id: Union[str, int],
|
||||
button_rows: list[list[dict]],
|
||||
) -> str:
|
||||
"""
|
||||
从按钮回调中提取稳定的 WebAgent 选项 ID。
|
||||
|
||||
:param message_id: 前端助手消息 ID
|
||||
:param button_rows: 已规范化的按钮行
|
||||
:return: 选项卡片 ID
|
||||
"""
|
||||
for row in button_rows:
|
||||
for button in row:
|
||||
callback_data = str(button.get("callback_data") or "").strip()
|
||||
if not callback_data:
|
||||
continue
|
||||
parts = callback_data.split(":")
|
||||
if len(parts) >= 2 and parts[1]:
|
||||
return parts[1]
|
||||
return callback_data
|
||||
return str(message_id)
|
||||
|
||||
|
||||
def build_web_agent_message_update_event(
|
||||
*,
|
||||
message_id: Union[str, int],
|
||||
title: Optional[str],
|
||||
text: str,
|
||||
buttons: Optional[list[list[dict]]],
|
||||
) -> dict:
|
||||
"""
|
||||
构造 WebAgent 原消息更新事件。
|
||||
|
||||
:param message_id: 前端助手消息 ID
|
||||
:param title: 更新后的标题
|
||||
:param text: 更新后的正文
|
||||
:param buttons: 更新后的按钮
|
||||
:return: 前端可应用到原消息的 SSE 事件
|
||||
"""
|
||||
button_rows = normalize_web_agent_button_rows(buttons)
|
||||
content_parts = [part for part in (title, text) if part]
|
||||
target_message = {
|
||||
"id": str(message_id),
|
||||
"content": "" if button_rows else "\n\n".join(content_parts),
|
||||
"choices": [],
|
||||
"attachments": [],
|
||||
"tools": [],
|
||||
"status": "done",
|
||||
}
|
||||
if button_rows:
|
||||
target_message["choices"].append({
|
||||
"id": _resolve_web_agent_choice_id(message_id, button_rows),
|
||||
"title": title,
|
||||
"prompt": text or "",
|
||||
"buttons": [button for row in button_rows for button in row],
|
||||
"button_rows": button_rows,
|
||||
"status": "pending",
|
||||
})
|
||||
return {
|
||||
"type": "message_update",
|
||||
"target_message": target_message,
|
||||
}
|
||||
|
||||
|
||||
def attach_web_agent_edit_queue(user_id: str, edit_queue: Queue[dict]) -> None:
|
||||
"""
|
||||
为当前 WebAgent 请求挂载原消息编辑事件队列。
|
||||
|
||||
:param user_id: 当前用户 ID
|
||||
:param edit_queue: 用于接收编辑事件的队列
|
||||
"""
|
||||
with _WEB_AGENT_EDIT_LOCK:
|
||||
_WEB_AGENT_EDIT_QUEUES.setdefault(str(user_id), []).append(edit_queue)
|
||||
|
||||
|
||||
def detach_web_agent_edit_queue(user_id: str, edit_queue: Queue[dict]) -> None:
|
||||
"""
|
||||
移除当前 WebAgent 请求的原消息编辑事件队列。
|
||||
|
||||
:param user_id: 当前用户 ID
|
||||
:param edit_queue: 需要移除的队列
|
||||
"""
|
||||
with _WEB_AGENT_EDIT_LOCK:
|
||||
queues = _WEB_AGENT_EDIT_QUEUES.get(str(user_id))
|
||||
if not queues:
|
||||
return
|
||||
_WEB_AGENT_EDIT_QUEUES[str(user_id)] = [
|
||||
item for item in queues if item is not edit_queue
|
||||
]
|
||||
if not _WEB_AGENT_EDIT_QUEUES[str(user_id)]:
|
||||
_WEB_AGENT_EDIT_QUEUES.pop(str(user_id), None)
|
||||
|
||||
|
||||
def dispatch_web_agent_edit_event(
|
||||
*,
|
||||
user_id: str,
|
||||
event: dict,
|
||||
) -> bool:
|
||||
"""
|
||||
将 WebAgent 原消息编辑事件分发给正在等待的请求队列。
|
||||
|
||||
:param user_id: 当前用户 ID
|
||||
:param event: 前端可应用的 SSE 事件
|
||||
:return: 是否存在接收本次编辑事件的请求队列
|
||||
"""
|
||||
with _WEB_AGENT_EDIT_LOCK:
|
||||
queues = list(_WEB_AGENT_EDIT_QUEUES.get(str(user_id)) or [])
|
||||
for edit_queue in queues:
|
||||
edit_queue.put(event)
|
||||
return bool(queues)
|
||||
|
||||
|
||||
def edit_web_agent_message(
|
||||
*,
|
||||
user_id: str,
|
||||
message_id: Union[str, int],
|
||||
title: Optional[str],
|
||||
text: str,
|
||||
buttons: Optional[list[list[dict]]] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
原地更新 WebAgent 前端消息卡片。
|
||||
|
||||
:param user_id: 当前用户 ID
|
||||
:param message_id: 前端助手消息 ID
|
||||
:param title: 更新后的标题
|
||||
:param text: 更新后的正文
|
||||
:param buttons: 更新后的按钮
|
||||
:return: 是否已投递编辑事件
|
||||
"""
|
||||
if not user_id:
|
||||
return False
|
||||
event = build_web_agent_message_update_event(
|
||||
message_id=message_id,
|
||||
title=title,
|
||||
text=text,
|
||||
buttons=buttons,
|
||||
)
|
||||
return dispatch_web_agent_edit_event(user_id=user_id, event=event)
|
||||
1009
app/application/messaging/interaction.py
Normal file
1009
app/application/messaging/interaction.py
Normal file
File diff suppressed because it is too large
Load Diff
1003
app/application/messaging/message.py
Normal file
1003
app/application/messaging/message.py
Normal file
File diff suppressed because it is too large
Load Diff
34
app/application/notification.py
Normal file
34
app/application/notification.py
Normal file
@@ -0,0 +1,34 @@
|
||||
from typing import Optional
|
||||
|
||||
from app.runtime.extensions.service_registry import ServiceBaseHelper
|
||||
from app.schemas import NotificationConf, ServiceInfo
|
||||
from app.schemas.types import ModuleType, SystemConfigKey
|
||||
|
||||
|
||||
class NotificationHelper(ServiceBaseHelper[NotificationConf]):
|
||||
"""提供按持久化配置发现通知服务的能力。"""
|
||||
|
||||
def __init__(self):
|
||||
"""绑定通知配置和通知模块类型。"""
|
||||
super().__init__(
|
||||
config_key=SystemConfigKey.Notifications,
|
||||
conf_type=NotificationConf,
|
||||
module_type=ModuleType.Notification,
|
||||
)
|
||||
|
||||
def is_notification(
|
||||
self,
|
||||
service_type: Optional[str] = None,
|
||||
service: Optional[ServiceInfo] = None,
|
||||
name: Optional[str] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判断通知服务是否属于指定类型。
|
||||
|
||||
:param service_type: 消息通知服务的类型名称
|
||||
:param service: 要判断的服务信息
|
||||
:param name: 未传入服务信息时用于查询的服务名称
|
||||
:return: 服务存在且类型匹配时返回 True
|
||||
"""
|
||||
service = service or self.get_service(name=name)
|
||||
return bool(service and service.type == service_type)
|
||||
24
app/application/recognition.py
Normal file
24
app/application/recognition.py
Normal file
@@ -0,0 +1,24 @@
|
||||
from typing import Optional
|
||||
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class RecognitionRuleService:
|
||||
"""集中读取用户持久化的媒体识别规则,供启动层注入纯领域匹配器。"""
|
||||
|
||||
def __init__(self, systemconfig: Optional[SystemConfigOper] = None) -> None:
|
||||
"""绑定系统配置访问器,测试可传入隔离替身。"""
|
||||
self._systemconfig = systemconfig or SystemConfigOper()
|
||||
|
||||
def get_customization(self) -> object:
|
||||
"""返回当前自定义占位符配置。"""
|
||||
return self._systemconfig.get(SystemConfigKey.Customization)
|
||||
|
||||
def get_release_groups(self) -> object:
|
||||
"""返回当前用户自定义制作组配置。"""
|
||||
return self._systemconfig.get(SystemConfigKey.CustomReleaseGroups)
|
||||
|
||||
def get_custom_words(self) -> object:
|
||||
"""返回当前自定义识别词配置。"""
|
||||
return self._systemconfig.get(SystemConfigKey.CustomIdentifiers)
|
||||
530
app/application/rss.py
Normal file
530
app/application/rss.py
Normal file
@@ -0,0 +1,530 @@
|
||||
import re
|
||||
import traceback
|
||||
from typing import List, Tuple, Union, Optional
|
||||
from urllib.parse import urljoin, urlparse
|
||||
|
||||
import dateutil.parser
|
||||
from lxml import etree
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.adapters.network.browser import PlaywrightHelper
|
||||
from app.runtime.log import logger
|
||||
from app.adapters.system import rust as rust_accel
|
||||
from app.adapters.network.http import RequestUtils
|
||||
|
||||
|
||||
class RssHelper:
|
||||
"""
|
||||
RSS帮助类,解析RSS报文、获取RSS地址等
|
||||
"""
|
||||
|
||||
# RSS解析限制配置
|
||||
MAX_RSS_SIZE = 50 * 1024 * 1024 # 50MB最大RSS文件大小
|
||||
MAX_RSS_ITEMS = 1000 # 最大解析条目数
|
||||
|
||||
# 各站点RSS链接获取配置
|
||||
rss_link_conf = {
|
||||
"default": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"hares.top": {
|
||||
"xpath": "//*[@id='layui-layer100001']/div[2]/div/p[4]/a/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"et8.org": {
|
||||
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/a[2]/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"pttime.org": {
|
||||
"xpath": "//*[@id='outer']/table/tbody/tr/td/table/tbody/tr/td/text()[5]",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"showrows": 10,
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1
|
||||
}
|
||||
},
|
||||
"ourbits.club": {
|
||||
"xpath": "//a[@class='gen_rsslink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"totheglory.im": {
|
||||
"xpath": "//textarea/text()",
|
||||
"url": "rsstools.php?c51=51&c52=52&c53=53&c54=54&c108=108&c109=109&c62=62&c63=63&c67=67&c69=69&c70=70&c73=73&c76=76&c75=75&c74=74&c87=87&c88=88&c99=99&c90=90&c58=58&c103=103&c101=101&c60=60",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"monikadesign.uk": {
|
||||
"xpath": "//a/@href",
|
||||
"url": "rss",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"zhuque.in": {
|
||||
"xpath": "//a/@href",
|
||||
"url": "user/rss",
|
||||
"render": True,
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
}
|
||||
},
|
||||
"hdchina.org": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"rsscart": 0
|
||||
}
|
||||
},
|
||||
"audiences.me": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"torrent_type": 1,
|
||||
"exp": 180
|
||||
}
|
||||
},
|
||||
"shadowflow.org": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"paid": 0,
|
||||
"search_mode": 0,
|
||||
"showrows": 30
|
||||
}
|
||||
},
|
||||
"hddolby.com": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"exp": 180
|
||||
}
|
||||
},
|
||||
"hdhome.org": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"exp": 180
|
||||
}
|
||||
},
|
||||
"pthome.net": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"exp": 180
|
||||
}
|
||||
},
|
||||
"ptsbao.club": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"size": 0
|
||||
}
|
||||
},
|
||||
"leaves.red": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 0,
|
||||
"paid": 2
|
||||
}
|
||||
},
|
||||
"hdtime.org": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 0,
|
||||
}
|
||||
},
|
||||
"m-team.io": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"showrows": 50,
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"https": 1
|
||||
}
|
||||
},
|
||||
"u2.dmhy.org": {
|
||||
"xpath": "//a[@class='faqlink']/@href",
|
||||
"url": "getrss.php",
|
||||
"params": {
|
||||
"inclbookmarked": 0,
|
||||
"itemsmalldescr": 1,
|
||||
"showrows": 50,
|
||||
"search_mode": 1,
|
||||
"inclautochecked": 1,
|
||||
"trackerssl": 1
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def _get_site_domain(cls, url: str) -> str:
|
||||
"""按 RSS 站点配置匹配域名,未命中时回退到最后两级域名。"""
|
||||
hostname = (urlparse(url).hostname or "").lower()
|
||||
for domain in cls.rss_link_conf:
|
||||
if domain == "default":
|
||||
continue
|
||||
if hostname == domain or hostname.endswith(f".{domain}"):
|
||||
return domain
|
||||
parts = hostname.split(".")
|
||||
return ".".join(parts[-2:]) if len(parts) >= 2 else hostname
|
||||
|
||||
@staticmethod
|
||||
def _parse_publish_time(value: str):
|
||||
"""将 RSS 常见日期表达解析为 datetime,无法解析时返回 None。"""
|
||||
try:
|
||||
return dateutil.parser.parse(value)
|
||||
except dateutil.parser.ParserError:
|
||||
return None
|
||||
|
||||
def __parse_with_rust(self, ret_xml: Optional[str]) -> Optional[list]:
|
||||
"""
|
||||
调用 Rust RSS 解析器,并统一处理基础 XML 校验和最大条目限制。
|
||||
"""
|
||||
if not ret_xml or not ret_xml.strip():
|
||||
return None
|
||||
ret_xml_stripped = ret_xml.strip()
|
||||
if not ret_xml_stripped.startswith('<'):
|
||||
return None
|
||||
rust_items = rust_accel.parse_rss_items(ret_xml, self.MAX_RSS_ITEMS + 1)
|
||||
if rust_items is None:
|
||||
return None
|
||||
if len(rust_items) > self.MAX_RSS_ITEMS:
|
||||
logger.warning(f"RSS条目过多: 超过{self.MAX_RSS_ITEMS},仅处理前{self.MAX_RSS_ITEMS}个")
|
||||
return rust_items[:self.MAX_RSS_ITEMS]
|
||||
|
||||
def parse(self, url, proxy: bool = False,
|
||||
timeout: Optional[int] = 15, headers: dict = None, ua: str = None) -> Union[List[dict], None, bool]:
|
||||
"""
|
||||
解析RSS订阅URL,获取RSS中的种子信息
|
||||
:param url: RSS地址
|
||||
:param proxy: 是否使用代理
|
||||
:param timeout: 请求超时
|
||||
:param headers: 自定义请求头
|
||||
:param ua: 自定义User-Agent
|
||||
:return: 种子信息列表,如为None代表Rss过期,如果为False则为错误
|
||||
"""
|
||||
# 开始处理
|
||||
ret_array: list = []
|
||||
if not url:
|
||||
return False
|
||||
|
||||
try:
|
||||
ret = RequestUtils(ua=ua,
|
||||
proxies=settings.PROXY if proxy else None,
|
||||
timeout=timeout or 30, headers=headers).get_res(url)
|
||||
if not ret:
|
||||
logger.error(f"获取RSS失败:请求返回空值,URL: {url}")
|
||||
return False
|
||||
except Exception as err:
|
||||
logger.error(f"获取RSS失败:{str(err)} - {traceback.format_exc()}")
|
||||
return False
|
||||
|
||||
if ret:
|
||||
# 检查HTTP状态码
|
||||
if ret.status_code != 200:
|
||||
logger.error(f"RSS请求失败,状态码: {ret.status_code}, URL: {url}")
|
||||
return False
|
||||
ret_xml = None
|
||||
root = None
|
||||
try:
|
||||
# 检查响应大小,避免处理过大的RSS文件
|
||||
raw_data = ret.content
|
||||
if raw_data and len(raw_data) > self.MAX_RSS_SIZE:
|
||||
logger.warning(f"RSS文件过大: {len(raw_data) / 1024 / 1024:.1f}MB,跳过解析")
|
||||
return False
|
||||
|
||||
if raw_data:
|
||||
ret_xml = RequestUtils.get_decoded_xml_content(
|
||||
ret,
|
||||
performance_mode=settings.ENCODING_DETECTION_PERFORMANCE_MODE,
|
||||
confidence_threshold=settings.ENCODING_DETECTION_MIN_CONFIDENCE
|
||||
)
|
||||
rust_items = self.__parse_with_rust(ret_xml)
|
||||
if rust_items is not None:
|
||||
return rust_items
|
||||
if not ret_xml:
|
||||
ret_xml = ret.text
|
||||
|
||||
# 验证RSS内容是否有效
|
||||
if not ret_xml or not ret_xml.strip():
|
||||
logger.error("RSS内容为空")
|
||||
return False
|
||||
|
||||
# 检查是否包含基本的RSS/XML结构
|
||||
ret_xml_stripped = ret_xml.strip()
|
||||
if not ret_xml_stripped.startswith('<'):
|
||||
logger.error("RSS内容不是有效的XML格式")
|
||||
return False
|
||||
|
||||
rust_items = self.__parse_with_rust(ret_xml)
|
||||
if rust_items is not None:
|
||||
return rust_items
|
||||
|
||||
# 使用lxml.etree解析XML
|
||||
parser = None
|
||||
try:
|
||||
# 创建解析器,禁用网络访问以提高安全性和性能
|
||||
parser = etree.XMLParser(
|
||||
recover=True, # 容错模式
|
||||
strip_cdata=False, # 保留CDATA
|
||||
resolve_entities=False, # 禁用外部实体解析
|
||||
no_network=True, # 禁用网络访问
|
||||
huge_tree=False # 禁用大文档解析,避免内存问题
|
||||
)
|
||||
root = etree.fromstring(ret_xml.encode('utf-8'), parser=parser)
|
||||
except etree.XMLSyntaxError as xml_error:
|
||||
logger.debug(f"XML解析失败:{str(xml_error)},尝试HTML解析")
|
||||
# 如果XML解析失败,尝试作为HTML解析
|
||||
try:
|
||||
root = etree.HTML(ret_xml)
|
||||
if root is not None:
|
||||
# 查找RSS根节点
|
||||
rss_root = root.xpath('//rss | //feed')
|
||||
if rss_root:
|
||||
root = rss_root[0]
|
||||
except Exception as e:
|
||||
logger.error(f"HTML解析也失败:{str(e)}")
|
||||
return False
|
||||
except Exception as general_error:
|
||||
logger.error(f"解析RSS时发生未预期错误:{str(general_error)}")
|
||||
return False
|
||||
finally:
|
||||
if parser is not None:
|
||||
try:
|
||||
parser.close()
|
||||
except Exception as close_error:
|
||||
logger.debug(f"关闭解析器时出错:{str(close_error)}")
|
||||
del parser
|
||||
|
||||
if root is None:
|
||||
logger.error("无法解析RSS内容")
|
||||
return False
|
||||
|
||||
# 查找所有item或entry节点
|
||||
items = root.xpath('.//item | .//entry')
|
||||
|
||||
# 限制处理的条目数量
|
||||
items_count = min(len(items), self.MAX_RSS_ITEMS)
|
||||
if len(items) > self.MAX_RSS_ITEMS:
|
||||
logger.warning(f"RSS条目过多: {len(items)},仅处理前{self.MAX_RSS_ITEMS}个")
|
||||
try:
|
||||
for item in items[:items_count]:
|
||||
try:
|
||||
# 使用xpath提取信息,更高效
|
||||
title_nodes = item.xpath('.//title')
|
||||
title = title_nodes[0].text if title_nodes and title_nodes[0].text else ""
|
||||
if not title:
|
||||
continue
|
||||
|
||||
# 描述
|
||||
desc_nodes = item.xpath('.//description | .//summary')
|
||||
description = desc_nodes[0].text if desc_nodes and desc_nodes[0].text else ""
|
||||
|
||||
# 种子页面
|
||||
link_nodes = item.xpath('.//link')
|
||||
if link_nodes:
|
||||
link = link_nodes[0].text if hasattr(link_nodes[0], 'text') and link_nodes[0].text else link_nodes[0].get('href', '')
|
||||
else:
|
||||
link = ""
|
||||
|
||||
# 种子链接
|
||||
enclosure_nodes = item.xpath('.//enclosure')
|
||||
enclosure = enclosure_nodes[0].get('url', '') if enclosure_nodes else ""
|
||||
if not enclosure and not link:
|
||||
continue
|
||||
# 部分RSS只有link没有enclosure
|
||||
if not enclosure and link:
|
||||
enclosure = link
|
||||
|
||||
# 大小
|
||||
size = 0
|
||||
if enclosure_nodes:
|
||||
size_attr = enclosure_nodes[0].get('length', '0')
|
||||
if size_attr and str(size_attr).isdigit():
|
||||
size = int(size_attr)
|
||||
|
||||
# 发布日期
|
||||
pubdate_nodes = item.xpath('./pubDate | ./published | ./updated')
|
||||
if not pubdate_nodes:
|
||||
pubdate_nodes = item.xpath('.//*[local-name()="pubDate"] | .//*[local-name()="published"] | .//*[local-name()="updated"]')
|
||||
|
||||
pubdate = ""
|
||||
if pubdate_nodes and pubdate_nodes[0].text:
|
||||
pubdate = self._parse_publish_time(pubdate_nodes[0].text)
|
||||
if pubdate is not None:
|
||||
# 转为本地时区
|
||||
pubdate = pubdate.astimezone(tz=None)
|
||||
|
||||
# 获取豆瓣昵称
|
||||
nickname_nodes = item.xpath('.//*[local-name()="creator"]')
|
||||
nickname = nickname_nodes[0].text if nickname_nodes and nickname_nodes[0].text else ""
|
||||
|
||||
# 返回对象
|
||||
tmp_dict = {
|
||||
'title': title,
|
||||
'enclosure': enclosure,
|
||||
'size': size,
|
||||
'description': description,
|
||||
'link': link,
|
||||
'pubdate': pubdate
|
||||
}
|
||||
# 如果豆瓣昵称不为空,返回数据增加豆瓣昵称,供doubansync插件获取
|
||||
if nickname:
|
||||
tmp_dict['nickname'] = nickname
|
||||
ret_array.append(tmp_dict)
|
||||
|
||||
except Exception as e1:
|
||||
logger.debug(f"解析RSS条目失败:{str(e1)} - {traceback.format_exc()}")
|
||||
continue
|
||||
finally:
|
||||
items.clear()
|
||||
del items
|
||||
|
||||
except Exception as e2:
|
||||
logger.error(f"解析RSS失败:{str(e2)} - {traceback.format_exc()}")
|
||||
# RSS过期检查
|
||||
_rss_expired_msg = [
|
||||
"RSS 链接已过期, 您需要获得一个新的!",
|
||||
"RSS Link has expired, You need to get a new one!",
|
||||
"RSS Link has expired, You need to get new!"
|
||||
]
|
||||
if ret_xml in _rss_expired_msg:
|
||||
return None
|
||||
return False
|
||||
finally:
|
||||
if root is not None:
|
||||
del root
|
||||
if ret_xml is not None:
|
||||
del ret_xml
|
||||
|
||||
return ret_array
|
||||
|
||||
def get_rss_link(self, url: str, cookie: str, ua: str, proxy: bool = False, timeout: int = None) -> Tuple[str, str]:
|
||||
"""
|
||||
获取站点rss地址
|
||||
:param url: 站点地址
|
||||
:param cookie: 站点cookie
|
||||
:param ua: 站点ua
|
||||
:param proxy: 是否使用代理
|
||||
:param timeout: 请求超时时间
|
||||
:return: rss地址、错误信息
|
||||
"""
|
||||
try:
|
||||
# 获取站点域名
|
||||
domain = self._get_site_domain(url)
|
||||
# 获取配置
|
||||
site_conf = self.rss_link_conf.get(domain) or self.rss_link_conf.get("default")
|
||||
# RSS地址
|
||||
rss_url = urljoin(url, site_conf.get("url"))
|
||||
# RSS请求参数
|
||||
rss_params = site_conf.get("params")
|
||||
# 请求RSS页面
|
||||
if site_conf.get("render"):
|
||||
html_text = PlaywrightHelper().get_page_source(
|
||||
url=rss_url,
|
||||
cookies=cookie,
|
||||
ua=ua,
|
||||
proxies=settings.PROXY_SERVER if proxy else None,
|
||||
timeout=timeout or 60
|
||||
)
|
||||
else:
|
||||
res = RequestUtils(
|
||||
cookies=cookie,
|
||||
timeout=timeout or 30,
|
||||
ua=ua,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
).post_res(url=rss_url, data=rss_params)
|
||||
if res:
|
||||
html_text = res.text
|
||||
elif res is not None:
|
||||
return "", f"获取 {url} RSS链接失败,错误码:{res.status_code},错误原因:{res.reason}"
|
||||
else:
|
||||
return "", f"获取RSS链接失败:无法连接 {url} "
|
||||
|
||||
# 解析HTML
|
||||
if html_text:
|
||||
html = None
|
||||
try:
|
||||
html = etree.HTML(html_text)
|
||||
if html is not None and len(html) > 0:
|
||||
rss_link = html.xpath(site_conf.get("xpath"))
|
||||
if rss_link:
|
||||
return str(rss_link[-1]), ""
|
||||
finally:
|
||||
if html is not None:
|
||||
del html
|
||||
|
||||
return "", f"获取RSS链接失败:{url}"
|
||||
except Exception as e:
|
||||
return "", f"获取 {url} RSS链接失败:{str(e)}"
|
||||
1
app/application/security/__init__.py
Normal file
1
app/application/security/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""认证授权、URL 安全、OTP、Cookie、Passkey 和双因素认证能力。"""
|
||||
442
app/application/security/access.py
Normal file
442
app/application/security/access.py
Normal file
@@ -0,0 +1,442 @@
|
||||
import base64
|
||||
import datetime
|
||||
import hashlib
|
||||
import hmac
|
||||
import json
|
||||
import os
|
||||
import traceback
|
||||
from datetime import timedelta
|
||||
from typing import Any, Union, Annotated, Optional, Callable
|
||||
|
||||
import jwt
|
||||
from Crypto.Cipher import AES
|
||||
from Crypto.Util.Padding import pad
|
||||
from cryptography.fernet import Fernet
|
||||
from fastapi import HTTPException, status, Security, Request, Response
|
||||
from fastapi.security import OAuth2PasswordBearer, APIKeyHeader, APIKeyQuery, APIKeyCookie, HTTPBearer
|
||||
from passlib.context import CryptContext
|
||||
|
||||
from app import schemas
|
||||
from app.runtime.cache import cached
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.log import logger
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
ALGORITHM = "HS256"
|
||||
SuperuserTokenPayloadProvider = Callable[[], schemas.TokenPayload]
|
||||
_superuser_token_payload_provider: Optional[SuperuserTokenPayloadProvider] = None
|
||||
|
||||
|
||||
def set_superuser_token_payload_provider(
|
||||
provider: SuperuserTokenPayloadProvider,
|
||||
) -> None:
|
||||
"""注入 API 密钥认证所需的超级用户载荷提供器。"""
|
||||
global _superuser_token_payload_provider
|
||||
_superuser_token_payload_provider = provider
|
||||
|
||||
# OAuth2PasswordBearer 用于 JWT Token 认证
|
||||
oauth2_scheme_manual_error = OAuth2PasswordBearer(
|
||||
auto_error=False, # 禁用自动错误处理,用以支持API令牌鉴权
|
||||
tokenUrl=f"{settings.API_V1_STR}/login/access-token"
|
||||
)
|
||||
|
||||
# RESOURCE TOKEN 通过 Cookie 认证
|
||||
resource_token_cookie = APIKeyCookie(name=settings.PROJECT_NAME, auto_error=False, scheme_name="resource_token_cookie")
|
||||
|
||||
# API TOKEN 通过 QUERY 认证
|
||||
api_token_query = APIKeyQuery(name="token", auto_error=False, scheme_name="api_token_query")
|
||||
|
||||
# API KEY 通过 Header 认证
|
||||
api_key_header = APIKeyHeader(name="X-API-KEY", auto_error=False, scheme_name="api_key_header")
|
||||
|
||||
# API KEY 通过 QUERY 认证
|
||||
api_key_query = APIKeyQuery(name="apikey", auto_error=False, scheme_name="api_key_query")
|
||||
|
||||
# OpenAI compatible Bearer Token 认证
|
||||
openai_bearer_scheme = HTTPBearer(auto_error=False)
|
||||
|
||||
# Anthropic compatible API Key 认证
|
||||
anthropic_api_key_header = APIKeyHeader(name="x-api-key", auto_error=False, scheme_name="anthropic_api_key_header")
|
||||
|
||||
|
||||
def __get_api_token(
|
||||
token_query: Annotated[str | None, Security(api_token_query)] = None
|
||||
) -> str | None:
|
||||
"""
|
||||
从 URL 查询参数中获取 API Token
|
||||
:param token_query: 从 URL 中的 `token` 查询参数获取 API Token
|
||||
:return: 返回获取到的 API Token,若无则返回 None
|
||||
"""
|
||||
return token_query
|
||||
|
||||
|
||||
def __get_api_key(
|
||||
key_query: Annotated[str | None, Security(api_key_query)] = None,
|
||||
key_header: Annotated[str | None, Security(api_key_header)] = None
|
||||
) -> str | None:
|
||||
"""
|
||||
从 URL 查询参数或请求头部获取 API Key,优先使用请求头
|
||||
:param key_query: URL 中的 `apikey` 查询参数
|
||||
:param key_header: 请求头中的 `X-API-KEY` 参数
|
||||
:return: 返回从 URL 或请求头中获取的 API Key,若无则返回 None
|
||||
"""
|
||||
return key_header or key_query # 首选请求头
|
||||
|
||||
|
||||
@cached(maxsize=1, ttl=600)
|
||||
def __create_superuser_token_payload() -> schemas.TokenPayload:
|
||||
"""
|
||||
创建管理员用户的TokenPayload
|
||||
|
||||
:return: 管理员TokenPayload
|
||||
"""
|
||||
if not _superuser_token_payload_provider:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="认证服务尚未初始化",
|
||||
)
|
||||
return _superuser_token_payload_provider()
|
||||
|
||||
|
||||
def create_access_token(
|
||||
userid: Union[str, Any],
|
||||
username: str,
|
||||
super_user: Optional[bool] = False,
|
||||
expires_delta: Optional[timedelta] = None,
|
||||
level: Optional[int] = 1,
|
||||
purpose: Optional[str] = "authentication"
|
||||
) -> str:
|
||||
"""
|
||||
创建 JWT 访问令牌,包含用户 ID、用户名、是否为超级用户以及权限等级
|
||||
:param userid: 用户的唯一标识符,通常是字符串或整数
|
||||
:param username: 用户名,用于标识用户的账户名
|
||||
:param super_user: 是否为超级用户,默认值为 False
|
||||
:param expires_delta: 令牌的有效期时长,如果不提供则根据用途使用默认过期时间
|
||||
:param level: 用户的权限级别,默认为 1
|
||||
:param purpose: 令牌的用途,"authentication" 或 "resource"
|
||||
:return: 编码后的 JWT 令牌字符串
|
||||
:raises ValueError: 如果 expires_delta 为负数
|
||||
"""
|
||||
if purpose == "resource":
|
||||
default_expire = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
|
||||
secret_key = settings.RESOURCE_SECRET_KEY
|
||||
else:
|
||||
default_expire = timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES)
|
||||
secret_key = settings.SECRET_KEY
|
||||
|
||||
if expires_delta is not None:
|
||||
if expires_delta.total_seconds() <= 0:
|
||||
raise ValueError("过期时间必须为正数")
|
||||
expire = datetime.datetime.now(datetime.UTC) + expires_delta
|
||||
else:
|
||||
expire = datetime.datetime.now(datetime.UTC) + default_expire
|
||||
|
||||
to_encode = {
|
||||
"exp": expire,
|
||||
"iat": datetime.datetime.now(datetime.UTC),
|
||||
"sub": str(userid),
|
||||
"username": username,
|
||||
"super_user": super_user,
|
||||
"level": level,
|
||||
"purpose": purpose
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(to_encode, secret_key, algorithm=ALGORITHM)
|
||||
return encoded_jwt
|
||||
|
||||
|
||||
def set_or_refresh_resource_token_cookie(
|
||||
request: Request, response: Response, payload: schemas.TokenPayload
|
||||
) -> None:
|
||||
"""
|
||||
设置资源令牌 Cookie
|
||||
:param request: 包含请求相关的上下文数据
|
||||
:param response: 用于在服务器响应时设置 Cookie
|
||||
:param payload: 已通过身份验证的 TokenPayload 对象
|
||||
"""
|
||||
resource_token = request.cookies.get(settings.PROJECT_NAME)
|
||||
|
||||
if resource_token:
|
||||
# 检查令牌剩余时间
|
||||
try:
|
||||
decoded_token = jwt.decode(resource_token, settings.RESOURCE_SECRET_KEY, algorithms=[ALGORITHM])
|
||||
exp = decoded_token.get("exp")
|
||||
if exp:
|
||||
remaining_time = datetime.datetime.fromtimestamp(exp, tz=datetime.UTC) - datetime.datetime.now(datetime.UTC)
|
||||
# 根据剩余时长提前刷新令牌
|
||||
if remaining_time < timedelta(seconds=(settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS / 3)):
|
||||
raise jwt.ExpiredSignatureError
|
||||
expected_claims = {
|
||||
"sub": str(payload.sub),
|
||||
"username": payload.username,
|
||||
"super_user": payload.super_user,
|
||||
"level": payload.level,
|
||||
"purpose": "resource",
|
||||
}
|
||||
if any(decoded_token.get(claim) != value for claim, value in expected_claims.items()):
|
||||
raise jwt.InvalidTokenError("资源令牌身份或权限上下文不匹配")
|
||||
except jwt.PyJWTError:
|
||||
logger.debug(f"Token error occurred. refreshing token")
|
||||
except Exception as e:
|
||||
logger.debug(f"Unexpected error occurred while decoding token: {e}")
|
||||
else:
|
||||
# 如果令牌有效且没有即将过期,则不需要刷新
|
||||
return
|
||||
|
||||
# 创建新的资源访问令牌
|
||||
resource_token_expires = timedelta(seconds=settings.RESOURCE_ACCESS_TOKEN_EXPIRE_SECONDS)
|
||||
resource_token = create_access_token(
|
||||
userid=payload.sub,
|
||||
username=payload.username,
|
||||
super_user=payload.super_user,
|
||||
expires_delta=resource_token_expires,
|
||||
level=payload.level,
|
||||
purpose="resource"
|
||||
)
|
||||
|
||||
# 判断请求是否为 HTTPS:直连协议为 https,或经反向代理转发时携带 X-Forwarded-Proto: https。
|
||||
# 无法确认为明文 HTTP 时按 fail-safe 默认设置 secure=True,避免代理终止 HTTPS 后以 HTTP 转发导致 Cookie 明文传输。
|
||||
is_https = (
|
||||
request.url.scheme == "https"
|
||||
or request.headers.get("x-forwarded-proto", "").lower() == "https"
|
||||
)
|
||||
|
||||
# 设置会话级别的 HttpOnly Cookie
|
||||
response.set_cookie(
|
||||
key=settings.PROJECT_NAME,
|
||||
value=resource_token,
|
||||
httponly=True,
|
||||
secure=is_https, # 根据当前请求协议(含反向代理转发标识)设置 secure 属性
|
||||
samesite="lax" # 不同浏览器对 "Strict" 的处理可能不同,设置 SameSite 为 "Lax",以平衡安全性和兼容性
|
||||
)
|
||||
|
||||
|
||||
def __verify_token(token: str, purpose: Optional[str] = "authentication") -> schemas.TokenPayload:
|
||||
"""
|
||||
使用 JWT Token 进行身份认证并解析 Token 的内容
|
||||
:param token: JWT 令牌
|
||||
:param purpose: 期望的令牌用途,默认为 "authentication"
|
||||
:return: 包含用户身份信息的 Token 负载数据
|
||||
:raises HTTPException: 如果令牌无效或用途不匹配
|
||||
"""
|
||||
try:
|
||||
if purpose == "resource":
|
||||
secret_key = settings.RESOURCE_SECRET_KEY
|
||||
else:
|
||||
secret_key = settings.SECRET_KEY
|
||||
|
||||
if not token:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"{purpose} token not found"
|
||||
)
|
||||
|
||||
payload = jwt.decode(
|
||||
token, secret_key, algorithms=[ALGORITHM]
|
||||
)
|
||||
|
||||
token_payload = schemas.TokenPayload(**payload)
|
||||
|
||||
if token_payload.purpose != purpose:
|
||||
raise jwt.InvalidTokenError("令牌用途不匹配")
|
||||
|
||||
return schemas.TokenPayload(**payload)
|
||||
except (jwt.DecodeError, jwt.InvalidTokenError, jwt.ImmatureSignatureError):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="token校验不通过",
|
||||
)
|
||||
|
||||
|
||||
def verify_token(
|
||||
request: Request,
|
||||
response: Response,
|
||||
jwt_token: Annotated[str | None, Security(oauth2_scheme_manual_error)],
|
||||
api_key: Annotated[str | None, Security(__get_api_key)],
|
||||
api_token: Annotated[str | None, Security(__get_api_token)],
|
||||
) -> schemas.TokenPayload:
|
||||
"""
|
||||
验证 JWT 令牌并自动处理 resource_token 写入
|
||||
|
||||
如果缺少JWT令牌再尝试用API令牌鉴权
|
||||
|
||||
:param request: 请求对象,用于访问 Cookie 和请求信息
|
||||
:param response: 响应对象,用于设置 Cookie
|
||||
:param jwt_token: 从 Authorization 头部获取的 JWT 令牌
|
||||
:param api_key: 从 查询参数`apikey` 或 请求头`X-API-KEY` 获取 API Token
|
||||
:param api_token: 从 查询参数`token` 获取 API Token
|
||||
:return: 解析后的 TokenPayload
|
||||
:raises HTTPException: 如果令牌无效或用途不匹配
|
||||
"""
|
||||
if jwt_token:
|
||||
# 验证并解析 JWT 认证令牌
|
||||
payload = __verify_token(token=jwt_token, purpose="authentication")
|
||||
|
||||
# 如果没有 resource_token,生成并写入到 Cookie
|
||||
set_or_refresh_resource_token_cookie(request, response, payload)
|
||||
|
||||
return payload
|
||||
elif api_key:
|
||||
verify_apikey(api_key)
|
||||
return __create_superuser_token_payload()
|
||||
elif api_token:
|
||||
verify_apitoken(api_token)
|
||||
return __create_superuser_token_payload()
|
||||
else:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Not authenticated",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
|
||||
def verify_resource_token(
|
||||
resource_token: Annotated[str, Security(resource_token_cookie)]
|
||||
) -> schemas.TokenPayload:
|
||||
"""
|
||||
验证资源访问令牌(从 Cookie 中获取)
|
||||
:param resource_token: 从 Cookie 中获取的资源访问令牌
|
||||
:return: 解析后的 TokenPayload
|
||||
:raises HTTPException: 如果资源访问令牌无效
|
||||
"""
|
||||
# 验证并解析资源访问令牌
|
||||
return __verify_token(token=resource_token, purpose="resource")
|
||||
|
||||
|
||||
def __verify_key(key: str | None, expected_key: str, key_type: str) -> str:
|
||||
"""
|
||||
通用的 API Key 或 Token 验证函数
|
||||
:param key: 从请求中获取的 API Key 或 Token
|
||||
:param expected_key: 系统配置中的期望值,用于验证的 API Key 或 Token
|
||||
:param key_type: 键的类型(例如 "API_KEY" 或 "API_TOKEN"),用于错误消息
|
||||
:return: 返回校验通过的 API Key 或 Token
|
||||
:raises HTTPException: 如果校验不通过,抛出 401 错误
|
||||
"""
|
||||
if not key or key != expected_key:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail=f"{key_type} 校验不通过"
|
||||
)
|
||||
return key
|
||||
|
||||
|
||||
def verify_apitoken(token: Annotated[str | None, Security(__get_api_token)]) -> str:
|
||||
"""
|
||||
使用 API Token 进行受信第三方集成认证。
|
||||
|
||||
校验值来自 settings.API_TOKEN;通过后只确认集成凭据有效,不生成 per-user 权限上下文。
|
||||
:param token: API Token,从 URL 查询参数中获取 token=xxx
|
||||
:return: 返回校验通过的 API Token
|
||||
"""
|
||||
return __verify_key(token, settings.API_TOKEN, "token")
|
||||
|
||||
|
||||
def verify_apikey(apikey: Annotated[str | None, Security(__get_api_key)]) -> str:
|
||||
"""
|
||||
使用 API Key 形式进行受信第三方集成认证。
|
||||
|
||||
请求字段名兼容 API Key,实际校验值来自 settings.API_TOKEN,不生成 per-user 权限上下文。
|
||||
:param apikey: API Key,从 URL 查询参数中获取 apikey=xxx,或请求头中获取 X-API-KEY=xxx
|
||||
:return: 返回校验通过的 API Key
|
||||
"""
|
||||
return __verify_key(apikey, settings.API_TOKEN, "apikey")
|
||||
|
||||
|
||||
def verify_password(plain_password: str, hashed_password: str) -> bool:
|
||||
"""校验明文密码是否匹配已保存的密码摘要。"""
|
||||
return pwd_context.verify(plain_password, hashed_password)
|
||||
|
||||
|
||||
def get_password_hash(password: str) -> str:
|
||||
"""生成适合持久化保存的密码摘要。"""
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def decrypt(data: bytes, key: bytes) -> Optional[bytes]:
|
||||
"""
|
||||
解密二进制数据
|
||||
"""
|
||||
fernet = Fernet(key)
|
||||
try:
|
||||
return fernet.decrypt(data)
|
||||
except Exception as e:
|
||||
logger.error(f"解密失败:{str(e)} - {traceback.format_exc()}")
|
||||
return None
|
||||
|
||||
|
||||
def encrypt_message(message: str, key: bytes) -> str:
|
||||
"""
|
||||
使用给定的key对消息进行加密,并返回加密后的字符串
|
||||
"""
|
||||
f = Fernet(key)
|
||||
encrypted_message = f.encrypt(message.encode())
|
||||
return encrypted_message.decode()
|
||||
|
||||
|
||||
def hash_sha256(message: str) -> str:
|
||||
"""
|
||||
对字符串做hash运算
|
||||
"""
|
||||
return hashlib.sha256(message.encode()).hexdigest()
|
||||
|
||||
|
||||
def aes_decrypt(data: str, key: str) -> str:
|
||||
"""
|
||||
AES解密
|
||||
"""
|
||||
if not data:
|
||||
return ""
|
||||
data = base64.b64decode(data)
|
||||
iv = data[:16]
|
||||
encrypted = data[16:]
|
||||
# 使用AES-256-CBC解密
|
||||
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC, iv)
|
||||
result = cipher.decrypt(encrypted)
|
||||
# 去除填充
|
||||
padding = result[-1]
|
||||
if padding < 1 or padding > AES.block_size:
|
||||
return ""
|
||||
result = result[:-padding]
|
||||
return result.decode('utf-8')
|
||||
|
||||
|
||||
def aes_encrypt(data: str, key: str) -> str:
|
||||
"""
|
||||
AES加密
|
||||
"""
|
||||
if not data:
|
||||
return ""
|
||||
# 使用AES-256-CBC加密
|
||||
cipher = AES.new(key.encode('utf-8'), AES.MODE_CBC)
|
||||
# 填充
|
||||
padding = AES.block_size - len(data) % AES.block_size
|
||||
data += chr(padding) * padding
|
||||
result = cipher.encrypt(data.encode('utf-8'))
|
||||
# 使用base64编码
|
||||
return base64.b64encode(cipher.iv + result).decode('utf-8')
|
||||
|
||||
|
||||
def nexusphp_encrypt(data_str: str, key: bytes) -> str:
|
||||
"""
|
||||
NexusPHP加密
|
||||
"""
|
||||
# 生成16字节长的随机字符串
|
||||
iv = os.urandom(16)
|
||||
# 对向量进行 Base64 编码
|
||||
iv_base64 = base64.b64encode(iv)
|
||||
# 加密数据
|
||||
cipher = AES.new(key, AES.MODE_CBC, iv)
|
||||
ciphertext = cipher.encrypt(pad(data_str.encode(), AES.block_size))
|
||||
ciphertext_base64 = base64.b64encode(ciphertext)
|
||||
# 对向量的字符串表示进行签名
|
||||
mac = hmac.new(key, msg=iv_base64 + ciphertext_base64, digestmod=hashlib.sha256).hexdigest()
|
||||
# 构造 JSON 字符串
|
||||
json_str = json.dumps({
|
||||
'iv': iv_base64.decode(),
|
||||
'value': ciphertext_base64.decode(),
|
||||
'mac': mac,
|
||||
'tag': ''
|
||||
})
|
||||
|
||||
# 对 JSON 字符串进行 Base64 编码
|
||||
return base64.b64encode(json_str.encode()).decode()
|
||||
166
app/application/security/auth.py
Normal file
166
app/application/security/auth.py
Normal file
@@ -0,0 +1,166 @@
|
||||
import secrets
|
||||
import threading
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from typing import Any, Optional
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from app import schemas
|
||||
from app.application.security import access as security
|
||||
from app.runtime.config import settings
|
||||
from app.db.models.user import User
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.db.user_oper import UserOper
|
||||
from app.application.site.sites import SitesHelper # pylint: disable=no-name-in-module
|
||||
from app.schemas.types import SystemConfigKey
|
||||
from app.foundation.singleton import Singleton
|
||||
|
||||
|
||||
class AuthTicketStore(metaclass=Singleton):
|
||||
"""
|
||||
插件认证一次性票据存储。
|
||||
"""
|
||||
|
||||
_ttl_seconds = 120
|
||||
_max_items = 1024
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化内存票据缓存。
|
||||
"""
|
||||
self._tickets: dict[str, dict[str, Any]] = {}
|
||||
self._lock = threading.RLock()
|
||||
|
||||
def create(self, user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
|
||||
"""
|
||||
创建短时一次性登录票据。
|
||||
|
||||
:param user_id: 已通过插件认证的本地用户 ID
|
||||
:param provider_id: 认证提供方 ID
|
||||
:param metadata: 插件侧附加信息
|
||||
:return: 一次性票据字符串
|
||||
"""
|
||||
ticket = secrets.token_urlsafe(32)
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
self._cleanup(now)
|
||||
self._tickets[ticket] = {
|
||||
"user_id": int(user_id),
|
||||
"provider_id": provider_id,
|
||||
"metadata": metadata or {},
|
||||
"created_at": now,
|
||||
}
|
||||
return ticket
|
||||
|
||||
def consume(self, ticket: str) -> Optional[dict[str, Any]]:
|
||||
"""
|
||||
消费并删除一次性登录票据。
|
||||
|
||||
:param ticket: 登录票据
|
||||
:return: 票据数据,票据不存在或过期时返回 None
|
||||
"""
|
||||
if not ticket:
|
||||
return None
|
||||
now = time.time()
|
||||
with self._lock:
|
||||
data = self._tickets.pop(ticket, None)
|
||||
self._cleanup(now)
|
||||
if not data:
|
||||
return None
|
||||
if now - float(data.get("created_at") or 0) > self._ttl_seconds:
|
||||
return None
|
||||
return data
|
||||
|
||||
def _cleanup(self, now: Optional[float] = None) -> None:
|
||||
"""
|
||||
清理过期或过量的票据缓存。
|
||||
|
||||
:param now: 当前时间戳,未传入时自动读取
|
||||
"""
|
||||
current = now or time.time()
|
||||
expired = [
|
||||
key
|
||||
for key, value in self._tickets.items()
|
||||
if current - float(value.get("created_at") or 0) > self._ttl_seconds
|
||||
]
|
||||
for key in expired:
|
||||
self._tickets.pop(key, None)
|
||||
if len(self._tickets) <= self._max_items:
|
||||
return
|
||||
ordered = sorted(
|
||||
self._tickets.items(),
|
||||
key=lambda item: float(item[1].get("created_at") or 0),
|
||||
)
|
||||
for key, _ in ordered[: len(self._tickets) - self._max_items]:
|
||||
self._tickets.pop(key, None)
|
||||
|
||||
|
||||
def create_plugin_auth_ticket(user_id: int, provider_id: str, metadata: Optional[dict[str, Any]] = None) -> str:
|
||||
"""
|
||||
为插件认证成功的用户创建一次性登录票据。
|
||||
|
||||
:param user_id: 本地用户 ID
|
||||
:param provider_id: 认证提供方 ID
|
||||
:param metadata: 插件侧附加信息
|
||||
:return: 一次性票据字符串
|
||||
"""
|
||||
return AuthTicketStore().create(user_id=user_id, provider_id=provider_id, metadata=metadata)
|
||||
|
||||
|
||||
def consume_plugin_auth_ticket(ticket: str) -> Optional[dict[str, Any]]:
|
||||
"""
|
||||
消费插件认证登录票据。
|
||||
|
||||
:param ticket: 登录票据
|
||||
:return: 票据数据,票据不存在或过期时返回 None
|
||||
"""
|
||||
return AuthTicketStore().consume(ticket)
|
||||
|
||||
|
||||
def build_superuser_token_payload() -> schemas.TokenPayload:
|
||||
"""从持久化用户和站点认证状态构造超级用户令牌载荷。"""
|
||||
user = UserOper().get_by_name(settings.SUPERUSER)
|
||||
if not user or not user.is_superuser:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="用户权限不足",
|
||||
)
|
||||
return schemas.TokenPayload(
|
||||
sub=user.id,
|
||||
username=user.name,
|
||||
super_user=user.is_superuser,
|
||||
level=SitesHelper().auth_level,
|
||||
purpose="authentication",
|
||||
)
|
||||
|
||||
|
||||
def build_token_response(user: User) -> schemas.Token:
|
||||
"""
|
||||
使用系统统一逻辑构造登录 Token 响应。
|
||||
|
||||
:param user: 已认证的本地用户
|
||||
:return: 标准 Token 响应
|
||||
"""
|
||||
level = SitesHelper().auth_level
|
||||
show_wizard = (
|
||||
not SystemConfigOper().get(SystemConfigKey.SetupWizardState)
|
||||
and not settings.ADVANCED_MODE
|
||||
)
|
||||
return schemas.Token(
|
||||
access_token=security.create_access_token(
|
||||
userid=user.id,
|
||||
username=user.name,
|
||||
super_user=user.is_superuser,
|
||||
expires_delta=timedelta(minutes=settings.ACCESS_TOKEN_EXPIRE_MINUTES),
|
||||
level=level,
|
||||
),
|
||||
token_type="bearer",
|
||||
super_user=user.is_superuser,
|
||||
user_id=user.id,
|
||||
user_name=user.name,
|
||||
avatar=user.avatar,
|
||||
level=level,
|
||||
permissions=user.permissions or {},
|
||||
wizard=show_wizard,
|
||||
)
|
||||
358
app/application/security/cookie.py
Normal file
358
app/application/security/cookie.py
Normal file
@@ -0,0 +1,358 @@
|
||||
import base64
|
||||
import time
|
||||
from typing import Tuple, Optional
|
||||
|
||||
from lxml import etree
|
||||
|
||||
from app.adapters.network.browser import BrowserPage, PlaywrightHelper
|
||||
from app.adapters.external.ocr import OcrHelper
|
||||
from app.application.security.twofactor import TwoFactorAuth
|
||||
from app.runtime.log import logger
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.domain.site import SiteUtils
|
||||
from app.domain.string import StringUtils
|
||||
|
||||
|
||||
class CookieHelper:
|
||||
"""处理站点登录表单、验证码和 Cookie 获取流程。"""
|
||||
|
||||
# 站点登录界面元素XPATH
|
||||
_SITE_LOGIN_XPATH = {
|
||||
"username": [
|
||||
'//input[@name="username"]',
|
||||
'//input[@id="form_item_username"]',
|
||||
'//input[@id="username"]',
|
||||
'//input[contains(@placeholder,"用户名")]',
|
||||
],
|
||||
"password": [
|
||||
'//input[@name="password"]',
|
||||
'//input[@id="form_item_password"]',
|
||||
'//input[@id="password"]',
|
||||
'//input[@type="password"]',
|
||||
],
|
||||
"captcha": [
|
||||
'//input[@name="imagestring"]',
|
||||
'//input[@name="captcha"]',
|
||||
'//input[@id="form_item_captcha"]',
|
||||
'//input[@placeholder="驗證碼"]',
|
||||
],
|
||||
"captcha_img": [
|
||||
'//img[@alt="captcha"]/@src',
|
||||
'//img[@alt="CAPTCHA"]/@src',
|
||||
'//img[@alt="SECURITY CODE"]/@src',
|
||||
'//img[@id="LAY-user-get-vercode"]/@src',
|
||||
'//img[contains(@src,"/api/getCaptcha")]/@src',
|
||||
],
|
||||
"submit": [
|
||||
'//input[@type="submit"]',
|
||||
'//button[@type="submit"]',
|
||||
'//button[@lay-filter="login"]',
|
||||
'//button[@lay-filter="formLogin"]',
|
||||
'//input[@type="button"][@value="登录"]',
|
||||
'//input[@id="submit-btn"]',
|
||||
],
|
||||
"error": [
|
||||
"//table[@class='main']//td[@class='text']/text()",
|
||||
],
|
||||
"remember": [
|
||||
'//input[@type="checkbox"][contains(@name,"remember") or contains(@id,"remember")]',
|
||||
'//*[@role="checkbox"][contains(.,"保持登录") or contains(.,"记住我") or contains(.,"自动登录")]',
|
||||
],
|
||||
"twostep": [
|
||||
'//input[@name="two_step_code"]',
|
||||
'//input[@name="2fa_secret"]',
|
||||
'//input[@name="otp"]',
|
||||
]
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_page_content(page: BrowserPage, retries: int = 3, interval: float = 1.0) -> Optional[str]:
|
||||
"""
|
||||
获取页面源码,页面跳转中(如登录前后的重定向)会导致 page.content() 抛出
|
||||
"Unable to retrieve content because the page is navigating" 异常,等待加载完成后重试
|
||||
:param page: 浏览器页面
|
||||
:param retries: 最大重试次数
|
||||
:param interval: 重试间隔(秒)
|
||||
:return: 页面源码
|
||||
"""
|
||||
for i in range(retries):
|
||||
# 等待加载失败不代表源码不可读取,最后一次等待失败时仍尝试直接获取源码
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=10 * 1000)
|
||||
except Exception as e:
|
||||
if i < retries - 1:
|
||||
logger.warning(f"等待页面加载完成失败:{str(e)},{interval}秒后重试 ({i + 1}/{retries - 1})")
|
||||
time.sleep(interval)
|
||||
continue
|
||||
logger.warning(f"等待页面加载完成失败:{str(e)},尝试直接获取源码")
|
||||
try:
|
||||
return page.content()
|
||||
except Exception as e:
|
||||
if i >= retries - 1:
|
||||
logger.error(f"获取页面源码失败:{str(e)}")
|
||||
return None
|
||||
logger.warning(f"获取页面源码失败:{str(e)},{interval}秒后重试 ({i + 1}/{retries - 1})")
|
||||
time.sleep(interval)
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def parse_cookies(cookies: list) -> str:
|
||||
"""
|
||||
将浏览器返回的cookies转化为字符串
|
||||
"""
|
||||
if not cookies:
|
||||
return ""
|
||||
cookie_str = ""
|
||||
for cookie in cookies:
|
||||
cookie_str += f"{cookie['name']}={cookie['value']}; "
|
||||
return cookie_str
|
||||
|
||||
def get_site_cookie_ua(self,
|
||||
url: str,
|
||||
username: str,
|
||||
password: str,
|
||||
two_step_code: Optional[str] = None,
|
||||
proxies: Optional[dict] = None,
|
||||
timeout: int = None) -> Tuple[Optional[str], Optional[str], str]:
|
||||
"""
|
||||
获取站点cookie和ua
|
||||
:param url: 站点地址
|
||||
:param username: 用户名
|
||||
:param password: 密码
|
||||
:param two_step_code: 二步验证码或密钥
|
||||
:param proxies: 代理
|
||||
:param timeout: 超时时间
|
||||
:return: cookie、ua、message
|
||||
"""
|
||||
|
||||
def __page_handler(page: BrowserPage) -> Tuple[Optional[str], Optional[str], str]:
|
||||
"""
|
||||
页面处理
|
||||
:return: Cookie和UA
|
||||
"""
|
||||
# 登录页面代码
|
||||
html_text = self.get_page_content(page)
|
||||
if not html_text:
|
||||
return None, None, "获取源码失败"
|
||||
# 查找用户名输入框
|
||||
html = etree.HTML(html_text)
|
||||
if html is None:
|
||||
return None, None, "解析网页源码失败"
|
||||
try:
|
||||
username_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("username"):
|
||||
if html.xpath(xpath):
|
||||
username_xpath = xpath
|
||||
break
|
||||
if not username_xpath:
|
||||
# 登录页可能为JS动态渲染(如SPA),等待用户名输入框出现后重试
|
||||
try:
|
||||
username_union_xpath = " | ".join(self._SITE_LOGIN_XPATH.get("username"))
|
||||
page.wait_for_selector(f"xpath={username_union_xpath}", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
html_text = self.get_page_content(page)
|
||||
html = etree.HTML(html_text) if html_text else None
|
||||
if html is None:
|
||||
return None, None, "解析网页源码失败"
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("username"):
|
||||
if html.xpath(xpath):
|
||||
username_xpath = xpath
|
||||
break
|
||||
if not username_xpath:
|
||||
return None, None, "未找到用户名输入框"
|
||||
# 查找密码输入框
|
||||
password_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("password"):
|
||||
if html.xpath(xpath):
|
||||
password_xpath = xpath
|
||||
break
|
||||
if not password_xpath:
|
||||
return None, None, "未找到密码输入框"
|
||||
# 处理二步验证码
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
# 查找二步验证码输入框
|
||||
twostep_xpath = None
|
||||
if otp_code:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
if html.xpath(xpath):
|
||||
twostep_xpath = xpath
|
||||
break
|
||||
# 查找验证码输入框
|
||||
captcha_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha"):
|
||||
if html.xpath(xpath):
|
||||
captcha_xpath = xpath
|
||||
break
|
||||
# 查找验证码图片
|
||||
captcha_img_url = None
|
||||
if captcha_xpath:
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("captcha_img"):
|
||||
if html.xpath(xpath):
|
||||
captcha_img_url = html.xpath(xpath)[0]
|
||||
break
|
||||
if not captcha_img_url:
|
||||
return None, None, "未找到验证码图片"
|
||||
# 查找登录按钮
|
||||
submit_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("submit"):
|
||||
if html.xpath(xpath):
|
||||
submit_xpath = xpath
|
||||
break
|
||||
if not submit_xpath:
|
||||
return None, None, "未找到登录按钮"
|
||||
|
||||
# 点击登录按钮
|
||||
try:
|
||||
# 等待登录按钮准备好
|
||||
page.wait_for_selector(submit_xpath)
|
||||
# 输入用户名
|
||||
page.fill(username_xpath, username)
|
||||
# 输入密码
|
||||
page.fill(password_xpath, password)
|
||||
# 勾选“记住我/保持登录”等选项,获取长期会话(部分站点默认发放短期会话)
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("remember"):
|
||||
remember_element = page.query_selector(xpath)
|
||||
if not remember_element:
|
||||
continue
|
||||
try:
|
||||
checked = remember_element.get_attribute("aria-checked")
|
||||
if checked is None:
|
||||
checked = "true" if remember_element.is_checked() else "false"
|
||||
if checked != "true":
|
||||
remember_element.click(timeout=3000)
|
||||
break
|
||||
except Exception as e:
|
||||
# 当前候选不可操作(如隐藏元素)时继续尝试后续候选
|
||||
logger.warning(f"勾选记住登录选项失败:{str(e)},尝试下一候选")
|
||||
continue
|
||||
# 输入二步验证码
|
||||
if twostep_xpath:
|
||||
page.fill(twostep_xpath, otp_code)
|
||||
# 识别验证码
|
||||
if captcha_xpath and captcha_img_url:
|
||||
captcha_element = page.query_selector(captcha_xpath)
|
||||
if captcha_element.is_visible():
|
||||
# 验证码图片地址
|
||||
code_url = self.__get_captcha_url(url, captcha_img_url)
|
||||
# 获取当前的cookie和ua
|
||||
cookie = self.parse_cookies(page.context.cookies())
|
||||
ua = page.evaluate("() => window.navigator.userAgent")
|
||||
# 自动OCR识别验证码
|
||||
captcha = self.__get_captcha_text(cookie=cookie, ua=ua, code_url=code_url)
|
||||
if captcha:
|
||||
logger.info("验证码地址为:%s,识别结果:%s" % (code_url, captcha))
|
||||
else:
|
||||
return None, None, "验证码识别失败"
|
||||
# 输入验证码
|
||||
captcha_element.fill(captcha)
|
||||
else:
|
||||
# 不可见元素不处理
|
||||
pass
|
||||
# 点击登录按钮
|
||||
page.click(submit_xpath)
|
||||
page.wait_for_load_state("networkidle", timeout=30 * 1000)
|
||||
except Exception as e:
|
||||
logger.error(f"仿真登录失败:{str(e)}")
|
||||
return None, None, f"仿真登录失败:{str(e)}"
|
||||
|
||||
# 对于某二次验证码为单页面的站点,输入二次验证码
|
||||
if "verify" in page.url:
|
||||
if not otp_code:
|
||||
return None, None, "需要二次验证码"
|
||||
html_text = self.get_page_content(page)
|
||||
if not html_text:
|
||||
return None, None, "获取网页源码失败"
|
||||
html = etree.HTML(html_text)
|
||||
if html is None:
|
||||
return None, None, "解析网页源码失败"
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("twostep"):
|
||||
if html.xpath(xpath):
|
||||
try:
|
||||
# 刷新一下 2fa code
|
||||
otp_code = TwoFactorAuth(two_step_code).get_code()
|
||||
page.fill(xpath, otp_code)
|
||||
# 登录按钮 xpath 理论上相同,不再重复查找
|
||||
page.click(submit_xpath)
|
||||
page.wait_for_load_state("networkidle", timeout=30 * 1000)
|
||||
except Exception as e:
|
||||
logger.error(f"二次验证码输入失败:{str(e)}")
|
||||
return None, None, f"二次验证码输入失败:{str(e)}"
|
||||
break
|
||||
|
||||
# 登录后的源码(部分站点登录成功后由前端脚本延迟跳转,等待并重试判定)
|
||||
html_text = None
|
||||
for i in range(3):
|
||||
if i:
|
||||
time.sleep(2)
|
||||
latest_text = self.get_page_content(page)
|
||||
if not latest_text:
|
||||
continue
|
||||
if SiteUtils.is_logged_in(latest_text):
|
||||
return self.parse_cookies(page.context.cookies()), \
|
||||
page.evaluate("() => window.navigator.userAgent"), ""
|
||||
# 保留首个快照用于失败时解析错误信息,避免提示被后续跳转或自动消失覆盖
|
||||
if html_text is None:
|
||||
html_text = latest_text
|
||||
# 页面已出现明确的登录错误信息时,以该快照为准并提前结束重试
|
||||
latest_html = etree.HTML(latest_text)
|
||||
if latest_html is not None and \
|
||||
any(latest_html.xpath(x) for x in self._SITE_LOGIN_XPATH.get("error")):
|
||||
html_text = latest_text
|
||||
break
|
||||
if not html_text:
|
||||
return None, None, "获取网页源码失败"
|
||||
else:
|
||||
# 从登录后的页面读取错误信息
|
||||
html = etree.HTML(html_text)
|
||||
if html is None:
|
||||
return None, None, "登录失败"
|
||||
error_xpath = None
|
||||
for xpath in self._SITE_LOGIN_XPATH.get("error"):
|
||||
if html.xpath(xpath):
|
||||
error_xpath = xpath
|
||||
break
|
||||
if not error_xpath:
|
||||
return None, None, "登录失败"
|
||||
else:
|
||||
error_msg = html.xpath(error_xpath)[0]
|
||||
return None, None, error_msg
|
||||
finally:
|
||||
if html:
|
||||
del html
|
||||
|
||||
if not url or not username or not password:
|
||||
return None, None, "参数错误"
|
||||
|
||||
return PlaywrightHelper().action(url=url,
|
||||
callback=__page_handler,
|
||||
proxies=proxies,
|
||||
timeout=timeout)
|
||||
|
||||
@staticmethod
|
||||
def __get_captcha_text(cookie: str, ua: str, code_url: str) -> str:
|
||||
"""
|
||||
识别验证码图片的内容
|
||||
"""
|
||||
if not code_url:
|
||||
return ""
|
||||
ret = RequestUtils(ua=ua, cookies=cookie).get_res(code_url)
|
||||
if ret:
|
||||
if not ret.content:
|
||||
return ""
|
||||
return OcrHelper().get_captcha_text(
|
||||
image_b64=base64.b64encode(ret.content).decode()
|
||||
)
|
||||
else:
|
||||
return ""
|
||||
|
||||
@staticmethod
|
||||
def __get_captcha_url(siteurl: str, imageurl: str) -> str:
|
||||
"""
|
||||
获取验证码图片的URL
|
||||
"""
|
||||
if not siteurl or not imageurl:
|
||||
return ""
|
||||
if imageurl.startswith("/"):
|
||||
imageurl = imageurl[1:]
|
||||
return "%s/%s" % (StringUtils.get_base_url(siteurl), imageurl)
|
||||
53
app/application/security/otp.py
Normal file
53
app/application/security/otp.py
Normal file
@@ -0,0 +1,53 @@
|
||||
from typing import Tuple
|
||||
|
||||
import pyotp
|
||||
|
||||
|
||||
class OtpUtils:
|
||||
"""提供基于 TOTP 的二次验证辅助能力。"""
|
||||
|
||||
@staticmethod
|
||||
def generate_secret_key(username: str) -> Tuple[str, str]:
|
||||
"""生成 TOTP 密钥及其配置 URI。"""
|
||||
try:
|
||||
secret = pyotp.random_base32()
|
||||
uri = pyotp.totp.TOTP(secret).provisioning_uri(name='MoviePilot',
|
||||
issuer_name='MoviePilot(' + username + ')')
|
||||
return secret, uri
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return "", ""
|
||||
|
||||
@staticmethod
|
||||
def is_legal(otp_uri: str, password: str) -> bool:
|
||||
"""
|
||||
校验二次验证是否正确
|
||||
"""
|
||||
try:
|
||||
return pyotp.TOTP(pyotp.parse_uri(otp_uri).secret).verify(password)
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def check(secret: str, password: str) -> bool:
|
||||
"""
|
||||
校验二次验证是否正确
|
||||
"""
|
||||
try:
|
||||
totp = pyotp.TOTP(secret)
|
||||
return totp.verify(password)
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_secret(otp_uri: str) -> str:
|
||||
"""
|
||||
获取uri中的secret
|
||||
"""
|
||||
try:
|
||||
return pyotp.parse_uri(otp_uri).secret
|
||||
except Exception as err:
|
||||
print(str(err))
|
||||
return ""
|
||||
451
app/application/security/passkey.py
Normal file
451
app/application/security/passkey.py
Normal file
@@ -0,0 +1,451 @@
|
||||
"""
|
||||
PassKey WebAuthn 辅助工具类
|
||||
"""
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import secrets
|
||||
import threading
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Dict, List, Literal, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from webauthn import (
|
||||
generate_registration_options,
|
||||
verify_registration_response,
|
||||
generate_authentication_options,
|
||||
verify_authentication_response,
|
||||
options_to_json
|
||||
)
|
||||
from webauthn.helpers import (
|
||||
parse_registration_credential_json,
|
||||
parse_authentication_credential_json
|
||||
)
|
||||
from webauthn.helpers.structs import (
|
||||
PublicKeyCredentialDescriptor,
|
||||
AuthenticatorTransport,
|
||||
UserVerificationRequirement,
|
||||
ResidentKeyRequirement,
|
||||
AuthenticatorSelectionCriteria
|
||||
)
|
||||
from webauthn.helpers.cose import COSEAlgorithmIdentifier
|
||||
from webauthn.helpers.exceptions import InvalidRegistrationResponse
|
||||
|
||||
from app.runtime.cache import TTLCache
|
||||
from app.runtime.config import settings
|
||||
from app.adapters.cache.redis import RedisHelper
|
||||
from app.runtime.log import logger
|
||||
|
||||
PASSKEY_CHALLENGE_TTL_SECONDS = 5 * 60
|
||||
PasskeyChallengePurpose = Literal["authentication", "registration"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PasskeyChallenge:
|
||||
"""服务端保存的一次性 Passkey challenge 及其认证边界。"""
|
||||
|
||||
challenge: str
|
||||
purpose: PasskeyChallengePurpose
|
||||
user_id: Optional[int]
|
||||
|
||||
|
||||
class PasskeyChallengeStore:
|
||||
"""使用当前缓存后端签发并原子消费短时 Passkey challenge。"""
|
||||
|
||||
_cache = TTLCache(
|
||||
region="passkey_challenge",
|
||||
maxsize=4096,
|
||||
ttl=PASSKEY_CHALLENGE_TTL_SECONDS,
|
||||
)
|
||||
_memory_consume_lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def issue(
|
||||
cls,
|
||||
*,
|
||||
challenge: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
user_id: Optional[int],
|
||||
) -> str:
|
||||
"""保存 challenge 并返回不携带认证事实的随机事务 token。"""
|
||||
transaction_token = secrets.token_urlsafe(32)
|
||||
cls._cache.set(
|
||||
transaction_token,
|
||||
PasskeyChallenge(
|
||||
challenge=challenge,
|
||||
purpose=purpose,
|
||||
user_id=user_id,
|
||||
),
|
||||
)
|
||||
return transaction_token
|
||||
|
||||
@classmethod
|
||||
def consume(
|
||||
cls,
|
||||
*,
|
||||
transaction_token: str,
|
||||
purpose: PasskeyChallengePurpose,
|
||||
) -> Optional[PasskeyChallenge]:
|
||||
"""原子领取 challenge;任何完成尝试都会使事务失效。"""
|
||||
if not transaction_token:
|
||||
return None
|
||||
|
||||
if cls._cache.is_redis():
|
||||
challenge = RedisHelper().pop(
|
||||
transaction_token,
|
||||
region="passkey_challenge",
|
||||
)
|
||||
else:
|
||||
with cls._memory_consume_lock:
|
||||
try:
|
||||
challenge = cls._cache.pop(transaction_token)
|
||||
except KeyError:
|
||||
challenge = None
|
||||
|
||||
if not isinstance(challenge, PasskeyChallenge):
|
||||
return None
|
||||
if challenge.purpose != purpose:
|
||||
return None
|
||||
return challenge
|
||||
|
||||
|
||||
class PassKeyRegistrationVerificationError(Exception):
|
||||
"""Passkey 注册响应未通过 WebAuthn 安全校验。"""
|
||||
|
||||
|
||||
class PassKeyRegistrationOriginMismatchError(PassKeyRegistrationVerificationError):
|
||||
"""浏览器来源与系统配置的 Passkey 注册来源不一致。"""
|
||||
|
||||
|
||||
class PassKeyHelper:
|
||||
"""
|
||||
PassKey WebAuthn 辅助类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_rp_id() -> str:
|
||||
"""
|
||||
获取 Relying Party ID
|
||||
"""
|
||||
if settings.APP_DOMAIN:
|
||||
app_domain = settings.APP_DOMAIN.strip()
|
||||
# 确保存在协议前缀,以便 urlparse 正确解析主机和端口
|
||||
if not app_domain.startswith(('http://', 'https://')):
|
||||
app_domain = f'https://{app_domain}'
|
||||
parsed = urlparse(app_domain)
|
||||
host = parsed.hostname
|
||||
if host:
|
||||
return host
|
||||
# 从 APP_DOMAIN 中提取域名
|
||||
host = settings.APP_DOMAIN.replace('https://', '').replace('http://', '')
|
||||
# 移除端口号
|
||||
if ':' in host:
|
||||
host = host.split(':')[0]
|
||||
return host
|
||||
# 只有在未配置 APP_DOMAIN 时,才默认为 localhost
|
||||
return 'localhost'
|
||||
|
||||
@staticmethod
|
||||
def get_rp_name() -> str:
|
||||
"""
|
||||
获取 Relying Party 名称
|
||||
"""
|
||||
return "MoviePilot"
|
||||
|
||||
@staticmethod
|
||||
def get_origin() -> str:
|
||||
"""
|
||||
获取源地址
|
||||
"""
|
||||
if settings.APP_DOMAIN:
|
||||
return settings.APP_DOMAIN.rstrip('/')
|
||||
# 如果未配置APP_DOMAIN,使用默认的localhost地址
|
||||
return f'http://localhost:{settings.NGINX_PORT}'
|
||||
|
||||
@staticmethod
|
||||
def standardize_credential_id(credential_id: str) -> str:
|
||||
"""
|
||||
标准化凭证ID(Base64 URL Safe)
|
||||
"""
|
||||
try:
|
||||
# Base64解码并重新编码以标准化格式
|
||||
decoded = base64.urlsafe_b64decode(credential_id + '==')
|
||||
return base64.urlsafe_b64encode(decoded).decode('utf-8').rstrip('=')
|
||||
except (binascii.Error, TypeError, ValueError) as e:
|
||||
logger.error(f"标准化凭证ID失败: {e}")
|
||||
return credential_id
|
||||
|
||||
@staticmethod
|
||||
def _base64_encode_urlsafe(data: bytes) -> str:
|
||||
"""
|
||||
Base64 URL Safe 编码(不带填充)
|
||||
|
||||
:param data: 要编码的字节数据
|
||||
:return: Base64 URL Safe 编码的字符串
|
||||
"""
|
||||
return base64.urlsafe_b64encode(data).decode('utf-8').rstrip('=')
|
||||
|
||||
@staticmethod
|
||||
def _base64_decode_urlsafe(data: str) -> bytes:
|
||||
"""
|
||||
Base64 URL Safe 解码(自动添加填充)
|
||||
|
||||
:param data: Base64 URL Safe 编码的字符串
|
||||
:return: 解码后的字节数据
|
||||
"""
|
||||
return base64.urlsafe_b64decode(data + '==')
|
||||
|
||||
@staticmethod
|
||||
def _parse_credential_list(credentials: List[Dict[str, Any]]) -> List[PublicKeyCredentialDescriptor]:
|
||||
"""
|
||||
解析凭证列表为 PublicKeyCredentialDescriptor 列表
|
||||
|
||||
:param credentials: 凭证字典列表
|
||||
:return: PublicKeyCredentialDescriptor 列表
|
||||
"""
|
||||
result = []
|
||||
for cred in credentials:
|
||||
try:
|
||||
result.append(
|
||||
PublicKeyCredentialDescriptor(
|
||||
id=PassKeyHelper._base64_decode_urlsafe(cred['credential_id']),
|
||||
transports=[
|
||||
AuthenticatorTransport(t) for t in cred.get('transports', '').split(',') if t
|
||||
] if cred.get('transports') else None
|
||||
)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"解析凭证失败: {e}")
|
||||
continue
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _get_user_verification_requirement(user_verification: Optional[str] = None) -> UserVerificationRequirement:
|
||||
"""
|
||||
获取用户验证要求
|
||||
|
||||
:param user_verification: 指定的用户验证要求,如果不指定则从配置中读取
|
||||
:return: UserVerificationRequirement
|
||||
"""
|
||||
if user_verification:
|
||||
return UserVerificationRequirement(user_verification)
|
||||
return UserVerificationRequirement.REQUIRED if settings.PASSKEY_REQUIRE_UV \
|
||||
else UserVerificationRequirement.PREFERRED
|
||||
|
||||
@staticmethod
|
||||
def _get_verification_params(
|
||||
expected_origin: Optional[str] = None,
|
||||
expected_rp_id: Optional[str] = None
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
获取验证参数(origin 和 rp_id)
|
||||
|
||||
:param expected_origin: 期望的源地址
|
||||
:param expected_rp_id: 期望的RP ID
|
||||
:return: (origin, rp_id)
|
||||
"""
|
||||
origin = expected_origin or PassKeyHelper.get_origin()
|
||||
rp_id = expected_rp_id or PassKeyHelper.get_rp_id()
|
||||
return origin, rp_id
|
||||
|
||||
@staticmethod
|
||||
def generate_registration_options(
|
||||
user_id: int,
|
||||
username: str,
|
||||
display_name: Optional[str] = None,
|
||||
existing_credentials: Optional[List[Dict[str, Any]]] = None
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
生成注册选项
|
||||
|
||||
:param user_id: 用户ID
|
||||
:param username: 用户名
|
||||
:param display_name: 显示名称
|
||||
:param existing_credentials: 已存在的凭证列表
|
||||
:return: (options_json, challenge)
|
||||
"""
|
||||
try:
|
||||
# 用户信息
|
||||
user_id_bytes = str(user_id).encode('utf-8')
|
||||
|
||||
# 排除已有的凭证
|
||||
exclude_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
|
||||
if existing_credentials else None
|
||||
|
||||
# 用户验证要求
|
||||
uv_requirement = PassKeyHelper._get_user_verification_requirement()
|
||||
|
||||
# 生成注册选项
|
||||
options = generate_registration_options(
|
||||
rp_id=PassKeyHelper.get_rp_id(),
|
||||
rp_name=PassKeyHelper.get_rp_name(),
|
||||
user_id=user_id_bytes,
|
||||
user_name=username,
|
||||
user_display_name=display_name or username,
|
||||
exclude_credentials=exclude_credentials,
|
||||
authenticator_selection=AuthenticatorSelectionCriteria(
|
||||
authenticator_attachment=None,
|
||||
resident_key=ResidentKeyRequirement.REQUIRED,
|
||||
user_verification=uv_requirement,
|
||||
),
|
||||
supported_pub_key_algs=[
|
||||
COSEAlgorithmIdentifier.ECDSA_SHA_256,
|
||||
COSEAlgorithmIdentifier.RSASSA_PKCS1_v1_5_SHA_256,
|
||||
]
|
||||
)
|
||||
|
||||
# 转换为JSON
|
||||
options_json = options_to_json(options)
|
||||
|
||||
# 提取challenge(用于后续验证)
|
||||
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
|
||||
|
||||
return options_json, challenge
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成注册选项失败: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def verify_registration_response(
|
||||
credential: Dict[str, Any],
|
||||
expected_challenge: str,
|
||||
expected_origin: Optional[str] = None,
|
||||
expected_rp_id: Optional[str] = None
|
||||
) -> Tuple[str, str, int, Optional[str]]:
|
||||
"""
|
||||
验证注册响应
|
||||
|
||||
:param credential: 客户端返回的凭证
|
||||
:param expected_challenge: 期望的challenge
|
||||
:param expected_origin: 期望的源地址
|
||||
:param expected_rp_id: 期望的RP ID
|
||||
:return: (credential_id, public_key, sign_count, aaguid)
|
||||
"""
|
||||
try:
|
||||
# 准备验证参数
|
||||
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
|
||||
# 解码challenge
|
||||
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
|
||||
|
||||
# 构建RegistrationCredential对象
|
||||
registration_credential = parse_registration_credential_json(json.dumps(credential))
|
||||
|
||||
# 验证注册响应
|
||||
verification = verify_registration_response(
|
||||
credential=registration_credential,
|
||||
expected_challenge=challenge_bytes,
|
||||
expected_rp_id=rp_id,
|
||||
expected_origin=origin,
|
||||
require_user_verification=settings.PASSKEY_REQUIRE_UV
|
||||
)
|
||||
|
||||
# 提取信息
|
||||
credential_id = PassKeyHelper._base64_encode_urlsafe(verification.credential_id)
|
||||
public_key = PassKeyHelper._base64_encode_urlsafe(verification.credential_public_key)
|
||||
sign_count = verification.sign_count
|
||||
# aaguid 可能已经是字符串格式,也可能是bytes
|
||||
if verification.aaguid:
|
||||
if isinstance(verification.aaguid, bytes):
|
||||
aaguid = verification.aaguid.hex()
|
||||
else:
|
||||
aaguid = str(verification.aaguid)
|
||||
else:
|
||||
aaguid = None
|
||||
|
||||
return credential_id, public_key, sign_count, aaguid
|
||||
|
||||
except InvalidRegistrationResponse as e:
|
||||
logger.error(f"验证注册响应失败: {e}")
|
||||
if str(e).startswith("Unexpected client data origin "):
|
||||
raise PassKeyRegistrationOriginMismatchError() from e
|
||||
raise PassKeyRegistrationVerificationError() from e
|
||||
except Exception as e:
|
||||
logger.error(f"验证注册响应失败: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def generate_authentication_options(
|
||||
existing_credentials: Optional[List[Dict[str, Any]]] = None,
|
||||
user_verification: Optional[str] = None
|
||||
) -> Tuple[str, str]:
|
||||
"""
|
||||
生成认证选项
|
||||
|
||||
:param existing_credentials: 已存在的凭证列表(用于限制可用凭证)
|
||||
:param user_verification: 用户验证要求,如果不指定则从配置中读取
|
||||
:return: (options_json, challenge)
|
||||
"""
|
||||
try:
|
||||
# 允许的凭证
|
||||
allow_credentials = PassKeyHelper._parse_credential_list(existing_credentials) \
|
||||
if existing_credentials else None
|
||||
|
||||
# 用户验证要求
|
||||
uv_requirement = PassKeyHelper._get_user_verification_requirement(user_verification)
|
||||
|
||||
# 生成认证选项
|
||||
options = generate_authentication_options(
|
||||
rp_id=PassKeyHelper.get_rp_id(),
|
||||
allow_credentials=allow_credentials,
|
||||
user_verification=uv_requirement
|
||||
)
|
||||
|
||||
# 转换为JSON
|
||||
options_json = options_to_json(options)
|
||||
|
||||
# 提取challenge
|
||||
challenge = PassKeyHelper._base64_encode_urlsafe(options.challenge)
|
||||
|
||||
return options_json, challenge
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"生成认证选项失败: {e}")
|
||||
raise
|
||||
|
||||
@staticmethod
|
||||
def verify_authentication_response(
|
||||
credential: Dict[str, Any],
|
||||
expected_challenge: str,
|
||||
credential_public_key: str,
|
||||
credential_current_sign_count: int,
|
||||
expected_origin: Optional[str] = None,
|
||||
expected_rp_id: Optional[str] = None
|
||||
) -> Tuple[bool, int]:
|
||||
"""
|
||||
验证认证响应
|
||||
|
||||
:param credential: 客户端返回的凭证
|
||||
:param expected_challenge: 期望的challenge
|
||||
:param credential_public_key: 凭证公钥
|
||||
:param credential_current_sign_count: 当前签名计数
|
||||
:param expected_origin: 期望的源地址
|
||||
:param expected_rp_id: 期望的RP ID
|
||||
:return: (验证成功, 新的签名计数)
|
||||
"""
|
||||
try:
|
||||
# 准备验证参数
|
||||
origin, rp_id = PassKeyHelper._get_verification_params(expected_origin, expected_rp_id)
|
||||
# 解码
|
||||
challenge_bytes = PassKeyHelper._base64_decode_urlsafe(expected_challenge)
|
||||
public_key_bytes = PassKeyHelper._base64_decode_urlsafe(credential_public_key)
|
||||
|
||||
# 构建AuthenticationCredential对象
|
||||
authentication_credential = parse_authentication_credential_json(json.dumps(credential))
|
||||
|
||||
# 验证认证响应
|
||||
verification = verify_authentication_response(
|
||||
credential=authentication_credential,
|
||||
expected_challenge=challenge_bytes,
|
||||
expected_rp_id=rp_id,
|
||||
expected_origin=origin,
|
||||
credential_public_key=public_key_bytes,
|
||||
credential_current_sign_count=credential_current_sign_count,
|
||||
require_user_verification=settings.PASSKEY_REQUIRE_UV
|
||||
)
|
||||
|
||||
return True, verification.new_sign_count
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"验证认证响应失败: {e}")
|
||||
return False, credential_current_sign_count
|
||||
48
app/application/security/twofactor.py
Normal file
48
app/application/security/twofactor.py
Normal file
@@ -0,0 +1,48 @@
|
||||
import base64
|
||||
import hashlib
|
||||
import hmac
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
|
||||
from app.runtime.log import logger
|
||||
|
||||
|
||||
class TwoFactorAuth:
|
||||
"""解析已有验证码或根据共享密钥生成 TOTP 验证码。"""
|
||||
|
||||
def __init__(self, code_or_secret: str):
|
||||
"""按长度区分用户验证码和 TOTP 共享密钥。"""
|
||||
if code_or_secret and len(code_or_secret) >= 16:
|
||||
self.code = None
|
||||
self.secret = code_or_secret
|
||||
else:
|
||||
self.code = code_or_secret
|
||||
self.secret = None
|
||||
|
||||
@staticmethod
|
||||
def __calc(secret_key: str) -> str:
|
||||
"""按 30 秒时间窗计算六位 TOTP 验证码。"""
|
||||
if not secret_key:
|
||||
return ""
|
||||
try:
|
||||
input_time = int(time.time()) // 30
|
||||
key = base64.b32decode(secret_key)
|
||||
msg = struct.pack(">Q", input_time)
|
||||
google_code = hmac.new(key, msg, hashlib.sha1).digest()
|
||||
o = (
|
||||
google_code[19] & 15
|
||||
if sys.version_info > (2, 7)
|
||||
else ord(str(google_code[19])) & 15
|
||||
)
|
||||
google_code = str(
|
||||
(struct.unpack(">I", google_code[o: o + 4])[0] & 0x7FFFFFFF) % 1000000
|
||||
)
|
||||
return f"0{google_code}" if len(google_code) == 5 else google_code
|
||||
except Exception as e:
|
||||
logger.error(f"计算动态验证码失败:{str(e)}")
|
||||
return ""
|
||||
|
||||
def get_code(self) -> str:
|
||||
"""返回显式验证码,或从共享密钥实时计算。"""
|
||||
return self.code or self.__calc(self.secret)
|
||||
983
app/application/security/url.py
Normal file
983
app/application/security/url.py
Normal file
@@ -0,0 +1,983 @@
|
||||
import asyncio
|
||||
import hmac
|
||||
import ipaddress
|
||||
import socket
|
||||
import threading
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from typing import Dict, Iterable, List, Optional, Set, Union
|
||||
from urllib.parse import parse_qsl, quote, urlencode, urlparse, urlunparse
|
||||
|
||||
from anyio import Path as AsyncPath
|
||||
from cachetools import TTLCache
|
||||
|
||||
from app.runtime.config import settings
|
||||
from app.runtime.log import logger
|
||||
from app.runtime.coalesce import (
|
||||
CoalesceDecision,
|
||||
CoalesceSummary,
|
||||
EventCoalescer,
|
||||
)
|
||||
|
||||
|
||||
# DNS 解析结果缓存。
|
||||
# 正向缓存 TTL 选择 120s,短于常见 CDN / fake-ip 的 DNS TTL,避免长期持有失效 IP;
|
||||
# 负向缓存 TTL 选择 15s,避免临时解析失败把目标长时间拉黑。
|
||||
_DNS_CACHE_MAXSIZE = 1024
|
||||
_DNS_CACHE_TTL_POSITIVE = 120
|
||||
_DNS_CACHE_TTL_NEGATIVE = 15
|
||||
_dns_positive_cache: "TTLCache[str, List[ipaddress._BaseAddress]]" = TTLCache(
|
||||
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_POSITIVE
|
||||
)
|
||||
_dns_negative_cache: "TTLCache[str, bool]" = TTLCache(
|
||||
maxsize=_DNS_CACHE_MAXSIZE, ttl=_DNS_CACHE_TTL_NEGATIVE
|
||||
)
|
||||
# 同步路径下保护 TTLCache 读写:`cachetools.TTLCache` 本身非线程安全。
|
||||
# 锁只覆盖缓存读写,不包 `getaddrinfo`,避免把 DNS 查询本身串行化。
|
||||
_dns_cache_lock = threading.Lock()
|
||||
# 同 hostname 的并发异步解析去重:同一 hostname 首次未命中时建立锁,
|
||||
# 后续并发请求 await 同一把锁,避免对同一目标重复发起 `getaddrinfo`。
|
||||
_dns_inflight_locks: Dict[str, asyncio.Lock] = {}
|
||||
_dns_inflight_meta_lock = threading.Lock()
|
||||
|
||||
|
||||
class UrlSafetyReason(str, Enum):
|
||||
"""
|
||||
`evaluate_url_safety` 返回的诊断原因枚举。
|
||||
|
||||
成员值为稳定的小写蛇形字符串,可直接作为日志字段或告警标签使用,
|
||||
扩展枚举时保留既有成员的取值,避免破坏下游聚合系统对原因的归类。
|
||||
"""
|
||||
|
||||
# 通过全部校验,URL 可被请求
|
||||
ALLOWED = "allowed"
|
||||
# 协议非 http/https,或 netloc 无效,或域名不在允许列表内
|
||||
DOMAIN_NOT_ALLOWED = "domain_not_allowed"
|
||||
# 已通过域名 allowlist,但 DNS 解析失败(无返回或抛错)
|
||||
DNS_RESOLUTION_FAILED = "dns_resolution_failed"
|
||||
# DNS 解析到至少一个非公网地址,且未配置 `allowed_private_ranges`
|
||||
NON_GLOBAL_DNS_RESULT = "non_global_dns_result"
|
||||
# 配置了 `allowed_private_ranges`,但仍存在不在允许网段内的解析结果
|
||||
MIXED_OR_DISALLOWED_PRIVATE_RESULT = "mixed_or_disallowed_private_result"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class UrlSafetyDiagnosis:
|
||||
"""
|
||||
URL 安全校验的结构化诊断结果,由 `evaluate_url_safety(_async)` 返回。
|
||||
|
||||
`is_safe_url` 仅使用 `allowed` 字段;日志、告警、运维诊断需要细分原因或
|
||||
解析 IP 时通过本对象消费。字段约束:
|
||||
- `host` 仅在通过域名 allowlist 后才被填充;DOMAIN_NOT_ALLOWED 场景为 None。
|
||||
- `ips` 仅在执行过 DNS 阶段后才可能非空;不含纯字符串协议失败场景。
|
||||
- `matched_private_ranges` 仅在通过 `allowed_private_ranges` 放行时填充。
|
||||
"""
|
||||
|
||||
# 是否放行
|
||||
allowed: bool
|
||||
# 放行/拦截的具体原因
|
||||
reason: UrlSafetyReason
|
||||
# 通过 allowlist 后从 URL 解析出的 hostname,未通过时为 None
|
||||
host: Optional[str] = None
|
||||
# DNS 解析结果(含命中或未命中私网放行的 IP),格式化为字符串
|
||||
ips: List[str] = field(default_factory=list)
|
||||
# 命中允许放行的非公网网段,仅 `ALLOWED` 且走私网放行分支时非空
|
||||
matched_private_ranges: List[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _resolve_addrinfo_to_ips(
|
||||
address_infos: Iterable,
|
||||
) -> Optional[List[ipaddress._BaseAddress]]:
|
||||
"""
|
||||
将 `socket.getaddrinfo` 返回的结果归一化为 IP 列表。
|
||||
|
||||
任一条目无法解析为 IP 即视为异常情况,整体返回 None 让上层按"不安全目标"
|
||||
处理,避免出现"部分 IP 漏校验"的情况。
|
||||
"""
|
||||
addresses: List[ipaddress._BaseAddress] = []
|
||||
for address_info in address_infos:
|
||||
try:
|
||||
addresses.append(ipaddress.ip_address(address_info[4][0]))
|
||||
except ValueError:
|
||||
return None
|
||||
return addresses or None
|
||||
|
||||
|
||||
class SecurityUtils:
|
||||
"""提供路径、URL、签名和网络目标安全校验能力。"""
|
||||
|
||||
_SIGNED_URL_PURPOSE = "image-proxy"
|
||||
_SUBTITLE_DOWNLOAD_PURPOSE_PREFIX = "subtitle-download"
|
||||
|
||||
@staticmethod
|
||||
def is_safe_path(base_path: Path, user_path: Path,
|
||||
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
|
||||
"""
|
||||
验证用户提供的路径是否在基准目录内,并检查文件类型是否合法,防止目录遍历攻击
|
||||
|
||||
:param base_path: 基准目录,允许访问的根目录
|
||||
:param user_path: 用户提供的路径,需检查其是否位于基准目录内
|
||||
:param allowed_suffixes: 允许的文件后缀名集合,用于验证文件类型
|
||||
:return: 如果用户路径安全且位于基准目录内,且文件类型合法,返回 True;否则返回 False
|
||||
:raises Exception: 如果解析路径时发生错误,则捕获并记录异常
|
||||
"""
|
||||
try:
|
||||
# resolve() 将相对路径转换为绝对路径,并处理符号链接和'..'
|
||||
base_path_resolved = base_path.resolve()
|
||||
user_path_resolved = user_path.resolve()
|
||||
|
||||
# 检查用户路径是否在基准目录或基准目录的子目录内
|
||||
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
|
||||
return False
|
||||
|
||||
if allowed_suffixes is not None:
|
||||
allowed_suffixes = set(allowed_suffixes)
|
||||
if user_path.suffix.lower() not in allowed_suffixes:
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error occurred while validating paths: {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
async def async_is_safe_path(base_path: AsyncPath, user_path: AsyncPath,
|
||||
allowed_suffixes: Optional[Union[Set[str], List[str]]] = None) -> bool:
|
||||
"""
|
||||
异步验证用户提供的路径是否在基准目录内,并检查文件类型是否合法,防止目录遍历攻击
|
||||
|
||||
:param base_path: 基准目录,允许访问的根目录
|
||||
:param user_path: 用户提供的路径,需检查其是否位于基准目录内
|
||||
:param allowed_suffixes: 允许的文件后缀名集合,用于验证文件类型
|
||||
:return: 如果用户路径安全且位于基准目录内,且文件类型合法,返回 True;否则返回 False
|
||||
:raises Exception: 如果解析路径时发生错误,则捕获并记录异常
|
||||
"""
|
||||
try:
|
||||
# resolve() 将相对路径转换为绝对路径,并处理符号链接和'..'
|
||||
base_path_resolved = await base_path.resolve()
|
||||
user_path_resolved = await user_path.resolve()
|
||||
|
||||
# 检查用户路径是否在基准目录或基准目录的子目录内
|
||||
if base_path_resolved != user_path_resolved and base_path_resolved not in user_path_resolved.parents:
|
||||
return False
|
||||
|
||||
if allowed_suffixes is not None:
|
||||
allowed_suffixes = set(allowed_suffixes)
|
||||
if user_path.suffix.lower() not in allowed_suffixes:
|
||||
return False
|
||||
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.debug(f"Error occurred while validating paths: {e}")
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _literal_ip(hostname: str) -> Optional[ipaddress._BaseAddress]:
|
||||
"""
|
||||
若 hostname 是字面量 IP(含 IPv6 的 `[::1]` 形式)则返回 IP 对象,否则 None。
|
||||
"""
|
||||
if not hostname:
|
||||
return None
|
||||
candidate = hostname
|
||||
if candidate.startswith("[") and candidate.endswith("]"):
|
||||
candidate = candidate[1:-1]
|
||||
try:
|
||||
return ipaddress.ip_address(candidate)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _cache_lookup(hostname: str) -> tuple[bool, Optional[List[ipaddress._BaseAddress]]]:
|
||||
"""
|
||||
在 TTL 缓存中查找 hostname,返回 (是否命中, 命中值)。
|
||||
|
||||
命中值为 `None` 表示命中负向缓存(先前解析失败)。
|
||||
"""
|
||||
with _dns_cache_lock:
|
||||
cached = _dns_positive_cache.get(hostname)
|
||||
if cached is not None:
|
||||
return True, cached
|
||||
if hostname in _dns_negative_cache:
|
||||
return True, None
|
||||
return False, None
|
||||
|
||||
@staticmethod
|
||||
def _cache_store(
|
||||
hostname: str, addresses: Optional[List[ipaddress._BaseAddress]]
|
||||
) -> None:
|
||||
"""
|
||||
将解析结果写入对应的正向/负向缓存。
|
||||
"""
|
||||
with _dns_cache_lock:
|
||||
if addresses is None:
|
||||
_dns_negative_cache[hostname] = True
|
||||
else:
|
||||
_dns_positive_cache[hostname] = addresses
|
||||
|
||||
@staticmethod
|
||||
def _hostname_addresses(hostname: str) -> Optional[List[ipaddress._BaseAddress]]:
|
||||
"""
|
||||
同步解析主机名并返回全部 IP 地址,结果走 TTL 缓存。
|
||||
|
||||
字面量 IP 直接返回自身;DNS 解析失败或结果异常时返回 None,由上层按
|
||||
不安全目标处理。async 调用方应使用 `_hostname_addresses_async`。
|
||||
"""
|
||||
if not hostname:
|
||||
return None
|
||||
literal = SecurityUtils._literal_ip(hostname)
|
||||
if literal is not None:
|
||||
return [literal]
|
||||
|
||||
hit, value = SecurityUtils._cache_lookup(hostname)
|
||||
if hit:
|
||||
return value
|
||||
|
||||
try:
|
||||
address_infos = socket.getaddrinfo(hostname, None, type=socket.SOCK_STREAM)
|
||||
except socket.gaierror:
|
||||
SecurityUtils._cache_store(hostname, None)
|
||||
return None
|
||||
addresses = _resolve_addrinfo_to_ips(address_infos)
|
||||
SecurityUtils._cache_store(hostname, addresses)
|
||||
return addresses
|
||||
|
||||
@staticmethod
|
||||
def _get_inflight_lock(hostname: str) -> asyncio.Lock:
|
||||
"""
|
||||
取得 hostname 对应的 in-flight 锁,不存在则按需创建。
|
||||
|
||||
用 `threading.Lock` 保护字典写入,避免多个事件循环线程并发创建出多把锁
|
||||
破坏去重语义;锁本身是 `asyncio.Lock`,归属当前事件循环。
|
||||
"""
|
||||
with _dns_inflight_meta_lock:
|
||||
lock = _dns_inflight_locks.get(hostname)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_dns_inflight_locks[hostname] = lock
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
def _release_inflight_lock(hostname: str, lock: asyncio.Lock) -> None:
|
||||
"""
|
||||
请求结束后清理 in-flight 锁,避免长期持有大量已闲置的 `asyncio.Lock`。
|
||||
|
||||
仅当字典中登记的仍是当前 lock,且 `lock.locked()` 为 False 时才删除。
|
||||
`asyncio.Lock` 公平 FIFO:持有者释放后若仍有等待者,锁会立刻被下一个
|
||||
等待者接走、`locked()` 重新变为 True,因此该守卫可同时排除"仍有持有者"
|
||||
与"刚被等待者接走"两种情况,避免误删后续协程仍在使用的字典条目。
|
||||
"""
|
||||
with _dns_inflight_meta_lock:
|
||||
current = _dns_inflight_locks.get(hostname)
|
||||
if current is lock and not lock.locked():
|
||||
_dns_inflight_locks.pop(hostname, None)
|
||||
|
||||
@staticmethod
|
||||
async def _hostname_addresses_async(
|
||||
hostname: str,
|
||||
) -> Optional[List[ipaddress._BaseAddress]]:
|
||||
"""
|
||||
异步解析主机名并返回全部 IP 地址,与同步版本共用同一份 TTL 缓存。
|
||||
|
||||
通过事件循环的默认线程池执行 `getaddrinfo`,不阻塞 asyncio 事件循环;
|
||||
同 hostname 的并发未命中请求通过 in-flight 锁去重,只发起一次 DNS 查询。
|
||||
"""
|
||||
if not hostname:
|
||||
return None
|
||||
literal = SecurityUtils._literal_ip(hostname)
|
||||
if literal is not None:
|
||||
return [literal]
|
||||
|
||||
hit, value = SecurityUtils._cache_lookup(hostname)
|
||||
if hit:
|
||||
return value
|
||||
|
||||
lock = SecurityUtils._get_inflight_lock(hostname)
|
||||
try:
|
||||
async with lock:
|
||||
# 等到锁后再查一次缓存,前一个持锁者可能已经回填结果
|
||||
hit, value = SecurityUtils._cache_lookup(hostname)
|
||||
if hit:
|
||||
return value
|
||||
|
||||
loop = asyncio.get_running_loop()
|
||||
try:
|
||||
address_infos = await loop.getaddrinfo(
|
||||
hostname, None, type=socket.SOCK_STREAM
|
||||
)
|
||||
except socket.gaierror:
|
||||
SecurityUtils._cache_store(hostname, None)
|
||||
return None
|
||||
addresses = _resolve_addrinfo_to_ips(address_infos)
|
||||
SecurityUtils._cache_store(hostname, addresses)
|
||||
return addresses
|
||||
finally:
|
||||
# 必须在 `async with` 释放锁之后再清理字典:`_release_inflight_lock`
|
||||
# 以 `not lock.locked()` 为清理守卫,持锁状态下调用会跳过 pop。
|
||||
SecurityUtils._release_inflight_lock(hostname, lock)
|
||||
|
||||
@staticmethod
|
||||
def _addresses_all_global(
|
||||
addresses: Optional[List[ipaddress._BaseAddress]],
|
||||
) -> bool:
|
||||
"""
|
||||
判断解析结果是否全部为公网地址(空列表/None 视为非公网)。
|
||||
"""
|
||||
if not addresses:
|
||||
return False
|
||||
return all(address.is_global for address in addresses)
|
||||
|
||||
@staticmethod
|
||||
def _is_global_hostname(hostname: str) -> bool:
|
||||
"""
|
||||
判断主机名解析结果是否全部为公网地址(同步版本)。
|
||||
|
||||
图片代理会访问用户可控的 URL,这里必须在 allowlist 命中前后都排除
|
||||
私有、回环、链路本地、保留地址等非公网目标,避免通过 DNS 或字面量 IP
|
||||
绕过域名白名单访问内网服务。
|
||||
"""
|
||||
return SecurityUtils._addresses_all_global(
|
||||
SecurityUtils._hostname_addresses(hostname)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _is_global_hostname_async(hostname: str) -> bool:
|
||||
"""
|
||||
判断主机名解析结果是否全部为公网地址(异步版本)。语义与 `_is_global_hostname` 一致。
|
||||
"""
|
||||
return SecurityUtils._addresses_all_global(
|
||||
await SecurityUtils._hostname_addresses_async(hostname)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _parse_ip_networks(ranges: Optional[Iterable[str]]) -> List[ipaddress._BaseNetwork]:
|
||||
"""
|
||||
解析用户配置的 IP/CIDR 网段。
|
||||
|
||||
配置错误的条目会被忽略并写入 debug 日志,避免单个无效值导致所有图片代理
|
||||
校验失败。调用方仍然需要先完成域名白名单匹配,不能单独依赖该网段放行。
|
||||
"""
|
||||
networks = []
|
||||
for value in ranges or []:
|
||||
if not value:
|
||||
continue
|
||||
try:
|
||||
networks.append(ipaddress.ip_network(str(value).strip(), strict=False))
|
||||
except ValueError:
|
||||
logger.debug(f"忽略无效的图片代理允许网段配置: {value}")
|
||||
return networks
|
||||
|
||||
@staticmethod
|
||||
def _match_private_addresses(
|
||||
addresses: Optional[List[ipaddress._BaseAddress]],
|
||||
networks: List[ipaddress._BaseNetwork],
|
||||
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
|
||||
"""
|
||||
在已解析出的地址列表中匹配显式允许的非公网网段。
|
||||
|
||||
所有解析地址都必须命中至少一个允许网段才放行;只要有一个 IP 落在允许
|
||||
网段外(或解析结果是全公网),就视为不匹配私网放行规则。
|
||||
"""
|
||||
if not addresses or not networks:
|
||||
return None
|
||||
if all(address.is_global for address in addresses):
|
||||
return None
|
||||
|
||||
matched_networks: List[ipaddress._BaseNetwork] = []
|
||||
for address in addresses:
|
||||
matched_for_address = [
|
||||
network for network in networks if address in network
|
||||
]
|
||||
if not matched_for_address:
|
||||
return None
|
||||
matched_networks.extend(matched_for_address)
|
||||
return addresses, list(dict.fromkeys(matched_networks))
|
||||
|
||||
@staticmethod
|
||||
def _is_allowed_private_hostname(
|
||||
hostname: str,
|
||||
allowed_private_ranges: Optional[Iterable[str]],
|
||||
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
|
||||
"""
|
||||
返回主机名命中的显式允许非公网地址和网段(同步版本)。
|
||||
|
||||
该能力只用于图片代理的受控例外,例如 TUN fake-ip 或内网 CDN。必须由
|
||||
`is_safe_url` 先完成域名 allowlist 校验后再调用,避免把任意用户 URL
|
||||
变成 SSRF 绕过入口。
|
||||
"""
|
||||
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
|
||||
if not networks:
|
||||
return None
|
||||
return SecurityUtils._match_private_addresses(
|
||||
SecurityUtils._hostname_addresses(hostname), networks
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def _is_allowed_private_hostname_async(
|
||||
hostname: str,
|
||||
allowed_private_ranges: Optional[Iterable[str]],
|
||||
) -> Optional[tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]]]:
|
||||
"""
|
||||
`_is_allowed_private_hostname` 的异步版本,语义保持一致。
|
||||
"""
|
||||
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
|
||||
if not networks:
|
||||
return None
|
||||
return SecurityUtils._match_private_addresses(
|
||||
await SecurityUtils._hostname_addresses_async(hostname), networks
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _url_signature_payload(url: str, purpose: str) -> bytes:
|
||||
"""
|
||||
构造 URL 签名载荷。
|
||||
|
||||
签名覆盖用途与完整 URL,确保同一个签名不能挪用到其它代理用途或其它 URL。
|
||||
"""
|
||||
return f"{purpose}\n{url}".encode("utf-8")
|
||||
|
||||
@staticmethod
|
||||
def _sign_url_payload(url: str, purpose: str) -> str:
|
||||
"""
|
||||
使用 RESOURCE_SECRET_KEY 对 URL 签名载荷生成 HMAC。
|
||||
|
||||
相同 `(url, purpose, RESOURCE_SECRET_KEY)` 组合在进程生命周期内输出
|
||||
完全一致;签名的失效边界绑定在 `RESOURCE_SECRET_KEY` 上,进程重启
|
||||
或显式轮换密钥时所有旧签名一起作废。
|
||||
"""
|
||||
return hmac.new(
|
||||
settings.RESOURCE_SECRET_KEY.encode("utf-8"),
|
||||
SecurityUtils._url_signature_payload(url, purpose),
|
||||
sha256,
|
||||
).hexdigest()
|
||||
|
||||
@staticmethod
|
||||
def strip_url_signature(url: str) -> str:
|
||||
"""
|
||||
移除 URL fragment 中的资源签名信息,得到真正要请求的地址。
|
||||
|
||||
签名放在 fragment 中,浏览器会把它传给 MoviePilot,但 HTTP 客户端
|
||||
请求外部资源前不能把这些内部参数带过去。
|
||||
"""
|
||||
if not url:
|
||||
return url
|
||||
parsed_url = urlparse(url)
|
||||
return urlunparse(parsed_url._replace(fragment=""))
|
||||
|
||||
@staticmethod
|
||||
def subtitle_download_purpose(site_id: int) -> str:
|
||||
"""
|
||||
构造字幕下载 URL 签名用途,签名必须绑定站点 ID,避免跨站点复用。
|
||||
"""
|
||||
return f"{SecurityUtils._SUBTITLE_DOWNLOAD_PURPOSE_PREFIX}:{site_id}"
|
||||
|
||||
@staticmethod
|
||||
def sign_url(
|
||||
url: str,
|
||||
purpose: str = _SIGNED_URL_PURPOSE,
|
||||
) -> str:
|
||||
"""
|
||||
给服务端返回的资源 URL 添加稳定签名。
|
||||
|
||||
签名作为后端资源能力凭证:外部请求边界可以用不同 `purpose` 绑定
|
||||
具体业务语义,避免一个场景签出的 URL 被挪用到另一个场景。
|
||||
|
||||
签名为 `(url, purpose, RESOURCE_SECRET_KEY)` 的确定性 HMAC,**不带
|
||||
过期时间**:相同 URL 多次调用结果完全一致,让浏览器与 Service Worker
|
||||
的缓存能稳定命中;失效边界由 `RESOURCE_SECRET_KEY` 控制——进程重启
|
||||
自动重生成、或者运维显式轮换后所有历史签名一起作废。
|
||||
"""
|
||||
if not url:
|
||||
return url
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
|
||||
return url
|
||||
clean_url = SecurityUtils.strip_url_signature(url)
|
||||
signature = SecurityUtils._sign_url_payload(clean_url, purpose)
|
||||
fragment = urlencode(
|
||||
{
|
||||
"mp_sig": signature,
|
||||
"mp_purpose": purpose,
|
||||
}
|
||||
)
|
||||
return urlunparse(urlparse(clean_url)._replace(fragment=fragment))
|
||||
|
||||
@staticmethod
|
||||
def verify_signed_url(
|
||||
url: str,
|
||||
purpose: str = _SIGNED_URL_PURPOSE,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
验证 URL fragment 中的资源签名,成功时返回去签名后的真实 URL。
|
||||
|
||||
签名只校验 `(url, purpose, RESOURCE_SECRET_KEY)`,密钥轮换/进程重启
|
||||
后旧签名自动失效。
|
||||
"""
|
||||
if not url:
|
||||
return None
|
||||
parsed_url = urlparse(url)
|
||||
if parsed_url.scheme not in {"http", "https"} or not parsed_url.netloc:
|
||||
return None
|
||||
fragment_params = dict(parse_qsl(parsed_url.fragment, keep_blank_values=True))
|
||||
signature = fragment_params.get("mp_sig")
|
||||
signed_purpose = fragment_params.get("mp_purpose")
|
||||
if not signature or signed_purpose != purpose:
|
||||
return None
|
||||
|
||||
clean_url = SecurityUtils.strip_url_signature(url)
|
||||
expected_signature = SecurityUtils._sign_url_payload(clean_url, purpose)
|
||||
if not hmac.compare_digest(signature, expected_signature):
|
||||
return None
|
||||
return clean_url
|
||||
|
||||
@staticmethod
|
||||
def _check_url_allowlist(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
strict: bool,
|
||||
) -> Optional[str]:
|
||||
"""
|
||||
执行"协议 + netloc + 域名白名单"前置校验,命中返回 hostname,未命中返回 None。
|
||||
|
||||
DNS 校验(SSRF 防御)由调用方自行接续,本方法不发起 DNS 查询。
|
||||
"""
|
||||
try:
|
||||
parsed_url = urlparse(url)
|
||||
except Exception as e: # noqa: BLE001 - 任何解析异常都视为不安全 URL
|
||||
logger.debug(f"Error occurred while validating URL: {e}")
|
||||
return None
|
||||
|
||||
# 如果 URL 没有包含有效的 scheme,或者无法从中提取到有效的 netloc,则认为该 URL 是无效的
|
||||
if not parsed_url.scheme or not parsed_url.netloc:
|
||||
return None
|
||||
# 仅允许 http 或 https 协议
|
||||
if parsed_url.scheme not in {"http", "https"}:
|
||||
return None
|
||||
|
||||
# 获取完整的 netloc(包括 IP 和端口)并转换为小写
|
||||
netloc = parsed_url.netloc.lower()
|
||||
if not netloc:
|
||||
return None
|
||||
|
||||
# 检查每个允许的域名
|
||||
normalized_allowed = {d.lower() for d in allowed_domains}
|
||||
domain_allowed = False
|
||||
for domain in normalized_allowed:
|
||||
parsed_allowed_url = urlparse(domain)
|
||||
allowed_netloc = parsed_allowed_url.netloc or parsed_allowed_url.path
|
||||
|
||||
if strict:
|
||||
# 严格模式下,要求完全匹配域名和端口
|
||||
if netloc == allowed_netloc:
|
||||
domain_allowed = True
|
||||
break
|
||||
else:
|
||||
# 非严格模式下,允许子域名匹配
|
||||
if netloc == allowed_netloc or netloc.endswith("." + allowed_netloc):
|
||||
domain_allowed = True
|
||||
break
|
||||
|
||||
if not domain_allowed:
|
||||
return None
|
||||
return parsed_url.hostname or ""
|
||||
|
||||
@staticmethod
|
||||
def _log_private_range_allowed(
|
||||
url: str,
|
||||
match: tuple[List[ipaddress._BaseAddress], List[ipaddress._BaseNetwork]],
|
||||
) -> None:
|
||||
"""
|
||||
记录"图片代理允许访问配置的非公网网段"放行日志,便于运维排查。
|
||||
"""
|
||||
addresses, matched_networks = match
|
||||
logger.debug(
|
||||
"图片代理允许访问配置的非公网网段: "
|
||||
f"url={url}, ips={','.join(map(str, addresses))}, "
|
||||
f"ranges={','.join(map(str, matched_networks))}"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def is_safe_url(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
strict: bool = False,
|
||||
block_private: bool = False,
|
||||
allowed_private_ranges: Optional[Iterable[str]] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
验证 URL 是否在允许的域名列表中,包括带有端口的域名(同步版本)。
|
||||
|
||||
:param url: 需要验证的 URL
|
||||
:param allowed_domains: 允许的域名集合,域名可以包含端口
|
||||
:param strict: 是否严格匹配一级域名(默认 False,允许多级域名)
|
||||
:param block_private: 是否拦截解析到非公网地址的 URL,防止 SSRF
|
||||
:param allowed_private_ranges: 域名命中后额外允许的非公网 IP/CIDR 网段
|
||||
:return: URL 合法且通过安全校验时返回 True,否则返回 False
|
||||
|
||||
校验细节与失败原因由 `evaluate_url_safety` 返回;本方法只暴露布尔结果,
|
||||
作为只关心通过/拒绝判断的调用方的最薄入口。`block_private=True` 时会
|
||||
同步调用 `getaddrinfo`;async 上下文请改用 `is_safe_url_async`。
|
||||
"""
|
||||
return SecurityUtils.evaluate_url_safety(
|
||||
url,
|
||||
allowed_domains,
|
||||
strict=strict,
|
||||
block_private=block_private,
|
||||
allowed_private_ranges=allowed_private_ranges,
|
||||
).allowed
|
||||
|
||||
@staticmethod
|
||||
async def is_safe_url_async(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
strict: bool = False,
|
||||
block_private: bool = False,
|
||||
allowed_private_ranges: Optional[Iterable[str]] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判定 URL 是否在允许的域名列表中,包括带有端口的域名。
|
||||
|
||||
DNS 解析通过事件循环线程池执行,并复用 TTL 缓存,不阻塞调用方所在的
|
||||
事件循环。参数与返回值含义同 `is_safe_url`;需要失败原因/解析 IP
|
||||
等结构化信息时调用 `evaluate_url_safety_async`。
|
||||
"""
|
||||
diagnosis = await SecurityUtils.evaluate_url_safety_async(
|
||||
url,
|
||||
allowed_domains,
|
||||
strict=strict,
|
||||
block_private=block_private,
|
||||
allowed_private_ranges=allowed_private_ranges,
|
||||
)
|
||||
return diagnosis.allowed
|
||||
|
||||
@staticmethod
|
||||
def evaluate_url_safety(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
strict: bool = False,
|
||||
block_private: bool = False,
|
||||
allowed_private_ranges: Optional[Iterable[str]] = None,
|
||||
) -> "UrlSafetyDiagnosis":
|
||||
"""
|
||||
在 `is_safe_url` 的判定路径上输出结构化诊断结果(同步版本)。
|
||||
|
||||
与 `is_safe_url` 共用同一套校验顺序:协议/域名 allowlist → 可选 DNS 解析
|
||||
→ 可选非公网放行匹配;本方法额外返回失败原因、解析到的 IP 列表和命中的
|
||||
私网网段,供日志与告警渲染消费。校验中遇到未预期异常时按默认拒绝原则
|
||||
归类为 `DOMAIN_NOT_ALLOWED`,避免任何解析路径漏过 SSRF 校验。
|
||||
"""
|
||||
try:
|
||||
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
|
||||
if hostname is None:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
|
||||
)
|
||||
if not block_private:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=True,
|
||||
reason=UrlSafetyReason.ALLOWED,
|
||||
host=hostname,
|
||||
)
|
||||
addresses = SecurityUtils._hostname_addresses(hostname)
|
||||
return SecurityUtils._diagnose_resolved_addresses(
|
||||
url, hostname, addresses, allowed_private_ranges
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - 默认拒绝,避免漏过 SSRF 校验
|
||||
logger.debug(f"Error occurred while validating URL: {e}")
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def evaluate_url_safety_async(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
strict: bool = False,
|
||||
block_private: bool = False,
|
||||
allowed_private_ranges: Optional[Iterable[str]] = None,
|
||||
) -> "UrlSafetyDiagnosis":
|
||||
"""
|
||||
输出与 `evaluate_url_safety` 完全一致的结构化诊断结果。
|
||||
|
||||
DNS 解析通过事件循环线程池执行,并复用 TTL 缓存,不阻塞调用方所在的
|
||||
事件循环;校验顺序、字段含义、异常归类均与同步版本相同。
|
||||
"""
|
||||
try:
|
||||
hostname = SecurityUtils._check_url_allowlist(url, allowed_domains, strict)
|
||||
if hostname is None:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
|
||||
)
|
||||
if not block_private:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=True,
|
||||
reason=UrlSafetyReason.ALLOWED,
|
||||
host=hostname,
|
||||
)
|
||||
addresses = await SecurityUtils._hostname_addresses_async(hostname)
|
||||
return SecurityUtils._diagnose_resolved_addresses(
|
||||
url, hostname, addresses, allowed_private_ranges
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - 默认拒绝,避免漏过 SSRF 校验
|
||||
logger.debug(f"Error occurred while validating URL: {e}")
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.DOMAIN_NOT_ALLOWED,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def is_safe_image_url_async(
|
||||
url: str,
|
||||
allowed_domains: Union[Set[str], List[str]],
|
||||
allowed_private_ranges: Optional[Iterable[str]] = None,
|
||||
) -> bool:
|
||||
"""
|
||||
判定 URL 是否可作为图片代理请求目标。
|
||||
|
||||
校验顺序:协议 + 域名 allowlist + DNS SSRF 拦截 + 非公网放行匹配;标准
|
||||
校验失败时再用 `verify_signed_url` 兜底,允许后端预签名的媒体服务器
|
||||
URL 跳过私网拦截。两者皆失败才视为拒绝。
|
||||
|
||||
拒绝路径会输出结构化阻断日志:单次拦截立即打印一条 warning,同
|
||||
`(host, reason)` 的连续命中在 `_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS`
|
||||
窗口内合并为一条聚合摘要,避免媒体详情页一次请求把日志刷爆。日志字段
|
||||
范围严格限定为 URL、host、reason、解析 IP 与允许网段配置;cookies、
|
||||
签名串、token、请求头等敏感材料一律不进入日志。
|
||||
"""
|
||||
diagnosis = await SecurityUtils.evaluate_url_safety_async(
|
||||
url,
|
||||
allowed_domains,
|
||||
block_private=True,
|
||||
allowed_private_ranges=allowed_private_ranges,
|
||||
)
|
||||
if diagnosis.allowed:
|
||||
return True
|
||||
if SecurityUtils.verify_signed_url(url) is not None:
|
||||
return True
|
||||
await _emit_image_proxy_block_warning(
|
||||
url=url,
|
||||
diagnosis=diagnosis,
|
||||
signature_carried=_url_carries_signature(url),
|
||||
allowed_private_ranges=allowed_private_ranges,
|
||||
)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _diagnose_resolved_addresses(
|
||||
url: str,
|
||||
hostname: str,
|
||||
addresses: Optional[List[ipaddress._BaseAddress]],
|
||||
allowed_private_ranges: Optional[Iterable[str]],
|
||||
) -> "UrlSafetyDiagnosis":
|
||||
"""
|
||||
对已完成 DNS 解析的地址列表执行非公网放行判断,并归一化诊断结果。
|
||||
|
||||
- 地址列表为空/None:视为 DNS 不可信,拒绝并标记 `DNS_RESOLUTION_FAILED`。
|
||||
- 全部公网地址:直接放行。
|
||||
- 存在非公网地址且未配置允许网段:拒绝并标记 `NON_GLOBAL_DNS_RESULT`,
|
||||
供日志附带"如使用 fake-ip 需要配置 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
|
||||
的提示。
|
||||
- 存在非公网地址且配置了允许网段但未全部命中:拒绝并标记
|
||||
`MIXED_OR_DISALLOWED_PRIVATE_RESULT`,提示存在不允许的解析结果。
|
||||
- 全部命中允许网段:放行并附带命中的 IP 与网段,由
|
||||
`_log_private_range_allowed` 输出排查日志。
|
||||
"""
|
||||
if not addresses:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.DNS_RESOLUTION_FAILED,
|
||||
host=hostname,
|
||||
)
|
||||
if SecurityUtils._addresses_all_global(addresses):
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=True,
|
||||
reason=UrlSafetyReason.ALLOWED,
|
||||
host=hostname,
|
||||
ips=[str(addr) for addr in addresses],
|
||||
)
|
||||
networks = SecurityUtils._parse_ip_networks(allowed_private_ranges)
|
||||
if not networks:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.NON_GLOBAL_DNS_RESULT,
|
||||
host=hostname,
|
||||
ips=[str(addr) for addr in addresses],
|
||||
)
|
||||
match = SecurityUtils._match_private_addresses(addresses, networks)
|
||||
if match is None:
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=False,
|
||||
reason=UrlSafetyReason.MIXED_OR_DISALLOWED_PRIVATE_RESULT,
|
||||
host=hostname,
|
||||
ips=[str(addr) for addr in addresses],
|
||||
)
|
||||
matched_addresses, matched_networks = match
|
||||
SecurityUtils._log_private_range_allowed(url, match)
|
||||
return UrlSafetyDiagnosis(
|
||||
allowed=True,
|
||||
reason=UrlSafetyReason.ALLOWED,
|
||||
host=hostname,
|
||||
ips=[str(addr) for addr in matched_addresses],
|
||||
matched_private_ranges=[str(net) for net in matched_networks],
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def sanitize_url_path(url: str, max_length: int = 120) -> str:
|
||||
"""
|
||||
将 URL 的路径部分进行编码,确保合法字符,并对路径长度进行压缩处理(如果超出最大长度)
|
||||
|
||||
:param url: 需要处理的 URL
|
||||
:param max_length: 路径允许的最大长度,超出时进行压缩
|
||||
:return: 处理后的路径字符串
|
||||
"""
|
||||
# 解析 URL,获取路径部分
|
||||
parsed_url = urlparse(url)
|
||||
path = parsed_url.path.lstrip("/")
|
||||
|
||||
# 对路径中的特殊字符进行编码
|
||||
safe_path = quote(path)
|
||||
|
||||
# 如果路径过长,进行压缩处理
|
||||
if len(safe_path) > max_length:
|
||||
# 使用 SHA-256 对路径进行哈希,取前 16 位作为压缩后的路径
|
||||
hash_value = sha256(safe_path.encode()).hexdigest()[:16]
|
||||
# 使用哈希值代替过长的路径,同时保留文件扩展名
|
||||
file_extension = Path(safe_path).suffix.lower() if Path(safe_path).suffix else ""
|
||||
safe_path = f"compressed_{hash_value}{file_extension}"
|
||||
|
||||
return safe_path
|
||||
|
||||
|
||||
# 图片代理阻断日志聚合窗口(秒)。媒体详情页一次请求会批量触发同 host/同原因的拦截,
|
||||
# 按 (host, reason) 合并后只输出首条 warning + 窗口结束的聚合摘要,避免日志刷屏。
|
||||
_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS = 60.0
|
||||
|
||||
# fake-ip / 旁路 DNS 用户最常因 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES 未配置而踩坑,
|
||||
# 在 reason=NON_GLOBAL_DNS_RESULT 且当前未配置允许网段时随 warning 一起输出,指向正确的修复开关。
|
||||
_IMAGE_PROXY_FAKEIP_HINT = (
|
||||
"提示:若使用 fake-ip / 旁路 DNS(常见网段 198.18.0.0/15、100.64.0.0/10),"
|
||||
"请将对应网段加入 IMAGE_PROXY_ALLOWED_PRIVATE_RANGES"
|
||||
)
|
||||
|
||||
# URL fragment 中实际携带代理签名但校验失败时附在 reason 末尾的标记。
|
||||
# 仅起标识作用,签名串本身不写入日志,避免泄露签名材料。
|
||||
_INVALID_SIGNATURE_TAG = "invalid_signature"
|
||||
|
||||
|
||||
def _url_carries_signature(url: str) -> bool:
|
||||
"""
|
||||
判断 URL 是否在 fragment 中显式携带代理签名参数 `mp_sig`。
|
||||
|
||||
仅做轻量字符串匹配,避免对普通图片 URL 跑完整签名校验路径;未携带签名
|
||||
的外链不会触发 `invalid_signature` 标记,避免阻断日志误导未签名调用方。
|
||||
"""
|
||||
if not url:
|
||||
return False
|
||||
fragment_start = url.find("#")
|
||||
if fragment_start < 0:
|
||||
return False
|
||||
return "mp_sig=" in url[fragment_start + 1:]
|
||||
|
||||
|
||||
def _format_image_proxy_block_warning(
|
||||
*,
|
||||
url: str,
|
||||
reason: str,
|
||||
host: Optional[str],
|
||||
ips: List[str],
|
||||
allowed_private_ranges: List[str],
|
||||
hint: Optional[str],
|
||||
) -> str:
|
||||
"""
|
||||
渲染图片代理首条阻断 warning 文案。
|
||||
|
||||
字段范围严格限定为 URL、host、reason、IP 与允许网段配置;hint 仅在
|
||||
reason 与配置缺失同时满足时由调用方填充。其余敏感材料(cookies、签名
|
||||
串、token、请求头)不允许进入该日志路径。
|
||||
"""
|
||||
fields = [
|
||||
f"url={url}",
|
||||
f"reason={reason}",
|
||||
f"host={host or ''}",
|
||||
f"ips={','.join(ips)}",
|
||||
f"allowed_private_ranges={','.join(allowed_private_ranges)}",
|
||||
]
|
||||
line = "Blocked unsafe image URL: " + ", ".join(fields)
|
||||
if hint:
|
||||
line = f"{line} | {hint}"
|
||||
return line
|
||||
|
||||
|
||||
def _log_image_proxy_block_summary(summary: CoalesceSummary) -> None:
|
||||
"""
|
||||
图片代理阻断日志聚合窗口到期回调,输出窗口内的命中计数与首条样例。
|
||||
|
||||
summary.key 由 `_emit_image_proxy_block_warning` 固定构造为
|
||||
`(host, reason_label)` 二元组;摘要保留首条事件的 URL 与解析 IP,
|
||||
避免运维只看到 count 而无法定位是哪批请求被合并。
|
||||
"""
|
||||
host, reason = summary.key
|
||||
payload = summary.first_payload or {}
|
||||
sample_ips = ",".join(payload.get("ips") or [])
|
||||
logger.warn(
|
||||
"Blocked unsafe image URL (aggregated): "
|
||||
f"host={host or ''}, reason={reason}, "
|
||||
f"count={summary.count}, window={summary.window_seconds:g}s, "
|
||||
f"sample_url={payload.get('url', '')}, sample_ips={sample_ips}"
|
||||
)
|
||||
|
||||
|
||||
# 图片代理阻断日志聚合器。同 (host, reason) 高频拦截在窗口内合并为一条聚合摘要,避免媒体详情页一次请求把日志刷爆;
|
||||
# 放行 debug 日志与诊断布尔结果不受聚合影响。
|
||||
_image_proxy_block_log_coalescer = EventCoalescer(
|
||||
window_seconds=_IMAGE_PROXY_BLOCK_LOG_WINDOW_SECONDS,
|
||||
on_flush=_log_image_proxy_block_summary,
|
||||
source="image_proxy",
|
||||
)
|
||||
|
||||
|
||||
async def _emit_image_proxy_block_warning(
|
||||
*,
|
||||
url: str,
|
||||
diagnosis: "UrlSafetyDiagnosis",
|
||||
signature_carried: bool,
|
||||
allowed_private_ranges: Optional[Iterable[str]],
|
||||
) -> None:
|
||||
"""
|
||||
把诊断结果转写为结构化阻断 warning,并交由 coalescer 决定是否实际输出。
|
||||
|
||||
`signature_carried=True` 表示请求 URL 在 fragment 里实际携带了代理签名但
|
||||
校验失败,此时在 reason 末尾追加 `invalid_signature` 标记,便于区分
|
||||
"未签名外链直接撞 allowlist"与"签名 URL 已失效"两种排查路径。
|
||||
"""
|
||||
# reason_label 既作为 warning 字段,也作为 coalescer 桶键的一部分;签名
|
||||
# 标记拼接到同一字符串里是为了让"带签名失败"的命中与"裸 URL 失败"分桶,
|
||||
# 各自独立计数与摘要,不要在不引入新桶维度的情况下拆开。
|
||||
reason_label = diagnosis.reason.value
|
||||
if signature_carried:
|
||||
reason_label = f"{reason_label}+{_INVALID_SIGNATURE_TAG}"
|
||||
allowed_ranges = [str(r) for r in (allowed_private_ranges or [])]
|
||||
hint = (
|
||||
_IMAGE_PROXY_FAKEIP_HINT
|
||||
if diagnosis.reason is UrlSafetyReason.NON_GLOBAL_DNS_RESULT
|
||||
and not allowed_ranges
|
||||
else None
|
||||
)
|
||||
key = (diagnosis.host or "", reason_label)
|
||||
payload = {"url": url, "ips": list(diagnosis.ips)}
|
||||
decision = await _image_proxy_block_log_coalescer.record(key=key, payload=payload)
|
||||
if decision is CoalesceDecision.EMIT:
|
||||
logger.warn(
|
||||
_format_image_proxy_block_warning(
|
||||
url=url,
|
||||
reason=reason_label,
|
||||
host=diagnosis.host,
|
||||
ips=list(diagnosis.ips),
|
||||
allowed_private_ranges=allowed_ranges,
|
||||
hint=hint,
|
||||
)
|
||||
)
|
||||
1
app/application/site/__init__.py
Normal file
1
app/application/site/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
"""站点目录、认证与索引资源的应用能力包。"""
|
||||
60
app/application/site/sites.pyi
Normal file
60
app/application/site/sites.pyi
Normal file
@@ -0,0 +1,60 @@
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
|
||||
class SitesHelper:
|
||||
"""声明 Cython 站点认证与索引扩展的宿主接口。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
"""返回进程内共享的站点助手实例。"""
|
||||
...
|
||||
|
||||
@property
|
||||
def auth_version(self) -> str:
|
||||
"""返回认证资源版本。"""
|
||||
...
|
||||
|
||||
@property
|
||||
def indexer_version(self) -> str:
|
||||
"""返回站点索引资源版本。"""
|
||||
...
|
||||
|
||||
@property
|
||||
def auth_level(self) -> int:
|
||||
"""返回当前用户认证等级。"""
|
||||
...
|
||||
|
||||
def check(self, domain: str) -> Tuple[bool, str]:
|
||||
"""检查站点域名是否触发访问频率限制。"""
|
||||
...
|
||||
|
||||
def get_indexers(self) -> List[dict]:
|
||||
"""返回全部可用站点索引配置。"""
|
||||
...
|
||||
|
||||
async def async_get_indexers(self) -> List[dict]:
|
||||
"""异步返回全部可用站点索引配置。"""
|
||||
...
|
||||
|
||||
def get_indexer(self, domain: str) -> Optional[dict]:
|
||||
"""按域名返回单个站点索引配置。"""
|
||||
...
|
||||
|
||||
async def async_get_indexer(self, domain: str) -> Optional[dict]:
|
||||
"""异步按域名返回单个站点索引配置。"""
|
||||
...
|
||||
|
||||
def get_authsites(self) -> dict:
|
||||
"""返回认证站点配置。"""
|
||||
...
|
||||
|
||||
def get_indexsites(self) -> dict:
|
||||
"""返回内置站点索引配置。"""
|
||||
...
|
||||
|
||||
def check_user(
|
||||
self,
|
||||
site: Optional[str] = None,
|
||||
params: Optional[dict] = None,
|
||||
) -> Tuple[bool, str]:
|
||||
"""校验用户站点认证信息并返回状态与消息。"""
|
||||
...
|
||||
82
app/application/storage.py
Normal file
82
app/application/storage.py
Normal file
@@ -0,0 +1,82 @@
|
||||
from typing import List, Optional
|
||||
|
||||
from app import schemas
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.schemas.types import SystemConfigKey
|
||||
|
||||
|
||||
class StorageHelper:
|
||||
"""
|
||||
存储帮助类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get_storagies() -> List[schemas.StorageConf]:
|
||||
"""
|
||||
获取所有存储设置
|
||||
"""
|
||||
storage_confs: List[dict] = SystemConfigOper().get(SystemConfigKey.Storages)
|
||||
if not storage_confs:
|
||||
return []
|
||||
return [schemas.StorageConf(**s) for s in storage_confs]
|
||||
|
||||
def get_storage(self, storage: str) -> Optional[schemas.StorageConf]:
|
||||
"""
|
||||
获取指定存储配置
|
||||
"""
|
||||
storagies = self.get_storagies()
|
||||
for s in storagies:
|
||||
if s.type == storage:
|
||||
return s
|
||||
return None
|
||||
|
||||
def set_storage(self, storage: str, conf: dict):
|
||||
"""
|
||||
设置存储配置
|
||||
"""
|
||||
storagies = self.get_storagies()
|
||||
if not storagies:
|
||||
storagies = [
|
||||
schemas.StorageConf(
|
||||
type=storage,
|
||||
config=conf
|
||||
)
|
||||
]
|
||||
else:
|
||||
for s in storagies:
|
||||
if s.type == storage:
|
||||
s.config = conf
|
||||
break
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
|
||||
def add_storage(self, storage: str, name: str, conf: dict):
|
||||
"""
|
||||
添加存储配置
|
||||
"""
|
||||
storagies = self.get_storagies()
|
||||
if not storagies:
|
||||
storagies = [
|
||||
schemas.StorageConf(
|
||||
type=storage,
|
||||
name=name,
|
||||
config=conf
|
||||
)
|
||||
]
|
||||
else:
|
||||
storagies.append(schemas.StorageConf(
|
||||
type=storage,
|
||||
name=name,
|
||||
config=conf
|
||||
))
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
|
||||
def reset_storage(self, storage: str):
|
||||
"""
|
||||
重置存储配置
|
||||
"""
|
||||
storagies = self.get_storagies()
|
||||
for s in storagies:
|
||||
if s.type == storage:
|
||||
s.config = {}
|
||||
break
|
||||
SystemConfigOper().set(SystemConfigKey.Storages, [s.model_dump() for s in storagies])
|
||||
627
app/application/torrent.py
Normal file
627
app/application/torrent.py
Normal file
@@ -0,0 +1,627 @@
|
||||
import datetime
|
||||
import re
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Tuple, Optional, List, Union, Dict, Any
|
||||
from urllib.parse import unquote
|
||||
|
||||
from torrentool.api import Torrent
|
||||
|
||||
from app.runtime.cache import TTLCache, FileCache
|
||||
from app.runtime.config import settings
|
||||
from app.domain.context import Context, TorrentInfo, MediaInfo
|
||||
from app.domain.meta.metabase import MetaBase
|
||||
from app.domain.meta.metamusic import audio_quality_tier, normalize_audio_format, parse_audio_quality
|
||||
from app.domain.metainfo import MetaInfo
|
||||
from app.db.site_oper import SiteOper
|
||||
from app.db.systemconfig_oper import SystemConfigOper
|
||||
from app.runtime.log import logger
|
||||
from app.schemas.types import MediaType, SystemConfigKey
|
||||
from app.adapters.network.http import RequestUtils
|
||||
from app.domain.media import resolve_media_identity
|
||||
from app.domain.string import StringUtils
|
||||
|
||||
|
||||
_SIZE_UNIT = 1024 * 1024
|
||||
|
||||
|
||||
@lru_cache(maxsize=512)
|
||||
def _compile_filter_pattern(pattern: str) -> re.Pattern:
|
||||
"""
|
||||
编译订阅/工作流附加过滤正则。
|
||||
用户输入沿用原本的正则语义,缓存只减少同一规则反复匹配大量种子时的编译成本。
|
||||
"""
|
||||
return re.compile(r"%s" % pattern, re.I)
|
||||
|
||||
|
||||
def _filter_pattern_search(pattern: Union[str, int, float], content: str) -> bool:
|
||||
"""
|
||||
按原有字符串插值语义执行过滤正则匹配。
|
||||
"""
|
||||
return bool(_compile_filter_pattern(str(pattern)).search(content))
|
||||
|
||||
|
||||
@lru_cache(maxsize=256)
|
||||
def _parse_filter_size_range(size_range: str) -> Tuple[str, float, Optional[float]]:
|
||||
"""
|
||||
解析附加过滤的大小范围,单位为 MB。
|
||||
"""
|
||||
if size_range.find("-") != -1:
|
||||
size_min, size_max = size_range.split("-")
|
||||
return "between", float(size_min.strip()) * _SIZE_UNIT, float(size_max.strip()) * _SIZE_UNIT
|
||||
if size_range.startswith(">"):
|
||||
return "gte", float(size_range[1:].strip()) * _SIZE_UNIT, None
|
||||
if size_range.startswith("<"):
|
||||
return "lte", 0, float(size_range[1:].strip()) * _SIZE_UNIT
|
||||
return "unknown", 0, None
|
||||
|
||||
|
||||
class TorrentHelper:
|
||||
"""
|
||||
种子帮助类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""初始化种子失败地址缓存"""
|
||||
self._invalid_torrents = TTLCache(region="invalid_torrents", maxsize=128, ttl=3600 * 24)
|
||||
|
||||
def download_torrent(self, url: str,
|
||||
cookie: Optional[str] = None,
|
||||
ua: Optional[str] = None,
|
||||
referer: Optional[str] = None,
|
||||
proxy: Optional[bool] = False,
|
||||
cache_invalid: bool = True) \
|
||||
-> Tuple[Optional[Path], Optional[Union[str, bytes]], Optional[str], Optional[list], Optional[str]]:
|
||||
"""
|
||||
把种子下载到本地
|
||||
:param url: 种子下载地址
|
||||
:param cookie: 站点 Cookie
|
||||
:param ua: 请求 User-Agent
|
||||
:param referer: 请求来源地址
|
||||
:param proxy: 是否使用系统代理
|
||||
:param cache_invalid: 是否缓存失败地址;短时凭证地址必须关闭
|
||||
:return: 种子缓存相对路径【用于索引缓存】, 种子内容、种子主目录、种子文件清单、错误信息
|
||||
"""
|
||||
if url.startswith("magnet:"):
|
||||
return None, url, "", [], f"磁力链接"
|
||||
# 构建 torrent 种子文件的缓存路径
|
||||
cache_path = Path(StringUtils.md5_hash(url)).with_suffix(".torrent")
|
||||
# 缓存处理器
|
||||
cache_backend = FileCache()
|
||||
# 读取缓存的种子文件
|
||||
torrent_content = cache_backend.get(cache_path.as_posix(), region="torrents")
|
||||
if torrent_content:
|
||||
# 缓存已存在
|
||||
try:
|
||||
# 获取种子目录和文件清单
|
||||
folder_name, file_list = self.get_fileinfo_from_torrent_content(torrent_content)
|
||||
# 无法获取信息,则认为缓存文件无效
|
||||
if not folder_name and not file_list:
|
||||
raise ValueError("无效的缓存种子文件")
|
||||
# 成功拿到种子数据
|
||||
return cache_path, torrent_content, folder_name, file_list, ""
|
||||
except Exception as err:
|
||||
logger.error(f"处理缓存的种子文件 {cache_path} 时出错: {err},将重新下载")
|
||||
# 下载种子文件
|
||||
req = RequestUtils(
|
||||
ua=ua,
|
||||
cookies=cookie,
|
||||
referer=referer,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
).get_res(url=url, allow_redirects=False)
|
||||
while req and req.status_code in [301, 302]:
|
||||
url = req.headers['Location']
|
||||
if url and url.startswith("magnet:"):
|
||||
return None, url, "", [], f"获取到磁力链接"
|
||||
req = RequestUtils(
|
||||
ua=ua,
|
||||
cookies=cookie,
|
||||
referer=referer,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
).get_res(url=url, allow_redirects=False)
|
||||
if req and req.status_code == 200:
|
||||
if not req.content:
|
||||
return cache_path, None, "", [], "未下载到种子数据"
|
||||
# 解析内容格式
|
||||
if req.content.startswith(b"magnet:"):
|
||||
# 磁力链接
|
||||
return cache_path, req.text, "", [], f"获取到磁力链接"
|
||||
if "下载种子文件".encode("utf-8") in req.content:
|
||||
# 首次下载提示页面
|
||||
skip_flag = False
|
||||
try:
|
||||
forms = re.findall(r'<form.*?action="(.*?)".*?>(.*?)</form>', req.text, re.S)
|
||||
for form in forms:
|
||||
action = form[0]
|
||||
if action != "?":
|
||||
continue
|
||||
action = url
|
||||
inputs = re.findall(r'<input.*?name="(.*?)".*?value="(.*?)".*?>', form[1], re.S)
|
||||
if inputs:
|
||||
data = {}
|
||||
for item in inputs:
|
||||
data[item[0]] = item[1]
|
||||
# 改写req
|
||||
req = RequestUtils(
|
||||
ua=ua,
|
||||
cookies=cookie,
|
||||
referer=referer,
|
||||
proxies=settings.PROXY if proxy else None
|
||||
).post_res(url=action, data=data)
|
||||
if req and req.status_code == 200:
|
||||
# 检查是不是种子文件,如果不是抛出异常
|
||||
Torrent.from_string(req.content)
|
||||
# 跳过成功
|
||||
logger.info("触发了站点首次种子下载,已自动跳过")
|
||||
skip_flag = True
|
||||
elif req is not None:
|
||||
logger.warn(f"触发了站点首次种子下载,且无法自动跳过,"
|
||||
f"返回码:{req.status_code},错误原因:{req.reason}")
|
||||
else:
|
||||
logger.warn("触发了站点首次种子下载,且无法自动跳过")
|
||||
break
|
||||
except Exception as err:
|
||||
logger.warn(f"触发了站点首次种子下载,尝试自动跳过时出现错误:{str(err)}")
|
||||
if not skip_flag:
|
||||
return cache_path, None, "", [], "种子数据有误,请确认链接是否正确,如为PT站点则需手工在站点下载一次种子"
|
||||
# 种子内容
|
||||
if req.content:
|
||||
# 检查是不是种子文件,如果不是仍然抛出异常
|
||||
try:
|
||||
# 获取种子目录和文件清单
|
||||
folder_name, file_list = self.get_fileinfo_from_torrent_content(req.content)
|
||||
if file_list:
|
||||
# 保存到缓存
|
||||
cache_backend.set(cache_path.as_posix(), req.content, region="torrents")
|
||||
# 成功拿到种子数据
|
||||
return cache_path, req.content, folder_name, file_list, ""
|
||||
except Exception as err:
|
||||
logger.error(f"种子文件解析失败:{str(err)}")
|
||||
# 种子数据仍然错误
|
||||
return cache_path, None, "", [], "种子数据有误,请确认链接是否正确"
|
||||
# 返回失败
|
||||
return cache_path, None, "", [], ""
|
||||
elif req is None:
|
||||
return cache_path, None, "", [], "无法打开链接"
|
||||
elif req.status_code == 429:
|
||||
return cache_path, None, "", [], "触发站点流控,请稍后重试"
|
||||
else:
|
||||
# 把错误的种子记下来,避免重复使用
|
||||
if cache_invalid:
|
||||
self.add_invalid(url)
|
||||
return cache_path, None, "", [], f"下载种子出错,状态码:{req.status_code}"
|
||||
|
||||
def get_torrent_info(self, torrent_path: Path) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
获取种子文件的文件夹名和文件清单
|
||||
:param torrent_path: 种子文件路径
|
||||
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
|
||||
"""
|
||||
if not torrent_path or not torrent_path.exists():
|
||||
return "", []
|
||||
try:
|
||||
torrentinfo = Torrent.from_file(torrent_path)
|
||||
# 获取文件清单
|
||||
return self.get_fileinfo_from_torrent(torrentinfo)
|
||||
except Exception as err:
|
||||
logger.error(f"种子文件解析失败:{str(err)}")
|
||||
return "", []
|
||||
|
||||
@staticmethod
|
||||
def get_fileinfo_from_torrent(torrent: Torrent) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
从种子文件中获取文件清单
|
||||
:param torrent: 种子文件对象
|
||||
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
|
||||
"""
|
||||
if not torrent or not torrent.files:
|
||||
return "", []
|
||||
# 获取文件清单
|
||||
if len(torrent.files) == 1 and torrent.files[0].name == torrent.name:
|
||||
# 单文件种子目录名返回空
|
||||
folder_name = ""
|
||||
# 单文件种子
|
||||
file_list = [torrent.name]
|
||||
else:
|
||||
# 目录名
|
||||
folder_name = torrent.name
|
||||
# 文件清单,如果一级目录与种子名相同则去掉
|
||||
file_list = []
|
||||
for fileinfo in torrent.files:
|
||||
file_path = Path(fileinfo.name)
|
||||
# 根路径
|
||||
root_path = file_path.parts[0]
|
||||
if root_path == folder_name:
|
||||
file_list.append(str(file_path.relative_to(root_path)))
|
||||
else:
|
||||
file_list.append(fileinfo.name)
|
||||
logger.debug(f"解析种子:{torrent.name} => 目录:{folder_name},文件清单:{file_list}")
|
||||
return folder_name, file_list
|
||||
|
||||
def get_fileinfo_from_torrent_content(self, torrent_content: Union[str, bytes]) -> Tuple[str, List[str]]:
|
||||
"""
|
||||
从种子内容中获取文件夹名和文件清单
|
||||
:param torrent_content: 种子内容
|
||||
:return: 文件夹名、文件清单,单文件种子返回空文件夹名
|
||||
"""
|
||||
|
||||
if not torrent_content:
|
||||
return "", []
|
||||
|
||||
# 检查是否为磁力链接
|
||||
if StringUtils.is_magnet_link(torrent_content):
|
||||
return "", []
|
||||
|
||||
try:
|
||||
# 解析种子内容
|
||||
torrentinfo = Torrent.from_string(torrent_content)
|
||||
# 获取文件清单
|
||||
return self.get_fileinfo_from_torrent(torrentinfo)
|
||||
except Exception as err:
|
||||
logger.error(f"种子内容解析失败:{str(err)}")
|
||||
return "", []
|
||||
|
||||
@staticmethod
|
||||
def get_url_filename(req: Any, url: str) -> str:
|
||||
"""
|
||||
从下载请求中获取种子文件名
|
||||
"""
|
||||
if not req:
|
||||
return ""
|
||||
disposition = req.headers.get('content-disposition') or ""
|
||||
file_name = re.findall(r"filename=\"?(.+)\"?", disposition)
|
||||
if file_name:
|
||||
file_name = unquote(str(file_name[0].encode('ISO-8859-1').decode()).split(";")[0].strip())
|
||||
if file_name.endswith('"'):
|
||||
file_name = file_name[:-1]
|
||||
elif url and url.endswith(".torrent"):
|
||||
file_name = unquote(url.split("/")[-1])
|
||||
else:
|
||||
file_name = str(datetime.datetime.now())
|
||||
return file_name
|
||||
|
||||
@staticmethod
|
||||
def sort_torrents(torrent_list: List[Context]) -> List[Context]:
|
||||
"""
|
||||
对种子对行排序:torrent、site、upload、seeder
|
||||
"""
|
||||
if not torrent_list:
|
||||
return []
|
||||
|
||||
# 下载规则
|
||||
priority_rule: List[str] = SystemConfigOper().get(
|
||||
SystemConfigKey.TorrentsPriority) or ["torrent", "upload", "seeder"]
|
||||
# 站点上传量
|
||||
site_uploads = {
|
||||
site.name: site.upload for site in SiteOper().get_userdata_latest()
|
||||
}
|
||||
|
||||
def get_sort_str(_context):
|
||||
"""
|
||||
拼装排序字段
|
||||
"""
|
||||
_meta = _context.meta_info
|
||||
_torrent = _context.torrent_info
|
||||
_media = _context.media_info
|
||||
# 标题
|
||||
_title = str(_media.title).ljust(200, ' ')
|
||||
# 站点优先级
|
||||
_site_order = str(999 - (_torrent.site_order or 0)).rjust(3, '0')
|
||||
# 站点上传量
|
||||
_site_upload = str(site_uploads.get(_torrent.site_name) or 0).rjust(30, '0')
|
||||
# 资源优先级
|
||||
_torrent_order = str(_torrent.pri_order or 0).rjust(3, '0')
|
||||
# 资源做种数
|
||||
_torrent_seeders = str(_torrent.seeders or 0).rjust(10, '0')
|
||||
# 季集
|
||||
if not _meta.episode_list:
|
||||
# 无集数的排最前面
|
||||
_season_episode = "%s%s" % (str(len(_meta.season_list)).rjust(3, '0'), "9999")
|
||||
else:
|
||||
# 集数越多的排越前面
|
||||
_season_episode = "%s%s" % (str(len(_meta.season_list)).rjust(3, '0'),
|
||||
str(len(_meta.episode_list)).rjust(4, '0'))
|
||||
# 根据下载规则的顺序拼装排序字符串
|
||||
_sort_str = _title
|
||||
for rule in priority_rule:
|
||||
if rule == "torrent":
|
||||
_sort_str += _torrent_order
|
||||
elif rule == "site":
|
||||
_sort_str += _site_order
|
||||
elif rule == "upload":
|
||||
_sort_str += _site_upload
|
||||
elif rule == "seeder":
|
||||
_sort_str += _torrent_seeders
|
||||
_sort_str += _season_episode
|
||||
return _sort_str
|
||||
|
||||
# 排序
|
||||
return sorted(torrent_list, key=lambda x: get_sort_str(x), reverse=True)
|
||||
|
||||
def sort_group_torrents(self, torrent_list: List[Context]) -> List[Context]:
|
||||
"""
|
||||
对媒体信息进行排序、去重
|
||||
"""
|
||||
if not torrent_list:
|
||||
return []
|
||||
|
||||
# 排序
|
||||
torrent_list = self.sort_torrents(torrent_list)
|
||||
|
||||
# 控重
|
||||
result = []
|
||||
_added = []
|
||||
# 排序后重新加入数组,按真实名称控重,即只取每个名称的第一个
|
||||
for context in torrent_list:
|
||||
# 控重的主链是名称、年份、季、集
|
||||
meta = context.meta_info
|
||||
media = context.media_info
|
||||
if media.type == MediaType.TV:
|
||||
media_name = "%s%s" % (media.title_year,
|
||||
meta.season_episode)
|
||||
else:
|
||||
media_name = media.title_year
|
||||
if media_name not in _added:
|
||||
_added.append(media_name)
|
||||
result.append(context)
|
||||
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def get_torrent_episodes(files: list, custom_words: Optional[List[str]] = None) -> list:
|
||||
"""
|
||||
从种子的文件清单中获取所有集数
|
||||
|
||||
:param files: 种子文件清单
|
||||
:param custom_words: 当前下载来源的临时自定义识别词
|
||||
:return: 识别到的全部集数
|
||||
"""
|
||||
episodes = []
|
||||
for file in files:
|
||||
if not file:
|
||||
continue
|
||||
file_path = Path(file)
|
||||
if not file_path.suffix or file_path.suffix.lower() not in settings.RMT_MEDIAEXT:
|
||||
continue
|
||||
# 只使用文件名识别
|
||||
meta = MetaInfo(file_path.name, custom_words=custom_words)
|
||||
if not meta.begin_episode:
|
||||
continue
|
||||
episodes = list(set(episodes).union(set(meta.episode_list)))
|
||||
return episodes
|
||||
|
||||
def is_invalid(self, url: Optional[str]) -> bool:
|
||||
"""
|
||||
判断种子是否是无效种子
|
||||
"""
|
||||
return url in self._invalid_torrents if url else True
|
||||
|
||||
def add_invalid(self, url: str):
|
||||
"""
|
||||
添加无效种子
|
||||
"""
|
||||
if url not in self._invalid_torrents:
|
||||
self._invalid_torrents[url] = True
|
||||
|
||||
@staticmethod
|
||||
def match_torrent(mediainfo: MediaInfo, torrent_meta: MetaBase, torrent: TorrentInfo) -> bool:
|
||||
"""
|
||||
检查种子是否匹配媒体信息
|
||||
:param mediainfo: 需要匹配的媒体信息
|
||||
:param torrent_meta: 种子识别信息
|
||||
:param torrent: 种子信息
|
||||
"""
|
||||
# 显式标题标签与识别结果使用同一主身份比较。
|
||||
torrent_identity = resolve_media_identity(media=torrent_meta)
|
||||
media_identity = resolve_media_identity(media=mediainfo)
|
||||
if all(torrent_identity) and torrent_identity == media_identity:
|
||||
logger.info(
|
||||
f'{mediainfo.title} 通过词表指定媒体身份 '
|
||||
f'{torrent_identity[0]}:{torrent_identity[1]} 匹配到资源:'
|
||||
f'{torrent.site_name} - {torrent.title}'
|
||||
)
|
||||
return True
|
||||
# 要匹配的媒体标题、原标题
|
||||
media_titles = {
|
||||
StringUtils.clear_upper(mediainfo.title),
|
||||
StringUtils.clear_upper(mediainfo.original_title)
|
||||
} - {""}
|
||||
# 要匹配的媒体别名、译名
|
||||
media_names = {StringUtils.clear_upper(name) for name in mediainfo.names if name}
|
||||
# 识别的种子中英文名
|
||||
meta_names = {
|
||||
StringUtils.clear_upper(torrent_meta.cn_name),
|
||||
StringUtils.clear_upper(torrent_meta.en_name)
|
||||
} - {""}
|
||||
# 比对种子识别类型
|
||||
if torrent_meta.type == MediaType.TV and mediainfo.type != MediaType.TV:
|
||||
logger.debug(f'{torrent.site_name} - {torrent.title} 种子标题类型为 {torrent_meta.type.value},'
|
||||
f'不匹配 {mediainfo.type.value}')
|
||||
return False
|
||||
# 比对种子在站点中的类型
|
||||
if torrent.category == MediaType.TV.value and mediainfo.type != MediaType.TV:
|
||||
logger.debug(f'{torrent.site_name} - {torrent.title} 种子在站点中归类为 {torrent.category},'
|
||||
f'不匹配 {mediainfo.type.value}')
|
||||
return False
|
||||
# 比对年份
|
||||
if mediainfo.year:
|
||||
if mediainfo.type == MediaType.TV:
|
||||
# 剧集年份,每季的年份可能不同,没年份时不比较年份(很多剧集种子不带年份)
|
||||
if torrent_meta.year and torrent_meta.year not in [year for year in
|
||||
mediainfo.season_years.values()]:
|
||||
logger.debug(f'{torrent.site_name} - {torrent.title} 年份不匹配 {mediainfo.season_years}')
|
||||
return False
|
||||
else:
|
||||
# 电影年份,上下浮动1年,没年份时不通过
|
||||
if not torrent_meta.year or torrent_meta.year not in [str(int(mediainfo.year) - 1),
|
||||
mediainfo.year,
|
||||
str(int(mediainfo.year) + 1)]:
|
||||
logger.debug(f'{torrent.site_name} - {torrent.title} 年份不匹配 {mediainfo.year}')
|
||||
return False
|
||||
# 比对标题和原语种标题
|
||||
if meta_names.intersection(media_titles):
|
||||
logger.info(f'{mediainfo.title} 通过标题匹配到资源:{torrent.site_name} - {torrent.title}')
|
||||
return True
|
||||
# 比对别名和译名
|
||||
if media_names:
|
||||
if meta_names.intersection(media_names):
|
||||
logger.info(f'{mediainfo.title} 通过别名或译名匹配到资源:{torrent.site_name} - {torrent.title}')
|
||||
return True
|
||||
# 标题拆分
|
||||
if torrent_meta.org_string:
|
||||
# 只拆分出标题中的非英文单词进行匹配,英文单词容易误匹配(带空格的多个单词组合除外)
|
||||
titles = [StringUtils.clear_upper(t) for t in re.split(
|
||||
r'[\s/【】.\[\]\-]+',
|
||||
torrent_meta.org_string
|
||||
) if not StringUtils.is_english_word(t)]
|
||||
# 在标题中判断是否存在标题、原语种标题
|
||||
if media_titles.intersection(titles):
|
||||
logger.info(f'{mediainfo.title} 通过标题匹配到资源:{torrent.site_name} - {torrent.title}')
|
||||
return True
|
||||
# 在副标题中(非英文单词)判断是否存在标题、原语种标题、别名、译名
|
||||
if torrent.description:
|
||||
subtitles = {StringUtils.clear_upper(t) for t in re.split(
|
||||
r'[\s/【】|]+',
|
||||
torrent.description) if not StringUtils.is_english_word(t)}
|
||||
if media_titles.intersection(subtitles) or media_names.intersection(subtitles):
|
||||
logger.info(f'{mediainfo.title} 通过副标题匹配到资源:{torrent.site_name} - {torrent.title},'
|
||||
f'副标题:{torrent.description}')
|
||||
return True
|
||||
# 未匹配
|
||||
logger.debug(f'{torrent.site_name} - {torrent.title} 标题不匹配,识别名称:{meta_names}')
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def filter_torrent(torrent_info: TorrentInfo,
|
||||
filter_params: Dict[str, Any]) -> bool:
|
||||
"""
|
||||
检查种子是否匹配订阅过滤规则
|
||||
"""
|
||||
|
||||
if not filter_params:
|
||||
return True
|
||||
|
||||
# 匹配内容
|
||||
content = (f"{torrent_info.title} "
|
||||
f"{torrent_info.description} "
|
||||
f"{' '.join(torrent_info.labels or [])} "
|
||||
f"{torrent_info.volume_factor}")
|
||||
|
||||
# 包含
|
||||
include = filter_params.get("include")
|
||||
if include:
|
||||
if not _filter_pattern_search(include, content):
|
||||
logger.info(f"{content} 不匹配包含规则 {include}")
|
||||
return False
|
||||
# 排除
|
||||
exclude = filter_params.get("exclude")
|
||||
if exclude:
|
||||
if _filter_pattern_search(exclude, content):
|
||||
logger.info(f"{content} 匹配排除规则 {exclude}")
|
||||
return False
|
||||
# 质量
|
||||
quality = filter_params.get("quality")
|
||||
if quality:
|
||||
if not _filter_pattern_search(quality, torrent_info.title):
|
||||
logger.info(f"{torrent_info.title} 不匹配质量规则 {quality}")
|
||||
return False
|
||||
# 分辨率
|
||||
resolution = filter_params.get("resolution")
|
||||
if resolution:
|
||||
if not _filter_pattern_search(resolution, torrent_info.title):
|
||||
logger.info(f"{torrent_info.title} 不匹配分辨率规则 {resolution}")
|
||||
return False
|
||||
# 特效
|
||||
effect = filter_params.get("effect")
|
||||
if effect:
|
||||
if not _filter_pattern_search(effect, torrent_info.title):
|
||||
logger.info(f"{torrent_info.title} 不匹配特效规则 {effect}")
|
||||
return False
|
||||
|
||||
# 音乐音质。技术参数从标题、副标题和标签统一解析,避免只靠用户正则筛选。
|
||||
audio_filters = {
|
||||
key: filter_params.get(key)
|
||||
for key in ("audio_quality", "audio_format", "min_bitrate", "min_bit_depth", "min_sample_rate")
|
||||
if filter_params.get(key) not in (None, "")
|
||||
}
|
||||
if audio_filters:
|
||||
specs = parse_audio_quality(content)
|
||||
tier = audio_quality_tier(**specs)
|
||||
audio_quality = audio_filters.get("audio_quality")
|
||||
quality_pattern = "hires|lossless" \
|
||||
if str(audio_quality).casefold() == "lossless" else audio_quality
|
||||
if audio_quality and (not tier or not _filter_pattern_search(quality_pattern, tier)):
|
||||
logger.info(f"{torrent_info.title} 不匹配音乐音质规则 {audio_quality}")
|
||||
return False
|
||||
audio_format = audio_filters.get("audio_format")
|
||||
normalized_format = normalize_audio_format(specs.get("audio_format"))
|
||||
if audio_format and (
|
||||
not normalized_format or not _filter_pattern_search(audio_format, normalized_format)
|
||||
):
|
||||
logger.info(f"{torrent_info.title} 不匹配音频格式规则 {audio_format}")
|
||||
return False
|
||||
for key, spec_key, label in (
|
||||
("min_bitrate", "bitrate", "码率"),
|
||||
("min_bit_depth", "bit_depth", "位深"),
|
||||
("min_sample_rate", "sample_rate", "采样率"),
|
||||
):
|
||||
minimum = audio_filters.get(key)
|
||||
actual = specs.get(spec_key)
|
||||
if minimum is not None and (actual is None or int(actual) < int(minimum)):
|
||||
logger.info(f"{torrent_info.title} 不满足最低{label} {minimum}")
|
||||
return False
|
||||
|
||||
# 大小
|
||||
size_range = filter_params.get("size")
|
||||
if size_range:
|
||||
size_rule, size_min, size_max = _parse_filter_size_range(size_range)
|
||||
if size_rule == "between":
|
||||
# 区间
|
||||
if torrent_info.size < size_min or torrent_info.size > size_max:
|
||||
return False
|
||||
elif size_rule == "gte":
|
||||
# 大于
|
||||
if torrent_info.size < size_min:
|
||||
return False
|
||||
elif size_rule == "lte":
|
||||
# 小于
|
||||
if torrent_info.size > size_max:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def match_season_episodes(torrent: TorrentInfo, meta: MetaBase, season_episodes: Dict[int, list]) -> bool:
|
||||
"""
|
||||
判断种子是否匹配季集数
|
||||
:param torrent: 种子信息
|
||||
:param meta: 种子元数据
|
||||
:param season_episodes: 季集数 {season:[episodes]}
|
||||
"""
|
||||
# 匹配季
|
||||
seasons = season_episodes.keys()
|
||||
seasons_set = set(seasons)
|
||||
# 种子季
|
||||
torrent_seasons = meta.season_list
|
||||
if not torrent_seasons:
|
||||
# 按第一季处理
|
||||
torrent_seasons = [1]
|
||||
# 种子集
|
||||
torrent_episodes = meta.episode_list
|
||||
if not set(torrent_seasons).issubset(seasons_set):
|
||||
# 种子季不在过滤季中
|
||||
logger.debug(
|
||||
f"种子 {torrent.site_name} - {torrent.title} 包含季 {torrent_seasons} 不是需要的季 {list(seasons)}")
|
||||
return False
|
||||
if not torrent_episodes:
|
||||
# 整季按匹配处理
|
||||
return True
|
||||
if len(torrent_seasons) == 1:
|
||||
need_episodes = season_episodes.get(torrent_seasons[0])
|
||||
if need_episodes \
|
||||
and not set(torrent_episodes).intersection(need_episodes):
|
||||
# 单季集没有交集的不要
|
||||
logger.debug(f"种子 {torrent.site_name} - {torrent.title} "
|
||||
f"集 {torrent_episodes} 没有需要的集:{need_episodes}")
|
||||
return False
|
||||
return True
|
||||
Reference in New Issue
Block a user