feat: expand metadata sources and media server sync (#6129)

This commit is contained in:
jxxghp
2026-07-21 11:34:18 +08:00
parent 1d708870c9
commit e3f80af74f
43 changed files with 2581 additions and 152 deletions

View File

@@ -0,0 +1,162 @@
import asyncio
from unittest.mock import AsyncMock, Mock
from xml.dom import minidom
import pytest
from app.core.context import MediaInfo
from app.core.meta import MetaBase
from app.helper.scraper import MediaScraperHelper
from app.modules.anilist import AniListModule
from app.modules.anilist.anilist import AniListApi
from app.schemas.types import MediaType
@pytest.fixture
def anilist_info() -> dict:
"""构造不依赖网络的AniList媒体详情。"""
return {
"id": 154587,
"title": {
"romaji": "Sousou no Frieren",
"english": "Frieren: Beyond Journey's End",
"native": "葬送のフリーレン",
},
"format": "TV",
"status": "FINISHED",
"description": "A <b>journey</b> after the adventure.",
"startDate": {"year": 2023, "month": 9, "day": 29},
"endDate": {"year": 2024, "month": 3, "day": 22},
"episodes": 28,
"duration": 24,
"countryOfOrigin": "JP",
"coverImage": {"extraLarge": "https://img.example/poster.jpg"},
"bannerImage": "https://img.example/backdrop.png",
"genres": ["Adventure", "Fantasy"],
"synonyms": ["Frieren"],
"averageScore": 91,
"popularity": 300000,
"isAdult": False,
"studios": {"nodes": [{"name": "Madhouse"}]},
"staff": {
"edges": [
{
"role": "Director",
"node": {
"name": {"full": "Keiichiro Saito"},
"image": {"large": "https://img.example/director.jpg"},
"siteUrl": "https://anilist.co/staff/1",
},
}
]
},
"characters": {
"edges": [
{
"node": {"name": {"full": "Frieren"}},
"voiceActors": [
{
"name": {"full": "Atsumi Tanezaki"},
"image": {"large": "https://img.example/actor.jpg"},
"siteUrl": "https://anilist.co/staff/2",
}
],
}
]
},
"externalLinks": [
{"site": "AniDB", "url": "https://anidb.net/anime/17617"}
],
}
def test_anilist_id_recognition_normalizes_media_info(anilist_info: dict) -> None:
"""AniList ID识别应生成可供整理和刮削复用的统一媒体信息。"""
module = AniListModule()
module.anilist_api = Mock()
module.anilist_api.detail.return_value = anilist_info
media = module.recognize_media(anilistid=154587)
assert media is not None
assert media.source == "anilist"
assert media.anilist_id == 154587
assert media.anidb_id == 17617
assert media.type == MediaType.TV
assert media.year == "2023"
assert media.number_of_episodes == 28
assert media.seasons[1] == list(range(1, 29))
assert media.genres == [
{"id": "Adventure", "name": "Adventure"},
{"id": "Fantasy", "name": "Fantasy"},
]
assert media.production_companies == [{"name": "Madhouse"}]
assert media.directors[0]["name"] == "Keiichiro Saito"
assert media.actors[0]["character"] == "Frieren"
module.anilist_api.detail.assert_called_once_with(154587)
def test_anilist_title_recognition_respects_request_source(anilist_info: dict) -> None:
"""标题识别仅在本次请求明确选择AniList时使用AniList候选项。"""
module = AniListModule()
module.anilist_api = Mock()
module.anilist_api.search.return_value = [anilist_info]
meta = MetaBase("Frieren")
meta.cn_name = "Frieren"
meta.type = MediaType.TV
meta.year = "2023"
media = module.recognize_media(meta=meta, source="anilist")
skipped = module.recognize_media(meta=meta, source="douban")
assert media is not None
assert media.anilist_id == 154587
assert skipped is None
module.anilist_api.search.assert_called_once_with("Frieren")
def test_async_anilist_title_recognition(anilist_info: dict) -> None:
"""异步AniList标题识别应与同步结果保持一致。"""
module = AniListModule()
module.anilist_api = Mock()
module.anilist_api.async_search = AsyncMock(return_value=[anilist_info])
meta = MetaBase("Frieren")
meta.cn_name = "Frieren"
meta.type = MediaType.TV
media = asyncio.run(
module.async_recognize_media(meta=meta, source="anilist")
)
assert media is not None
assert media.anilist_id == 154587
module.anilist_api.async_search.assert_awaited_once_with("Frieren")
def test_anilist_scraper_generates_nfo_and_images(anilist_info: dict) -> None:
"""AniList媒体信息应生成带来源ID的NFO以及主海报和背景图。"""
module = AniListModule()
module.scraper = MediaScraperHelper()
media = MediaInfo(anilist_info=anilist_info)
media.scrape_source = "anilist"
nfo = module.metadata_nfo(media)
images = module.metadata_img(media)
document = minidom.parseString(nfo)
unique_id = document.getElementsByTagName("uniqueid")[0]
assert document.documentElement.tagName == "tvshow"
assert unique_id.firstChild.data == "154587"
assert unique_id.getAttribute("type") == "anilist"
assert images == {
"poster.jpg": "https://img.example/poster.jpg",
"backdrop.png": "https://img.example/backdrop.png",
}
def test_anilist_api_extracts_graphql_errors_without_network() -> None:
"""AniList客户端应把GraphQL错误响应统一视为无结果。"""
response = Mock(status_code=200)
response.json.return_value = {"errors": [{"message": "invalid"}]}
assert AniListApi._extract_response(response) is None

View File

@@ -0,0 +1,71 @@
from pathlib import Path
from unittest.mock import patch
import pytest
from app.core.metainfo import MetaInfo, MetaInfoPath, find_metainfo
from app.schemas.types import MediaType
@pytest.mark.parametrize(
("title", "source", "media_id"),
[
("葬送的芙莉莲 {[bangumiid=400602;type=tv;s=1]}", "bangumi", "400602"),
("Frieren {[anilistid=154587;type=tv;s=1]}", "anilist", "154587"),
("Frieren [anilist=154587] S01E01", "anilist", "154587"),
],
)
def test_find_metainfo_supports_anime_source_ids(
title: str,
source: str,
media_id: str,
) -> None:
"""显式动画来源标签应提取统一来源ID并从标题中移除。"""
parsed_title, metainfo = find_metainfo(title)
assert metainfo["media_source"] == source
assert metainfo["media_id"] == media_id
assert f"{source}id=" not in parsed_title
assert f"{source}=" not in parsed_title
def test_metainfo_custom_words_support_anilist_id() -> None:
"""自定义识别词替换结果中的AniList ID应进入统一元数据字段。"""
meta = MetaInfo(
"Sousou no Frieren 01",
custom_words=[
"Sousou no Frieren => Frieren {[anilistid=154587;type=tv;s=1]}"
],
)
assert meta.media_source == "anilist"
assert meta.media_id == "154587"
assert meta.type == MediaType.TV
assert meta.begin_season == 1
def test_metainfo_path_inherits_bangumi_id_from_parent() -> None:
"""文件路径识别应从父目录继承Bangumi来源ID。"""
meta = MetaInfoPath(
Path("/anime/葬送的芙莉莲 [bangumi=400602]/Frieren.S01E01.mkv")
)
assert meta.media_source == "bangumi"
assert meta.media_id == "400602"
assert meta.begin_season == 1
assert meta.begin_episode == 1
def test_extended_ids_fall_back_when_installed_rust_is_old() -> None:
"""当前Rust扩展缺少新字段时应直接使用Python解析器。"""
with patch(
"app.core.metainfo.rust_accel.supports_extended_media_ids",
return_value=False,
), patch(
"app.core.metainfo.rust_accel.find_metainfo",
side_effect=AssertionError("旧Rust扩展不应处理扩展来源ID"),
):
_, metainfo = find_metainfo("Frieren [anilist=154587]")
assert metainfo["media_source"] == "anilist"
assert metainfo["media_id"] == "154587"

View File

@@ -0,0 +1,78 @@
from unittest.mock import Mock
from xml.dom import minidom
from app.core.meta import MetaBase
from app.helper.scraper import MediaScraperHelper
from app.modules.bangumi import BangumiModule
from app.schemas.types import MediaType
def _bangumi_info() -> dict:
"""构造Bangumi识别与刮削测试详情。"""
return {
"id": 400602,
"name": "Sousou no Frieren",
"name_cn": "葬送的芙莉莲",
"platform": "TV",
"date": "2023-09-29",
"eps": 28,
"summary": "勇者一行击败魔王后的故事。",
"images": {"large": "https://lain.example/poster.jpg"},
"rating": {"score": 8.9},
"tags": [{"name": "奇幻"}, {"name": "冒险"}],
"infobox": [
{"key": "动画制作", "value": "MADHOUSE"},
{"key": "导演", "value": [{"v": "斋藤圭一郎"}]},
],
}
def test_bangumi_title_recognition_loads_detail_and_people() -> None:
"""Bangumi标题识别应搜索候选、读取详情并补齐演职员。"""
module = BangumiModule()
module.bangumiapi = Mock()
module.bangumiapi.search.return_value = [
{"id": 400602, "name": "Sousou no Frieren", "name_cn": "葬送的芙莉莲"}
]
module.bangumiapi.detail.return_value = _bangumi_info()
module.bangumiapi.credits.return_value = [
{"name": "种崎敦美", "career": ["芙莉莲"]}
]
meta = MetaBase("葬送的芙莉莲")
meta.cn_name = "葬送的芙莉莲"
meta.type = MediaType.TV
meta.year = "2023"
media = module.recognize_media(meta=meta, source="bangumi")
assert media is not None
assert media.source == "bangumi"
assert media.bangumi_id == 400602
assert media.number_of_episodes == 28
assert media.genres == [
{"id": "奇幻", "name": "奇幻"},
{"id": "冒险", "name": "冒险"},
]
assert media.production_companies == [{"name": "MADHOUSE"}]
assert media.directors == [{"name": "斋藤圭一郎"}]
assert media.actors[0]["name"] == "种崎敦美"
def test_bangumi_scraper_generates_source_nfo() -> None:
"""Bangumi来源应可生成带Bangumi唯一ID的NFO与图片清单。"""
module = BangumiModule()
module.bangumiapi = Mock()
module.scraper = MediaScraperHelper()
module.bangumiapi.detail.return_value = _bangumi_info()
module.bangumiapi.credits.return_value = []
media = module.recognize_media(bangumiid=400602)
media.scrape_source = "bangumi"
nfo = module.metadata_nfo(media)
images = module.metadata_img(media)
document = minidom.parseString(nfo)
unique_id = document.getElementsByTagName("uniqueid")[0]
assert unique_id.firstChild.data == "400602"
assert unique_id.getAttribute("type") == "bangumi"
assert images == {"poster.jpg": "https://lain.example/poster.jpg"}

View File

@@ -0,0 +1,118 @@
from types import SimpleNamespace
from app import schemas
from app.api.endpoints import download as download_endpoint
from app.core.context import MediaInfo
from app.schemas.types import MediaType
def test_download_add_passes_generic_media_source(monkeypatch) -> None:
"""不含媒体信息的下载应按统一来源ID执行精确识别。"""
captured = {}
media = MediaInfo(
anilist_info={
"id": 154587,
"title": {"english": "Frieren"},
"format": "TV",
}
)
class FakeMediaChain:
"""记录下载接口传入的媒体识别参数。"""
def recognize_media(self, **kwargs):
"""返回固定媒体信息并保存识别参数。"""
captured["recognize"] = kwargs
return media
class FakeDownloadChain:
"""模拟下载任务提交。"""
def download_single(self, **kwargs):
"""保存下载上下文并返回任务ID。"""
captured["download"] = kwargs
return "download-1"
monkeypatch.setattr(download_endpoint, "MediaChain", FakeMediaChain)
monkeypatch.setattr(download_endpoint, "DownloadChain", FakeDownloadChain)
response = download_endpoint.add(
torrent_in=schemas.TorrentInfo(title="Frieren S01E01"),
media_source="anilist",
media_id="154587",
current_user=SimpleNamespace(name="tester"),
)
assert response.success is True
assert captured["recognize"]["source"] == "anilist"
assert captured["recognize"]["mediaid"] == "154587"
assert captured["download"]["context"].media_info is media
def test_download_add_uses_selected_source_for_title_recognition(monkeypatch) -> None:
"""只选择来源而未填写ID时应在该来源内按标题识别。"""
captured = {}
media = MediaInfo(title="测试动画", type=MediaType.TV, bangumi_id=1)
class FakeMediaChain:
"""记录按标题识别的请求级来源。"""
def recognize_by_meta(self, metainfo, **kwargs):
"""返回固定媒体信息并保存来源。"""
captured["metainfo"] = metainfo
captured["kwargs"] = kwargs
return media
class FakeDownloadChain:
"""模拟下载任务提交。"""
@staticmethod
def download_single(**kwargs):
"""返回固定下载任务ID。"""
return "download-2"
monkeypatch.setattr(download_endpoint, "MediaChain", FakeMediaChain)
monkeypatch.setattr(download_endpoint, "DownloadChain", FakeDownloadChain)
response = download_endpoint.add(
torrent_in=schemas.TorrentInfo(title="测试动画 S01E01"),
media_source="bangumi",
current_user=SimpleNamespace(name="tester"),
)
assert response.success is True
assert captured["kwargs"]["source"] == "bangumi"
def test_subtitle_download_passes_generic_media_source(monkeypatch) -> None:
"""字幕下载接口应把统一来源ID传递到下载链。"""
captured = {}
class FakeDownloadChain:
"""记录字幕下载参数。"""
def download_subtitle(self, **kwargs):
"""保存参数并返回固定成功结果。"""
captured.update(kwargs)
return True, "字幕下载成功", ["/tmp/subtitle.ass"]
monkeypatch.setattr(
download_endpoint,
"_prepare_subtitle_download",
lambda _subtitle: (True, ""),
)
monkeypatch.setattr(download_endpoint, "DownloadChain", FakeDownloadChain)
response = download_endpoint.download_subtitle(
subtitle_in=schemas.SubtitleInfo(
title="Frieren S01E01",
enclosure="https://example.com/subtitle.ass",
),
media_source="anilist",
media_id="154587",
current_user=SimpleNamespace(name="tester"),
)
assert response.success is True
assert captured["media_source"] == "anilist"
assert captured["media_id"] == "154587"

View File

@@ -317,6 +317,18 @@ def test_schedule_info_auto_fills_i18n_display_fields():
assert schedule.progress_detail.text_i18n == "Media server Emby has no libraries to sync"
assert schedule.progress_detail.error_i18n == "Background service does not exist"
token = LocaleHelper.set_current_locale("en-US")
try:
dynamic_schedule = ScheduleInfo(
id="mediaserver_sync_example",
name="同步媒体服务器 - Plex",
progress_text="同步媒体服务器 - Plex 开始执行 ...",
)
finally:
LocaleHelper.reset_current_locale(token)
assert dynamic_schedule.name_i18n == "Sync Media Server - Plex"
assert dynamic_schedule.progress_text_i18n == "Starting media server sync - Plex ..."
def test_scheduler_progress_patterns_translate_dynamic_texts():
"""定时任务进度动态模板应翻译固定词并保留业务变量。"""

View File

@@ -0,0 +1,106 @@
from unittest.mock import Mock, patch
from app.chain import ChainBase
from app.core.context import MediaInfo
from app.core.meta import MetaBase
from app.schemas.types import MediaType
def _chain_without_init() -> ChainBase:
"""构造不加载真实模块和外部服务的识别链实例。"""
return object.__new__(ChainBase)
def test_generic_source_id_wins_over_legacy_ids() -> None:
"""显式source与media_id应优先于同一请求残留的兼容ID字段。"""
resolved = ChainBase._resolve_media_source_params(
source="anilist",
mediaid="154587",
tmdbid=999,
doubanid="888",
)
assert resolved == ("anilist", None, None, None, 154587)
def test_explicit_source_recognition_runs_system_modules_only() -> None:
"""显式选择数据源时应跳过插件并只向系统模块传递该来源ID。"""
chain = _chain_without_init()
media = MediaInfo(
anilist_info={
"id": 154587,
"title": {"english": "Frieren"},
"format": "TV",
}
)
chain.run_module = Mock(return_value=media)
with patch(
"app.chain.MoviePilotServerHelper.report_recognize_share",
return_value=False,
):
result = chain.recognize_media(
source="anilist",
mediaid="154587",
tmdbid=999,
mtype=MediaType.TV,
)
assert result is media
call = chain.run_module.call_args
assert call.kwargs["system_only"] is True
assert call.kwargs["source"] == "anilist"
assert call.kwargs["anilistid"] == 154587
assert call.kwargs["tmdbid"] is None
def test_default_recognition_preserves_plugin_method_contract() -> None:
"""未显式选择来源时不应向既有插件额外传递source参数。"""
chain = _chain_without_init()
media = MediaInfo(title="测试电影", type=MediaType.MOVIE, tmdb_id=1)
chain.run_module = Mock(return_value=media)
meta = MetaBase("测试电影")
meta.cn_name = "测试电影"
meta.type = MediaType.MOVIE
with patch(
"app.chain.MoviePilotServerHelper.report_recognize_share",
return_value=False,
):
result = chain.recognize_media(meta=meta)
assert result is media
call = chain.run_module.call_args
assert call.kwargs["system_only"] is False
assert "source" not in call.kwargs
assert "anilistid" not in call.kwargs
def test_system_only_module_dispatch_skips_plugins() -> None:
"""模块调度的system_only模式不得执行插件模块。"""
chain = _chain_without_init()
chain._ChainBase__execute_plugin_modules = Mock(return_value="plugin")
chain._ChainBase__execute_system_modules = Mock(return_value="system")
result = chain.run_module("search_medias", system_only=True, meta=MetaBase("test"))
assert result == "system"
chain._ChainBase__execute_plugin_modules.assert_not_called()
chain._ChainBase__execute_system_modules.assert_called_once()
def test_explicit_search_source_uses_system_only_dispatch() -> None:
"""请求级搜索来源应进入仅系统模块调度。"""
chain = _chain_without_init()
chain.run_module = Mock(return_value=[])
meta = MetaBase("Frieren")
result = chain.search_medias(meta, source="anilist")
assert result == []
chain.run_module.assert_called_once_with(
"search_medias",
meta=meta,
source="anilist",
system_only=True,
)

View File

@@ -248,3 +248,33 @@ def test_sync_queries_counts_before_items_and_reports_media_progress(database):
assert media_progress[2]["data"]["media_finished"] == 3
progress_values = [snapshot["value"] for snapshot in progress_snapshots]
assert progress_values == sorted(progress_values)
def test_sync_targets_one_server_without_excluding_other_enabled_servers(monkeypatch):
"""定向同步只访问目标服务器,缓存清理仍保留其他已启用服务器。"""
chain = object.__new__(MediaServerChain)
library_calls = []
excluded_server_calls = []
class FakeMediaServerOper:
"""记录媒体服务器缓存清理参数的测试替身。"""
def delete_excluded_servers(self, servers):
"""记录应保留的全部已启用服务器名称。"""
excluded_server_calls.append(servers)
chain.librarys = lambda server: library_calls.append(server) or []
monkeypatch.setattr(MEDIA_SERVER_CHAIN_MODULE, "MediaServerOper", FakeMediaServerOper)
monkeypatch.setattr(
MEDIA_SERVER_CHAIN_MODULE.ServiceConfigHelper,
"get_mediaserver_configs",
lambda: [
SimpleNamespace(name="plex-a", enabled=True, sync_libraries=["all"]),
SimpleNamespace(name="plex-b", enabled=True, sync_libraries=["all"]),
],
)
chain.sync(server="plex-a")
assert library_calls == ["plex-a"]
assert excluded_server_calls == [["plex-a", "plex-b"]]

View File

@@ -0,0 +1,33 @@
from app.schemas.system import MediaServerConf
from app.scheduler import Scheduler
def test_build_mediaserver_sync_schedules_uses_server_interval_and_legacy_fallback():
"""媒体服务器自动任务应支持独立周期,并在缺省时回退旧全局值。"""
schedules = Scheduler._build_mediaserver_sync_schedules(
mediaservers=[
MediaServerConf(name="default", enabled=True),
MediaServerConf(name="custom", enabled=True, sync_interval=12),
MediaServerConf(name="disabled-sync", enabled=True, sync_interval=0),
MediaServerConf(name="disabled-server", enabled=False, sync_interval=3),
],
default_interval=6,
)
assert [(item["server"], item["interval"]) for item in schedules] == [
("default", 6),
("custom", 12),
]
assert len({item["id"] for item in schedules}) == 2
assert all(item["id"].startswith("mediaserver_sync_") for item in schedules)
def test_build_mediaserver_sync_schedules_keeps_ids_stable():
"""同名媒体服务器重载配置后应生成稳定的自动任务标识。"""
mediaservers = [MediaServerConf(name="My Plex", enabled=True, sync_interval=8)]
first = Scheduler._build_mediaserver_sync_schedules(mediaservers, 6)
second = Scheduler._build_mediaserver_sync_schedules(mediaservers, 24)
assert first[0]["id"] == second[0]["id"]
assert first[0]["interval"] == second[0]["interval"] == 8

View File

@@ -0,0 +1,82 @@
import importlib
from types import SimpleNamespace
from unittest.mock import Mock
import sqlalchemy as sa
from alembic.migration import MigrationContext
from alembic.operations import Operations
from app.core.meta import MetaBase
from app.db.transferhistory_oper import TransferHistoryOper
from app.schemas import FileItem
def test_transferhistory_migration_backfills_existing_source_ids(monkeypatch) -> None:
"""迁移应把存量TMDB和豆瓣字段回填到统一来源字段。"""
migration = importlib.import_module(
"database.versions.e6a1c4b8d2f0_2_2_13"
)
engine = sa.create_engine("sqlite://")
metadata = sa.MetaData()
transfer_history = sa.Table(
"transferhistory",
metadata,
sa.Column("id", sa.Integer(), primary_key=True),
sa.Column("tmdbid", sa.Integer()),
sa.Column("doubanid", sa.String()),
)
with engine.begin() as connection:
metadata.create_all(connection)
connection.execute(
transfer_history.insert(),
[
{"id": 1, "tmdbid": 123, "doubanid": "ignored"},
{"id": 2, "tmdbid": None, "doubanid": "456"},
{"id": 3, "tmdbid": None, "doubanid": None},
],
)
context = MigrationContext.configure(connection)
monkeypatch.setattr(migration, "op", Operations(context))
migration.upgrade()
migrated = sa.Table(
"transferhistory",
sa.MetaData(),
autoload_with=connection,
)
rows = connection.execute(
sa.select(migrated).order_by(migrated.c.id)
).mappings().all()
assert rows[0]["media_source"] == "themoviedb"
assert rows[0]["media_id"] == "123"
assert rows[1]["media_source"] == "douban"
assert rows[1]["media_id"] == "456"
assert rows[2]["media_source"] is None
assert rows[2]["media_id"] is None
def test_failed_transfer_history_preserves_explicit_media_source() -> None:
"""识别失败记录也应保存文件名中显式指定的数据源ID。"""
oper = object.__new__(TransferHistoryOper)
oper.add_force = Mock(return_value=SimpleNamespace(id=1))
meta = MetaBase("Frieren")
meta.cn_name = "Frieren"
meta.media_source = "anilist"
meta.media_id = "154587"
oper.add_fail(
fileitem=FileItem(
storage="local",
path="/downloads/Frieren.mkv",
type="file",
),
mode="copy",
meta=meta,
)
call = oper.add_force.call_args
assert call.kwargs["media_source"] == "anilist"
assert call.kwargs["media_id"] == "154587"