refactor(chain): type download and media server ports

This commit is contained in:
jxxghp
2026-08-28 07:47:37 +08:00
parent c3f115d4fb
commit 5fb62108ab
25 changed files with 1262 additions and 331 deletions
+54 -35
View File
@@ -6,7 +6,7 @@ import re
import shutil
import time
from pathlib import Path
from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
from typing import Dict, List, Optional, Set, Tuple, Union
from urllib.parse import parse_qs, urlencode, urljoin, urlparse
from app.adapters.network.http import RequestUtils
@@ -19,6 +19,11 @@ from app.application.chain.data import (
from app.application.configuration import get_chain_runtime_config_snapshot
from app.application.directory import DirectoryHelper, validate_download_save_path
from app.application.download import selection as _selection
from app.application.download.failures import (
DownloadFailureRepository,
DownloadFailureSnapshot,
DownloadFailureWrite,
)
from app.application.download.tasks import DownloadTaskService
from app.application.torrent import TorrentHelper
from app.chain import ChainBase
@@ -63,12 +68,6 @@ from app.schemas.types import (
)
from app.schemas.workflow import FileItem as _SchemaFileItem
if TYPE_CHECKING:
from typing import Any
DownloadFailure = Any
DOWNLOAD_FAILURE_RESOURCE_TTL_SECONDS = 24 * 60 * 60
DOWNLOAD_FAILURE_TRANSIENT_TTL_SECONDS = 60 * 60
DOWNLOAD_FAILURE_RESOURCE_ERROR_KEYWORDS = (
@@ -880,25 +879,40 @@ class DownloadChain(ChainBase):
torrent = context.torrent_info
site = getattr(torrent, "site", None)
try:
get_chain_download_failure_port().record_failure(
fingerprint=fingerprint,
now_time=now_time,
next_retry_at=next_retry_at,
type=getattr(getattr(media, "type", None), "value", getattr(media, "type", None)),
title=getattr(media, "title", None),
year=getattr(media, "year", None),
media_source=media_source,
media_id=media_id,
seasons=getattr(meta, "season", None),
episodes=episode_rules.format_ranges(list(episodes)) if episodes else self._format_failure_episodes(meta),
site=site if isinstance(site, int) else None,
site_name=getattr(torrent, "site_name", None),
torrent_id=self._torrent_resource_key(torrent),
torrent_name=getattr(torrent, "title", None),
torrent_size=getattr(torrent, "size", None),
downloader=downloader,
source=str(source)[:1000] if source else None,
error_message=str(error_msg or "")[:1000],
repository: DownloadFailureRepository = get_chain_download_failure_port()
repository.record_failure(
DownloadFailureWrite(
fingerprint=fingerprint,
failed_at=now_time,
next_retry_at=next_retry_at,
media_type=getattr(
getattr(media, "type", None),
"value",
getattr(media, "type", None),
),
title=getattr(media, "title", None),
year=getattr(media, "year", None),
media_source=media_source,
media_id=media_id,
seasons=(
str(getattr(meta, "season", None))
if getattr(meta, "season", None) is not None
else None
),
episodes=(
episode_rules.format_ranges(list(episodes))
if episodes
else self._format_failure_episodes(meta)
),
site=site if isinstance(site, int) else None,
site_name=getattr(torrent, "site_name", None),
torrent_id=self._torrent_resource_key(torrent),
torrent_name=getattr(torrent, "title", None),
torrent_size=getattr(torrent, "size", None),
downloader=downloader,
source=str(source)[:1000] if source else None,
error_message=str(error_msg or "")[:1000],
)
)
except Exception as err:
logger.error(f"记录下载失败冷却失败:{str(err)}")
@@ -908,7 +922,7 @@ class DownloadChain(ChainBase):
self,
contexts: List[Context],
source: Optional[str],
) -> Dict[str, "DownloadFailure"]:
) -> Dict[str, DownloadFailureSnapshot]:
"""
查询当前订阅候选中仍处于冷却期的失败记录,返回指纹到失败记录的映射。
"""
@@ -926,7 +940,8 @@ class DownloadChain(ChainBase):
return {}
now_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
try:
return get_chain_download_failure_port().get_active_by_fingerprints(
repository: DownloadFailureRepository = get_chain_download_failure_port()
return repository.get_active_by_fingerprints(
fingerprints=fingerprints, now_time=now_time,
)
except Exception as err:
@@ -936,7 +951,7 @@ class DownloadChain(ChainBase):
@staticmethod
def _log_download_failure_cooldown(
context: Context,
failure: Optional["DownloadFailure"],
failure: Optional[DownloadFailureSnapshot],
) -> None:
"""记录候选资源处于失败冷却期时的跳过原因和下次重试时间。"""
reason = getattr(failure, "error_message", None) or "未知原因"
@@ -957,7 +972,7 @@ class DownloadChain(ChainBase):
contexts: List[Context],
downloader: Optional[str],
source: Optional[str],
) -> Tuple[List[Context], Dict[str, "DownloadFailure"]]:
) -> Tuple[List[Context], Dict[str, Optional[DownloadFailureSnapshot]]]:
"""
执行批量下载前的资源选择、排序和失败冷却准备。
@@ -979,16 +994,20 @@ class DownloadChain(ChainBase):
)
contexts = event_data.updated_contexts
contexts = TorrentHelper().sort_torrents(contexts)
return contexts, self._active_download_failure_fingerprints(
contexts=contexts,
source=source,
)
active_failures: Dict[str, Optional[DownloadFailureSnapshot]] = {
fingerprint: failure
for fingerprint, failure in self._active_download_failure_fingerprints(
contexts=contexts,
source=source,
).items()
}
return contexts, active_failures
def _download_movie_music_candidates(
self,
contexts: List[Context],
downloaded_list: List[Context],
active_failure_records: Dict[str, "DownloadFailure"],
active_failure_records: Dict[str, Optional[DownloadFailureSnapshot]],
save_path: Optional[str],
channel: Optional[NotificationChannel],
source: Optional[str],
+26 -16
View File
@@ -3,7 +3,11 @@ from datetime import datetime
from typing import Any, Callable, Dict, Generator, List, Optional, Tuple, Union
from app.application.chain.data import get_chain_media_server_port
from app.application.mediaserver import get_mediaserver_configs
from app.application.mediaserver import (
MediaServerRepository,
MediaServerSyncItem,
get_mediaserver_configs,
)
from app.application.security.url import SecurityUtils
from app.chain import ChainBase
from app.runtime.log import logger
@@ -271,6 +275,7 @@ class MediaServerChain(ChainBase):
self,
mediaservers: List[Any],
server: Optional[str],
repository: MediaServerRepository,
) -> Tuple[List[Any], int, Dict[str, Any], Optional[int]]:
"""
准备启用媒体服务器的同步库和进度总量。
@@ -281,8 +286,7 @@ class MediaServerChain(ChainBase):
item.name for item in mediaservers
if item and item.enabled and item.name
]
dboper = get_chain_media_server_port()
dboper.delete_excluded_servers(enabled_servers)
repository.delete_excluded_servers(enabled_servers)
selected_servers = [
item for item in mediaservers
if item and item.enabled and (not server or item.name == server)
@@ -332,7 +336,7 @@ class MediaServerChain(ChainBase):
server_name: str,
selected_libraries: List[Any],
library_media_counts: Dict[str, Optional[int]],
dboper: Any,
repository: MediaServerRepository,
sync_time: str,
progress_callback: Optional[Callable[..., None]],
server_index: int,
@@ -363,14 +367,17 @@ class MediaServerChain(ChainBase):
item_type = self._normalize_item_type(item.item_type)
if item_type == MediaType.TV.value:
for episode in self.episodes(server_name, item.item_id) or []:
seasoninfo[episode.season] = episode.episodes
item_dict = item.model_dump()
item_dict.update({
"seasoninfo": seasoninfo,
"item_type": item_type,
"lst_mod_date": sync_time,
})
dboper.upsert(**item_dict)
if episode.season is None:
continue
seasoninfo[episode.season] = episode.episodes or []
repository.upsert(
MediaServerSyncItem.from_item(
item,
item_type=item_type,
seasoninfo=seasoninfo,
sync_time=sync_time,
)
)
if progress_callback:
if global_media_total:
progress_value = min(global_media_finished / global_media_total, 1) * 100
@@ -423,7 +430,10 @@ class MediaServerChain(ChainBase):
"media_finished": global_media_finished,
},
)
stale_count = dboper.delete_stale(server=server_name, sync_time=sync_time)
stale_count = repository.delete_stale(
server=server_name,
sync_time=sync_time,
)
logger.info(f"媒体服务器 {server_name} 清理陈旧数据完成,删除数量:{stale_count}")
return total_count, global_media_finished
@@ -463,9 +473,9 @@ class MediaServerChain(ChainBase):
with lock:
# 汇总统计
total_count = 0
dboper = get_chain_media_server_port()
repository = get_chain_media_server_port()
mediaservers, total_servers, server_sync_contexts, global_media_total = (
self._prepare_sync_contexts(mediaservers, server)
self._prepare_sync_contexts(mediaservers, server, repository)
)
if progress_callback:
progress_callback(
@@ -543,7 +553,7 @@ class MediaServerChain(ChainBase):
server_name=server_name,
selected_libraries=selected_libraries,
library_media_counts=library_media_counts,
dboper=dboper,
repository=repository,
sync_time=sync_time,
progress_callback=progress_callback,
server_index=server_index,