fix: handle OpenList delayed transfer metadata

This commit is contained in:
jxxghp
2026-05-20 13:08:45 +08:00
parent a5745af484
commit 3ec0964a01
5 changed files with 229 additions and 26 deletions
+62 -16
View File
@@ -882,6 +882,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
transferhis = TransferHistoryOper() transferhis = TransferHistoryOper()
target_dir_path = self.__get_transfer_target_dir_path(transferinfo)
# 转移失败 # 转移失败
if not transferinfo.success: if not transferinfo.success:
@@ -999,9 +1000,7 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
else: else:
# 转移成功 # 转移成功
logger.info( logger.info(f"{task.fileitem.name} 入库成功:{target_dir_path or ''}")
f"{task.fileitem.name} 入库成功:{transferinfo.target_diritem.path}"
)
# 新增task转移成功历史记录 # 新增task转移成功历史记录
history = transferhis.add_success( history = transferhis.add_success(
@@ -1059,13 +1058,13 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
) )
# task登记转移成功文件清单 # task登记转移成功文件清单
target_dir_path = transferinfo.target_diritem.path
target_files = transferinfo.file_list_new target_files = transferinfo.file_list_new
with job_lock: if target_dir_path:
if self._success_target_files.get(target_dir_path): with job_lock:
self._success_target_files[target_dir_path].extend(target_files) if self._success_target_files.get(target_dir_path):
else: self._success_target_files[target_dir_path].extend(target_files)
self._success_target_files[target_dir_path] = target_files else:
self._success_target_files[target_dir_path] = target_files
# 设置任务成功 # 设置任务成功
self.jobview.finish_task(task) self.jobview.finish_task(task)
@@ -1077,9 +1076,12 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
if self.jobview.is_finished(task): if self.jobview.is_finished(task):
# 更新文件清单 # 更新文件清单
with job_lock: with job_lock:
transferinfo.file_list_new = self._success_target_files.pop( if target_dir_path:
transferinfo.target_diritem.path, [] transferinfo.file_list_new = self._success_target_files.pop(
) target_dir_path, []
)
else:
transferinfo.file_list_new = transferinfo.file_list_new or []
__notify() __notify()
if not task.transfer_batch_id: if not task.transfer_batch_id:
self.__send_metadata_scrape_event(task, transferinfo) self.__send_metadata_scrape_event(task, transferinfo)
@@ -1121,6 +1123,45 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
return ret_status, ret_message return ret_status, ret_message
def __get_transfer_target_dir_path(
self, transferinfo: Optional[TransferInfo]
) -> Optional[str]:
"""
获取整理目标目录路径,兼容 OpenList 等成功后目录项短时间不可见的存储。
"""
if not transferinfo:
return None
if transferinfo.target_diritem and transferinfo.target_diritem.path:
return transferinfo.target_diritem.path
if transferinfo.target_item and transferinfo.target_item.path:
return Path(transferinfo.target_item.path).parent.as_posix()
if transferinfo.file_list_new:
return Path(transferinfo.file_list_new[0]).parent.as_posix()
return None
def __build_transfer_target_diritem(
self, transferinfo: Optional[TransferInfo]
) -> Optional[FileItem]:
"""
构建整理目标目录项,避免成功结果缺少 target_diritem 时阻断后续流程。
"""
if not transferinfo:
return None
if transferinfo.target_diritem:
return transferinfo.target_diritem
target_dir_path = self.__get_transfer_target_dir_path(transferinfo)
if not target_dir_path:
return None
target_path = Path(target_dir_path)
storage = transferinfo.target_item.storage if transferinfo.target_item else "local"
return FileItem(
storage=storage,
path=target_dir_path,
type="dir",
name=target_path.name,
basename=target_path.stem,
)
def put_to_queue(self, task: TransferTask) -> bool: def put_to_queue(self, task: TransferTask) -> bool:
""" """
添加到待整理队列 添加到待整理队列
@@ -1170,17 +1211,20 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
not task not task
or not transferinfo or not transferinfo
or not transferinfo.need_scrape or not transferinfo.need_scrape
or not transferinfo.target_diritem
or not self.__is_media_file(task.fileitem) or not self.__is_media_file(task.fileitem)
): ):
return return
target_diritem = self.__build_transfer_target_diritem(transferinfo)
if not target_diritem:
return
self.eventmanager.send_event( self.eventmanager.send_event(
EventType.MetadataScrape, EventType.MetadataScrape,
{ {
"meta": task.meta, "meta": task.meta,
"mediainfo": task.mediainfo, "mediainfo": task.mediainfo,
"fileitem": transferinfo.target_diritem, "fileitem": target_diritem,
"file_list": transferinfo.file_list_new, "file_list": transferinfo.file_list_new,
"overwrite": False, "overwrite": False,
}, },
@@ -1230,12 +1274,14 @@ class TransferChain(ChainBase, ConfigReloadMixin, metaclass=Singleton):
or not task.transfer_batch_id or not task.transfer_batch_id
or not transferinfo or not transferinfo
or not transferinfo.need_scrape or not transferinfo.need_scrape
or not transferinfo.target_diritem
or not self.__is_media_file(task.fileitem) or not self.__is_media_file(task.fileitem)
): ):
return return
target_diritem = transferinfo.target_diritem target_diritem = self.__build_transfer_target_diritem(transferinfo)
if not target_diritem:
return
target_files = transferinfo.file_list_new or [] target_files = transferinfo.file_list_new or []
target_key = (target_diritem.storage, target_diritem.path) target_key = (target_diritem.storage, target_diritem.path)
with job_lock: with job_lock:
+53
View File
@@ -61,6 +61,24 @@ class Alist(StorageBase, metaclass=WeakSingleton):
return fileitem return fileitem
return None return None
def __build_transfer_item(
self, source_item: schemas.FileItem, target_path: Path
) -> schemas.FileItem:
"""
根据目标路径构造文件项,用于 OpenList 操作成功但元数据短时间不可见的场景。
"""
return schemas.FileItem(
storage=self.schema.value,
type=source_item.type,
path=target_path.as_posix(),
name=target_path.name,
basename=target_path.stem,
extension=target_path.suffix[1:] if source_item.type != "dir" else None,
size=getattr(source_item, "size", None),
modify_time=getattr(source_item, "modify_time", None),
thumbnail=getattr(source_item, "thumbnail", None),
)
@property @property
def __get_base_url(self) -> str: def __get_base_url(self) -> str:
""" """
@@ -799,6 +817,28 @@ class Alist(StorageBase, metaclass=WeakSingleton):
self.rename(new_item, new_name) self.rename(new_item, new_name)
return True return True
def copy_item(
self, fileitem: schemas.FileItem, path: Path, new_name: str
) -> Optional[schemas.FileItem]:
"""
复制文件并返回目标文件项,兼容 OpenList 成功响应不携带目标对象的格式。
"""
if not self.copy(fileitem=fileitem, path=path, new_name=new_name):
return None
target_path = path / new_name
target_item = self._delay_get_item(target_path, refresh=True)
if target_item:
return target_item
if fileitem.name == new_name:
return self.__build_transfer_item(fileitem, target_path)
copied_item = self._delay_get_item(path / fileitem.name, refresh=True)
if copied_item and self.rename(copied_item, new_name):
return self._delay_get_item(
target_path, refresh=True
) or self.__build_transfer_item(fileitem, target_path)
return None
def move(self, fileitem: schemas.FileItem, path: Path, new_name: str) -> bool: def move(self, fileitem: schemas.FileItem, path: Path, new_name: str) -> bool:
""" """
移动文件 移动文件
@@ -852,6 +892,19 @@ class Alist(StorageBase, metaclass=WeakSingleton):
return False return False
return True return True
def move_item(
self, fileitem: schemas.FileItem, path: Path, new_name: str
) -> Optional[schemas.FileItem]:
"""
移动文件并返回目标文件项,兼容 OpenList 成功响应不携带目标对象的格式。
"""
if not self.move(fileitem=fileitem, path=path, new_name=new_name):
return None
target_path = path / new_name
return self._delay_get_item(target_path, refresh=True) or self.__build_transfer_item(
fileitem, target_path
)
def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool: def link(self, fileitem: schemas.FileItem, target_file: Path) -> bool:
""" """
硬链接文件 硬链接文件
+26 -10
View File
@@ -709,12 +709,20 @@ class TransHandler:
# 复制文件到新目录 # 复制文件到新目录
target_fileitem = target_oper.get_folder(target_file.parent) target_fileitem = target_oper.get_folder(target_file.parent)
if target_fileitem: if target_fileitem:
if source_oper.copy( copy_item = getattr(source_oper, "copy_item", None)
fileitem, Path(target_fileitem.path), target_file.name if callable(copy_item):
new_item = copy_item(
fileitem, Path(target_fileitem.path), target_file.name
)
if new_item:
return new_item, ""
elif source_oper.copy(
fileitem, Path(target_fileitem.path), target_file.name
): ):
return target_oper.get_item(target_file), "" new_item = target_oper.get_item(target_file)
else: if new_item:
return None, f"{target_storage}{fileitem.path} 复制文件失败" return new_item, ""
return None, f"{target_storage}{fileitem.path} 复制文件失败"
else: else:
return ( return (
None, None,
@@ -724,12 +732,20 @@ class TransHandler:
# 移动文件到新目录 # 移动文件到新目录
target_fileitem = target_oper.get_folder(target_file.parent) target_fileitem = target_oper.get_folder(target_file.parent)
if target_fileitem: if target_fileitem:
if source_oper.move( move_item = getattr(source_oper, "move_item", None)
fileitem, Path(target_fileitem.path), target_file.name if callable(move_item):
new_item = move_item(
fileitem, Path(target_fileitem.path), target_file.name
)
if new_item:
return new_item, ""
elif source_oper.move(
fileitem, Path(target_fileitem.path), target_file.name
): ):
return target_oper.get_item(target_file), "" new_item = target_oper.get_item(target_file)
else: if new_item:
return None, f"{target_storage}{fileitem.path} 移动文件失败" return new_item, ""
return None, f"{target_storage}{fileitem.path} 移动文件失败"
else: else:
return ( return (
None, None,
+38
View File
@@ -200,6 +200,44 @@ class AlistStorageTest(unittest.TestCase):
self.assertEqual(50, len(items)) self.assertEqual(50, len(items))
self.assertEqual(1, request_utils.post_res.call_count) self.assertEqual(1, request_utils.post_res.call_count)
def test_move_item_returns_target_when_openlist_metadata_is_delayed(self):
"""
OpenList 操作成功但目标元数据延迟可见时,应返回可用的目标文件项。
"""
source = FileItem(
storage="alist",
type="file",
path="/downloads/Test.Show.S01E01.mkv",
name="Test.Show.S01E01.mkv",
basename="Test.Show.S01E01",
extension="mkv",
size=1024,
modify_time=1715939275.0,
)
request_utils = MagicMock()
request_utils.post_res.return_value = _FakeResponse(
{"code": 200, "message": "success", "data": None}
)
with patch.object(Alist, "get_conf", return_value={"url": "http://openlist.test", "token": "token"}):
with patch.object(self.storage, "_Alist__get_header_with_token", return_value={}):
with patch.object(alist_module, "RequestUtils", return_value=request_utils):
with patch.object(self.storage, "_delay_get_item", return_value=None):
target = self.storage.move_item(
source,
Path("/library/Test Show (2026)/Season 1"),
"Test.Show.S01E01.mkv",
)
self.assertIsNotNone(target)
self.assertEqual(
"/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv",
target.path,
)
self.assertEqual("alist", target.storage)
self.assertEqual("file", target.type)
self.assertEqual(1024, target.size)
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()
+50
View File
@@ -725,6 +725,56 @@ class TransferJobManagerTest(unittest.TestCase):
event_data["file_list"], event_data["file_list"],
) )
def test_success_callback_handles_missing_target_diritem(self):
"""
成功结果缺少目标目录项时,回调不应把已入库任务误判为失败。
"""
chain = make_transfer_chain()
chain.eventmanager = MagicMock()
chain.transfer_completed = lambda *args, **kwargs: None
task = make_task(1)
task.mediainfo = FakeMedia()
task.background = False
task.manual = True
self.assertTrue(chain._TransferChain__put_to_jobview(task))
target_item = FileItem(
storage="alist",
path="/library/Test Show (2026)/Season 1/Test.Show.S01E01.mkv",
type="file",
name="Test.Show.S01E01.mkv",
extension="mkv",
)
transferinfo = TransferInfo(
success=True,
fileitem=task.fileitem,
target_item=target_item,
file_list_new=[target_item.path],
transfer_type="copy",
need_scrape=True,
need_notify=False,
)
with patch(
"app.chain.transfer.TransferHistoryOper",
return_value=SimpleNamespace(add_success=lambda **kwargs: SimpleNamespace(id=1)),
):
state, errmsg = chain._TransferChain__default_callback(task, transferinfo)
self.assertTrue(state)
self.assertEqual("", errmsg)
metadata_calls = [
call
for call in chain.eventmanager.send_event.call_args_list
if call.args[0] == EventType.MetadataScrape
]
self.assertEqual(1, len(metadata_calls))
event_data = metadata_calls[0].args[1]
self.assertEqual("alist", event_data["fileitem"].storage)
self.assertEqual("/library/Test Show (2026)/Season 1", event_data["fileitem"].path)
self.assertEqual([target_item.path], event_data["file_list"])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()