fix(transfer): 修复部分情况下无法正确统计已完成任务总大小的问题

- get_directory_size 使用 os.scandir 递归遍历提升性能
- 当任务文件项存储类型为 local 时,若其大小为空,则通过 SystemUtils 获取目录大小以确保
完成任务的准确统计。

fix(cache): 修改 fresh 和 async_fresh 默认参数为 True

refactor(filemanager): 移除整理后总大小计算逻辑

- 删除 TransHandler 中对整理目录总大小的冗余计算,提升性能并简化流程。

perf(system): 使用 scandir 优化文件扫描性能

- 重构 SystemUtils 中的文件扫描方法(list_files、exists_file、list_sub_files),
- 采用 os.scandir 替代 glob 实现,并预编译正则表达式以提升目录遍历与文件匹配性能。
This commit is contained in:
Attente
2025-10-23 19:21:24 +08:00
parent 0c58eae5e7
commit 7421ca09cc
4 changed files with 88 additions and 49 deletions

View File

@@ -33,6 +33,7 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey,
SystemConfigKey, ChainEventType, ContentType SystemConfigKey, ChainEventType, ContentType
from app.utils.singleton import Singleton from app.utils.singleton import Singleton
from app.utils.string import StringUtils from app.utils.string import StringUtils
from app.utils.system import SystemUtils
downloader_lock = threading.Lock() downloader_lock = threading.Lock()
job_lock = threading.Lock() job_lock = threading.Lock()
@@ -329,8 +330,12 @@ class JobManager:
# 计算状态为完成的任务数 # 计算状态为完成的任务数
if __mediaid__ not in self._job_view: if __mediaid__ not in self._job_view:
return 0 return 0
return sum([task.fileitem.size for task in self._job_view[__mediaid__].tasks if return sum([
task.state == "completed" and task.fileitem.size is not None]) task.fileitem.size if task.fileitem.size is not None
else (SystemUtils.get_directory_size(Path(task.fileitem.path)) if task.fileitem.storage == "local" else 0)
for task in self._job_view[__mediaid__].tasks
if task.state == "completed"
])
def total(self) -> int: def total(self) -> int:
""" """

View File

@@ -1016,7 +1016,7 @@ class AsyncFileBackend(AsyncCacheBackend):
@contextmanager @contextmanager
def fresh(fresh: bool = False): def fresh(fresh: bool = True):
""" """
是否获取新数据(不使用缓存的值) 是否获取新数据(不使用缓存的值)
@@ -1031,7 +1031,7 @@ def fresh(fresh: bool = False):
_fresh.reset(token) _fresh.reset(token)
@asynccontextmanager @asynccontextmanager
async def async_fresh(fresh: bool = False): async def async_fresh(fresh: bool = True):
""" """
是否获取新数据(不使用缓存的值) 是否获取新数据(不使用缓存的值)

View File

@@ -150,14 +150,11 @@ class TransHandler:
return self.result.copy() return self.result.copy()
logger.info(f"文件夹 {fileitem.path} 整理成功") logger.info(f"文件夹 {fileitem.path} 整理成功")
# 计算目录下所有文件大小
total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
# 返回整理后的路径 # 返回整理后的路径
self.__set_result(success=True, self.__set_result(success=True,
fileitem=fileitem, fileitem=fileitem,
target_item=new_diritem, target_item=new_diritem,
target_diritem=new_diritem, target_diritem=new_diritem,
total_size=total_size,
need_scrape=need_scrape, need_scrape=need_scrape,
need_notify=need_notify, need_notify=need_notify,
transfer_type=transfer_type) transfer_type=transfer_type)

View File

@@ -7,7 +7,6 @@ import shutil
import subprocess import subprocess
import sys import sys
import uuid import uuid
from glob import glob
from pathlib import Path from pathlib import Path
from typing import List, Optional, Tuple, Union from typing import List, Optional, Tuple, Union
@@ -225,23 +224,31 @@ class SystemUtils:
if directory.is_file(): if directory.is_file():
return [directory] return [directory]
if not min_filesize:
min_filesize = 0
files = [] files = []
# 预编译正则表达式
if extensions: if extensions:
pattern = r".*(" + "|".join(extensions) + ")$" pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
else: else:
pattern = r".*" pattern = re.compile(r".*")
# 遍历目录及子目录 def _scan_directory(dir_path: Path, is_recursive: bool):
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True): try:
path = directory.joinpath(matched_glob) with os.scandir(dir_path) as entries:
if path.is_file() \ for entry in entries:
and re.match(pattern, path.name, re.IGNORECASE) \ try:
and path.stat().st_size >= min_filesize * 1024 * 1024: if entry.is_file(follow_symlinks=False):
files.append(path) entry_path = Path(entry.path)
if (pattern.match(entry.name) and
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
files.append(entry_path)
elif entry.is_dir() and is_recursive:
_scan_directory(Path(entry.path), is_recursive)
except (OSError, PermissionError):
continue
except (OSError, PermissionError):
pass
_scan_directory(directory, recursive)
return files return files
@staticmethod @staticmethod
@@ -256,29 +263,44 @@ class SystemUtils:
:return: True存在 False不存在 :return: True存在 False不存在
""" """
if not min_filesize:
min_filesize = 0
if not directory.exists(): if not directory.exists():
return False return False
# 预编译正则表达式
if extensions:
pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
else:
pattern = re.compile(r".*")
if directory.is_file(): if directory.is_file():
# 检查单个文件是否符合条件
if extensions and not pattern.match(directory.name):
return False
if min_filesize > 0 and directory.stat().st_size < min_filesize * 1024 * 1024:
return False
return True return True
if not min_filesize: def _search_files(dir_path: Path, is_recursive: bool) -> bool:
min_filesize = 0 try:
with os.scandir(dir_path) as entries:
for entry in entries:
try:
if entry.is_file(follow_symlinks=False):
# 检查文件是否符合条件
if (pattern.match(entry.name) and
(min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
return True
elif entry.is_dir() and is_recursive:
# 递归搜索子目录
if _search_files(Path(entry.path), is_recursive):
return True
except (OSError, PermissionError):
continue
except (OSError, PermissionError):
pass
return False
pattern = r".*(" + "|".join(extensions) + ")$" return _search_files(directory, recursive)
# 遍历目录及子目录
for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
path = directory.joinpath(matched_glob)
if path.is_file() \
and re.match(pattern, path.name, re.IGNORECASE) \
and path.stat().st_size >= min_filesize * 1024 * 1024:
return True
return False
@staticmethod @staticmethod
def list_sub_files(directory: Path, extensions: list) -> List[Path]: def list_sub_files(directory: Path, extensions: list) -> List[Path]:
@@ -292,12 +314,20 @@ class SystemUtils:
return [directory] return [directory]
files = [] files = []
pattern = r".*(" + "|".join(extensions) + ")$"
# 遍历目录 # 预编译正则表达式
for path in directory.iterdir(): if extensions:
if path.is_file() and re.match(pattern, path.name, re.IGNORECASE): pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
files.append(path) else:
pattern = re.compile(r".*")
try:
with os.scandir(directory) as entries:
for entry in entries:
if entry.is_file() and pattern.match(entry.name):
files.append(Path(entry.path))
except OSError:
pass
return files return files
@@ -346,7 +376,7 @@ class SystemUtils:
return items return items
@staticmethod @staticmethod
def get_directory_size(path: Path) -> float: def get_directory_size(path: Path) -> int:
""" """
计算目录的大小 计算目录的大小
@@ -358,14 +388,21 @@ class SystemUtils:
""" """
if not path or not path.exists(): if not path or not path.exists():
return 0 return 0
if path.is_file():
return path.stat().st_size
total_size = 0
for path in path.glob('**/*'):
if path.is_file():
total_size += path.stat().st_size
return total_size def _calc_dir_size(dir_path):
total = 0
try:
with os.scandir(dir_path) as entries:
for entry in entries:
if entry.is_file():
total += entry.stat().st_size
elif entry.is_dir():
total += _calc_dir_size(entry.path)
except OSError:
pass
return total
return _calc_dir_size(path) if path.is_dir() else path.stat().st_size
@staticmethod @staticmethod
def space_usage(dir_list: Union[Path, List[Path]]) -> Tuple[float, float]: def space_usage(dir_list: Union[Path, List[Path]]) -> Tuple[float, float]: