Merge pull request #5071 from wikrin/optimize-file-size

app/chain/transfer.py

@@ -33,6 +33,7 @@ from app.schemas.types import TorrentStatus, EventType, MediaType, ProgressKey,
     SystemConfigKey, ChainEventType, ContentType
 from app.utils.singleton import Singleton
 from app.utils.string import StringUtils
+from app.utils.system import SystemUtils
 
 downloader_lock = threading.Lock()
 job_lock = threading.Lock()
@@ -329,8 +330,12 @@ class JobManager:
             # 计算状态为完成的任务数
             if __mediaid__ not in self._job_view:
                 return 0
-            return sum([task.fileitem.size for task in self._job_view[__mediaid__].tasks if
+            return sum([
-                        task.state == "completed" and task.fileitem.size is not None])
+                task.fileitem.size if task.fileitem.size is not None
+                else (SystemUtils.get_directory_size(Path(task.fileitem.path)) if task.fileitem.storage == "local" else 0)
+                for task in self._job_view[__mediaid__].tasks
+                if task.state == "completed"
+            ])
 
     def total(self) -> int:
         """

app/core/cache.py

@@ -1016,7 +1016,7 @@ class AsyncFileBackend(AsyncCacheBackend):
 
 
 @contextmanager
-def fresh(fresh: bool = False):
+def fresh(fresh: bool = True):
     """
     是否获取新数据（不使用缓存的值）
 
@@ -1031,7 +1031,7 @@ def fresh(fresh: bool = False):
         _fresh.reset(token)
 
 @asynccontextmanager
-async def async_fresh(fresh: bool = False):
+async def async_fresh(fresh: bool = True):
     """
     是否获取新数据（不使用缓存的值）
 

app/modules/filemanager/transhandler.py

@@ -150,14 +150,11 @@ class TransHandler:
                     return self.result.copy()
 
                 logger.info(f"文件夹 {fileitem.path} 整理成功")
-                # 计算目录下所有文件大小
-                total_size = sum(file.stat().st_size for file in Path(fileitem.path).rglob('*') if file.is_file())
                 # 返回整理后的路径
                 self.__set_result(success=True,
                                   fileitem=fileitem,
                                   target_item=new_diritem,
                                   target_diritem=new_diritem,
-                                  total_size=total_size,
                                   need_scrape=need_scrape,
                                   need_notify=need_notify,
                                   transfer_type=transfer_type)

app/utils/system.py

@@ -7,7 +7,6 @@ import shutil
 import subprocess
 import sys
 import uuid
-from glob import glob
 from pathlib import Path
 from typing import List, Optional, Tuple, Union
 
@@ -225,23 +224,31 @@ class SystemUtils:
         if directory.is_file():
             return [directory]
 
-        if not min_filesize:
-            min_filesize = 0
-
         files = []
+        # 预编译正则表达式
         if extensions:
-            pattern = r".*(" + "|".join(extensions) + ")$"
+            pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
         else:
-            pattern = r".*"
+            pattern = re.compile(r".*")
 
-        # 遍历目录及子目录
+        def _scan_directory(dir_path: Path, is_recursive: bool):
-        for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
+            try:
-            path = directory.joinpath(matched_glob)
+                with os.scandir(dir_path) as entries:
-            if path.is_file() \
+                    for entry in entries:
-                    and re.match(pattern, path.name, re.IGNORECASE) \
+                        try:
-                    and path.stat().st_size >= min_filesize * 1024 * 1024:
+                            if entry.is_file(follow_symlinks=False):
-                files.append(path)
+                                entry_path = Path(entry.path)
+                                if (pattern.match(entry.name) and
+                                    (min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
+                                    files.append(entry_path)
+                            elif entry.is_dir() and is_recursive:
+                                _scan_directory(Path(entry.path), is_recursive)
+                        except (OSError, PermissionError):
+                            continue
+            except (OSError, PermissionError):
+                pass
 
+        _scan_directory(directory, recursive)
         return files
 
     @staticmethod
@@ -256,29 +263,44 @@ class SystemUtils:
         :return: True存在 False不存在
         """
 
-        if not min_filesize:
-            min_filesize = 0
-
         if not directory.exists():
             return False
 
+        # 预编译正则表达式
+        if extensions:
+            pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
+        else:
+            pattern = re.compile(r".*")
+
         if directory.is_file():
+            # 检查单个文件是否符合条件
+            if extensions and not pattern.match(directory.name):
+                return False
+            if min_filesize > 0 and directory.stat().st_size < min_filesize * 1024 * 1024:
+                return False
             return True
 
-        if not min_filesize:
+        def _search_files(dir_path: Path, is_recursive: bool) -> bool:
-            min_filesize = 0
+            try:
+                with os.scandir(dir_path) as entries:
+                    for entry in entries:
+                        try:
+                            if entry.is_file(follow_symlinks=False):
+                                # 检查文件是否符合条件
+                                if (pattern.match(entry.name) and
+                                    (min_filesize <= 0 or entry.stat().st_size >= min_filesize * 1024 * 1024)):
+                                    return True
+                            elif entry.is_dir() and is_recursive:
+                                # 递归搜索子目录
+                                if _search_files(Path(entry.path), is_recursive):
+                                    return True
+                        except (OSError, PermissionError):
+                            continue
+            except (OSError, PermissionError):
+                pass
+            return False
 
-        pattern = r".*(" + "|".join(extensions) + ")$"
+        return _search_files(directory, recursive)
-
-        # 遍历目录及子目录
-        for matched_glob in glob('**', root_dir=directory, recursive=recursive, include_hidden=True):
-            path = directory.joinpath(matched_glob)
-            if path.is_file() \
-                    and re.match(pattern, path.name, re.IGNORECASE) \
-                    and path.stat().st_size >= min_filesize * 1024 * 1024:
-                return True
-
-        return False
 
     @staticmethod
     def list_sub_files(directory: Path, extensions: list) -> List[Path]:
@@ -292,12 +314,20 @@ class SystemUtils:
             return [directory]
 
         files = []
-        pattern = r".*(" + "|".join(extensions) + ")$"
 
-        # 遍历目录
+        # 预编译正则表达式
-        for path in directory.iterdir():
+        if extensions:
-            if path.is_file() and re.match(pattern, path.name, re.IGNORECASE):
+            pattern = re.compile(r".*(" + "|".join(extensions) + r")$", re.IGNORECASE)
-                files.append(path)
+        else:
+            pattern = re.compile(r".*")
+
+        try:
+            with os.scandir(directory) as entries:
+                for entry in entries:
+                    if entry.is_file() and pattern.match(entry.name):
+                        files.append(Path(entry.path))
+        except OSError:
+            pass
 
         return files
 
@@ -346,7 +376,7 @@ class SystemUtils:
         return items
 
     @staticmethod
-    def get_directory_size(path: Path) -> float:
+    def get_directory_size(path: Path) -> int:
         """
         计算目录的大小
 
@@ -358,14 +388,21 @@ class SystemUtils:
         """
         if not path or not path.exists():
             return 0
-        if path.is_file():
-            return path.stat().st_size
-        total_size = 0
-        for path in path.glob('**/*'):
-            if path.is_file():
-                total_size += path.stat().st_size
 
-        return total_size
+        def _calc_dir_size(dir_path):
+            total = 0
+            try:
+                with os.scandir(dir_path) as entries:
+                    for entry in entries:
+                        if entry.is_file():
+                            total += entry.stat().st_size
+                        elif entry.is_dir():
+                            total += _calc_dir_size(entry.path)
+            except OSError:
+                pass
+            return total
+
+        return _calc_dir_size(path) if path.is_dir() else path.stat().st_size
 
     @staticmethod
     def space_usage(dir_list: Union[Path, List[Path]]) -> Tuple[float, float]: