fix(openlist): paginate Alist directory listings

Fetch all OpenList/AList fs/list pages when using the default per_page=0 to avoid truncating directories at 200 entries. Add an isolated regression test for the auto-pagination behavior.

Fixes #5723

app/modules/filemanager/storages/alist.py

@@ -176,86 +176,101 @@ class Alist(StorageBase, metaclass=WeakSingleton):
             if item:
                 return [item]
             return []
-        resp = RequestUtils(headers=self.__get_header_with_token()).post_res(
+        items = []
-            self.__get_api_url("/api/fs/list"),
+        current_page = page
-            json={
+        while True:
-                "path": fileitem.path,
+            resp = RequestUtils(headers=self.__get_header_with_token()).post_res(
-                "password": password,
+                self.__get_api_url("/api/fs/list"),
-                "page": page,
+                json={
-                "per_page": per_page,
+                    "path": fileitem.path,
-                "refresh": refresh,
+                    "password": password,
-            },
+                    "page": current_page,
-        )
+                    "per_page": per_page,
-        """
+                    "refresh": refresh,
-        {
+                },
-            "path": "/t",
+            )
-            "password": "",
+            """
-            "page": 1,
+            {
-            "per_page": 0,
+                "path": "/t",
-            "refresh": false
+                "password": "",
-        }
+                "page": 1,
-        ======================================
+                "per_page": 0,
-        {
+                "refresh": false
-            "code": 200,
-            "message": "success",
-            "data": {
-                "content": [
-                {
-                    "name": "Alist V3.md",
-                    "size": 1592,
-                    "is_dir": false,
-                    "modified": "2024-05-17T13:47:55.4174917+08:00",
-                    "created": "2024-05-17T13:47:47.5725906+08:00",
-                    "sign": "",
-                    "thumb": "",
-                    "type": 4,
-                    "hashinfo": "null",
-                    "hash_info": null
-                }
-                ],
-                "total": 1,
-                "readme": "",
-                "header": "",
-                "write": true,
-                "provider": "Local"
             }
-        }
+            ======================================
-        """
+            {
+                "code": 200,
+                "message": "success",
+                "data": {
+                    "content": [
+                    {
+                        "name": "Alist V3.md",
+                        "size": 1592,
+                        "is_dir": false,
+                        "modified": "2024-05-17T13:47:55.4174917+08:00",
+                        "created": "2024-05-17T13:47:47.5725906+08:00",
+                        "sign": "",
+                        "thumb": "",
+                        "type": 4,
+                        "hashinfo": "null",
+                        "hash_info": null
+                    }
+                    ],
+                    "total": 1,
+                    "readme": "",
+                    "header": "",
+                    "write": true,
+                    "provider": "Local"
+                }
+            }
+            """
 
-        if resp is None:
+            if resp is None:
-            logger.warn(
+                logger.warn(
-                f"【OpenList】请求获取目录 {fileitem.path} 的文件列表失败，无法连接alist服务"
+                    f"【OpenList】请求获取目录 {fileitem.path} 的文件列表失败，无法连接alist服务"
-            )
+                )
-            return []
+                return []
-        if resp.status_code != 200:
+            if resp.status_code != 200:
-            logger.warn(
+                logger.warn(
-                f"【OpenList】请求获取目录 {fileitem.path} 的文件列表失败，状态码：{resp.status_code}"
+                    f"【OpenList】请求获取目录 {fileitem.path} 的文件列表失败，状态码：{resp.status_code}"
-            )
+                )
-            return []
+                return []
 
-        result = resp.json()
+            result = resp.json()
 
-        if result["code"] != 200:
+            if result["code"] != 200:
-            logger.warn(
+                logger.warn(
-                f"【OpenList】获取目录 {fileitem.path} 的文件列表失败，错误信息：{result['message']}"
+                    f"【OpenList】获取目录 {fileitem.path} 的文件列表失败，错误信息：{result['message']}"
-            )
+                )
-            return []
+                return []
 
-        return [
+            page_content = result["data"].get("content") or []
-            schemas.FileItem(
+            items.extend(
-                storage=self.schema.value,
+                [
-                type="dir" if item["is_dir"] else "file",
+                    schemas.FileItem(
-                path=(Path(fileitem.path) / item["name"]).as_posix()
+                        storage=self.schema.value,
-                + ("/" if item["is_dir"] else ""),
+                        type="dir" if item["is_dir"] else "file",
-                name=item["name"],
+                        path=(Path(fileitem.path) / item["name"]).as_posix()
-                basename=Path(item["name"]).stem,
+                        + ("/" if item["is_dir"] else ""),
-                extension=Path(item["name"]).suffix[1:] if not item["is_dir"] else None,
+                        name=item["name"],
-                size=item["size"] if not item["is_dir"] else None,
+                        basename=Path(item["name"]).stem,
-                modify_time=self.__parse_timestamp(item["modified"]),
+                        extension=Path(item["name"]).suffix[1:] if not item["is_dir"] else None,
-                thumbnail=item["thumb"],
+                        size=item["size"] if not item["is_dir"] else None,
+                        modify_time=self.__parse_timestamp(item["modified"]),
+                        thumbnail=item["thumb"],
+                    )
+                    for item in page_content
+                ]
             )
-            for item in result["data"]["content"] or []
+
-        ]
+            if per_page > 0:
+                return items
+
+            total = result["data"].get("total") or 0
+            if not page_content or len(items) >= total:
+                return items
+
+            current_page += 1
 
     def create_folder(
         self, fileitem: schemas.FileItem, name: str

tests/test_alist_storage.py

@@ -0,0 +1,205 @@
+import importlib.util
+import sys
+import types
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+
+def _load_alist_module():
+    module_name = "_test_alist_module"
+    app_module = types.ModuleType("app")
+    schemas_module = types.ModuleType("app.schemas")
+    cache_module = types.ModuleType("app.core.cache")
+    config_module = types.ModuleType("app.core.config")
+    log_module = types.ModuleType("app.log")
+    storages_module = types.ModuleType("app.modules.filemanager.storages")
+    exception_module = types.ModuleType("app.schemas.exception")
+    types_module = types.ModuleType("app.schemas.types")
+    http_module = types.ModuleType("app.utils.http")
+    singleton_module = types.ModuleType("app.utils.singleton")
+    url_module = types.ModuleType("app.utils.url")
+
+    class _FileItem:
+        def __init__(self, **kwargs):
+            for key, value in kwargs.items():
+                setattr(self, key, value)
+
+    class _StorageSchemaValue:
+        def __init__(self, value):
+            self.value = value
+
+    class _Logger:
+        def debug(self, *_args, **_kwargs):
+            pass
+
+        def warn(self, *_args, **_kwargs):
+            pass
+
+        def warning(self, *_args, **_kwargs):
+            pass
+
+        def error(self, *_args, **_kwargs):
+            pass
+
+        def critical(self, *_args, **_kwargs):
+            pass
+
+        def info(self, *_args, **_kwargs):
+            pass
+
+    class _StorageBase:
+        def __init__(self):
+            pass
+
+        def get_conf(self):
+            return {}
+
+    class _OperationInterrupted(Exception):
+        pass
+
+    class _RequestUtils:
+        def __init__(self, *args, **kwargs):
+            pass
+
+    class _UrlUtils:
+        @staticmethod
+        def standardize_base_url(url):
+            return url.rstrip("/") if url else ""
+
+        @staticmethod
+        def adapt_request_url(base, path):
+            return f"{base() if callable(base) else base}{path}"
+
+        @staticmethod
+        def quote(path):
+            return path
+
+    def _cached(*_args, **_kwargs):
+        def decorator(func):
+            func.cache_clear = lambda: None
+            return func
+
+        return decorator
+
+    schemas_module.FileItem = _FileItem
+    schemas_module.StorageUsage = object
+    cache_module.cached = _cached
+    config_module.settings = types.SimpleNamespace(
+        OPENLIST_SNAPSHOT_CHECK_FOLDER_MODTIME=True,
+        TEMP_PATH=Path("/tmp"),
+    )
+    config_module.global_vars = types.SimpleNamespace(
+        is_transfer_stopped=lambda *_args, **_kwargs: False
+    )
+    log_module.logger = _Logger()
+    storages_module.StorageBase = _StorageBase
+    storages_module.transfer_process = lambda *_args, **_kwargs: (lambda *_a, **_k: None)
+    exception_module.OperationInterrupted = _OperationInterrupted
+    types_module.StorageSchema = types.SimpleNamespace(Alist=_StorageSchemaValue("alist"))
+    http_module.RequestUtils = _RequestUtils
+    singleton_module.WeakSingleton = type
+    url_module.UrlUtils = _UrlUtils
+
+    app_module.schemas = schemas_module
+
+    stub_modules = {
+        "app": app_module,
+        "app.schemas": schemas_module,
+        "app.core.cache": cache_module,
+        "app.core.config": config_module,
+        "app.log": log_module,
+        "app.modules.filemanager.storages": storages_module,
+        "app.schemas.exception": exception_module,
+        "app.schemas.types": types_module,
+        "app.utils.http": http_module,
+        "app.utils.singleton": singleton_module,
+        "app.utils.url": url_module,
+    }
+    for stub_module in stub_modules.values():
+        stub_module._alist_test_stub = True
+
+    alist_path = Path(__file__).resolve().parents[1] / "app" / "modules" / "filemanager" / "storages" / "alist.py"
+    spec = importlib.util.spec_from_file_location(module_name, alist_path)
+    module = importlib.util.module_from_spec(spec)
+    assert spec and spec.loader
+    with patch.dict(sys.modules, stub_modules):
+        spec.loader.exec_module(module)
+    return module
+
+
+alist_module = _load_alist_module()
+Alist = alist_module.Alist
+FileItem = alist_module.schemas.FileItem
+
+
+class _FakeResponse:
+    def __init__(self, payload: dict, status_code: int = 200):
+        self._payload = payload
+        self.status_code = status_code
+
+    def json(self):
+        return self._payload
+
+
+class AlistStorageTest(unittest.TestCase):
+    def setUp(self):
+        self.storage = Alist()
+
+    @staticmethod
+    def _dir_item(path: str = "/"):
+        return FileItem(storage="alist", type="dir", path=path)
+
+    @staticmethod
+    def _page_payload(start: int, count: int, total: int) -> dict:
+        return {
+            "code": 200,
+            "message": "success",
+            "data": {
+                "content": [
+                    {
+                        "name": f"dir-{index}",
+                        "size": 0,
+                        "is_dir": True,
+                        "modified": "2024-05-17T13:47:55.4174917+08:00",
+                        "thumb": "",
+                    }
+                    for index in range(start, start + count)
+                ],
+                "total": total,
+            },
+        }
+
+    def test_list_fetches_all_pages_when_per_page_is_default(self):
+        responses = [
+            _FakeResponse(self._page_payload(0, 200, 205)),
+            _FakeResponse(self._page_payload(200, 5, 205)),
+        ]
+        request_utils = MagicMock()
+        request_utils.post_res.side_effect = responses
+
+        with patch.object(Alist, "get_conf", return_value={"url": "http://openlist.test", "token": "token"}):
+            with patch.object(alist_module, "RequestUtils", return_value=request_utils):
+                items = self.storage.list(self._dir_item())
+
+        self.assertEqual(205, len(items))
+        self.assertEqual("/dir-0/", items[0].path)
+        self.assertEqual("/dir-204/", items[-1].path)
+        self.assertEqual(2, request_utils.post_res.call_count)
+        self.assertEqual(1, request_utils.post_res.call_args_list[0].kwargs["json"]["page"])
+        self.assertEqual(2, request_utils.post_res.call_args_list[1].kwargs["json"]["page"])
+
+    def test_list_respects_explicit_per_page_without_auto_paging(self):
+        request_utils = MagicMock()
+        request_utils.post_res.return_value = _FakeResponse(self._page_payload(0, 50, 205))
+
+        with patch.object(Alist, "get_conf", return_value={"url": "http://openlist.test", "token": "token"}):
+            with patch.object(alist_module, "RequestUtils", return_value=request_utils):
+                items = self.storage.list(self._dir_item(), per_page=50)
+
+        self.assertEqual(50, len(items))
+        self.assertEqual(1, request_utils.post_res.call_count)
+
+
+if __name__ == "__main__":
+    unittest.main()