refactor: strengthen architecture CI gates and runtime contracts

This commit is contained in:
jxxghp
2026-08-25 17:02:29 +08:00
parent 605bf8174a
commit b914e8b6e4
123 changed files with 4240 additions and 1342 deletions
+6 -6
View File
@@ -1,17 +1,17 @@
from abc import ABCMeta, abstractmethod
from pathlib import Path, PurePosixPath
from typing import Optional, List, Dict, Tuple, Callable, Union
from typing import Callable, Dict, List, Optional, Tuple, Union
from tqdm import tqdm
from app.application.storage import StorageHelper
from app.foundation.crypto import HashUtils
from app.runtime.log import logger
from app.runtime.progress import ProgressHelper
from app.schemas.exception import StorageQueryError
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.system import StorageConf as _SchemaStorageConf
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.progress import ProgressHelper
from app.application.storage import StorageHelper
from app.runtime.log import logger
from app.schemas.exception import StorageQueryError
from app.foundation.crypto import HashUtils
def transfer_process(path: str) -> Callable[[int | float], None]:
+8 -8
View File
@@ -8,19 +8,19 @@ from typing import List, Optional, Tuple, Union
import requests
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.stop import runtime_stop_state
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.config import global_vars
settings = RuntimeSettingsCompat()
from app.runtime.log import logger
from app.adapters.network.http import RequestUtils
from app.foundation import temporal as time_tools
from app.foundation.singleton import WeakSingleton
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.adapters.network.http import RequestUtils
from app.foundation.singleton import WeakSingleton
from app.foundation import temporal as time_tools
lock = threading.Lock()
@@ -645,7 +645,7 @@ class AliPan(StorageBase, metaclass=WeakSingleton):
uploaded_size = 0
with open(local_path, "rb") as f:
for part_info in part_info_list:
if global_vars.is_transfer_stopped(local_path.as_posix()):
if runtime_stop_state.consume_transfer_stop(local_path.as_posix()):
logger.info(f"【阿里云盘】{target_name} 上传已取消!")
return None
@@ -780,7 +780,7 @@ class AliPan(StorageBase, metaclass=WeakSingleton):
downloaded_size = 0
with open(local_path, "wb") as f:
for chunk in r.iter_content(chunk_size=self.chunk_size):
if global_vars.is_transfer_stopped(fileitem.path):
if runtime_stop_state.consume_transfer_stop(fileitem.path):
logger.info(f"【阿里云盘】{fileitem.path} 下载已取消!")
return None
if chunk:
+10 -11
View File
@@ -3,23 +3,22 @@ import json
import time
from datetime import datetime
from pathlib import Path
from typing import Optional, List
from typing import List, Optional
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.cache import cached
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.config import global_vars
from app.runtime.stop import runtime_stop_state
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
settings = RuntimeSettingsCompat()
from app.runtime.log import logger
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.schemas.exception import OperationInterrupted, StorageQueryError
from app.schemas.types import StorageSchema
from app.adapters.network.http import RequestUtils
from app.foundation.singleton import WeakSingleton
from app.foundation.url import UrlUtils
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.schemas.exception import OperationInterrupted, StorageQueryError
from app.schemas.types import StorageSchema
# OpenList/AList 在 per_page<=0 时会退回后端默认 200,显式指定最大页大小避免大目录被截断。
OPENLIST_MAX_LIST_PAGE_SIZE = 500
@@ -703,7 +702,7 @@ class Alist(StorageBase, metaclass=WeakSingleton):
r.raise_for_status()
with open(local_path, "wb") as f:
for chunk in r.iter_content(chunk_size=8192):
if global_vars.is_transfer_stopped(fileitem.path):
if runtime_stop_state.consume_transfer_stop(fileitem.path):
logger.info(f"【OpenList】{fileitem.path} 下载已取消!")
return None
f.write(chunk)
@@ -760,7 +759,7 @@ class Alist(StorageBase, metaclass=WeakSingleton):
return self.file_size
def read(self, size=-1):
if global_vars.is_transfer_stopped(path.as_posix()):
if runtime_stop_state.consume_transfer_stop(path.as_posix()):
logger.info(f"【OpenList】{path} 上传已取消!")
raise OperationInterrupted(f"Upload cancelled: {path}")
chunk = self.file.read(size)
+8 -8
View File
@@ -2,21 +2,21 @@ import os
import shutil
import time
from pathlib import Path
from typing import Optional, List
from typing import List, Optional
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.stop import runtime_stop_state
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.config import global_vars
from app.runtime.settings import RuntimeSettingsCompat
settings = RuntimeSettingsCompat()
from app.application.directory import DirectoryHelper
from app.runtime.log import logger
from app.adapters.system.fsproxy import fsproxy
from app.adapters.system.host import SystemUtils
from app.application.directory import DirectoryHelper
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.adapters.system.host import SystemUtils
class LocalStorage(StorageBase):
@@ -301,7 +301,7 @@ class LocalStorage(StorageBase):
copied = fsproxy.copy(
src, partial,
progress_cb=progress_callback,
cancel_cb=lambda: global_vars.is_transfer_stopped(src.as_posix()),
cancel_cb=lambda: runtime_stop_state.consume_transfer_stop(src.as_posix()),
chunk_size=self.chunk_size,
)
if not copied:
@@ -352,7 +352,7 @@ class LocalStorage(StorageBase):
try:
with open(src, "rb") as fsrc, open(dest, "wb") as fdst:
while True:
if global_vars.is_transfer_stopped(src.as_posix()):
if runtime_stop_state.consume_transfer_stop(src.as_posix()):
logger.info(f"【本地】{src} 复制已取消!")
return False
buf = fsrc.read(self.chunk_size)
+5 -5
View File
@@ -4,19 +4,19 @@ import threading
import time
from collections import OrderedDict
from pathlib import Path
from typing import Optional, List, Union
from typing import List, Optional, Union
from app.runtime.settings import RuntimeSettingsCompat
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.settings import RuntimeSettingsCompat
settings = RuntimeSettingsCompat()
from app.runtime.log import logger
from app.adapters.system.host import SystemUtils
from app.foundation import temporal as time_tools
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.foundation import temporal as time_tools
from app.adapters.system.host import SystemUtils
_MAX_FOLDER_LOCKS = 4096
_folder_locks: OrderedDict[str, threading.Lock] = OrderedDict()
+7 -7
View File
@@ -7,22 +7,22 @@ from typing import List, Optional, Union
import smbclient
from smbclient import ClientConfig, register_session, reset_connection_cache
from smbprotocol.exceptions import (
SMBAuthenticationError,
SMBException,
SMBResponseException,
SMBAuthenticationError,
)
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.stop import runtime_stop_state
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.config import global_vars
settings = RuntimeSettingsCompat()
from app.runtime.log import logger
from app.foundation.singleton import WeakSingleton
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.foundation.singleton import WeakSingleton
lock = threading.Lock()
@@ -572,7 +572,7 @@ class SMB(StorageBase, metaclass=WeakSingleton):
with open(local_path, "wb") as dst_file:
downloaded_size = 0
while True:
if global_vars.is_transfer_stopped(fileitem.path):
if runtime_stop_state.consume_transfer_stop(fileitem.path):
logger.info(f"【SMB】{fileitem.path} 下载已取消!")
return None
chunk = src_file.read(self.chunk_size)
@@ -622,7 +622,7 @@ class SMB(StorageBase, metaclass=WeakSingleton):
with smbclient.open_file(smb_path, mode="wb") as dst_file:
uploaded_size = 0
while True:
if global_vars.is_transfer_stopped(path.as_posix()):
if runtime_stop_state.consume_transfer_stop(path.as_posix()):
logger.info(f"【SMB】{path} 上传已取消!")
return None
chunk = src_file.read(self.chunk_size)
+11 -12
View File
@@ -1,31 +1,30 @@
import base64
import secrets
import time
from hashlib import sha256
from pathlib import Path
from threading import Lock
from typing import List, Optional, Tuple, Union
from hashlib import sha256
import oss2
import httpx
import oss2
from cryptography.hazmat.primitives import hashes
from oss2 import SizedFileAdapter, determine_part_size
from oss2.models import PartInfo
from cryptography.hazmat.primitives import hashes
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.stop import runtime_stop_state
from app.schemas.file import StorageUsage as _SchemaStorageUsage
from app.schemas.workflow import FileItem as _SchemaFileItem
from app.runtime.settings import RuntimeSettingsCompat
from app.runtime.config import global_vars
settings = RuntimeSettingsCompat()
from app.runtime.log import logger
from app.foundation import size as size_tools
from app.foundation.singleton import WeakSingleton
from app.modules.filemanager.storages import StorageBase, transfer_process
from app.runtime.log import logger
from app.runtime.rate import QpsRateLimiter, RateStats
from app.schemas.exception import StorageQueryError
from app.schemas.types import StorageSchema
from app.foundation.singleton import WeakSingleton
from app.foundation import size as size_tools
from app.runtime.rate import QpsRateLimiter, RateStats
lock = Lock()
@@ -778,7 +777,7 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
part_number = 1
offset = 0
while offset < file_size:
if global_vars.is_transfer_stopped(local_path.as_posix()):
if runtime_stop_state.consume_transfer_stop(local_path.as_posix()):
logger.info(f"【115】{local_path} 上传已取消!")
return None
num_to_upload = min(part_size, file_size - offset)
@@ -929,7 +928,7 @@ class U115Pan(StorageBase, metaclass=WeakSingleton):
with open(local_path, "wb") as f:
for chunk in r.iter_bytes(chunk_size=self.chunk_size):
if global_vars.is_transfer_stopped(fileitem.path):
if runtime_stop_state.consume_transfer_stop(fileitem.path):
logger.info(f"【115】{fileitem.path} 下载已取消!")
r.close()
return None